From 20ed5b1f45871612570d3bd447121ac43e083c6a Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl@google.com>
Date: Thu, 11 Apr 2024 12:47:51 -0700
Subject: [PATCH 001/464] [nfc]Fix test gold/x86/thinlto.ll with GV Summary
 change (#88429)

This gold thinlto test should have been updated in commit dda7333
(https://github.com/llvm/llvm-project/commit/dda73336ad22bd0b5ecda17040c50fb10fcbe5fb).

It's ok to keep {Inputs/emit-llvm.foo.ll, Inputs/emit-llvm.bar.ll} the
current way for `llvm-as` since the field is optional (see example in
https://github.com/llvm/llvm-project/blob/5d6d8dcd292e0a107b11d378932eee9c2f9ccfc7/llvm/test/Assembler/thinlto-summary.ll#L12-L16)
---
 llvm/test/tools/gold/X86/thinlto.ll | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/llvm/test/tools/gold/X86/thinlto.ll b/llvm/test/tools/gold/X86/thinlto.ll
index 1745f0382b81..9d56afe9d58d 100644
--- a/llvm/test/tools/gold/X86/thinlto.ll
+++ b/llvm/test/tools/gold/X86/thinlto.ll
@@ -123,11 +123,11 @@
 
 ; DIS1: ^0 = module: (path: "{{.*}}thinlto.ll.tmp.o", hash: (0, 0, 0, 0, 0))
 ; DIS1: ^1 = module: (path: "{{.*}}thinlto.ll.tmp2.o", hash: (0, 0, 0, 0, 0))
-; DIS1: ^2 = gv: (guid: 13146401226427987378, summaries: (function: (module: ^1, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 1, dsoLocal: 0, canAutoHide: 0), insts: 1)))
-; DIS1: ^3 = gv: (guid: 14740650423002898831, summaries: (function: (module: ^0, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 1, dsoLocal: 0, canAutoHide: 0), insts: 2, calls: ((callee: ^2)))))
+; DIS1: ^2 = gv: (guid: 13146401226427987378, summaries: (function: (module: ^1, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 1, dsoLocal: 0, canAutoHide: 0, importType: definition), insts: 1)))
+; DIS1: ^3 = gv: (guid: 14740650423002898831, summaries: (function: (module: ^0, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 1, dsoLocal: 0, canAutoHide: 0, importType: definition), insts: 2, calls: ((callee: ^2)))))
 
 ; DIS2: ^0 = module: (path: "{{.*}}thinlto.ll.tmp2.o", hash: (0, 0, 0, 0, 0))
-; DIS2: ^1 = gv: (guid: 13146401226427987378, summaries: (function: (module: ^0, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 1, dsoLocal: 0, canAutoHide: 0), insts: 1)))
+; DIS2: ^1 = gv: (guid: 13146401226427987378, summaries: (function: (module: ^0, flags: (linkage: external, visibility: default, notEligibleToImport: 0, live: 1, dsoLocal: 0, canAutoHide: 0, importType: definition), insts: 1)))
 
 ; COMBINED: <MODULE_STRTAB_BLOCK
 ; COMBINED-NEXT: <ENTRY {{.*}} record string = '{{.*}}/test/tools/gold/X86/Output/thinlto.ll.tmp{{.*}}.o'
-- 
GitLab


From aa80f3ec48419a73aafcc2ff947c6dd1e3734481 Mon Sep 17 00:00:00 2001
From: Krystian Stasiowski <sdkrystian@gmail.com>
Date: Thu, 11 Apr 2024 16:19:29 -0400
Subject: [PATCH 002/464] Reapply "[Clang][Sema] Fix crash when 'this' is used
 in a dependent class scope function template specialization that instantiates
 to a static member function (#87541)" (#88311)

Reapplies #87541 and addresses the bug which caused expressions naming
overload sets to be incorrectly rebuilt.
---
 clang/docs/ReleaseNotes.rst                   |   2 +
 clang/include/clang/Sema/Sema.h               |  16 ++-
 clang/lib/Sema/SemaExpr.cpp                   |  28 +----
 clang/lib/Sema/SemaExprCXX.cpp                |  61 +++++-----
 clang/lib/Sema/SemaExprMember.cpp             |  63 ++++++++---
 .../lib/Sema/SemaTemplateInstantiateDecl.cpp  |   8 ++
 clang/lib/Sema/TreeTransform.h                |  65 ++++++-----
 .../SemaTemplate/instantiate-using-decl.cpp   |   2 +-
 ...ms-function-specialization-class-scope.cpp | 106 +++++++++++++++++-
 9 files changed, 252 insertions(+), 99 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 93a380411604..45a9a79739a4 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -526,6 +526,8 @@ Bug Fixes to C++ Support
 - Fix an issue caused by not handling invalid cases when substituting into the parameter mapping of a constraint. Fixes (#GH86757).
 - Fixed a bug that prevented member function templates of class templates declared with a deduced return type
   from being explicitly specialized for a given implicit instantiation of the class template.
+- Fixed a crash when ``this`` is used in a dependent class scope function template specialization
+  that instantiates to a static member function.
 
 - Fix crash when inheriting from a cv-qualified type. Fixes:
   (`#35603 <https://github.com/llvm/llvm-project/issues/35603>`_)
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index f2c55b13b6d3..351b8e1f3113 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -5447,7 +5447,8 @@ public:
 
   ExprResult BuildDeclarationNameExpr(const CXXScopeSpec &SS, LookupResult &R,
                                       bool NeedsADL,
-                                      bool AcceptInvalidDecl = false);
+                                      bool AcceptInvalidDecl = false,
+                                      bool NeedUnresolved = false);
   ExprResult BuildDeclarationNameExpr(
       const CXXScopeSpec &SS, const DeclarationNameInfo &NameInfo, NamedDecl *D,
       NamedDecl *FoundD = nullptr,
@@ -6590,7 +6591,10 @@ public:
                             SourceLocation RParenLoc);
 
   //// ActOnCXXThis -  Parse 'this' pointer.
-  ExprResult ActOnCXXThis(SourceLocation loc);
+  ExprResult ActOnCXXThis(SourceLocation Loc);
+
+  /// Check whether the type of 'this' is valid in the current context.
+  bool CheckCXXThisType(SourceLocation Loc, QualType Type);
 
   /// Build a CXXThisExpr and mark it referenced in the current context.
   Expr *BuildCXXThisExpr(SourceLocation Loc, QualType Type, bool IsImplicit);
@@ -7013,10 +7017,14 @@ private:
   ///@{
 
 public:
+  /// Check whether an expression might be an implicit class member access.
+  bool isPotentialImplicitMemberAccess(const CXXScopeSpec &SS, LookupResult &R,
+                                       bool IsAddressOfOperand);
+
   ExprResult BuildPossibleImplicitMemberExpr(
       const CXXScopeSpec &SS, SourceLocation TemplateKWLoc, LookupResult &R,
-      const TemplateArgumentListInfo *TemplateArgs, const Scope *S,
-      UnresolvedLookupExpr *AsULE = nullptr);
+      const TemplateArgumentListInfo *TemplateArgs, const Scope *S);
+
   ExprResult
   BuildImplicitMemberExpr(const CXXScopeSpec &SS, SourceLocation TemplateKWLoc,
                           LookupResult &R,
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 4d4ef9b16381..846e7d5d3ab9 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -2912,26 +2912,9 @@ Sema::ActOnIdExpression(Scope *S, CXXScopeSpec &SS,
   // to get this right here so that we don't end up making a
   // spuriously dependent expression if we're inside a dependent
   // instance method.
-  if (getLangOpts().CPlusPlus && !R.empty() &&
-      (*R.begin())->isCXXClassMember()) {
-    bool MightBeImplicitMember;
-    if (!IsAddressOfOperand)
-      MightBeImplicitMember = true;
-    else if (!SS.isEmpty())
-      MightBeImplicitMember = false;
-    else if (R.isOverloadedResult())
-      MightBeImplicitMember = false;
-    else if (R.isUnresolvableResult())
-      MightBeImplicitMember = true;
-    else
-      MightBeImplicitMember = isa<FieldDecl>(R.getFoundDecl()) ||
-                              isa<IndirectFieldDecl>(R.getFoundDecl()) ||
-                              isa<MSPropertyDecl>(R.getFoundDecl());
-
-    if (MightBeImplicitMember)
-      return BuildPossibleImplicitMemberExpr(SS, TemplateKWLoc,
-                                             R, TemplateArgs, S);
-  }
+  if (isPotentialImplicitMemberAccess(SS, R, IsAddressOfOperand))
+    return BuildPossibleImplicitMemberExpr(SS, TemplateKWLoc, R, TemplateArgs,
+                                           S);
 
   if (TemplateArgs || TemplateKWLoc.isValid()) {
 
@@ -3442,10 +3425,11 @@ static bool ShouldLookupResultBeMultiVersionOverload(const LookupResult &R) {
 
 ExprResult Sema::BuildDeclarationNameExpr(const CXXScopeSpec &SS,
                                           LookupResult &R, bool NeedsADL,
-                                          bool AcceptInvalidDecl) {
+                                          bool AcceptInvalidDecl,
+                                          bool NeedUnresolved) {
   // If this is a single, fully-resolved result and we don't need ADL,
   // just build an ordinary singleton decl ref.
-  if (!NeedsADL && R.isSingleResult() &&
+  if (!NeedUnresolved && !NeedsADL && R.isSingleResult() &&
       !R.getAsSingle<FunctionTemplateDecl>() &&
       !ShouldLookupResultBeMultiVersionOverload(R))
     return BuildDeclarationNameExpr(SS, R.getLookupNameInfo(), R.getFoundDecl(),
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index 12f42f66e5e2..763901d4418d 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -1414,26 +1414,42 @@ bool Sema::CheckCXXThisCapture(SourceLocation Loc, const bool Explicit,
 }
 
 ExprResult Sema::ActOnCXXThis(SourceLocation Loc) {
-  /// C++ 9.3.2: In the body of a non-static member function, the keyword this
-  /// is a non-lvalue expression whose value is the address of the object for
-  /// which the function is called.
+  // C++20 [expr.prim.this]p1:
+  //   The keyword this names a pointer to the object for which an
+  //   implicit object member function is invoked or a non-static
+  //   data member's initializer is evaluated.
   QualType ThisTy = getCurrentThisType();
 
-  if (ThisTy.isNull()) {
-    DeclContext *DC = getFunctionLevelDeclContext();
+  if (CheckCXXThisType(Loc, ThisTy))
+    return ExprError();
 
-    if (const auto *Method = dyn_cast<CXXMethodDecl>(DC);
-        Method && Method->isExplicitObjectMemberFunction()) {
-      return Diag(Loc, diag::err_invalid_this_use) << 1;
-    }
+  return BuildCXXThisExpr(Loc, ThisTy, /*IsImplicit=*/false);
+}
 
-    if (isLambdaCallWithExplicitObjectParameter(CurContext))
-      return Diag(Loc, diag::err_invalid_this_use) << 1;
+bool Sema::CheckCXXThisType(SourceLocation Loc, QualType Type) {
+  if (!Type.isNull())
+    return false;
 
-    return Diag(Loc, diag::err_invalid_this_use) << 0;
+  // C++20 [expr.prim.this]p3:
+  //   If a declaration declares a member function or member function template
+  //   of a class X, the expression this is a prvalue of type
+  //   "pointer to cv-qualifier-seq X" wherever X is the current class between
+  //   the optional cv-qualifier-seq and the end of the function-definition,
+  //   member-declarator, or declarator. It shall not appear within the
+  //   declaration of either a static member function or an explicit object
+  //   member function of the current class (although its type and value
+  //   category are defined within such member functions as they are within
+  //   an implicit object member function).
+  DeclContext *DC = getFunctionLevelDeclContext();
+  if (const auto *Method = dyn_cast<CXXMethodDecl>(DC);
+      Method && Method->isExplicitObjectMemberFunction()) {
+    Diag(Loc, diag::err_invalid_this_use) << 1;
+  } else if (isLambdaCallWithExplicitObjectParameter(CurContext)) {
+    Diag(Loc, diag::err_invalid_this_use) << 1;
+  } else {
+    Diag(Loc, diag::err_invalid_this_use) << 0;
   }
-
-  return BuildCXXThisExpr(Loc, ThisTy, /*IsImplicit=*/false);
+  return true;
 }
 
 Expr *Sema::BuildCXXThisExpr(SourceLocation Loc, QualType Type,
@@ -8626,21 +8642,8 @@ static ExprResult attemptRecovery(Sema &SemaRef,
 
       // Detect and handle the case where the decl might be an implicit
       // member.
-      bool MightBeImplicitMember;
-      if (!Consumer.isAddressOfOperand())
-        MightBeImplicitMember = true;
-      else if (!NewSS.isEmpty())
-        MightBeImplicitMember = false;
-      else if (R.isOverloadedResult())
-        MightBeImplicitMember = false;
-      else if (R.isUnresolvableResult())
-        MightBeImplicitMember = true;
-      else
-        MightBeImplicitMember = isa<FieldDecl>(ND) ||
-                                isa<IndirectFieldDecl>(ND) ||
-                                isa<MSPropertyDecl>(ND);
-
-      if (MightBeImplicitMember)
+      if (SemaRef.isPotentialImplicitMemberAccess(
+              NewSS, R, Consumer.isAddressOfOperand()))
         return SemaRef.BuildPossibleImplicitMemberExpr(
             NewSS, /*TemplateKWLoc*/ SourceLocation(), R,
             /*TemplateArgs*/ nullptr, /*S*/ nullptr);
diff --git a/clang/lib/Sema/SemaExprMember.cpp b/clang/lib/Sema/SemaExprMember.cpp
index 32998ae60eaf..eeac753a3489 100644
--- a/clang/lib/Sema/SemaExprMember.cpp
+++ b/clang/lib/Sema/SemaExprMember.cpp
@@ -61,6 +61,10 @@ enum IMAKind {
   /// The reference is a contextually-permitted abstract member reference.
   IMA_Abstract,
 
+  /// Whether the context is static is dependent on the enclosing template (i.e.
+  /// in a dependent class scope explicit specialization).
+  IMA_Dependent,
+
   /// The reference may be to an unresolved using declaration and the
   /// context is not an instance method.
   IMA_Unresolved_StaticOrExplicitContext,
@@ -91,10 +95,18 @@ static IMAKind ClassifyImplicitMemberAccess(Sema &SemaRef,
 
   DeclContext *DC = SemaRef.getFunctionLevelDeclContext();
 
-  bool isStaticOrExplicitContext =
-      SemaRef.CXXThisTypeOverride.isNull() &&
-      (!isa<CXXMethodDecl>(DC) || cast<CXXMethodDecl>(DC)->isStatic() ||
-       cast<CXXMethodDecl>(DC)->isExplicitObjectMemberFunction());
+  bool couldInstantiateToStatic = false;
+  bool isStaticOrExplicitContext = SemaRef.CXXThisTypeOverride.isNull();
+
+  if (auto *MD = dyn_cast<CXXMethodDecl>(DC)) {
+    if (MD->isImplicitObjectMemberFunction()) {
+      isStaticOrExplicitContext = false;
+      // A dependent class scope function template explicit specialization
+      // that is neither declared 'static' nor with an explicit object
+      // parameter could instantiate to a static or non-static member function.
+      couldInstantiateToStatic = MD->getDependentSpecializationInfo();
+    }
+  }
 
   if (R.isUnresolvableResult())
     return isStaticOrExplicitContext ? IMA_Unresolved_StaticOrExplicitContext
@@ -123,6 +135,9 @@ static IMAKind ClassifyImplicitMemberAccess(Sema &SemaRef,
   if (Classes.empty())
     return IMA_Static;
 
+  if (couldInstantiateToStatic)
+    return IMA_Dependent;
+
   // C++11 [expr.prim.general]p12:
   //   An id-expression that denotes a non-static data member or non-static
   //   member function of a class can only be used:
@@ -263,32 +278,52 @@ static void diagnoseInstanceReference(Sema &SemaRef,
   }
 }
 
+bool Sema::isPotentialImplicitMemberAccess(const CXXScopeSpec &SS,
+                                           LookupResult &R,
+                                           bool IsAddressOfOperand) {
+  if (!getLangOpts().CPlusPlus)
+    return false;
+  else if (R.empty() || !R.begin()->isCXXClassMember())
+    return false;
+  else if (!IsAddressOfOperand)
+    return true;
+  else if (!SS.isEmpty())
+    return false;
+  else if (R.isOverloadedResult())
+    return false;
+  else if (R.isUnresolvableResult())
+    return true;
+  else
+    return isa<FieldDecl, IndirectFieldDecl, MSPropertyDecl>(R.getFoundDecl());
+}
+
 /// Builds an expression which might be an implicit member expression.
 ExprResult Sema::BuildPossibleImplicitMemberExpr(
     const CXXScopeSpec &SS, SourceLocation TemplateKWLoc, LookupResult &R,
-    const TemplateArgumentListInfo *TemplateArgs, const Scope *S,
-    UnresolvedLookupExpr *AsULE) {
-  switch (ClassifyImplicitMemberAccess(*this, R)) {
+    const TemplateArgumentListInfo *TemplateArgs, const Scope *S) {
+  switch (IMAKind Classification = ClassifyImplicitMemberAccess(*this, R)) {
   case IMA_Instance:
-    return BuildImplicitMemberExpr(SS, TemplateKWLoc, R, TemplateArgs, true, S);
-
   case IMA_Mixed:
   case IMA_Mixed_Unrelated:
   case IMA_Unresolved:
-    return BuildImplicitMemberExpr(SS, TemplateKWLoc, R, TemplateArgs, false,
-                                   S);
-
+    return BuildImplicitMemberExpr(
+        SS, TemplateKWLoc, R, TemplateArgs,
+        /*IsKnownInstance=*/Classification == IMA_Instance, S);
   case IMA_Field_Uneval_Context:
     Diag(R.getNameLoc(), diag::warn_cxx98_compat_non_static_member_use)
       << R.getLookupNameInfo().getName();
     [[fallthrough]];
   case IMA_Static:
   case IMA_Abstract:
+  case IMA_Dependent:
   case IMA_Mixed_StaticOrExplicitContext:
   case IMA_Unresolved_StaticOrExplicitContext:
     if (TemplateArgs || TemplateKWLoc.isValid())
-      return BuildTemplateIdExpr(SS, TemplateKWLoc, R, false, TemplateArgs);
-    return AsULE ? AsULE : BuildDeclarationNameExpr(SS, R, false);
+      return BuildTemplateIdExpr(SS, TemplateKWLoc, R, /*RequiresADL=*/false,
+                                 TemplateArgs);
+    return BuildDeclarationNameExpr(
+        SS, R, /*NeedsADL=*/false, /*AcceptInvalidDecl=*/false,
+        /*NeedUnresolved=*/Classification == IMA_Dependent);
 
   case IMA_Error_StaticOrExplicitContext:
   case IMA_Error_Unrelated:
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index 2aa696d35d1d..707446e13209 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -5095,6 +5095,14 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
   EnterExpressionEvaluationContext EvalContext(
       *this, Sema::ExpressionEvaluationContext::PotentiallyEvaluated);
 
+  Qualifiers ThisTypeQuals;
+  CXXRecordDecl *ThisContext = nullptr;
+  if (CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(Function)) {
+    ThisContext = Method->getParent();
+    ThisTypeQuals = Method->getMethodQualifiers();
+  }
+  CXXThisScopeRAII ThisScope(*this, ThisContext, ThisTypeQuals);
+
   // Introduce a new scope where local variable instantiations will be
   // recorded, unless we're actually a member function within a local
   // class, in which case we need to merge our results with the parent
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 33a9356e82f4..284c9173e68e 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -795,6 +795,9 @@ public:
       ParenExpr *PE, DependentScopeDeclRefExpr *DRE, bool IsAddressOfOperand,
       TypeSourceInfo **RecoveryTSI);
 
+  ExprResult TransformUnresolvedLookupExpr(UnresolvedLookupExpr *E,
+                                           bool IsAddressOfOperand);
+
   StmtResult TransformOMPExecutableDirective(OMPExecutableDirective *S);
 
 // FIXME: We use LLVM_ATTRIBUTE_NOINLINE because inlining causes a ridiculous
@@ -3309,12 +3312,13 @@ public:
 
   /// Build a new C++ "this" expression.
   ///
-  /// By default, builds a new "this" expression without performing any
-  /// semantic analysis. Subclasses may override this routine to provide
-  /// different behavior.
+  /// By default, performs semantic analysis to build a new "this" expression.
+  /// Subclasses may override this routine to provide different behavior.
   ExprResult RebuildCXXThisExpr(SourceLocation ThisLoc,
                                 QualType ThisType,
                                 bool isImplicit) {
+    if (getSema().CheckCXXThisType(ThisLoc, ThisType))
+      return ExprError();
     return getSema().BuildCXXThisExpr(ThisLoc, ThisType, isImplicit);
   }
 
@@ -11351,7 +11355,11 @@ template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformAddressOfOperand(Expr *E) {
   if (DependentScopeDeclRefExpr *DRE = dyn_cast<DependentScopeDeclRefExpr>(E))
-    return getDerived().TransformDependentScopeDeclRefExpr(DRE, true, nullptr);
+    return getDerived().TransformDependentScopeDeclRefExpr(
+        DRE, /*IsAddressOfOperand=*/true, nullptr);
+  else if (UnresolvedLookupExpr *ULE = dyn_cast<UnresolvedLookupExpr>(E))
+    return getDerived().TransformUnresolvedLookupExpr(
+        ULE, /*IsAddressOfOperand=*/true);
   else
     return getDerived().TransformExpr(E);
 }
@@ -13057,10 +13065,16 @@ bool TreeTransform<Derived>::TransformOverloadExprDecls(OverloadExpr *Old,
   return false;
 }
 
-template<typename Derived>
+template <typename Derived>
+ExprResult TreeTransform<Derived>::TransformUnresolvedLookupExpr(
+    UnresolvedLookupExpr *Old) {
+  return TransformUnresolvedLookupExpr(Old, /*IsAddressOfOperand=*/false);
+}
+
+template <typename Derived>
 ExprResult
-TreeTransform<Derived>::TransformUnresolvedLookupExpr(
-                                                  UnresolvedLookupExpr *Old) {
+TreeTransform<Derived>::TransformUnresolvedLookupExpr(UnresolvedLookupExpr *Old,
+                                                      bool IsAddressOfOperand) {
   LookupResult R(SemaRef, Old->getName(), Old->getNameLoc(),
                  Sema::LookupOrdinaryName);
 
@@ -13092,26 +13106,8 @@ TreeTransform<Derived>::TransformUnresolvedLookupExpr(
     R.setNamingClass(NamingClass);
   }
 
+  // Rebuild the template arguments, if any.
   SourceLocation TemplateKWLoc = Old->getTemplateKeywordLoc();
-
-  // If we have neither explicit template arguments, nor the template keyword,
-  // it's a normal declaration name or member reference.
-  if (!Old->hasExplicitTemplateArgs() && !TemplateKWLoc.isValid()) {
-    NamedDecl *D = R.getAsSingle<NamedDecl>();
-    // In a C++11 unevaluated context, an UnresolvedLookupExpr might refer to an
-    // instance member. In other contexts, BuildPossibleImplicitMemberExpr will
-    // give a good diagnostic.
-    if (D && D->isCXXInstanceMember()) {
-      return SemaRef.BuildPossibleImplicitMemberExpr(SS, TemplateKWLoc, R,
-                                                     /*TemplateArgs=*/nullptr,
-                                                     /*Scope=*/nullptr);
-    }
-
-    return getDerived().RebuildDeclarationNameExpr(SS, R, Old->requiresADL());
-  }
-
-  // If we have template arguments, rebuild them, then rebuild the
-  // templateid expression.
   TemplateArgumentListInfo TransArgs(Old->getLAngleLoc(), Old->getRAngleLoc());
   if (Old->hasExplicitTemplateArgs() &&
       getDerived().TransformTemplateArguments(Old->getTemplateArgs(),
@@ -13121,6 +13117,23 @@ TreeTransform<Derived>::TransformUnresolvedLookupExpr(
     return ExprError();
   }
 
+  // An UnresolvedLookupExpr can refer to a class member. This occurs e.g. when
+  // a non-static data member is named in an unevaluated operand, or when
+  // a member is named in a dependent class scope function template explicit
+  // specialization that is neither declared static nor with an explicit object
+  // parameter.
+  if (SemaRef.isPotentialImplicitMemberAccess(SS, R, IsAddressOfOperand))
+    return SemaRef.BuildPossibleImplicitMemberExpr(
+        SS, TemplateKWLoc, R,
+        Old->hasExplicitTemplateArgs() ? &TransArgs : nullptr,
+        /*S=*/nullptr);
+
+  // If we have neither explicit template arguments, nor the template keyword,
+  // it's a normal declaration name or member reference.
+  if (!Old->hasExplicitTemplateArgs() && !TemplateKWLoc.isValid())
+    return getDerived().RebuildDeclarationNameExpr(SS, R, Old->requiresADL());
+
+  // If we have template arguments, then rebuild the template-id expression.
   return getDerived().RebuildTemplateIdExpr(SS, TemplateKWLoc, R,
                                             Old->requiresADL(), &TransArgs);
 }
diff --git a/clang/test/SemaTemplate/instantiate-using-decl.cpp b/clang/test/SemaTemplate/instantiate-using-decl.cpp
index 0bbb3ca9c88c..28d837643851 100644
--- a/clang/test/SemaTemplate/instantiate-using-decl.cpp
+++ b/clang/test/SemaTemplate/instantiate-using-decl.cpp
@@ -121,7 +121,7 @@ template <typename Scalar> struct Derived : Base<Scalar> {
     (void)&field;
     // expected-error@+1 {{call to non-static member function without an object argument}}
     (void)method;
-    // expected-error@+1 {{call to non-static member function without an object argument}}
+    // expected-error@+1 {{must explicitly qualify name of member function when taking its address}}
     (void)&method;
     // expected-error@+1 {{call to non-static member function without an object argument}}
     method();
diff --git a/clang/test/SemaTemplate/ms-function-specialization-class-scope.cpp b/clang/test/SemaTemplate/ms-function-specialization-class-scope.cpp
index dcab9bfaeabc..f7b59f2ac715 100644
--- a/clang/test/SemaTemplate/ms-function-specialization-class-scope.cpp
+++ b/clang/test/SemaTemplate/ms-function-specialization-class-scope.cpp
@@ -1,7 +1,6 @@
-// RUN: %clang_cc1 -fms-extensions -fsyntax-only -verify %s
-// RUN: %clang_cc1 -fms-extensions -fdelayed-template-parsing -fsyntax-only -verify %s
+// RUN: %clang_cc1 -fms-extensions -fsyntax-only -Wno-unused-value -verify %s
+// RUN: %clang_cc1 -fms-extensions -fdelayed-template-parsing -fsyntax-only -Wno-unused-value -verify %s
 
-// expected-no-diagnostics
 class A {
 public:
   template<class U> A(U p) {}
@@ -76,3 +75,104 @@ struct S {
   int f<0>(int);
 };
 }
+
+namespace UsesThis {
+  template<typename T>
+  struct A {
+    int x;
+
+    static inline int y;
+
+    template<typename U = void>
+    static void f();
+
+    template<typename U = void>
+    void g();
+
+    template<typename U>
+    static auto h() -> A*;
+
+    void i();
+
+    static void j();
+
+    template<>
+    void f<int>() {
+      this->x; // expected-error {{invalid use of 'this' outside of a non-static member function}}
+      x; // expected-error {{invalid use of member 'x' in static member function}}
+      A::x; // expected-error {{invalid use of member 'x' in static member function}}
+      +x; // expected-error {{invalid use of member 'x' in static member function}}
+      +A::x; // expected-error {{invalid use of member 'x' in static member function}}
+      &x; // expected-error {{invalid use of member 'x' in static member function}}
+      &A::x;
+      this->y; // expected-error {{invalid use of 'this' outside of a non-static member function}}
+      y;
+      A::y;
+      +y;
+      +A::y;
+      &y;
+      &A::y;
+      f();
+      f<void>();
+      g(); // expected-error {{call to non-static member function without an object argument}}
+      g<void>(); // expected-error {{call to non-static member function without an object argument}}
+      i(); // expected-error {{call to non-static member function without an object argument}}
+      j();
+      &i; // expected-error 2{{must explicitly qualify name of member function when taking its address}}
+      &j;
+      &A::i;
+      &A::j;
+    }
+
+    template<>
+    void g<int>() {
+      this->x;
+      x;
+      A::x;
+      +x;
+      +A::x;
+      &x;
+      &A::x;
+      this->y;
+      y;
+      A::y;
+      +y;
+      +A::y;
+      &y;
+      &A::y;
+      f();
+      f<void>();
+      g();
+      g<void>();
+      i();
+      j();
+      &i; // expected-error 2{{must explicitly qualify name of member function when taking its address}}
+      &j;
+      &A::i;
+      &A::j;
+    }
+
+    template<>
+    auto h<int>() -> decltype(this); // expected-error {{'this' cannot be used in a static member function declaration}}
+  };
+
+  template struct A<int>; // expected-note 3{{in instantiation of}}
+
+  template <typename T>
+  struct Foo {
+    template <typename X>
+    int bar(X x) {
+      return 0;
+    }
+
+    template <>
+    int bar(int x) {
+      return bar(5.0); // ok
+    }
+  };
+
+  void call() {
+    Foo<double> f;
+    f.bar(1);
+  }
+}
-- 
GitLab


From 08e210c6af68005e1bd27488d92de2efba548f58 Mon Sep 17 00:00:00 2001
From: Mingming Liu <mingmingl@google.com>
Date: Thu, 11 Apr 2024 13:28:20 -0700
Subject: [PATCH 003/464] [NFC][IndirectCallProm] Refactor function-based
 conditional devirtualization and indirect call value profile update into one
 helper function (#80762)

* The motivation is to move indirect callee profile update inside the
function-based speculative indirect-call promotion, so that there are
fewer diffs the vtable-based transformation and profile update is
implemented in a follow-up patch.
* The Parent patch is https://github.com/llvm/llvm-project/pull/79381
---
 llvm/include/llvm/ProfileData/InstrProf.h     |  3 +-
 llvm/lib/ProfileData/InstrProf.cpp            |  2 +
 .../Instrumentation/IndirectCallPromotion.cpp | 49 +++++++++++--------
 3 files changed, 32 insertions(+), 22 deletions(-)

diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index eb3c10bcba1c..d5c1ba62911f 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -277,6 +277,8 @@ void annotateValueSite(Module &M, Instruction &Inst,
                        uint32_t MaxMDCount = 3);
 
 /// Same as the above interface but using an ArrayRef, as well as \p Sum.
+/// This function will not annotate !prof metadata on the instruction if the
+/// referenced array is empty.
 void annotateValueSite(Module &M, Instruction &Inst,
                        ArrayRef<InstrProfValueData> VDs, uint64_t Sum,
                        InstrProfValueKind ValueKind, uint32_t MaxMDCount);
@@ -465,7 +467,6 @@ private:
   StringSet<> VTableNames;
   // A map from MD5 keys to function name strings.
   std::vector<std::pair<uint64_t, StringRef>> MD5NameMap;
-
   // A map from MD5 keys to function define. We only populate this map
   // when build the Symtab from a Module.
   std::vector<std::pair<uint64_t, Function *>> MD5FuncMap;
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index 95f900d0fff1..a5abf63b010f 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -1246,6 +1246,8 @@ void annotateValueSite(Module &M, Instruction &Inst,
                        ArrayRef<InstrProfValueData> VDs,
                        uint64_t Sum, InstrProfValueKind ValueKind,
                        uint32_t MaxMDCount) {
+  if (VDs.empty())
+    return;
   LLVMContext &Ctx = M.getContext();
   MDBuilder MDHelper(Ctx);
   SmallVector<Metadata *, 3> Vals;
diff --git a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
index 7344fea17517..23a7c6a20aec 100644
--- a/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
+++ b/llvm/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
@@ -136,11 +136,13 @@ private:
       const CallBase &CB, const ArrayRef<InstrProfValueData> &ValueDataRef,
       uint64_t TotalCount, uint32_t NumCandidates);
 
-  // Promote a list of targets for one indirect-call callsite. Return
-  // the number of promotions.
-  uint32_t tryToPromote(CallBase &CB,
-                        const std::vector<PromotionCandidate> &Candidates,
-                        uint64_t &TotalCount);
+  // Promote a list of targets for one indirect-call callsite by comparing
+  // indirect callee with functions. Returns true if there are IR
+  // transformations and false otherwise.
+  bool tryToPromoteWithFuncCmp(
+      CallBase &CB, const std::vector<PromotionCandidate> &Candidates,
+      uint64_t TotalCount, ArrayRef<InstrProfValueData> ICallProfDataRef,
+      uint32_t NumCandidates);
 
 public:
   IndirectCallPromoter(Function &Func, InstrProfSymtab *Symtab, bool SamplePGO,
@@ -273,9 +275,10 @@ CallBase &llvm::pgo::promoteIndirectCall(CallBase &CB, Function *DirectCallee,
 }
 
 // Promote indirect-call to conditional direct-call for one callsite.
-uint32_t IndirectCallPromoter::tryToPromote(
+bool IndirectCallPromoter::tryToPromoteWithFuncCmp(
     CallBase &CB, const std::vector<PromotionCandidate> &Candidates,
-    uint64_t &TotalCount) {
+    uint64_t TotalCount, ArrayRef<InstrProfValueData> ICallProfDataRef,
+    uint32_t NumCandidates) {
   uint32_t NumPromoted = 0;
 
   for (const auto &C : Candidates) {
@@ -287,7 +290,22 @@ uint32_t IndirectCallPromoter::tryToPromote(
     NumOfPGOICallPromotion++;
     NumPromoted++;
   }
-  return NumPromoted;
+
+  if (NumPromoted == 0)
+    return false;
+
+  // Adjust the MD.prof metadata. First delete the old one.
+  CB.setMetadata(LLVMContext::MD_prof, nullptr);
+
+  assert(NumPromoted <= ICallProfDataRef.size() &&
+         "Number of promoted functions should not be greater than the number "
+         "of values in profile metadata");
+  // Annotate the remaining value profiles if counter is not zero.
+  if (TotalCount != 0)
+    annotateValueSite(*F.getParent(), CB, ICallProfDataRef.slice(NumPromoted),
+                      TotalCount, IPVK_IndirectCallTarget, NumCandidates);
+
+  return true;
 }
 
 // Traverse all the indirect-call callsite and get the value profile
@@ -305,19 +323,8 @@ bool IndirectCallPromoter::processFunction(ProfileSummaryInfo *PSI) {
       continue;
     auto PromotionCandidates = getPromotionCandidatesForCallSite(
         *CB, ICallProfDataRef, TotalCount, NumCandidates);
-    uint32_t NumPromoted = tryToPromote(*CB, PromotionCandidates, TotalCount);
-    if (NumPromoted == 0)
-      continue;
-
-    Changed = true;
-    // Adjust the MD.prof metadata. First delete the old one.
-    CB->setMetadata(LLVMContext::MD_prof, nullptr);
-    // If all promoted, we don't need the MD.prof metadata.
-    if (TotalCount == 0 || NumPromoted == NumVals)
-      continue;
-    // Otherwise we need update with the un-promoted records back.
-    annotateValueSite(*F.getParent(), *CB, ICallProfDataRef.slice(NumPromoted),
-                      TotalCount, IPVK_IndirectCallTarget, NumCandidates);
+    Changed |= tryToPromoteWithFuncCmp(*CB, PromotionCandidates, TotalCount,
+                                       ICallProfDataRef, NumCandidates);
   }
   return Changed;
 }
-- 
GitLab


From 4df854e1a98ed47c955838d2c94bf2913c50041f Mon Sep 17 00:00:00 2001
From: Chelsea Cassanova <chelsea_cassanova@apple.com>
Date: Thu, 11 Apr 2024 13:38:26 -0700
Subject: [PATCH 004/464] [lldb][lit] Add MallocNanoZone envvar to Darwin ASan
 builds (#88431)

When builds on Darwin are configured with ASan, running tests will emit
a warning about malloc's nano zone (`malloc: nano zone abandoned due to inability to reserve vm space`) that can interfere with the test
output and cause failures. Setting the environment variable
`MallocNanoZone` to 0 will remove this warning and allow the tests to
run as normal.
---
 lldb/test/API/lit.cfg.py   | 1 +
 lldb/test/Shell/lit.cfg.py | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/lldb/test/API/lit.cfg.py b/lldb/test/API/lit.cfg.py
index f9497b632fc5..9ea389c639a0 100644
--- a/lldb/test/API/lit.cfg.py
+++ b/lldb/test/API/lit.cfg.py
@@ -121,6 +121,7 @@ if is_configured("llvm_use_sanitizer"):
             config.environment["DYLD_INSERT_LIBRARIES"] = find_sanitizer_runtime(
                 "libclang_rt.asan_osx_dynamic.dylib"
             )
+            config.environment["MallocNanoZone"] = "0"
 
     if "Thread" in config.llvm_use_sanitizer:
         config.environment["TSAN_OPTIONS"] = "halt_on_error=1"
diff --git a/lldb/test/Shell/lit.cfg.py b/lldb/test/Shell/lit.cfg.py
index 31afe5151c06..345627835d2b 100644
--- a/lldb/test/Shell/lit.cfg.py
+++ b/lldb/test/Shell/lit.cfg.py
@@ -52,6 +52,8 @@ llvm_config.with_system_environment(
 # Enable sanitizer runtime flags.
 config.environment["ASAN_OPTIONS"] = "detect_stack_use_after_return=1"
 config.environment["TSAN_OPTIONS"] = "halt_on_error=1"
+if "Darwin" in config.host_os:
+    config.environment["MallocNanoZone"] = "0"
 
 # Support running the test suite under the lldb-repro wrapper. This makes it
 # possible to capture a test suite run and then rerun all the test from the
-- 
GitLab


From 1f5d130df85c2d0550dc8687ad0fa1d96856c318 Mon Sep 17 00:00:00 2001
From: Chelsea Cassanova <chelsea_cassanova@apple.com>
Date: Thu, 11 Apr 2024 13:46:32 -0700
Subject: [PATCH 005/464] Revert "[lldb][lit] Add MallocNanoZone envvar to
 Darwin ASan builds" (#88436)

Reverts llvm/llvm-project#88431. A config variable for the host OS is
unrecognized on the Linux builds and is causing a build failure.
---
 lldb/test/API/lit.cfg.py   | 1 -
 lldb/test/Shell/lit.cfg.py | 2 --
 2 files changed, 3 deletions(-)

diff --git a/lldb/test/API/lit.cfg.py b/lldb/test/API/lit.cfg.py
index 9ea389c639a0..f9497b632fc5 100644
--- a/lldb/test/API/lit.cfg.py
+++ b/lldb/test/API/lit.cfg.py
@@ -121,7 +121,6 @@ if is_configured("llvm_use_sanitizer"):
             config.environment["DYLD_INSERT_LIBRARIES"] = find_sanitizer_runtime(
                 "libclang_rt.asan_osx_dynamic.dylib"
             )
-            config.environment["MallocNanoZone"] = "0"
 
     if "Thread" in config.llvm_use_sanitizer:
         config.environment["TSAN_OPTIONS"] = "halt_on_error=1"
diff --git a/lldb/test/Shell/lit.cfg.py b/lldb/test/Shell/lit.cfg.py
index 345627835d2b..31afe5151c06 100644
--- a/lldb/test/Shell/lit.cfg.py
+++ b/lldb/test/Shell/lit.cfg.py
@@ -52,8 +52,6 @@ llvm_config.with_system_environment(
 # Enable sanitizer runtime flags.
 config.environment["ASAN_OPTIONS"] = "detect_stack_use_after_return=1"
 config.environment["TSAN_OPTIONS"] = "halt_on_error=1"
-if "Darwin" in config.host_os:
-    config.environment["MallocNanoZone"] = "0"
 
 # Support running the test suite under the lldb-repro wrapper. This makes it
 # possible to capture a test suite run and then rerun all the test from the
-- 
GitLab


From 44de2bb6949f0ca62a2e16506fe3d91be14e6d23 Mon Sep 17 00:00:00 2001
From: Nathan Lanza <nathanlanza@gmail.com>
Date: Thu, 11 Apr 2024 16:56:31 -0400
Subject: [PATCH 006/464] [CIR][cmake] Add support for cmake variable
 CLANG_ENABLE_CIR

Introduce a cmake variable that guards the inclusion of ClangIR into the
build of clang. Guard that we aren't trying to build without MLIR. Add
two subdirectories that, as of now, don't do anything.

Reviewers: bcardosolopes, erichkeane, petrhosek, Ericson2314

Reviewed By: bcardosolopes

Pull Request: https://github.com/llvm/llvm-project/pull/86078
---
 clang/CMakeLists.txt                      | 7 +++++++
 clang/include/clang/CIR/CMakeLists.txt    | 0
 clang/include/clang/CMakeLists.txt        | 3 +++
 clang/include/clang/Config/config.h.cmake | 3 +++
 clang/lib/CIR/CMakeLists.txt              | 0
 clang/lib/CMakeLists.txt                  | 4 ++++
 clang/test/CMakeLists.txt                 | 1 +
 clang/test/lit.site.cfg.py.in             | 1 +
 8 files changed, 19 insertions(+)
 create mode 100644 clang/include/clang/CIR/CMakeLists.txt
 create mode 100644 clang/lib/CIR/CMakeLists.txt

diff --git a/clang/CMakeLists.txt b/clang/CMakeLists.txt
index 284b2af24dda..f092766fa19f 100644
--- a/clang/CMakeLists.txt
+++ b/clang/CMakeLists.txt
@@ -165,6 +165,13 @@ if(CLANG_ENABLE_LIBXML2)
   endif()
 endif()
 
+if(CLANG_ENABLE_CIR)
+  if (NOT "${LLVM_ENABLE_PROJECTS}" MATCHES "MLIR|mlir")
+    message(FATAL_ERROR
+      "Cannot build ClangIR without MLIR in LLVM_ENABLE_PROJECTS")
+  endif()
+endif()
+
 include(CheckIncludeFile)
 check_include_file(sys/resource.h CLANG_HAVE_RLIMITS)
 
diff --git a/clang/include/clang/CIR/CMakeLists.txt b/clang/include/clang/CIR/CMakeLists.txt
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/include/clang/CMakeLists.txt b/clang/include/clang/CMakeLists.txt
index 0dc9ea5ed8ac..47ac70cd2169 100644
--- a/clang/include/clang/CMakeLists.txt
+++ b/clang/include/clang/CMakeLists.txt
@@ -1,5 +1,8 @@
 add_subdirectory(AST)
 add_subdirectory(Basic)
+if(CLANG_ENABLE_CIR)
+  add_subdirectory(CIR)
+endif()
 add_subdirectory(Driver)
 add_subdirectory(Parse)
 add_subdirectory(Sema)
diff --git a/clang/include/clang/Config/config.h.cmake b/clang/include/clang/Config/config.h.cmake
index 4015ac804086..27ed69e21562 100644
--- a/clang/include/clang/Config/config.h.cmake
+++ b/clang/include/clang/Config/config.h.cmake
@@ -83,4 +83,7 @@
 /* Spawn a new process clang.exe for the CC1 tool invocation, when necessary */
 #cmakedefine01 CLANG_SPAWN_CC1
 
+/* Whether CIR is built into Clang */
+#cmakedefine01 CLANG_ENABLE_CIR
+
 #endif
diff --git a/clang/lib/CIR/CMakeLists.txt b/clang/lib/CIR/CMakeLists.txt
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/lib/CMakeLists.txt b/clang/lib/CMakeLists.txt
index 0cac86451f39..14ba55360fe0 100644
--- a/clang/lib/CMakeLists.txt
+++ b/clang/lib/CMakeLists.txt
@@ -31,3 +31,7 @@ if(CLANG_INCLUDE_TESTS)
 endif()
 add_subdirectory(Interpreter)
 add_subdirectory(Support)
+
+if(CLANG_ENABLE_CIR)
+  add_subdirectory(CIR)
+endif()
diff --git a/clang/test/CMakeLists.txt b/clang/test/CMakeLists.txt
index fcfca354f4a7..df34a5707da3 100644
--- a/clang/test/CMakeLists.txt
+++ b/clang/test/CMakeLists.txt
@@ -9,6 +9,7 @@ llvm_canonicalize_cmake_booleans(
   CLANG_ENABLE_STATIC_ANALYZER
   CLANG_PLUGIN_SUPPORT
   CLANG_SPAWN_CC1
+  CLANG_ENABLE_CIR
   ENABLE_BACKTRACES
   LLVM_ENABLE_ZLIB
   LLVM_ENABLE_ZSTD
diff --git a/clang/test/lit.site.cfg.py.in b/clang/test/lit.site.cfg.py.in
index ef75770a2c3c..6641811c5883 100644
--- a/clang/test/lit.site.cfg.py.in
+++ b/clang/test/lit.site.cfg.py.in
@@ -27,6 +27,7 @@ config.clang_default_pie_on_linux = @CLANG_DEFAULT_PIE_ON_LINUX@
 config.clang_default_cxx_stdlib = "@CLANG_DEFAULT_CXX_STDLIB@"
 config.clang_staticanalyzer = @CLANG_ENABLE_STATIC_ANALYZER@
 config.clang_staticanalyzer_z3 = @LLVM_WITH_Z3@
+config.clang_enable_cir = @CLANG_ENABLE_CIR@
 config.clang_examples = @CLANG_BUILD_EXAMPLES@
 config.enable_shared = @ENABLE_SHARED@
 config.enable_backtrace = @ENABLE_BACKTRACES@
-- 
GitLab


From a952c123880eb1168f1021b116485e27170d48ca Mon Sep 17 00:00:00 2001
From: Marius Brehler <marius.brehler@iml.fraunhofer.de>
Date: Thu, 11 Apr 2024 22:58:47 +0200
Subject: [PATCH 007/464] [GitHub] Set top level token permission (#87326)

---
 .github/workflows/pr-code-format.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/pr-code-format.yml b/.github/workflows/pr-code-format.yml
index 10b18f245d89..983838858ba4 100644
--- a/.github/workflows/pr-code-format.yml
+++ b/.github/workflows/pr-code-format.yml
@@ -1,4 +1,8 @@
 name: "Check code formatting"
+
+permissions:
+  contents: read
+
 on:
   pull_request:
     branches:
-- 
GitLab


From 84df7a09f8da5809b85fd097015e5ac6cc8a3f88 Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda@apple.com>
Date: Thu, 11 Apr 2024 14:44:55 -0700
Subject: [PATCH 008/464] [clang][modules] Do not resolve `HeaderFileInfo`
 externally in `ASTWriter` (#87848)

Clang uses the `HeaderFileInfo` struct to track bits of information on
header files, which gets used throughout the compiler. We also use this
to compute the set of affecting module maps in `ASTWriter` and in the
end serialize the information into the `HEADER_SEARCH_TABLE` record of a
PCM file, allowing clients to learn about headers from the module. In
doing so, Clang asks for existing `HeaderFileInfo` for all known
`FileEntries`. Note that this asks the loaded PCM files for the
information they have on each header file in question. This seems
unnecessary: we only want to serialize information on header files that
either belong to the current module or that got included textually.
Loaded PCM files can't provide us with any useful information.

For explicit modules with lazy loading (using `-fmodule-map-file=<path>`
with `-fmodule-file=<name>=<path>`) the compiler knows about header
files listed in the module map files on the command-line. This can be a
large number.

Asking for existing `HeaderFileInfo` can trigger deserialization of
`HEADER_SEARCH_TABLE` from loaded PCM files. Keys of the on-disk hash
table consist of the header file size and modification time. However,
with explicit modules Clang zeroes out the modification time. Moreover,
if you import lots of modules, some of their header files end up having
identical sizes. This means lots of hash collisions that can only be
resolved by running the serialized filename through `FileManager` and
comparing equality of the `FileEntry`. This ends up being super
expensive, essentially re-stating lots of the transitively loaded SDK
header files.

This patch cleans up the API for getting `HeaderFileInfo` and makes sure
`ASTWriter` uses the version that doesn't ask loaded PCM files for more
information. This removes the excessive stat traffic coming from
`ASTWriter` hopefully without changing observable behavior.
---
 .../clangd/index/SymbolCollector.cpp          |  2 +-
 clang/include/clang/Lex/HeaderSearch.h        | 24 +++---
 clang/lib/Lex/HeaderSearch.cpp                | 77 ++++++++++---------
 clang/lib/Serialization/ASTWriter.cpp         | 11 +--
 4 files changed, 60 insertions(+), 54 deletions(-)

diff --git a/clang-tools-extra/clangd/index/SymbolCollector.cpp b/clang-tools-extra/clangd/index/SymbolCollector.cpp
index 85b8fc549b01..969f27af6fb1 100644
--- a/clang-tools-extra/clangd/index/SymbolCollector.cpp
+++ b/clang-tools-extra/clangd/index/SymbolCollector.cpp
@@ -409,7 +409,7 @@ private:
     // Framework headers are spelled as <FrameworkName/Foo.h>, not
     // "path/FrameworkName.framework/Headers/Foo.h".
     auto &HS = PP->getHeaderSearchInfo();
-    if (const auto *HFI = HS.getExistingFileInfo(*FE, /*WantExternal*/ false))
+    if (const auto *HFI = HS.getExistingLocalFileInfo(*FE))
       if (!HFI->Framework.empty())
         if (auto Spelling =
                 getFrameworkHeaderIncludeSpelling(*FE, HFI->Framework, HS))
diff --git a/clang/include/clang/Lex/HeaderSearch.h b/clang/include/clang/Lex/HeaderSearch.h
index 855f81f775f8..c5f90ef4cb36 100644
--- a/clang/include/clang/Lex/HeaderSearch.h
+++ b/clang/include/clang/Lex/HeaderSearch.h
@@ -547,14 +547,15 @@ public:
   /// Return whether the specified file is a normal header,
   /// a system header, or a C++ friendly system header.
   SrcMgr::CharacteristicKind getFileDirFlavor(FileEntryRef File) {
-    return (SrcMgr::CharacteristicKind)getFileInfo(File).DirInfo;
+    if (const HeaderFileInfo *HFI = getExistingFileInfo(File))
+      return (SrcMgr::CharacteristicKind)HFI->DirInfo;
+    return (SrcMgr::CharacteristicKind)HeaderFileInfo().DirInfo;
   }
 
   /// Mark the specified file as a "once only" file due to
   /// \#pragma once.
   void MarkFileIncludeOnce(FileEntryRef File) {
-    HeaderFileInfo &FI = getFileInfo(File);
-    FI.isPragmaOnce = true;
+    getFileInfo(File).isPragmaOnce = true;
   }
 
   /// Mark the specified file as a system header, e.g. due to
@@ -834,16 +835,17 @@ public:
 
   unsigned header_file_size() const { return FileInfo.size(); }
 
-  /// Return the HeaderFileInfo structure for the specified FileEntry,
-  /// in preparation for updating it in some way.
+  /// Return the HeaderFileInfo structure for the specified FileEntry, in
+  /// preparation for updating it in some way.
   HeaderFileInfo &getFileInfo(FileEntryRef FE);
 
-  /// Return the HeaderFileInfo structure for the specified FileEntry,
-  /// if it has ever been filled in.
-  /// \param WantExternal Whether the caller wants purely-external header file
-  ///        info (where \p External is true).
-  const HeaderFileInfo *getExistingFileInfo(FileEntryRef FE,
-                                            bool WantExternal = true) const;
+  /// Return the HeaderFileInfo structure for the specified FileEntry, if it has
+  /// ever been filled in (either locally or externally).
+  const HeaderFileInfo *getExistingFileInfo(FileEntryRef FE) const;
+
+  /// Return the headerFileInfo structure for the specified FileEntry, if it has
+  /// ever been filled in locally.
+  const HeaderFileInfo *getExistingLocalFileInfo(FileEntryRef FE) const;
 
   SearchDirIterator search_dir_begin() { return {*this, 0}; }
   SearchDirIterator search_dir_end() { return {*this, SearchDirs.size()}; }
diff --git a/clang/lib/Lex/HeaderSearch.cpp b/clang/lib/Lex/HeaderSearch.cpp
index f0750e5336b6..0632882b2961 100644
--- a/clang/lib/Lex/HeaderSearch.cpp
+++ b/clang/lib/Lex/HeaderSearch.cpp
@@ -946,9 +946,13 @@ OptionalFileEntryRef HeaderSearch::LookupFile(
       // If we have no includer, that means we're processing a #include
       // from a module build. We should treat this as a system header if we're
       // building a [system] module.
-      bool IncluderIsSystemHeader =
-          Includer ? getFileInfo(*Includer).DirInfo != SrcMgr::C_User :
-          BuildSystemModule;
+      bool IncluderIsSystemHeader = [&]() {
+        if (!Includer)
+          return BuildSystemModule;
+        const HeaderFileInfo *HFI = getExistingFileInfo(*Includer);
+        assert(HFI && "includer without file info");
+        return HFI->DirInfo != SrcMgr::C_User;
+      }();
       if (OptionalFileEntryRef FE = getFileAndSuggestModule(
               TmpDir, IncludeLoc, IncluderAndDir.second, IncluderIsSystemHeader,
               RequestingModule, SuggestedModule)) {
@@ -963,10 +967,11 @@ OptionalFileEntryRef HeaderSearch::LookupFile(
         // Note that we only use one of FromHFI/ToHFI at once, due to potential
         // reallocation of the underlying vector potentially making the first
         // reference binding dangling.
-        HeaderFileInfo &FromHFI = getFileInfo(*Includer);
-        unsigned DirInfo = FromHFI.DirInfo;
-        bool IndexHeaderMapHeader = FromHFI.IndexHeaderMapHeader;
-        StringRef Framework = FromHFI.Framework;
+        const HeaderFileInfo *FromHFI = getExistingFileInfo(*Includer);
+        assert(FromHFI && "includer without file info");
+        unsigned DirInfo = FromHFI->DirInfo;
+        bool IndexHeaderMapHeader = FromHFI->IndexHeaderMapHeader;
+        StringRef Framework = FromHFI->Framework;
 
         HeaderFileInfo &ToHFI = getFileInfo(*FE);
         ToHFI.DirInfo = DirInfo;
@@ -1153,10 +1158,12 @@ OptionalFileEntryRef HeaderSearch::LookupFile(
   // "Foo" is the name of the framework in which the including header was found.
   if (!Includers.empty() && Includers.front().first && !isAngled &&
       !Filename.contains('/')) {
-    HeaderFileInfo &IncludingHFI = getFileInfo(*Includers.front().first);
-    if (IncludingHFI.IndexHeaderMapHeader) {
+    const HeaderFileInfo *IncludingHFI =
+        getExistingFileInfo(*Includers.front().first);
+    assert(IncludingHFI && "includer without file info");
+    if (IncludingHFI->IndexHeaderMapHeader) {
       SmallString<128> ScratchFilename;
-      ScratchFilename += IncludingHFI.Framework;
+      ScratchFilename += IncludingHFI->Framework;
       ScratchFilename += '/';
       ScratchFilename += Filename;
 
@@ -1286,11 +1293,11 @@ OptionalFileEntryRef HeaderSearch::LookupSubframeworkHeader(
   }
 
   // This file is a system header or C++ unfriendly if the old file is.
-  //
-  // Note that the temporary 'DirInfo' is required here, as either call to
-  // getFileInfo could resize the vector and we don't want to rely on order
-  // of evaluation.
-  unsigned DirInfo = getFileInfo(ContextFileEnt).DirInfo;
+  const HeaderFileInfo *ContextHFI = getExistingFileInfo(ContextFileEnt);
+  assert(ContextHFI && "context file without file info");
+  // Note that the temporary 'DirInfo' is required here, as the call to
+  // getFileInfo could resize the vector and might invalidate 'ContextHFI'.
+  unsigned DirInfo = ContextHFI->DirInfo;
   getFileInfo(*File).DirInfo = DirInfo;
 
   FrameworkName.pop_back(); // remove the trailing '/'
@@ -1348,8 +1355,6 @@ static void mergeHeaderFileInfo(HeaderFileInfo &HFI,
     HFI.Framework = OtherHFI.Framework;
 }
 
-/// getFileInfo - Return the HeaderFileInfo structure for the specified
-/// FileEntry.
 HeaderFileInfo &HeaderSearch::getFileInfo(FileEntryRef FE) {
   if (FE.getUID() >= FileInfo.size())
     FileInfo.resize(FE.getUID() + 1);
@@ -1366,27 +1371,20 @@ HeaderFileInfo &HeaderSearch::getFileInfo(FileEntryRef FE) {
   }
 
   HFI->IsValid = true;
-  // We have local information about this header file, so it's no longer
-  // strictly external.
+  // We assume the caller has local information about this header file, so it's
+  // no longer strictly external.
   HFI->External = false;
   return *HFI;
 }
 
-const HeaderFileInfo *
-HeaderSearch::getExistingFileInfo(FileEntryRef FE, bool WantExternal) const {
-  // If we have an external source, ensure we have the latest information.
-  // FIXME: Use a generation count to check whether this is really up to date.
+const HeaderFileInfo *HeaderSearch::getExistingFileInfo(FileEntryRef FE) const {
   HeaderFileInfo *HFI;
   if (ExternalSource) {
-    if (FE.getUID() >= FileInfo.size()) {
-      if (!WantExternal)
-        return nullptr;
+    if (FE.getUID() >= FileInfo.size())
       FileInfo.resize(FE.getUID() + 1);
-    }
 
     HFI = &FileInfo[FE.getUID()];
-    if (!WantExternal && (!HFI->IsValid || HFI->External))
-      return nullptr;
+    // FIXME: Use a generation count to check whether this is really up to date.
     if (!HFI->Resolved) {
       auto ExternalHFI = ExternalSource->GetHeaderFileInfo(FE);
       if (ExternalHFI.IsValid) {
@@ -1395,16 +1393,25 @@ HeaderSearch::getExistingFileInfo(FileEntryRef FE, bool WantExternal) const {
           mergeHeaderFileInfo(*HFI, ExternalHFI);
       }
     }
-  } else if (FE.getUID() >= FileInfo.size()) {
-    return nullptr;
-  } else {
+  } else if (FE.getUID() < FileInfo.size()) {
     HFI = &FileInfo[FE.getUID()];
+  } else {
+    HFI = nullptr;
   }
 
-  if (!HFI->IsValid || (HFI->External && !WantExternal))
-    return nullptr;
+  return (HFI && HFI->IsValid) ? HFI : nullptr;
+}
+
+const HeaderFileInfo *
+HeaderSearch::getExistingLocalFileInfo(FileEntryRef FE) const {
+  HeaderFileInfo *HFI;
+  if (FE.getUID() < FileInfo.size()) {
+    HFI = &FileInfo[FE.getUID()];
+  } else {
+    HFI = nullptr;
+  }
 
-  return HFI;
+  return (HFI && HFI->IsValid && !HFI->External) ? HFI : nullptr;
 }
 
 bool HeaderSearch::isFileMultipleIncludeGuarded(FileEntryRef File) const {
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index ffc53292e391..330dbd623350 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -185,8 +185,7 @@ GetAffectingModuleMaps(const Preprocessor &PP, Module *RootModule) {
     if (!File)
       continue;
 
-    const HeaderFileInfo *HFI =
-        HS.getExistingFileInfo(*File, /*WantExternal*/ false);
+    const HeaderFileInfo *HFI = HS.getExistingLocalFileInfo(*File);
     if (!HFI || (HFI->isModuleHeader && !HFI->isCompilingModuleHeader))
       continue;
 
@@ -2052,14 +2051,12 @@ void ASTWriter::WriteHeaderSearch(const HeaderSearch &HS) {
     if (!File)
       continue;
 
-    // Get the file info. This will load info from the external source if
-    // necessary. Skip emitting this file if we have no information on it
-    // as a header file (in which case HFI will be null) or if it hasn't
+    // Get the file info. Skip emitting this file if we have no information on
+    // it as a header file (in which case HFI will be null) or if it hasn't
     // changed since it was loaded. Also skip it if it's for a modular header
     // from a different module; in that case, we rely on the module(s)
     // containing the header to provide this information.
-    const HeaderFileInfo *HFI =
-        HS.getExistingFileInfo(*File, /*WantExternal*/!Chain);
+    const HeaderFileInfo *HFI = HS.getExistingLocalFileInfo(*File);
     if (!HFI || (HFI->isModuleHeader && !HFI->isCompilingModuleHeader))
       continue;
 
-- 
GitLab


From 95fbd8d19dff10b5a734e3db1b29cba2da7a983f Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda@apple.com>
Date: Thu, 11 Apr 2024 15:22:35 -0700
Subject: [PATCH 009/464] [clangd] Don't ignore external HFI in
 `SymbolCollector` (#88446)

The `FileEntry` corresponds to a `FileID` containing the
`SourceLocation` of a `NamedDecl` which (I think) might've been
deserialized from a PCM file. Considering external `HeaderFileInfo` here
is most likely the right thing to do here in order to get the correct
spelling in case the current compiler instance has not register this
file as a header yet.
---
 clang-tools-extra/clangd/index/SymbolCollector.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang-tools-extra/clangd/index/SymbolCollector.cpp b/clang-tools-extra/clangd/index/SymbolCollector.cpp
index 969f27af6fb1..5c4e2150cf31 100644
--- a/clang-tools-extra/clangd/index/SymbolCollector.cpp
+++ b/clang-tools-extra/clangd/index/SymbolCollector.cpp
@@ -409,7 +409,7 @@ private:
     // Framework headers are spelled as <FrameworkName/Foo.h>, not
     // "path/FrameworkName.framework/Headers/Foo.h".
     auto &HS = PP->getHeaderSearchInfo();
-    if (const auto *HFI = HS.getExistingLocalFileInfo(*FE))
+    if (const auto *HFI = HS.getExistingFileInfo(*FE))
       if (!HFI->Framework.empty())
         if (auto Spelling =
                 getFrameworkHeaderIncludeSpelling(*FE, HFI->Framework, HS))
-- 
GitLab


From 9a36077e4db30c7da603620762036d4a430e4e62 Mon Sep 17 00:00:00 2001
From: jimingham <jingham@apple.com>
Date: Thu, 11 Apr 2024 15:22:58 -0700
Subject: [PATCH 010/464] Fix error in unrecognized register name handling for
 "SBFrame.register" (#88047)

The code returned lldb.SBValue() when you passed in an unrecognized
register name. But referring to "lldb" is apparently not legal within
the module.

I changed this to just return SBValue(), but then this construct:

(lldb) script
>>> for reg_set in lldb.target.process.thread[0].frames[0].register
...    print(reg)

Runs forever printing "No Value". The __getitem__(key) gets called with
a monotonically increasing by 1 series of integers. I don't know why
Python decided the class we defined should have a generator that returns
positive integers in order, but we can add a more useful one here by
returning an iterator over the flattened list of registers.

Note, the not very aptly named "SBFrame.registers" is an iterator over
register sets, not registers, so the two are not redundant.
---
 lldb/bindings/interface/SBFrameExtensions.i  | 12 +++++++++++-
 lldb/test/API/python_api/frame/TestFrames.py | 12 ++++++++++++
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/lldb/bindings/interface/SBFrameExtensions.i b/lldb/bindings/interface/SBFrameExtensions.i
index 43b22ed7a6b3..e0472280666a 100644
--- a/lldb/bindings/interface/SBFrameExtensions.i
+++ b/lldb/bindings/interface/SBFrameExtensions.i
@@ -44,6 +44,16 @@ STRING_EXTENSION_OUTSIDE(SBFrame)
                 def __init__(self, regs):
                     self.regs = regs
 
+                def __iter__(self):
+                    return self.get_registers()
+
+                def get_registers(self):
+                    for i in range(0,len(self.regs)):
+                        rs = self.regs[i]
+                        for j in range (0,rs.num_children):
+                            reg = rs.GetChildAtIndex(j)
+                            yield reg
+                          
                 def __getitem__(self, key):
                     if type(key) is str:
                         for i in range(0,len(self.regs)):
@@ -52,7 +62,7 @@ STRING_EXTENSION_OUTSIDE(SBFrame)
                                 reg = rs.GetChildAtIndex(j)
                                 if reg.name == key: return reg
                     else:
-                        return lldb.SBValue()
+                        return SBValue()
 
             return registers_access(self.registers)
 
diff --git a/lldb/test/API/python_api/frame/TestFrames.py b/lldb/test/API/python_api/frame/TestFrames.py
index a82b129bc809..dfa96d51830b 100644
--- a/lldb/test/API/python_api/frame/TestFrames.py
+++ b/lldb/test/API/python_api/frame/TestFrames.py
@@ -73,7 +73,19 @@ class FrameAPITestCase(TestBase):
                 gpr_reg_set = lldbutil.get_GPRs(frame)
                 pc_value = gpr_reg_set.GetChildMemberWithName("pc")
                 self.assertTrue(pc_value, "We should have a valid PC.")
+                # Make sure we can also get this from the "register" property:
+                iterator_pc_value = 0
+                found_pc = False
+                for reg in frame.register:
+                    if reg.name == "pc":
+                        found_pc = True
+                        iterator_pc_value = int(reg.GetValue(), 0)
+                        break
+
                 pc_value_int = int(pc_value.GetValue(), 0)
+                self.assertTrue(found_pc, "Found the PC value in the register list")
+                self.assertEqual(iterator_pc_value, pc_value_int, "The methods of finding pc match")
+    
                 # Make sure on arm targets we dont mismatch PC value on the basis of thumb bit.
                 # Frame PC will not have thumb bit set in case of a thumb
                 # instruction as PC.
-- 
GitLab


From 4036a6946e5420bb77a93037c83732be600b9b0b Mon Sep 17 00:00:00 2001
From: Farzon Lotfi <1802579+farzonl@users.noreply.github.com>
Date: Thu, 11 Apr 2024 15:26:25 -0700
Subject: [PATCH 011/464] [HLSL] move rcp to cgbuiltins (#88401)

Removing the intrinsic because there is no opCodes for rcp in DXIL or
SPIR-V.
Moving means we don't have to re-implement this feature for each
backend.

fixes #87784

Co-authored-by: Farzon Lotfi <farzon@farzon.com>
---
 clang/lib/CodeGen/CGBuiltin.cpp               |  13 +-
 clang/test/CodeGenHLSL/builtins/rcp.hlsl      | 127 ++++++++++++------
 .../Target/DirectX/DXILIntrinsicExpansion.cpp |  22 ---
 llvm/test/CodeGen/DirectX/rcp.ll              |  52 -------
 .../test/CodeGen/SPIRV/hlsl-intrinsics/rcp.ll | 126 +++++++++++++++++
 5 files changed, 224 insertions(+), 116 deletions(-)
 delete mode 100644 llvm/test/CodeGen/DirectX/rcp.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-intrinsics/rcp.ll

diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index c052367d2878..c7b219dcfcec 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -18303,9 +18303,16 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
     Value *Op0 = EmitScalarExpr(E->getArg(0));
     if (!E->getArg(0)->getType()->hasFloatingRepresentation())
       llvm_unreachable("rcp operand must have a float representation");
-    return Builder.CreateIntrinsic(
-        /*ReturnType=*/Op0->getType(), Intrinsic::dx_rcp,
-        ArrayRef<Value *>{Op0}, nullptr, "dx.rcp");
+    llvm::Type *Ty = Op0->getType();
+    llvm::Type *EltTy = Ty->getScalarType();
+    Constant *One =
+        Ty->isVectorTy()
+            ? ConstantVector::getSplat(
+                  ElementCount::getFixed(
+                      dyn_cast<FixedVectorType>(Ty)->getNumElements()),
+                  ConstantFP::get(EltTy, 1.0))
+            : ConstantFP::get(EltTy, 1.0);
+    return Builder.CreateFDiv(One, Op0, "hlsl.rcp");
   }
   case Builtin::BI__builtin_hlsl_elementwise_rsqrt: {
     Value *Op0 = EmitScalarExpr(E->getArg(0));
diff --git a/clang/test/CodeGenHLSL/builtins/rcp.hlsl b/clang/test/CodeGenHLSL/builtins/rcp.hlsl
index 9b8406e1f0b6..eb89bcc4c7c0 100644
--- a/clang/test/CodeGenHLSL/builtins/rcp.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/rcp.hlsl
@@ -1,53 +1,102 @@
 // RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
 // RUN:   dxil-pc-shadermodel6.3-library %s -fnative-half-type \
 // RUN:   -emit-llvm -disable-llvm-passes -o - | FileCheck %s \ 
-// RUN:   --check-prefixes=CHECK,NATIVE_HALF
+// RUN:   --check-prefixes=CHECK,DXIL_CHECK,DXIL_NATIVE_HALF,NATIVE_HALF
 // RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
 // RUN:   dxil-pc-shadermodel6.3-library %s -emit-llvm -disable-llvm-passes \
-// RUN:   -o - | FileCheck %s --check-prefixes=CHECK,NO_HALF
-
-// NATIVE_HALF: define noundef half @
-// NATIVE_HALF: %dx.rcp = call half @llvm.dx.rcp.f16(
-// NATIVE_HALF: ret half %dx.rcp
-// NO_HALF: define noundef float @"?test_rcp_half@@YA$halff@$halff@@Z"(
-// NO_HALF: %dx.rcp = call float @llvm.dx.rcp.f32(
-// NO_HALF: ret float %dx.rcp
+// RUN:   -o - | FileCheck %s --check-prefixes=CHECK,DXIL_CHECK,NO_HALF,DXIL_NO_HALF
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN:   spirv-unknown-vulkan-compute %s -fnative-half-type \
+// RUN:   -emit-llvm -disable-llvm-passes -o - | FileCheck %s \ 
+// RUN:   --check-prefixes=CHECK,NATIVE_HALF,SPIR_NATIVE_HALF,SPIR_CHECK
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN:   spirv-unknown-vulkan-compute %s -emit-llvm -disable-llvm-passes \
+// RUN:   -o - | FileCheck %s --check-prefixes=CHECK,NO_HALF,SPIR_NO_HALF,SPIR_CHECK
+
+// DXIL_NATIVE_HALF: define noundef half @
+// SPIR_NATIVE_HALF: define spir_func noundef half @
+// NATIVE_HALF: %hlsl.rcp = fdiv half 0xH3C00, %{{.*}} 
+// NATIVE_HALF: ret half %hlsl.rcp
+// DXIL_NO_HALF: define noundef float @
+// SPIR_NO_HALF: define spir_func noundef float @
+// NO_HALF: %hlsl.rcp = fdiv float 1.000000e+00, %{{.*}}
+// NO_HALF: ret float %hlsl.rcp
 half test_rcp_half(half p0) { return rcp(p0); }
-// NATIVE_HALF: define noundef <2 x half> @
-// NATIVE_HALF: %dx.rcp = call <2 x half> @llvm.dx.rcp.v2f16
-// NATIVE_HALF: ret <2 x half> %dx.rcp
-// NO_HALF: define noundef <2 x float> @
-// NO_HALF: %dx.rcp = call <2 x float> @llvm.dx.rcp.v2f32(
-// NO_HALF: ret <2 x float> %dx.rcp
+
+// DXIL_NATIVE_HALF: define noundef <2 x half> @
+// SPIR_NATIVE_HALF: define spir_func noundef <2 x half> @
+// NATIVE_HALF: %hlsl.rcp = fdiv <2 x half> <half  0xH3C00, half  0xH3C00>, %{{.*}} 
+// NATIVE_HALF: ret <2 x half> %hlsl.rcp
+// DXIL_NO_HALF: define noundef <2 x float> @
+// SPIR_NO_HALF: define spir_func noundef <2 x float> @
+// NO_HALF: %hlsl.rcp = fdiv <2 x float> <float 1.000000e+00, float 1.000000e+00>, %{{.*}}
+// NO_HALF: ret <2 x float> %hlsl.rcp
 half2 test_rcp_half2(half2 p0) { return rcp(p0); }
-// NATIVE_HALF: define noundef <3 x half> @
-// NATIVE_HALF: %dx.rcp = call <3 x half> @llvm.dx.rcp.v3f16
-// NATIVE_HALF: ret <3 x half> %dx.rcp
-// NO_HALF: define noundef <3 x float> @
-// NO_HALF: %dx.rcp = call <3 x float> @llvm.dx.rcp.v3f32(
-// NO_HALF: ret <3 x float> %dx.rcp
+
+// DXIL_NATIVE_HALF: define noundef <3 x half> @
+// SPIR_NATIVE_HALF: define spir_func noundef <3 x half> @
+// NATIVE_HALF: %hlsl.rcp = fdiv <3 x half> <half  0xH3C00, half  0xH3C00, half  0xH3C00>, %{{.*}} 
+// NATIVE_HALF: ret <3 x half> %hlsl.rcp
+// DXIL_NO_HALF: define noundef <3 x float> @
+// SPIR_NO_HALF: define spir_func noundef <3 x float> @
+// NO_HALF: %hlsl.rcp = fdiv <3 x float> <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>, %{{.*}}
+// NO_HALF: ret <3 x float> %hlsl.rcp
 half3 test_rcp_half3(half3 p0) { return rcp(p0); }
-// NATIVE_HALF: define noundef <4 x half> @
-// NATIVE_HALF: %dx.rcp = call <4 x half> @llvm.dx.rcp.v4f16
-// NATIVE_HALF: ret <4 x half> %dx.rcp
-// NO_HALF: define noundef <4 x float> @
-// NO_HALF: %dx.rcp = call <4 x float> @llvm.dx.rcp.v4f32(
-// NO_HALF: ret <4 x float> %dx.rcp
+
+// DXIL_NATIVE_HALF: define noundef <4 x half> @
+// SPIR_NATIVE_HALF: define spir_func noundef <4 x half> @
+// NATIVE_HALF: %hlsl.rcp = fdiv <4 x half> <half  0xH3C00, half  0xH3C00, half  0xH3C00, half  0xH3C00>, %{{.*}} 
+// NATIVE_HALF: ret <4 x half> %hlsl.rcp
+// DXIL_NO_HALF: define noundef <4 x float> @
+// SPIR_NO_HALF: define spir_func noundef <4 x float> @
+// NO_HALF: %hlsl.rcp = fdiv <4 x float> <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>, %{{.*}}
+// NO_HALF: ret <4 x float> %hlsl.rcp
 half4 test_rcp_half4(half4 p0) { return rcp(p0); }
 
-// CHECK: define noundef float @
-// CHECK: %dx.rcp = call float @llvm.dx.rcp.f32(
-// CHECK: ret float %dx.rcp
+// DXIL_CHECK: define noundef float @
+// SPIR_CHECK: define spir_func noundef float @
+// CHECK: %hlsl.rcp = fdiv float 1.000000e+00, %{{.*}}
+// CHECK: ret float %hlsl.rcp
 float test_rcp_float(float p0) { return rcp(p0); }
-// CHECK: define noundef <2 x float> @
-// CHECK: %dx.rcp = call <2 x float> @llvm.dx.rcp.v2f32
-// CHECK: ret <2 x float> %dx.rcp
+
+// DXIL_CHECK: define noundef <2 x float> @
+// SPIR_CHECK: define spir_func noundef <2 x float> @
+// CHECK: %hlsl.rcp = fdiv <2 x float> <float 1.000000e+00, float 1.000000e+00>, %{{.*}}
+// CHECK: ret <2 x float> %hlsl.rcp
 float2 test_rcp_float2(float2 p0) { return rcp(p0); }
-// CHECK: define noundef <3 x float> @
-// CHECK: %dx.rcp = call <3 x float> @llvm.dx.rcp.v3f32
-// CHECK: ret <3 x float> %dx.rcp
+
+// DXIL_CHECK: define noundef <3 x float> @
+// SPIR_CHECK: define spir_func noundef <3 x float> @
+// CHECK: %hlsl.rcp = fdiv <3 x float> <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>, %{{.*}}
+// CHECK: ret <3 x float> %hlsl.rcp
 float3 test_rcp_float3(float3 p0) { return rcp(p0); }
-// CHECK: define noundef <4 x float> @
-// CHECK: %dx.rcp = call <4 x float> @llvm.dx.rcp.v4f32
-// CHECK: ret <4 x float> %dx.rcp
+
+// DXIL_CHECK: define noundef <4 x float> @
+// SPIR_CHECK: define spir_func noundef <4 x float> @
+// CHECK: %hlsl.rcp = fdiv <4 x float> <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>, %{{.*}}
+// CHECK: ret <4 x float> %hlsl.rcp
 float4 test_rcp_float4(float4 p0) { return rcp(p0); }
+
+// DXIL_CHECK: define noundef double @
+// SPIR_CHECK: define spir_func noundef double @
+// CHECK: %hlsl.rcp = fdiv double 1.000000e+00, %{{.*}} 
+// CHECK: ret double %hlsl.rcp
+double test_rcp_double(double p0) { return rcp(p0); }
+
+// DXIL_CHECK: define noundef <2 x double> @
+// SPIR_CHECK: define spir_func noundef <2 x double> @
+// CHECK: %hlsl.rcp = fdiv <2 x double> <double 1.000000e+00, double 1.000000e+00>, %{{.*}}
+// CHECK: ret <2 x double> %hlsl.rcp
+double2 test_rcp_double2(double2 p0) { return rcp(p0); }
+
+// DXIL_CHECK: define noundef <3 x double> @
+// SPIR_CHECK: define spir_func noundef <3 x double> @
+// CHECK: %hlsl.rcp = fdiv <3 x double> <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>, %{{.*}}
+// CHECK: ret <3 x double> %hlsl.rcp
+double3 test_rcp_double3(double3 p0) { return rcp(p0); }
+
+// DXIL_CHECK: define noundef <4 x double> @
+// SPIR_CHECK: define spir_func noundef <4 x double> @
+// CHECK: %hlsl.rcp = fdiv <4 x double> <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>, %{{.*}}
+// CHECK: ret <4 x double> %hlsl.rcp
+double4 test_rcp_double4(double4 p0) { return rcp(p0); }
diff --git a/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp b/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp
index 3e2d10f5ee7a..4d99bc006900 100644
--- a/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp
+++ b/llvm/lib/Target/DirectX/DXILIntrinsicExpansion.cpp
@@ -42,7 +42,6 @@ static bool isIntrinsicExpansion(Function &F) {
   case Intrinsic::dx_clamp:
   case Intrinsic::dx_uclamp:
   case Intrinsic::dx_lerp:
-  case Intrinsic::dx_rcp:
   case Intrinsic::dx_sdot:
   case Intrinsic::dx_udot:
     return true;
@@ -218,25 +217,6 @@ static bool expandPowIntrinsic(CallInst *Orig) {
   return true;
 }
 
-static bool expandRcpIntrinsic(CallInst *Orig) {
-  Value *X = Orig->getOperand(0);
-  IRBuilder<> Builder(Orig->getParent());
-  Builder.SetInsertPoint(Orig);
-  Type *Ty = X->getType();
-  Type *EltTy = Ty->getScalarType();
-  Constant *One =
-      Ty->isVectorTy()
-          ? ConstantVector::getSplat(
-                ElementCount::getFixed(
-                    dyn_cast<FixedVectorType>(Ty)->getNumElements()),
-                ConstantFP::get(EltTy, 1.0))
-          : ConstantFP::get(EltTy, 1.0);
-  auto *Result = Builder.CreateFDiv(One, X, "dx.rcp");
-  Orig->replaceAllUsesWith(Result);
-  Orig->eraseFromParent();
-  return true;
-}
-
 static Intrinsic::ID getMaxForClamp(Type *ElemTy,
                                     Intrinsic::ID ClampIntrinsic) {
   if (ClampIntrinsic == Intrinsic::dx_uclamp)
@@ -300,8 +280,6 @@ static bool expandIntrinsic(Function &F, CallInst *Orig) {
     return expandClampIntrinsic(Orig, F.getIntrinsicID());
   case Intrinsic::dx_lerp:
     return expandLerpIntrinsic(Orig);
-  case Intrinsic::dx_rcp:
-    return expandRcpIntrinsic(Orig);
   case Intrinsic::dx_sdot:
   case Intrinsic::dx_udot:
     return expandIntegerDot(Orig, F.getIntrinsicID());
diff --git a/llvm/test/CodeGen/DirectX/rcp.ll b/llvm/test/CodeGen/DirectX/rcp.ll
deleted file mode 100644
index 65abe832db53..000000000000
--- a/llvm/test/CodeGen/DirectX/rcp.ll
+++ /dev/null
@@ -1,52 +0,0 @@
-; RUN: opt -S -dxil-op-lower < %s | FileCheck %s
-
-; Make sure dxil operation function calls for rcp are generated for float, double, and half.
-
-; CHECK-LABEL: rcp_float4
-; CHECK: fdiv <4 x float> <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>, %{{.*}}
-define noundef <4 x float> @rcp_float4(<4 x float> noundef %p0) {
-entry:
-  %p0.addr = alloca <4 x float>, align 16
-  store <4 x float> %p0, ptr %p0.addr, align 16
-  %0 = load <4 x float>, ptr %p0.addr, align 16
-  %dx.rcp = call <4 x float> @llvm.dx.rcp.v4f32(<4 x float> %0)
-  ret <4 x float> %dx.rcp
-}
-
-; CHECK-LABEL: rcp_double4
-; CHECK: fdiv <4 x double> <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>, %{{.*}}
-define noundef <4 x double> @rcp_double4(<4 x double> noundef %p0) {
-entry:
-  %p0.addr = alloca <4 x double>, align 16
-  store <4 x double> %p0, ptr %p0.addr, align 16
-  %0 = load <4 x double>, ptr %p0.addr, align 16
-  %dx.rcp = call <4 x double> @llvm.dx.rcp.v4f64(<4 x double> %0)
-  ret <4 x double> %dx.rcp
-}
-
-; CHECK-LABEL: rcp_half4
-; CHECK: fdiv <4 x half> <half  0xH3C00, half  0xH3C00, half  0xH3C00, half  0xH3C00>, %{{.*}} 
-define noundef <4 x half> @rcp_half4(<4 x half> noundef %p0) {
-entry:
-  %p0.addr = alloca <4 x half>, align 16
-  store <4 x half> %p0, ptr %p0.addr, align 16
-  %0 = load <4 x half>, ptr %p0.addr, align 16
-  %dx.rcp = call <4 x half> @llvm.dx.rcp.v4f16(<4 x half> %0)
-  ret <4 x half> %dx.rcp
-}
-
-; CHECK-LABEL: rcp_half
-; CHECK: fdiv half 0xH3C00, %{{.*}} 
-define noundef half @rcp_half(half noundef %p0) {
-entry:
-  %p0.addr = alloca half, align 2
-  store half %p0, ptr %p0.addr, align 2
-  %0 = load half, ptr %p0.addr, align 2
-  %dx.rcp = call half @llvm.dx.rcp.f16(half %0)
-  ret half %dx.rcp
-}
-
-declare half @llvm.dx.rcp.f16(half)
-declare <4 x half> @llvm.dx.rcp.v4f16(<4 x half>)
-declare <4 x float> @llvm.dx.rcp.v4f32(<4 x float>)
-declare <4 x double> @llvm.dx.rcp.v4f64(<4 x double>)
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/rcp.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/rcp.ll
new file mode 100644
index 000000000000..95962c0fdc96
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/rcp.ll
@@ -0,0 +1,126 @@
+ ; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s
+
+; CHECK-DAG: %[[#float_64:]] = OpTypeFloat 64
+; CHECK-DAG: %[[#float_32:]] = OpTypeFloat 32
+; CHECK-DAG: %[[#float_16:]] = OpTypeFloat 16
+; CHECK-DAG: %[[#vec2_float_16:]] = OpTypeVector %[[#float_16]] 2
+; CHECK-DAG: %[[#vec2_float_32:]] = OpTypeVector %[[#float_32]] 2
+; CHECK-DAG: %[[#vec2_float_64:]] = OpTypeVector %[[#float_64]] 2
+; CHECK-DAG: %[[#vec3_float_16:]] = OpTypeVector %[[#float_16]] 3
+; CHECK-DAG: %[[#vec3_float_32:]] = OpTypeVector %[[#float_32]] 3
+; CHECK-DAG: %[[#vec3_float_64:]] = OpTypeVector %[[#float_64]] 3
+; CHECK-DAG: %[[#vec4_float_16:]] = OpTypeVector %[[#float_16]] 4
+; CHECK-DAG: %[[#vec4_float_32:]] = OpTypeVector %[[#float_32]] 4
+; CHECK-DAG: %[[#vec4_float_64:]] = OpTypeVector %[[#float_64]] 4
+; CHECK-DAG: %[[#const_f64_1:]] = OpConstant %[[#float_64]] 1
+; CHECK-DAG: %[[#const_f32_1:]] = OpConstant %[[#float_32:]] 1
+; CHECK-DAG: %[[#const_f16_1:]] = OpConstant %[[#float_16:]] 1
+
+; CHECK-DAG: %[[#vec2_const_ones_f16:]] = OpConstantComposite %[[#vec2_float_16:]] %[[#const_f16_1:]] %[[#const_f16_1:]]
+; CHECK-DAG: %[[#vec3_const_ones_f16:]] = OpConstantComposite %[[#vec3_float_16:]] %[[#const_f16_1:]] %[[#const_f16_1:]] %[[#const_f16_1:]]
+; CHECK-DAG: %[[#vec4_const_ones_f16:]] = OpConstantComposite %[[#vec4_float_16:]] %[[#const_f16_1:]] %[[#const_f16_1:]] %[[#const_f16_1:]] %[[#const_f16_1:]]
+
+; CHECK-DAG: %[[#vec2_const_ones_f32:]] = OpConstantComposite %[[#vec2_float_32:]] %[[#const_f32_1:]] %[[#const_f32_1:]]
+; CHECK-DAG: %[[#vec3_const_ones_f32:]] = OpConstantComposite %[[#vec3_float_32:]] %[[#const_f32_1:]] %[[#const_f32_1:]] %[[#const_f32_1:]]
+; CHECK-DAG: %[[#vec4_const_ones_f32:]] = OpConstantComposite %[[#vec4_float_32:]] %[[#const_f32_1:]] %[[#const_f32_1:]] %[[#const_f32_1:]] %[[#const_f32_1:]]
+
+; CHECK-DAG: %[[#vec2_const_ones_f64:]] = OpConstantComposite %[[#vec2_float_64:]] %[[#const_f64_1:]] %[[#const_f64_1:]]
+; CHECK-DAG: %[[#vec3_const_ones_f64:]] = OpConstantComposite %[[#vec3_float_64:]] %[[#const_f64_1:]] %[[#const_f64_1:]] %[[#const_f64_1:]]
+; CHECK-DAG: %[[#vec4_const_ones_f64:]] = OpConstantComposite %[[#vec4_float_64:]] %[[#const_f64_1:]] %[[#const_f64_1:]] %[[#const_f64_1:]] %[[#const_f64_1:]]
+
+
+define spir_func noundef half @test_rcp_half(half noundef %p0) #0 {
+entry:
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#float_16:]]
+  ; CHECK: OpFDiv %[[#float_16:]] %[[#const_f16_1:]] %[[#arg0:]]
+  %hlsl.rcp = fdiv half 0xH3C00, %p0
+  ret half %hlsl.rcp
+}
+
+define spir_func noundef <2 x half> @test_rcp_half2(<2 x half> noundef %p0) #0 {
+entry:
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#vec2_float_16:]]
+  ; CHECK: OpFDiv %[[#vec2_float_16:]] %[[#vec2_const_ones_f16:]] %[[#arg0:]]
+  %hlsl.rcp = fdiv <2 x half> <half 0xH3C00, half 0xH3C00>, %p0
+  ret <2 x half> %hlsl.rcp
+}
+
+define spir_func noundef <3 x half> @test_rcp_half3(<3 x half> noundef %p0) #0 {
+entry:
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#vec3_float_16:]]
+  ; CHECK: OpFDiv %[[#vec3_float_16:]] %[[#vec3_const_ones_f16:]] %[[#arg0:]]
+  %hlsl.rcp = fdiv <3 x half> <half 0xH3C00, half 0xH3C00, half 0xH3C00>, %p0
+  ret <3 x half> %hlsl.rcp
+}
+
+define spir_func noundef <4 x half> @test_rcp_half4(<4 x half> noundef %p0) #0 {
+entry:
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#vec4_float_16:]]
+  ; CHECK: OpFDiv %[[#vec4_float_16:]] %[[#vec4_const_ones_f16:]] %[[#arg0:]]
+  %hlsl.rcp = fdiv <4 x half> <half 0xH3C00, half 0xH3C00, half 0xH3C00, half 0xH3C00>, %p0
+  ret <4 x half> %hlsl.rcp
+}
+
+define spir_func noundef float @test_rcp_float(float noundef %p0) #0 {
+entry:
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#float_32:]]
+  ; CHECK: OpFDiv %[[#float_32:]] %[[#const_f32_1:]] %[[#arg0:]]
+  %hlsl.rcp = fdiv float 1.000000e+00, %p0
+  ret float %hlsl.rcp
+}
+
+define spir_func noundef <2 x float> @test_rcp_float2(<2 x float> noundef %p0) #0 {
+entry:
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#vec2_float_32:]]
+  ; CHECK: OpFDiv %[[#vec2_float_32:]] %[[#vec2_const_ones_f32:]] %[[#arg0:]]
+  %hlsl.rcp = fdiv <2 x float> <float 1.000000e+00, float 1.000000e+00>, %p0
+  ret <2 x float> %hlsl.rcp
+}
+
+define spir_func noundef <3 x float> @test_rcp_float3(<3 x float> noundef %p0) #0 {
+entry:
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#vec3_float_32:]]
+  ; CHECK: OpFDiv %[[#vec3_float_32:]] %[[#vec3_const_ones_f32:]] %[[#arg0:]]
+  %hlsl.rcp = fdiv <3 x float> <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>, %p0
+  ret <3 x float> %hlsl.rcp
+}
+
+define spir_func noundef <4 x float> @test_rcp_float4(<4 x float> noundef %p0) #0 {
+entry:
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#vec4_float_32:]]
+  ; CHECK: OpFDiv %[[#vec4_float_32:]] %[[#vec4_const_ones_f32:]] %[[#arg0:]]
+  %hlsl.rcp = fdiv <4 x float> <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>, %p0
+  ret <4 x float> %hlsl.rcp
+}
+
+define spir_func noundef double @test_rcp_double(double noundef %p0) #0 {
+entry:
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#float_64:]]
+  ; CHECK: OpFDiv %[[#float_64:]] %[[#const_f64_1:]] %[[#arg0:]]
+  %hlsl.rcp = fdiv double 1.000000e+00, %p0
+  ret double %hlsl.rcp
+}
+
+define spir_func noundef <2 x double> @test_rcp_double2(<2 x double> noundef %p0) #0 {
+entry:
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#vec2_float_64:]]
+  ; CHECK: OpFDiv %[[#vec2_float_64:]] %[[#vec2_const_ones_f64:]] %[[#arg0:]]
+  %hlsl.rcp = fdiv <2 x double> <double 1.000000e+00, double 1.000000e+00>, %p0
+  ret <2 x double> %hlsl.rcp
+}
+
+define spir_func noundef <3 x double> @test_rcp_double3(<3 x double> noundef %p0) #0 {
+entry:
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#vec3_float_64:]]
+  ; CHECK: OpFDiv %[[#vec3_float_64:]] %[[#vec3_const_ones_f64:]] %[[#arg0:]]
+  %hlsl.rcp = fdiv <3 x double> <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>, %p0
+  ret <3 x double> %hlsl.rcp
+}
+
+define spir_func noundef <4 x double> @test_rcp_double4(<4 x double> noundef %p0) #0 {
+entry:
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#vec4_float_64:]]
+  ; CHECK: OpFDiv %[[#vec4_float_64:]] %[[#vec4_const_ones_f64:]] %[[#arg0:]]
+  %hlsl.rcp = fdiv <4 x double> <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>, %p0
+  ret <4 x double> %hlsl.rcp
+}
-- 
GitLab


From 00a4f091bad199053b33af983a5abbf2c5abe949 Mon Sep 17 00:00:00 2001
From: Cyndy Ishida <cyndy_ishida@apple.com>
Date: Thu, 11 Apr 2024 15:51:27 -0700
Subject: [PATCH 012/464] [clang][docs] Modernize attribute docs for darwin
 specifics (#88448)

* Generally recommend target triples. But replace `m*version-min` with
`mtargetos`.
* Also include test coverage for -mtargetos=visionos
---
 clang/include/clang/Basic/AttrDocs.td | 37 ++++++++++++++++++---------
 clang/test/Driver/mtargetos-darwin.c  |  2 ++
 2 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index 0ca4ea377fc3..8687c4f57d3f 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -1604,27 +1604,40 @@ specifies availability for the current target platform, the availability
 attributes are ignored. Supported platforms are:
 
 ``ios``
-  Apple's iOS operating system. The minimum deployment target is specified by
-  the ``-mios-version-min=*version*`` or ``-miphoneos-version-min=*version*``
-  command-line arguments.
+  Apple's iOS operating system. The minimum deployment target is specified 
+  as part of the ``-target *arch*-apple-ios*version*`` command line argument. 
+  Alternatively, it can be specified by the ``-mtargetos=ios*version*`` 
+  command-line argument.
 
 ``macos``
-  Apple's macOS operating system. The minimum deployment target is
-  specified by the ``-mmacosx-version-min=*version*`` command-line argument.
-  ``macosx`` is supported for backward-compatibility reasons, but it is
-  deprecated.
+  Apple's macOS operating system. The minimum deployment target is specified 
+  as part of the ``-target *arch*-apple-macos*version*`` command line argument. 
+  Alternatively, it can be specified by the ``-mtargetos=macos*version*`` 
+  command-line argument. ``macosx`` is supported for 
+  backward-compatibility reasons, but it is deprecated.
 
 ``tvos``
-  Apple's tvOS operating system. The minimum deployment target is specified by
-  the ``-mtvos-version-min=*version*`` command-line argument.
+  Apple's tvOS operating system. The minimum deployment target is specified 
+  as part of the ``-target *arch*-apple-tvos*version*`` command line argument. 
+  Alternatively, it can be specified by the ``-mtargetos=tvos*version*`` 
+  command-line argument.
 
 ``watchos``
-  Apple's watchOS operating system. The minimum deployment target is specified by
-  the ``-mwatchos-version-min=*version*`` command-line argument.
+  Apple's watchOS operating system. The minimum deployment target is specified
+  as part of the ``-target *arch*-apple-watchos*version*`` command line argument. 
+  Alternatively, it can be specified by the ``-mtargetos=watchos*version*`` 
+  command-line argument.
+
+``visionos``
+  Apple's visionOS operating system. The minimum deployment target is specified
+  as part of the ``-target *arch*-apple-visionos*version*`` command line argument. 
+  Alternatively, it can be specified by the ``-mtargetos=visionos*version*`` 
+  command-line argument.
 
 ``driverkit``
   Apple's DriverKit userspace kernel extensions. The minimum deployment target
-  is specified as part of the triple.
+  is specified as part of the ``-target *arch*-apple-driverkit*version*`` 
+  command line argument.
 
 A declaration can typically be used even when deploying back to a platform
 version prior to when the declaration was introduced. When this happens, the
diff --git a/clang/test/Driver/mtargetos-darwin.c b/clang/test/Driver/mtargetos-darwin.c
index e706be37a371..7e86ab15279b 100644
--- a/clang/test/Driver/mtargetos-darwin.c
+++ b/clang/test/Driver/mtargetos-darwin.c
@@ -4,6 +4,7 @@
 // RUN: %clang -mtargetos=ios14-macabi -arch arm64 -c %s -o %t.o -### 2>&1 | FileCheck --check-prefix=MACCATALYST %s
 // RUN: %clang -mtargetos=tvos14 -arch arm64 -c %s -o %t.o -### 2>&1 | FileCheck --check-prefix=TVOS %s
 // RUN: %clang -mtargetos=watchos7.1 -arch arm64 -c %s -o %t.o -### 2>&1 | FileCheck --check-prefix=WATCHOS %s
+// RUN: %clang -mtargetos=visionos1 -arch arm64 -c %s -o %t.o -### 2>&1 | FileCheck --check-prefix=VISIONOS %s
 
 // RUN: not %clang -target arm64-apple-ios14 -mtargetos=ios14 -arch arm64 -c %s -o %t.o -### 2>&1 | FileCheck --check-prefix=NOMIX1 %s
 // RUN: not %clang -mtargetos=ios14 -arch arm64 -miphoneos-version-min=14 -c %s -o %t.o -### 2>&1 | FileCheck --check-prefix=NOMIX2 %s
@@ -19,6 +20,7 @@
 // MACCATALYST: "-cc1" "-triple" "arm64-apple-ios14.0.0-macabi"
 // TVOS: "-cc1" "-triple" "arm64-apple-tvos14.0.0"
 // WATCHOS: "-cc1" "-triple" "arm64-apple-watchos7.1.0"
+// VISIONOS: "-cc1" "-triple" "arm64-apple-xros1.0.0"
 
 // NOMIX1: error: cannot specify '-mtargetos=ios14' along with '-target arm64-apple-ios14'
 // NOMIX2: error: cannot specify '-miphoneos-version-min=14' along with '-mtargetos=ios14'
-- 
GitLab


From 43d0891d3bb1fc40ff5dcea91c28d1582978caff Mon Sep 17 00:00:00 2001
From: Maksim Panchenko <maks@fb.com>
Date: Thu, 11 Apr 2024 16:11:00 -0700
Subject: [PATCH 013/464] [BOLT] Fix handling of trailing entries in jump
 tables (#88444)

If a jump table has entries at the end that are a result of
__builtin_unreachable() targets, BOLT can confuse them with function
pointers. In such case, we should exclude these targets from the table
as we risk incorrectly updating the function pointers. It is safe to
exclude them as branching on such targets is considered an undefined
behavior.
---
 bolt/lib/Core/BinaryContext.cpp      |  26 ++++-
 bolt/test/runtime/X86/jt-confusion.s | 164 +++++++++++++++++++++++++++
 2 files changed, 185 insertions(+), 5 deletions(-)
 create mode 100644 bolt/test/runtime/X86/jt-confusion.s

diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp
index 47eae964e816..7c2d8c52287b 100644
--- a/bolt/lib/Core/BinaryContext.cpp
+++ b/bolt/lib/Core/BinaryContext.cpp
@@ -555,6 +555,9 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
                                      const uint64_t NextJTAddress,
                                      JumpTable::AddressesType *EntriesAsAddress,
                                      bool *HasEntryInFragment) const {
+  // Target address of __builtin_unreachable.
+  const uint64_t UnreachableAddress = BF.getAddress() + BF.getSize();
+
   // Is one of the targets __builtin_unreachable?
   bool HasUnreachable = false;
 
@@ -564,9 +567,15 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
   // Number of targets other than __builtin_unreachable.
   uint64_t NumRealEntries = 0;
 
-  auto addEntryAddress = [&](uint64_t EntryAddress) {
-    if (EntriesAsAddress)
-      EntriesAsAddress->emplace_back(EntryAddress);
+  // Size of the jump table without trailing __builtin_unreachable entries.
+  size_t TrimmedSize = 0;
+
+  auto addEntryAddress = [&](uint64_t EntryAddress, bool Unreachable = false) {
+    if (!EntriesAsAddress)
+      return;
+    EntriesAsAddress->emplace_back(EntryAddress);
+    if (!Unreachable)
+      TrimmedSize = EntriesAsAddress->size();
   };
 
   ErrorOr<const BinarySection &> Section = getSectionForAddress(Address);
@@ -618,8 +627,8 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
             : *getPointerAtAddress(EntryAddress);
 
     // __builtin_unreachable() case.
-    if (Value == BF.getAddress() + BF.getSize()) {
-      addEntryAddress(Value);
+    if (Value == UnreachableAddress) {
+      addEntryAddress(Value, /*Unreachable*/ true);
       HasUnreachable = true;
       LLVM_DEBUG(dbgs() << formatv("OK: {0:x} __builtin_unreachable\n", Value));
       continue;
@@ -673,6 +682,13 @@ bool BinaryContext::analyzeJumpTable(const uint64_t Address,
     addEntryAddress(Value);
   }
 
+  // Trim direct/normal jump table to exclude trailing unreachable entries that
+  // can collide with a function address.
+  if (Type == JumpTable::JTT_NORMAL && EntriesAsAddress &&
+      TrimmedSize != EntriesAsAddress->size() &&
+      getBinaryFunctionAtAddress(UnreachableAddress))
+    EntriesAsAddress->resize(TrimmedSize);
+
   // It's a jump table if the number of real entries is more than 1, or there's
   // one real entry and one or more special targets. If there are only multiple
   // special targets, then it's not a jump table.
diff --git a/bolt/test/runtime/X86/jt-confusion.s b/bolt/test/runtime/X86/jt-confusion.s
new file mode 100644
index 000000000000..f15c83b35b6a
--- /dev/null
+++ b/bolt/test/runtime/X86/jt-confusion.s
@@ -0,0 +1,164 @@
+# REQUIRES: system-linux
+
+# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o
+# RUN: llvm-strip --strip-unneeded %t.o
+# RUN: %clang %cflags -no-pie -nostartfiles -nostdlib -lc %t.o -o %t.exe -Wl,-q
+
+# RUN: llvm-bolt %t.exe -o %t.exe.bolt --relocs=1 --lite=0
+
+# RUN: %t.exe.bolt
+
+## Check that BOLT's jump table detection diffrentiates between
+## __builtin_unreachable() targets and function pointers.
+
+## The test case was built from the following two source files and
+## modiffied for standalone build. main became _start, etc.
+## $ $(CC) a.c -O1 -S -o a.s
+## $ $(CC) b.c -O0 -S -o b.s
+
+## a.c:
+
+## typedef int (*fptr)(int);
+## void check_fptr(fptr, int);
+##
+## int foo(int a) {
+##   check_fptr(foo, 0);
+##   switch (a) {
+##   default:
+##     __builtin_unreachable();
+##   case 0:
+##     return 3;
+##   case 1:
+##     return 5;
+##   case 2:
+##     return 7;
+##   case 3:
+##     return 11;
+##   case 4:
+##     return 13;
+##   case 5:
+##     return 17;
+##   }
+##   return 0;
+## }
+##
+## int main(int argc) {
+##   check_fptr(main, 1);
+##   return foo(argc);
+## }
+##
+## const fptr funcs[2] = {foo, main};
+
+## b.c.:
+
+## typedef int (*fptr)(int);
+## extern const fptr funcs[2];
+##
+## #define assert(C) { if (!(C)) (*(unsigned long long *)0) = 0; }
+## void check_fptr(fptr f, int i) {
+##   assert(f == funcs[i]);
+## }
+
+
+	.text
+	.globl	foo
+	.type	foo, @function
+foo:
+.LFB0:
+	.cfi_startproc
+	pushq	%rbx
+	.cfi_def_cfa_offset 16
+	.cfi_offset 3, -16
+	movl	%edi, %ebx
+	movl	$0, %esi
+	movl	$foo, %edi
+	call	check_fptr
+	movl	%ebx, %ebx
+	jmp	*.L4(,%rbx,8)
+.L8:
+	movl	$5, %eax
+	jmp	.L1
+.L7:
+	movl	$7, %eax
+	jmp	.L1
+.L6:
+	movl	$11, %eax
+	jmp	.L1
+.L5:
+	movl	$13, %eax
+	jmp	.L1
+.L3:
+	movl	$17, %eax
+	jmp	.L1
+.L10:
+	movl	$3, %eax
+.L1:
+	popq	%rbx
+	.cfi_def_cfa_offset 8
+	ret
+	.cfi_endproc
+.LFE0:
+	.size	foo, .-foo
+	.globl	_start
+	.type	_start, @function
+_start:
+.LFB1:
+	.cfi_startproc
+	pushq	%rbx
+	.cfi_def_cfa_offset 16
+	.cfi_offset 3, -16
+	movl	%edi, %ebx
+	movl	$1, %esi
+	movl	$_start, %edi
+	call	check_fptr
+	movl	$1, %edi
+	call	foo
+	popq	%rbx
+	.cfi_def_cfa_offset 8
+  callq exit@PLT
+	.cfi_endproc
+.LFE1:
+	.size	_start, .-_start
+	.globl	check_fptr
+	.type	check_fptr, @function
+check_fptr:
+.LFB2:
+	.cfi_startproc
+	pushq	%rbp
+	.cfi_def_cfa_offset 16
+	.cfi_offset 6, -16
+	movq	%rsp, %rbp
+	.cfi_def_cfa_register 6
+	movq	%rdi, -8(%rbp)
+	movl	%esi, -12(%rbp)
+	movl	-12(%rbp), %eax
+	cltq
+	movq	funcs(,%rax,8), %rax
+	cmpq	%rax, -8(%rbp)
+	je	.L33
+	movl	$0, %eax
+	movq	$0, (%rax)
+.L33:
+	nop
+	popq	%rbp
+	.cfi_def_cfa 7, 8
+	ret
+	.cfi_endproc
+
+	.section	.rodata
+	.align 8
+	.align 4
+.L4:
+	.quad	.L10
+	.quad	.L8
+	.quad	.L7
+	.quad	.L6
+	.quad	.L5
+	.quad	.L3
+
+	.globl	funcs
+	.type	funcs, @object
+	.size	funcs, 16
+funcs:
+	.quad	foo
+	.quad	_start
-- 
GitLab


From 2f55056c89dd1ccb1aa69a9aed68048a59413d4d Mon Sep 17 00:00:00 2001
From: Mark Rowe <mrowe@bdash.net.nz>
Date: Thu, 11 Apr 2024 16:13:54 -0700
Subject: [PATCH 014/464] [compiler-rt] Don't explicitly ad-hoc code sign
 dylibs if using Apple's new linker (#88323)

Apple's new linker reports itself as ld rather than ld64 and does not
match the version detection regex.

Invert the logic to look only for older versions of ld64. This ensures
the runtime dylibs are left with a linker-generated code signature that
tools such as `strip` will preserve.

Co-authored-by: Mark Rowe <markrowe@chromium.org>
---
 compiler-rt/cmake/Modules/AddCompilerRT.cmake | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/compiler-rt/cmake/Modules/AddCompilerRT.cmake b/compiler-rt/cmake/Modules/AddCompilerRT.cmake
index e0400a8ea952..dc36f16c661c 100644
--- a/compiler-rt/cmake/Modules/AddCompilerRT.cmake
+++ b/compiler-rt/cmake/Modules/AddCompilerRT.cmake
@@ -405,11 +405,14 @@ function(add_compiler_rt_runtime name type)
         if (HAD_ERROR)
           message(FATAL_ERROR "${CMAKE_LINKER} failed with status ${HAD_ERROR}")
         endif()
-        set(NEED_EXPLICIT_ADHOC_CODESIGN 1)
+        set(NEED_EXPLICIT_ADHOC_CODESIGN 0)
+        # Apple introduced a new linker by default in Xcode 15. This linker reports itself as ld
+        # rather than ld64 and does not match this version regex. That's ok since it never needs
+        # the explicit ad-hoc code signature.
         if ("${LD_V_OUTPUT}" MATCHES ".*ld64-([0-9.]+).*")
           string(REGEX REPLACE ".*ld64-([0-9.]+).*" "\\1" HOST_LINK_VERSION ${LD_V_OUTPUT})
-          if (HOST_LINK_VERSION VERSION_GREATER_EQUAL 609)
-            set(NEED_EXPLICIT_ADHOC_CODESIGN 0)
+          if (HOST_LINK_VERSION VERSION_LESS 609)
+            set(NEED_EXPLICIT_ADHOC_CODESIGN 1)
           endif()
         endif()
         if (NEED_EXPLICIT_ADHOC_CODESIGN)
-- 
GitLab


From 5fb821560afd0543e4875c3c81d58f68f73eb03b Mon Sep 17 00:00:00 2001
From: Michael Jones <michaelrj@google.com>
Date: Thu, 11 Apr 2024 16:26:49 -0700
Subject: [PATCH 015/464] [libc] add remaining epoll functions, pipe (#84587)

The epoll_wait functions need the rest of the epoll functions (create,
ctl) to be available to actually test them, as well as pipe to create a
usable file descriptor. This patch adds epoll_create, epoll_create1,
epoll_ctl, and pipe. These have tests, and the tests for epoll_wait,
epoll_pwait, and epoll_pwait2 (currently disabled) are updated to use
these newly available functions.
---
 libc/config/linux/x86_64/entrypoints.txt      |   4 +
 libc/hdr/CMakeLists.txt                       |  18 +++
 libc/hdr/signal_macros.h                      |  22 ++++
 libc/hdr/sys_epoll_macros.h                   |  22 ++++
 libc/include/CMakeLists.txt                   |   1 +
 libc/include/llvm-libc-macros/CMakeLists.txt  |   6 +
 .../llvm-libc-macros/linux/CMakeLists.txt     |   6 +
 .../llvm-libc-macros/linux/sys-epoll-macros.h |  40 +++++++
 .../llvm-libc-macros/sys-epoll-macros.h       |  16 +++
 libc/include/llvm-libc-types/sigset_t.h       |   4 +-
 .../llvm-libc-types/struct_epoll_event.h      |   6 +-
 libc/include/sys/epoll.h.def                  |   2 +
 libc/spec/posix.td                            |   5 +
 libc/src/sys/epoll/CMakeLists.txt             |  21 ++++
 libc/src/sys/epoll/epoll_create.h             |  18 +++
 libc/src/sys/epoll/epoll_create1.h            |  18 +++
 libc/src/sys/epoll/epoll_ctl.h                |  21 ++++
 libc/src/sys/epoll/epoll_pwait.h              |   4 +-
 libc/src/sys/epoll/linux/CMakeLists.txt       |  45 +++++++-
 libc/src/sys/epoll/linux/epoll_create.cpp     |  38 +++++++
 libc/src/sys/epoll/linux/epoll_create1.cpp    |  31 ++++++
 libc/src/sys/epoll/linux/epoll_ctl.cpp        |  34 ++++++
 libc/src/sys/epoll/linux/epoll_pwait.cpp      |   5 +-
 libc/src/sys/epoll/linux/epoll_pwait2.cpp     |   5 +-
 libc/src/sys/epoll/linux/epoll_wait.cpp       |   5 +-
 libc/src/unistd/CMakeLists.txt                |   7 ++
 libc/src/unistd/linux/CMakeLists.txt          |  13 +++
 libc/src/unistd/linux/pipe.cpp                |  33 ++++++
 libc/src/unistd/pipe.h                        |  18 +++
 libc/test/src/sys/epoll/linux/CMakeLists.txt  |  68 +++++++++++-
 .../sys/epoll/linux/epoll_create1_test.cpp    |  31 ++++++
 .../src/sys/epoll/linux/epoll_create_test.cpp |  26 +++++
 .../src/sys/epoll/linux/epoll_ctl_test.cpp    |  47 ++++++++
 .../src/sys/epoll/linux/epoll_pwait2_test.cpp |  47 +++++++-
 .../src/sys/epoll/linux/epoll_pwait_test.cpp  |  42 ++++++-
 .../src/sys/epoll/linux/epoll_wait_test.cpp   |  37 ++++++-
 libc/test/src/unistd/CMakeLists.txt           |  15 +++
 libc/test/src/unistd/pipe_test.cpp            |  26 +++++
 .../llvm-project-overlay/libc/BUILD.bazel     | 103 ++++++++++++++++--
 .../libc/test/src/sys/epoll/BUILD.bazel       |  61 +++++++++++
 40 files changed, 933 insertions(+), 38 deletions(-)
 create mode 100644 libc/hdr/signal_macros.h
 create mode 100644 libc/hdr/sys_epoll_macros.h
 create mode 100644 libc/include/llvm-libc-macros/linux/sys-epoll-macros.h
 create mode 100644 libc/include/llvm-libc-macros/sys-epoll-macros.h
 create mode 100644 libc/src/sys/epoll/epoll_create.h
 create mode 100644 libc/src/sys/epoll/epoll_create1.h
 create mode 100644 libc/src/sys/epoll/epoll_ctl.h
 create mode 100644 libc/src/sys/epoll/linux/epoll_create.cpp
 create mode 100644 libc/src/sys/epoll/linux/epoll_create1.cpp
 create mode 100644 libc/src/sys/epoll/linux/epoll_ctl.cpp
 create mode 100644 libc/src/unistd/linux/pipe.cpp
 create mode 100644 libc/src/unistd/pipe.h
 create mode 100644 libc/test/src/sys/epoll/linux/epoll_create1_test.cpp
 create mode 100644 libc/test/src/sys/epoll/linux/epoll_create_test.cpp
 create mode 100644 libc/test/src/sys/epoll/linux/epoll_ctl_test.cpp
 create mode 100644 libc/test/src/unistd/pipe_test.cpp

diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index 6bb53cb76220..8fdd4575e27e 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -215,6 +215,9 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.stdio.fileno
 
     # sys/epoll.h entrypoints
+    libc.src.sys.epoll.epoll_create
+    libc.src.sys.epoll.epoll_create1
+    libc.src.sys.epoll.epoll_ctl
     libc.src.sys.epoll.epoll_wait
     libc.src.sys.epoll.epoll_pwait
     # TODO: Need to check if pwait2 is available before providing.
@@ -308,6 +311,7 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.unistd.link
     libc.src.unistd.linkat
     libc.src.unistd.lseek
+    libc.src.unistd.pipe
     libc.src.unistd.pread
     libc.src.unistd.pwrite
     libc.src.unistd.read
diff --git a/libc/hdr/CMakeLists.txt b/libc/hdr/CMakeLists.txt
index 38ef56e3f04c..fb7c342f92b7 100644
--- a/libc/hdr/CMakeLists.txt
+++ b/libc/hdr/CMakeLists.txt
@@ -41,4 +41,22 @@ add_proxy_header_library(
     libc.include.fenv
 )
 
+add_proxy_header_library(
+  signal_macros
+  HDRS
+    signal_macros.h
+  FULL_BUILD_DEPENDS
+    libc.include.llvm-libc-macros.signal_macros
+    libc.include.signal
+)
+
+add_proxy_header_library(
+  sys_epoll_macros
+  HDRS
+    sys_epoll_macros.h
+  FULL_BUILD_DEPENDS
+    libc.include.sys_epoll
+    libc.include.llvm-libc-macros.sys_epoll_macros
+)
+
 add_subdirectory(types)
diff --git a/libc/hdr/signal_macros.h b/libc/hdr/signal_macros.h
new file mode 100644
index 000000000000..867d17a4ca8c
--- /dev/null
+++ b/libc/hdr/signal_macros.h
@@ -0,0 +1,22 @@
+//===-- Definition of macros from signal.h --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_SIGNAL_MACROS_H
+#define LLVM_LIBC_HDR_SIGNAL_MACROS_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-macros/signal-macros.h"
+
+#else // Overlay mode
+
+#include <signal.h>
+
+#endif // LLVM_LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_SIGNAL_MACROS_H
diff --git a/libc/hdr/sys_epoll_macros.h b/libc/hdr/sys_epoll_macros.h
new file mode 100644
index 000000000000..db047f161634
--- /dev/null
+++ b/libc/hdr/sys_epoll_macros.h
@@ -0,0 +1,22 @@
+//===-- Definition of macros from sys/epoll.h -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_SYS_EPOLL_MACROS_H
+#define LLVM_LIBC_HDR_SYS_EPOLL_MACROS_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-macros/sys-epoll-macros.h"
+
+#else // Overlay mode
+
+#include <sys/epoll.h>
+
+#endif // LLVM_LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_SYS/EPOLL_MACROS_H
diff --git a/libc/include/CMakeLists.txt b/libc/include/CMakeLists.txt
index 02c7dc8fbc0b..b85366c8deaf 100644
--- a/libc/include/CMakeLists.txt
+++ b/libc/include/CMakeLists.txt
@@ -379,6 +379,7 @@ add_gen_header(
     .llvm-libc-types.struct_epoll_event
     .llvm-libc-types.struct_epoll_data
     .llvm-libc-types.sigset_t
+    .llvm-libc-macros.sys_epoll_macros
 )
 
 add_gen_header(
diff --git a/libc/include/llvm-libc-macros/CMakeLists.txt b/libc/include/llvm-libc-macros/CMakeLists.txt
index 635ccadfb49e..382cb8ee417e 100644
--- a/libc/include/llvm-libc-macros/CMakeLists.txt
+++ b/libc/include/llvm-libc-macros/CMakeLists.txt
@@ -143,6 +143,12 @@ add_macro_header(
     sys-auxv-macros.h
 )
 
+add_macro_header(
+  sys_epoll_macros
+  HDR
+    sys-epoll-macros.h
+)
+
 add_macro_header(
   sys_ioctl_macros
   HDR
diff --git a/libc/include/llvm-libc-macros/linux/CMakeLists.txt b/libc/include/llvm-libc-macros/linux/CMakeLists.txt
index d934c3a0fca6..4ee429d1db16 100644
--- a/libc/include/llvm-libc-macros/linux/CMakeLists.txt
+++ b/libc/include/llvm-libc-macros/linux/CMakeLists.txt
@@ -10,6 +10,12 @@ add_header(
     sched-macros.h
 )
 
+add_header(
+  sys_epoll_macros
+  HDR
+    sys-epoll-macros.h
+)
+
 add_header(
   sys_ioctl_macros
   HDR
diff --git a/libc/include/llvm-libc-macros/linux/sys-epoll-macros.h b/libc/include/llvm-libc-macros/linux/sys-epoll-macros.h
new file mode 100644
index 000000000000..f73d690908fd
--- /dev/null
+++ b/libc/include/llvm-libc-macros/linux/sys-epoll-macros.h
@@ -0,0 +1,40 @@
+//===-- Macros defined in sys/epoll.h header file -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_MACROS_LINUX_SYS_EPOLL_MACROS_H
+#define LLVM_LIBC_MACROS_LINUX_SYS_EPOLL_MACROS_H
+
+#include "fcntl-macros.h"
+
+// These are also defined in <linux/eventpoll.h> but that also contains a
+// different definition of the epoll_event struct that is different from the
+// userspace version.
+
+#define EPOLL_CLOEXEC O_CLOEXEC
+
+#define EPOLL_CTL_ADD 1
+#define EPOLL_CTL_DEL 2
+#define EPOLL_CTL_MOD 3
+
+#define EPOLLIN 0x1
+#define EPOLLPRI 0x2
+#define EPOLLOUT 0x4
+#define EPOLLERR 0x8
+#define EPOLLHUP 0x10
+#define EPOLLRDNORM 0x40
+#define EPOLLRDBAND 0x80
+#define EPOLLWRNORM 0x100
+#define EPOLLWRBAND 0x200
+#define EPOLLMSG 0x400
+#define EPOLLRDHUP 0x2000
+#define EPOLLEXCLUSIVE 0x10000000
+#define EPOLLWAKEUP 0x20000000
+#define EPOLLONESHOT 0x40000000
+#define EPOLLET 0x80000000
+
+#endif // LLVM_LIBC_MACROS_LINUX_SYS_EPOLL_MACROS_H
diff --git a/libc/include/llvm-libc-macros/sys-epoll-macros.h b/libc/include/llvm-libc-macros/sys-epoll-macros.h
new file mode 100644
index 000000000000..8212df252b8e
--- /dev/null
+++ b/libc/include/llvm-libc-macros/sys-epoll-macros.h
@@ -0,0 +1,16 @@
+//===-- Macros defined in sys/epoll.h header file -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_MACROS_SYS_EPOLL_MACROS_H
+#define LLVM_LIBC_MACROS_SYS_EPOLL_MACROS_H
+
+#ifdef __linux__
+#include "linux/sys-epoll-macros.h"
+#endif
+
+#endif // LLVM_LIBC_MACROS_SYS_EPOLL_MACROS_H
diff --git a/libc/include/llvm-libc-types/sigset_t.h b/libc/include/llvm-libc-types/sigset_t.h
index 311a92b823ff..5c3e4e6f0431 100644
--- a/libc/include/llvm-libc-types/sigset_t.h
+++ b/libc/include/llvm-libc-types/sigset_t.h
@@ -13,8 +13,8 @@
 
 // This definition can be adjusted/specialized for different targets and
 // platforms as necessary. This definition works for Linux on most targets.
-typedef struct {
+struct sigset_t {
   unsigned long __signals[__NSIGSET_WORDS];
-} sigset_t;
+};
 
 #endif // LLVM_LIBC_TYPES_SIGSET_T_H
diff --git a/libc/include/llvm-libc-types/struct_epoll_event.h b/libc/include/llvm-libc-types/struct_epoll_event.h
index 66cf86c1e2a0..b7685df20bd7 100644
--- a/libc/include/llvm-libc-types/struct_epoll_event.h
+++ b/libc/include/llvm-libc-types/struct_epoll_event.h
@@ -11,7 +11,11 @@
 
 #include "llvm-libc-types/struct_epoll_data.h"
 
-typedef struct epoll_event {
+typedef struct
+#ifdef __x86_64__
+    [[gnu::packed]] // Necessary for compatibility.
+#endif
+    epoll_event {
   __UINT32_TYPE__ events;
   epoll_data_t data;
 } epoll_event;
diff --git a/libc/include/sys/epoll.h.def b/libc/include/sys/epoll.h.def
index 85f7d9ad6091..98d96bf42925 100644
--- a/libc/include/sys/epoll.h.def
+++ b/libc/include/sys/epoll.h.def
@@ -11,6 +11,8 @@
 
 #include "__llvm-libc-common.h"
 
+#include "llvm-libc-macros/sys-epoll-macros.h"
+
 %%public_api()
 
 #endif // LLVM_LIBC_SYS_EPOLL_H
diff --git a/libc/spec/posix.td b/libc/spec/posix.td
index 45f7ecfe84e9..7095a3964ee3 100644
--- a/libc/spec/posix.td
+++ b/libc/spec/posix.td
@@ -572,6 +572,11 @@ def POSIX : StandardSpec<"POSIX"> {
           RetValSpec<OffTType>,
           [ArgSpec<IntType>, ArgSpec<OffTType>, ArgSpec<IntType>]
         >,
+        FunctionSpec<
+          "pipe",
+          RetValSpec<IntType>,
+          [ArgSpec<IntPtr>] //TODO: make this int[2]
+        >,
         FunctionSpec<
           "pread",
           RetValSpec<SSizeTType>,
diff --git a/libc/src/sys/epoll/CMakeLists.txt b/libc/src/sys/epoll/CMakeLists.txt
index d4991a238e2a..e785f535bf0c 100644
--- a/libc/src/sys/epoll/CMakeLists.txt
+++ b/libc/src/sys/epoll/CMakeLists.txt
@@ -2,6 +2,27 @@ if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
   add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
 endif()
 
+add_entrypoint_object(
+  epoll_create
+  ALIAS
+  DEPENDS
+    .${LIBC_TARGET_OS}.epoll_create
+)
+
+add_entrypoint_object(
+  epoll_create1
+  ALIAS
+  DEPENDS
+    .${LIBC_TARGET_OS}.epoll_create1
+)
+
+add_entrypoint_object(
+  epoll_ctl
+  ALIAS
+  DEPENDS
+    .${LIBC_TARGET_OS}.epoll_ctl
+)
+
 add_entrypoint_object(
   epoll_wait
   ALIAS
diff --git a/libc/src/sys/epoll/epoll_create.h b/libc/src/sys/epoll/epoll_create.h
new file mode 100644
index 000000000000..a1eeabd567e4
--- /dev/null
+++ b/libc/src/sys/epoll/epoll_create.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for epoll_create function ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_SYS_EPOLL_EPOLL_CREATE_H
+#define LLVM_LIBC_SRC_SYS_EPOLL_EPOLL_CREATE_H
+
+namespace LIBC_NAMESPACE {
+
+int epoll_create(int size);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_SYS_EPOLL_EPOLL_CREATE_H
diff --git a/libc/src/sys/epoll/epoll_create1.h b/libc/src/sys/epoll/epoll_create1.h
new file mode 100644
index 000000000000..70f446b59266
--- /dev/null
+++ b/libc/src/sys/epoll/epoll_create1.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for epoll_create1 function --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_SYS_EPOLL_EPOLL_CREATE1_H
+#define LLVM_LIBC_SRC_SYS_EPOLL_EPOLL_CREATE1_H
+
+namespace LIBC_NAMESPACE {
+
+int epoll_create1(int flags);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_SYS_EPOLL_EPOLL_CREATE1_H
diff --git a/libc/src/sys/epoll/epoll_ctl.h b/libc/src/sys/epoll/epoll_ctl.h
new file mode 100644
index 000000000000..42402949d06b
--- /dev/null
+++ b/libc/src/sys/epoll/epoll_ctl.h
@@ -0,0 +1,21 @@
+//===-- Implementation header for epoll_ctl function ------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_SYS_EPOLL_EPOLL_CTL_H
+#define LLVM_LIBC_SRC_SYS_EPOLL_EPOLL_CTL_H
+
+#include "hdr/types/struct_epoll_event.h"
+
+namespace LIBC_NAMESPACE {
+
+// TODO: event should be nullable
+int epoll_ctl(int epfd, int op, int fd, struct epoll_event *event);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_SYS_EPOLL_EPOLL_CTL_H
diff --git a/libc/src/sys/epoll/epoll_pwait.h b/libc/src/sys/epoll/epoll_pwait.h
index 801aa9761000..bcae72b3e2c2 100644
--- a/libc/src/sys/epoll/epoll_pwait.h
+++ b/libc/src/sys/epoll/epoll_pwait.h
@@ -15,8 +15,8 @@
 namespace LIBC_NAMESPACE {
 
 // TODO: sigmask should be nullable
-int epoll_pwait(int epfd, epoll_event *events, int maxevents, int timeout,
-                const sigset_t *sigmask);
+int epoll_pwait(int epfd, struct epoll_event *events, int maxevents,
+                int timeout, const sigset_t *sigmask);
 
 } // namespace LIBC_NAMESPACE
 
diff --git a/libc/src/sys/epoll/linux/CMakeLists.txt b/libc/src/sys/epoll/linux/CMakeLists.txt
index 586aac7055dc..4e661b262b85 100644
--- a/libc/src/sys/epoll/linux/CMakeLists.txt
+++ b/libc/src/sys/epoll/linux/CMakeLists.txt
@@ -1,3 +1,41 @@
+add_entrypoint_object(
+  epoll_create
+  SRCS
+    epoll_create.cpp
+  HDRS
+    ../epoll_create.h
+  DEPENDS
+    libc.include.sys_syscall
+    libc.src.__support.OSUtil.osutil
+    libc.src.errno.errno
+)
+
+add_entrypoint_object(
+  epoll_create1
+  SRCS
+    epoll_create1.cpp
+  HDRS
+    ../epoll_create1.h
+  DEPENDS
+    libc.include.sys_syscall
+    libc.src.__support.OSUtil.osutil
+    libc.src.errno.errno
+)
+
+add_entrypoint_object(
+  epoll_ctl
+  SRCS
+    epoll_ctl.cpp
+  HDRS
+    ../epoll_ctl.h
+  DEPENDS
+    libc.hdr.types.struct_epoll_event
+    libc.hdr.sys_epoll_macros
+    libc.include.sys_syscall
+    libc.src.__support.OSUtil.osutil
+    libc.src.errno.errno
+)
+
 add_entrypoint_object(
   epoll_wait
   SRCS
@@ -23,7 +61,8 @@ add_entrypoint_object(
     libc.hdr.types.sigset_t
     libc.hdr.types.struct_epoll_event
     libc.hdr.types.struct_timespec
-    libc.include.signal
+    libc.hdr.sys_epoll_macros
+    libc.hdr.signal_macros
     libc.include.sys_syscall
     libc.src.__support.OSUtil.osutil
     libc.src.errno.errno
@@ -39,9 +78,9 @@ add_entrypoint_object(
     libc.hdr.types.sigset_t
     libc.hdr.types.struct_epoll_event
     libc.hdr.types.struct_timespec
-    libc.include.signal
+    libc.hdr.sys_epoll_macros
+    libc.hdr.signal_macros
     libc.include.sys_syscall
-    libc.include.time
     libc.src.__support.OSUtil.osutil
     libc.src.errno.errno
 )
diff --git a/libc/src/sys/epoll/linux/epoll_create.cpp b/libc/src/sys/epoll/linux/epoll_create.cpp
new file mode 100644
index 000000000000..d4995c476074
--- /dev/null
+++ b/libc/src/sys/epoll/linux/epoll_create.cpp
@@ -0,0 +1,38 @@
+//===---------- Linux implementation of the epoll_create function ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/sys/epoll/epoll_create.h"
+
+#include "src/__support/OSUtil/syscall.h" // For internal syscall function.
+#include "src/__support/common.h"
+#include "src/errno/libc_errno.h"
+#include <sys/syscall.h> // For syscall numbers.
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(int, epoll_create, ([[maybe_unused]] int size)) {
+#ifdef SYS_epoll_create
+  int ret = LIBC_NAMESPACE::syscall_impl<int>(SYS_epoll_create, size);
+#elif defined(SYS_epoll_create1)
+  int ret = LIBC_NAMESPACE::syscall_impl<int>(SYS_epoll_create1, 0);
+#else
+#error                                                                         \
+    "epoll_create and epoll_create1 are unavailable. Unable to build epoll_create."
+#endif
+
+  // A negative return value indicates an error with the magnitude of the
+  // value being the error code.
+  if (ret < 0) {
+    libc_errno = -ret;
+    return -1;
+  }
+
+  return ret;
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/sys/epoll/linux/epoll_create1.cpp b/libc/src/sys/epoll/linux/epoll_create1.cpp
new file mode 100644
index 000000000000..30f1a9910274
--- /dev/null
+++ b/libc/src/sys/epoll/linux/epoll_create1.cpp
@@ -0,0 +1,31 @@
+//===---------- Linux implementation of the epoll_create1 function --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/sys/epoll/epoll_create1.h"
+
+#include "src/__support/OSUtil/syscall.h" // For internal syscall function.
+#include "src/__support/common.h"
+#include "src/errno/libc_errno.h"
+#include <sys/syscall.h> // For syscall numbers.
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(int, epoll_create1, (int flags)) {
+  int ret = LIBC_NAMESPACE::syscall_impl<int>(SYS_epoll_create1, flags);
+
+  // A negative return value indicates an error with the magnitude of the
+  // value being the error code.
+  if (ret < 0) {
+    libc_errno = -ret;
+    return -1;
+  }
+
+  return ret;
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/sys/epoll/linux/epoll_ctl.cpp b/libc/src/sys/epoll/linux/epoll_ctl.cpp
new file mode 100644
index 000000000000..c111c5e4e0ee
--- /dev/null
+++ b/libc/src/sys/epoll/linux/epoll_ctl.cpp
@@ -0,0 +1,34 @@
+//===---------- Linux implementation of the epoll_ctl function ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/sys/epoll/epoll_ctl.h"
+
+#include "hdr/types/struct_epoll_event.h"
+#include "src/__support/OSUtil/syscall.h" // For internal syscall function.
+#include "src/__support/common.h"
+#include "src/errno/libc_errno.h"
+#include <sys/syscall.h> // For syscall numbers.
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(int, epoll_ctl,
+                   (int epfd, int op, int fd, epoll_event *event)) {
+  int ret = LIBC_NAMESPACE::syscall_impl<int>(SYS_epoll_ctl, epfd, op, fd,
+                                              reinterpret_cast<long>(event));
+
+  // A negative return value indicates an error with the magnitude of the
+  // value being the error code.
+  if (ret < 0) {
+    libc_errno = -ret;
+    return -1;
+  }
+
+  return ret;
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/sys/epoll/linux/epoll_pwait.cpp b/libc/src/sys/epoll/linux/epoll_pwait.cpp
index ac012944a957..8f498d18d547 100644
--- a/libc/src/sys/epoll/linux/epoll_pwait.cpp
+++ b/libc/src/sys/epoll/linux/epoll_pwait.cpp
@@ -8,6 +8,7 @@
 
 #include "src/sys/epoll/epoll_pwait.h"
 
+#include "hdr/signal_macros.h" // for NSIG
 #include "hdr/types/sigset_t.h"
 #include "hdr/types/struct_epoll_event.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
@@ -23,7 +24,7 @@ LLVM_LIBC_FUNCTION(int, epoll_pwait,
                     int timeout, const sigset_t *sigmask)) {
   int ret = LIBC_NAMESPACE::syscall_impl<int>(
       SYS_epoll_pwait, epfd, reinterpret_cast<long>(events), maxevents, timeout,
-      reinterpret_cast<long>(sigmask), sizeof(sigset_t));
+      reinterpret_cast<long>(sigmask), NSIG / 8);
 
   // A negative return value indicates an error with the magnitude of the
   // value being the error code.
@@ -32,7 +33,7 @@ LLVM_LIBC_FUNCTION(int, epoll_pwait,
     return -1;
   }
 
-  return 0;
+  return ret;
 }
 
 } // namespace LIBC_NAMESPACE
diff --git a/libc/src/sys/epoll/linux/epoll_pwait2.cpp b/libc/src/sys/epoll/linux/epoll_pwait2.cpp
index 3c42e38deb22..bd33cb6325ce 100644
--- a/libc/src/sys/epoll/linux/epoll_pwait2.cpp
+++ b/libc/src/sys/epoll/linux/epoll_pwait2.cpp
@@ -8,6 +8,7 @@
 
 #include "src/sys/epoll/epoll_pwait2.h"
 
+#include "hdr/signal_macros.h" // for NSIG
 #include "hdr/types/sigset_t.h"
 #include "hdr/types/struct_epoll_event.h"
 #include "hdr/types/struct_timespec.h"
@@ -25,7 +26,7 @@ LLVM_LIBC_FUNCTION(int, epoll_pwait2,
   int ret = LIBC_NAMESPACE::syscall_impl<int>(
       SYS_epoll_pwait2, epfd, reinterpret_cast<long>(events), maxevents,
       reinterpret_cast<long>(timeout), reinterpret_cast<long>(sigmask),
-      sizeof(sigset_t));
+      NSIG / 8);
 
   // A negative return value indicates an error with the magnitude of the
   // value being the error code.
@@ -34,7 +35,7 @@ LLVM_LIBC_FUNCTION(int, epoll_pwait2,
     return -1;
   }
 
-  return 0;
+  return ret;
 }
 
 } // namespace LIBC_NAMESPACE
diff --git a/libc/src/sys/epoll/linux/epoll_wait.cpp b/libc/src/sys/epoll/linux/epoll_wait.cpp
index 18dd6e2b8354..95238d872d52 100644
--- a/libc/src/sys/epoll/linux/epoll_wait.cpp
+++ b/libc/src/sys/epoll/linux/epoll_wait.cpp
@@ -8,6 +8,7 @@
 
 #include "src/sys/epoll/epoll_wait.h"
 
+#include "hdr/signal_macros.h" // for NSIG
 #include "hdr/types/sigset_t.h"
 #include "hdr/types/struct_epoll_event.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
@@ -27,7 +28,7 @@ LLVM_LIBC_FUNCTION(int, epoll_wait,
 #elif defined(SYS_epoll_pwait)
   int ret = LIBC_NAMESPACE::syscall_impl<int>(
       SYS_epoll_pwait, epfd, reinterpret_cast<long>(events), maxevents, timeout,
-      reinterpret_cast<long>(nullptr), sizeof(sigset_t));
+      reinterpret_cast<long>(nullptr), NSIG / 8);
 #else
 #error "epoll_wait and epoll_pwait are unavailable. Unable to build epoll_wait."
 #endif
@@ -38,7 +39,7 @@ LLVM_LIBC_FUNCTION(int, epoll_wait,
     return -1;
   }
 
-  return 0;
+  return ret;
 }
 
 } // namespace LIBC_NAMESPACE
diff --git a/libc/src/unistd/CMakeLists.txt b/libc/src/unistd/CMakeLists.txt
index 58b44f5307ca..77db76518350 100644
--- a/libc/src/unistd/CMakeLists.txt
+++ b/libc/src/unistd/CMakeLists.txt
@@ -160,6 +160,13 @@ add_entrypoint_object(
     .${LIBC_TARGET_OS}.lseek
 )
 
+add_entrypoint_object(
+  pipe
+  ALIAS
+  DEPENDS
+    .${LIBC_TARGET_OS}.pipe
+)
+
 add_entrypoint_object(
   pread
   ALIAS
diff --git a/libc/src/unistd/linux/CMakeLists.txt b/libc/src/unistd/linux/CMakeLists.txt
index df85d44e9e9e..7d831f9c29c7 100644
--- a/libc/src/unistd/linux/CMakeLists.txt
+++ b/libc/src/unistd/linux/CMakeLists.txt
@@ -273,6 +273,19 @@ add_entrypoint_object(
     libc.src.errno.errno
 )
 
+add_entrypoint_object(
+  pipe
+  SRCS
+    pipe.cpp
+  HDRS
+    ../pipe.h
+  DEPENDS
+    libc.include.unistd
+    libc.include.sys_syscall
+    libc.src.__support.OSUtil.osutil
+    libc.src.errno.errno
+)
+
 add_entrypoint_object(
   pread
   SRCS
diff --git a/libc/src/unistd/linux/pipe.cpp b/libc/src/unistd/linux/pipe.cpp
new file mode 100644
index 000000000000..b4e8b9b7d9c8
--- /dev/null
+++ b/libc/src/unistd/linux/pipe.cpp
@@ -0,0 +1,33 @@
+//===-- Linux implementation of pipe --------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/unistd/pipe.h"
+
+#include "src/__support/OSUtil/syscall.h" // For internal syscall function.
+#include "src/__support/common.h"
+#include "src/errno/libc_errno.h"
+#include <sys/syscall.h> // For syscall numbers.
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(int, pipe, (int pipefd[2])) {
+#ifdef SYS_pipe
+  int ret = LIBC_NAMESPACE::syscall_impl<int>(SYS_pipe,
+                                              reinterpret_cast<long>(pipefd));
+#elif defined(SYS_pipe2)
+  int ret = LIBC_NAMESPACE::syscall_impl<int>(
+      SYS_pipe2, reinterpret_cast<long>(pipefd), 0);
+#endif
+  if (ret < 0) {
+    libc_errno = -ret;
+    return -1;
+  }
+  return ret;
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/unistd/pipe.h b/libc/src/unistd/pipe.h
new file mode 100644
index 000000000000..0e20bb49c058
--- /dev/null
+++ b/libc/src/unistd/pipe.h
@@ -0,0 +1,18 @@
+//===-- Implementation header for pipe --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_UNISTD_PIPE_H
+#define LLVM_LIBC_SRC_UNISTD_PIPE_H
+
+namespace LIBC_NAMESPACE {
+
+int pipe(int pipefd[2]);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_UNISTD_PIPE_H
diff --git a/libc/test/src/sys/epoll/linux/CMakeLists.txt b/libc/test/src/sys/epoll/linux/CMakeLists.txt
index e346af2328db..8f4b698ef141 100644
--- a/libc/test/src/sys/epoll/linux/CMakeLists.txt
+++ b/libc/test/src/sys/epoll/linux/CMakeLists.txt
@@ -1,4 +1,50 @@
 add_custom_target(libc_sys_epoll_unittests)
+
+add_libc_unittest(
+  epoll_create_test
+  SUITE
+    libc_sys_epoll_unittests
+  SRCS
+    epoll_create_test.cpp
+  DEPENDS
+    libc.include.sys_epoll
+    libc.src.errno.errno
+    libc.src.sys.epoll.epoll_create
+    libc.src.unistd.close
+    libc.test.UnitTest.ErrnoSetterMatcher
+)
+
+add_libc_unittest(
+  epoll_create1_test
+  SUITE
+    libc_sys_epoll_unittests
+  SRCS
+    epoll_create1_test.cpp
+  DEPENDS
+    libc.hdr.sys_epoll_macros
+    libc.src.errno.errno
+    libc.src.sys.epoll.epoll_create1
+    libc.src.unistd.close
+    libc.test.UnitTest.ErrnoSetterMatcher
+)
+
+add_libc_unittest(
+  epoll_ctl_test
+  SUITE
+    libc_sys_epoll_unittests
+  SRCS
+    epoll_ctl_test.cpp
+  DEPENDS
+    libc.hdr.sys_epoll_macros
+    libc.hdr.types.struct_epoll_event
+    libc.src.errno.errno
+    libc.src.sys.epoll.epoll_create1
+    libc.src.sys.epoll.epoll_ctl
+    libc.src.unistd.pipe
+    libc.src.unistd.close
+    libc.test.UnitTest.ErrnoSetterMatcher
+)
+
 add_libc_unittest(
   epoll_wait_test
   SUITE
@@ -6,9 +52,14 @@ add_libc_unittest(
   SRCS
     epoll_wait_test.cpp
   DEPENDS
-    libc.include.sys_epoll
+    libc.hdr.sys_epoll_macros
+    libc.hdr.types.struct_epoll_event
     libc.src.errno.errno
+    libc.src.sys.epoll.epoll_create1
+    libc.src.sys.epoll.epoll_ctl
     libc.src.sys.epoll.epoll_wait
+    libc.src.unistd.pipe
+    libc.src.unistd.close
     libc.test.UnitTest.ErrnoSetterMatcher
 )
 
@@ -19,9 +70,14 @@ add_libc_unittest(
   SRCS
     epoll_pwait_test.cpp
   DEPENDS
-    libc.include.sys_epoll
+    libc.hdr.sys_epoll_macros
+    libc.hdr.types.struct_epoll_event
     libc.src.errno.errno
+    libc.src.sys.epoll.epoll_create1
+    libc.src.sys.epoll.epoll_ctl
     libc.src.sys.epoll.epoll_pwait
+    libc.src.unistd.pipe
+    libc.src.unistd.close
     libc.test.UnitTest.ErrnoSetterMatcher
 )
 
@@ -32,8 +88,14 @@ add_libc_unittest(
   SRCS
     epoll_pwait2_test.cpp
   DEPENDS
-    libc.include.sys_epoll
+    libc.hdr.sys_epoll_macros
+    libc.hdr.types.struct_epoll_event
+    libc.hdr.types.struct_timespec
     libc.src.errno.errno
+    libc.src.sys.epoll.epoll_create1
+    libc.src.sys.epoll.epoll_ctl
     libc.src.sys.epoll.epoll_pwait2
+    libc.src.unistd.pipe
+    libc.src.unistd.close
     libc.test.UnitTest.ErrnoSetterMatcher
 )
diff --git a/libc/test/src/sys/epoll/linux/epoll_create1_test.cpp b/libc/test/src/sys/epoll/linux/epoll_create1_test.cpp
new file mode 100644
index 000000000000..4059afe16b80
--- /dev/null
+++ b/libc/test/src/sys/epoll/linux/epoll_create1_test.cpp
@@ -0,0 +1,31 @@
+//===-- Unittests for epoll_create1 ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "hdr/sys_epoll_macros.h"
+#include "src/errno/libc_errno.h"
+#include "src/sys/epoll/epoll_create1.h"
+#include "src/unistd/close.h"
+#include "test/UnitTest/ErrnoSetterMatcher.h"
+#include "test/UnitTest/Test.h"
+
+using namespace LIBC_NAMESPACE::testing::ErrnoSetterMatcher;
+
+TEST(LlvmLibcEpollCreate1Test, Basic) {
+  int fd = LIBC_NAMESPACE::epoll_create1(0);
+  ASSERT_GT(fd, 0);
+  ASSERT_ERRNO_SUCCESS();
+
+  ASSERT_THAT(LIBC_NAMESPACE::close(fd), Succeeds());
+}
+
+TEST(LlvmLibcEpollCreate1Test, CloseOnExecute) {
+  int fd = LIBC_NAMESPACE::epoll_create1(EPOLL_CLOEXEC);
+  ASSERT_GT(fd, 0);
+  ASSERT_ERRNO_SUCCESS();
+
+  ASSERT_THAT(LIBC_NAMESPACE::close(fd), Succeeds());
+}
diff --git a/libc/test/src/sys/epoll/linux/epoll_create_test.cpp b/libc/test/src/sys/epoll/linux/epoll_create_test.cpp
new file mode 100644
index 000000000000..fdcdcf8eb427
--- /dev/null
+++ b/libc/test/src/sys/epoll/linux/epoll_create_test.cpp
@@ -0,0 +1,26 @@
+//===-- Unittests for epoll_create ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "src/errno/libc_errno.h"
+#include "src/sys/epoll/epoll_create.h"
+#include "src/unistd/close.h"
+#include "test/UnitTest/ErrnoSetterMatcher.h"
+#include "test/UnitTest/Test.h"
+
+using namespace LIBC_NAMESPACE::testing::ErrnoSetterMatcher;
+
+TEST(LlvmLibcEpollCreateTest, Basic) {
+  int fd = LIBC_NAMESPACE::epoll_create(1);
+  ASSERT_GT(fd, 0);
+  ASSERT_ERRNO_SUCCESS();
+
+  ASSERT_THAT(LIBC_NAMESPACE::close(fd), Succeeds());
+}
+
+TEST(LlvmLibcEpollCreateTest, Fails) {
+  ASSERT_THAT(LIBC_NAMESPACE::epoll_create(0), Fails(EINVAL));
+}
diff --git a/libc/test/src/sys/epoll/linux/epoll_ctl_test.cpp b/libc/test/src/sys/epoll/linux/epoll_ctl_test.cpp
new file mode 100644
index 000000000000..fa2d358c5796
--- /dev/null
+++ b/libc/test/src/sys/epoll/linux/epoll_ctl_test.cpp
@@ -0,0 +1,47 @@
+//===-- Unittests for epoll_ctl -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/sys_epoll_macros.h"
+#include "hdr/types/struct_epoll_event.h"
+#include "src/errno/libc_errno.h"
+#include "src/sys/epoll/epoll_create1.h"
+#include "src/sys/epoll/epoll_ctl.h"
+#include "src/unistd/close.h"
+#include "src/unistd/pipe.h"
+#include "test/UnitTest/ErrnoSetterMatcher.h"
+#include "test/UnitTest/Test.h"
+
+using namespace LIBC_NAMESPACE::testing::ErrnoSetterMatcher;
+
+TEST(LlvmLibcEpollCtlTest, Basic) {
+  int epfd = LIBC_NAMESPACE::epoll_create1(0);
+  ASSERT_GT(epfd, 0);
+  ASSERT_ERRNO_SUCCESS();
+
+  int pipefd[2];
+
+  ASSERT_THAT(LIBC_NAMESPACE::pipe(pipefd), Succeeds());
+
+  epoll_event event;
+  event.events = EPOLLOUT;
+  event.data.fd = pipefd[0];
+
+  ASSERT_THAT(LIBC_NAMESPACE::epoll_ctl(epfd, EPOLL_CTL_ADD, pipefd[0], &event),
+              Succeeds());
+
+  // adding the same file fail.
+  ASSERT_THAT(LIBC_NAMESPACE::epoll_ctl(epfd, EPOLL_CTL_ADD, pipefd[0], &event),
+              Fails(EEXIST));
+
+  ASSERT_THAT(LIBC_NAMESPACE::epoll_ctl(epfd, EPOLL_CTL_DEL, pipefd[0], &event),
+              Succeeds());
+
+  ASSERT_THAT(LIBC_NAMESPACE::close(pipefd[0]), Succeeds());
+  ASSERT_THAT(LIBC_NAMESPACE::close(pipefd[1]), Succeeds());
+  ASSERT_THAT(LIBC_NAMESPACE::close(epfd), Succeeds());
+}
diff --git a/libc/test/src/sys/epoll/linux/epoll_pwait2_test.cpp b/libc/test/src/sys/epoll/linux/epoll_pwait2_test.cpp
index 83fe12bb8eb6..2f4c9854be1a 100644
--- a/libc/test/src/sys/epoll/linux/epoll_pwait2_test.cpp
+++ b/libc/test/src/sys/epoll/linux/epoll_pwait2_test.cpp
@@ -5,16 +5,53 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+#include "hdr/sys_epoll_macros.h"
+#include "hdr/types/struct_epoll_event.h"
+#include "hdr/types/struct_timespec.h"
 #include "src/errno/libc_errno.h"
+#include "src/sys/epoll/epoll_create1.h"
+#include "src/sys/epoll/epoll_ctl.h"
 #include "src/sys/epoll/epoll_pwait2.h"
+#include "src/unistd/close.h"
+#include "src/unistd/pipe.h"
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
 using namespace LIBC_NAMESPACE::testing::ErrnoSetterMatcher;
 
-TEST(LlvmLibcEpollWaitTest, Basic) {
-  EXPECT_THAT(LIBC_NAMESPACE::epoll_pwait2(-1, nullptr, 0, nullptr, nullptr),
-              returns(EQ(-1ul)).with_errno(EQ(EINVAL)));
-}
+TEST(LlvmLibcEpollPwaitTest, Basic) {
+  int epfd = LIBC_NAMESPACE::epoll_create1(0);
+  ASSERT_GT(epfd, 0);
+  ASSERT_ERRNO_SUCCESS();
+
+  int pipefd[2];
+
+  ASSERT_THAT(LIBC_NAMESPACE::pipe(pipefd), Succeeds());
+
+  epoll_event event;
+  event.events = EPOLLOUT;
+  event.data.fd = pipefd[0];
+
+  timespec time_spec;
+  time_spec.tv_sec = 0;
+  time_spec.tv_nsec = 0;
 
-// TODO: Complete these tests when epoll_create is implemented.
+  ASSERT_THAT(LIBC_NAMESPACE::epoll_ctl(epfd, EPOLL_CTL_ADD, pipefd[0], &event),
+              Succeeds());
+
+  // Timeout of 0 causes immediate return. We just need to check that the
+  // interface works, we're not testing the kernel behavior here.
+  ASSERT_THAT(
+      LIBC_NAMESPACE::epoll_pwait2(epfd, &event, 1, &time_spec, nullptr),
+      Succeeds());
+
+  ASSERT_THAT(LIBC_NAMESPACE::epoll_pwait2(-1, &event, 1, &time_spec, nullptr),
+              Fails(EBADF));
+
+  ASSERT_THAT(LIBC_NAMESPACE::epoll_ctl(epfd, EPOLL_CTL_DEL, pipefd[0], &event),
+              Succeeds());
+
+  ASSERT_THAT(LIBC_NAMESPACE::close(pipefd[0]), Succeeds());
+  ASSERT_THAT(LIBC_NAMESPACE::close(pipefd[1]), Succeeds());
+  ASSERT_THAT(LIBC_NAMESPACE::close(epfd), Succeeds());
+}
diff --git a/libc/test/src/sys/epoll/linux/epoll_pwait_test.cpp b/libc/test/src/sys/epoll/linux/epoll_pwait_test.cpp
index 217facbfcebb..8e14aea8b9d5 100644
--- a/libc/test/src/sys/epoll/linux/epoll_pwait_test.cpp
+++ b/libc/test/src/sys/epoll/linux/epoll_pwait_test.cpp
@@ -5,16 +5,48 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+#include "hdr/sys_epoll_macros.h"
+#include "hdr/types/struct_epoll_event.h"
 #include "src/errno/libc_errno.h"
+#include "src/sys/epoll/epoll_create1.h"
+#include "src/sys/epoll/epoll_ctl.h"
 #include "src/sys/epoll/epoll_pwait.h"
+#include "src/unistd/close.h"
+#include "src/unistd/pipe.h"
+
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
 using namespace LIBC_NAMESPACE::testing::ErrnoSetterMatcher;
 
-TEST(LlvmLibcEpollWaitTest, Basic) {
-  EXPECT_THAT(LIBC_NAMESPACE::epoll_pwait(-1, nullptr, 0, 0, nullptr),
-              returns(EQ(-1ul)).with_errno(EQ(EINVAL)));
-}
+TEST(LlvmLibcEpollPwaitTest, Basic) {
+  int epfd = LIBC_NAMESPACE::epoll_create1(0);
+  ASSERT_GT(epfd, 0);
+  ASSERT_ERRNO_SUCCESS();
+
+  int pipefd[2];
+
+  ASSERT_THAT(LIBC_NAMESPACE::pipe(pipefd), Succeeds());
+
+  epoll_event event;
+  event.events = EPOLLOUT;
+  event.data.fd = pipefd[0];
 
-// TODO: Complete these tests when epoll_create is implemented.
+  ASSERT_THAT(LIBC_NAMESPACE::epoll_ctl(epfd, EPOLL_CTL_ADD, pipefd[0], &event),
+              Succeeds());
+
+  // Timeout of 0 causes immediate return. We just need to check that the
+  // interface works, we're not testing the kernel behavior here.
+  ASSERT_THAT(LIBC_NAMESPACE::epoll_pwait(epfd, &event, 1, 0, nullptr),
+              Succeeds());
+
+  ASSERT_THAT(LIBC_NAMESPACE::epoll_pwait(-1, &event, 1, 0, nullptr),
+              Fails(EBADF));
+
+  ASSERT_THAT(LIBC_NAMESPACE::epoll_ctl(epfd, EPOLL_CTL_DEL, pipefd[0], &event),
+              Succeeds());
+
+  ASSERT_THAT(LIBC_NAMESPACE::close(pipefd[0]), Succeeds());
+  ASSERT_THAT(LIBC_NAMESPACE::close(pipefd[1]), Succeeds());
+  ASSERT_THAT(LIBC_NAMESPACE::close(epfd), Succeeds());
+}
diff --git a/libc/test/src/sys/epoll/linux/epoll_wait_test.cpp b/libc/test/src/sys/epoll/linux/epoll_wait_test.cpp
index 57fef3f690b6..f9e855a891b0 100644
--- a/libc/test/src/sys/epoll/linux/epoll_wait_test.cpp
+++ b/libc/test/src/sys/epoll/linux/epoll_wait_test.cpp
@@ -5,16 +5,45 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+#include "hdr/sys_epoll_macros.h"
+#include "hdr/types/struct_epoll_event.h"
 #include "src/errno/libc_errno.h"
+#include "src/sys/epoll/epoll_create1.h"
+#include "src/sys/epoll/epoll_ctl.h"
 #include "src/sys/epoll/epoll_wait.h"
+#include "src/unistd/close.h"
+#include "src/unistd/pipe.h"
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
 using namespace LIBC_NAMESPACE::testing::ErrnoSetterMatcher;
 
 TEST(LlvmLibcEpollWaitTest, Basic) {
-  EXPECT_THAT(LIBC_NAMESPACE::epoll_wait(-1, nullptr, 0, 0),
-              returns(EQ(-1ul)).with_errno(EQ(EINVAL)));
-}
+  int epfd = LIBC_NAMESPACE::epoll_create1(0);
+  ASSERT_GT(epfd, 0);
+  ASSERT_ERRNO_SUCCESS();
+
+  int pipefd[2];
+
+  ASSERT_THAT(LIBC_NAMESPACE::pipe(pipefd), Succeeds());
+
+  epoll_event event;
+  event.events = EPOLLOUT;
+  event.data.fd = pipefd[0];
+
+  ASSERT_THAT(LIBC_NAMESPACE::epoll_ctl(epfd, EPOLL_CTL_ADD, pipefd[0], &event),
+              Succeeds());
 
-// TODO: Complete these tests when epoll_create is implemented.
+  // Timeout of 0 causes immediate return. We just need to check that the
+  // interface works, we're not testing the kernel behavior here.
+  ASSERT_THAT(LIBC_NAMESPACE::epoll_wait(epfd, &event, 1, 0), Succeeds());
+
+  ASSERT_THAT(LIBC_NAMESPACE::epoll_wait(-1, &event, 1, 0), Fails(EBADF));
+
+  ASSERT_THAT(LIBC_NAMESPACE::epoll_ctl(epfd, EPOLL_CTL_DEL, pipefd[0], &event),
+              Succeeds());
+
+  ASSERT_THAT(LIBC_NAMESPACE::close(pipefd[0]), Succeeds());
+  ASSERT_THAT(LIBC_NAMESPACE::close(pipefd[1]), Succeeds());
+  ASSERT_THAT(LIBC_NAMESPACE::close(epfd), Succeeds());
+}
diff --git a/libc/test/src/unistd/CMakeLists.txt b/libc/test/src/unistd/CMakeLists.txt
index 9de5081f0094..de3e8d9ccbb6 100644
--- a/libc/test/src/unistd/CMakeLists.txt
+++ b/libc/test/src/unistd/CMakeLists.txt
@@ -212,6 +212,21 @@ add_libc_unittest(
     libc.test.UnitTest.ErrnoSetterMatcher
 )
 
+add_libc_unittest(
+  pipe_test
+  SUITE
+    libc_unistd_unittests
+  SRCS
+    pipe_test.cpp
+  DEPENDS
+    libc.include.errno
+    libc.include.unistd
+    libc.src.errno.errno
+    libc.src.unistd.close
+    libc.src.unistd.pipe
+    libc.test.UnitTest.ErrnoSetterMatcher
+)
+
 add_libc_unittest(
   rmdir_test
   SUITE
diff --git a/libc/test/src/unistd/pipe_test.cpp b/libc/test/src/unistd/pipe_test.cpp
new file mode 100644
index 000000000000..9c633dea1dba
--- /dev/null
+++ b/libc/test/src/unistd/pipe_test.cpp
@@ -0,0 +1,26 @@
+//===-- Unittests for pipe ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "src/unistd/close.h"
+#include "src/unistd/pipe.h"
+
+#include "test/UnitTest/ErrnoSetterMatcher.h"
+#include "test/UnitTest/Test.h"
+
+using namespace LIBC_NAMESPACE::testing::ErrnoSetterMatcher;
+
+TEST(LlvmLibcPipeTest, SmokeTest) {
+
+  int pipefd[2];
+
+  ASSERT_THAT(LIBC_NAMESPACE::pipe(pipefd), Succeeds());
+
+  ASSERT_THAT(LIBC_NAMESPACE::close(pipefd[0]), Succeeds());
+  ASSERT_THAT(LIBC_NAMESPACE::close(pipefd[1]), Succeeds());
+}
+
+// TODO: Functionality tests
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index c61904a96711..9270a9d75b01 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -106,7 +106,7 @@ libc_support_library(
     hdrs = ["include/llvm-libc-macros/linux/fcntl-macros.h"],
 )
 
-############################ Proxy Header Files ################################
+########################### Macro Proxy Header Files ###########################
 
 libc_support_library(
     name = "hdr_math_macros",
@@ -118,6 +118,33 @@ libc_support_library(
     hdrs = ["hdr/fenv_macros.h"],
 )
 
+libc_support_library(
+    name = "hdr_signal_macros",
+    hdrs = ["hdr/signal_macros.h"],
+)
+
+libc_support_library(
+    name = "hdr_sys_epoll_macros",
+    hdrs = ["hdr/sys_epoll_macros.h"],
+)
+
+############################ Type Proxy Header Files ###########################
+
+libc_support_library(
+    name = "types_sigset_t",
+    hdrs = ["hdr/types/sigset_t.h"],
+)
+
+libc_support_library(
+    name = "types_struct_epoll_event",
+    hdrs = ["hdr/types/struct_epoll_event.h"],
+)
+
+libc_support_library(
+    name = "types_struct_timespec",
+    hdrs = ["hdr/types/struct_timespec.h"],
+)
+
 ############################### Support libraries ##############################
 
 libc_support_library(
@@ -2941,6 +2968,17 @@ libc_function(
     ],
 )
 
+libc_function(
+    name = "pipe",
+    srcs = ["src/unistd/linux/pipe.cpp"],
+    hdrs = ["src/unistd/pipe.h"],
+    deps = [
+        ":__support_common",
+        ":__support_osutil_syscall",
+        ":errno",
+    ],
+)
+
 libc_function(
     name = "lseek",
     srcs = ["src/unistd/linux/lseek.cpp"],
@@ -3417,13 +3455,51 @@ libc_function(
 
 ############################## sys/epoll targets ###############################
 
-libc_support_library(
-  name = "types_sigset_t",
-  hdrs = ["hdr/types/sigset_t.h"],
+libc_function(
+    name = "epoll_create",
+    srcs = ["src/sys/epoll/linux/epoll_create.cpp"],
+    hdrs = ["src/sys/epoll/epoll_create.h"],
+    target_compatible_with = select({
+        "@platforms//os:linux": [],
+        "//conditions:default": ["@platforms//:incompatible"],
+    }),
+    weak = True,
+    deps = [
+        ":__support_osutil_syscall",
+        ":errno",
+    ],
 )
-libc_support_library(
-  name = "types_struct_epoll_event",
-  hdrs = ["hdr/types/struct_epoll_event.h"],
+
+libc_function(
+    name = "epoll_create1",
+    srcs = ["src/sys/epoll/linux/epoll_create1.cpp"],
+    hdrs = ["src/sys/epoll/epoll_create1.h"],
+    target_compatible_with = select({
+        "@platforms//os:linux": [],
+        "//conditions:default": ["@platforms//:incompatible"],
+    }),
+    weak = True,
+    deps = [
+        ":__support_osutil_syscall",
+        ":errno",
+    ],
+)
+
+libc_function(
+    name = "epoll_ctl",
+    srcs = ["src/sys/epoll/linux/epoll_ctl.cpp"],
+    hdrs = ["src/sys/epoll/epoll_ctl.h"],
+    target_compatible_with = select({
+        "@platforms//os:linux": [],
+        "//conditions:default": ["@platforms//:incompatible"],
+    }),
+    weak = True,
+    deps = [
+        ":__support_osutil_syscall",
+        ":errno",
+        ":hdr_sys_epoll_macros",
+        ":types_struct_epoll_event",
+    ],
 )
 
 libc_function(
@@ -3438,6 +3514,8 @@ libc_function(
     deps = [
         ":__support_osutil_syscall",
         ":errno",
+        ":hdr_signal_macros",
+        ":hdr_sys_epoll_macros",
         ":types_sigset_t",
         ":types_struct_epoll_event",
     ],
@@ -3455,6 +3533,8 @@ libc_function(
     deps = [
         ":__support_osutil_syscall",
         ":errno",
+        ":hdr_signal_macros",
+        ":hdr_sys_epoll_macros",
         ":types_sigset_t",
         ":types_struct_epoll_event",
     ],
@@ -3466,9 +3546,18 @@ libc_function(
 #     name = "epoll_pwait2",
 #     srcs = ["src/sys/epoll/linux/epoll_pwait2.cpp"],
 #     hdrs = ["src/sys/epoll/epoll_pwait2.h"],
+#     target_compatible_with = select({
+#         "@platforms//os:linux": [],
+#         "//conditions:default": ["@platforms//:incompatible"],
+#     }),
 #     weak = True,
 #     deps = [
 #         ":__support_osutil_syscall",
 #         ":errno",
+#         ":hdr_signal_macros",
+#         ":hdr_sys_epoll_macros",
+#         ":types_sigset_t",
+#         ":types_struct_epoll_event",
+#         ":types_struct_timespec",
 #     ],
 # )
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/sys/epoll/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/sys/epoll/BUILD.bazel
index e6b63235757e..b090bde35b88 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/sys/epoll/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/sys/epoll/BUILD.bazel
@@ -10,11 +10,55 @@ package(default_visibility = ["//visibility:public"])
 
 licenses(["notice"])
 
+libc_test(
+    name = "epoll_create_test",
+    srcs = ["linux/epoll_create_test.cpp"],
+    libc_function_deps = [
+        "//libc:epoll_create",
+        "//libc:close",
+    ],
+)
+
+libc_test(
+    name = "epoll_create1_test",
+    srcs = ["linux/epoll_create1_test.cpp"],
+    libc_function_deps = [
+        "//libc:epoll_create1",
+        "//libc:close",
+    ],
+    deps = [
+        "//libc:hdr_sys_epoll_macros",
+    ],
+)
+
+libc_test(
+    name = "epoll_ctl_test",
+    srcs = ["linux/epoll_ctl_test.cpp"],
+    libc_function_deps = [
+        "//libc:epoll_create1",
+        "//libc:epoll_ctl",
+        "//libc:pipe",
+        "//libc:close",
+    ],
+    deps = [
+        "//libc:hdr_sys_epoll_macros",
+        "//libc:types_struct_epoll_event",
+    ],
+)
+
 libc_test(
     name = "epoll_wait_test",
     srcs = ["linux/epoll_wait_test.cpp"],
     libc_function_deps = [
         "//libc:epoll_wait",
+        "//libc:epoll_create1",
+        "//libc:epoll_ctl",
+        "//libc:pipe",
+        "//libc:close",
+    ],
+    deps = [
+        "//libc:hdr_sys_epoll_macros",
+        "//libc:types_struct_epoll_event",
     ],
 )
 
@@ -23,6 +67,14 @@ libc_test(
     srcs = ["linux/epoll_pwait_test.cpp"],
     libc_function_deps = [
         "//libc:epoll_pwait",
+        "//libc:epoll_create1",
+        "//libc:epoll_ctl",
+        "//libc:pipe",
+        "//libc:close",
+    ],
+    deps = [
+        "//libc:hdr_sys_epoll_macros",
+        "//libc:types_struct_epoll_event",
     ],
 )
 
@@ -33,5 +85,14 @@ libc_test(
 #     srcs = ["linux/epoll_pwait2_test.cpp"],
 #     libc_function_deps = [
 #         "//libc:epoll_pwait2",
+#         "//libc:epoll_create1",
+#         "//libc:epoll_ctl",
+#         "//libc:pipe",
+#         "//libc:close",
+#     ],
+#     deps = [
+#         "//libc:hdr_sys_epoll_macros",
+#         "//libc:types_struct_epoll_event",
+#         "//libc:types_struct_timespec",
 #     ],
 # )
-- 
GitLab


From 7aa371687ace40b85f04e21956e03f1e93052b56 Mon Sep 17 00:00:00 2001
From: Nathan Ridge <zeratul976@hotmail.com>
Date: Thu, 11 Apr 2024 19:34:53 -0400
Subject: [PATCH 016/464] [clangd] Avoid using CompletionItemKind.Text for
 macro completions from the index (#88236)

This was fixed in https://github.com/clangd/clangd/issues/1484 for Sema
completions but the fix did not apply to index completions.

Fixes https://github.com/clangd/clangd/issues/2002
---
 clang-tools-extra/clangd/CodeComplete.cpp        | 16 +++++++++++++---
 .../clangd/unittests/CodeCompleteTests.cpp       |  7 +++++--
 clang-tools-extra/clangd/unittests/TestIndex.cpp |  7 ++++++-
 clang-tools-extra/clangd/unittests/TestIndex.h   |  4 +++-
 4 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/clang-tools-extra/clangd/CodeComplete.cpp b/clang-tools-extra/clangd/CodeComplete.cpp
index 9e321dce4c50..89eee392837a 100644
--- a/clang-tools-extra/clangd/CodeComplete.cpp
+++ b/clang-tools-extra/clangd/CodeComplete.cpp
@@ -89,7 +89,11 @@ const CodeCompleteOptions::CodeCompletionRankingModel
 
 namespace {
 
-CompletionItemKind toCompletionItemKind(index::SymbolKind Kind) {
+// Note: changes to this function should also be reflected in the
+// CodeCompletionResult overload where appropriate.
+CompletionItemKind
+toCompletionItemKind(index::SymbolKind Kind,
+                     const llvm::StringRef *Signature = nullptr) {
   using SK = index::SymbolKind;
   switch (Kind) {
   case SK::Unknown:
@@ -99,7 +103,10 @@ CompletionItemKind toCompletionItemKind(index::SymbolKind Kind) {
   case SK::NamespaceAlias:
     return CompletionItemKind::Module;
   case SK::Macro:
-    return CompletionItemKind::Text;
+    // Use macro signature (if provided) to tell apart function-like and
+    // object-like macros.
+    return Signature && Signature->contains('(') ? CompletionItemKind::Function
+                                                 : CompletionItemKind::Constant;
   case SK::Enum:
     return CompletionItemKind::Enum;
   case SK::Struct:
@@ -150,6 +157,8 @@ CompletionItemKind toCompletionItemKind(index::SymbolKind Kind) {
   llvm_unreachable("Unhandled clang::index::SymbolKind.");
 }
 
+// Note: changes to this function should also be reflected in the
+// index::SymbolKind overload where appropriate.
 CompletionItemKind toCompletionItemKind(const CodeCompletionResult &Res,
                                         CodeCompletionContext::Kind CtxKind) {
   if (Res.Declaration)
@@ -379,7 +388,8 @@ struct CodeCompletionBuilder {
       if (Completion.Scope.empty())
         Completion.Scope = std::string(C.IndexResult->Scope);
       if (Completion.Kind == CompletionItemKind::Missing)
-        Completion.Kind = toCompletionItemKind(C.IndexResult->SymInfo.Kind);
+        Completion.Kind = toCompletionItemKind(C.IndexResult->SymInfo.Kind,
+                                               &C.IndexResult->Signature);
       if (Completion.Name.empty())
         Completion.Name = std::string(C.IndexResult->Name);
       if (Completion.FilterText.empty())
diff --git a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
index 49337bddf98d..8fbac73cb653 100644
--- a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
+++ b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
@@ -671,7 +671,8 @@ TEST(CompletionTest, Kinds) {
           #define MACRO 10
           int X = ^
       )cpp",
-      {func("indexFunction"), var("indexVariable"), cls("indexClass")});
+      {func("indexFunction"), var("indexVariable"), cls("indexClass"),
+       macro("indexObjMacro"), macro("indexFuncMacro", "(x, y)")});
   EXPECT_THAT(Results.Completions,
               AllOf(has("function", CompletionItemKind::Function),
                     has("variable", CompletionItemKind::Variable),
@@ -680,7 +681,9 @@ TEST(CompletionTest, Kinds) {
                     has("MACRO", CompletionItemKind::Constant),
                     has("indexFunction", CompletionItemKind::Function),
                     has("indexVariable", CompletionItemKind::Variable),
-                    has("indexClass", CompletionItemKind::Class)));
+                    has("indexClass", CompletionItemKind::Class),
+                    has("indexObjMacro", CompletionItemKind::Constant),
+                    has("indexFuncMacro", CompletionItemKind::Function)));
 
   Results = completions("nam^");
   EXPECT_THAT(Results.Completions,
diff --git a/clang-tools-extra/clangd/unittests/TestIndex.cpp b/clang-tools-extra/clangd/unittests/TestIndex.cpp
index 278336bdde2e..b13a5d32d175 100644
--- a/clang-tools-extra/clangd/unittests/TestIndex.cpp
+++ b/clang-tools-extra/clangd/unittests/TestIndex.cpp
@@ -38,7 +38,7 @@ static std::string replace(llvm::StringRef Haystack, llvm::StringRef Needle,
 // Helpers to produce fake index symbols for memIndex() or completions().
 // USRFormat is a regex replacement string for the unqualified part of the USR.
 Symbol sym(llvm::StringRef QName, index::SymbolKind Kind,
-           llvm::StringRef USRFormat) {
+           llvm::StringRef USRFormat, llvm::StringRef Signature) {
   Symbol Sym;
   std::string USR = "c:"; // We synthesize a few simple cases of USRs by hand!
   size_t Pos = QName.rfind("::");
@@ -55,6 +55,7 @@ Symbol sym(llvm::StringRef QName, index::SymbolKind Kind,
   Sym.SymInfo.Kind = Kind;
   Sym.Flags |= Symbol::IndexedForCodeCompletion;
   Sym.Origin = SymbolOrigin::Static;
+  Sym.Signature = Signature;
   return Sym;
 }
 
@@ -86,6 +87,10 @@ Symbol conceptSym(llvm::StringRef Name) {
   return sym(Name, index::SymbolKind::Concept, "@CT@\\0");
 }
 
+Symbol macro(llvm::StringRef Name, llvm::StringRef ArgList) {
+  return sym(Name, index::SymbolKind::Macro, "@macro@\\0", ArgList);
+}
+
 Symbol objcSym(llvm::StringRef Name, index::SymbolKind Kind,
                llvm::StringRef USRPrefix) {
   Symbol Sym;
diff --git a/clang-tools-extra/clangd/unittests/TestIndex.h b/clang-tools-extra/clangd/unittests/TestIndex.h
index 9280b0b12a67..0699b29392d7 100644
--- a/clang-tools-extra/clangd/unittests/TestIndex.h
+++ b/clang-tools-extra/clangd/unittests/TestIndex.h
@@ -20,7 +20,7 @@ Symbol symbol(llvm::StringRef QName);
 // Helpers to produce fake index symbols with proper SymbolID.
 // USRFormat is a regex replacement string for the unqualified part of the USR.
 Symbol sym(llvm::StringRef QName, index::SymbolKind Kind,
-           llvm::StringRef USRFormat);
+           llvm::StringRef USRFormat, llvm::StringRef Signature = {});
 // Creats a function symbol assuming no function arg.
 Symbol func(llvm::StringRef Name);
 // Creates a class symbol.
@@ -35,6 +35,8 @@ Symbol var(llvm::StringRef Name);
 Symbol ns(llvm::StringRef Name);
 // Create a C++20 concept symbol.
 Symbol conceptSym(llvm::StringRef Name);
+// Create a macro symbol.
+Symbol macro(llvm::StringRef Name, llvm::StringRef ArgList = {});
 
 // Create an Objective-C symbol.
 Symbol objcSym(llvm::StringRef Name, index::SymbolKind Kind,
-- 
GitLab


From c42a262560a659d1cfb9ec46712ad66420314f2d Mon Sep 17 00:00:00 2001
From: Alexandre Eichenberger <alexe@us.ibm.com>
Date: Thu, 11 Apr 2024 20:04:26 -0400
Subject: [PATCH 017/464] [MLIR] Bug Fix: affine.prefetch replaceAffineOp
 invoked during canonicalization (#88346)

Signed-off-by: Alexandre Eichenberger <alexe@us.ibm.com>
---
 mlir/lib/Dialect/Affine/IR/AffineOps.cpp   |  5 ++---
 mlir/test/Dialect/Affine/canonicalize.mlir | 14 ++++++++++++++
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
index c591e5056480..c9c0a7b4cc68 100644
--- a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
@@ -1487,9 +1487,8 @@ void SimplifyAffineOp<AffinePrefetchOp>::replaceAffineOp(
     PatternRewriter &rewriter, AffinePrefetchOp prefetch, AffineMap map,
     ArrayRef<Value> mapOperands) const {
   rewriter.replaceOpWithNewOp<AffinePrefetchOp>(
-      prefetch, prefetch.getMemref(), map, mapOperands,
-      prefetch.getLocalityHint(), prefetch.getIsWrite(),
-      prefetch.getIsDataCache());
+      prefetch, prefetch.getMemref(), map, mapOperands, prefetch.getIsWrite(),
+      prefetch.getLocalityHint(), prefetch.getIsDataCache());
 }
 template <>
 void SimplifyAffineOp<AffineStoreOp>::replaceAffineOp(
diff --git a/mlir/test/Dialect/Affine/canonicalize.mlir b/mlir/test/Dialect/Affine/canonicalize.mlir
index 7c0930eedc85..730808dbbdf1 100644
--- a/mlir/test/Dialect/Affine/canonicalize.mlir
+++ b/mlir/test/Dialect/Affine/canonicalize.mlir
@@ -1452,3 +1452,17 @@ func.func @mod_of_mod(%lb: index, %ub: index, %step: index) -> (index, index) {
   %1 = affine.apply affine_map<()[s0, s1, s2] -> ((s0 - ((s0 - s2) mod s1) - s2) mod s1)> ()[%ub, %step, %lb]
   return %0, %1 : index, index
 }
+
+// -----
+
+// CHECK-LABEL:  func.func @prefetch_canonicalize
+// CHECK-SAME:   ([[PARAM_0_:%.+]]: memref<512xf32>) {
+func.func @prefetch_canonicalize(%arg0: memref<512xf32>) -> () {
+  // CHECK: affine.for [[I_0_:%.+]] = 0 to 8 {
+  affine.for %arg3 = 0 to 8  {
+    %1 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%arg3]
+    // CHECK: affine.prefetch [[PARAM_0_]][symbol([[I_0_]]) * 64], read, locality<3>, data : memref<512xf32>
+    affine.prefetch %arg0[%1], read, locality<3>, data : memref<512xf32>
+  }
+  return
+}
-- 
GitLab


From 56954a53e58282d7584e31ec14a2b1052cd861e8 Mon Sep 17 00:00:00 2001
From: pawelszczerbuk <153013546+pawelszczerbuk@users.noreply.github.com>
Date: Thu, 11 Apr 2024 17:09:44 -0700
Subject: [PATCH 018/464] [MLIR][LoopPipelining] Improve schedule verifier, so
 it checks also operands of nested operations (#88450)

`verifySchedule` was not looking at the operands of nested operations,
which caused incorrect schedule to be allowed in some cases, potentially
leading to crash during expansion.
There is also minor fix in `cloneAndUpdateOperands` in the pipeline
expander that prevents double visit of the cloned op. This one has no
functional impact, so no test for it.
---
 .../Dialect/SCF/Transforms/LoopPipelining.cpp | 18 +++++++++++----
 mlir/test/Dialect/SCF/loop-pipelining.mlir    | 23 +++++++++++++++++++
 2 files changed, 37 insertions(+), 4 deletions(-)

diff --git a/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp b/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
index 9eda1a4597ba..82ec95d31f52 100644
--- a/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
@@ -203,6 +203,17 @@ bool LoopPipelinerInternal::initializeLoopInfo(
   return true;
 }
 
+/// Find operands of all the nested operations within `op`.
+static SetVector<Value> getNestedOperands(Operation *op) {
+  SetVector<Value> operands;
+  op->walk([&](Operation *nestedOp) {
+    for (Value operand : nestedOp->getOperands()) {
+      operands.insert(operand);
+    }
+  });
+  return operands;
+}
+
 /// Compute unrolled cycles of each op (consumer) and verify that each op is
 /// scheduled after its operands (producers) while adjusting for the distance
 /// between producer and consumer.
@@ -219,7 +230,7 @@ bool LoopPipelinerInternal::verifySchedule() {
   }
   for (Operation *consumer : opOrder) {
     int64_t consumerCycle = unrolledCyles[consumer];
-    for (Value operand : consumer->getOperands()) {
+    for (Value operand : getNestedOperands(consumer)) {
       auto [producer, distance] = getDefiningOpAndDistance(operand);
       if (!producer)
         continue;
@@ -245,9 +256,8 @@ static Operation *
 cloneAndUpdateOperands(RewriterBase &rewriter, Operation *op,
                        function_ref<void(OpOperand *newOperand)> callback) {
   Operation *clone = rewriter.clone(*op);
-  for (OpOperand &operand : clone->getOpOperands())
-    callback(&operand);
-  clone->walk([&](Operation *nested) {
+  clone->walk<WalkOrder::PreOrder>([&](Operation *nested) {
+    // 'clone' itself will be visited first.
     for (OpOperand &operand : nested->getOpOperands()) {
       Operation *def = operand.get().getDefiningOp();
       if ((def && !clone->isAncestor(def)) || isa<BlockArgument>(operand.get()))
diff --git a/mlir/test/Dialect/SCF/loop-pipelining.mlir b/mlir/test/Dialect/SCF/loop-pipelining.mlir
index 8d6f454d1875..46e7feca4329 100644
--- a/mlir/test/Dialect/SCF/loop-pipelining.mlir
+++ b/mlir/test/Dialect/SCF/loop-pipelining.mlir
@@ -846,3 +846,26 @@ func.func @invalid_schedule2(%A: memref<?xf32>, %result: memref<?xf32>) {
   }  { __test_pipelining_loop__ }
   return
 }
+
+// -----
+
+func.func @invalid_schedule3(%A: memref<?xf32>, %result: memref<?xf32>, %ext: index) {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  %r = scf.for %i0 = %c0 to %c4 step %c1 iter_args(%idx = %c0) -> (index) {
+    %cnd = arith.cmpi slt, %ext, %c4 { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 0 } : index
+    // expected-error@+1 {{operation scheduled before its operands}}
+    %idx1 = scf.if %cnd -> (index) {
+      %idxinc = arith.addi %idx, %c1 : index
+      scf.yield %idxinc : index
+    } else {
+      scf.yield %idx : index
+    } { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 1 }
+    %A_elem = memref.load %A[%idx1] { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 2 } : memref<?xf32>
+    %idx2 = arith.addi %idx1, %c1 { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 3 } : index
+    memref.store %A_elem, %result[%idx1] { __test_pipelining_stage__ = 2, __test_pipelining_op_order__ = 4 } : memref<?xf32>
+    scf.yield %idx2 : index
+  }  { __test_pipelining_loop__ }
+  return
+}
-- 
GitLab


From bf1d7b8df287d69ee265b91be40dec37267b2d5c Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Thu, 11 Apr 2024 16:55:22 -0700
Subject: [PATCH 019/464] [RISCV] Give HWASAN_CHECK_MEMACCESS_SHORTGRANULES
 pseudo the same code size as PseudoCall.

This is converted to a PseudoCall in RISCVAsmPrinter.cpp so it should
have the same size of 8 bytes.
---
 llvm/lib/Target/RISCV/RISCVInstrInfo.td | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
index 2cfad7f7c061..f9dadc6c0d48 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
@@ -1861,7 +1861,7 @@ def : Pat<(trap), (UNIMP)>;
 def : Pat<(debugtrap), (EBREAK)>;
 
 let Predicates = [IsRV64], Uses = [X5],
-    Defs = [X1, X6, X7, X28, X29, X30, X31] in
+    Defs = [X1, X6, X7, X28, X29, X30, X31], Size = 8 in
 def HWASAN_CHECK_MEMACCESS_SHORTGRANULES
   : Pseudo<(outs), (ins GPRJALR:$ptr, i32imm:$accessinfo),
            [(int_hwasan_check_memaccess_shortgranules (i64 X5), GPRJALR:$ptr,
-- 
GitLab


From 09d51a841dcfbc41c3d7f3274b109b5f9fb09bb0 Mon Sep 17 00:00:00 2001
From: "Felix (Ting Wang)" <Ting.Wang.SH@ibm.com>
Date: Fri, 12 Apr 2024 08:18:01 +0800
Subject: [PATCH 020/464] [PowerPC][AIX] Enable aix-small-local-dynamic-tls
 target attribute (#86641)

Following the aix-small-local-exec-tls target attribute, this patch adds
a target attribute for an AIX-specific option in llc that informs the
compiler that it can use a faster access sequence for the local-dynamic
TLS model (formally named aix-small-local-dynamic-tls) when TLS
variables are less than ~32KB in size.

The patch either produces an addi/la with a displacement off of module
handle (return value from .__tls_get_mod) when the address is
calculated, or it produces an addi/la followed by a load/store when the
address is calculated and used for further accesses.

---------

Co-authored-by: Amy Kwan <amy.kwan1@ibm.com>
---
 .../MCTargetDesc/PPCXCOFFObjectWriter.cpp     |    4 +
 llvm/lib/Target/PowerPC/PPC.td                |    9 +
 llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp     |   20 +-
 llvm/lib/Target/PowerPC/PPCISelLowering.cpp   |   32 +-
 llvm/lib/Target/PowerPC/PPCMCInstLower.cpp    |   13 +-
 llvm/lib/Target/PowerPC/PPCSubtarget.cpp      |   26 +-
 ...aix-small-local-dynamic-tls-largeaccess.ll |  182 +-
 .../aix-small-local-dynamic-tls-types.ll      | 1601 ++++++++++++++---
 ...ix-small-local-exec-tls-opt-IRattribute.ll |   25 +-
 .../check-aix-small-local-exec-tls-opt.ll     |   14 +-
 10 files changed, 1530 insertions(+), 396 deletions(-)

diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCXCOFFObjectWriter.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCXCOFFObjectWriter.cpp
index f4998e9b9dcb..714ce64a3939 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCXCOFFObjectWriter.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCXCOFFObjectWriter.cpp
@@ -71,6 +71,8 @@ std::pair<uint8_t, uint8_t> PPCXCOFFObjectWriter::getRelocTypeAndSignSize(
       return {XCOFF::RelocationType::R_TOCL, SignAndSizeForHalf16};
     case MCSymbolRefExpr::VK_PPC_AIX_TLSLE:
       return {XCOFF::RelocationType::R_TLS_LE, SignAndSizeForHalf16};
+    case MCSymbolRefExpr::VK_PPC_AIX_TLSLD:
+      return {XCOFF::RelocationType::R_TLS_LD, SignAndSizeForHalf16};
     }
   } break;
   case PPC::fixup_ppc_half16ds:
@@ -86,6 +88,8 @@ std::pair<uint8_t, uint8_t> PPCXCOFFObjectWriter::getRelocTypeAndSignSize(
       return {XCOFF::RelocationType::R_TOCL, 15};
     case MCSymbolRefExpr::VK_PPC_AIX_TLSLE:
       return {XCOFF::RelocationType::R_TLS_LE, 15};
+    case MCSymbolRefExpr::VK_PPC_AIX_TLSLD:
+      return {XCOFF::RelocationType::R_TLS_LD, 15};
     }
   } break;
   case PPC::fixup_ppc_br24:
diff --git a/llvm/lib/Target/PowerPC/PPC.td b/llvm/lib/Target/PowerPC/PPC.td
index 535616d33a80..b962ed28d720 100644
--- a/llvm/lib/Target/PowerPC/PPC.td
+++ b/llvm/lib/Target/PowerPC/PPC.td
@@ -329,6 +329,15 @@ def FeatureAIXLocalExecTLS :
                    "Produce a TOC-free local-exec TLS sequence for this function "
                    "for 64-bit AIX">;
 
+// Specifies that local-dynamic TLS accesses in any function with this target
+// attribute should use the optimized sequence (where the offset is an immediate
+// off the module-handle for which the linker might add fix-up code for if the
+// immediate is too large).
+def FeatureAIXLocalDynamicTLS :
+  SubtargetFeature<"aix-small-local-dynamic-tls", "HasAIXSmallLocalDynamicTLS",
+                   "true", "Produce a faster local-dynamic TLS sequence for this "
+                   "function for 64-bit AIX">;
+
 def FeaturePredictableSelectIsExpensive :
   SubtargetFeature<"predictable-select-expensive",
                    "PredictableSelectIsExpensive",
diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
index 16942c6893a1..1c57b92057ff 100644
--- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
+++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
@@ -803,7 +803,8 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
   MCInst TmpInst;
   const bool IsPPC64 = Subtarget->isPPC64();
   const bool IsAIX = Subtarget->isAIXABI();
-  const bool HasAIXSmallLocalExecTLS = Subtarget->hasAIXSmallLocalExecTLS();
+  const bool HasAIXSmallLocalTLS = Subtarget->hasAIXSmallLocalExecTLS() ||
+                                   Subtarget->hasAIXSmallLocalDynamicTLS();
   const Module *M = MF->getFunction().getParent();
   PICLevel::Level PL = M->getPICLevel();
 
@@ -1612,11 +1613,11 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
   case PPC::LFD:
   case PPC::STFD:
   case PPC::ADDI8: {
-    // A faster non-TOC-based local-exec sequence is represented by `addi`
-    // or a load/store instruction (that directly loads or stores off of the
-    // thread pointer) with an immediate operand having the MO_TPREL_FLAG.
+    // A faster non-TOC-based local-[exec|dynamic] sequence is represented by
+    // `addi` or a load/store instruction (that directly loads or stores off of
+    // the thread pointer) with an immediate operand having the MO_TPREL_FLAG.
     // Such instructions do not otherwise arise.
-    if (!HasAIXSmallLocalExecTLS)
+    if (!HasAIXSmallLocalTLS)
       break;
     bool IsMIADDI8 = MI->getOpcode() == PPC::ADDI8;
     unsigned OpNum = IsMIADDI8 ? 2 : 1;
@@ -1624,7 +1625,7 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
     unsigned Flag = MO.getTargetFlags();
     if (Flag == PPCII::MO_TPREL_FLAG ||
         Flag == PPCII::MO_GOT_TPREL_PCREL_FLAG ||
-        Flag == PPCII::MO_TPREL_PCREL_FLAG) {
+        Flag == PPCII::MO_TPREL_PCREL_FLAG || Flag == PPCII::MO_TLSLD_FLAG) {
       LowerPPCMachineInstrToMCInst(MI, TmpInst, *this);
 
       const MCExpr *Expr = getAdjustedLocalExecExpr(MO, MO.getOffset());
@@ -1672,7 +1673,12 @@ const MCExpr *PPCAsmPrinter::getAdjustedLocalExecExpr(const MachineOperand &MO,
 
   assert(MO.isGlobal() && "Only expecting a global MachineOperand here!");
   const GlobalValue *GValue = MO.getGlobal();
-  assert(TM.getTLSModel(GValue) == TLSModel::LocalExec &&
+  // TODO: Handle the aix-small-local-dynamic-tls non-zero offset case.
+  TLSModel::Model Model = TM.getTLSModel(GValue);
+  if (Model == TLSModel::LocalDynamic) {
+    return nullptr;
+  }
+  assert(Model == TLSModel::LocalExec &&
          "Only local-exec accesses are handled!");
 
   bool IsGlobalADeclaration = GValue->isDeclarationForLinker();
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 52d5b7136705..d27932f2915f 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -153,10 +153,10 @@ static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl);
 
 static const char AIXSSPCanaryWordName[] = "__ssp_canary_word";
 
-// A faster local-exec TLS access sequence (enabled with the
-// -maix-small-local-exec-tls option) can be produced for TLS variables;
-// consistent with the IBM XL compiler, we apply a max size of slightly under
-// 32KB.
+// A faster local-[exec|dynamic] TLS access sequence (enabled with the
+// -maix-small-local-[exec|dynamic]-tls option) can be produced for TLS
+// variables; consistent with the IBM XL compiler, we apply a max size of
+// slightly under 32KB.
 constexpr uint64_t AIXSmallTlsPolicySizeLimit = 32751;
 
 // FIXME: Remove this once the bug has been fixed!
@@ -3434,6 +3434,14 @@ SDValue PPCTargetLowering::LowerGlobalTLSAddressAIX(SDValue Op,
   }
 
   if (Model == TLSModel::LocalDynamic) {
+    bool HasAIXSmallLocalDynamicTLS = Subtarget.hasAIXSmallLocalDynamicTLS();
+
+    // We do not implement the 32-bit version of the faster access sequence
+    // for local-dynamic that is controlled by -maix-small-local-dynamic-tls.
+    if (!Is64Bit && HasAIXSmallLocalDynamicTLS)
+      report_fatal_error("The small-local-dynamic TLS access sequence is "
+                         "currently only supported on AIX (64-bit mode).");
+
     // For local-dynamic on AIX, we need to generate one TOC entry for each
     // variable offset, and a single module-handle TOC entry for the entire
     // file.
@@ -3454,6 +3462,22 @@ SDValue PPCTargetLowering::LowerGlobalTLSAddressAIX(SDValue Op,
     SDValue ModuleHandle =
         DAG.getNode(PPCISD::TLSLD_AIX, dl, PtrVT, ModuleHandleTOC);
 
+    // With the -maix-small-local-dynamic-tls option, produce a faster access
+    // sequence for local-dynamic TLS variables where the offset from the
+    // module-handle is encoded as an immediate operand.
+    //
+    // We only utilize the faster local-dynamic access sequence when the TLS
+    // variable has a size within the policy limit. We treat types that are
+    // not sized or are empty as being over the policy size limit.
+    if (HasAIXSmallLocalDynamicTLS) {
+      Type *GVType = GV->getValueType();
+      if (GVType->isSized() && !GVType->isEmptyTy() &&
+          GV->getParent()->getDataLayout().getTypeAllocSize(GVType) <=
+              AIXSmallTlsPolicySizeLimit)
+        return DAG.getNode(PPCISD::Lo, dl, PtrVT, VariableOffsetTGA,
+                           ModuleHandle);
+    }
+
     return DAG.getNode(ISD::ADD, dl, PtrVT, ModuleHandle, VariableOffset);
   }
 
diff --git a/llvm/lib/Target/PowerPC/PPCMCInstLower.cpp b/llvm/lib/Target/PowerPC/PPCMCInstLower.cpp
index 9a3ca5a78293..c05bb37e58bf 100644
--- a/llvm/lib/Target/PowerPC/PPCMCInstLower.cpp
+++ b/llvm/lib/Target/PowerPC/PPCMCInstLower.cpp
@@ -96,15 +96,18 @@ static MCOperand GetSymbolRef(const MachineOperand &MO, const MCSymbol *Symbol,
     RefKind = MCSymbolRefExpr::VK_PPC_GOT_TLSLD_PCREL;
   else if (MO.getTargetFlags() == PPCII::MO_GOT_TPREL_PCREL_FLAG)
     RefKind = MCSymbolRefExpr::VK_PPC_GOT_TPREL_PCREL;
-  else if (MO.getTargetFlags() == PPCII::MO_TPREL_FLAG) {
+  else if (MO.getTargetFlags() == PPCII::MO_TPREL_FLAG ||
+           MO.getTargetFlags() == PPCII::MO_TLSLD_FLAG) {
     assert(MO.isGlobal() && "Only expecting a global MachineOperand here!");
     TLSModel::Model Model = TM.getTLSModel(MO.getGlobal());
-    // For the local-exec TLS model, we may generate the offset from the TLS
-    // base as an immediate operand (instead of using a TOC entry).
-    // Set the relocation type in case the result is used for purposes other
-    // than a TOC reference. In TOC reference cases, this result is discarded.
+    // For the local-[exec|dynamic] TLS model, we may generate the offset from
+    // the TLS base as an immediate operand (instead of using a TOC entry). Set
+    // the relocation type in case the result is used for purposes other than a
+    // TOC reference. In TOC reference cases, this result is discarded.
     if (Model == TLSModel::LocalExec)
       RefKind = MCSymbolRefExpr::VK_PPC_AIX_TLSLE;
+    else if (Model == TLSModel::LocalDynamic)
+      RefKind = MCSymbolRefExpr::VK_PPC_AIX_TLSLD;
   }
 
   const MachineInstr *MI = MO.getParent();
diff --git a/llvm/lib/Target/PowerPC/PPCSubtarget.cpp b/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
index 653d9bda9919..d1722555f1fc 100644
--- a/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
+++ b/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
@@ -124,22 +124,22 @@ void PPCSubtarget::initSubtargetFeatures(StringRef CPU, StringRef TuneCPU,
   // Determine endianness.
   IsLittleEndian = TM.isLittleEndian();
 
-  if (HasAIXSmallLocalExecTLS) {
+  if (HasAIXSmallLocalExecTLS || HasAIXSmallLocalDynamicTLS) {
     if (!TargetTriple.isOSAIX() || !IsPPC64)
-      report_fatal_error(
-          "The aix-small-local-exec-tls attribute is only supported on AIX in "
-          "64-bit mode.\n",
-          false);
-    // The aix-small-local-exec-tls attribute should only be used with
+      report_fatal_error("The aix-small-local-[exec|dynamic]-tls attribute is "
+                         "only supported on AIX in "
+                         "64-bit mode.\n",
+                         false);
+    // The aix-small-local-[exec|dynamic]-tls attribute should only be used with
     // -data-sections, as having data sections turned off with this option
-    // is not ideal for performance. Moreover, the small-local-exec-tls region
-    // is a limited resource, and should not be used for variables that may
-    // be replaced.
+    // is not ideal for performance. Moreover, the
+    // small-local-[exec|dynamic]-tls region is a limited resource, and should
+    // not be used for variables that may be replaced.
     if (!TM.getDataSections())
-      report_fatal_error(
-          "The aix-small-local-exec-tls attribute can only be specified with "
-          "-data-sections.\n",
-          false);
+      report_fatal_error("The aix-small-local-[exec|dynamic]-tls attribute can "
+                         "only be specified with "
+                         "-data-sections.\n",
+                         false);
   }
 }
 
diff --git a/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-largeaccess.ll b/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-largeaccess.ll
index eb16bae67150..7db1048c258c 100644
--- a/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-largeaccess.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-largeaccess.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
 ; RUN: llc  -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \
-; RUN:      -mtriple powerpc64-ibm-aix-xcoff < %s \
+; RUN:      -mattr=+aix-small-local-dynamic-tls -mtriple powerpc64-ibm-aix-xcoff < %s \
 ; RUN:      | FileCheck %s --check-prefix=SMALL-LOCAL-DYNAMIC-SMALLCM64
 ; RUN: llc  -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \
-; RUN:      -mtriple powerpc64-ibm-aix-xcoff --code-model=large \
+; RUN:      -mattr=+aix-small-local-dynamic-tls -mtriple powerpc64-ibm-aix-xcoff --code-model=large \
 ; RUN:      < %s | FileCheck %s \
 ; RUN:      --check-prefix=SMALL-LOCAL-DYNAMIC-LARGECM64
 
@@ -39,27 +39,23 @@ define signext i32 @test1() {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r6, L..C1(r2) # target-flags(ppc-tlsld) @ElementIntTLS2
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r7, L..C2(r2) # target-flags(ppc-tlsld) @ElementIntTLS3
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r8, L..C3(r2) # target-flags(ppc-tlsld) @ElementIntTLS4
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r9, L..C4(r2) # target-flags(ppc-tlsld) @ElementIntTLS5
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r6, 4
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r5, L..C5(r2) # target-flags(ppc-tlsld) @ElementIntTLSv1
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r4, 1
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r6, r3, r6
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r7, r3, r7
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r8, r3, r8
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r9, r3, r9
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stwux r4, r3, r5
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r4, 4
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r4, 24(r3)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r3, 2
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r3, 320(r6)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r3, 3
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r3, 324(r7)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r3, 88
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r4, 328(r8)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r3, 332(r9)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r5, 1
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r4, ElementIntTLSv1[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r5, ElementIntTLSv1[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r5, ElementIntTLS2[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r6, 24(r4)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r4, 2
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r4, 320(r5)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r4, ElementIntTLS3[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r5, 3
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r5, 324(r4)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r4, ElementIntTLS4[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r3, ElementIntTLS5[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r6, 328(r4)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r4, 88
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r4, 332(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r3, 102
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
@@ -71,34 +67,25 @@ define signext i32 @test1() {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C1@u(r2)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r7, L..C2@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r6, 4
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r8, L..C3@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r9, L..C4@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r7, L..C2@l(r7)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r8, L..C3@l(r8)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r9, L..C4@l(r9)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r5, L..C1@l(r6)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C5@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 1
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r6, L..C5@l(r6)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r7, r3, r7
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r8, r3, r8
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r9, r3, r9
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r6, r3, r6
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stwux r4, r3, r5
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 4
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r4, 24(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r3, 2
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r3, 320(r6)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r3, 3
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r3, 324(r7)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r3, 88
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r4, 328(r8)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r3, 332(r9)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r5, 1
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r4, ElementIntTLSv1[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r5, ElementIntTLSv1[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r5, ElementIntTLS2[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r6, 24(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 2
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r4, 320(r5)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r4, ElementIntTLS3[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r5, 3
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r5, 324(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r4, ElementIntTLS4[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r3, ElementIntTLS5[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r6, 328(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 88
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r4, 332(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r3, 102
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
@@ -144,31 +131,26 @@ define i64 @test2() {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C6(r2) # target-flags(ppc-tlsld) @ElementLongTLS6
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mr r6, r3
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r3, 212
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r4, r6, r4
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r3, 424(r4)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C7(r2) # target-flags(ppc-tlsld) @ElementLongTLS2
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r3, ElementLongTLS6[UL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r4, 212
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r4, 424(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r3, ElementLongTLS2[TL]@ld(r6)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r4, 203
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r3, r6, r3
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r4, 1200(r3)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C8(r2) # target-flags(ppc-tlsgdm) @MyTLSGDVar
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C9(r2) # target-flags(ppc-tlsgd) @MyTLSGDVar
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C1(r2) # target-flags(ppc-tlsgdm) @MyTLSGDVar
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C2(r2) # target-flags(ppc-tlsgd) @MyTLSGDVar
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_addr[PR]
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r4, 44
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r4, 440(r3)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C10(r2) # target-flags(ppc-tlsld) @ElementLongTLS3
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r3, ElementLongTLS3[TL]@ld(r6)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r4, 6
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r3, r6, r3
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r4, 2000(r3)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C11(r2) # target-flags(ppc-tlsld) @ElementLongTLS4
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r3, 100
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r4, r6, r4
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r3, 6800(r4)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C12(r2) # target-flags(ppc-tlsld) @ElementLongTLS5
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r3, ElementLongTLS4[TL]@ld(r6)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r4, 100
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r4, 6800(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r3, ElementLongTLS5[TL]@ld(r6)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r4, 882
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r3, r6, r3
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r4, 8400(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r3, 1191
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
@@ -182,40 +164,30 @@ define i64 @test2() {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r7, L..C6@u(r2)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 212
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mr r6, r3
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C6@l(r7)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r3, r6, r3
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r3, ElementLongTLS6[UL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r4, 424(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C7@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r3, ElementLongTLS2[TL]@ld(r6)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 203
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C7@l(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r3, r6, r3
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r4, 1200(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C8@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r4, L..C9@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C8@l(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C9@l(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C1@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r4, L..C2@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C1@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C2@l(r4)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_addr[PR]
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 44
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r4, 440(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C10@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r3, ElementLongTLS3[TL]@ld(r6)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 6
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C10@l(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r3, r6, r3
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r4, 2000(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C11@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r3, ElementLongTLS4[TL]@ld(r6)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 100
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C11@l(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r3, r6, r3
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r4, 6800(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C12@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r3, ElementLongTLS5[TL]@ld(r6)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 882
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C12@l(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r3, r6, r3
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r4, 8400(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r3, 1191
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
@@ -257,27 +229,23 @@ define signext i32 @test3() {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r8, L..C3(r2) # target-flags(ppc-tlsld) @ElementIntTLSv2
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r6, L..C1(r2) # target-flags(ppc-tlsld) @ElementIntTLS2
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r7, L..C2(r2) # target-flags(ppc-tlsld) @ElementIntTLS3
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r8, L..C3(r2) # target-flags(ppc-tlsld) @ElementIntTLS4
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r9, L..C4(r2) # target-flags(ppc-tlsld) @ElementIntTLS5
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r5, L..C13(r2) # target-flags(ppc-tlsld) @ElementIntTLSv2
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r4, 1
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r6, r3, r6
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r7, r3, r7
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r8, r3, r8
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r9, r3, r9
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stwux r4, r3, r5
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r5, ElementIntTLS2[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r6, ElementIntTLS3[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r7, ElementIntTLS4[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r9, ElementIntTLS5[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stwux r4, r3, r8
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r4, 4
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r4, 24(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r3, 2
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r3, 320(r6)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r3, 320(r5)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r3, 3
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r3, 324(r7)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r3, 324(r6)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r3, 88
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r4, 328(r8)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r4, 328(r7)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stw r3, 332(r9)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    li r3, 102
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
@@ -290,29 +258,21 @@ define signext i32 @test3() {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C13@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C3@u(r2)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r7, L..C2@u(r2)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r8, L..C3@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r9, L..C4@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r7, L..C2@l(r7)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r8, L..C3@l(r8)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r9, L..C4@l(r9)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r6, L..C3@l(r6)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r5, L..C13@l(r6)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C5@u(r2)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 1
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r6, L..C5@l(r6)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r7, r3, r7
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r8, r3, r8
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r9, r3, r9
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r6, r3, r6
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stwux r4, r3, r5
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r5, ElementIntTLS2[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r7, ElementIntTLS3[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r8, ElementIntTLS4[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r9, ElementIntTLS5[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stwux r4, r3, r6
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r4, 4
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r4, 24(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r3, 2
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r3, 320(r6)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r3, 320(r5)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r3, 3
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stw r3, 324(r7)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    li r3, 88
diff --git a/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-types.ll b/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-types.ll
index d996d86a23d8..161a58a90296 100644
--- a/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-types.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-small-local-dynamic-tls-types.ll
@@ -1,18 +1,30 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
 ; RUN: llc  -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \
-; RUN:      -mtriple powerpc64-ibm-aix-xcoff < %s \
+; RUN:      -mattr=+aix-small-local-dynamic-tls -mtriple powerpc64-ibm-aix-xcoff < %s \
 ; RUN:      | FileCheck %s --check-prefix=SMALL-LOCAL-DYNAMIC-SMALLCM64
 ; RUN: llc  -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \
+; RUN:      -mattr=+aix-small-local-dynamic-tls -mtriple powerpc64-ibm-aix-xcoff --code-model=large \
+; RUN:      < %s | FileCheck %s --check-prefix=SMALL-LOCAL-DYNAMIC-LARGECM64
+; RUN: llc  -O0 -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \
+; RUN:      -mattr=+aix-small-local-dynamic-tls -mtriple powerpc64-ibm-aix-xcoff < %s \
+; RUN:      | FileCheck %s --check-prefix=SMALL-LOCAL-DYNAMIC-SMALLCM64-O0
+; RUN: llc  -O0 -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \
+; RUN:      -mattr=+aix-small-local-dynamic-tls -mtriple powerpc64-ibm-aix-xcoff --code-model=large \
+; RUN:      < %s | FileCheck %s --check-prefix=SMALL-LOCAL-DYNAMIC-LARGECM64-O0
+; RUN: llc  -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \
+; RUN:      -mtriple powerpc64-ibm-aix-xcoff < %s \
+; RUN:      | FileCheck %s --check-prefix=DEFAULT-LOCAL-DYNAMIC-SMALLCM64
+; RUN: llc  -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \
 ; RUN:      -mtriple powerpc64-ibm-aix-xcoff --code-model=large \
 ; RUN:      < %s | FileCheck %s \
-; RUN:      --check-prefix=SMALL-LOCAL-DYNAMIC-LARGECM64
+; RUN:      --check-prefix=DEFAULT-LOCAL-DYNAMIC-LARGECM64
 ; RUN: llc  -O0 -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \
 ; RUN:      -mtriple powerpc64-ibm-aix-xcoff < %s \
-; RUN:      | FileCheck %s --check-prefix=SMALL-LOCAL-DYNAMIC-SMALLCM64-O0
+; RUN:      | FileCheck %s --check-prefix=DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0
 ; RUN: llc  -O0 -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \
 ; RUN:      -mtriple powerpc64-ibm-aix-xcoff --code-model=large \
 ; RUN:      < %s | FileCheck %s \
-; RUN:      --check-prefix=SMALL-LOCAL-DYNAMIC-LARGECM64-O0
+; RUN:      --check-prefix=DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0
 
 declare nonnull ptr @llvm.threadlocal.address.p0(ptr nonnull) #1
 @tlv_int_init = local_unnamed_addr global i32 87, align 4
@@ -27,7 +39,8 @@ declare nonnull ptr @llvm.threadlocal.address.p0(ptr nonnull) #1
 @internal_tlv_double = internal thread_local(localdynamic) global double 1.000000e+00, align 8
 
 %struct.anon = type { i32 }
-@ThreadLocalStruct = thread_local(localdynamic) global %struct.anon zeroinitializer, align 1
+@ThreadLocalStructAlign1 = thread_local(localdynamic) global %struct.anon zeroinitializer, align 1
+@ThreadLocalStructAlign4 = thread_local(localdynamic) global %struct.anon zeroinitializer, align 4
 @a = thread_local(localdynamic) global [87 x i32] zeroinitializer, align 4
 
 define nonnull ptr @AddrTest1() local_unnamed_addr {
@@ -38,8 +51,7 @@ define nonnull ptr @AddrTest1() local_unnamed_addr {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C1(r2) # target-flags(ppc-tlsld) @a
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r3, a[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r3, r3, 12
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
@@ -51,12 +63,10 @@ define nonnull ptr @AddrTest1() local_unnamed_addr {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C1@u(r2)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C1@l(r6)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r3, a[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r3, r3, 12
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
@@ -70,8 +80,7 @@ define nonnull ptr @AddrTest1() local_unnamed_addr {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C1(r2) # target-flags(ppc-tlsld) @a
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    la r3, a[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r3, r3, 12
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
@@ -81,27 +90,212 @@ define nonnull ptr @AddrTest1() local_unnamed_addr {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: AddrTest1:
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C0@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C0@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 56(r1) # 8-byte Folded Reload
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C0@l(r4)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    la r3, a[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r3, r3, 12
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-LABEL: AddrTest1:
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C1(r2) # target-flags(ppc-tlsld) @a
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r3, r3, 12
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-LABEL: AddrTest1:
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C1@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C1@l(r6)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r3, r3, 12
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: AddrTest1:
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C1(r2) # target-flags(ppc-tlsld) @a
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r3, r3, 12
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: AddrTest1:
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C0@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 56(r1) # 8-byte Folded Reload
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C0@l(r4)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r3, r3, 12
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
 entry:
   %tlv_addr = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @a)
   %arrayidx = getelementptr inbounds [87 x i32], ptr %tlv_addr, i64 0, i64 3
   ret ptr %arrayidx
 }
 
+; Demonstrates when offset is zero, addi for the offset is optimized out.
+define nonnull ptr @AddrTest2() local_unnamed_addr {
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: AddrTest2:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r3, a[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: AddrTest2:
+; SMALL-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r3, a[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: AddrTest2:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    la r3, a[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: AddrTest2:
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C0@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    la r3, a[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-LABEL: AddrTest2:
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C1(r2) # target-flags(ppc-tlsld) @a
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-LABEL: AddrTest2:
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C1@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C1@l(r6)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: AddrTest2:
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C1(r2) # target-flags(ppc-tlsld) @a
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: AddrTest2:
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C0@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 56(r1) # 8-byte Folded Reload
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C0@l(r4)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
+entry:
+  %tlv_addr = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @a)
+  %arrayidx = getelementptr inbounds [87 x i32], ptr %tlv_addr, i64 0, i64 0
+  ret ptr %arrayidx
+}
+
 define signext i32 @testUnaligned() {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testUnaligned:
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
@@ -110,8 +304,8 @@ define signext i32 @testUnaligned() {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C2(r2) # target-flags(ppc-tlsld) @ThreadLocalStruct
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lwax r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    la r3, ThreadLocalStructAlign1[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lwa r3, 0(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
@@ -122,12 +316,11 @@ define signext i32 @testUnaligned() {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C2@u(r2)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C2@l(r6)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    lwax r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    la r3, ThreadLocalStructAlign1[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    lwa r3, 0(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
@@ -140,8 +333,7 @@ define signext i32 @testUnaligned() {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C2(r2) # target-flags(ppc-tlsld) @ThreadLocalStruct
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    la r3, ThreadLocalStructAlign1[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lwa r3, 0(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
@@ -151,26 +343,213 @@ define signext i32 @testUnaligned() {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testUnaligned:
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C2@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C0@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 56(r1) # 8-byte Folded Reload
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C2@l(r4)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    la r3, ThreadLocalStructAlign1[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lwa r3, 0(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-LABEL: testUnaligned:
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C2(r2) # target-flags(ppc-tlsld) @ThreadLocalStructAlign1
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lwax r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-LABEL: testUnaligned:
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C2@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C2@l(r6)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    lwax r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testUnaligned:
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C2(r2) # target-flags(ppc-tlsld) @ThreadLocalStructAlign1
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lwa r3, 0(r3)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testUnaligned:
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C2@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 56(r1) # 8-byte Folded Reload
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C2@l(r4)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lwa r3, 0(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
 entry:
-  %tlv_addr = call align 1 ptr @llvm.threadlocal.address.p0(ptr align 1 @ThreadLocalStruct)
-  %x = getelementptr inbounds %struct.anon, ptr %tlv_addr, i32 0, i32 0
-  %value = load i32, ptr %x, align 1
-  ret i32 %value
+  %tlv_addr1 = call align 1 ptr @llvm.threadlocal.address.p0(ptr align 1 @ThreadLocalStructAlign1)
+  %x = getelementptr inbounds %struct.anon, ptr %tlv_addr1, i32 0, i32 0
+  %value1 = load i32, ptr %x, align 1
+  ret i32 %value1
+}
+
+define signext i32 @testAligned() {
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-LABEL: testAligned:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lwa r3, ThreadLocalStructAlign4[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-LABEL: testAligned:
+; SMALL-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    lwa r3, ThreadLocalStructAlign4[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testAligned:
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    la r3, ThreadLocalStructAlign4[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lwa r3, 0(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    blr
+;
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testAligned:
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C0@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    la r3, ThreadLocalStructAlign4[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lwa r3, 0(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 48
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-LABEL: testAligned:
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C3(r2) # target-flags(ppc-tlsld) @ThreadLocalStructAlign4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lwax r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-LABEL: testAligned:
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C3@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C3@l(r6)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    lwax r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testAligned:
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C3(r2) # target-flags(ppc-tlsld) @ThreadLocalStructAlign4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lwa r3, 0(r3)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testAligned:
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C3@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 56(r1) # 8-byte Folded Reload
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C3@l(r4)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lwa r3, 0(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
+entry:
+  %tlv_addr1 = call align 1 ptr @llvm.threadlocal.address.p0(ptr align 4 @ThreadLocalStructAlign4)
+  %x = getelementptr inbounds %struct.anon, ptr %tlv_addr1, i32 0, i32 0
+  %value1 = load i32, ptr %x, align 1
+  ret i32 %value1
 }
 
 define void @testChar(i8 noundef signext %x) {
@@ -182,8 +561,7 @@ define void @testChar(i8 noundef signext %x) {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C3(r2) # target-flags(ppc-tlsld) @tlv_char
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stbx r6, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stb r6, tlv_char[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
@@ -195,12 +573,10 @@ define void @testChar(i8 noundef signext %x) {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mr r6, r3
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r7, L..C3@u(r2)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C3@l(r7)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stbx r6, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stb r6, tlv_char[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
@@ -217,8 +593,7 @@ define void @testChar(i8 noundef signext %x) {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mr r4, r3
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lwz r3, 60(r1) # 4-byte Folded Reload
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r5, L..C3(r2) # target-flags(ppc-tlsld) @tlv_char
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r4, r4, r5
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    la r4, tlv_char[TL]@ld(r4)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stb r3, 0(r4)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 64
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
@@ -232,21 +607,91 @@ define void @testChar(i8 noundef signext %x) {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    # kill: def $r3 killed $r3 killed $x3
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stw r3, 60(r1) # 4-byte Folded Spill
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C3@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 48(r1) # 8-byte Folded Spill
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C0@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r5, 48(r1) # 8-byte Folded Reload
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mr r4, r3
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lwz r3, 60(r1) # 4-byte Folded Reload
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r5, L..C3@l(r5)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r4, r4, r5
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    la r4, tlv_char[TL]@ld(r4)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stb r3, 0(r4)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-LABEL: testChar:
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mr r6, r3
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C4(r2) # target-flags(ppc-tlsld) @tlv_char
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stbx r6, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-LABEL: testChar:
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    mr r6, r3
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r7, L..C4@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C4@l(r7)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    stbx r6, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testChar:
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stdu r1, -64(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 80(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    # kill: def $r3 killed $r3 killed $x3
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stw r3, 60(r1) # 4-byte Folded Spill
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mr r4, r3
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lwz r3, 60(r1) # 4-byte Folded Reload
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r5, L..C4(r2) # target-flags(ppc-tlsld) @tlv_char
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r4, r4, r5
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stb r3, 0(r4)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 64
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testChar:
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    # kill: def $r3 killed $r3 killed $x3
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stw r3, 60(r1) # 4-byte Folded Spill
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C4@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 48(r1) # 8-byte Folded Spill
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r5, 48(r1) # 8-byte Folded Reload
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mr r4, r3
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lwz r3, 60(r1) # 4-byte Folded Reload
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r5, L..C4@l(r5)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r4, r4, r5
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stb r3, 0(r4)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
 entry:
   %tlv_addr = tail call align 1 ptr @llvm.threadlocal.address.p0(ptr align 1 @tlv_char)
   store i8 %x, ptr %tlv_addr, align 1
@@ -262,8 +707,7 @@ define void @testShort(i16 noundef signext %x) {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C4(r2) # target-flags(ppc-tlsld) @tlv_short
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    sthx r6, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    sth r6, tlv_short[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
@@ -275,12 +719,10 @@ define void @testShort(i16 noundef signext %x) {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mr r6, r3
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r7, L..C4@u(r2)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C4@l(r7)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    sthx r6, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    sth r6, tlv_short[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
@@ -297,8 +739,7 @@ define void @testShort(i16 noundef signext %x) {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mr r4, r3
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lwz r3, 60(r1) # 4-byte Folded Reload
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r5, L..C4(r2) # target-flags(ppc-tlsld) @tlv_short
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r4, r4, r5
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    la r4, tlv_short[TL]@ld(r4)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    sth r3, 0(r4)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 64
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
@@ -312,21 +753,91 @@ define void @testShort(i16 noundef signext %x) {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    # kill: def $r3 killed $r3 killed $x3
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stw r3, 60(r1) # 4-byte Folded Spill
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C4@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 48(r1) # 8-byte Folded Spill
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C0@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r5, 48(r1) # 8-byte Folded Reload
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mr r4, r3
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lwz r3, 60(r1) # 4-byte Folded Reload
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r5, L..C4@l(r5)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r4, r4, r5
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    la r4, tlv_short[TL]@ld(r4)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    sth r3, 0(r4)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-LABEL: testShort:
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mr r6, r3
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C5(r2) # target-flags(ppc-tlsld) @tlv_short
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    sthx r6, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-LABEL: testShort:
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    mr r6, r3
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r7, L..C5@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C5@l(r7)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    sthx r6, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testShort:
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stdu r1, -64(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 80(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    # kill: def $r3 killed $r3 killed $x3
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stw r3, 60(r1) # 4-byte Folded Spill
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mr r4, r3
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lwz r3, 60(r1) # 4-byte Folded Reload
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r5, L..C5(r2) # target-flags(ppc-tlsld) @tlv_short
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r4, r4, r5
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    sth r3, 0(r4)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 64
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testShort:
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    # kill: def $r3 killed $r3 killed $x3
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stw r3, 60(r1) # 4-byte Folded Spill
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C5@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 48(r1) # 8-byte Folded Spill
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r5, 48(r1) # 8-byte Folded Reload
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mr r4, r3
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lwz r3, 60(r1) # 4-byte Folded Reload
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r5, L..C5@l(r5)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r4, r4, r5
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    sth r3, 0(r4)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
 entry:
   %tlv_addr = tail call align 2 ptr @llvm.threadlocal.address.p0(ptr align 2 @tlv_short)
   store i16 %x, ptr %tlv_addr, align 2
@@ -341,8 +852,7 @@ define signext i32 @testInt1() {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C5(r2) # target-flags(ppc-tlsld) @tlv_int
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lwax r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lwa r3, tlv_int[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
@@ -353,12 +863,10 @@ define signext i32 @testInt1() {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C5@u(r2)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C5@l(r6)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    lwax r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    lwa r3, tlv_int[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
@@ -371,8 +879,7 @@ define signext i32 @testInt1() {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C5(r2) # target-flags(ppc-tlsld) @tlv_int
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    la r3, tlv_int[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lwa r3, 0(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
@@ -382,21 +889,81 @@ define signext i32 @testInt1() {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testInt1:
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C5@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C0@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 56(r1) # 8-byte Folded Reload
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C5@l(r4)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    la r3, tlv_int[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lwa r3, 0(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-LABEL: testInt1:
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C6(r2) # target-flags(ppc-tlsld) @tlv_int
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lwax r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-LABEL: testInt1:
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C6@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C6@l(r6)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    lwax r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testInt1:
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C6(r2) # target-flags(ppc-tlsld) @tlv_int
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lwa r3, 0(r3)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testInt1:
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C6@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 56(r1) # 8-byte Folded Reload
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C6@l(r4)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lwa r3, 0(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
 entry:
   %tlv_addr = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @tlv_int)
   %value = load i32, ptr %tlv_addr, align 4
@@ -411,9 +978,8 @@ define signext i32 @testInt2() {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C6(r2) # target-flags(ppc-tlsld) @internal_tlv_int
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lwzx r3, r3, r4
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C7(r2) # @tlv_int_init
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C1(r2) # @tlv_int_init
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lwz r3, internal_tlv_int[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lwz r4, 0(r4)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r3, r4, r3
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    extsw r3, r3
@@ -427,14 +993,12 @@ define signext i32 @testInt2() {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C6@u(r2)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C6@l(r6)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    lwzx r3, r3, r4
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r4, L..C7@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C7@l(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r4, L..C1@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C1@l(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    lwz r3, internal_tlv_int[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    lwz r4, 0(r4)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r3, r4, r3
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    extsw r3, r3
@@ -450,10 +1014,9 @@ define signext i32 @testInt2() {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C6(r2) # target-flags(ppc-tlsld) @internal_tlv_int
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    la r3, internal_tlv_int[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lwz r4, 0(r3)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C7(r2) # @tlv_int_init
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C1(r2) # @tlv_int_init
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lwz r3, 0(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r3, r3, r4
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    extsw r3, r3
@@ -465,26 +1028,104 @@ define signext i32 @testInt2() {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testInt2:
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C6@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C0@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 56(r1) # 8-byte Folded Reload
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C6@l(r4)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    la r3, internal_tlv_int[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lwz r4, 0(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C7@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C7@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lwz r3, 0(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r3, r3, r4
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    extsw r3, r3
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-LABEL: testInt2:
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C7(r2) # target-flags(ppc-tlsld) @internal_tlv_int
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lwzx r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C8(r2) # @tlv_int_init
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lwz r4, 0(r4)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    add r3, r4, r3
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    extsw r3, r3
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-LABEL: testInt2:
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C7@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C7@l(r6)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    lwzx r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r4, L..C8@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C8@l(r4)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    lwz r4, 0(r4)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    add r3, r4, r3
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    extsw r3, r3
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testInt2:
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C7(r2) # target-flags(ppc-tlsld) @internal_tlv_int
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lwz r4, 0(r3)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C8(r2) # @tlv_int_init
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lwz r3, 0(r3)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    extsw r3, r3
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testInt2:
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C7@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 56(r1) # 8-byte Folded Reload
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C7@l(r4)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lwz r4, 0(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C8@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C8@l(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lwz r3, 0(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    extsw r3, r3
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
 entry:
   %tlv_addr = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @internal_tlv_int)
   %tlv_val = load i32, ptr %tlv_addr, align 4
@@ -501,8 +1142,7 @@ define signext i64 @testLong1() {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C8(r2) # target-flags(ppc-tlsld) @tlv_long
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ldx r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, tlv_long[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
@@ -513,12 +1153,10 @@ define signext i64 @testLong1() {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C8@u(r2)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C8@l(r6)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ldx r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, tlv_long[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
@@ -531,8 +1169,7 @@ define signext i64 @testLong1() {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C8(r2) # target-flags(ppc-tlsld) @tlv_long
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    la r3, tlv_long[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, 0(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
@@ -542,21 +1179,81 @@ define signext i64 @testLong1() {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testLong1:
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C8@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C0@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 56(r1) # 8-byte Folded Reload
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C8@l(r4)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    la r3, tlv_long[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, 0(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-LABEL: testLong1:
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C9(r2) # target-flags(ppc-tlsld) @tlv_long
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ldx r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-LABEL: testLong1:
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C9@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C9@l(r6)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ldx r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testLong1:
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C9(r2) # target-flags(ppc-tlsld) @tlv_long
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, 0(r3)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testLong1:
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C9@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 56(r1) # 8-byte Folded Reload
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C9@l(r4)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, 0(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
 entry:
   %tlv_addr = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @tlv_long)
   %value = load i64, ptr %tlv_addr, align 4
@@ -571,10 +1268,9 @@ define void @testLong2(i64 noundef signext %x) {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C9(r2) # target-flags(ppc-tlsld) @internal_tlv_long
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ldx r5, r3, r4
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r5, r5, 9
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdx r5, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, internal_tlv_long[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r4, r4, 9
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r4, internal_tlv_long[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
@@ -585,14 +1281,12 @@ define void @testLong2(i64 noundef signext %x) {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C9@u(r2)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C9@l(r6)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ldx r5, r3, r4
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r5, r5, 9
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdx r5, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, internal_tlv_long[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r4, r4, 9
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r4, internal_tlv_long[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
@@ -605,8 +1299,7 @@ define void @testLong2(i64 noundef signext %x) {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C9(r2) # target-flags(ppc-tlsld) @internal_tlv_long
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r4, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    la r4, internal_tlv_long[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, 0(r4)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r3, r3, 9
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r3, 0(r4)
@@ -618,23 +1311,91 @@ define void @testLong2(i64 noundef signext %x) {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testLong2:
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C9@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C0@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 56(r1) # 8-byte Folded Reload
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C9@l(r4)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r4, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    la r4, internal_tlv_long[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, 0(r4)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r3, r3, 9
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 0(r4)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-LABEL: testLong2:
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C10(r2) # target-flags(ppc-tlsld) @internal_tlv_long
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ldx r5, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r5, r5, 9
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdx r5, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-LABEL: testLong2:
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C10@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C10@l(r6)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ldx r5, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r5, r5, 9
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdx r5, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testLong2:
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C10(r2) # target-flags(ppc-tlsld) @internal_tlv_long
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r4, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, 0(r4)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r3, r3, 9
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r3, 0(r4)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testLong2:
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C10@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 56(r1) # 8-byte Folded Reload
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C10@l(r4)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r4, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, 0(r4)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r3, r3, 9
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 0(r4)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
 entry:
   %tlv_addr = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @internal_tlv_long)
   %value = load i64, ptr %tlv_addr, align 8
@@ -651,8 +1412,7 @@ define i32 @testLong3() {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C8(r2) # target-flags(ppc-tlsld) @tlv_long
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ldx r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, tlv_long[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
@@ -663,12 +1423,10 @@ define i32 @testLong3() {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C8@u(r2)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C8@l(r6)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ldx r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, tlv_long[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
@@ -681,8 +1439,7 @@ define i32 @testLong3() {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C8(r2) # target-flags(ppc-tlsld) @tlv_long
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    la r3, tlv_long[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, 0(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    # kill: def $r3 killed $r3 killed $x3
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    clrldi r3, r3, 32
@@ -694,23 +1451,87 @@ define i32 @testLong3() {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testLong3:
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C8@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C0@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 56(r1) # 8-byte Folded Reload
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C8@l(r4)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    la r3, tlv_long[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, 0(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    # kill: def $r3 killed $r3 killed $x3
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    clrldi r3, r3, 32
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-LABEL: testLong3:
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C9(r2) # target-flags(ppc-tlsld) @tlv_long
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ldx r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-LABEL: testLong3:
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C9@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C9@l(r6)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ldx r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testLong3:
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C9(r2) # target-flags(ppc-tlsld) @tlv_long
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, 0(r3)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    # kill: def $r3 killed $r3 killed $x3
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    clrldi r3, r3, 32
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testLong3:
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C9@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 56(r1) # 8-byte Folded Reload
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C9@l(r4)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, 0(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    # kill: def $r3 killed $r3 killed $x3
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    clrldi r3, r3, 32
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
 entry:
   %tlv_addr = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @tlv_long)
   %value = load i64, ptr %tlv_addr, align 8
@@ -723,18 +1544,17 @@ define void @testFloat1(float noundef %x) {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    vspltisw v2, 1
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    vspltisw v3, 8
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    xvcvsxwdp vs0, vs34
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C10(r2) # target-flags(ppc-tlsld) @tlv_float
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lfsx f1, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lfs f1, tlv_float[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    fadds f0, f1, f0
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    xvcvsxwdp vs1, vs35
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    fadds f0, f0, f1
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stfsx f0, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stfs f0, tlv_float[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
@@ -744,20 +1564,18 @@ define void @testFloat1(float noundef %x) {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    vspltisw v2, 1
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C10@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    vspltisw v3, 8
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    xvcvsxwdp vs0, vs34
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C10@l(r6)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    lfsx f1, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    lfs f1, tlv_float[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    fadds f0, f1, f0
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    xvcvsxwdp vs1, vs35
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    fadds f0, f0, f1
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stfsx f0, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stfs f0, tlv_float[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
@@ -770,13 +1588,12 @@ define void @testFloat1(float noundef %x) {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C10(r2) # target-flags(ppc-tlsld) @tlv_float
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    la r3, tlv_float[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lfs f0, 0(r3)
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C11(r2) # %const.1
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C2(r2) # %const.1
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lfs f1, 0(r4)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    fadds f0, f0, f1
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C12(r2) # %const.0
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C3(r2) # %const.0
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lfs f1, 0(r4)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    fadds f0, f0, f1
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stfs f0, 0(r3)
@@ -788,30 +1605,120 @@ define void @testFloat1(float noundef %x) {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testFloat1:
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C10@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C0@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 56(r1) # 8-byte Folded Reload
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C10@l(r4)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    la r3, tlv_float[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lfs f0, 0(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r4, L..C11@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C11@l(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r4, L..C2@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C2@l(r4)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lfs f1, 0(r4)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    fadds f0, f0, f1
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r4, L..C12@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C12@l(r4)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r4, L..C3@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C3@l(r4)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lfs f1, 0(r4)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    fadds f0, f0, f1
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stfs f0, 0(r3)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-LABEL: testFloat1:
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    vspltisw v2, 1
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    vspltisw v3, 8
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    xvcvsxwdp vs0, vs34
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C11(r2) # target-flags(ppc-tlsld) @tlv_float
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lfsx f1, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    fadds f0, f1, f0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    xvcvsxwdp vs1, vs35
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    fadds f0, f0, f1
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stfsx f0, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-LABEL: testFloat1:
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    vspltisw v2, 1
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C11@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    vspltisw v3, 8
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    xvcvsxwdp vs0, vs34
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C11@l(r6)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    lfsx f1, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    fadds f0, f1, f0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    xvcvsxwdp vs1, vs35
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    fadds f0, f0, f1
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    stfsx f0, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testFloat1:
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C11(r2) # target-flags(ppc-tlsld) @tlv_float
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lfs f0, 0(r3)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C12(r2) # %const.1
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lfs f1, 0(r4)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    fadds f0, f0, f1
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C13(r2) # %const.0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lfs f1, 0(r4)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    fadds f0, f0, f1
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stfs f0, 0(r3)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testFloat1:
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C11@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 56(r1) # 8-byte Folded Reload
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C11@l(r4)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lfs f0, 0(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r4, L..C12@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C12@l(r4)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lfs f1, 0(r4)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    fadds f0, f0, f1
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r4, L..C13@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C13@l(r4)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lfs f1, 0(r4)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    fadds f0, f0, f1
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stfs f0, 0(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
 entry:
   %tlv_addr = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @tlv_float)
   %value = load float, ptr %tlv_addr, align 4
@@ -829,8 +1736,7 @@ define i32 @testFloat2() {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 80(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C10(r2) # target-flags(ppc-tlsld) @tlv_float
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lfsx f0, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lfs f0, tlv_float[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r3, r1, 60
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    xscvdpsxws f0, f0
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stfiwx f0, 0, r3
@@ -845,12 +1751,10 @@ define i32 @testFloat2() {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -64(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C10@u(r2)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 80(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C10@l(r6)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    lfsx f0, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    lfs f0, tlv_float[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r3, r1, 60
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    xscvdpsxws f0, f0
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stfiwx f0, 0, r3
@@ -867,8 +1771,7 @@ define i32 @testFloat2() {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 80(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C10(r2) # target-flags(ppc-tlsld) @tlv_float
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    la r3, tlv_float[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lfs f0, 0(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    fctiwz f0, f0
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stfd f0, 56(r1)
@@ -884,14 +1787,10 @@ define i32 @testFloat2() {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C10@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 48(r1) # 8-byte Folded Spill
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C0@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 48(r1) # 8-byte Folded Reload
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C10@l(r4)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    la r3, tlv_float[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lfs f0, 0(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    fctiwz f0, f0
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stfd f0, 56(r1)
@@ -901,6 +1800,86 @@ define i32 @testFloat2() {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-LABEL: testFloat2:
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -64(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 80(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C11(r2) # target-flags(ppc-tlsld) @tlv_float
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lfsx f0, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r3, r1, 60
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    xscvdpsxws f0, f0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stfiwx f0, 0, r3
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lwz r3, 60(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 64
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-LABEL: testFloat2:
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -64(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C11@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 80(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C11@l(r6)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    lfsx f0, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r3, r1, 60
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    xscvdpsxws f0, f0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    stfiwx f0, 0, r3
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    lwz r3, 60(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 64
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testFloat2:
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stdu r1, -64(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 80(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C11(r2) # target-flags(ppc-tlsld) @tlv_float
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lfs f0, 0(r3)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    fctiwz f0, f0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stfd f0, 56(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lwa r3, 60(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    clrldi r3, r3, 32
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 64
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testFloat2:
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C11@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 48(r1) # 8-byte Folded Spill
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 48(r1) # 8-byte Folded Reload
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C11@l(r4)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lfs f0, 0(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    fctiwz f0, f0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stfd f0, 56(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lwa r3, 60(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    clrldi r3, r3, 32
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
 entry:
   %tlv_addr = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @tlv_float)
   %value = load float, ptr %tlv_addr, align 4
@@ -916,8 +1895,7 @@ define void @testDouble1(double noundef %x) {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C11(r2) # target-flags(ppc-tlsld) @internal_tlv_double
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stfdx f1, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stfd f1, internal_tlv_double[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
@@ -928,12 +1906,10 @@ define void @testDouble1(double noundef %x) {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C11@u(r2)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C11@l(r6)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stfdx f1, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stfd f1, internal_tlv_double[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
@@ -946,8 +1922,7 @@ define void @testDouble1(double noundef %x) {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 64(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C13(r2) # target-flags(ppc-tlsld) @internal_tlv_double
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    la r3, internal_tlv_double[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stxsdx f1, 0, r3
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
@@ -957,21 +1932,81 @@ define void @testDouble1(double noundef %x) {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testDouble1:
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C13@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -48(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C0@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 56(r1) # 8-byte Folded Reload
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C13@l(r4)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r3, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    la r3, internal_tlv_double[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stxsdx f1, 0, r3
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 48
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-LABEL: testDouble1:
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C12(r2) # target-flags(ppc-tlsld) @internal_tlv_double
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stfdx f1, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-LABEL: testDouble1:
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C12@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C12@l(r6)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    stfdx f1, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testDouble1:
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stdu r1, -48(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 64(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C14(r2) # target-flags(ppc-tlsld) @internal_tlv_double
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    add r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stxsdx f1, 0, r3
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 48
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testDouble1:
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C14@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 56(r1) # 8-byte Folded Reload
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C14@l(r4)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    add r3, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stxsdx f1, 0, r3
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
 entry:
   %tlv_addr = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @internal_tlv_double)
   store double %x, ptr %tlv_addr, align 8
@@ -986,8 +2021,7 @@ define i32 @testDouble2() {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 80(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C11(r2) # target-flags(ppc-tlsld) @internal_tlv_double
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lfdx f0, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lfd f0, internal_tlv_double[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r3, r1, 60
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    xscvdpsxws f0, f0
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stfiwx f0, 0, r3
@@ -1002,12 +2036,10 @@ define i32 @testDouble2() {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -64(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C11@u(r2)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 80(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C11@l(r6)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    lfdx f0, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    lfd f0, internal_tlv_double[TL]@ld(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r3, r1, 60
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    xscvdpsxws f0, f0
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-NEXT:    stfiwx f0, 0, r3
@@ -1024,8 +2056,9 @@ define i32 @testDouble2() {
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 80(r1)
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C13(r2) # target-flags(ppc-tlsld) @internal_tlv_double
-; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lfdx f0, r3, r4
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    la r3, internal_tlv_double[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lfd f0, 0(r3)
+; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    # kill: def $f1 killed $f0
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    xscvdpsxws f0, f0
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r3, r1, 52
 ; SMALL-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stfiwx f0, 0, r3
@@ -1039,25 +2072,103 @@ define i32 @testDouble2() {
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testDouble2:
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -80(r1)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 96(r1)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C13@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -64(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 80(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C0@u(r2)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C0@l(r3)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 56(r1) # 8-byte Folded Reload
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C13@l(r4)
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lfdx f0, r3, r4
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    la r3, internal_tlv_double[TL]@ld(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lfd f0, 0(r3)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    # kill: def $f1 killed $f0
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    xscvdpsxws f0, f0
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r3, r1, 68
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r3, r1, 52
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stfiwx f0, 0, r3
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lwz r3, 68(r1)
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lwz r3, 52(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    clrldi r3, r3, 32
-; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 80
+; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 64
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
 ; SMALL-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-LABEL: testDouble2:
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stdu r1, -64(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    std r0, 80(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r4, L..C12(r2) # target-flags(ppc-tlsld) @internal_tlv_double
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lfdx f0, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r3, r1, 60
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    xscvdpsxws f0, f0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    stfiwx f0, 0, r3
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    lwz r3, 60(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    addi r1, r1, 64
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-LABEL: testDouble2:
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    stdu r1, -64(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r3, L..C0@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addis r6, L..C12@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    std r0, 80(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r3, L..C0@l(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r4, L..C12@l(r6)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    lfdx f0, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r3, r1, 60
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    xscvdpsxws f0, f0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    stfiwx f0, 0, r3
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    lwz r3, 60(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    addi r1, r1, 64
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-LABEL: testDouble2:
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stdu r1, -64(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    std r0, 80(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tlsldm) @"_$TLSML"
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r4, L..C14(r2) # target-flags(ppc-tlsld) @internal_tlv_double
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lfdx f0, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    xscvdpsxws f0, f0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r3, r1, 52
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    stfiwx f0, 0, r3
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    lwz r3, 52(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    clrldi r3, r3, 32
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    addi r1, r1, 64
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-SMALLCM64-O0-NEXT:    blr
+;
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-LABEL: testDouble2:
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0:       # %bb.0: # %entry
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mflr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stdu r1, -80(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r0, 96(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C14@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    std r3, 56(r1) # 8-byte Folded Spill
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addis r3, L..C1@u(r2)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r3, L..C1@l(r3)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    bla .__tls_get_mod[PR]
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, 56(r1) # 8-byte Folded Reload
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r4, L..C14@l(r4)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lfdx f0, r3, r4
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    xscvdpsxws f0, f0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r3, r1, 68
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    stfiwx f0, 0, r3
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    lwz r3, 68(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    clrldi r3, r3, 32
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    addi r1, r1, 80
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    ld r0, 16(r1)
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    mtlr r0
+; DEFAULT-LOCAL-DYNAMIC-LARGECM64-O0-NEXT:    blr
 entry:
   %tlv_addr = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @internal_tlv_double)
   %value = load double, ptr %tlv_addr, align 8
diff --git a/llvm/test/CodeGen/PowerPC/check-aix-small-local-exec-tls-opt-IRattribute.ll b/llvm/test/CodeGen/PowerPC/check-aix-small-local-exec-tls-opt-IRattribute.ll
index dc78ae8436df..fc49e9416803 100644
--- a/llvm/test/CodeGen/PowerPC/check-aix-small-local-exec-tls-opt-IRattribute.ll
+++ b/llvm/test/CodeGen/PowerPC/check-aix-small-local-exec-tls-opt-IRattribute.ll
@@ -8,23 +8,32 @@
 ; RUN:   -data-sections=false < %s 2>&1 | \
 ; RUN: FileCheck %s --check-prefix=CHECK-UNSUPPORTED-NO-DATASEC
 
-define dso_local signext i32 @testWithIRAttr() #0 {
+define dso_local signext i32 @testWithIRAttrLocalExec() #0 {
 entry:
   ret i32 0
 }
-; Check that the aix-small-local-exec-tls attribute is not supported on Linux and AIX (32-bit).
-; CHECK-NOT-SUPPORTED: The aix-small-local-exec-tls attribute is only supported on AIX in 64-bit mode.
 
-; Check that the aix-small-local-exec-tls attribute is only supported when
+define dso_local signext i32 @testWithIRAttrLocalDynamic() #1 {
+entry:
+  ret i32 0
+}
+
+; Check that the aix-small-local-[exec|dynamic]-tls attribute is not supported on Linux and AIX (32-bit).
+; CHECK-NOT-SUPPORTED: The aix-small-local-[exec|dynamic]-tls attribute is only supported on AIX in 64-bit mode.
+
+; Check that the aix-small-local-[exec|dynamic]-tls attribute is only supported when
 ; data sections are enabled.
-; CHECK-UNSUPPORTED-NO-DATASEC: The aix-small-local-exec-tls attribute can only be specified with -data-sections.
+; CHECK-UNSUPPORTED-NO-DATASEC: The aix-small-local-[exec|dynamic]-tls attribute can only be specified with -data-sections.
 
 ; Make sure that the test was actually compiled successfully after using the
-; aix-small-local-exec-tls attribute.
-; CHECK-LABEL: testWithIRAttr:
+; aix-small-local-[exec|dynamic]-tls attribute.
+; CHECK-LABEL: testWithIRAttrLocalExec:
+; CHECK:        li r3, 0
+; CHECK-NEXT:   blr
+; CHECK-LABEL: testWithIRAttrLocalDynamic:
 ; CHECK:        li r3, 0
 ; CHECK-NEXT:   blr
-
 
 attributes #0 = { "target-features"="+aix-small-local-exec-tls" }
+attributes #1 = { "target-features"="+aix-small-local-dynamic-tls" }
 
diff --git a/llvm/test/CodeGen/PowerPC/check-aix-small-local-exec-tls-opt.ll b/llvm/test/CodeGen/PowerPC/check-aix-small-local-exec-tls-opt.ll
index 5104b7c6c839..25bab5b5efcd 100644
--- a/llvm/test/CodeGen/PowerPC/check-aix-small-local-exec-tls-opt.ll
+++ b/llvm/test/CodeGen/PowerPC/check-aix-small-local-exec-tls-opt.ll
@@ -6,17 +6,25 @@
 ; RUN: not llc -mtriple powerpc64le-unknown-linux-gnu -mattr=+aix-small-local-exec-tls \
 ; RUN:   -ppc-asm-full-reg-names < %s 2>&1 | \
 ; RUN:   FileCheck %s --check-prefix=CHECK-NOT-SUPPORTED
+; RUN: llc -mtriple powerpc64-ibm-aix-xcoff -mattr=+aix-small-local-dynamic-tls \
+; RUN:   -ppc-asm-full-reg-names < %s | FileCheck %s
+; RUN: not llc -mtriple powerpc-ibm-aix-xcoff -mattr=+aix-small-local-dynamic-tls \
+; RUN:   -ppc-asm-full-reg-names < %s 2>&1 | \
+; RUN:   FileCheck %s --check-prefix=CHECK-NOT-SUPPORTED
+; RUN: not llc -mtriple powerpc64le-unknown-linux-gnu -mattr=+aix-small-local-dynamic-tls \
+; RUN:   -ppc-asm-full-reg-names < %s 2>&1 | \
+; RUN:   FileCheck %s --check-prefix=CHECK-NOT-SUPPORTED
 
 define dso_local signext i32 @testNoIRAttr() {
 entry:
   ret i32 0
 }
 
-; Check that the aix-small-local-exec-tls attribute is not supported on Linux and AIX (32-bit).
-; CHECK-NOT-SUPPORTED: The aix-small-local-exec-tls attribute is only supported on AIX in 64-bit mode.
+; Check that the aix-small-local-[exec|dynamic]-tls attribute is not supported on Linux and AIX (32-bit).
+; CHECK-NOT-SUPPORTED: The aix-small-local-[exec|dynamic]-tls attribute is only supported on AIX in 64-bit mode.
 
 ; Make sure that the test was actually compiled successfully after using the
-; aix-small-local-exec-tls attribute.
+; aix-small-local-[exec|dynamic]-tls attribute.
 ; CHECK-LABEL: testNoIRAttr:
 ; CHECK:        li r3, 0
 ; CHECK-NEXT:   blr
-- 
GitLab


From 86df55efa1242e308535679cf05c2793ff38f3d9 Mon Sep 17 00:00:00 2001
From: Phoebe Wang <phoebe.wang@intel.com>
Date: Fri, 12 Apr 2024 09:21:17 +0800
Subject: [PATCH 021/464] [X86][BF16] Promote vector ADD/SUB/MUL/DIV to f32
 (#87858)

---
 llvm/lib/Target/X86/X86ISelLowering.cpp |  18 +-
 llvm/test/CodeGen/X86/bfloat.ll         | 286 ++----------------------
 2 files changed, 31 insertions(+), 273 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index f274da6f6f77..b7cb4b7dafeb 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -1978,10 +1978,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::STRICT_FP_ROUND, MVT::v16f16, Custom);
     setOperationAction(ISD::FP_EXTEND, MVT::v16f32, Custom);
     setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v16f32, Custom);
-    for (unsigned Opc : {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV}) {
-      setOperationPromotedToType(Opc, MVT::v16f16, MVT::v16f32);
+    for (unsigned Opc : {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV})
       setOperationPromotedToType(Opc, MVT::v32f16, MVT::v32f32);
-    }
 
     for (auto VT : { MVT::v16i32, MVT::v8i64, MVT::v16f32, MVT::v8f64 }) {
       setOperationAction(ISD::MLOAD,               VT, Legal);
@@ -2296,15 +2294,15 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::bf16, Custom);
     for (auto VT : {MVT::v8bf16, MVT::v16bf16}) {
       setF16Action(VT, Expand);
-      setOperationAction(ISD::FADD, VT, Expand);
-      setOperationAction(ISD::FSUB, VT, Expand);
-      setOperationAction(ISD::FMUL, VT, Expand);
-      setOperationAction(ISD::FDIV, VT, Expand);
       setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
       setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
       setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal);
       setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
     }
+    for (unsigned Opc : {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV}) {
+      setOperationPromotedToType(Opc, MVT::v8bf16, MVT::v8f32);
+      setOperationPromotedToType(Opc, MVT::v16bf16, MVT::v16f32);
+    }
     setOperationAction(ISD::FP_ROUND, MVT::v8bf16, Custom);
     addLegalFPImmediate(APFloat::getZero(APFloat::BFloat()));
   }
@@ -2312,10 +2310,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
   if (!Subtarget.useSoftFloat() && Subtarget.hasBF16()) {
     addRegisterClass(MVT::v32bf16, &X86::VR512RegClass);
     setF16Action(MVT::v32bf16, Expand);
-    setOperationAction(ISD::FADD, MVT::v32bf16, Expand);
-    setOperationAction(ISD::FSUB, MVT::v32bf16, Expand);
-    setOperationAction(ISD::FMUL, MVT::v32bf16, Expand);
-    setOperationAction(ISD::FDIV, MVT::v32bf16, Expand);
+    for (unsigned Opc : {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV})
+      setOperationPromotedToType(Opc, MVT::v32bf16, MVT::v32f32);
     setOperationAction(ISD::BUILD_VECTOR, MVT::v32bf16, Custom);
     setOperationAction(ISD::FP_ROUND, MVT::v16bf16, Custom);
     setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v32bf16, Custom);
diff --git a/llvm/test/CodeGen/X86/bfloat.ll b/llvm/test/CodeGen/X86/bfloat.ll
index 6e3020d628ed..8a2109a1c78d 100644
--- a/llvm/test/CodeGen/X86/bfloat.ll
+++ b/llvm/test/CodeGen/X86/bfloat.ll
@@ -525,101 +525,13 @@ define bfloat @fold_ext_trunc2(bfloat %a) nounwind {
 define <8 x bfloat> @addv(<8 x bfloat> %a, <8 x bfloat> %b) nounwind {
 ; X86-LABEL: addv:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl %ebp
-; X86-NEXT:    pushl %ebx
-; X86-NEXT:    pushl %edi
-; X86-NEXT:    pushl %esi
-; X86-NEXT:    vmovw %xmm1, %eax
-; X86-NEXT:    shll $16, %eax
-; X86-NEXT:    vmovd %eax, %xmm2
-; X86-NEXT:    vmovw %xmm0, %eax
-; X86-NEXT:    shll $16, %eax
-; X86-NEXT:    vmovd %eax, %xmm3
-; X86-NEXT:    vaddss %xmm2, %xmm3, %xmm2
-; X86-NEXT:    vcvtneps2bf16 %xmm2, %xmm2
-; X86-NEXT:    vmovw %xmm2, %ecx
-; X86-NEXT:    vpextrw $1, %xmm1, %eax
-; X86-NEXT:    shll $16, %eax
-; X86-NEXT:    vmovd %eax, %xmm2
-; X86-NEXT:    vpextrw $1, %xmm0, %eax
-; X86-NEXT:    shll $16, %eax
-; X86-NEXT:    vmovd %eax, %xmm3
-; X86-NEXT:    vaddss %xmm2, %xmm3, %xmm2
-; X86-NEXT:    vcvtneps2bf16 %xmm2, %xmm2
-; X86-NEXT:    vmovw %xmm2, %eax
-; X86-NEXT:    vpextrw $2, %xmm1, %edx
-; X86-NEXT:    shll $16, %edx
-; X86-NEXT:    vmovd %edx, %xmm2
-; X86-NEXT:    vpextrw $2, %xmm0, %edx
-; X86-NEXT:    shll $16, %edx
-; X86-NEXT:    vmovd %edx, %xmm3
-; X86-NEXT:    vaddss %xmm2, %xmm3, %xmm2
-; X86-NEXT:    vcvtneps2bf16 %xmm2, %xmm2
-; X86-NEXT:    vmovw %xmm2, %edx
-; X86-NEXT:    vpextrw $3, %xmm1, %esi
-; X86-NEXT:    shll $16, %esi
-; X86-NEXT:    vmovd %esi, %xmm2
-; X86-NEXT:    vpextrw $3, %xmm0, %esi
-; X86-NEXT:    shll $16, %esi
-; X86-NEXT:    vmovd %esi, %xmm3
-; X86-NEXT:    vaddss %xmm2, %xmm3, %xmm2
-; X86-NEXT:    vcvtneps2bf16 %xmm2, %xmm2
-; X86-NEXT:    vmovw %xmm2, %esi
-; X86-NEXT:    vpextrw $4, %xmm1, %edi
-; X86-NEXT:    shll $16, %edi
-; X86-NEXT:    vmovd %edi, %xmm2
-; X86-NEXT:    vpextrw $4, %xmm0, %edi
-; X86-NEXT:    shll $16, %edi
-; X86-NEXT:    vmovd %edi, %xmm3
-; X86-NEXT:    vaddss %xmm2, %xmm3, %xmm2
-; X86-NEXT:    vcvtneps2bf16 %xmm2, %xmm2
-; X86-NEXT:    vmovw %xmm2, %ebx
-; X86-NEXT:    vpextrw $5, %xmm1, %edi
-; X86-NEXT:    shll $16, %edi
-; X86-NEXT:    vmovd %edi, %xmm2
-; X86-NEXT:    vpextrw $5, %xmm0, %edi
-; X86-NEXT:    shll $16, %edi
-; X86-NEXT:    vmovd %edi, %xmm3
-; X86-NEXT:    vaddss %xmm2, %xmm3, %xmm2
-; X86-NEXT:    vcvtneps2bf16 %xmm2, %xmm2
-; X86-NEXT:    vmovw %xmm2, %edi
-; X86-NEXT:    vpextrw $6, %xmm1, %ebp
-; X86-NEXT:    shll $16, %ebp
-; X86-NEXT:    vmovd %ebp, %xmm2
-; X86-NEXT:    vpextrw $6, %xmm0, %ebp
-; X86-NEXT:    shll $16, %ebp
-; X86-NEXT:    vmovd %ebp, %xmm3
-; X86-NEXT:    vaddss %xmm2, %xmm3, %xmm3
-; X86-NEXT:    vmovw %ecx, %xmm2
-; X86-NEXT:    vcvtneps2bf16 %xmm3, %xmm3
-; X86-NEXT:    vmovw %xmm3, %ecx
-; X86-NEXT:    vmovw %ebx, %xmm3
-; X86-NEXT:    vpextrw $7, %xmm1, %ebx
-; X86-NEXT:    shll $16, %ebx
-; X86-NEXT:    vmovd %ebx, %xmm1
-; X86-NEXT:    vpextrw $7, %xmm0, %ebx
-; X86-NEXT:    shll $16, %ebx
-; X86-NEXT:    vmovd %ebx, %xmm0
-; X86-NEXT:    vaddss %xmm1, %xmm0, %xmm0
-; X86-NEXT:    vmovw %ecx, %xmm1
-; X86-NEXT:    vcvtneps2bf16 %xmm0, %xmm0
-; X86-NEXT:    vmovw %xmm0, %ecx
-; X86-NEXT:    vmovw %ecx, %xmm0
-; X86-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; X86-NEXT:    vmovw %edi, %xmm1
-; X86-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; X86-NEXT:    vmovw %edx, %xmm3
-; X86-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; X86-NEXT:    vmovw %esi, %xmm1
-; X86-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
-; X86-NEXT:    vmovw %eax, %xmm3
-; X86-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; X86-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; X86-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; X86-NEXT:    popl %esi
-; X86-NEXT:    popl %edi
-; X86-NEXT:    popl %ebx
-; X86-NEXT:    popl %ebp
+; X86-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; X86-NEXT:    vpslld $16, %ymm1, %ymm1
+; X86-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X86-NEXT:    vpslld $16, %ymm0, %ymm0
+; X86-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; X86-NEXT:    vcvtneps2bf16 %ymm0, %xmm0
+; X86-NEXT:    vzeroupper
 ; X86-NEXT:    retl
 ;
 ; SSE2-LABEL: addv:
@@ -756,176 +668,26 @@ define <8 x bfloat> @addv(<8 x bfloat> %a, <8 x bfloat> %b) nounwind {
 ; SSE2-NEXT:    popq %rbp
 ; SSE2-NEXT:    retq
 ;
-; FP16-LABEL: addv:
-; FP16:       # %bb.0:
-; FP16-NEXT:    vmovw %xmm1, %eax
-; FP16-NEXT:    shll $16, %eax
-; FP16-NEXT:    vmovd %eax, %xmm2
-; FP16-NEXT:    vmovw %xmm0, %eax
-; FP16-NEXT:    shll $16, %eax
-; FP16-NEXT:    vmovd %eax, %xmm3
-; FP16-NEXT:    vaddss %xmm2, %xmm3, %xmm2
-; FP16-NEXT:    vcvtneps2bf16 %xmm2, %xmm2
-; FP16-NEXT:    vmovw %xmm2, %eax
-; FP16-NEXT:    vmovw %eax, %xmm2
-; FP16-NEXT:    vpextrw $1, %xmm1, %eax
-; FP16-NEXT:    shll $16, %eax
-; FP16-NEXT:    vmovd %eax, %xmm3
-; FP16-NEXT:    vpextrw $1, %xmm0, %eax
-; FP16-NEXT:    shll $16, %eax
-; FP16-NEXT:    vmovd %eax, %xmm4
-; FP16-NEXT:    vaddss %xmm3, %xmm4, %xmm3
-; FP16-NEXT:    vcvtneps2bf16 %xmm3, %xmm3
-; FP16-NEXT:    vmovw %xmm3, %eax
-; FP16-NEXT:    vmovw %eax, %xmm3
-; FP16-NEXT:    vpextrw $2, %xmm1, %eax
-; FP16-NEXT:    shll $16, %eax
-; FP16-NEXT:    vmovd %eax, %xmm4
-; FP16-NEXT:    vpextrw $2, %xmm0, %eax
-; FP16-NEXT:    shll $16, %eax
-; FP16-NEXT:    vmovd %eax, %xmm5
-; FP16-NEXT:    vaddss %xmm4, %xmm5, %xmm4
-; FP16-NEXT:    vcvtneps2bf16 %xmm4, %xmm4
-; FP16-NEXT:    vmovw %xmm4, %eax
-; FP16-NEXT:    vmovw %eax, %xmm4
-; FP16-NEXT:    vpextrw $3, %xmm1, %eax
-; FP16-NEXT:    shll $16, %eax
-; FP16-NEXT:    vmovd %eax, %xmm5
-; FP16-NEXT:    vpextrw $3, %xmm0, %eax
-; FP16-NEXT:    shll $16, %eax
-; FP16-NEXT:    vmovd %eax, %xmm6
-; FP16-NEXT:    vaddss %xmm5, %xmm6, %xmm5
-; FP16-NEXT:    vcvtneps2bf16 %xmm5, %xmm5
-; FP16-NEXT:    vmovw %xmm5, %eax
-; FP16-NEXT:    vmovw %eax, %xmm5
-; FP16-NEXT:    vpextrw $4, %xmm1, %eax
-; FP16-NEXT:    shll $16, %eax
-; FP16-NEXT:    vmovd %eax, %xmm6
-; FP16-NEXT:    vpextrw $4, %xmm0, %eax
-; FP16-NEXT:    shll $16, %eax
-; FP16-NEXT:    vmovd %eax, %xmm7
-; FP16-NEXT:    vaddss %xmm6, %xmm7, %xmm6
-; FP16-NEXT:    vcvtneps2bf16 %xmm6, %xmm6
-; FP16-NEXT:    vmovw %xmm6, %eax
-; FP16-NEXT:    vmovw %eax, %xmm6
-; FP16-NEXT:    vpextrw $5, %xmm1, %eax
-; FP16-NEXT:    shll $16, %eax
-; FP16-NEXT:    vmovd %eax, %xmm7
-; FP16-NEXT:    vpextrw $5, %xmm0, %eax
-; FP16-NEXT:    shll $16, %eax
-; FP16-NEXT:    vmovd %eax, %xmm8
-; FP16-NEXT:    vaddss %xmm7, %xmm8, %xmm7
-; FP16-NEXT:    vcvtneps2bf16 %xmm7, %xmm7
-; FP16-NEXT:    vmovw %xmm7, %eax
-; FP16-NEXT:    vmovw %eax, %xmm7
-; FP16-NEXT:    vpextrw $6, %xmm1, %eax
-; FP16-NEXT:    shll $16, %eax
-; FP16-NEXT:    vmovd %eax, %xmm8
-; FP16-NEXT:    vpextrw $6, %xmm0, %eax
-; FP16-NEXT:    shll $16, %eax
-; FP16-NEXT:    vmovd %eax, %xmm9
-; FP16-NEXT:    vaddss %xmm8, %xmm9, %xmm8
-; FP16-NEXT:    vcvtneps2bf16 %xmm8, %xmm8
-; FP16-NEXT:    vmovw %xmm8, %eax
-; FP16-NEXT:    vmovw %eax, %xmm8
-; FP16-NEXT:    vpextrw $7, %xmm1, %eax
-; FP16-NEXT:    shll $16, %eax
-; FP16-NEXT:    vmovd %eax, %xmm1
-; FP16-NEXT:    vpextrw $7, %xmm0, %eax
-; FP16-NEXT:    shll $16, %eax
-; FP16-NEXT:    vmovd %eax, %xmm0
-; FP16-NEXT:    vaddss %xmm1, %xmm0, %xmm0
-; FP16-NEXT:    vcvtneps2bf16 %xmm0, %xmm0
-; FP16-NEXT:    vmovw %xmm0, %eax
-; FP16-NEXT:    vmovw %eax, %xmm0
-; FP16-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm8[0],xmm0[0],xmm8[1],xmm0[1],xmm8[2],xmm0[2],xmm8[3],xmm0[3]
-; FP16-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3]
-; FP16-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; FP16-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
-; FP16-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; FP16-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; FP16-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; FP16-NEXT:    retq
+; F16-LABEL: addv:
+; F16:       # %bb.0:
+; F16-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; F16-NEXT:    vpslld $16, %ymm1, %ymm1
+; F16-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; F16-NEXT:    vpslld $16, %ymm0, %ymm0
+; F16-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; F16-NEXT:    vcvtneps2bf16 %ymm0, %xmm0
+; F16-NEXT:    vzeroupper
+; F16-NEXT:    retq
 ;
 ; AVXNC-LABEL: addv:
 ; AVXNC:       # %bb.0:
-; AVXNC-NEXT:    vpextrw $7, %xmm1, %eax
-; AVXNC-NEXT:    shll $16, %eax
-; AVXNC-NEXT:    vmovd %eax, %xmm2
-; AVXNC-NEXT:    vpextrw $7, %xmm0, %eax
-; AVXNC-NEXT:    shll $16, %eax
-; AVXNC-NEXT:    vmovd %eax, %xmm3
-; AVXNC-NEXT:    vaddss %xmm2, %xmm3, %xmm2
-; AVXNC-NEXT:    {vex} vcvtneps2bf16 %xmm2, %xmm2
-; AVXNC-NEXT:    vmovd %xmm2, %eax
-; AVXNC-NEXT:    vpextrw $6, %xmm1, %ecx
-; AVXNC-NEXT:    shll $16, %ecx
-; AVXNC-NEXT:    vmovd %ecx, %xmm2
-; AVXNC-NEXT:    vpextrw $6, %xmm0, %ecx
-; AVXNC-NEXT:    shll $16, %ecx
-; AVXNC-NEXT:    vmovd %ecx, %xmm3
-; AVXNC-NEXT:    vaddss %xmm2, %xmm3, %xmm2
-; AVXNC-NEXT:    {vex} vcvtneps2bf16 %xmm2, %xmm2
-; AVXNC-NEXT:    vmovd %xmm2, %ecx
-; AVXNC-NEXT:    vpextrw $5, %xmm1, %edx
-; AVXNC-NEXT:    shll $16, %edx
-; AVXNC-NEXT:    vmovd %edx, %xmm2
-; AVXNC-NEXT:    vpextrw $5, %xmm0, %edx
-; AVXNC-NEXT:    shll $16, %edx
-; AVXNC-NEXT:    vmovd %edx, %xmm3
-; AVXNC-NEXT:    vaddss %xmm2, %xmm3, %xmm2
-; AVXNC-NEXT:    {vex} vcvtneps2bf16 %xmm2, %xmm2
-; AVXNC-NEXT:    vmovd %xmm2, %edx
-; AVXNC-NEXT:    vpextrw $4, %xmm1, %esi
-; AVXNC-NEXT:    shll $16, %esi
-; AVXNC-NEXT:    vmovd %esi, %xmm2
-; AVXNC-NEXT:    vpextrw $4, %xmm0, %esi
-; AVXNC-NEXT:    shll $16, %esi
-; AVXNC-NEXT:    vmovd %esi, %xmm3
-; AVXNC-NEXT:    vaddss %xmm2, %xmm3, %xmm2
-; AVXNC-NEXT:    {vex} vcvtneps2bf16 %xmm2, %xmm2
-; AVXNC-NEXT:    vmovd %xmm2, %esi
-; AVXNC-NEXT:    vpextrw $3, %xmm1, %edi
-; AVXNC-NEXT:    shll $16, %edi
-; AVXNC-NEXT:    vmovd %edi, %xmm2
-; AVXNC-NEXT:    vpextrw $3, %xmm0, %edi
-; AVXNC-NEXT:    shll $16, %edi
-; AVXNC-NEXT:    vmovd %edi, %xmm3
-; AVXNC-NEXT:    vaddss %xmm2, %xmm3, %xmm2
-; AVXNC-NEXT:    {vex} vcvtneps2bf16 %xmm2, %xmm2
-; AVXNC-NEXT:    vmovd %xmm2, %edi
-; AVXNC-NEXT:    vpextrw $2, %xmm1, %r8d
-; AVXNC-NEXT:    shll $16, %r8d
-; AVXNC-NEXT:    vmovd %r8d, %xmm2
-; AVXNC-NEXT:    vpextrw $2, %xmm0, %r8d
-; AVXNC-NEXT:    shll $16, %r8d
-; AVXNC-NEXT:    vmovd %r8d, %xmm3
-; AVXNC-NEXT:    vaddss %xmm2, %xmm3, %xmm2
-; AVXNC-NEXT:    {vex} vcvtneps2bf16 %xmm2, %xmm2
-; AVXNC-NEXT:    vmovd %xmm2, %r8d
-; AVXNC-NEXT:    vpextrw $1, %xmm1, %r9d
-; AVXNC-NEXT:    shll $16, %r9d
-; AVXNC-NEXT:    vmovd %r9d, %xmm2
-; AVXNC-NEXT:    vpextrw $1, %xmm0, %r9d
-; AVXNC-NEXT:    shll $16, %r9d
-; AVXNC-NEXT:    vmovd %r9d, %xmm3
-; AVXNC-NEXT:    vaddss %xmm2, %xmm3, %xmm2
-; AVXNC-NEXT:    {vex} vcvtneps2bf16 %xmm2, %xmm2
-; AVXNC-NEXT:    vmovd %xmm1, %r9d
-; AVXNC-NEXT:    shll $16, %r9d
-; AVXNC-NEXT:    vmovd %r9d, %xmm1
-; AVXNC-NEXT:    vmovd %xmm0, %r9d
-; AVXNC-NEXT:    shll $16, %r9d
-; AVXNC-NEXT:    vmovd %r9d, %xmm0
-; AVXNC-NEXT:    vaddss %xmm1, %xmm0, %xmm0
-; AVXNC-NEXT:    {vex} vcvtneps2bf16 %xmm0, %xmm0
-; AVXNC-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; AVXNC-NEXT:    vpinsrw $2, %r8d, %xmm0, %xmm0
-; AVXNC-NEXT:    vpinsrw $3, %edi, %xmm0, %xmm0
-; AVXNC-NEXT:    vpinsrw $4, %esi, %xmm0, %xmm0
-; AVXNC-NEXT:    vpinsrw $5, %edx, %xmm0, %xmm0
-; AVXNC-NEXT:    vpinsrw $6, %ecx, %xmm0, %xmm0
-; AVXNC-NEXT:    vpinsrw $7, %eax, %xmm0, %xmm0
+; AVXNC-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVXNC-NEXT:    vpslld $16, %ymm1, %ymm1
+; AVXNC-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVXNC-NEXT:    vpslld $16, %ymm0, %ymm0
+; AVXNC-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVXNC-NEXT:    {vex} vcvtneps2bf16 %ymm0, %xmm0
+; AVXNC-NEXT:    vzeroupper
 ; AVXNC-NEXT:    retq
   %add = fadd <8 x bfloat> %a, %b
   ret <8 x bfloat> %add
-- 
GitLab


From 27ce513788e15c95f8c623566fbfca9304bb1844 Mon Sep 17 00:00:00 2001
From: Corbin Robeck <corbin.robeck@amd.com>
Date: Thu, 11 Apr 2024 22:05:01 -0400
Subject: [PATCH 022/464] [AMDGPU] Add Clang builtins for amdgcn s_ttrace
 intrinsics (#88076)

---
 clang/include/clang/Basic/BuiltinsAMDGPU.def  |  2 ++
 .../CodeGenOpenCL/builtins-amdgcn-gfx12.cl    | 22 +++++++++++++++++++
 llvm/include/llvm/IR/IntrinsicsAMDGPU.td      |  3 +++
 3 files changed, 27 insertions(+)

diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.def b/clang/include/clang/Basic/BuiltinsAMDGPU.def
index c660582cc98e..3e21a2fe2ac6 100644
--- a/clang/include/clang/Basic/BuiltinsAMDGPU.def
+++ b/clang/include/clang/Basic/BuiltinsAMDGPU.def
@@ -61,6 +61,7 @@ BUILTIN(__builtin_amdgcn_s_waitcnt, "vIi", "n")
 BUILTIN(__builtin_amdgcn_s_sendmsg, "vIiUi", "n")
 BUILTIN(__builtin_amdgcn_s_sendmsghalt, "vIiUi", "n")
 BUILTIN(__builtin_amdgcn_s_barrier, "v", "n")
+BUILTIN(__builtin_amdgcn_s_ttracedata, "vi", "n")
 BUILTIN(__builtin_amdgcn_wave_barrier, "v", "n")
 BUILTIN(__builtin_amdgcn_sched_barrier, "vIi", "n")
 BUILTIN(__builtin_amdgcn_sched_group_barrier, "vIiIiIi", "n")
@@ -267,6 +268,7 @@ TARGET_BUILTIN(__builtin_amdgcn_dot4_f32_bf8_bf8, "fUiUif", "nc", "dot11-insts")
 TARGET_BUILTIN(__builtin_amdgcn_permlane16, "UiUiUiUiUiIbIb", "nc", "gfx10-insts")
 TARGET_BUILTIN(__builtin_amdgcn_permlanex16, "UiUiUiUiUiIbIb", "nc", "gfx10-insts")
 TARGET_BUILTIN(__builtin_amdgcn_mov_dpp8, "UiUiIUi", "nc", "gfx10-insts")
+TARGET_BUILTIN(__builtin_amdgcn_s_ttracedata_imm, "vIs", "n", "gfx10-insts")
 
 //===----------------------------------------------------------------------===//
 // Raytracing builtins.
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx12.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx12.cl
index ebd367bba0cd..26c0ee483062 100644
--- a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx12.cl
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx12.cl
@@ -235,3 +235,25 @@ unsigned test_s_get_barrier_state(int a)
   unsigned State = __builtin_amdgcn_s_get_barrier_state(a);
   return State;
 }
+
+// CHECK-LABEL: @test_s_ttracedata(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    call void @llvm.amdgcn.s.ttracedata(i32 1)
+// CHECK-NEXT:    ret void
+//
+void test_s_ttracedata()
+{
+  __builtin_amdgcn_s_ttracedata(1);
+}
+
+// CHECK-LABEL: @test_s_ttracedata_imm(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    call void @llvm.amdgcn.s.ttracedata.imm(i16 1)
+// CHECK-NEXT:    ret void
+//
+void test_s_ttracedata_imm()
+{
+  __builtin_amdgcn_s_ttracedata_imm(1);
+}
+
+
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index 6bbc13f1de86..ee9a5d7a3439 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -1887,9 +1887,12 @@ def int_amdgcn_s_setprio :
                                 IntrHasSideEffects]>;
 
 def int_amdgcn_s_ttracedata :
+  ClangBuiltin<"__builtin_amdgcn_s_ttracedata">,
   DefaultAttrsIntrinsic<[], [llvm_i32_ty],
                         [IntrNoMem, IntrHasSideEffects]>;
+
 def int_amdgcn_s_ttracedata_imm :
+  ClangBuiltin<"__builtin_amdgcn_s_ttracedata_imm">,
   DefaultAttrsIntrinsic<[], [llvm_i16_ty],
                         [IntrNoMem, IntrHasSideEffects, ImmArg<ArgIndex<0>>]>;
 
-- 
GitLab


From 6b6f272f355a2cbc753e9057dd7aefb1a74a0409 Mon Sep 17 00:00:00 2001
From: Feng Zou <feng.zou@intel.com>
Date: Fri, 12 Apr 2024 10:13:56 +0800
Subject: [PATCH 023/464] [DAGCombiner] Require same type of splat & element
 for build_vector (#88284)

Only allow to change build_vector to concat_vector when the splat type
and vector element type is same. It's to fix assertion of failing to
bitcast types of different sizes.
---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp |  9 ++++---
 llvm/test/CodeGen/X86/buildvec-bitcast.ll     | 25 +++++++++++++++++++
 2 files changed, 31 insertions(+), 3 deletions(-)
 create mode 100644 llvm/test/CodeGen/X86/buildvec-bitcast.ll

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 95c1cde0b934..24f50b87c4cf 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -23428,7 +23428,10 @@ SDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) {
   // If X is a build_vector itself, the concat can become a larger build_vector.
   // TODO: Maybe this is useful for non-splat too?
   if (!LegalOperations) {
-    if (SDValue Splat = cast<BuildVectorSDNode>(N)->getSplatValue()) {
+    SDValue Splat = cast<BuildVectorSDNode>(N)->getSplatValue();
+    // Only change build_vector to a concat_vector if the splat value type is
+    // same as the vector element type.
+    if (Splat && Splat.getValueType() == VT.getVectorElementType()) {
       Splat = peekThroughBitcasts(Splat);
       EVT SrcVT = Splat.getValueType();
       if (SrcVT.isVector()) {
@@ -23437,8 +23440,8 @@ SDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) {
                                      SrcVT.getVectorElementType(), NumElts);
         if (!LegalTypes || TLI.isTypeLegal(NewVT)) {
           SmallVector<SDValue, 8> Ops(N->getNumOperands(), Splat);
-          SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N),
-                                       NewVT, Ops);
+          SDValue Concat =
+              DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), NewVT, Ops);
           return DAG.getBitcast(VT, Concat);
         }
       }
diff --git a/llvm/test/CodeGen/X86/buildvec-bitcast.ll b/llvm/test/CodeGen/X86/buildvec-bitcast.ll
new file mode 100644
index 000000000000..980fd2dee091
--- /dev/null
+++ b/llvm/test/CodeGen/X86/buildvec-bitcast.ll
@@ -0,0 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc < %s -mtriple=x86_64 -mattr=avx512bw | FileCheck %s
+
+; Verify that the DAGCombiner doesn't change build_vector to concat_vectors if
+; the vector element type is different than splat type. The example here:
+;   v8i1 = build_vector (i8 (bitcast (v8i1 X))), ..., (i8 (bitcast (v8i1 X))))
+
+define <8 x i1> @foo(<8 x i1> %mask.i1) {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+entry:
+  %0 = and <8 x i1> %mask.i1, <i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false>
+  %1 = bitcast <8 x i1> %0 to i8
+  %2 = icmp ne i8 %1, 0
+  %insert54 = insertelement <8 x i1> zeroinitializer, i1 %2, i64 0
+  %splat55 = shufflevector <8 x i1> %insert54, <8 x i1> zeroinitializer, <8 x i32> zeroinitializer
+  %3 = and <8 x i1> %0, %splat55
+  br label %end
+
+end:                           ; preds = %entry
+  %4 = select <8 x i1> %3, <8 x i1> zeroinitializer, <8 x i1> zeroinitializer
+  ret <8 x i1> %4
+}
-- 
GitLab


From 3a2817749eb8a2210320aa551fa20a5f9c8c32e3 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 11 Apr 2024 19:47:58 -0700
Subject: [PATCH 024/464] [llvm-exegesis] Let ArrayRef infer the array size
 (NFC) (#88241)

---
 llvm/tools/llvm-exegesis/lib/X86/Target.cpp | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/llvm/tools/llvm-exegesis/lib/X86/Target.cpp b/llvm/tools/llvm-exegesis/lib/X86/Target.cpp
index 31bec2545926..dbb2bf409316 100644
--- a/llvm/tools/llvm-exegesis/lib/X86/Target.cpp
+++ b/llvm/tools/llvm-exegesis/lib/X86/Target.cpp
@@ -769,11 +769,9 @@ private:
 
   ArrayRef<unsigned> getUnavailableRegisters() const override {
     if (DisableUpperSSERegisters)
-      return ArrayRef(kUnavailableRegistersSSE,
-                      sizeof(kUnavailableRegistersSSE) /
-                          sizeof(kUnavailableRegistersSSE[0]));
+      return ArrayRef(kUnavailableRegistersSSE);
 
-    return ArrayRef(kUnavailableRegisters, std::size(kUnavailableRegisters));
+    return ArrayRef(kUnavailableRegisters);
   }
 
   bool allowAsBackToBack(const Instruction &Instr) const override {
-- 
GitLab


From 334e07f18e6f2fbc84579f7cac3cdd29d0d7cce0 Mon Sep 17 00:00:00 2001
From: Piyou Chen <piyou.chen@sifive.com>
Date: Fri, 12 Apr 2024 10:59:06 +0800
Subject: [PATCH 025/464] [RISCV] Disallow target attribute use in
 multiversioning (#85899)

For RISC-V target only `target_clones` and `target_version` can enable
function multiversion(FMV).

This patch make target attribute trigger redefinition instead of emit
FMV.

Here is spec
https://github.com/riscv-non-isa/riscv-c-api-doc/blob/master/riscv-c-api.md#__attribute__targetattr-string
---
 clang/lib/Sema/SemaDecl.cpp         | 8 +++++++-
 clang/test/Sema/attr-target-riscv.c | 6 ++++++
 2 files changed, 13 insertions(+), 1 deletion(-)
 create mode 100644 clang/test/Sema/attr-target-riscv.c

diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index a4699d6ba2c7..691e170cd051 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -11911,8 +11911,14 @@ static bool CheckMultiVersionFunction(Sema &S, FunctionDecl *NewFD,
     return false;
   }
 
+  const llvm::Triple &T = S.getASTContext().getTargetInfo().getTriple();
+
   // Target attribute on AArch64 is not used for multiversioning
-  if (NewTA && S.getASTContext().getTargetInfo().getTriple().isAArch64())
+  if (NewTA && T.isAArch64())
+    return false;
+
+  // Target attribute on RISCV is not used for multiversioning
+  if (NewTA && T.isRISCV())
     return false;
 
   if (!OldDecl || !OldDecl->getAsFunction() ||
diff --git a/clang/test/Sema/attr-target-riscv.c b/clang/test/Sema/attr-target-riscv.c
new file mode 100644
index 000000000000..ed4e2915d6c6
--- /dev/null
+++ b/clang/test/Sema/attr-target-riscv.c
@@ -0,0 +1,6 @@
+// RUN: %clang_cc1 -triple riscv64-linux-gnu -target-feature +i -fsyntax-only -verify -std=c2x %s
+
+//expected-note@+1 {{previous definition is here}}
+int __attribute__((target("arch=rv64g"))) foo(void) { return 0; }
+//expected-error@+1 {{redefinition of 'foo'}}
+int __attribute__((target("arch=rv64gc"))) foo(void) { return 0; }
-- 
GitLab


From c24efffabbf96c7a138439bb8e219850c4d78887 Mon Sep 17 00:00:00 2001
From: Cyndy Ishida <cyndy_ishida@apple.com>
Date: Thu, 11 Apr 2024 20:01:55 -0700
Subject: [PATCH 026/464] [InstallAPI] Handle zippered frameworks (#88205)

A zippered framework is a single framework that can be loaded in both
macOS and macatalyst processes. Broadly to InstallAPI, it means the same
interface can represent two separate platforms.

A dylib's symbol table does not distinguish between macOS/macCatalyst.
  `InstallAPI` provides the ability for the tbd file to distinct
symbols between them.
The verifier handles this special logic by tracking all unavailable and
obsoleted APIs in this context and checking against those when
determining dylib symbols with no matching declaration.

* If there exists an available decl for either platform, do not warn.
* If there is no available decl, emit a diagnostic and print the source
location for both decls.
---
 .../clang/Basic/DiagnosticInstallAPIKinds.td  |   4 +
 .../include/clang/InstallAPI/DylibVerifier.h  |  32 +-
 clang/lib/InstallAPI/DylibVerifier.cpp        |  98 ++-
 .../Inputs/MacOSX13.0.sdk/SDKSettings.json    |  19 +
 .../test/InstallAPI/diagnostics-zippered.test | 765 ++++++++++++++++++
 .../InstallAPI/driver-invalid-options.test    |   7 +
 clang/tools/clang-installapi/Options.cpp      |  52 +-
 clang/tools/clang-installapi/Options.h        |   3 +
 8 files changed, 964 insertions(+), 16 deletions(-)
 create mode 100644 clang/test/InstallAPI/Inputs/MacOSX13.0.sdk/SDKSettings.json
 create mode 100644 clang/test/InstallAPI/diagnostics-zippered.test

diff --git a/clang/include/clang/Basic/DiagnosticInstallAPIKinds.td b/clang/include/clang/Basic/DiagnosticInstallAPIKinds.td
index 0a477da7186b..396bff0146a3 100644
--- a/clang/include/clang/Basic/DiagnosticInstallAPIKinds.td
+++ b/clang/include/clang/Basic/DiagnosticInstallAPIKinds.td
@@ -20,6 +20,10 @@ def warn_no_such_excluded_header_file : Warning<"no such excluded %select{public
 def warn_glob_did_not_match: Warning<"glob '%0' did not match any header file">, InGroup<InstallAPIViolation>;
 def err_no_such_umbrella_header_file : Error<"%select{public|private|project}1 umbrella header file not found in input: '%0'">;
 def err_cannot_find_reexport : Error<"cannot find re-exported %select{framework|library}0: '%1'">;
+def err_no_matching_target : Error<"no matching target found for target variant '%0'">;
+def err_unsupported_vendor : Error<"vendor '%0' is not supported: '%1'">;
+def err_unsupported_environment : Error<"environment '%0' is not supported: '%1'">;
+def err_unsupported_os : Error<"os '%0' is not supported: '%1'">;
 } // end of command line category.
 
 let CategoryName = "Verification" in {
diff --git a/clang/include/clang/InstallAPI/DylibVerifier.h b/clang/include/clang/InstallAPI/DylibVerifier.h
index a3df25f10de4..31de212fc423 100644
--- a/clang/include/clang/InstallAPI/DylibVerifier.h
+++ b/clang/include/clang/InstallAPI/DylibVerifier.h
@@ -28,6 +28,16 @@ enum class VerificationMode {
 using LibAttrs = llvm::StringMap<ArchitectureSet>;
 using ReexportedInterfaces = llvm::SmallVector<llvm::MachO::InterfaceFile, 8>;
 
+// Pointers to information about a zippered declaration used for
+// querying and reporting violations against different
+// declarations that all map to the same symbol.
+struct ZipperedDeclSource {
+  const FrontendAttrs *FA;
+  clang::SourceManager *SrcMgr;
+  Target T;
+};
+using ZipperedDeclSources = std::vector<ZipperedDeclSource>;
+
 /// Service responsible to tracking state of verification across the
 /// lifetime of InstallAPI.
 /// As declarations are collected during AST traversal, they are
@@ -68,10 +78,10 @@ public:
   DylibVerifier() = default;
 
   DylibVerifier(llvm::MachO::Records &&Dylib, ReexportedInterfaces &&Reexports,
-                DiagnosticsEngine *Diag, VerificationMode Mode, bool Demangle,
-                StringRef DSYMPath)
+                DiagnosticsEngine *Diag, VerificationMode Mode, bool Zippered,
+                bool Demangle, StringRef DSYMPath)
       : Dylib(std::move(Dylib)), Reexports(std::move(Reexports)), Mode(Mode),
-        Demangle(Demangle), DSYMPath(DSYMPath),
+        Zippered(Zippered), Demangle(Demangle), DSYMPath(DSYMPath),
         Exports(std::make_unique<SymbolSet>()), Ctx(VerifierContext{Diag}) {}
 
   Result verify(GlobalRecord *R, const FrontendAttrs *FA);
@@ -118,6 +128,15 @@ private:
   /// symbols should be omitted from the text-api file.
   bool shouldIgnoreReexport(const Record *R, SymbolContext &SymCtx) const;
 
+  // Ignore and omit unavailable symbols in zippered libraries.
+  bool shouldIgnoreZipperedAvailability(const Record *R, SymbolContext &SymCtx);
+
+  // Check if an internal declaration in zippered library has an
+  // external declaration for a different platform. This results
+  // in the symbol being in a "seperate" platform slice.
+  bool shouldIgnoreInternalZipperedSymbol(const Record *R,
+                                          const SymbolContext &SymCtx) const;
+
   /// Compare the visibility declarations to the linkage of symbol found in
   /// dylib.
   Result compareVisibility(const Record *R, SymbolContext &SymCtx,
@@ -173,6 +192,9 @@ private:
   // Controls what class of violations to report.
   VerificationMode Mode = VerificationMode::Invalid;
 
+  // Library is zippered.
+  bool Zippered = false;
+
   // Attempt to demangle when reporting violations.
   bool Demangle = false;
 
@@ -182,6 +204,10 @@ private:
   // Valid symbols in final text file.
   std::unique_ptr<SymbolSet> Exports = std::make_unique<SymbolSet>();
 
+  // Unavailable or obsoleted declarations for a zippered library.
+  // These are cross referenced against symbols in the dylib.
+  llvm::StringMap<ZipperedDeclSources> DeferredZipperedSymbols;
+
   // Track current state of verification while traversing AST.
   VerifierContext Ctx;
 
diff --git a/clang/lib/InstallAPI/DylibVerifier.cpp b/clang/lib/InstallAPI/DylibVerifier.cpp
index 4fa2d4e9292c..84d9b5892e88 100644
--- a/clang/lib/InstallAPI/DylibVerifier.cpp
+++ b/clang/lib/InstallAPI/DylibVerifier.cpp
@@ -176,7 +176,13 @@ void DylibVerifier::addSymbol(const Record *R, SymbolContext &SymCtx,
 
 bool DylibVerifier::shouldIgnoreObsolete(const Record *R, SymbolContext &SymCtx,
                                          const Record *DR) {
-  return SymCtx.FA->Avail.isObsoleted();
+  if (!SymCtx.FA->Avail.isObsoleted())
+    return false;
+
+  if (Zippered)
+    DeferredZipperedSymbols[SymCtx.SymbolName].emplace_back(ZipperedDeclSource{
+        SymCtx.FA, &Ctx.Diag->getSourceManager(), Ctx.Target});
+  return true;
 }
 
 bool DylibVerifier::shouldIgnoreReexport(const Record *R,
@@ -195,6 +201,28 @@ bool DylibVerifier::shouldIgnoreReexport(const Record *R,
   return false;
 }
 
+bool DylibVerifier::shouldIgnoreInternalZipperedSymbol(
+    const Record *R, const SymbolContext &SymCtx) const {
+  if (!Zippered)
+    return false;
+
+  return Exports->findSymbol(SymCtx.Kind, SymCtx.SymbolName,
+                             SymCtx.ObjCIFKind) != nullptr;
+}
+
+bool DylibVerifier::shouldIgnoreZipperedAvailability(const Record *R,
+                                                     SymbolContext &SymCtx) {
+  if (!(Zippered && SymCtx.FA->Avail.isUnavailable()))
+    return false;
+
+  // Collect source location incase there is an exported symbol to diagnose
+  // during `verifyRemainingSymbols`.
+  DeferredZipperedSymbols[SymCtx.SymbolName].emplace_back(
+      ZipperedDeclSource{SymCtx.FA, SourceManagers.back().get(), Ctx.Target});
+
+  return true;
+}
+
 bool DylibVerifier::compareObjCInterfaceSymbols(const Record *R,
                                                 SymbolContext &SymCtx,
                                                 const ObjCInterfaceRecord *DR) {
@@ -294,6 +322,9 @@ DylibVerifier::Result DylibVerifier::compareVisibility(const Record *R,
     if (shouldIgnorePrivateExternAttr(SymCtx.FA->D))
       return Result::Ignore;
 
+    if (shouldIgnoreInternalZipperedSymbol(R, SymCtx))
+      return Result::Ignore;
+
     unsigned ID;
     Result Outcome;
     if (Mode == VerificationMode::ErrorsAndWarnings) {
@@ -321,6 +352,9 @@ DylibVerifier::Result DylibVerifier::compareAvailability(const Record *R,
   if (!SymCtx.FA->Avail.isUnavailable())
     return Result::Valid;
 
+  if (shouldIgnoreZipperedAvailability(R, SymCtx))
+    return Result::Ignore;
+
   const bool IsDeclAvailable = SymCtx.FA->Avail.isUnavailable();
 
   switch (Mode) {
@@ -588,13 +622,58 @@ void DylibVerifier::visitSymbolInDylib(const Record &R, SymbolContext &SymCtx) {
     }
   }
 
+  const bool IsLinkerSymbol = SymbolName.starts_with("$ld$");
+
+  if (R.isVerified()) {
+    // Check for unavailable symbols.
+    // This should only occur in the zippered case where we ignored
+    // availability until all headers have been parsed.
+    auto It = DeferredZipperedSymbols.find(SymCtx.SymbolName);
+    if (It == DeferredZipperedSymbols.end()) {
+      updateState(Result::Valid);
+      return;
+    }
+
+    ZipperedDeclSources Locs;
+    for (const ZipperedDeclSource &ZSource : It->second) {
+      if (ZSource.FA->Avail.isObsoleted()) {
+        updateState(Result::Ignore);
+        return;
+      }
+      if (ZSource.T.Arch != Ctx.Target.Arch)
+        continue;
+      Locs.emplace_back(ZSource);
+    }
+    assert(Locs.size() == 2 && "Expected two decls for zippered symbol");
+
+    // Print violating declarations per platform.
+    for (const ZipperedDeclSource &ZSource : Locs) {
+      unsigned DiagID = 0;
+      if (Mode == VerificationMode::Pedantic || IsLinkerSymbol) {
+        updateState(Result::Invalid);
+        DiagID = diag::err_header_availability_mismatch;
+      } else if (Mode == VerificationMode::ErrorsAndWarnings) {
+        updateState(Result::Ignore);
+        DiagID = diag::warn_header_availability_mismatch;
+      } else {
+        updateState(Result::Ignore);
+        return;
+      }
+      // Bypass emitDiag banner and print the target everytime.
+      Ctx.Diag->setSourceManager(ZSource.SrcMgr);
+      Ctx.Diag->Report(diag::warn_target) << getTargetTripleName(ZSource.T);
+      Ctx.Diag->Report(ZSource.FA->Loc, DiagID)
+          << getAnnotatedName(&R, SymCtx) << ZSource.FA->Avail.isUnavailable()
+          << ZSource.FA->Avail.isUnavailable();
+    }
+    return;
+  }
+
   if (shouldIgnoreCpp(SymbolName, R.isWeakDefined())) {
     updateState(Result::Valid);
     return;
   }
 
-  const bool IsLinkerSymbol = SymbolName.starts_with("$ld$");
-
   // All checks at this point classify as some kind of violation.
   // The different verification modes dictate whether they are reported to the
   // user.
@@ -647,8 +726,6 @@ void DylibVerifier::visitSymbolInDylib(const Record &R, SymbolContext &SymCtx) {
 }
 
 void DylibVerifier::visitGlobal(const GlobalRecord &R) {
-  if (R.isVerified())
-    return;
   SymbolContext SymCtx;
   SimpleSymbol Sym = parseSymbol(R.getName());
   SymCtx.SymbolName = Sym.Name;
@@ -658,8 +735,6 @@ void DylibVerifier::visitGlobal(const GlobalRecord &R) {
 
 void DylibVerifier::visitObjCIVar(const ObjCIVarRecord &R,
                                   const StringRef Super) {
-  if (R.isVerified())
-    return;
   SymbolContext SymCtx;
   SymCtx.SymbolName = ObjCIVarRecord::createScopedName(Super, R.getName());
   SymCtx.Kind = EncodeKind::ObjectiveCInstanceVariable;
@@ -679,8 +754,6 @@ void DylibVerifier::accumulateSrcLocForDylibSymbols() {
 }
 
 void DylibVerifier::visitObjCInterface(const ObjCInterfaceRecord &R) {
-  if (R.isVerified())
-    return;
   SymbolContext SymCtx;
   SymCtx.SymbolName = R.getName();
   SymCtx.ObjCIFKind = assignObjCIFSymbolKind(&R);
@@ -713,9 +786,12 @@ DylibVerifier::Result DylibVerifier::verifyRemainingSymbols() {
 
   DWARFContext DWARFInfo;
   DWARFCtx = &DWARFInfo;
-  Ctx.DiscoveredFirstError = false;
-  Ctx.PrintArch = true;
+  Ctx.Target = Target(Architecture::AK_unknown, PlatformType::PLATFORM_UNKNOWN);
   for (std::shared_ptr<RecordsSlice> Slice : Dylib) {
+    if (Ctx.Target.Arch == Slice->getTarget().Arch)
+      continue;
+    Ctx.DiscoveredFirstError = false;
+    Ctx.PrintArch = true;
     Ctx.Target = Slice->getTarget();
     Ctx.DylibSlice = Slice.get();
     Slice->visit(*this);
diff --git a/clang/test/InstallAPI/Inputs/MacOSX13.0.sdk/SDKSettings.json b/clang/test/InstallAPI/Inputs/MacOSX13.0.sdk/SDKSettings.json
new file mode 100644
index 000000000000..258d8288fc6b
--- /dev/null
+++ b/clang/test/InstallAPI/Inputs/MacOSX13.0.sdk/SDKSettings.json
@@ -0,0 +1,19 @@
+{
+  "DefaultVariant": "macos", "DisplayName": "macOS 13",
+  "Version": "13.0",
+  "MaximumDeploymentTarget": "13.0.99",
+  "PropertyConditionFallbackNames": [], "VersionMap": {
+    "iOSMac_macOS": {
+      "16.1": "13.0",
+      "15.0": "12.0",
+      "13.1": "10.15",
+      "14.0": "11.0"
+    },
+    "macOS_iOSMac": {
+      "13.0": "16.1",
+      "12.0": "15.0",
+      "11.0": "14.0",
+      "10.15": "13.1"
+    }
+  }
+}
diff --git a/clang/test/InstallAPI/diagnostics-zippered.test b/clang/test/InstallAPI/diagnostics-zippered.test
new file mode 100644
index 000000000000..59fd0bb673a3
--- /dev/null
+++ b/clang/test/InstallAPI/diagnostics-zippered.test
@@ -0,0 +1,765 @@
+; RUN: rm -rf %t
+; RUN: split-file %s %t
+; RUN: cp -r %S/Inputs/Foundation/Foundation.framework %t/System/Library/Frameworks/
+; RUN: sed -e "s|DSTROOT|%/t|g" %t/inputs.json.in > %t/inputs.json
+; RUN: yaml2obj %t/Mismatch.yaml -o %t/System/Library/Frameworks/Mismatch.framework/Mismatch
+
+; RUN: not clang-installapi --target=arm64-apple-macos13 \
+; RUN: --target=x86_64-apple-macos13 -darwin-target-variant x86_64-apple-ios16.0-macabi \
+; RUN: -darwin-target-variant arm64-apple-ios16.0-macabi \
+; RUN: -F %t/System/Library/Frameworks -isysroot %S/Inputs/MacOSX13.0.sdk \
+; RUN: -install_name /System/Library/Frameworks/Mismatch.framework/Versions/A/Mismatch \
+; RUN: -current_version 1 -compatibility_version 1 %t/inputs.json -o %t/Mismatch.tbd \
+; RUN: --verify-against=%t/System/Library/Frameworks/Mismatch.framework/Mismatch \
+; RUN: --verify-mode=Pedantic 2>&1 | FileCheck %s 
+
+; CHECK:      warning: violations found for x86_64-apple-macos13
+; CHECK:      error: declaration has external linkage, but dynamic library doesn't have symbol 'foo_arch_x86'
+; CHECK-NEXT: extern int foo_arch_x86();
+
+; CHECK:      error: symbol exported in dynamic library, but marked hidden in declaration 'baz'
+; CHECK-NEXT: int baz() __attribute__((visibility("hidden")));
+
+; CHECK-NOT:  inlinedFunc
+
+; CHECK:      error: declaration has external linkage, but symbol has internal linkage in dynamic library 'bar'
+; CHECK-NEXT: extern int bar();
+
+; CHECK:      error: declaration has external linkage, but dynamic library doesn't have symbol 'NSFoo'
+; CHECK-NEXT: @interface NSFoo
+
+; CHECK:      error: declaration has external linkage, but dynamic library doesn't have symbol 'NSFoo.nsint'
+; CHECK-NEXT: int nsint;
+
+; CHECK:      error: declaration has external linkage, but dynamic library doesn't have symbol 'NSFooExcept'
+; CHECK-NEXT: @interface NSFooExcept
+ 
+; CHECK:      warning: violations found for arm64-apple-macos13
+; CHECK:      error: declaration has external linkage, but dynamic library doesn't have symbol 'foo_arch_arm'
+; CHECK-NEXT: extern int foo_arch_arm();
+ 
+; CHECK:      warning: violations found for x86_64 
+; CHECK-NEXT: error: no declaration found for exported symbol '_fooBar' in dynamic library
+ 
+; CHECK:      warning: violations found for arm64 
+; CHECK-NEXT: error: no declaration found for exported symbol '_fooBar' in dynamic library
+; CHECK:      warning: violations found for arm64-apple-macos13
+; CHECK-NEXT: error: declaration 'unavailableSymbol' is marked unavailable, but symbol is exported in dynamic library
+; CHECK-NEXT: extern int unavailableSymbol
+; CHECK:      warning: violations found for arm64-apple-ios16.0-macabi
+; CHECK-NEXT: error: declaration 'unavailableSymbol' is marked unavailable, but symbol is exported in dynamic library
+; CHECK-NEXT: extern int unavailableSymbol
+
+
+;--- System/Library/Frameworks/Mismatch.framework/Headers/Mismatch.h
+#if defined(__x86_64__)
+extern int foo_arch_x86();
+#else
+extern int foo_arch_arm();
+#endif
+
+int foo();
+int baz() __attribute__((visibility("hidden")));
+
+__attribute__((visibility("hidden"))) inline int inlinedFunc() { return 1; }
+
+;--- System/Library/Frameworks/Mismatch.framework/PrivateHeaders/Mismatch_Private.h
+#include <Foundation/Foundation.h>
+extern int bar();
+
+@interface NSFoo : NSObject {
+  int nsint;
+}
+@end
+
+__attribute__((objc_exception))
+@interface NSFooExcept : NSObject
+@end
+
+extern int unavailableSymbol __attribute__((
+    availability(macosx, unavailable), availability(macCatalyst, unavailable)));
+
+
+// Generated from source:
+// *** 
+//     int inlinedFunc() { return 1; }
+//     int foo() { return 1; }
+//     __attribute__((visibility("hidden"))) int bar() { return 1; }
+//     int baz() { return 1; }
+//     int fooBar = 1;
+//     int unavailableSymbol = 1;
+// *** 
+// xcrun -sdk macosx clang -target x86_64-apple-macos13 --target-variant=x86_64-apple-ios16-macabi -dynamiclib
+// xcrun -sdk macosx clang -target arm64-apple-macos13 --target-variant=arm64-apple-ios16-macabi -dynamiclib
+// then lipo'ed together. 
+
+;--- Mismatch.yaml
+--- !fat-mach-o
+FatHeader:
+  magic:           0xCAFEBABE
+  nfat_arch:       2
+FatArchs:
+  - cputype:         0x1000007
+    cpusubtype:      0x3
+    offset:          0x1000
+    size:            20768
+    align:           12
+  - cputype:         0x100000C
+    cpusubtype:      0x0
+    offset:          0x8000
+    size:            49984
+    align:           14
+Slices:
+  - !mach-o
+    FileHeader:
+      magic:           0xFEEDFACF
+      cputype:         0x1000007
+      cpusubtype:      0x3
+      filetype:        0x6
+      ncmds:           15
+      sizeofcmds:      1584
+      flags:           0x100085
+      reserved:        0x0
+    LoadCommands:
+      - cmd:             LC_SEGMENT_64
+        cmdsize:         312
+        segname:         __TEXT
+        vmaddr:          0
+        vmsize:          12288
+        fileoff:         0
+        filesize:        12288
+        maxprot:         5
+        initprot:        5
+        nsects:          3
+        flags:           0
+        Sections:
+          - sectname:        __text
+            segname:         __TEXT
+            addr:            0x1090
+            size:            59
+            offset:          0x1090
+            align:           4
+            reloff:          0x0
+            nreloc:          0
+            flags:           0x80000400
+            reserved1:       0x0
+            reserved2:       0x0
+            reserved3:       0x0
+            content:         554889E5B8010000005DC30F1F440000554889E5B8010000005DC30F1F440000554889E5B8010000005DC30F1F440000554889E5B8010000005DC3
+          - sectname:        __unwind_info
+            segname:         __TEXT
+            addr:            0x10CC
+            size:            4152
+            offset:          0x10CC
+            align:           2
+            reloff:          0x0
+            nreloc:          0
+            flags:           0x0
+            reserved1:       0x0
+            reserved2:       0x0
+            reserved3:       0x0
+            content:         010000001C000000010000002000000000000000200000000200000000000001901000003800000038000000CB1000000000000038000000030000000C0001001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
+          - sectname:        __eh_frame
+            segname:         __TEXT
+            addr:            0x2108
+            size:            24
+            offset:          0x2108
+            align:           3
+            reloff:          0x0
+            nreloc:          0
+            flags:           0x6000000B
+            reserved1:       0x0
+            reserved2:       0x0
+            reserved3:       0x0
+            content:         1400000000000000017A520001781001100C070890010000
+      - cmd:             LC_SEGMENT_64
+        cmdsize:         152
+        segname:         __DATA_CONST
+        vmaddr:          12288
+        vmsize:          4096
+        fileoff:         12288
+        filesize:        4096
+        maxprot:         3
+        initprot:        3
+        nsects:          1
+        flags:           16
+        Sections:
+          - sectname:        __objc_imageinfo
+            segname:         __DATA_CONST
+            addr:            0x3000
+            size:            8
+            offset:          0x3000
+            align:           0
+            reloff:          0x0
+            nreloc:          0
+            flags:           0x0
+            reserved1:       0x0
+            reserved2:       0x0
+            reserved3:       0x0
+            content:         '0000000040000000'
+      - cmd:             LC_SEGMENT_64
+        cmdsize:         152
+        segname:         __DATA
+        vmaddr:          16384
+        vmsize:          4096
+        fileoff:         16384
+        filesize:        4096
+        maxprot:         3
+        initprot:        3
+        nsects:          1
+        flags:           0
+        Sections:
+          - sectname:        __data
+            segname:         __DATA
+            addr:            0x4000
+            size:            8
+            offset:          0x4000
+            align:           2
+            reloff:          0x0
+            nreloc:          0
+            flags:           0x0
+            reserved1:       0x0
+            reserved2:       0x0
+            reserved3:       0x0
+            content:         '0100000001000000'
+      - cmd:             LC_SEGMENT_64
+        cmdsize:         72
+        segname:         __LINKEDIT
+        vmaddr:          20480
+        vmsize:          288
+        fileoff:         20480
+        filesize:        288
+        maxprot:         1
+        initprot:        1
+        nsects:          0
+        flags:           0
+      - cmd:             LC_DYLD_INFO_ONLY
+        cmdsize:         48
+        rebase_off:      0
+        rebase_size:     0
+        bind_off:        0
+        bind_size:       0
+        weak_bind_off:   0
+        weak_bind_size:  0
+        lazy_bind_off:   0
+        lazy_bind_size:  0
+        export_off:      20480
+        export_size:     88
+      - cmd:             LC_SYMTAB
+        cmdsize:         24
+        symoff:          20576
+        nsyms:           7
+        stroff:          20688
+        strsize:         80
+      - cmd:             LC_DYSYMTAB
+        cmdsize:         80
+        ilocalsym:       0
+        nlocalsym:       1
+        iextdefsym:      1
+        nextdefsym:      5
+        iundefsym:       6
+        nundefsym:       1
+        tocoff:          0
+        ntoc:            0
+        modtaboff:       0
+        nmodtab:         0
+        extrefsymoff:    0
+        nextrefsyms:     0
+        indirectsymoff:  0
+        nindirectsyms:   0
+        extreloff:       0
+        nextrel:         0
+        locreloff:       0
+        nlocrel:         0
+      - cmd:             LC_ID_DYLIB
+        cmdsize:         96
+        dylib:
+          name:            24
+          timestamp:       0
+          current_version: 65536
+          compatibility_version: 65536
+        Content:         '/System/Library/Frameworks/Mismatch.framework/Versions/A/Mismatch'
+        ZeroPadBytes:    7
+      - cmd:             LC_UUID
+        cmdsize:         24
+        uuid:            4C4C4421-5555-3144-A16C-E4AA63D66775
+      - cmd:             LC_BUILD_VERSION
+        cmdsize:         32
+        platform:        1
+        minos:           851968
+        sdk:             983040
+        ntools:          1
+        Tools:
+          - tool:            4
+            version:         1245184
+      - cmd:             LC_BUILD_VERSION
+        cmdsize:         32
+        platform:        6
+        minos:           1048576
+        sdk:             1048576
+        ntools:          1
+        Tools:
+          - tool:            4
+            version:         1245184
+      - cmd:             LC_LOAD_DYLIB
+        cmdsize:         56
+        dylib:
+          name:            24
+          timestamp:       0
+          current_version: 14942208
+          compatibility_version: 65536
+        Content:         '/usr/lib/libobjc.A.dylib'
+        ZeroPadBytes:    8
+      - cmd:             LC_LOAD_DYLIB
+        cmdsize:         56
+        dylib:
+          name:            24
+          timestamp:       0
+          current_version: 88539136
+          compatibility_version: 65536
+        Content:         '/usr/lib/libSystem.B.dylib'
+        ZeroPadBytes:    6
+      - cmd:             LC_FUNCTION_STARTS
+        cmdsize:         16
+        dataoff:         20568
+        datasize:        8
+      - cmd:             LC_DATA_IN_CODE
+        cmdsize:         16
+        dataoff:         20576
+        datasize:        0
+    LinkEditData:
+      ExportTrie:
+        TerminalSize:    0
+        NodeOffset:      0
+        Name:            ''
+        Flags:           0x0
+        Address:         0x0
+        Other:           0x0
+        ImportName:      ''
+        Children:
+          - TerminalSize:    0
+            NodeOffset:      5
+            Name:            _
+            Flags:           0x0
+            Address:         0x0
+            Other:           0x0
+            ImportName:      ''
+            Children:
+              - TerminalSize:    3
+                NodeOffset:      49
+                Name:            inlinedFunc
+                Flags:           0x0
+                Address:         0x1090
+                Other:           0x0
+                ImportName:      ''
+              - TerminalSize:    3
+                NodeOffset:      54
+                Name:            foo
+                Flags:           0x0
+                Address:         0x10A0
+                Other:           0x0
+                ImportName:      ''
+                Children:
+                  - TerminalSize:    4
+                    NodeOffset:      64
+                    Name:            Bar
+                    Flags:           0x0
+                    Address:         0x4000
+                    Other:           0x0
+                    ImportName:      ''
+              - TerminalSize:    4
+                NodeOffset:      70
+                Name:            unavailableSymbol
+                Flags:           0x0
+                Address:         0x4004
+                Other:           0x0
+                ImportName:      ''
+              - TerminalSize:    3
+                NodeOffset:      76
+                Name:            baz
+                Flags:           0x0
+                Address:         0x10C0
+                Other:           0x0
+                ImportName:      ''
+      NameList:
+        - n_strx:          20
+          n_type:          0x1E
+          n_sect:          1
+          n_desc:          0
+          n_value:         4272
+        - n_strx:          2
+          n_type:          0xF
+          n_sect:          1
+          n_desc:          0
+          n_value:         4240
+        - n_strx:          15
+          n_type:          0xF
+          n_sect:          1
+          n_desc:          0
+          n_value:         4256
+        - n_strx:          25
+          n_type:          0xF
+          n_sect:          1
+          n_desc:          0
+          n_value:         4288
+        - n_strx:          30
+          n_type:          0xF
+          n_sect:          5
+          n_desc:          0
+          n_value:         16384
+        - n_strx:          38
+          n_type:          0xF
+          n_sect:          5
+          n_desc:          0
+          n_value:         16388
+        - n_strx:          57
+          n_type:          0x1
+          n_sect:          0
+          n_desc:          512
+          n_value:         0
+      StringTable:
+        - ' '
+        - _inlinedFunc
+        - _foo
+        - _bar
+        - _baz
+        - _fooBar
+        - _unavailableSymbol
+        - dyld_stub_binder
+        - ''
+        - ''
+        - ''
+        - ''
+        - ''
+        - ''
+      FunctionStarts:  [ 0x1090, 0x10A0, 0x10B0, 0x10C0 ]
+  - !mach-o
+    FileHeader:
+      magic:           0xFEEDFACF
+      cputype:         0x100000C
+      cpusubtype:      0x0
+      filetype:        0x6
+      ncmds:           16
+      sizeofcmds:      1104
+      flags:           0x100085
+      reserved:        0x0
+    LoadCommands:
+      - cmd:             LC_SEGMENT_64
+        cmdsize:         232
+        segname:         __TEXT
+        vmaddr:          0
+        vmsize:          16384
+        fileoff:         0
+        filesize:        16384
+        maxprot:         5
+        initprot:        5
+        nsects:          2
+        flags:           0
+        Sections:
+          - sectname:        __text
+            segname:         __TEXT
+            addr:            0x1050
+            size:            32
+            offset:          0x1050
+            align:           2
+            reloff:          0x0
+            nreloc:          0
+            flags:           0x80000400
+            reserved1:       0x0
+            reserved2:       0x0
+            reserved3:       0x0
+            content:         20008052C0035FD620008052C0035FD620008052C0035FD620008052C0035FD6
+          - sectname:        __unwind_info
+            segname:         __TEXT
+            addr:            0x1070
+            size:            4152
+            offset:          0x1070
+            align:           2
+            reloff:          0x0
+            nreloc:          0
+            flags:           0x0
+            reserved1:       0x0
+            reserved2:       0x0
+            reserved3:       0x0
+            content:         010000001C000000010000002000000000000000200000000200000000000002501000003800000038000000701000000000000038000000030000000C0001001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
+      - cmd:             LC_SEGMENT_64
+        cmdsize:         152
+        segname:         __DATA_CONST
+        vmaddr:          16384
+        vmsize:          16384
+        fileoff:         16384
+        filesize:        16384
+        maxprot:         3
+        initprot:        3
+        nsects:          1
+        flags:           16
+        Sections:
+          - sectname:        __objc_imageinfo
+            segname:         __DATA_CONST
+            addr:            0x4000
+            size:            8
+            offset:          0x4000
+            align:           0
+            reloff:          0x0
+            nreloc:          0
+            flags:           0x0
+            reserved1:       0x0
+            reserved2:       0x0
+            reserved3:       0x0
+            content:         '0000000040000000'
+      - cmd:             LC_SEGMENT_64
+        cmdsize:         152
+        segname:         __DATA
+        vmaddr:          32768
+        vmsize:          16384
+        fileoff:         32768
+        filesize:        16384
+        maxprot:         3
+        initprot:        3
+        nsects:          1
+        flags:           0
+        Sections:
+          - sectname:        __data
+            segname:         __DATA
+            addr:            0x8000
+            size:            8
+            offset:          0x8000
+            align:           2
+            reloff:          0x0
+            nreloc:          0
+            flags:           0x0
+            reserved1:       0x0
+            reserved2:       0x0
+            reserved3:       0x0
+            content:         '0100000001000000'
+      - cmd:             LC_SEGMENT_64
+        cmdsize:         72
+        segname:         __LINKEDIT
+        vmaddr:          49152
+        vmsize:          832
+        fileoff:         49152
+        filesize:        832
+        maxprot:         1
+        initprot:        1
+        nsects:          0
+        flags:           0
+      - cmd:             LC_DYLD_INFO_ONLY
+        cmdsize:         48
+        rebase_off:      0
+        rebase_size:     0
+        bind_off:        0
+        bind_size:       0
+        weak_bind_off:   0
+        weak_bind_size:  0
+        lazy_bind_off:   0
+        lazy_bind_size:  0
+        export_off:      49152
+        export_size:     88
+      - cmd:             LC_SYMTAB
+        cmdsize:         24
+        symoff:          49248
+        nsyms:           7
+        stroff:          49360
+        strsize:         80
+      - cmd:             LC_DYSYMTAB
+        cmdsize:         80
+        ilocalsym:       0
+        nlocalsym:       1
+        iextdefsym:      1
+        nextdefsym:      5
+        iundefsym:       6
+        nundefsym:       1
+        tocoff:          0
+        ntoc:            0
+        modtaboff:       0
+        nmodtab:         0
+        extrefsymoff:    0
+        nextrefsyms:     0
+        indirectsymoff:  0
+        nindirectsyms:   0
+        extreloff:       0
+        nextrel:         0
+        locreloff:       0
+        nlocrel:         0
+      - cmd:             LC_ID_DYLIB
+        cmdsize:         96
+        dylib:
+          name:            24
+          timestamp:       0
+          current_version: 65536
+          compatibility_version: 65536
+        Content:         '/System/Library/Frameworks/Mismatch.framework/Versions/A/Mismatch'
+        ZeroPadBytes:    7
+      - cmd:             LC_UUID
+        cmdsize:         24
+        uuid:            4C4C4428-5555-3144-A12B-7130A3C946D2
+      - cmd:             LC_BUILD_VERSION
+        cmdsize:         32
+        platform:        1
+        minos:           851968
+        sdk:             983040
+        ntools:          1
+        Tools:
+          - tool:            4
+            version:         1245184
+      - cmd:             LC_BUILD_VERSION
+        cmdsize:         32
+        platform:        6
+        minos:           1048576
+        sdk:             1048576
+        ntools:          1
+        Tools:
+          - tool:            4
+            version:         1245184
+      - cmd:             LC_LOAD_DYLIB
+        cmdsize:         56
+        dylib:
+          name:            24
+          timestamp:       0
+          current_version: 14942208
+          compatibility_version: 65536
+        Content:         '/usr/lib/libobjc.A.dylib'
+        ZeroPadBytes:    8
+      - cmd:             LC_LOAD_DYLIB
+        cmdsize:         56
+        dylib:
+          name:            24
+          timestamp:       0
+          current_version: 88539136
+          compatibility_version: 65536
+        Content:         '/usr/lib/libSystem.B.dylib'
+        ZeroPadBytes:    6
+      - cmd:             LC_FUNCTION_STARTS
+        cmdsize:         16
+        dataoff:         49240
+        datasize:        8
+      - cmd:             LC_DATA_IN_CODE
+        cmdsize:         16
+        dataoff:         49248
+        datasize:        0
+      - cmd:             LC_CODE_SIGNATURE
+        cmdsize:         16
+        dataoff:         49440
+        datasize:        544
+    LinkEditData:
+      ExportTrie:
+        TerminalSize:    0
+        NodeOffset:      0
+        Name:            ''
+        Flags:           0x0
+        Address:         0x0
+        Other:           0x0
+        ImportName:      ''
+        Children:
+          - TerminalSize:    0
+            NodeOffset:      5
+            Name:            _
+            Flags:           0x0
+            Address:         0x0
+            Other:           0x0
+            ImportName:      ''
+            Children:
+              - TerminalSize:    3
+                NodeOffset:      49
+                Name:            inlinedFunc
+                Flags:           0x0
+                Address:         0x1050
+                Other:           0x0
+                ImportName:      ''
+              - TerminalSize:    3
+                NodeOffset:      54
+                Name:            foo
+                Flags:           0x0
+                Address:         0x1058
+                Other:           0x0
+                ImportName:      ''
+                Children:
+                  - TerminalSize:    4
+                    NodeOffset:      64
+                    Name:            Bar
+                    Flags:           0x0
+                    Address:         0x8000
+                    Other:           0x0
+                    ImportName:      ''
+              - TerminalSize:    4
+                NodeOffset:      70
+                Name:            unavailableSymbol
+                Flags:           0x0
+                Address:         0x8004
+                Other:           0x0
+                ImportName:      ''
+              - TerminalSize:    3
+                NodeOffset:      76
+                Name:            baz
+                Flags:           0x0
+                Address:         0x1068
+                Other:           0x0
+                ImportName:      ''
+      NameList:
+        - n_strx:          20
+          n_type:          0x1E
+          n_sect:          1
+          n_desc:          0
+          n_value:         4192
+        - n_strx:          2
+          n_type:          0xF
+          n_sect:          1
+          n_desc:          0
+          n_value:         4176
+        - n_strx:          15
+          n_type:          0xF
+          n_sect:          1
+          n_desc:          0
+          n_value:         4184
+        - n_strx:          25
+          n_type:          0xF
+          n_sect:          1
+          n_desc:          0
+          n_value:         4200
+        - n_strx:          30
+          n_type:          0xF
+          n_sect:          4
+          n_desc:          0
+          n_value:         32768
+        - n_strx:          38
+          n_type:          0xF
+          n_sect:          4
+          n_desc:          0
+          n_value:         32772
+        - n_strx:          57
+          n_type:          0x1
+          n_sect:          0
+          n_desc:          512
+          n_value:         0
+      StringTable:
+        - ' '
+        - _inlinedFunc
+        - _foo
+        - _bar
+        - _baz
+        - _fooBar
+        - _unavailableSymbol
+        - dyld_stub_binder
+        - ''
+        - ''
+        - ''
+        - ''
+        - ''
+        - ''
+      FunctionStarts:  [ 0x1050, 0x1058, 0x1060, 0x1068 ]
+...
+
+;--- inputs.json.in
+{
+  "headers": [ 
+  {
+    "path" : "DSTROOT/System/Library/Frameworks/Mismatch.framework/PrivateHeaders/Mismatch_Private.h",
+    "type" : "private"
+  }, 
+  {
+    "path" : "DSTROOT/System/Library/Frameworks/Mismatch.framework/Headers/Mismatch.h",
+    "type" : "public"
+  }
+  ],
+  "version": "3"
+}
diff --git a/clang/test/InstallAPI/driver-invalid-options.test b/clang/test/InstallAPI/driver-invalid-options.test
index 2b2c551fca20..8efefd1b8c59 100644
--- a/clang/test/InstallAPI/driver-invalid-options.test
+++ b/clang/test/InstallAPI/driver-invalid-options.test
@@ -19,3 +19,10 @@
 // RUN: -isysroot /no/such/path -o tmp.tbd 2> %t 
 // RUN: FileCheck --check-prefix INVALID_ISYSROOT -input-file %t %s 
 // INVALID_ISYSROOT: error: no such sysroot directory: {{.*}}no/such/path'
+
+/// Check invalid target-variant.
+// RUN: not clang-installapi -target x86_64-apple-macosx12 \
+// RUN: -darwin-target-variant arm64-apple-ios15-macabi \
+// RUN: -install_name tmp.dylib -o tmp.tbd 2> %t 
+// RUN: FileCheck --check-prefix INVALID_TARGET_VARIANT -input-file %t %s
+// INVALID_TARGET_VARIANT: error: no matching target found for target variant 'arm64-apple-ios15-macabi'
diff --git a/clang/tools/clang-installapi/Options.cpp b/clang/tools/clang-installapi/Options.cpp
index d1f911812800..120ff3da8998 100644
--- a/clang/tools/clang-installapi/Options.cpp
+++ b/clang/tools/clang-installapi/Options.cpp
@@ -104,7 +104,7 @@ bool Options::processDriverOptions(InputArgList &Args) {
   auto *ArgArch = Args.getLastArgNoClaim(drv::OPT_arch);
   auto *ArgTarget = Args.getLastArgNoClaim(drv::OPT_target);
   auto *ArgTargetVariant =
-      Args.getLastArgNoClaim(drv::OPT_darwin_target_variant_triple);
+      Args.getLastArgNoClaim(drv::OPT_darwin_target_variant);
   if (ArgArch && (ArgTarget || ArgTargetVariant)) {
     Diags->Report(clang::diag::err_drv_argument_not_allowed_with)
         << ArgArch->getAsString(Args)
@@ -135,6 +135,53 @@ bool Options::processDriverOptions(InputArgList &Args) {
     }
   }
 
+  // Capture target variants.
+  DriverOpts.Zippered = ArgTargetVariant != nullptr;
+  for (Arg *A : Args.filtered(drv::OPT_darwin_target_variant)) {
+    A->claim();
+    Triple Variant(A->getValue());
+    if (Variant.getVendor() != Triple::Apple) {
+      Diags->Report(diag::err_unsupported_vendor)
+          << Variant.getVendorName() << A->getAsString(Args);
+      return false;
+    }
+
+    switch (Variant.getOS()) {
+    default:
+      Diags->Report(diag::err_unsupported_os)
+          << Variant.getOSName() << A->getAsString(Args);
+      return false;
+    case Triple::MacOSX:
+    case Triple::IOS:
+      break;
+    }
+
+    switch (Variant.getEnvironment()) {
+    default:
+      Diags->Report(diag::err_unsupported_environment)
+          << Variant.getEnvironmentName() << A->getAsString(Args);
+      return false;
+    case Triple::UnknownEnvironment:
+    case Triple::MacABI:
+      break;
+    }
+
+    Target TAPIVariant(Variant);
+    // See if there is a matching --target option for this --target-variant
+    // option.
+    auto It = find_if(DriverOpts.Targets, [&](const auto &T) {
+      return (T.first.Arch == TAPIVariant.Arch) &&
+             (T.first.Platform != PlatformType::PLATFORM_UNKNOWN);
+    });
+
+    if (It == DriverOpts.Targets.end()) {
+      Diags->Report(diag::err_no_matching_target) << Variant.str();
+      return false;
+    }
+
+    DriverOpts.Targets[TAPIVariant] = Variant;
+  }
+
   DriverOpts.Verbose = Args.hasArgNoClaim(drv::OPT_v);
 
   return true;
@@ -803,7 +850,8 @@ InstallAPIContext Options::createContext() {
 
   Ctx.Verifier = std::make_unique<DylibVerifier>(
       std::move(*Slices), std::move(ReexportedIFs), Diags,
-      DriverOpts.VerifyMode, DriverOpts.Demangle, DriverOpts.DSYMPath);
+      DriverOpts.VerifyMode, DriverOpts.Zippered, DriverOpts.Demangle,
+      DriverOpts.DSYMPath);
   return Ctx;
 }
 
diff --git a/clang/tools/clang-installapi/Options.h b/clang/tools/clang-installapi/Options.h
index 6da1469de2c8..984366c94e91 100644
--- a/clang/tools/clang-installapi/Options.h
+++ b/clang/tools/clang-installapi/Options.h
@@ -75,6 +75,9 @@ struct DriverOptions {
   /// \brief Verification mode for comparing symbols.
   VerificationMode VerifyMode = VerificationMode::Pedantic;
 
+  /// \brief Whether the library is zippered.
+  bool Zippered = false;
+
   /// \brief Print demangled symbols when reporting errors.
   bool Demangle = false;
 
-- 
GitLab


From 9f14914753599f3879e4c273191959e2f1b3632c Mon Sep 17 00:00:00 2001
From: Chelsea Cassanova <chelsea_cassanova@apple.com>
Date: Thu, 11 Apr 2024 20:15:43 -0700
Subject: [PATCH 027/464] Revert "Fix error in unrecognized register name
 handling for "SBFrame.register"" (#88468)

Reverts llvm/llvm-project#88047. TestFrames.py is failing on x86_64
GreenDragon:
https://green.lab.llvm.org/job/llvm.org/view/LLDB/job/lldb-cmake/983/
---
 lldb/bindings/interface/SBFrameExtensions.i  | 12 +-----------
 lldb/test/API/python_api/frame/TestFrames.py | 12 ------------
 2 files changed, 1 insertion(+), 23 deletions(-)

diff --git a/lldb/bindings/interface/SBFrameExtensions.i b/lldb/bindings/interface/SBFrameExtensions.i
index e0472280666a..43b22ed7a6b3 100644
--- a/lldb/bindings/interface/SBFrameExtensions.i
+++ b/lldb/bindings/interface/SBFrameExtensions.i
@@ -44,16 +44,6 @@ STRING_EXTENSION_OUTSIDE(SBFrame)
                 def __init__(self, regs):
                     self.regs = regs
 
-                def __iter__(self):
-                    return self.get_registers()
-
-                def get_registers(self):
-                    for i in range(0,len(self.regs)):
-                        rs = self.regs[i]
-                        for j in range (0,rs.num_children):
-                            reg = rs.GetChildAtIndex(j)
-                            yield reg
-                          
                 def __getitem__(self, key):
                     if type(key) is str:
                         for i in range(0,len(self.regs)):
@@ -62,7 +52,7 @@ STRING_EXTENSION_OUTSIDE(SBFrame)
                                 reg = rs.GetChildAtIndex(j)
                                 if reg.name == key: return reg
                     else:
-                        return SBValue()
+                        return lldb.SBValue()
 
             return registers_access(self.registers)
 
diff --git a/lldb/test/API/python_api/frame/TestFrames.py b/lldb/test/API/python_api/frame/TestFrames.py
index dfa96d51830b..a82b129bc809 100644
--- a/lldb/test/API/python_api/frame/TestFrames.py
+++ b/lldb/test/API/python_api/frame/TestFrames.py
@@ -73,19 +73,7 @@ class FrameAPITestCase(TestBase):
                 gpr_reg_set = lldbutil.get_GPRs(frame)
                 pc_value = gpr_reg_set.GetChildMemberWithName("pc")
                 self.assertTrue(pc_value, "We should have a valid PC.")
-                # Make sure we can also get this from the "register" property:
-                iterator_pc_value = 0
-                found_pc = False
-                for reg in frame.register:
-                    if reg.name == "pc":
-                        found_pc = True
-                        iterator_pc_value = int(reg.GetValue(), 0)
-                        break
-
                 pc_value_int = int(pc_value.GetValue(), 0)
-                self.assertTrue(found_pc, "Found the PC value in the register list")
-                self.assertEqual(iterator_pc_value, pc_value_int, "The methods of finding pc match")
-    
                 # Make sure on arm targets we dont mismatch PC value on the basis of thumb bit.
                 # Frame PC will not have thumb bit set in case of a thumb
                 # instruction as PC.
-- 
GitLab


From c1a44568911b75c98675a3c19a55b4741d861437 Mon Sep 17 00:00:00 2001
From: Chelsea Cassanova <chelsea_cassanova@apple.com>
Date: Thu, 11 Apr 2024 20:21:16 -0700
Subject: [PATCH 028/464] =?UTF-8?q?Reland=20"[lldb][lit]=20Add=20MallocNan?=
 =?UTF-8?q?oZone=20envvar=20to=20Darwin=20ASan=20builds"=20=E2=80=A6=20(#8?=
 =?UTF-8?q?8442)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…(#88436)"

This reverts commit 1f5d130df85c2d0550dc8687ad0fa1d96856c318. The
original commit checks that the host system is "Darwin" before setting
the `MallocNanoZone` envvar, but on the Shell lit config this attribute
does not exist at the point where it is being checked which leads to a
build failure.

This commit checks the host OS correctly.
---
 lldb/test/API/lit.cfg.py   | 1 +
 lldb/test/Shell/lit.cfg.py | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/lldb/test/API/lit.cfg.py b/lldb/test/API/lit.cfg.py
index f9497b632fc5..9ea389c639a0 100644
--- a/lldb/test/API/lit.cfg.py
+++ b/lldb/test/API/lit.cfg.py
@@ -121,6 +121,7 @@ if is_configured("llvm_use_sanitizer"):
             config.environment["DYLD_INSERT_LIBRARIES"] = find_sanitizer_runtime(
                 "libclang_rt.asan_osx_dynamic.dylib"
             )
+            config.environment["MallocNanoZone"] = "0"
 
     if "Thread" in config.llvm_use_sanitizer:
         config.environment["TSAN_OPTIONS"] = "halt_on_error=1"
diff --git a/lldb/test/Shell/lit.cfg.py b/lldb/test/Shell/lit.cfg.py
index 31afe5151c06..290569576ac8 100644
--- a/lldb/test/Shell/lit.cfg.py
+++ b/lldb/test/Shell/lit.cfg.py
@@ -52,6 +52,8 @@ llvm_config.with_system_environment(
 # Enable sanitizer runtime flags.
 config.environment["ASAN_OPTIONS"] = "detect_stack_use_after_return=1"
 config.environment["TSAN_OPTIONS"] = "halt_on_error=1"
+if platform.system() == "Darwin":
+    config.environment["MallocNanoZone"] = "0"
 
 # Support running the test suite under the lldb-repro wrapper. This makes it
 # possible to capture a test suite run and then rerun all the test from the
-- 
GitLab


From b45c9c313c5107b1942cd325e8ab3b4235948a08 Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Fri, 12 Apr 2024 07:44:58 +0400
Subject: [PATCH 029/464] [clang][NFC] Move more functions to `SemaHLSL`
 (#88354)

A follow-up to #87912. I'm moving more HLSL-related functions from
`Sema` to `SemaHLSL`. I'm also dropping `HLSL` from their names in the
process.
---
 clang/include/clang/Sema/Sema.h     |  15 ---
 clang/include/clang/Sema/SemaHLSL.h |  27 +++-
 clang/lib/Parse/ParseHLSL.cpp       |  10 +-
 clang/lib/Sema/SemaDecl.cpp         | 130 +------------------
 clang/lib/Sema/SemaDeclAttr.cpp     |  54 +-------
 clang/lib/Sema/SemaHLSL.cpp         | 186 +++++++++++++++++++++++++++-
 6 files changed, 218 insertions(+), 204 deletions(-)

diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 351b8e1f3113..c4a603cc5a4a 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -2940,13 +2940,6 @@ public:
                                       QualType NewT, QualType OldT);
   void CheckMain(FunctionDecl *FD, const DeclSpec &D);
   void CheckMSVCRTEntryPoint(FunctionDecl *FD);
-  void ActOnHLSLTopLevelFunction(FunctionDecl *FD);
-  void CheckHLSLEntryPoint(FunctionDecl *FD);
-  void CheckHLSLSemanticAnnotation(FunctionDecl *EntryPoint, const Decl *Param,
-                                   const HLSLAnnotationAttr *AnnotationAttr);
-  void DiagnoseHLSLAttrStageMismatch(
-      const Attr *A, HLSLShaderAttr::ShaderType Stage,
-      std::initializer_list<HLSLShaderAttr::ShaderType> AllowedStages);
   Attr *getImplicitCodeSegOrSectionAttrForFunction(const FunctionDecl *FD,
                                                    bool IsDefinition);
   void CheckFunctionOrTemplateParamDeclarator(Scope *S, Declarator &D);
@@ -3708,14 +3701,6 @@ public:
                           StringRef UuidAsWritten, MSGuidDecl *GuidDecl);
 
   BTFDeclTagAttr *mergeBTFDeclTagAttr(Decl *D, const BTFDeclTagAttr &AL);
-  HLSLNumThreadsAttr *mergeHLSLNumThreadsAttr(Decl *D,
-                                              const AttributeCommonInfo &AL,
-                                              int X, int Y, int Z);
-  HLSLShaderAttr *mergeHLSLShaderAttr(Decl *D, const AttributeCommonInfo &AL,
-                                      HLSLShaderAttr::ShaderType ShaderType);
-  HLSLParamModifierAttr *
-  mergeHLSLParamModifierAttr(Decl *D, const AttributeCommonInfo &AL,
-                             HLSLParamModifierAttr::Spelling Spelling);
 
   WebAssemblyImportNameAttr *
   mergeImportNameAttr(Decl *D, const WebAssemblyImportNameAttr &AL);
diff --git a/clang/include/clang/Sema/SemaHLSL.h b/clang/include/clang/Sema/SemaHLSL.h
index acc675963c23..34acaf19517f 100644
--- a/clang/include/clang/Sema/SemaHLSL.h
+++ b/clang/include/clang/Sema/SemaHLSL.h
@@ -13,12 +13,16 @@
 #ifndef LLVM_CLANG_SEMA_SEMAHLSL_H
 #define LLVM_CLANG_SEMA_SEMAHLSL_H
 
+#include "clang/AST/Attr.h"
+#include "clang/AST/Decl.h"
 #include "clang/AST/DeclBase.h"
 #include "clang/AST/Expr.h"
+#include "clang/Basic/AttributeCommonInfo.h"
 #include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/SemaBase.h"
+#include <initializer_list>
 
 namespace clang {
 
@@ -26,10 +30,25 @@ class SemaHLSL : public SemaBase {
 public:
   SemaHLSL(Sema &S);
 
-  Decl *ActOnStartHLSLBuffer(Scope *BufferScope, bool CBuffer,
-                             SourceLocation KwLoc, IdentifierInfo *Ident,
-                             SourceLocation IdentLoc, SourceLocation LBrace);
-  void ActOnFinishHLSLBuffer(Decl *Dcl, SourceLocation RBrace);
+  Decl *ActOnStartBuffer(Scope *BufferScope, bool CBuffer, SourceLocation KwLoc,
+                         IdentifierInfo *Ident, SourceLocation IdentLoc,
+                         SourceLocation LBrace);
+  void ActOnFinishBuffer(Decl *Dcl, SourceLocation RBrace);
+  HLSLNumThreadsAttr *mergeNumThreadsAttr(Decl *D,
+                                          const AttributeCommonInfo &AL, int X,
+                                          int Y, int Z);
+  HLSLShaderAttr *mergeShaderAttr(Decl *D, const AttributeCommonInfo &AL,
+                                  HLSLShaderAttr::ShaderType ShaderType);
+  HLSLParamModifierAttr *
+  mergeParamModifierAttr(Decl *D, const AttributeCommonInfo &AL,
+                         HLSLParamModifierAttr::Spelling Spelling);
+  void ActOnTopLevelFunction(FunctionDecl *FD);
+  void CheckEntryPoint(FunctionDecl *FD);
+  void CheckSemanticAnnotation(FunctionDecl *EntryPoint, const Decl *Param,
+                               const HLSLAnnotationAttr *AnnotationAttr);
+  void DiagnoseAttrStageMismatch(
+      const Attr *A, HLSLShaderAttr::ShaderType Stage,
+      std::initializer_list<HLSLShaderAttr::ShaderType> AllowedStages);
 };
 
 } // namespace clang
diff --git a/clang/lib/Parse/ParseHLSL.cpp b/clang/lib/Parse/ParseHLSL.cpp
index 5afc958600fa..d97985d42369 100644
--- a/clang/lib/Parse/ParseHLSL.cpp
+++ b/clang/lib/Parse/ParseHLSL.cpp
@@ -72,9 +72,9 @@ Decl *Parser::ParseHLSLBuffer(SourceLocation &DeclEnd) {
     return nullptr;
   }
 
-  Decl *D = Actions.HLSL().ActOnStartHLSLBuffer(
-      getCurScope(), IsCBuffer, BufferLoc, Identifier, IdentifierLoc,
-      T.getOpenLocation());
+  Decl *D = Actions.HLSL().ActOnStartBuffer(getCurScope(), IsCBuffer, BufferLoc,
+                                            Identifier, IdentifierLoc,
+                                            T.getOpenLocation());
 
   while (Tok.isNot(tok::r_brace) && Tok.isNot(tok::eof)) {
     // FIXME: support attribute on constants inside cbuffer/tbuffer.
@@ -88,7 +88,7 @@ Decl *Parser::ParseHLSLBuffer(SourceLocation &DeclEnd) {
       T.skipToEnd();
       DeclEnd = T.getCloseLocation();
       BufferScope.Exit();
-      Actions.HLSL().ActOnFinishHLSLBuffer(D, DeclEnd);
+      Actions.HLSL().ActOnFinishBuffer(D, DeclEnd);
       return nullptr;
     }
   }
@@ -96,7 +96,7 @@ Decl *Parser::ParseHLSLBuffer(SourceLocation &DeclEnd) {
   T.consumeClose();
   DeclEnd = T.getCloseLocation();
   BufferScope.Exit();
-  Actions.HLSL().ActOnFinishHLSLBuffer(D, DeclEnd);
+  Actions.HLSL().ActOnFinishBuffer(D, DeclEnd);
 
   Actions.ProcessDeclAttributeList(Actions.CurScope, D, Attrs);
   return D;
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 691e170cd051..afd9e407946a 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -45,6 +45,7 @@
 #include "clang/Sema/ParsedTemplate.h"
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/ScopeInfo.h"
+#include "clang/Sema/SemaHLSL.h"
 #include "clang/Sema/SemaInternal.h"
 #include "clang/Sema/Template.h"
 #include "llvm/ADT/SmallString.h"
@@ -2972,10 +2973,10 @@ static bool mergeDeclAttribute(Sema &S, NamedDecl *D,
   else if (const auto *BTFA = dyn_cast<BTFDeclTagAttr>(Attr))
     NewAttr = S.mergeBTFDeclTagAttr(D, *BTFA);
   else if (const auto *NT = dyn_cast<HLSLNumThreadsAttr>(Attr))
-    NewAttr =
-        S.mergeHLSLNumThreadsAttr(D, *NT, NT->getX(), NT->getY(), NT->getZ());
+    NewAttr = S.HLSL().mergeNumThreadsAttr(D, *NT, NT->getX(), NT->getY(),
+                                           NT->getZ());
   else if (const auto *SA = dyn_cast<HLSLShaderAttr>(Attr))
-    NewAttr = S.mergeHLSLShaderAttr(D, *SA, SA->getType());
+    NewAttr = S.HLSL().mergeShaderAttr(D, *SA, SA->getType());
   else if (isa<SuppressAttr>(Attr))
     // Do nothing. Each redeclaration should be suppressed separately.
     NewAttr = nullptr;
@@ -10808,10 +10809,10 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
   if (getLangOpts().HLSL && D.isFunctionDefinition()) {
     // Any top level function could potentially be specified as an entry.
     if (!NewFD->isInvalidDecl() && S->getDepth() == 0 && Name.isIdentifier())
-      ActOnHLSLTopLevelFunction(NewFD);
+      HLSL().ActOnTopLevelFunction(NewFD);
 
     if (NewFD->hasAttr<HLSLShaderAttr>())
-      CheckHLSLEntryPoint(NewFD);
+      HLSL().CheckEntryPoint(NewFD);
   }
 
   // If this is the first declaration of a library builtin function, add
@@ -12665,125 +12666,6 @@ void Sema::CheckMSVCRTEntryPoint(FunctionDecl *FD) {
   }
 }
 
-void Sema::ActOnHLSLTopLevelFunction(FunctionDecl *FD) {
-  auto &TargetInfo = getASTContext().getTargetInfo();
-
-  if (FD->getName() != TargetInfo.getTargetOpts().HLSLEntry)
-    return;
-
-  StringRef Env = TargetInfo.getTriple().getEnvironmentName();
-  HLSLShaderAttr::ShaderType ShaderType;
-  if (HLSLShaderAttr::ConvertStrToShaderType(Env, ShaderType)) {
-    if (const auto *Shader = FD->getAttr<HLSLShaderAttr>()) {
-      // The entry point is already annotated - check that it matches the
-      // triple.
-      if (Shader->getType() != ShaderType) {
-        Diag(Shader->getLocation(), diag::err_hlsl_entry_shader_attr_mismatch)
-            << Shader;
-        FD->setInvalidDecl();
-      }
-    } else {
-      // Implicitly add the shader attribute if the entry function isn't
-      // explicitly annotated.
-      FD->addAttr(HLSLShaderAttr::CreateImplicit(Context, ShaderType,
-                                                 FD->getBeginLoc()));
-    }
-  } else {
-    switch (TargetInfo.getTriple().getEnvironment()) {
-    case llvm::Triple::UnknownEnvironment:
-    case llvm::Triple::Library:
-      break;
-    default:
-      llvm_unreachable("Unhandled environment in triple");
-    }
-  }
-}
-
-void Sema::CheckHLSLEntryPoint(FunctionDecl *FD) {
-  const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>();
-  assert(ShaderAttr && "Entry point has no shader attribute");
-  HLSLShaderAttr::ShaderType ST = ShaderAttr->getType();
-
-  switch (ST) {
-  case HLSLShaderAttr::Pixel:
-  case HLSLShaderAttr::Vertex:
-  case HLSLShaderAttr::Geometry:
-  case HLSLShaderAttr::Hull:
-  case HLSLShaderAttr::Domain:
-  case HLSLShaderAttr::RayGeneration:
-  case HLSLShaderAttr::Intersection:
-  case HLSLShaderAttr::AnyHit:
-  case HLSLShaderAttr::ClosestHit:
-  case HLSLShaderAttr::Miss:
-  case HLSLShaderAttr::Callable:
-    if (const auto *NT = FD->getAttr<HLSLNumThreadsAttr>()) {
-      DiagnoseHLSLAttrStageMismatch(NT, ST,
-                                    {HLSLShaderAttr::Compute,
-                                     HLSLShaderAttr::Amplification,
-                                     HLSLShaderAttr::Mesh});
-      FD->setInvalidDecl();
-    }
-    break;
-
-  case HLSLShaderAttr::Compute:
-  case HLSLShaderAttr::Amplification:
-  case HLSLShaderAttr::Mesh:
-    if (!FD->hasAttr<HLSLNumThreadsAttr>()) {
-      Diag(FD->getLocation(), diag::err_hlsl_missing_numthreads)
-          << HLSLShaderAttr::ConvertShaderTypeToStr(ST);
-      FD->setInvalidDecl();
-    }
-    break;
-  }
-
-  for (ParmVarDecl *Param : FD->parameters()) {
-    if (const auto *AnnotationAttr = Param->getAttr<HLSLAnnotationAttr>()) {
-      CheckHLSLSemanticAnnotation(FD, Param, AnnotationAttr);
-    } else {
-      // FIXME: Handle struct parameters where annotations are on struct fields.
-      // See: https://github.com/llvm/llvm-project/issues/57875
-      Diag(FD->getLocation(), diag::err_hlsl_missing_semantic_annotation);
-      Diag(Param->getLocation(), diag::note_previous_decl) << Param;
-      FD->setInvalidDecl();
-    }
-  }
-  // FIXME: Verify return type semantic annotation.
-}
-
-void Sema::CheckHLSLSemanticAnnotation(
-    FunctionDecl *EntryPoint, const Decl *Param,
-    const HLSLAnnotationAttr *AnnotationAttr) {
-  auto *ShaderAttr = EntryPoint->getAttr<HLSLShaderAttr>();
-  assert(ShaderAttr && "Entry point has no shader attribute");
-  HLSLShaderAttr::ShaderType ST = ShaderAttr->getType();
-
-  switch (AnnotationAttr->getKind()) {
-  case attr::HLSLSV_DispatchThreadID:
-  case attr::HLSLSV_GroupIndex:
-    if (ST == HLSLShaderAttr::Compute)
-      return;
-    DiagnoseHLSLAttrStageMismatch(AnnotationAttr, ST,
-                                  {HLSLShaderAttr::Compute});
-    break;
-  default:
-    llvm_unreachable("Unknown HLSLAnnotationAttr");
-  }
-}
-
-void Sema::DiagnoseHLSLAttrStageMismatch(
-    const Attr *A, HLSLShaderAttr::ShaderType Stage,
-    std::initializer_list<HLSLShaderAttr::ShaderType> AllowedStages) {
-  SmallVector<StringRef, 8> StageStrings;
-  llvm::transform(AllowedStages, std::back_inserter(StageStrings),
-                  [](HLSLShaderAttr::ShaderType ST) {
-                    return StringRef(
-                        HLSLShaderAttr::ConvertShaderTypeToStr(ST));
-                  });
-  Diag(A->getLoc(), diag::err_hlsl_attr_unsupported_in_stage)
-      << A << HLSLShaderAttr::ConvertShaderTypeToStr(Stage)
-      << (AllowedStages.size() != 1) << join(StageStrings, ", ");
-}
-
 bool Sema::CheckForConstantInitializer(Expr *Init, QualType DclT) {
   // FIXME: Need strict checking.  In C89, we need to check for
   // any assignment, increment, decrement, function-calls, or
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 8bce04640e74..b91064e28e41 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -39,6 +39,7 @@
 #include "clang/Sema/ParsedAttr.h"
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/ScopeInfo.h"
+#include "clang/Sema/SemaHLSL.h"
 #include "clang/Sema/SemaInternal.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringExtras.h"
@@ -7238,24 +7239,11 @@ static void handleHLSLNumThreadsAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
     return;
   }
 
-  HLSLNumThreadsAttr *NewAttr = S.mergeHLSLNumThreadsAttr(D, AL, X, Y, Z);
+  HLSLNumThreadsAttr *NewAttr = S.HLSL().mergeNumThreadsAttr(D, AL, X, Y, Z);
   if (NewAttr)
     D->addAttr(NewAttr);
 }
 
-HLSLNumThreadsAttr *Sema::mergeHLSLNumThreadsAttr(Decl *D,
-                                                  const AttributeCommonInfo &AL,
-                                                  int X, int Y, int Z) {
-  if (HLSLNumThreadsAttr *NT = D->getAttr<HLSLNumThreadsAttr>()) {
-    if (NT->getX() != X || NT->getY() != Y || NT->getZ() != Z) {
-      Diag(NT->getLocation(), diag::err_hlsl_attribute_param_mismatch) << AL;
-      Diag(AL.getLoc(), diag::note_conflicting_attribute);
-    }
-    return nullptr;
-  }
-  return ::new (Context) HLSLNumThreadsAttr(Context, AL, X, Y, Z);
-}
-
 static bool isLegalTypeForHLSLSV_DispatchThreadID(QualType T) {
   if (!T->hasUnsignedIntegerRepresentation())
     return false;
@@ -7299,24 +7287,11 @@ static void handleHLSLShaderAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
 
   // FIXME: check function match the shader stage.
 
-  HLSLShaderAttr *NewAttr = S.mergeHLSLShaderAttr(D, AL, ShaderType);
+  HLSLShaderAttr *NewAttr = S.HLSL().mergeShaderAttr(D, AL, ShaderType);
   if (NewAttr)
     D->addAttr(NewAttr);
 }
 
-HLSLShaderAttr *
-Sema::mergeHLSLShaderAttr(Decl *D, const AttributeCommonInfo &AL,
-                          HLSLShaderAttr::ShaderType ShaderType) {
-  if (HLSLShaderAttr *NT = D->getAttr<HLSLShaderAttr>()) {
-    if (NT->getType() != ShaderType) {
-      Diag(NT->getLocation(), diag::err_hlsl_attribute_param_mismatch) << AL;
-      Diag(AL.getLoc(), diag::note_conflicting_attribute);
-    }
-    return nullptr;
-  }
-  return HLSLShaderAttr::Create(Context, ShaderType, AL);
-}
-
 static void handleHLSLResourceBindingAttr(Sema &S, Decl *D,
                                           const ParsedAttr &AL) {
   StringRef Space = "space0";
@@ -7391,34 +7366,13 @@ static void handleHLSLResourceBindingAttr(Sema &S, Decl *D,
 
 static void handleHLSLParamModifierAttr(Sema &S, Decl *D,
                                         const ParsedAttr &AL) {
-  HLSLParamModifierAttr *NewAttr = S.mergeHLSLParamModifierAttr(
+  HLSLParamModifierAttr *NewAttr = S.HLSL().mergeParamModifierAttr(
       D, AL,
       static_cast<HLSLParamModifierAttr::Spelling>(AL.getSemanticSpelling()));
   if (NewAttr)
     D->addAttr(NewAttr);
 }
 
-HLSLParamModifierAttr *
-Sema::mergeHLSLParamModifierAttr(Decl *D, const AttributeCommonInfo &AL,
-                                 HLSLParamModifierAttr::Spelling Spelling) {
-  // We can only merge an `in` attribute with an `out` attribute. All other
-  // combinations of duplicated attributes are ill-formed.
-  if (HLSLParamModifierAttr *PA = D->getAttr<HLSLParamModifierAttr>()) {
-    if ((PA->isIn() && Spelling == HLSLParamModifierAttr::Keyword_out) ||
-        (PA->isOut() && Spelling == HLSLParamModifierAttr::Keyword_in)) {
-      D->dropAttr<HLSLParamModifierAttr>();
-      SourceRange AdjustedRange = {PA->getLocation(), AL.getRange().getEnd()};
-      return HLSLParamModifierAttr::Create(
-          Context, /*MergedSpelling=*/true, AdjustedRange,
-          HLSLParamModifierAttr::Keyword_inout);
-    }
-    Diag(AL.getLoc(), diag::err_hlsl_duplicate_parameter_modifier) << AL;
-    Diag(PA->getLocation(), diag::note_conflicting_attribute);
-    return nullptr;
-  }
-  return HLSLParamModifierAttr::Create(Context, AL);
-}
-
 static void handleMSInheritanceAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   if (!S.LangOpts.CPlusPlus) {
     S.Diag(AL.getLoc(), diag::err_attribute_not_supported_in_lang)
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index 681849d6e6c8..bb9e37f18d37 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -9,17 +9,25 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Sema/SemaHLSL.h"
+#include "clang/Basic/DiagnosticSema.h"
+#include "clang/Basic/LLVM.h"
+#include "clang/Basic/TargetInfo.h"
 #include "clang/Sema/Sema.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/TargetParser/Triple.h"
+#include <iterator>
 
 using namespace clang;
 
 SemaHLSL::SemaHLSL(Sema &S) : SemaBase(S) {}
 
-Decl *SemaHLSL::ActOnStartHLSLBuffer(Scope *BufferScope, bool CBuffer,
-                                     SourceLocation KwLoc,
-                                     IdentifierInfo *Ident,
-                                     SourceLocation IdentLoc,
-                                     SourceLocation LBrace) {
+Decl *SemaHLSL::ActOnStartBuffer(Scope *BufferScope, bool CBuffer,
+                                 SourceLocation KwLoc, IdentifierInfo *Ident,
+                                 SourceLocation IdentLoc,
+                                 SourceLocation LBrace) {
   // For anonymous namespace, take the location of the left brace.
   DeclContext *LexicalParent = SemaRef.getCurLexicalContext();
   HLSLBufferDecl *Result = HLSLBufferDecl::Create(
@@ -31,8 +39,174 @@ Decl *SemaHLSL::ActOnStartHLSLBuffer(Scope *BufferScope, bool CBuffer,
   return Result;
 }
 
-void SemaHLSL::ActOnFinishHLSLBuffer(Decl *Dcl, SourceLocation RBrace) {
+void SemaHLSL::ActOnFinishBuffer(Decl *Dcl, SourceLocation RBrace) {
   auto *BufDecl = cast<HLSLBufferDecl>(Dcl);
   BufDecl->setRBraceLoc(RBrace);
   SemaRef.PopDeclContext();
 }
+
+HLSLNumThreadsAttr *SemaHLSL::mergeNumThreadsAttr(Decl *D,
+                                                  const AttributeCommonInfo &AL,
+                                                  int X, int Y, int Z) {
+  if (HLSLNumThreadsAttr *NT = D->getAttr<HLSLNumThreadsAttr>()) {
+    if (NT->getX() != X || NT->getY() != Y || NT->getZ() != Z) {
+      Diag(NT->getLocation(), diag::err_hlsl_attribute_param_mismatch) << AL;
+      Diag(AL.getLoc(), diag::note_conflicting_attribute);
+    }
+    return nullptr;
+  }
+  return ::new (getASTContext())
+      HLSLNumThreadsAttr(getASTContext(), AL, X, Y, Z);
+}
+
+HLSLShaderAttr *
+SemaHLSL::mergeShaderAttr(Decl *D, const AttributeCommonInfo &AL,
+                          HLSLShaderAttr::ShaderType ShaderType) {
+  if (HLSLShaderAttr *NT = D->getAttr<HLSLShaderAttr>()) {
+    if (NT->getType() != ShaderType) {
+      Diag(NT->getLocation(), diag::err_hlsl_attribute_param_mismatch) << AL;
+      Diag(AL.getLoc(), diag::note_conflicting_attribute);
+    }
+    return nullptr;
+  }
+  return HLSLShaderAttr::Create(getASTContext(), ShaderType, AL);
+}
+
+HLSLParamModifierAttr *
+SemaHLSL::mergeParamModifierAttr(Decl *D, const AttributeCommonInfo &AL,
+                                 HLSLParamModifierAttr::Spelling Spelling) {
+  // We can only merge an `in` attribute with an `out` attribute. All other
+  // combinations of duplicated attributes are ill-formed.
+  if (HLSLParamModifierAttr *PA = D->getAttr<HLSLParamModifierAttr>()) {
+    if ((PA->isIn() && Spelling == HLSLParamModifierAttr::Keyword_out) ||
+        (PA->isOut() && Spelling == HLSLParamModifierAttr::Keyword_in)) {
+      D->dropAttr<HLSLParamModifierAttr>();
+      SourceRange AdjustedRange = {PA->getLocation(), AL.getRange().getEnd()};
+      return HLSLParamModifierAttr::Create(
+          getASTContext(), /*MergedSpelling=*/true, AdjustedRange,
+          HLSLParamModifierAttr::Keyword_inout);
+    }
+    Diag(AL.getLoc(), diag::err_hlsl_duplicate_parameter_modifier) << AL;
+    Diag(PA->getLocation(), diag::note_conflicting_attribute);
+    return nullptr;
+  }
+  return HLSLParamModifierAttr::Create(getASTContext(), AL);
+}
+
+void SemaHLSL::ActOnTopLevelFunction(FunctionDecl *FD) {
+  auto &TargetInfo = getASTContext().getTargetInfo();
+
+  if (FD->getName() != TargetInfo.getTargetOpts().HLSLEntry)
+    return;
+
+  StringRef Env = TargetInfo.getTriple().getEnvironmentName();
+  HLSLShaderAttr::ShaderType ShaderType;
+  if (HLSLShaderAttr::ConvertStrToShaderType(Env, ShaderType)) {
+    if (const auto *Shader = FD->getAttr<HLSLShaderAttr>()) {
+      // The entry point is already annotated - check that it matches the
+      // triple.
+      if (Shader->getType() != ShaderType) {
+        Diag(Shader->getLocation(), diag::err_hlsl_entry_shader_attr_mismatch)
+            << Shader;
+        FD->setInvalidDecl();
+      }
+    } else {
+      // Implicitly add the shader attribute if the entry function isn't
+      // explicitly annotated.
+      FD->addAttr(HLSLShaderAttr::CreateImplicit(getASTContext(), ShaderType,
+                                                 FD->getBeginLoc()));
+    }
+  } else {
+    switch (TargetInfo.getTriple().getEnvironment()) {
+    case llvm::Triple::UnknownEnvironment:
+    case llvm::Triple::Library:
+      break;
+    default:
+      llvm_unreachable("Unhandled environment in triple");
+    }
+  }
+}
+
+void SemaHLSL::CheckEntryPoint(FunctionDecl *FD) {
+  const auto *ShaderAttr = FD->getAttr<HLSLShaderAttr>();
+  assert(ShaderAttr && "Entry point has no shader attribute");
+  HLSLShaderAttr::ShaderType ST = ShaderAttr->getType();
+
+  switch (ST) {
+  case HLSLShaderAttr::Pixel:
+  case HLSLShaderAttr::Vertex:
+  case HLSLShaderAttr::Geometry:
+  case HLSLShaderAttr::Hull:
+  case HLSLShaderAttr::Domain:
+  case HLSLShaderAttr::RayGeneration:
+  case HLSLShaderAttr::Intersection:
+  case HLSLShaderAttr::AnyHit:
+  case HLSLShaderAttr::ClosestHit:
+  case HLSLShaderAttr::Miss:
+  case HLSLShaderAttr::Callable:
+    if (const auto *NT = FD->getAttr<HLSLNumThreadsAttr>()) {
+      DiagnoseAttrStageMismatch(NT, ST,
+                                {HLSLShaderAttr::Compute,
+                                 HLSLShaderAttr::Amplification,
+                                 HLSLShaderAttr::Mesh});
+      FD->setInvalidDecl();
+    }
+    break;
+
+  case HLSLShaderAttr::Compute:
+  case HLSLShaderAttr::Amplification:
+  case HLSLShaderAttr::Mesh:
+    if (!FD->hasAttr<HLSLNumThreadsAttr>()) {
+      Diag(FD->getLocation(), diag::err_hlsl_missing_numthreads)
+          << HLSLShaderAttr::ConvertShaderTypeToStr(ST);
+      FD->setInvalidDecl();
+    }
+    break;
+  }
+
+  for (ParmVarDecl *Param : FD->parameters()) {
+    if (const auto *AnnotationAttr = Param->getAttr<HLSLAnnotationAttr>()) {
+      CheckSemanticAnnotation(FD, Param, AnnotationAttr);
+    } else {
+      // FIXME: Handle struct parameters where annotations are on struct fields.
+      // See: https://github.com/llvm/llvm-project/issues/57875
+      Diag(FD->getLocation(), diag::err_hlsl_missing_semantic_annotation);
+      Diag(Param->getLocation(), diag::note_previous_decl) << Param;
+      FD->setInvalidDecl();
+    }
+  }
+  // FIXME: Verify return type semantic annotation.
+}
+
+void SemaHLSL::CheckSemanticAnnotation(
+    FunctionDecl *EntryPoint, const Decl *Param,
+    const HLSLAnnotationAttr *AnnotationAttr) {
+  auto *ShaderAttr = EntryPoint->getAttr<HLSLShaderAttr>();
+  assert(ShaderAttr && "Entry point has no shader attribute");
+  HLSLShaderAttr::ShaderType ST = ShaderAttr->getType();
+
+  switch (AnnotationAttr->getKind()) {
+  case attr::HLSLSV_DispatchThreadID:
+  case attr::HLSLSV_GroupIndex:
+    if (ST == HLSLShaderAttr::Compute)
+      return;
+    DiagnoseAttrStageMismatch(AnnotationAttr, ST, {HLSLShaderAttr::Compute});
+    break;
+  default:
+    llvm_unreachable("Unknown HLSLAnnotationAttr");
+  }
+}
+
+void SemaHLSL::DiagnoseAttrStageMismatch(
+    const Attr *A, HLSLShaderAttr::ShaderType Stage,
+    std::initializer_list<HLSLShaderAttr::ShaderType> AllowedStages) {
+  SmallVector<StringRef, 8> StageStrings;
+  llvm::transform(AllowedStages, std::back_inserter(StageStrings),
+                  [](HLSLShaderAttr::ShaderType ST) {
+                    return StringRef(
+                        HLSLShaderAttr::ConvertShaderTypeToStr(ST));
+                  });
+  Diag(A->getLoc(), diag::err_hlsl_attr_unsupported_in_stage)
+      << A << HLSLShaderAttr::ConvertShaderTypeToStr(Stage)
+      << (AllowedStages.size() != 1) << join(StageStrings, ", ");
+}
-- 
GitLab


From 505a9ae81d620b92284f7b2cbe874c936cf19583 Mon Sep 17 00:00:00 2001
From: Sirraide <aeternalmail@gmail.com>
Date: Fri, 12 Apr 2024 06:49:53 +0200
Subject: [PATCH 030/464] [Clang] Look through type sugar when accessing
 FunctionProtoType (#88428)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This fixes a bug introduced by #84473: if a lambda’s type is type sugar
(e.g. an `AttributedType`), we need to use `getAs()` instead of `cast()`
to retrieve the `FunctionProtoType`.
---
 clang/lib/Sema/SemaExpr.cpp                |  2 +-
 clang/lib/Sema/SemaExprCXX.cpp             |  2 +-
 clang/test/SemaCXX/cxx2b-deducing-this.cpp | 12 ++++++++++++
 clang/test/SemaCXX/lambda-expressions.cpp  |  9 +++++++++
 4 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 846e7d5d3ab9..ec5ca2b9352e 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -20704,7 +20704,7 @@ static void FixDependencyOfIdExpressionsInLambdaWithDependentObjectParameter(
       if (MD->getType().isNull())
         continue;
 
-      const auto *Ty = cast<FunctionProtoType>(MD->getType());
+      const auto *Ty = MD->getType()->getAs<FunctionProtoType>();
       if (!Ty || !MD->isExplicitObjectMemberFunction() ||
           !Ty->getParamType(0)->isDependentType())
         continue;
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index 763901d4418d..ac39cb99fc93 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -1488,7 +1488,7 @@ void Sema::MarkThisReferenced(CXXThisExpr *This) {
         if (MD->getType().isNull())
           return false;
 
-        const auto *Ty = cast<FunctionProtoType>(MD->getType());
+        const auto *Ty = MD->getType()->getAs<FunctionProtoType>();
         return Ty && MD->isExplicitObjectMemberFunction() &&
                Ty->getParamType(0)->isDependentType();
       }
diff --git a/clang/test/SemaCXX/cxx2b-deducing-this.cpp b/clang/test/SemaCXX/cxx2b-deducing-this.cpp
index c14c971afd23..5f29a955e053 100644
--- a/clang/test/SemaCXX/cxx2b-deducing-this.cpp
+++ b/clang/test/SemaCXX/cxx2b-deducing-this.cpp
@@ -312,6 +312,18 @@ void TestMutationInLambda() {
       l1();
       l2();
     }
+
+    // Check that we don't crash if the lambda has type sugar.
+    const auto l15 = [=](this auto&&) [[clang::annotate_type("foo")]] [[clang::annotate_type("bar")]] {
+        return x;
+    };
+
+    const auto l16 = [=]() [[clang::annotate_type("foo")]] [[clang::annotate_type("bar")]] {
+        return x;
+    };
+
+    l15();
+    l16();
 }
 
 struct Over_Call_Func_Example {
diff --git a/clang/test/SemaCXX/lambda-expressions.cpp b/clang/test/SemaCXX/lambda-expressions.cpp
index 151d74f21d64..6be338064452 100644
--- a/clang/test/SemaCXX/lambda-expressions.cpp
+++ b/clang/test/SemaCXX/lambda-expressions.cpp
@@ -762,3 +762,12 @@ template auto t::operator()<int>(int a) const; // expected-note {{in instantiati
 
 }
 #endif
+
+namespace GH84473_bug {
+void f1() {
+  int b;
+  (void) [=] [[gnu::regcall]] () { // expected-warning {{an attribute specifier sequence in this position is a C++23 extension}}
+    (void) b;
+  };
+}
+}
-- 
GitLab


From f21ead06750b670cf8ce72d6666e3550b04056a2 Mon Sep 17 00:00:00 2001
From: Chuanqi Xu <yedeng.yd@linux.alibaba.com>
Date: Fri, 12 Apr 2024 12:51:58 +0800
Subject: [PATCH 031/464] [C++20] [Modules] [Reduced BMI] Remove unreachable
 decls GMF in redued BMI (#88359)

Following of https://github.com/llvm/llvm-project/pull/76930

This follows the idea of "only writes what we writes", which I think is
the most natural and efficient way to implement this optimization.

We start writing the BMI from the first declaration in module purview
instead of the global module fragment, so that everything in the GMF
untouched won't be written in the BMI naturally.

The exception is, as I said in
https://github.com/llvm/llvm-project/pull/76930, when we write a
declaration we need to write its decl context, and when we write the
decl context, we need to write everything from it. So when we see
`std::vector`, we basically need to write everything under namespace
std. This violates our intention. To fix this, this patch delays the
writing of namespace in the GMF.

From my local measurement, the size of the BMI decrease to 90M from 112M
for a local modules build. I think this is significant.

This feature will be covered under the experimental reduced BMI so that
it won't affect any existing users. So I'd like to land this when the CI
gets green.

Documents will be added seperately.
---
 .../include/clang/Serialization/ASTBitCodes.h |   4 +
 clang/include/clang/Serialization/ASTReader.h |  14 ++
 clang/include/clang/Serialization/ASTWriter.h |  26 +++-
 clang/lib/Serialization/ASTReader.cpp         |  23 +++
 clang/lib/Serialization/ASTReaderDecl.cpp     |   9 ++
 clang/lib/Serialization/ASTWriter.cpp         | 139 +++++++++++++++---
 clang/lib/Serialization/ASTWriterDecl.cpp     |  27 +++-
 .../module.glob.frag/cxx20-10-4-ex2.cppm      |  58 ++++++++
 .../inconsistent-deduction-guide-linkage.cppm |   8 +
 clang/test/Modules/named-modules-adl-2.cppm   |   7 +-
 clang/test/Modules/named-modules-adl.cppm     |   7 +-
 .../test/Modules/placement-new-reachable.cpp  |   9 +-
 clang/test/Modules/polluted-operator.cppm     |  15 +-
 clang/test/Modules/pr62589.cppm               |   4 +
 clang/test/Modules/preferred_name.cppm        |   2 +
 .../redundant-template-default-arg3.cpp       |  10 ++
 .../template-function-specialization.cpp      |   8 +-
 17 files changed, 337 insertions(+), 33 deletions(-)
 create mode 100644 clang/test/CXX/module/module.glob.frag/cxx20-10-4-ex2.cppm

diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h
index f762116fea95..500098dd3dab 100644
--- a/clang/include/clang/Serialization/ASTBitCodes.h
+++ b/clang/include/clang/Serialization/ASTBitCodes.h
@@ -698,6 +698,10 @@ enum ASTRecordTypes {
   /// Record code for an unterminated \#pragma clang assume_nonnull begin
   /// recorded in a preamble.
   PP_ASSUME_NONNULL_LOC = 67,
+
+  /// Record code for lexical and visible block for delayed namespace in
+  /// reduced BMI.
+  DELAYED_NAMESPACE_LEXICAL_VISIBLE_RECORD = 68,
 };
 
 /// Record types used within a source manager block.
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index e8b9f28690d9..6656c1c58dec 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -517,6 +517,20 @@ private:
   /// in the chain.
   DeclUpdateOffsetsMap DeclUpdateOffsets;
 
+  using DelayedNamespaceOffsetMapTy = llvm::DenseMap<
+      serialization::DeclID,
+      std::pair</*LexicalOffset*/ uint64_t, /*VisibleOffset*/ uint64_t>>;
+
+  /// Mapping from global declaration IDs to the lexical and visible block
+  /// offset for delayed namespace in reduced BMI.
+  ///
+  /// We can't use the existing DeclUpdate mechanism since the DeclUpdate
+  /// may only be applied in an outer most read. However, we need to know
+  /// whether or not a DeclContext has external storage during the recursive
+  /// reading. So we need to apply the offset immediately after we read the
+  /// namespace as if it is not delayed.
+  DelayedNamespaceOffsetMapTy DelayedNamespaceOffsetMap;
+
   struct PendingUpdateRecord {
     Decl *D;
     serialization::GlobalDeclID ID;
diff --git a/clang/include/clang/Serialization/ASTWriter.h b/clang/include/clang/Serialization/ASTWriter.h
index 214eb3601148..443f77031047 100644
--- a/clang/include/clang/Serialization/ASTWriter.h
+++ b/clang/include/clang/Serialization/ASTWriter.h
@@ -201,6 +201,16 @@ private:
   /// The declarations and types to emit.
   std::queue<DeclOrType> DeclTypesToEmit;
 
+  /// The delayed namespace to emit. Only meaningful for reduced BMI.
+  ///
+  /// In reduced BMI, we want to elide the unreachable declarations in
+  /// the global module fragment. However, in ASTWriterDecl, when we see
+  /// a namespace, all the declarations in the namespace would be emitted.
+  /// So the optimization become meaningless. To solve the issue, we
+  /// delay recording all the declarations until we emit all the declarations.
+  /// Then we can safely record the reached declarations only.
+  llvm::SmallVector<NamespaceDecl *, 16> DelayedNamespace;
+
   /// The first ID number we can use for our own declarations.
   serialization::DeclID FirstDeclID = serialization::NUM_PREDEF_DECL_IDS;
 
@@ -529,7 +539,8 @@ private:
   void WriteType(QualType T);
 
   bool isLookupResultExternal(StoredDeclsList &Result, DeclContext *DC);
-  bool isLookupResultEntirelyExternal(StoredDeclsList &Result, DeclContext *DC);
+  bool isLookupResultEntirelyExternalOrUnreachable(StoredDeclsList &Result,
+                                                   DeclContext *DC);
 
   void GenerateNameLookupTable(const DeclContext *DC,
                                llvm::SmallVectorImpl<char> &LookupTable);
@@ -704,6 +715,15 @@ public:
   /// declaration.
   serialization::DeclID getDeclID(const Decl *D);
 
+  /// Whether or not the declaration got emitted. If not, it wouldn't be
+  /// emitted.
+  ///
+  /// This may only be called after we've done the job to write the
+  /// declarations (marked by DoneWritingDeclsAndTypes).
+  ///
+  /// A declaration may only be omitted in reduced BMI.
+  bool wasDeclEmitted(const Decl *D) const;
+
   unsigned getAnonymousDeclarationNumber(const NamedDecl *D);
 
   /// Add a string to the given record.
@@ -798,6 +818,10 @@ public:
     return WritingModule && WritingModule->isNamedModule();
   }
 
+  bool isGeneratingReducedBMI() const { return GeneratingReducedBMI; }
+
+  bool getDoneWritingDeclsAndTypes() const { return DoneWritingDeclsAndTypes; }
+
 private:
   // ASTDeserializationListener implementation
   void ReaderInitialized(ASTReader *Reader) override;
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 4f6987f92fc8..ce96ce2bdbcc 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -3795,6 +3795,29 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
       }
       break;
 
+    case DELAYED_NAMESPACE_LEXICAL_VISIBLE_RECORD: {
+      if (Record.size() % 3 != 0)
+        return llvm::createStringError(
+            std::errc::illegal_byte_sequence,
+            "invalid DELAYED_NAMESPACE_LEXICAL_VISIBLE_RECORD block in AST "
+            "file");
+      for (unsigned I = 0, N = Record.size(); I != N; I += 3) {
+        GlobalDeclID ID = getGlobalDeclID(F, Record[I]);
+
+        uint64_t BaseOffset = F.DeclsBlockStartOffset;
+        assert(BaseOffset && "Invalid DeclsBlockStartOffset for module file!");
+        uint64_t LexicalOffset = Record[I + 1] ? BaseOffset + Record[I + 1] : 0;
+        uint64_t VisibleOffset = Record[I + 2] ? BaseOffset + Record[I + 2] : 0;
+
+        DelayedNamespaceOffsetMap[ID] = {LexicalOffset, VisibleOffset};
+
+        assert(!GetExistingDecl(ID) &&
+               "We shouldn't load the namespace in the front of delayed "
+               "namespace lexical and visible block");
+      }
+      break;
+    }
+
     case OBJC_CATEGORIES_MAP:
       if (F.LocalNumObjCCategoriesInMap != 0)
         return llvm::createStringError(
diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index 5b7b9f19a106..ff72fc30c718 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -4125,6 +4125,15 @@ Decl *ASTReader::ReadDeclRecord(DeclID ID) {
   // offsets for its tables of lexical and visible declarations.
   if (auto *DC = dyn_cast<DeclContext>(D)) {
     std::pair<uint64_t, uint64_t> Offsets = Reader.VisitDeclContext(DC);
+
+    // Get the lexical and visible block for the delayed namespace.
+    // It is sufficient to judge if ID is in DelayedNamespaceOffsetMap.
+    // But it may be more efficient to filter the other cases.
+    if (!Offsets.first && !Offsets.second && isa<NamespaceDecl>(D))
+      if (auto Iter = DelayedNamespaceOffsetMap.find(ID);
+          Iter != DelayedNamespaceOffsetMap.end())
+        Offsets = Iter->second;
+
     if (Offsets.first &&
         ReadLexicalDeclContextStorage(*Loc.F, DeclsCursor, Offsets.first, DC))
       return nullptr;
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 330dbd623350..d9ba10ab6087 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -869,6 +869,7 @@ void ASTWriter::WriteBlockInfoBlock() {
   RECORD(WEAK_UNDECLARED_IDENTIFIERS);
   RECORD(PENDING_IMPLICIT_INSTANTIATIONS);
   RECORD(UPDATE_VISIBLE);
+  RECORD(DELAYED_NAMESPACE_LEXICAL_VISIBLE_RECORD);
   RECORD(DECL_UPDATE_OFFSETS);
   RECORD(DECL_UPDATES);
   RECORD(CUDA_SPECIAL_DECL_REFS);
@@ -3026,10 +3027,12 @@ void ASTWriter::WriteSubmodules(Module *WritingModule) {
       Stream.EmitRecordWithBlob(ConfigMacroAbbrev, Record, CM);
     }
 
-    // Emit the initializers, if any.
+    // Emit the reachable initializers.
+    // The initializer may only be unreachable in reduced BMI.
     RecordData Inits;
     for (Decl *D : Context->getModuleInitializers(Mod))
-      Inits.push_back(GetDeclRef(D));
+      if (wasDeclEmitted(D))
+        Inits.push_back(GetDeclRef(D));
     if (!Inits.empty())
       Stream.EmitRecord(SUBMODULE_INITIALIZERS, Inits);
 
@@ -3208,6 +3211,9 @@ uint64_t ASTWriter::WriteDeclContextLexicalBlock(ASTContext &Context,
   uint64_t Offset = Stream.GetCurrentBitNo();
   SmallVector<uint32_t, 128> KindDeclPairs;
   for (const auto *D : DC->decls()) {
+    if (DoneWritingDeclsAndTypes && !wasDeclEmitted(D))
+      continue;
+
     KindDeclPairs.push_back(D->getKind());
     KindDeclPairs.push_back(GetDeclRef(D));
   }
@@ -3862,8 +3868,14 @@ public:
   data_type getData(const Coll &Decls) {
     unsigned Start = DeclIDs.size();
     for (NamedDecl *D : Decls) {
-      DeclIDs.push_back(
-          Writer.GetDeclRef(getDeclForLocalLookup(Writer.getLangOpts(), D)));
+      NamedDecl *DeclForLocalLookup =
+          getDeclForLocalLookup(Writer.getLangOpts(), D);
+
+      if (Writer.getDoneWritingDeclsAndTypes() &&
+          !Writer.wasDeclEmitted(DeclForLocalLookup))
+        continue;
+
+      DeclIDs.push_back(Writer.GetDeclRef(DeclForLocalLookup));
     }
     return std::make_pair(Start, DeclIDs.size());
   }
@@ -3972,11 +3984,20 @@ bool ASTWriter::isLookupResultExternal(StoredDeclsList &Result,
          DC->hasNeedToReconcileExternalVisibleStorage();
 }
 
-bool ASTWriter::isLookupResultEntirelyExternal(StoredDeclsList &Result,
-                                               DeclContext *DC) {
-  for (auto *D : Result.getLookupResult())
-    if (!getDeclForLocalLookup(getLangOpts(), D)->isFromASTFile())
-      return false;
+bool ASTWriter::isLookupResultEntirelyExternalOrUnreachable(
+    StoredDeclsList &Result, DeclContext *DC) {
+  for (auto *D : Result.getLookupResult()) {
+    auto *LocalD = getDeclForLocalLookup(getLangOpts(), D);
+    if (LocalD->isFromASTFile())
+      continue;
+
+    // We can only be sure whether the local declaration is reachable
+    // after we done writing the declarations and types.
+    if (DoneWritingDeclsAndTypes && !wasDeclEmitted(LocalD))
+      continue;
+
+    return false;
+  }
 
   return true;
 }
@@ -4014,8 +4035,17 @@ ASTWriter::GenerateNameLookupTable(const DeclContext *ConstDC,
     // don't need to write an entry for the name at all. If we can't
     // write out a lookup set without performing more deserialization,
     // just skip this entry.
-    if (isLookupResultExternal(Result, DC) &&
-        isLookupResultEntirelyExternal(Result, DC))
+    //
+    // Also in reduced BMI, we'd like to avoid writing unreachable
+    // declarations in GMF, so we need to avoid writing declarations
+    // that entirely external or unreachable.
+    //
+    // FIMXE: It looks sufficient to test
+    // isLookupResultEntirelyExternalOrUnreachable here. But due to bug we have
+    // to test isLookupResultExternal here. See
+    // https://github.com/llvm/llvm-project/issues/61065 for details.
+    if ((GeneratingReducedBMI || isLookupResultExternal(Result, DC)) &&
+        isLookupResultEntirelyExternalOrUnreachable(Result, DC))
       continue;
 
     // We also skip empty results. If any of the results could be external and
@@ -4206,9 +4236,15 @@ uint64_t ASTWriter::WriteDeclContextVisibleBlock(ASTContext &Context,
         continue;
       }
 
-      for (NamedDecl *ND : Result)
-        if (!ND->isFromASTFile())
-          GetDeclRef(ND);
+      for (NamedDecl *ND : Result) {
+        if (ND->isFromASTFile())
+          continue;
+
+        if (DoneWritingDeclsAndTypes && !wasDeclEmitted(ND))
+          continue;
+
+        GetDeclRef(ND);
+      }
     }
 
     return 0;
@@ -4976,9 +5012,18 @@ ASTFileSignature ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot,
   const TranslationUnitDecl *TU = Context.getTranslationUnitDecl();
 
   // Force all top level declarations to be emitted.
-  for (const auto *D : TU->noload_decls())
-    if (!D->isFromASTFile())
-      GetDeclRef(D);
+  //
+  // We start emitting top level declarations from the module purview to
+  // implement the eliding unreachable declaration feature.
+  for (const auto *D : TU->noload_decls()) {
+    if (D->isFromASTFile())
+      continue;
+
+    if (GeneratingReducedBMI && D->isFromExplicitGlobalModule())
+      continue;
+
+    GetDeclRef(D);
+  }
 
   // If the translation unit has an anonymous namespace, and we don't already
   // have an update block for it, write it as an update block.
@@ -5288,24 +5333,59 @@ void ASTWriter::WriteDeclAndTypes(ASTContext &Context) {
         WriteDecl(Context, DOT.getDecl());
     }
   } while (!DeclUpdates.empty());
-  Stream.ExitBlock();
 
   DoneWritingDeclsAndTypes = true;
 
+  // DelayedNamespace is only meaningful in reduced BMI.
+  // See the comments of DelayedNamespace for details.
+  assert(DelayedNamespace.empty() || GeneratingReducedBMI);
+  RecordData DelayedNamespaceRecord;
+  for (NamespaceDecl *NS : DelayedNamespace) {
+    uint64_t LexicalOffset = WriteDeclContextLexicalBlock(Context, NS);
+    uint64_t VisibleOffset = WriteDeclContextVisibleBlock(Context, NS);
+
+    // Write the offset relative to current block.
+    if (LexicalOffset)
+      LexicalOffset -= DeclTypesBlockStartOffset;
+
+    if (VisibleOffset)
+      VisibleOffset -= DeclTypesBlockStartOffset;
+
+    DelayedNamespaceRecord.push_back(getDeclID(NS));
+    DelayedNamespaceRecord.push_back(LexicalOffset);
+    DelayedNamespaceRecord.push_back(VisibleOffset);
+  }
+
+  // The process of writing lexical and visible block for delayed namespace
+  // shouldn't introduce any new decls, types or update to emit.
+  assert(DeclTypesToEmit.empty());
+  assert(DeclUpdates.empty());
+
+  Stream.ExitBlock();
+
   // These things can only be done once we've written out decls and types.
   WriteTypeDeclOffsets();
   if (!DeclUpdatesOffsetsRecord.empty())
     Stream.EmitRecord(DECL_UPDATE_OFFSETS, DeclUpdatesOffsetsRecord);
 
+  if (!DelayedNamespaceRecord.empty())
+    Stream.EmitRecord(DELAYED_NAMESPACE_LEXICAL_VISIBLE_RECORD,
+                      DelayedNamespaceRecord);
+
   const TranslationUnitDecl *TU = Context.getTranslationUnitDecl();
   // Create a lexical update block containing all of the declarations in the
   // translation unit that do not come from other AST files.
   SmallVector<uint32_t, 128> NewGlobalKindDeclPairs;
   for (const auto *D : TU->noload_decls()) {
-    if (!D->isFromASTFile()) {
-      NewGlobalKindDeclPairs.push_back(D->getKind());
-      NewGlobalKindDeclPairs.push_back(GetDeclRef(D));
-    }
+    if (D->isFromASTFile())
+      continue;
+
+    // In reduced BMI, skip unreached declarations.
+    if (!wasDeclEmitted(D))
+      continue;
+
+    NewGlobalKindDeclPairs.push_back(D->getKind());
+    NewGlobalKindDeclPairs.push_back(GetDeclRef(D));
   }
 
   auto Abv = std::make_shared<llvm::BitCodeAbbrev>();
@@ -5814,6 +5894,21 @@ DeclID ASTWriter::getDeclID(const Decl *D) {
   return DeclIDs[D];
 }
 
+bool ASTWriter::wasDeclEmitted(const Decl *D) const {
+  assert(D);
+
+  assert(DoneWritingDeclsAndTypes &&
+         "wasDeclEmitted should only be called after writing declarations");
+
+  if (D->isFromASTFile())
+    return true;
+
+  bool Emitted = DeclIDs.contains(D);
+  assert((Emitted || GeneratingReducedBMI) &&
+         "The declaration can only be omitted in reduced BMI.");
+  return Emitted;
+}
+
 void ASTWriter::associateDeclWithFile(const Decl *D, DeclID ID) {
   assert(ID);
   assert(D);
diff --git a/clang/lib/Serialization/ASTWriterDecl.cpp b/clang/lib/Serialization/ASTWriterDecl.cpp
index 87e773be54fa..4b417909751f 100644
--- a/clang/lib/Serialization/ASTWriterDecl.cpp
+++ b/clang/lib/Serialization/ASTWriterDecl.cpp
@@ -1719,6 +1719,15 @@ void ASTDeclWriter::VisitClassTemplateDecl(ClassTemplateDecl *D) {
 
   if (D->isFirstDecl())
     AddTemplateSpecializations(D);
+
+  // Force emitting the corresponding deduction guide in reduced BMI mode.
+  // Otherwise, the deduction guide may be optimized out incorrectly.
+  if (Writer.isGeneratingReducedBMI()) {
+    auto Name = Context.DeclarationNames.getCXXDeductionGuideName(D);
+    for (auto *DG : D->getDeclContext()->noload_lookup(Name))
+      Writer.GetDeclRef(DG);
+  }
+
   Code = serialization::DECL_CLASS_TEMPLATE;
 }
 
@@ -1963,8 +1972,22 @@ void ASTDeclWriter::VisitDeclContext(DeclContext *DC) {
                 "You need to update the serializer after you change the "
                 "DeclContextBits");
 
-  Record.AddOffset(Writer.WriteDeclContextLexicalBlock(Context, DC));
-  Record.AddOffset(Writer.WriteDeclContextVisibleBlock(Context, DC));
+  uint64_t LexicalOffset = 0;
+  uint64_t VisibleOffset = 0;
+
+  if (Writer.isGeneratingReducedBMI() && isa<NamespaceDecl>(DC) &&
+      cast<NamespaceDecl>(DC)->isFromExplicitGlobalModule()) {
+    // In reduced BMI, delay writing lexical and visible block for namespace
+    // in the global module fragment. See the comments of DelayedNamespace for
+    // details.
+    Writer.DelayedNamespace.push_back(cast<NamespaceDecl>(DC));
+  } else {
+    LexicalOffset = Writer.WriteDeclContextLexicalBlock(Context, DC);
+    VisibleOffset = Writer.WriteDeclContextVisibleBlock(Context, DC);
+  }
+
+  Record.AddOffset(LexicalOffset);
+  Record.AddOffset(VisibleOffset);
 }
 
 const Decl *ASTWriter::getFirstLocalDecl(const Decl *D) {
diff --git a/clang/test/CXX/module/module.glob.frag/cxx20-10-4-ex2.cppm b/clang/test/CXX/module/module.glob.frag/cxx20-10-4-ex2.cppm
new file mode 100644
index 000000000000..edb208cdf4de
--- /dev/null
+++ b/clang/test/CXX/module/module.glob.frag/cxx20-10-4-ex2.cppm
@@ -0,0 +1,58 @@
+// RUN: rm -rf %t
+// RUN: mkdir %t
+// RUN: split-file %s %t
+
+// RUN: %clang_cc1 -std=c++20 %t/std-10-4-ex2-interface.cppm -emit-reduced-module-interface \
+// RUN:     -o %t/M.pcm -Wno-unused-value
+// RUN: %clang_cc1 -std=c++20 %t/std-10-4-ex2-implementation.cpp -fmodule-file=M=%t/M.pcm \
+// RUN:     -fsyntax-only -verify -Wno-unused-value
+
+//--- std-10-4-ex2.h
+
+namespace N {
+struct X {};
+int d();
+int e();
+inline int f(X, int = d()) { return e(); }
+int g(X);
+int h(X);
+} // namespace N
+
+//--- std-10-4-ex2-interface.cppm
+module;
+#include "std-10-4-ex2.h"
+export module M;
+
+template <typename T> int use_f() {
+  N::X x;           // N::X, N, and  ::  are decl-reachable from use_f
+  return f(x, 123); // N::f is decl-reachable from use_f,
+                    // N::e is indirectly decl-reachable from use_f
+                    //   because it is decl-reachable from N::f, and
+                    // N::d is decl-reachable from use_f
+                    //   because it is decl-reachable from N::f
+                    //   even though it is not used in this call
+}
+
+template <typename T> int use_g() {
+  N::X x;             // N::X, N, and :: are decl-reachable from use_g
+  return g((T(), x)); // N::g is not decl-reachable from use_g
+}
+
+template <typename T> int use_h() {
+  N::X x;             // N::X, N, and :: are decl-reachable from use_h
+  return h((T(), x)); // N::h is not decl-reachable from use_h, but
+                      // N::h is decl-reachable from use_h<int>
+}
+
+int k = use_h<int>();
+// use_h<int> is decl-reachable from k, so
+// N::h is decl-reachable from k
+
+//--- std-10-4-ex2-implementation.cpp
+module M;
+
+int a = use_f<int>();
+int b = use_g<int>();
+// expected-error@std-10-4-ex2-interface.cppm:17 {{use of undeclared identifier 'g'}}
+// expected-note@-2 {{in instantiation of function template specialization 'use_g<int>' requested here}}
+int c = use_h<int>();
diff --git a/clang/test/Modules/inconsistent-deduction-guide-linkage.cppm b/clang/test/Modules/inconsistent-deduction-guide-linkage.cppm
index 3991e47ce215..eaf033d1d928 100644
--- a/clang/test/Modules/inconsistent-deduction-guide-linkage.cppm
+++ b/clang/test/Modules/inconsistent-deduction-guide-linkage.cppm
@@ -26,6 +26,10 @@ module;
 #include "C.h"
 export module B;
 
+namespace foo {
+    export using foo::bar;
+}
+
 //--- C.h
 namespace foo {
   template<class T, class U> struct bar { // expected-error {{declaration of 'bar' in module baz:A follows declaration in the global module}} // expected-note {{previous declaration is here}}
@@ -40,6 +44,10 @@ module;
 #include "E.h"
 export module D;
 
+namespace foo {
+    export using foo::bar;
+}
+
 //--- D-part.cppm
 export module D:part;
 import D;
diff --git a/clang/test/Modules/named-modules-adl-2.cppm b/clang/test/Modules/named-modules-adl-2.cppm
index a14b9a68d74e..88f58d85bc7c 100644
--- a/clang/test/Modules/named-modules-adl-2.cppm
+++ b/clang/test/Modules/named-modules-adl-2.cppm
@@ -7,7 +7,7 @@
 // RUN: %clang_cc1 -std=c++20 %t/c.cppm -fmodule-file=a=%t/a.pcm -fmodule-file=b=%t/b.pcm -fsyntax-only -verify
 
 // RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-reduced-module-interface -o %t/a.pcm
-// RUN: %clang_cc1 -std=c++20 %t/b.cppm -fmodule-file=a=%t/a.pcm -emit-reduced-module-interface -o %t/b.pcm
+// RUN: %clang_cc1 -std=c++20 %t/b.cppm -fmodule-file=a=%t/a.pcm -emit-reduced-module-interface -o %t/b.pcm -DREDUCED
 // RUN: %clang_cc1 -std=c++20 %t/c.cppm -fmodule-file=a=%t/a.pcm -fmodule-file=b=%t/b.pcm -fsyntax-only -verify
 
 //--- a.cppm
@@ -35,6 +35,11 @@ void b() {
 	a(s());
 }
 
+#ifdef REDUCED
+// Mention it to avoid the compiler optimizing it out.
+using ::operator+;
+#endif
+
 //--- c.cppm
 // expected-no-diagnostics
 export module c;
diff --git a/clang/test/Modules/named-modules-adl.cppm b/clang/test/Modules/named-modules-adl.cppm
index ef250023f91e..079d816200b2 100644
--- a/clang/test/Modules/named-modules-adl.cppm
+++ b/clang/test/Modules/named-modules-adl.cppm
@@ -5,7 +5,7 @@
 // RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-module-interface -o %t/a.pcm
 // RUN: %clang_cc1 -std=c++20 %t/b.cppm -fmodule-file=a=%t/a.pcm -fsyntax-only -verify
 
-// RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-reduced-module-interface -o %t/a.pcm
+// RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-reduced-module-interface -o %t/a.pcm -DREDUCED
 // RUN: %clang_cc1 -std=c++20 %t/b.cppm -fmodule-file=a=%t/a.pcm -fsyntax-only -verify
 
 //--- a.h
@@ -28,6 +28,11 @@ void a(T x) {
 	n::s() + x;
 }
 
+#ifdef REDUCED
+// Use it to make sure it is not optimized out in reduced BMI.
+using n::operator+;
+#endif
+
 //--- b.cppm
 // expected-no-diagnostics
 export module b;
diff --git a/clang/test/Modules/placement-new-reachable.cpp b/clang/test/Modules/placement-new-reachable.cpp
index 6b495a60306b..244029470474 100644
--- a/clang/test/Modules/placement-new-reachable.cpp
+++ b/clang/test/Modules/placement-new-reachable.cpp
@@ -31,12 +31,17 @@ private:
     void *ptr;
 };
 
+// The use of operator new in the current module unit is only in the non-inline
+// function definitions. So it may be optimized out.
+using ::operator new;
+
 //--- Use.cpp
 // expected-no-diagnostics
 import A;
 void bar(int *);
 void foo(void *ptr) {
-    A<int>(nullptr); // Good. It should be OK to construct A.
-    void *p = ::operator new(sizeof(int), ptr); // Bad. The function shouldn't be visible here.
+    A<int> a(nullptr); // Good. It should be OK to construct A.
+    B b(nullptr);
+    void *p = ::operator new(sizeof(int), ptr); // Bad. The placement allocation in module A is not visible.
     void *q = new (ptr) int(43); // Good. We don't call the placement allocation function directly.
 }
diff --git a/clang/test/Modules/polluted-operator.cppm b/clang/test/Modules/polluted-operator.cppm
index 2179fa098064..45cc5e37d6a6 100644
--- a/clang/test/Modules/polluted-operator.cppm
+++ b/clang/test/Modules/polluted-operator.cppm
@@ -12,7 +12,8 @@
 // RUN:   -emit-module-interface -DSKIP_ODR_CHECK_IN_GMF -o %t/b.pcm -verify
 
 // RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/a.cppm -o %t/a.pcm
-// RUN: %clang_cc1 -std=c++20 %t/b.cppm -fprebuilt-module-path=%t -emit-reduced-module-interface -o %t/b.pcm -verify
+// RUN: %clang_cc1 -std=c++20 %t/b.cppm -fprebuilt-module-path=%t -emit-reduced-module-interface \
+// RUN:     -o %t/b.pcm -verify -DREDUCED
 
 //--- foo.h
 
@@ -53,18 +54,26 @@ module;
 #include "foo.h"
 #include "bar.h"
 export module a;
+export namespace std {
+  using std::variant;
+  using std::_Traits;
+  using std::operator&&;
+}
 
 //--- b.cppm
 module;
 #include "bar.h"
 export module b;
 import a;
+export namespace std {
+  using std::variant;
+  using std::_Traits;
+  using std::operator&&;
+}
 
 #ifdef SKIP_ODR_CHECK_IN_GMF
 // expected-no-diagnostics
 #else
 // expected-error@* {{has different definitions in different modules; first difference is defined here found data member '_S_copy_ctor' with an initializer}}
 // expected-note@* {{but in 'a.<global>' found data member '_S_copy_ctor' with a different initializer}}
-// expected-error@* {{from module 'a.<global>' is not present in definition of 'variant<_Types...>' provided earlier}}
-// expected-note@* {{declaration of 'swap' does not match}}
 #endif
diff --git a/clang/test/Modules/pr62589.cppm b/clang/test/Modules/pr62589.cppm
index c5aec3ed8184..54f2ecef22e1 100644
--- a/clang/test/Modules/pr62589.cppm
+++ b/clang/test/Modules/pr62589.cppm
@@ -73,6 +73,10 @@ export module a;
 export using ::a;
 export using ::a_view;
 
+// We need to mention the 'operator==' explicitly to make sure it won't be
+// discarded.
+export using ::operator==;
+
 //--- b.cpp
 // expected-no-diagnostics
 import a;
diff --git a/clang/test/Modules/preferred_name.cppm b/clang/test/Modules/preferred_name.cppm
index 2f1705867845..806781a81c5c 100644
--- a/clang/test/Modules/preferred_name.cppm
+++ b/clang/test/Modules/preferred_name.cppm
@@ -42,6 +42,7 @@ inline foo_templ<char> bar()
 module;
 #include "foo.h"
 export module A;
+export using ::foo_templ;
 
 //--- Use.cppm
 // expected-no-diagnostics
@@ -49,6 +50,7 @@ module;
 #include "foo.h"
 export module Use;
 import A;
+export using ::foo_templ;
 
 //--- Use1.cpp
 import A;         // expected-warning@foo.h:8 {{attribute declaration must precede definition}}
diff --git a/clang/test/Modules/redundant-template-default-arg3.cpp b/clang/test/Modules/redundant-template-default-arg3.cpp
index e4464c40e976..da6fe01a5357 100644
--- a/clang/test/Modules/redundant-template-default-arg3.cpp
+++ b/clang/test/Modules/redundant-template-default-arg3.cpp
@@ -91,6 +91,16 @@ int v9;
 module;
 #include "foo.h"
 export module foo;
+export using ::v;
+export using ::v2;
+export using ::my_array;
+export using ::v3;
+export using ::v4;
+export using ::v5;
+export using ::v6;
+export using ::v7;
+export using ::v8;
+export using ::v9;
 
 //--- use.cpp
 import foo;
diff --git a/clang/test/Modules/template-function-specialization.cpp b/clang/test/Modules/template-function-specialization.cpp
index 1b6bf2de6ba1..d5d7d7e81239 100644
--- a/clang/test/Modules/template-function-specialization.cpp
+++ b/clang/test/Modules/template-function-specialization.cpp
@@ -6,7 +6,7 @@
 // RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -verify -fsyntax-only
 
 // RUN: %clang_cc1 -std=c++20 -emit-reduced-module-interface %t/foo.cppm -o %t/foo.pcm
-// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -verify -fsyntax-only
+// RUN: %clang_cc1 -std=c++20 -fprebuilt-module-path=%t %t/Use.cpp -verify -fsyntax-only -DREDUCED
 
 //--- foo.cppm
 module;
@@ -51,10 +51,16 @@ import foo;
 void use() {
   foo<short>();
   foo<int>();
+#ifdef REDUCED
+  // In reduced BMI, the foo2 template function is optimized out.
+  foo2<short>(); // expected-error {{use of undeclared identifier 'foo2'}}
+  foo2<int>();   // expected-error {{use of undeclared identifier 'foo2'}}
+#else
   foo2<short>(); // expected-error {{missing '#include'; 'foo2' must be declared before it is used}}
                  // expected-note@* {{declaration here is not visible}}
   foo2<int>();   // expected-error {{missing '#include'; 'foo2' must be declared before it is used}}
                  // expected-note@* {{declaration here is not visible}}
+#endif
   foo3<short>();
   foo3<int>();
 
-- 
GitLab


From 506ff547924ab81123865964b7d998a5a378330a Mon Sep 17 00:00:00 2001
From: NimishMishra <42909663+NimishMishra@users.noreply.github.com>
Date: Thu, 11 Apr 2024 22:12:14 -0700
Subject: [PATCH 032/464] [flang][Semantics] Fix updating flags of
 threadprivate symbols in presence of default clause (#78283)

Current semantic checks of default clause incorrectly update symbol
flags related to threadprivate symbols. This patch adds an additional
check to skip such updation should a symbol be already declared
threadprivate.

Fixes https://github.com/llvm/llvm-project/issues/78282
---
 flang/lib/Semantics/resolve-directives.cpp |  5 ++++
 flang/test/Lower/OpenMP/default-clause.f90 | 29 ++++++++++++++++++++++
 2 files changed, 34 insertions(+)

diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 27af192f606b..f4ac7f198d85 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -2021,6 +2021,11 @@ void OmpAttributeVisitor::Post(const parser::Name &name) {
         }
       }
     }
+
+    if (Symbol * found{currScope().FindSymbol(name.source)}) {
+      if (found->test(semantics::Symbol::Flag::OmpThreadprivate))
+        return;
+    }
     std::vector<Symbol *> defaultDSASymbols;
     for (int dirDepth{0}; dirDepth < (int)dirContext_.size(); ++dirDepth) {
       DirContext &dirContext = dirContext_[dirDepth];
diff --git a/flang/test/Lower/OpenMP/default-clause.f90 b/flang/test/Lower/OpenMP/default-clause.f90
index f86b51aef4e0..9a47e561338f 100644
--- a/flang/test/Lower/OpenMP/default-clause.f90
+++ b/flang/test/Lower/OpenMP/default-clause.f90
@@ -383,3 +383,32 @@ subroutine skipped_default_clause_checks()
           iii=it(11)
        !$omp end parallel
 end subroutine
+
+!CHECK: func.func @_QPthreadprivate_with_default() {
+!CHECK: %[[VAR_I:.*]] = fir.alloca i32 {bindc_name = "i", uniq_name = "_QFthreadprivate_with_defaultEi"}
+!CHECK: %[[VAR_I_DECLARE:.*]] = hlfir.declare %[[VAR_I]] {uniq_name = "_QFthreadprivate_with_defaultEi"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK: %[[BLK_ADDR:.*]] = fir.address_of(@blk_) : !fir.ref<!fir.array<4xi8>>
+!CHECK: %[[BLK_THREADPRIVATE_OUTER:.*]] = omp.threadprivate %[[BLK_ADDR]] : !fir.ref<!fir.array<4xi8>> -> !fir.ref<!fir.array<4xi8>>
+!CHECK: %[[CONVERT:.*]] = fir.convert %[[BLK_THREADPRIVATE_OUTER]] : (!fir.ref<!fir.array<4xi8>>) -> !fir.ref<!fir.array<?xi8>>
+!CHECK: %[[VAR_C:.*]] = arith.constant 0 : index
+!CHECK: %[[BLK_REF:.*]] = fir.coordinate_of %[[CONVERT]], %[[VAR_C]] : (!fir.ref<!fir.array<?xi8>>, index) -> !fir.ref<i8>
+!CHECK: %[[CONVERT:.*]] = fir.convert %[[BLK_REF]] : (!fir.ref<i8>) -> !fir.ref<i32>
+!CHECK: %[[VAR_X_DECLARE:.*]] = hlfir.declare %[[CONVERT]] {uniq_name = "_QFthreadprivate_with_defaultEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+!CHECK: omp.parallel {
+!CHECK:   %[[BLK_THREADPRIVATE_INNER:.*]] = omp.threadprivate %[[BLK_ADDR]] : !fir.ref<!fir.array<4xi8>> -> !fir.ref<!fir.array<4xi8>>
+!CHECK:   %[[CONVERT_INNER:.*]] = fir.convert %[[BLK_THREADPRIVATE_INNER]] : (!fir.ref<!fir.array<4xi8>>) -> !fir.ref<!fir.array<?xi8>>
+!CHECK:   %[[VAR_C_INNER:.*]] = arith.constant 0 : index
+!CHECK:   %[[BLK_REF_INNER:.*]] = fir.coordinate_of %[[CONVERT_INNER]], %[[VAR_C_INNER]] : (!fir.ref<!fir.array<?xi8>>, index) -> !fir.ref<i8>
+!CHECK:   %[[CONVERT_INNER:.*]] = fir.convert %[[BLK_REF_INNER]] : (!fir.ref<i8>) -> !fir.ref<i32>
+!CHECK:   %[[VAR_X_DECLARE_INNER:.*]] = hlfir.declare %[[CONVERT_INNER]] {uniq_name = "_QFthreadprivate_with_defaultEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+subroutine threadprivate_with_default
+        integer :: x
+        common /blk/ x 
+        !$omp threadprivate (/blk/)
+
+        !$omp parallel do default(private)
+           do i = 1, 4
+              x = 4
+           end do
+        !$omp end parallel do
+end subroutine
-- 
GitLab


From b502dc596ba3ed1e857e163e21ed6426125760ef Mon Sep 17 00:00:00 2001
From: Sirraide <aeternalmail@gmail.com>
Date: Fri, 12 Apr 2024 07:12:57 +0200
Subject: [PATCH 033/464] [Clang] Fix test broken due to unsupported calling
 convention (#88472)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

#88428 ended up breaking CI because it included a test that uses the
`regcall` calling convention, which isn’t supported on all targets; I’ve
moved it into a separate file that sets the triple.
---
 clang/test/SemaCXX/gh84473.cpp            | 11 +++++++++++
 clang/test/SemaCXX/lambda-expressions.cpp |  9 ---------
 2 files changed, 11 insertions(+), 9 deletions(-)
 create mode 100644 clang/test/SemaCXX/gh84473.cpp

diff --git a/clang/test/SemaCXX/gh84473.cpp b/clang/test/SemaCXX/gh84473.cpp
new file mode 100644
index 000000000000..72f5e9c45b0c
--- /dev/null
+++ b/clang/test/SemaCXX/gh84473.cpp
@@ -0,0 +1,11 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -std=c++23 -fsyntax-only -verify %s
+// expected-no-diagnostics
+
+namespace GH84473_bug {
+void f1() {
+  int b;
+  (void) [=] [[gnu::regcall]] () {
+    (void) b;
+  };
+}
+}
diff --git a/clang/test/SemaCXX/lambda-expressions.cpp b/clang/test/SemaCXX/lambda-expressions.cpp
index 6be338064452..151d74f21d64 100644
--- a/clang/test/SemaCXX/lambda-expressions.cpp
+++ b/clang/test/SemaCXX/lambda-expressions.cpp
@@ -762,12 +762,3 @@ template auto t::operator()<int>(int a) const; // expected-note {{in instantiati
 
 }
 #endif
-
-namespace GH84473_bug {
-void f1() {
-  int b;
-  (void) [=] [[gnu::regcall]] () { // expected-warning {{an attribute specifier sequence in this position is a C++23 extension}}
-    (void) b;
-  };
-}
-}
-- 
GitLab


From 469758c944dfb492102ded9e50eac5e478f85f6a Mon Sep 17 00:00:00 2001
From: Dominik Adamski <dominik.adamski@amd.com>
Date: Fri, 12 Apr 2024 09:11:26 +0200
Subject: [PATCH 034/464] [Flang] Add options frtlib-add-rpath and resource-dir
 (#88280)

frtlib-add-rpath option is used by clang and should also be visible in
flang. It is already handled by the toolchains used by both clang and
flang.

Reported issue: https://github.com/llvm/llvm-project/issues/82553
---
 clang/include/clang/Driver/Options.td         |  6 ++--
 clang/lib/Driver/ToolChains/Flang.cpp         |  4 +++
 .../Driver/arch-specific-libdir-rpath.f95     | 32 +++++++++++++++++++
 flang/test/Driver/driver-help-hidden.f90      |  3 ++
 flang/test/Driver/driver-help.f90             |  2 ++
 5 files changed, 45 insertions(+), 2 deletions(-)
 create mode 100644 flang/test/Driver/arch-specific-libdir-rpath.f95

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 7ac36222644a..400f72f2250a 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -5464,21 +5464,23 @@ def rdynamic : Flag<["-"], "rdynamic">, Group<Link_Group>,
   Visibility<[ClangOption, FlangOption]>;
 def resource_dir : Separate<["-"], "resource-dir">,
   Flags<[NoXarchOption, HelpHidden]>,
-  Visibility<[ClangOption, CC1Option, CLOption, DXCOption]>,
+  Visibility<[ClangOption, CC1Option, CLOption, DXCOption, FlangOption, FC1Option]>,
   HelpText<"The directory which holds the compiler resource files">,
   MarshallingInfoString<HeaderSearchOpts<"ResourceDir">>;
 def resource_dir_EQ : Joined<["-"], "resource-dir=">, Flags<[NoXarchOption]>,
-  Visibility<[ClangOption, CLOption, DXCOption]>,
+  Visibility<[ClangOption, CLOption, DXCOption, FlangOption]>,
   Alias<resource_dir>;
 def rpath : Separate<["-"], "rpath">, Flags<[LinkerInput]>, Group<Link_Group>,
   Visibility<[ClangOption, CLOption, DXCOption, FlangOption]>;
 def rtlib_EQ : Joined<["-", "--"], "rtlib=">, Visibility<[ClangOption, CLOption]>,
   HelpText<"Compiler runtime library to use">;
 def frtlib_add_rpath: Flag<["-"], "frtlib-add-rpath">, Flags<[NoArgumentUnused]>,
+  Visibility<[ClangOption, FlangOption]>,
   HelpText<"Add -rpath with architecture-specific resource directory to the linker flags. "
   "When --hip-link is specified, also add -rpath with HIP runtime library directory to the linker flags">;
 def fno_rtlib_add_rpath: Flag<["-"], "fno-rtlib-add-rpath">,
   Flags<[NoArgumentUnused]>,
+  Visibility<[ClangOption, FlangOption]>,
   HelpText<"Do not add -rpath with architecture-specific resource directory to the linker flags. "
   "When --hip-link is specified, do not add -rpath with HIP runtime library directory to the linker flags">;
 def offload_add_rpath: Flag<["--"], "offload-add-rpath">,
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index b00068c8098b..b46bac24503c 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -786,6 +786,10 @@ void Flang::ConstructJob(Compilation &C, const JobAction &JA,
     }
   }
 
+  // Pass the path to compiler resource files.
+  CmdArgs.push_back("-resource-dir");
+  CmdArgs.push_back(D.ResourceDir.c_str());
+
   // Offloading related options
   addOffloadOptions(C, Inputs, JA, Args, CmdArgs);
 
diff --git a/flang/test/Driver/arch-specific-libdir-rpath.f95 b/flang/test/Driver/arch-specific-libdir-rpath.f95
new file mode 100644
index 000000000000..cc09938f7d1e
--- /dev/null
+++ b/flang/test/Driver/arch-specific-libdir-rpath.f95
@@ -0,0 +1,32 @@
+! REQUIRES: x86-registered-target
+! Test that the driver adds an arch-specific subdirectory in
+! {RESOURCE_DIR}/lib/linux to the linker search path and to '-rpath'
+!
+! Test the default behavior when neither -frtlib-add-rpath nor
+! -fno-rtlib-add-rpath is specified, which is to skip -rpath
+! RUN: %flang %s -### --target=x86_64-linux \
+! RUN:     -resource-dir=%S/../../../clang/test/Driver/Inputs/resource_dir_with_arch_subdir 2>&1 \
+! RUN:   | FileCheck --check-prefixes=RESDIR,LIBPATH-X86_64,NO-RPATH-X86_64 %s
+!
+! Test that -rpath is not added under -fno-rtlib-add-rpath
+! RUN: %flang %s -### --target=x86_64-linux \
+! RUN:     -resource-dir=%S/../../../clang/test/Driver/Inputs/resource_dir_with_arch_subdir \
+! RUN:     -fno-rtlib-add-rpath 2>&1 \
+! RUN:   | FileCheck --check-prefixes=RESDIR,LIBPATH-X86_64,NO-RPATH-X86_64 %s
+!
+! Test that -rpath is added
+!
+! Add LIBPATH, RPATH for OpenMP
+!
+! RUN: %flang %s -### --target=x86_64-linux -fopenmp \
+! RUN:     -resource-dir=%S/../../../clang/test/Driver/Inputs/resource_dir_with_arch_subdir \
+! RUN:     -frtlib-add-rpath 2>&1 \
+! RUN:   | FileCheck --check-prefixes=RESDIR,LIBPATH-X86_64,RPATH-X86_64 %s
+!
+!
+! RESDIR: "-resource-dir" "[[RESDIR:[^"]*]]"
+!
+! LIBPATH-X86_64: -L[[RESDIR]]{{(/|\\\\)lib(/|\\\\)linux(/|\\\\)x86_64}}
+! RPATH-X86_64:   "-rpath" "[[RESDIR]]{{(/|\\\\)lib(/|\\\\)linux(/|\\\\)x86_64}}"
+!
+! NO-RPATH-X86_64-NOT:   "-rpath" "[[RESDIR]]{{(/|\\\\)lib(/|\\\\)linux(/|\\\\)x86_64}}"
diff --git a/flang/test/Driver/driver-help-hidden.f90 b/flang/test/Driver/driver-help-hidden.f90
index 35b188ad3a9e..eb00949c2a95 100644
--- a/flang/test/Driver/driver-help-hidden.f90
+++ b/flang/test/Driver/driver-help-hidden.f90
@@ -68,6 +68,7 @@
 ! CHECK-NEXT: -fno-lto                Disable LTO mode (default)
 ! CHECK-NEXT: -fno-ppc-native-vector-element-order
 ! CHECK-NEXT:                         Specifies PowerPC non-native vector element order
+! CHECK-NEXT: -fno-rtlib-add-rpath Do not add -rpath with architecture-specific resource directory to the linker flags. When --hip-link is specified, do not add -rpath with HIP runtime library directory to the linker flags
 ! CHECK-NEXT: -fno-signed-zeros       Allow optimizations that ignore the sign of floating point zeros
 ! CHECK-NEXT: -fno-stack-arrays       Allocate array temporaries on the heap (default)
 ! CHECK-NEXT: -fno-version-loops-for-stride
@@ -93,6 +94,7 @@
 ! CHECK-NEXT:                         Specifies PowerPC native vector element order (default)
 ! CHECK-NEXT: -freciprocal-math       Allow division operations to be reassociated
 ! CHECK-NEXT: -fropi                  Generate read-only position independent code (ARM only)
+! CHECK-NEXT: -frtlib-add-rpath Add -rpath with architecture-specific resource directory to the linker flags. When --hip-link is specified, also add -rpath with HIP runtime library directory to the linker flags
 ! CHECK-NEXT: -frwpi                  Generate read-write position independent code (ARM only)
 ! CHECK-NEXT: -fsave-optimization-record=<format>
 ! CHECK-NEXT:                         Generate an optimization record file in a specific format
@@ -145,6 +147,7 @@
 ! CHECK-NEXT: -print-target-triple    Print the normalized target triple
 ! CHECK-NEXT: -pthread                Support POSIX threads in generated code
 ! CHECK-NEXT: -P                      Disable linemarker output in -E mode
+! CHECK-NEXT: -resource-dir <value>   The directory which holds the compiler resource files
 ! CHECK-NEXT: -Rpass-analysis=<value> Report transformation analysis from optimization passes whose name matches the given POSIX regular expression
 ! CHECK-NEXT: -Rpass-missed=<value>   Report missed transformations by optimization passes whose name matches the given POSIX regular expression
 ! CHECK-NEXT: -Rpass=<value>          Report transformations performed by optimization passes whose name matches the given POSIX regular expression
diff --git a/flang/test/Driver/driver-help.f90 b/flang/test/Driver/driver-help.f90
index d4dab55f40e8..05f50ea348c2 100644
--- a/flang/test/Driver/driver-help.f90
+++ b/flang/test/Driver/driver-help.f90
@@ -60,6 +60,7 @@
 ! HELP-NEXT: -fno-lto                Disable LTO mode (default)
 ! HELP-NEXT: -fno-ppc-native-vector-element-order
 ! HELP-NEXT:                         Specifies PowerPC non-native vector element order
+! HELP-NEXT:  -fno-rtlib-add-rpath Do not add -rpath with architecture-specific resource directory to the linker flags. When --hip-link is specified, do not add -rpath with HIP runtime library directory to the linker flags
 ! HELP-NEXT: -fno-signed-zeros       Allow optimizations that ignore the sign of floating point zeros
 ! HELP-NEXT: -fno-stack-arrays       Allocate array temporaries on the heap (default)
 ! HELP-NEXT: -fno-version-loops-for-stride
@@ -81,6 +82,7 @@
 ! HELP-NEXT:                         Specifies PowerPC native vector element order (default)
 ! HELP-NEXT: -freciprocal-math       Allow division operations to be reassociated
 ! HELP-NEXT: -fropi                  Generate read-only position independent code (ARM only)
+! HELP-NEXT: -frtlib-add-rpath Add -rpath with architecture-specific resource directory to the linker flags. When --hip-link is specified, also add -rpath with HIP runtime library directory to the linker flags
 ! HELP-NEXT: -frwpi                  Generate read-write position independent code (ARM only)
 ! HELP-NEXT: -fsave-optimization-record=<format>
 ! HELP-NEXT:                         Generate an optimization record file in a specific format
-- 
GitLab


From 1b8830c56abd01c9ab70bbbb71a00c2506cf2116 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Fri, 12 Apr 2024 08:52:18 +0200
Subject: [PATCH 035/464] [clang][Interp] Fix comparing to integral function
 pointers

We need to account for the fact that the Function pointer is not
accessible here. Add FunctionPointer::isWeak() for that.
---
 clang/lib/AST/Interp/FunctionPointer.h |  6 ++++++
 clang/lib/AST/Interp/Interp.h          |  2 +-
 clang/test/AST/Interp/c.c              | 10 ++++++++--
 3 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/clang/lib/AST/Interp/FunctionPointer.h b/clang/lib/AST/Interp/FunctionPointer.h
index 840c1101f396..c2ea295b82bd 100644
--- a/clang/lib/AST/Interp/FunctionPointer.h
+++ b/clang/lib/AST/Interp/FunctionPointer.h
@@ -32,6 +32,12 @@ public:
 
   const Function *getFunction() const { return Func; }
   bool isZero() const { return !Func; }
+  bool isWeak() const {
+    if (!Func || !Valid)
+      return false;
+
+    return Func->getDecl()->isWeak();
+  }
 
   APValue toAPValue() const {
     if (!Func)
diff --git a/clang/lib/AST/Interp/Interp.h b/clang/lib/AST/Interp/Interp.h
index c7012aa4ec68..4182254357eb 100644
--- a/clang/lib/AST/Interp/Interp.h
+++ b/clang/lib/AST/Interp/Interp.h
@@ -758,7 +758,7 @@ inline bool CmpHelperEQ<FunctionPointer>(InterpState &S, CodePtr OpPC,
 
   // We cannot compare against weak declarations at compile time.
   for (const auto &FP : {LHS, RHS}) {
-    if (!FP.isZero() && FP.getFunction()->getDecl()->isWeak()) {
+    if (FP.isWeak()) {
       const SourceInfo &Loc = S.Current->getSource(OpPC);
       S.FFDiag(Loc, diag::note_constexpr_pointer_weak_comparison)
           << FP.toDiagnosticString(S.getCtx());
diff --git a/clang/test/AST/Interp/c.c b/clang/test/AST/Interp/c.c
index cdecd3e83a99..e0b18120fd21 100644
--- a/clang/test/AST/Interp/c.c
+++ b/clang/test/AST/Interp/c.c
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -triple x86_64-linux -fexperimental-new-constant-interpreter -verify=expected,all -std=c11 -Wcast-qual %s
-// RUN: %clang_cc1 -triple x86_64-linux -fexperimental-new-constant-interpreter -pedantic -verify=pedantic-expected,all -std=c11 -Wcast-qual %s
+// RUN: %clang_cc1 -triple x86_64-linux -fexperimental-new-constant-interpreter -pedantic -verify=pedantic,pedantic-expected,all -std=c11 -Wcast-qual %s
 // RUN: %clang_cc1 -triple x86_64-linux -verify=ref,all -std=c11 -Wcast-qual %s
-// RUN: %clang_cc1 -triple x86_64-linux -pedantic -verify=pedantic-ref,all -std=c11 -Wcast-qual %s
+// RUN: %clang_cc1 -triple x86_64-linux -pedantic -verify=pedantic,pedantic-ref,all -std=c11 -Wcast-qual %s
 
 typedef __INTPTR_TYPE__ intptr_t;
 typedef __PTRDIFF_TYPE__ ptrdiff_t;
@@ -227,3 +227,9 @@ int castViaInt[*(int*)(unsigned long)"test"]; // ref-error {{variable length arr
                                               // pedantic-ref-error {{variable length array}} \
                                               // expected-error {{variable length array}} \
                                               // pedantic-expected-error {{variable length array}}
+
+const void (*const funcp)(void) = (void*)123; // pedantic-warning {{converts between void pointer and function pointer}}
+_Static_assert(funcp == (void*)0, ""); // all-error {{failed due to requirement 'funcp == (void *)0'}} \
+                                       // pedantic-warning {{expression is not an integer constant expression}}
+_Static_assert(funcp == (void*)123, ""); // pedantic-warning {{equality comparison between function pointer and void pointer}} \
+                                         // pedantic-warning {{expression is not an integer constant expression}}
-- 
GitLab


From ff9bc3a0fa4e3790c437e62450903bf399df37f9 Mon Sep 17 00:00:00 2001
From: Johannes Reifferscheid <jreiffers@google.com>
Date: Fri, 12 Apr 2024 09:42:04 +0200
Subject: [PATCH 036/464] Fix overflows in complex sqrt lowering. (#88480)

This ports XLA's complex sqrt lowering. The accuracy was tested with its
exhaustive_unary_test_complex test.

Note: rsqrt is still broken.
---
 .../ComplexToStandard/ComplexToStandard.cpp   | 166 ++++++----
 .../convert-to-standard.mlir                  | 296 +++++++++++-------
 .../ComplexToStandard/full-conversion.mlir    |   2 +-
 3 files changed, 280 insertions(+), 184 deletions(-)

diff --git a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
index 9c82e8105f06..0664b053fc9e 100644
--- a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
+++ b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
@@ -27,35 +27,52 @@ using namespace mlir;
 
 namespace {
 
+// Returns the absolute value or its square root.
+Value computeAbs(Value real, Value imag, arith::FastMathFlags fmf,
+                 ImplicitLocOpBuilder &b, bool returnSqrt = false) {
+  Value one = b.create<arith::ConstantOp>(real.getType(),
+                                          b.getFloatAttr(real.getType(), 1.0));
+
+  Value absReal = b.create<math::AbsFOp>(real, fmf);
+  Value absImag = b.create<math::AbsFOp>(imag, fmf);
+
+  Value max = b.create<arith::MaximumFOp>(absReal, absImag, fmf);
+  Value min = b.create<arith::MinimumFOp>(absReal, absImag, fmf);
+  Value ratio = b.create<arith::DivFOp>(min, max, fmf);
+  Value ratioSq = b.create<arith::MulFOp>(ratio, ratio, fmf);
+  Value ratioSqPlusOne = b.create<arith::AddFOp>(ratioSq, one, fmf);
+  Value result;
+
+  if (returnSqrt) {
+    Value quarter = b.create<arith::ConstantOp>(
+        real.getType(), b.getFloatAttr(real.getType(), 0.25));
+    // sqrt(sqrt(a*b)) would avoid the pow, but will overflow more easily.
+    Value sqrt = b.create<math::SqrtOp>(max, fmf);
+    Value p025 = b.create<math::PowFOp>(ratioSqPlusOne, quarter, fmf);
+    result = b.create<arith::MulFOp>(sqrt, p025, fmf);
+  } else {
+    Value sqrt = b.create<math::SqrtOp>(ratioSqPlusOne, fmf);
+    result = b.create<arith::MulFOp>(max, sqrt, fmf);
+  }
+
+  Value isNaN =
+      b.create<arith::CmpFOp>(arith::CmpFPredicate::UNO, result, result, fmf);
+  return b.create<arith::SelectOp>(isNaN, min, result);
+}
+
 struct AbsOpConversion : public OpConversionPattern<complex::AbsOp> {
   using OpConversionPattern<complex::AbsOp>::OpConversionPattern;
 
   LogicalResult
   matchAndRewrite(complex::AbsOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
-    mlir::ImplicitLocOpBuilder b(op.getLoc(), rewriter);
+    ImplicitLocOpBuilder b(op.getLoc(), rewriter);
 
     arith::FastMathFlags fmf = op.getFastMathFlagsAttr().getValue();
 
-    Type elementType = op.getType();
-    Value one = b.create<arith::ConstantOp>(elementType,
-                                            b.getFloatAttr(elementType, 1.0));
-
     Value real = b.create<complex::ReOp>(adaptor.getComplex());
     Value imag = b.create<complex::ImOp>(adaptor.getComplex());
-    Value absReal = b.create<math::AbsFOp>(real, fmf);
-    Value absImag = b.create<math::AbsFOp>(imag, fmf);
-
-    Value max = b.create<arith::MaximumFOp>(absReal, absImag, fmf);
-    Value min = b.create<arith::MinimumFOp>(absReal, absImag, fmf);
-    Value ratio = b.create<arith::DivFOp>(min, max, fmf);
-    Value ratioSq = b.create<arith::MulFOp>(ratio, ratio, fmf);
-    Value ratioSqPlusOne = b.create<arith::AddFOp>(ratioSq, one, fmf);
-    Value sqrt = b.create<math::SqrtOp>(ratioSqPlusOne, fmf);
-    Value result = b.create<arith::MulFOp>(max, sqrt, fmf);
-    Value isNaN =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::UNO, result, result, fmf);
-    rewriter.replaceOpWithNewOp<arith::SelectOp>(op, isNaN, min, result);
+    rewriter.replaceOp(op, computeAbs(real, imag, fmf, b));
 
     return success();
   }
@@ -829,60 +846,71 @@ struct SqrtOpConversion : public OpConversionPattern<complex::SqrtOp> {
   LogicalResult
   matchAndRewrite(complex::SqrtOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
-    mlir::ImplicitLocOpBuilder b(op.getLoc(), rewriter);
+    ImplicitLocOpBuilder b(op.getLoc(), rewriter);
 
     auto type = cast<ComplexType>(op.getType());
-    Type elementType = type.getElementType();
-    Value arg = adaptor.getComplex();
-    arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
-
-    Value zero =
-        b.create<arith::ConstantOp>(elementType, b.getZeroAttr(elementType));
-
-    Value real = b.create<complex::ReOp>(elementType, adaptor.getComplex());
-    Value imag = b.create<complex::ImOp>(elementType, adaptor.getComplex());
-
-    Value absLhs = b.create<math::AbsFOp>(real, fmf);
-    Value absArg = b.create<complex::AbsOp>(elementType, arg, fmf);
-    Value addAbs = b.create<arith::AddFOp>(absLhs, absArg, fmf);
+    auto elementType = type.getElementType().cast<FloatType>();
+    arith::FastMathFlags fmf = op.getFastMathFlagsAttr().getValue();
 
+    auto cst = [&](APFloat v) {
+      return b.create<arith::ConstantOp>(elementType,
+                                         b.getFloatAttr(elementType, v));
+    };
+    const auto &floatSemantics = elementType.getFloatSemantics();
+    Value zero = cst(APFloat::getZero(floatSemantics));
     Value half = b.create<arith::ConstantOp>(elementType,
                                              b.getFloatAttr(elementType, 0.5));
-    Value halfAddAbs = b.create<arith::MulFOp>(addAbs, half, fmf);
-    Value sqrtAddAbs = b.create<math::SqrtOp>(halfAddAbs, fmf);
-
-    Value realIsNegative =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, real, zero);
-    Value imagIsNegative =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, imag, zero);
-
-    Value resultReal = sqrtAddAbs;
-
-    Value imagDivTwoResultReal = b.create<arith::DivFOp>(
-        imag, b.create<arith::AddFOp>(resultReal, resultReal, fmf), fmf);
-
-    Value negativeResultReal = b.create<arith::NegFOp>(resultReal);
 
+    Value real = b.create<complex::ReOp>(elementType, adaptor.getComplex());
+    Value imag = b.create<complex::ImOp>(elementType, adaptor.getComplex());
+    Value absSqrt = computeAbs(real, imag, fmf, b, /*returnSqrt=*/true);
+    Value argArg = b.create<math::Atan2Op>(imag, real, fmf);
+    Value sqrtArg = b.create<arith::MulFOp>(argArg, half, fmf);
+    Value cos = b.create<math::CosOp>(sqrtArg, fmf);
+    Value sin = b.create<math::SinOp>(sqrtArg, fmf);
+    // sin(atan2(0, inf)) = 0, sqrt(abs(inf)) = inf, but we can't multiply
+    // 0 * inf.
+    Value sinIsZero =
+        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, sin, zero, fmf);
+
+    Value resultReal = b.create<arith::MulFOp>(absSqrt, cos, fmf);
     Value resultImag = b.create<arith::SelectOp>(
-        realIsNegative,
-        b.create<arith::SelectOp>(imagIsNegative, negativeResultReal,
-                                  resultReal),
-        imagDivTwoResultReal);
-
-    resultReal = b.create<arith::SelectOp>(
-        realIsNegative,
-        b.create<arith::DivFOp>(
-            imag, b.create<arith::AddFOp>(resultImag, resultImag, fmf), fmf),
-        resultReal);
-
-    Value realIsZero =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, real, zero);
-    Value imagIsZero =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, imag, zero);
-    Value argIsZero = b.create<arith::AndIOp>(realIsZero, imagIsZero);
-
-    resultReal = b.create<arith::SelectOp>(argIsZero, zero, resultReal);
-    resultImag = b.create<arith::SelectOp>(argIsZero, zero, resultImag);
+        sinIsZero, zero, b.create<arith::MulFOp>(absSqrt, sin, fmf));
+    if (!arith::bitEnumContainsAll(fmf, arith::FastMathFlags::nnan |
+                                            arith::FastMathFlags::ninf)) {
+      Value inf = cst(APFloat::getInf(floatSemantics));
+      Value negInf = cst(APFloat::getInf(floatSemantics, true));
+      Value nan = cst(APFloat::getNaN(floatSemantics));
+      Value absImag = b.create<math::AbsFOp>(elementType, imag, fmf);
+
+      Value absImagIsInf =
+          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absImag, inf, fmf);
+      Value absImagIsNotInf =
+          b.create<arith::CmpFOp>(arith::CmpFPredicate::ONE, absImag, inf, fmf);
+      Value realIsInf =
+          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, real, inf, fmf);
+      Value realIsNegInf =
+          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, real, negInf, fmf);
+
+      resultReal = b.create<arith::SelectOp>(
+          b.create<arith::AndIOp>(realIsNegInf, absImagIsNotInf), zero,
+          resultReal);
+      resultReal = b.create<arith::SelectOp>(
+          b.create<arith::OrIOp>(absImagIsInf, realIsInf), inf, resultReal);
+
+      Value imagSignInf = b.create<math::CopySignOp>(inf, imag, fmf);
+      resultImag = b.create<arith::SelectOp>(
+          b.create<arith::CmpFOp>(arith::CmpFPredicate::UNO, absSqrt, absSqrt),
+          nan, resultImag);
+      resultImag = b.create<arith::SelectOp>(
+          b.create<arith::OrIOp>(absImagIsInf, realIsNegInf), imagSignInf,
+          resultImag);
+    }
+
+    Value resultIsZero =
+        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absSqrt, zero, fmf);
+    resultReal = b.create<arith::SelectOp>(resultIsZero, zero, resultReal);
+    resultImag = b.create<arith::SelectOp>(resultIsZero, zero, resultImag);
 
     rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
                                                    resultImag);
@@ -1065,7 +1093,7 @@ static Value powOpConversionImpl(mlir::ImplicitLocOpBuilder &builder,
   // Case 2:
   // 1^(c + d*i) = 1 + 0*i
   Value lhsEqOne = builder.create<arith::AndIOp>(
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, a, one),
+      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, a, one, fmf),
       bEqZero);
   Value cutoff2 =
       builder.create<arith::SelectOp>(lhsEqOne, complexOne, cutoff1);
@@ -1073,11 +1101,11 @@ static Value powOpConversionImpl(mlir::ImplicitLocOpBuilder &builder,
   // Case 3:
   // inf^(c + 0*i) = inf + 0*i, c > 0
   Value lhsEqInf = builder.create<arith::AndIOp>(
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, a, inf),
+      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, a, inf, fmf),
       bEqZero);
   Value rhsGt0 = builder.create<arith::AndIOp>(
       dEqZero,
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, c, zero));
+      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OGT, c, zero, fmf));
   Value cutoff3 = builder.create<arith::SelectOp>(
       builder.create<arith::AndIOp>(lhsEqInf, rhsGt0), complexInf, cutoff2);
 
@@ -1085,7 +1113,7 @@ static Value powOpConversionImpl(mlir::ImplicitLocOpBuilder &builder,
   // inf^(c + 0*i) = 0 + 0*i, c < 0
   Value rhsLt0 = builder.create<arith::AndIOp>(
       dEqZero,
-      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, c, zero));
+      builder.create<arith::CmpFOp>(arith::CmpFPredicate::OLT, c, zero, fmf));
   Value cutoff4 = builder.create<arith::SelectOp>(
       builder.create<arith::AndIOp>(lhsEqInf, rhsLt0), complexZero, cutoff3);
 
diff --git a/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir b/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir
index 8d2fb09daa87..b22c1acacaea 100644
--- a/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir
+++ b/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir
@@ -8,9 +8,9 @@ func.func @complex_abs(%arg: complex<f32>) -> f32 {
   return %abs : f32
 }
 
-// CHECK: %[[ONE:.*]] = arith.constant 1.000000e+00 : f32
 // CHECK: %[[REAL:.*]] = complex.re %[[ARG]] : complex<f32>
 // CHECK: %[[IMAG:.*]] = complex.im %[[ARG]] : complex<f32>
+// CHECK: %[[ONE:.*]] = arith.constant 1.000000e+00 : f32
 // CHECK: %[[ABS_REAL:.*]] = math.absf %[[REAL]] : f32
 // CHECK: %[[ABS_IMAG:.*]] = math.absf %[[IMAG]] : f32
 // CHECK: %[[MAX:.*]] = arith.maximumf %[[ABS_REAL]], %[[ABS_IMAG]] : f32
@@ -250,9 +250,9 @@ func.func @complex_log(%arg: complex<f32>) -> complex<f32> {
   %log = complex.log %arg: complex<f32>
   return %log : complex<f32>
 }
-// CHECK: %[[ONE:.*]] = arith.constant 1.000000e+00 : f32
 // CHECK: %[[REAL:.*]] = complex.re %[[ARG]] : complex<f32>
 // CHECK: %[[IMAG:.*]] = complex.im %[[ARG]] : complex<f32>
+// CHECK: %[[ONE:.*]] = arith.constant 1.000000e+00 : f32
 // CHECK: %[[ABS_REAL:.*]] = math.absf %[[REAL]] : f32
 // CHECK: %[[ABS_IMAG:.*]] = math.absf %[[IMAG]] : f32
 // CHECK: %[[MAX:.*]] = arith.maximumf %[[ABS_REAL]], %[[ABS_IMAG]] : f32
@@ -493,9 +493,9 @@ func.func @complex_sign(%arg: complex<f32>) -> complex<f32> {
 // CHECK: %[[REAL_IS_ZERO:.*]] = arith.cmpf oeq, %[[REAL]], %[[ZERO]] : f32
 // CHECK: %[[IMAG_IS_ZERO:.*]] = arith.cmpf oeq, %[[IMAG]], %[[ZERO]] : f32
 // CHECK: %[[IS_ZERO:.*]] = arith.andi %[[REAL_IS_ZERO]], %[[IMAG_IS_ZERO]] : i1
-// CHECK: %[[ONE:.*]] = arith.constant 1.000000e+00 : f32
 // CHECK: %[[REAL2:.*]] = complex.re %[[ARG]] : complex<f32>
 // CHECK: %[[IMAG2:.*]] = complex.im %[[ARG]] : complex<f32>
+// CHECK: %[[ONE:.*]] = arith.constant 1.000000e+00 : f32
 // CHECK: %[[ABS_REAL:.*]] = math.absf %[[REAL2]] : f32
 // CHECK: %[[ABS_IMAG:.*]] = math.absf %[[IMAG2]] : f32
 // CHECK: %[[MAX:.*]] = arith.maximumf %[[ABS_REAL]], %[[ABS_IMAG]] : f32
@@ -697,45 +697,95 @@ func.func @complex_sqrt(%arg: complex<f32>) -> complex<f32> {
   return %sqrt : complex<f32>
 }
 
-// CHECK: %[[CST:.*]]  = arith.constant 0.000000e+00 : f32
-// CHECK: %[[VAR0:.*]] = complex.re %[[ARG]] : complex<f32>
-// CHECK: %[[VAR1:.*]] = complex.im %[[ARG]] : complex<f32>
-// CHECK: %[[VAR2:.*]] = math.absf %[[VAR0]] : f32
+// CHECK: %[[ZERO:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK: %[[HALF:.*]] = arith.constant 5.000000e-01 : f32
+// CHECK: %[[RE:.*]] = complex.re %[[ARG]] : complex<f32>
+// CHECK: %[[IM:.*]] = complex.im %[[ARG]] : complex<f32>
 // CHECK: %[[ONE:.*]] = arith.constant 1.000000e+00 : f32
-// CHECK: %[[REAL:.*]] = complex.re %[[ARG]] : complex<f32>
-// CHECK: %[[IMAG:.*]] = complex.im %[[ARG]] : complex<f32>
-// CHECK: %[[ABS_REAL:.*]] = math.absf %[[REAL]] : f32
-// CHECK: %[[ABS_IMAG:.*]] = math.absf %[[IMAG]] : f32
-// CHECK: %[[MAX:.*]] = arith.maximumf %[[ABS_REAL]], %[[ABS_IMAG]] : f32
-// CHECK: %[[MIN:.*]] = arith.minimumf %[[ABS_REAL]], %[[ABS_IMAG]] : f32
+// CHECK: %[[ABSRE:.*]] = math.absf %[[RE]] : f32
+// CHECK: %[[ABSIM:.*]] = math.absf %[[IM]] : f32
+// CHECK: %[[MAX:.*]] = arith.maximumf %[[ABSRE]], %[[ABSIM]] : f32
+// CHECK: %[[MIN:.*]] = arith.minimumf %[[ABSRE]], %[[ABSIM]] : f32
 // CHECK: %[[RATIO:.*]] = arith.divf %[[MIN]], %[[MAX]] : f32
 // CHECK: %[[RATIO_SQ:.*]] = arith.mulf %[[RATIO]], %[[RATIO]] : f32
 // CHECK: %[[RATIO_SQ_PLUS_ONE:.*]] = arith.addf %[[RATIO_SQ]], %[[ONE]] : f32
-// CHECK: %[[SQRT:.*]] = math.sqrt %[[RATIO_SQ_PLUS_ONE]] : f32
-// CHECK: %[[ABS_OR_NAN:.*]] = arith.mulf %[[MAX]], %[[SQRT]] : f32
-// CHECK: %[[IS_NAN:.*]] = arith.cmpf uno, %[[ABS_OR_NAN]], %[[ABS_OR_NAN]] : f32
-// CHECK: %[[ABS:.*]] = arith.select %[[IS_NAN]], %[[MIN]], %[[ABS_OR_NAN]] : f32
-// CHECK: %[[VAR23:.*]] = arith.addf %[[VAR2]], %[[ABS]] : f32
-// CHECK: %[[CST2:.*]]  = arith.constant 5.000000e-01 : f32
-// CHECK: %[[VAR24:.*]] = arith.mulf %[[VAR23]], %[[CST2]] : f32
-// CHECK: %[[VAR25:.*]] = math.sqrt %[[VAR24]] : f32
-// CHECK: %[[VAR26:.*]] = arith.cmpf olt, %[[VAR0]], %cst : f32
-// CHECK: %[[VAR27:.*]] = arith.cmpf olt, %[[VAR1]], %cst : f32
-// CHECK: %[[VAR28:.*]] = arith.addf %[[VAR25]], %[[VAR25]] : f32
-// CHECK: %[[VAR29:.*]] = arith.divf %[[VAR1]], %[[VAR28]] : f32
-// CHECK: %[[VAR30:.*]] = arith.negf %[[VAR25]] : f32
-// CHECK: %[[VAR31:.*]] = arith.select %[[VAR27]], %[[VAR30]], %[[VAR25]] : f32
-// CHECK: %[[VAR32:.*]] = arith.select %[[VAR26]], %[[VAR31]], %[[VAR29]] : f32
-// CHECK: %[[VAR33:.*]] = arith.addf %[[VAR32]], %[[VAR32]] : f32
-// CHECK: %[[VAR34:.*]] = arith.divf %[[VAR1]], %[[VAR33]] : f32
-// CHECK: %[[VAR35:.*]] = arith.select %[[VAR26]], %[[VAR34]], %[[VAR25]] : f32
-// CHECK: %[[VAR36:.*]] = arith.cmpf oeq, %[[VAR0]], %cst : f32
-// CHECK: %[[VAR37:.*]] = arith.cmpf oeq, %[[VAR1]], %cst : f32
-// CHECK: %[[VAR38:.*]] = arith.andi %[[VAR36]], %[[VAR37]] : i1
-// CHECK: %[[VAR39:.*]] = arith.select %[[VAR38]], %cst, %[[VAR35]] : f32
-// CHECK: %[[VAR40:.*]] = arith.select %[[VAR38]], %cst, %[[VAR32]] : f32
-// CHECK: %[[VAR41:.*]] = complex.create %[[VAR39]], %[[VAR40]] : complex<f32>
-// CHECK: return %[[VAR41]] : complex<f32>
+// CHECK: %[[QUARTER:.*]] = arith.constant 2.500000e-01 : f32
+// CHECK: %[[SQRT_MAX:.*]] = math.sqrt %[[MAX]] : f32
+// CHECK: %[[POW:.*]] = math.powf %[[RATIO_SQ_PLUS_ONE]], %[[QUARTER]] : f32
+// CHECK: %[[SQRT_ABS_OR_NAN:.*]] = arith.mulf %[[SQRT_MAX]], %[[POW]] : f32
+// CHECK: %[[IS_NAN:.*]] = arith.cmpf uno, %[[SQRT_ABS_OR_NAN]], %[[SQRT_ABS_OR_NAN]] : f32
+// CHECK: %[[SQRT_ABS:.*]] = arith.select %[[IS_NAN]], %[[MIN]], %[[SQRT_ABS_OR_NAN]] : f32
+// CHECK: %[[ARGARG:.*]] = math.atan2 %[[IM]], %[[RE]] : f32
+// CHECK: %[[SQRTARG:.*]] = arith.mulf %[[ARGARG]], %[[HALF]] : f32
+// CHECK: %[[COS:.*]] = math.cos %[[SQRTARG]] : f32
+// CHECK: %[[SIN:.*]] = math.sin %[[SQRTARG]] : f32
+// CHECK: %[[SIN_ZERO:.*]] = arith.cmpf oeq, %[[SIN]], %[[ZERO]] : f32
+// CHECK: %[[RESULT_RE:.*]] = arith.mulf %[[SQRT_ABS]], %[[COS]] : f32
+// CHECK: %[[RESULT_IM:.*]] = arith.mulf %[[SQRT_ABS]], %[[SIN]] : f32
+// CHECK: %[[RESULT_IM2:.*]] = arith.select %[[SIN_ZERO]], %[[ZERO]], %[[RESULT_IM]] : f32
+// CHECK: %[[INF:.*]] = arith.constant 0x7F800000 : f32
+// CHECK: %[[NINF:.*]] = arith.constant 0xFF800000 : f32
+// CHECK: %[[NAN:.*]] = arith.constant 0x7FC00000 : f32
+// CHECK: %[[ABSIM:.*]] = math.absf %[[IM]] : f32
+// CHECK: %[[ABSIMINF:.*]] = arith.cmpf oeq, %[[ABSIM]], %[[INF]] : f32
+// CHECK: %[[ABSIMNOTINF:.*]] = arith.cmpf one, %[[ABSIM]], %[[INF]] : f32
+// CHECK: %[[REINF:.*]] = arith.cmpf oeq, %[[RE]], %[[INF]] : f32
+// CHECK: %[[RENINF:.*]] = arith.cmpf oeq, %[[RE]], %[[NINF]] : f32
+// CHECK: %[[RESULT_RE_ZERO:.*]] = arith.andi %[[RENINF]], %[[ABSIMNOTINF]] : i1
+// CHECK: %[[RESULT_RE2:.*]] = arith.select %[[RESULT_RE_ZERO]], %[[ZERO]], %[[RESULT_RE]] : f32
+// CHECK: %[[RESUL_IM_INF:.*]] = arith.ori %[[ABSIMINF]], %[[REINF]] : i1
+// CHECK: %[[RESULT_RE3:.*]] = arith.select %[[RESUL_IM_INF]], %[[INF]], %[[RESULT_RE2]] : f32
+// CHECK: %[[INF_IM_SIGN:.*]] = math.copysign %[[INF]], %[[IM]] : f32
+// CHECK: %[[RESULT_IM_NAN:.*]] = arith.cmpf uno, %[[SQRT_ABS]], %[[SQRT_ABS]] : f32
+// CHECK: %[[RESULT_IM3:.*]] = arith.select %[[RESULT_IM_NAN]], %[[NAN]], %[[RESULT_IM2]] : f32
+// CHECK: %[[RESULT_IM_INF:.*]] = arith.ori %[[ABSIMINF]], %[[RENINF]] : i1
+// CHECK: %[[RESULT_IM4:.*]] = arith.select %[[RESULT_IM_INF]], %[[INF_IM_SIGN]], %[[RESULT_IM3]] : f32
+// CHECK: %[[RESULT_ZERO:.*]] = arith.cmpf oeq, %[[SQRT_ABS]], %[[ZERO]] : f32
+// CHECK: %[[RESULT_RE4:.*]] = arith.select %[[RESULT_ZERO]], %[[ZERO]], %[[RESULT_RE3]] : f32
+// CHECK: %[[RESULT_IM5:.*]] = arith.select %[[RESULT_ZERO]], %[[ZERO]], %[[RESULT_IM4]] : f32
+// CHECK: %[[RESULT:.*]] = complex.create %[[RESULT_RE4]], %[[RESULT_IM5]] : complex<f32>
+// CHECK: return %[[RESULT]] : complex<f32>
+
+// -----
+
+// CHECK-LABEL: func @complex_sqrt_nnan_ninf
+// CHECK-SAME: %[[ARG:.*]]: complex<f32>
+func.func @complex_sqrt_nnan_ninf(%arg: complex<f32>) -> complex<f32> {
+  %sqrt = complex.sqrt %arg fastmath<nnan,ninf> : complex<f32>
+  return %sqrt : complex<f32>
+}
+
+// CHECK: %[[ZERO:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK: %[[HALF:.*]] = arith.constant 5.000000e-01 : f32
+// CHECK: %[[RE:.*]] = complex.re %[[ARG]] : complex<f32>
+// CHECK: %[[IM:.*]] = complex.im %[[ARG]] : complex<f32>
+// CHECK: %[[ONE:.*]] = arith.constant 1.000000e+00 : f32
+// CHECK: %[[ABSRE:.*]] = math.absf %[[RE]] fastmath<nnan,ninf> : f32
+// CHECK: %[[ABSIM:.*]] = math.absf %[[IM]] fastmath<nnan,ninf> : f32
+// CHECK: %[[MAX:.*]] = arith.maximumf %[[ABSRE]], %[[ABSIM]] fastmath<nnan,ninf> : f32
+// CHECK: %[[MIN:.*]] = arith.minimumf %[[ABSRE]], %[[ABSIM]] fastmath<nnan,ninf> : f32
+// CHECK: %[[RATIO:.*]] = arith.divf %[[MIN]], %[[MAX]] fastmath<nnan,ninf> : f32
+// CHECK: %[[RATIO_SQ:.*]] = arith.mulf %[[RATIO]], %[[RATIO]] fastmath<nnan,ninf> : f32
+// CHECK: %[[RATIO_SQ_PLUS_ONE:.*]] = arith.addf %[[RATIO_SQ]], %[[ONE]] fastmath<nnan,ninf> : f32
+// CHECK: %[[QUARTER:.*]] = arith.constant 2.500000e-01 : f32
+// CHECK: %[[SQRT_MAX:.*]] = math.sqrt %[[MAX]] fastmath<nnan,ninf> : f32
+// CHECK: %[[POW:.*]] = math.powf %[[RATIO_SQ_PLUS_ONE]], %[[QUARTER]] fastmath<nnan,ninf> : f32
+// CHECK: %[[SQRT_ABS_OR_NAN:.*]] = arith.mulf %[[SQRT_MAX]], %[[POW]] fastmath<nnan,ninf> : f32
+// CHECK: %[[IS_NAN:.*]] = arith.cmpf uno, %[[SQRT_ABS_OR_NAN]], %[[SQRT_ABS_OR_NAN]] fastmath<nnan,ninf> : f32
+// CHECK: %[[SQRT_ABS:.*]] = arith.select %[[IS_NAN]], %[[MIN]], %[[SQRT_ABS_OR_NAN]] : f32
+// CHECK: %[[ARGARG:.*]] = math.atan2 %[[IM]], %[[RE]] fastmath<nnan,ninf> : f32
+// CHECK: %[[SQRTARG:.*]] = arith.mulf %[[ARGARG]], %[[HALF]] fastmath<nnan,ninf> : f32
+// CHECK: %[[COS:.*]] = math.cos %[[SQRTARG]] fastmath<nnan,ninf> : f32
+// CHECK: %[[SIN:.*]] = math.sin %[[SQRTARG]] fastmath<nnan,ninf> : f32
+// CHECK: %[[SIN_ZERO:.*]] = arith.cmpf oeq, %[[SIN]], %[[ZERO]] fastmath<nnan,ninf> : f32
+// CHECK: %[[RESULT_RE:.*]] = arith.mulf %[[SQRT_ABS]], %[[COS]] fastmath<nnan,ninf> : f32
+// CHECK: %[[RESULT_IM:.*]] = arith.mulf %[[SQRT_ABS]], %[[SIN]] fastmath<nnan,ninf> : f32
+// CHECK: %[[RESULT_IM2:.*]] = arith.select %[[SIN_ZERO]], %[[ZERO]], %[[RESULT_IM]] : f32
+// CHECK: %[[RESULT_ZERO:.*]] = arith.cmpf oeq, %[[SQRT_ABS]], %[[ZERO]] fastmath<nnan,ninf> : f32
+// CHECK: %[[RESULT_RE2:.*]] = arith.select %[[RESULT_ZERO]], %[[ZERO]], %[[RESULT_RE]] : f32
+// CHECK: %[[RESULT_IM3:.*]] = arith.select %[[RESULT_ZERO]], %[[ZERO]], %[[RESULT_IM2]] : f32
+// CHECK: %[[RESULT:.*]] = complex.create %[[RESULT_RE2]], %[[RESULT_IM3]] : complex<f32>
+// CHECK: return %[[RESULT]] : complex<f32>
 
 // -----
 
@@ -808,9 +858,9 @@ func.func @complex_abs_with_fmf(%arg: complex<f32>) -> f32 {
   %abs = complex.abs %arg fastmath<nnan,contract> : complex<f32>
   return %abs : f32
 }
-// CHECK: %[[ONE:.*]] = arith.constant 1.000000e+00 : f32
 // CHECK: %[[REAL:.*]] = complex.re %[[ARG]] : complex<f32>
 // CHECK: %[[IMAG:.*]] = complex.im %[[ARG]] : complex<f32>
+// CHECK: %[[ONE:.*]] = arith.constant 1.000000e+00 : f32
 // CHECK: %[[ABS_REAL:.*]] = math.absf %[[REAL]] fastmath<nnan,contract> : f32
 // CHECK: %[[ABS_IMAG:.*]] = math.absf %[[IMAG]] fastmath<nnan,contract> : f32
 // CHECK: %[[MAX:.*]] = arith.maximumf %[[ABS_REAL]], %[[ABS_IMAG]] fastmath<nnan,contract> : f32
@@ -907,9 +957,9 @@ func.func @complex_log_with_fmf(%arg: complex<f32>) -> complex<f32> {
   %log = complex.log %arg fastmath<nnan,contract> : complex<f32>
   return %log : complex<f32>
 }
-// CHECK: %[[ONE:.*]] = arith.constant 1.000000e+00 : f32
 // CHECK: %[[REAL:.*]] = complex.re %[[ARG]] : complex<f32>
 // CHECK: %[[IMAG:.*]] = complex.im %[[ARG]] : complex<f32>
+// CHECK: %[[ONE:.*]] = arith.constant 1.000000e+00 : f32
 // CHECK: %[[ABS_REAL:.*]] = math.absf %[[REAL]] fastmath<nnan,contract> : f32
 // CHECK: %[[ABS_IMAG:.*]] = math.absf %[[IMAG]] fastmath<nnan,contract> : f32
 // CHECK: %[[MAX:.*]] = arith.maximumf %[[ABS_REAL]], %[[ABS_IMAG]] fastmath<nnan,contract> : f32
@@ -1285,44 +1335,53 @@ func.func @complex_atan2_with_fmf(%lhs: complex<f32>,
 // CHECK: %[[VAR184:.*]] = complex.im %[[VAR179]] : complex<f32>
 // CHECK: %[[VAR185:.*]] = arith.addf %[[VAR183]], %[[VAR184]] fastmath<nnan,contract> : f32
 // CHECK: %[[VAR186:.*]] = complex.create %[[VAR182]], %[[VAR185]] : complex<f32>
-// CHECK: %[[CST_6:.*]] = arith.constant 0.000000e+00 : f32
-// CHECK: %[[VAR187:.*]] = complex.re %[[VAR186]] : complex<f32>
-// CHECK: %[[VAR188:.*]] = complex.im %[[VAR186]] : complex<f32>
-// CHECK: %[[VAR189:.*]] = math.absf %[[VAR187]] fastmath<nnan,contract> : f32
+// CHECK: %[[ZERO:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK: %[[HALF:.*]] = arith.constant 5.000000e-01 : f32
+// CHECK: %[[RE:.*]] = complex.re %[[VAR186]] : complex<f32>
+// CHECK: %[[IM:.*]] = complex.im %[[VAR186]] : complex<f32>
 // CHECK: %[[ONE:.*]] = arith.constant 1.000000e+00 : f32
-// CHECK: %[[REAL:.*]] = complex.re %[[VAR186]] : complex<f32>
-// CHECK: %[[IMAG:.*]] = complex.im %[[VAR186]] : complex<f32>
-// CHECK: %[[ABS_REAL:.*]] = math.absf %[[REAL]] fastmath<nnan,contract>  : f32
-// CHECK: %[[ABS_IMAG:.*]] = math.absf %[[IMAG]] fastmath<nnan,contract>  : f32
-// CHECK: %[[MAX:.*]] = arith.maximumf %[[ABS_REAL]], %[[ABS_IMAG]] fastmath<nnan,contract>  : f32
-// CHECK: %[[MIN:.*]] = arith.minimumf %[[ABS_REAL]], %[[ABS_IMAG]] fastmath<nnan,contract>  : f32
-// CHECK: %[[RATIO:.*]] = arith.divf %[[MIN]], %[[MAX]] fastmath<nnan,contract>  : f32
-// CHECK: %[[RATIO_SQ:.*]] = arith.mulf %[[RATIO]], %[[RATIO]] fastmath<nnan,contract>  : f32
-// CHECK: %[[RATIO_SQ_PLUS_ONE:.*]] = arith.addf %[[RATIO_SQ]], %[[ONE]] fastmath<nnan,contract>  : f32
-// CHECK: %[[SQRT:.*]] = math.sqrt %[[RATIO_SQ_PLUS_ONE]] fastmath<nnan,contract>  : f32
-// CHECK: %[[ABS_OR_NAN:.*]] = arith.mulf %[[MAX]], %[[SQRT]] fastmath<nnan,contract>  : f32
-// CHECK: %[[IS_NAN:.*]] = arith.cmpf uno, %[[ABS_OR_NAN]], %[[ABS_OR_NAN]] fastmath<nnan,contract>  : f32
-// CHECK: %[[ABS:.*]] = arith.select %[[IS_NAN]], %[[MIN]], %[[ABS_OR_NAN]] : f32
-// CHECK: %[[VAR210:.*]] = arith.addf %[[VAR189]], %[[ABS]] fastmath<nnan,contract> : f32
-// CHECK: %[[CST_9:.*]] = arith.constant 5.000000e-01 : f32
-// CHECK: %[[VAR211:.*]] = arith.mulf %[[VAR210]], %[[CST_9]] fastmath<nnan,contract> : f32
-// CHECK: %[[VAR212:.*]] = math.sqrt %[[VAR211]] fastmath<nnan,contract> : f32
-// CHECK: %[[VAR213:.*]] = arith.cmpf olt, %[[VAR187]], %[[CST_6]] : f32
-// CHECK: %[[VAR214:.*]] = arith.cmpf olt, %[[VAR188]], %[[CST_6]] : f32
-// CHECK: %[[VAR215:.*]] = arith.addf %[[VAR212]], %[[VAR212]] fastmath<nnan,contract> : f32
-// CHECK: %[[VAR216:.*]] = arith.divf %[[VAR188]], %[[VAR215]] fastmath<nnan,contract> : f32
-// CHECK: %[[VAR217:.*]] = arith.negf %[[VAR212]] : f32
-// CHECK: %[[VAR218:.*]] = arith.select %[[VAR214]], %[[VAR217]], %[[VAR212]] : f32
-// CHECK: %[[VAR219:.*]] = arith.select %[[VAR213]], %[[VAR218]], %[[VAR216]] : f32
-// CHECK: %[[VAR220:.*]] = arith.addf %[[VAR219]], %[[VAR219]] fastmath<nnan,contract> : f32
-// CHECK: %[[VAR221:.*]] = arith.divf %[[VAR188]], %[[VAR220]] fastmath<nnan,contract> : f32
-// CHECK: %[[VAR222:.*]] = arith.select %[[VAR213]], %[[VAR221]], %[[VAR212]] : f32
-// CHECK: %[[VAR223:.*]] = arith.cmpf oeq, %[[VAR187]], %[[CST_6]] : f32
-// CHECK: %[[VAR224:.*]] = arith.cmpf oeq, %[[VAR188]], %[[CST_6]] : f32
-// CHECK: %[[VAR225:.*]] = arith.andi %[[VAR223]], %[[VAR224]] : i1
-// CHECK: %[[VAR226:.*]] = arith.select %[[VAR225]], %[[CST_6]], %[[VAR222]] : f32
-// CHECK: %[[VAR227:.*]] = arith.select %[[VAR225]], %[[CST_6]], %[[VAR219]] : f32
-// CHECK: %[[VAR228:.*]] = complex.create %[[VAR226]], %[[VAR227]] : complex<f32>
+// CHECK: %[[ABSRE:.*]] = math.absf %[[RE]] fastmath<nnan,contract> : f32
+// CHECK: %[[ABSIM:.*]] = math.absf %[[IM]] fastmath<nnan,contract> : f32
+// CHECK: %[[MAX:.*]] = arith.maximumf %[[ABSRE]], %[[ABSIM]] fastmath<nnan,contract> : f32
+// CHECK: %[[MIN:.*]] = arith.minimumf %[[ABSRE]], %[[ABSIM]] fastmath<nnan,contract> : f32
+// CHECK: %[[RATIO:.*]] = arith.divf %[[MIN]], %[[MAX]] fastmath<nnan,contract> : f32
+// CHECK: %[[RATIO_SQ:.*]] = arith.mulf %[[RATIO]], %[[RATIO]] fastmath<nnan,contract> : f32
+// CHECK: %[[RATIO_SQ_PLUS_ONE:.*]] = arith.addf %[[RATIO_SQ]], %[[ONE]] fastmath<nnan,contract> : f32
+// CHECK: %[[QUARTER:.*]] = arith.constant 2.500000e-01 : f32
+// CHECK: %[[SQRT_MAX:.*]] = math.sqrt %[[MAX]] fastmath<nnan,contract> : f32
+// CHECK: %[[POW:.*]] = math.powf %[[RATIO_SQ_PLUS_ONE]], %[[QUARTER]] fastmath<nnan,contract> : f32
+// CHECK: %[[SQRT_ABS_OR_NAN:.*]] = arith.mulf %[[SQRT_MAX]], %[[POW]] fastmath<nnan,contract> : f32
+// CHECK: %[[IS_NAN:.*]] = arith.cmpf uno, %[[SQRT_ABS_OR_NAN]], %[[SQRT_ABS_OR_NAN]] fastmath<nnan,contract> : f32
+// CHECK: %[[SQRT_ABS:.*]] = arith.select %[[IS_NAN]], %[[MIN]], %[[SQRT_ABS_OR_NAN]] : f32
+// CHECK: %[[ARGARG:.*]] = math.atan2 %[[IM]], %[[RE]] fastmath<nnan,contract> : f32
+// CHECK: %[[SQRTARG:.*]] = arith.mulf %[[ARGARG]], %[[HALF]] fastmath<nnan,contract> : f32
+// CHECK: %[[COS:.*]] = math.cos %[[SQRTARG]] fastmath<nnan,contract> : f32
+// CHECK: %[[SIN:.*]] = math.sin %[[SQRTARG]] fastmath<nnan,contract> : f32
+// CHECK: %[[SIN_ZERO:.*]] = arith.cmpf oeq, %[[SIN]], %[[ZERO]] fastmath<nnan,contract> : f32
+// CHECK: %[[RESULT_RE:.*]] = arith.mulf %[[SQRT_ABS]], %[[COS]] fastmath<nnan,contract> : f32
+// CHECK: %[[RESULT_IM:.*]] = arith.mulf %[[SQRT_ABS]], %[[SIN]] fastmath<nnan,contract> : f32
+// CHECK: %[[RESULT_IM2:.*]] = arith.select %[[SIN_ZERO]], %[[ZERO]], %[[RESULT_IM]] : f32
+// CHECK: %[[INF:.*]] = arith.constant 0x7F800000 : f32
+// CHECK: %[[NINF:.*]] = arith.constant 0xFF800000 : f32
+// CHECK: %[[NAN:.*]] = arith.constant 0x7FC00000 : f32
+// CHECK: %[[ABSIM:.*]] = math.absf %[[IM]] fastmath<nnan,contract> : f32
+// CHECK: %[[ABSIMINF:.*]] = arith.cmpf oeq, %[[ABSIM]], %[[INF]] fastmath<nnan,contract> : f32
+// CHECK: %[[ABSIMNOTINF:.*]] = arith.cmpf one, %[[ABSIM]], %[[INF]] fastmath<nnan,contract> : f32
+// CHECK: %[[REINF:.*]] = arith.cmpf oeq, %[[RE]], %[[INF]] fastmath<nnan,contract> : f32
+// CHECK: %[[RENINF:.*]] = arith.cmpf oeq, %[[RE]], %[[NINF]] fastmath<nnan,contract> : f32
+// CHECK: %[[RESULT_RE_ZERO:.*]] = arith.andi %[[RENINF]], %[[ABSIMNOTINF]] : i1
+// CHECK: %[[RESULT_RE2:.*]] = arith.select %[[RESULT_RE_ZERO]], %[[ZERO]], %[[RESULT_RE]] : f32
+// CHECK: %[[RESUL_IM_INF:.*]] = arith.ori %[[ABSIMINF]], %[[REINF]] : i1
+// CHECK: %[[RESULT_RE3:.*]] = arith.select %[[RESUL_IM_INF]], %[[INF]], %[[RESULT_RE2]] : f32
+// CHECK: %[[INF_IM_SIGN:.*]] = math.copysign %[[INF]], %[[IM]] fastmath<nnan,contract> : f32
+// CHECK: %[[RESULT_IM_NAN:.*]] = arith.cmpf uno, %[[SQRT_ABS]], %[[SQRT_ABS]] : f32
+// CHECK: %[[RESULT_IM3:.*]] = arith.select %[[RESULT_IM_NAN]], %[[NAN]], %[[RESULT_IM2]] : f32
+// CHECK: %[[RESULT_IM_INF:.*]] = arith.ori %[[ABSIMINF]], %[[RENINF]] : i1
+// CHECK: %[[RESULT_IM4:.*]] = arith.select %[[RESULT_IM_INF]], %[[INF_IM_SIGN]], %[[RESULT_IM3]] : f32
+// CHECK: %[[RESULT_ZERO:.*]] = arith.cmpf oeq, %[[SQRT_ABS]], %[[ZERO]] fastmath<nnan,contract> : f32
+// CHECK: %[[RESULT_RE4:.*]] = arith.select %[[RESULT_ZERO]], %[[ZERO]], %[[RESULT_RE3]] : f32
+// CHECK: %[[RESULT_IM5:.*]] = arith.select %[[RESULT_ZERO]], %[[ZERO]], %[[RESULT_IM4]] : f32
+// CHECK: %[[VAR228:.*]] = complex.create %[[RESULT_RE4]], %[[RESULT_IM5]] : complex<f32>
 // CHECK: %[[CST_10:.*]] = arith.constant 0.000000e+00 : f32
 // CHECK: %[[CST_11:.*]] = arith.constant 1.000000e+00 : f32
 // CHECK: %[[VAR229:.*]] = complex.create %[[CST_10]], %[[CST_11]] : complex<f32>
@@ -1519,9 +1578,9 @@ func.func @complex_atan2_with_fmf(%lhs: complex<f32>,
 // CHECK: %[[VAR413:.*]] = arith.select %[[VAR412]], %[[VAR408]], %[[VAR402]] : f32
 // CHECK: %[[VAR414:.*]] = arith.select %[[VAR412]], %[[VAR409]], %[[VAR403]] : f32
 // CHECK: %[[VAR415:.*]] = complex.create %[[VAR413]], %[[VAR414]] : complex<f32>
-// CHECK: %[[ONE:.*]] = arith.constant 1.000000e+00 : f32
 // CHECK: %[[REAL:.*]] = complex.re %[[VAR415]] : complex<f32>
 // CHECK: %[[IMAG:.*]] = complex.im %[[VAR415]] : complex<f32>
+// CHECK: %[[ONE:.*]] = arith.constant 1.000000e+00 : f32
 // CHECK: %[[ABS_REAL:.*]] = math.absf %[[REAL]] fastmath<nnan,contract> : f32
 // CHECK: %[[ABS_IMAG:.*]] = math.absf %[[IMAG]] fastmath<nnan,contract> : f32
 // CHECK: %[[MAX:.*]] = arith.maximumf %[[ABS_REAL]], %[[ABS_IMAG]] fastmath<nnan,contract> : f32
@@ -1756,45 +1815,54 @@ func.func @complex_sqrt_with_fmf(%arg: complex<f32>) -> complex<f32> {
   return %sqrt : complex<f32>
 }
 
-// CHECK: %[[CST:.*]]  = arith.constant 0.000000e+00 : f32
-// CHECK: %[[VAR0:.*]] = complex.re %[[ARG]] : complex<f32>
-// CHECK: %[[VAR1:.*]] = complex.im %[[ARG]] : complex<f32>
-// CHECK: %[[VAR2:.*]] = math.absf %[[VAR0]] fastmath<nnan,contract> : f32
+// CHECK: %[[ZERO:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK: %[[HALF:.*]] = arith.constant 5.000000e-01 : f32
+// CHECK: %[[RE:.*]] = complex.re %[[ARG]] : complex<f32>
+// CHECK: %[[IM:.*]] = complex.im %[[ARG]] : complex<f32>
 // CHECK: %[[ONE:.*]] = arith.constant 1.000000e+00 : f32
-// CHECK: %[[REAL:.*]] = complex.re %[[ARG]] : complex<f32>
-// CHECK: %[[IMAG:.*]] = complex.im %[[ARG]] : complex<f32>
-// CHECK: %[[ABS_REAL:.*]] = math.absf %[[REAL]] fastmath<nnan,contract> : f32
-// CHECK: %[[ABS_IMAG:.*]] = math.absf %[[IMAG]] fastmath<nnan,contract> : f32
-// CHECK: %[[MAX:.*]] = arith.maximumf %[[ABS_REAL]], %[[ABS_IMAG]] fastmath<nnan,contract> : f32
-// CHECK: %[[MIN:.*]] = arith.minimumf %[[ABS_REAL]], %[[ABS_IMAG]] fastmath<nnan,contract> : f32
+// CHECK: %[[ABSRE:.*]] = math.absf %[[RE]] fastmath<nnan,contract> : f32
+// CHECK: %[[ABSIM:.*]] = math.absf %[[IM]] fastmath<nnan,contract> : f32
+// CHECK: %[[MAX:.*]] = arith.maximumf %[[ABSRE]], %[[ABSIM]] fastmath<nnan,contract> : f32
+// CHECK: %[[MIN:.*]] = arith.minimumf %[[ABSRE]], %[[ABSIM]] fastmath<nnan,contract> : f32
 // CHECK: %[[RATIO:.*]] = arith.divf %[[MIN]], %[[MAX]] fastmath<nnan,contract> : f32
 // CHECK: %[[RATIO_SQ:.*]] = arith.mulf %[[RATIO]], %[[RATIO]] fastmath<nnan,contract> : f32
 // CHECK: %[[RATIO_SQ_PLUS_ONE:.*]] = arith.addf %[[RATIO_SQ]], %[[ONE]] fastmath<nnan,contract> : f32
-// CHECK: %[[SQRT:.*]] = math.sqrt %[[RATIO_SQ_PLUS_ONE]] fastmath<nnan,contract> : f32
-// CHECK: %[[ABS_OR_NAN:.*]] = arith.mulf %[[MAX]], %[[SQRT]] fastmath<nnan,contract> : f32
-// CHECK: %[[IS_NAN:.*]] = arith.cmpf uno, %[[ABS_OR_NAN]], %[[ABS_OR_NAN]] fastmath<nnan,contract> : f32
-// CHECK: %[[ABS:.*]] = arith.select %[[IS_NAN]], %[[MIN]], %[[ABS_OR_NAN]] : f32
-// CHECK: %[[VAR23:.*]] = arith.addf %[[VAR2]], %[[ABS]] fastmath<nnan,contract> : f32
-// CHECK: %[[CST2:.*]]  = arith.constant 5.000000e-01 : f32
-// CHECK: %[[VAR24:.*]] = arith.mulf %[[VAR23]], %[[CST2]] fastmath<nnan,contract> : f32
-// CHECK: %[[VAR25:.*]] = math.sqrt %[[VAR24]] fastmath<nnan,contract> : f32
-// CHECK: %[[VAR26:.*]] = arith.cmpf olt, %[[VAR0]], %cst : f32
-// CHECK: %[[VAR27:.*]] = arith.cmpf olt, %[[VAR1]], %cst : f32
-// CHECK: %[[VAR28:.*]] = arith.addf %[[VAR25]], %[[VAR25]] fastmath<nnan,contract> : f32
-// CHECK: %[[VAR29:.*]] = arith.divf %[[VAR1]], %[[VAR28]] fastmath<nnan,contract> : f32
-// CHECK: %[[VAR30:.*]] = arith.negf %[[VAR25]] : f32
-// CHECK: %[[VAR31:.*]] = arith.select %[[VAR27]], %[[VAR30]], %[[VAR25]] : f32
-// CHECK: %[[VAR32:.*]] = arith.select %[[VAR26]], %[[VAR31]], %[[VAR29]] : f32
-// CHECK: %[[VAR33:.*]] = arith.addf %[[VAR32]], %[[VAR32]] fastmath<nnan,contract> : f32
-// CHECK: %[[VAR34:.*]] = arith.divf %[[VAR1]], %[[VAR33]] fastmath<nnan,contract> : f32
-// CHECK: %[[VAR35:.*]] = arith.select %[[VAR26]], %[[VAR34]], %[[VAR25]] : f32
-// CHECK: %[[VAR36:.*]] = arith.cmpf oeq, %[[VAR0]], %cst : f32
-// CHECK: %[[VAR37:.*]] = arith.cmpf oeq, %[[VAR1]], %cst : f32
-// CHECK: %[[VAR38:.*]] = arith.andi %[[VAR36]], %[[VAR37]] : i1
-// CHECK: %[[VAR39:.*]] = arith.select %[[VAR38]], %cst, %[[VAR35]] : f32
-// CHECK: %[[VAR40:.*]] = arith.select %[[VAR38]], %cst, %[[VAR32]] : f32
-// CHECK: %[[VAR41:.*]] = complex.create %[[VAR39]], %[[VAR40]] : complex<f32>
-// CHECK: return %[[VAR41]] : complex<f32>
+// CHECK: %[[QUARTER:.*]] = arith.constant 2.500000e-01 : f32
+// CHECK: %[[SQRT_MAX:.*]] = math.sqrt %[[MAX]] fastmath<nnan,contract> : f32
+// CHECK: %[[POW:.*]] = math.powf %[[RATIO_SQ_PLUS_ONE]], %[[QUARTER]] fastmath<nnan,contract> : f32
+// CHECK: %[[SQRT_ABS_OR_NAN:.*]] = arith.mulf %[[SQRT_MAX]], %[[POW]] fastmath<nnan,contract> : f32
+// CHECK: %[[IS_NAN:.*]] = arith.cmpf uno, %[[SQRT_ABS_OR_NAN]], %[[SQRT_ABS_OR_NAN]] fastmath<nnan,contract> : f32
+// CHECK: %[[SQRT_ABS:.*]] = arith.select %[[IS_NAN]], %[[MIN]], %[[SQRT_ABS_OR_NAN]] : f32
+// CHECK: %[[ARGARG:.*]] = math.atan2 %[[IM]], %[[RE]] fastmath<nnan,contract> : f32
+// CHECK: %[[SQRTARG:.*]] = arith.mulf %[[ARGARG]], %[[HALF]] fastmath<nnan,contract> : f32
+// CHECK: %[[COS:.*]] = math.cos %[[SQRTARG]] fastmath<nnan,contract> : f32
+// CHECK: %[[SIN:.*]] = math.sin %[[SQRTARG]] fastmath<nnan,contract> : f32
+// CHECK: %[[SIN_ZERO:.*]] = arith.cmpf oeq, %[[SIN]], %[[ZERO]] fastmath<nnan,contract> : f32
+// CHECK: %[[RESULT_RE:.*]] = arith.mulf %[[SQRT_ABS]], %[[COS]] fastmath<nnan,contract> : f32
+// CHECK: %[[RESULT_IM:.*]] = arith.mulf %[[SQRT_ABS]], %[[SIN]] fastmath<nnan,contract> : f32
+// CHECK: %[[RESULT_IM2:.*]] = arith.select %[[SIN_ZERO]], %[[ZERO]], %[[RESULT_IM]] : f32
+// CHECK: %[[INF:.*]] = arith.constant 0x7F800000 : f32
+// CHECK: %[[NINF:.*]] = arith.constant 0xFF800000 : f32
+// CHECK: %[[NAN:.*]] = arith.constant 0x7FC00000 : f32
+// CHECK: %[[ABSIM:.*]] = math.absf %[[IM]] fastmath<nnan,contract> : f32
+// CHECK: %[[ABSIMINF:.*]] = arith.cmpf oeq, %[[ABSIM]], %[[INF]] fastmath<nnan,contract> : f32
+// CHECK: %[[ABSIMNOTINF:.*]] = arith.cmpf one, %[[ABSIM]], %[[INF]] fastmath<nnan,contract> : f32
+// CHECK: %[[REINF:.*]] = arith.cmpf oeq, %[[RE]], %[[INF]] fastmath<nnan,contract> : f32
+// CHECK: %[[RENINF:.*]] = arith.cmpf oeq, %[[RE]], %[[NINF]] fastmath<nnan,contract> : f32
+// CHECK: %[[RESULT_RE_ZERO:.*]] = arith.andi %[[RENINF]], %[[ABSIMNOTINF]] : i1
+// CHECK: %[[RESULT_RE2:.*]] = arith.select %[[RESULT_RE_ZERO]], %[[ZERO]], %[[RESULT_RE]] : f32
+// CHECK: %[[RESUL_IM_INF:.*]] = arith.ori %[[ABSIMINF]], %[[REINF]] : i1
+// CHECK: %[[RESULT_RE3:.*]] = arith.select %[[RESUL_IM_INF]], %[[INF]], %[[RESULT_RE2]] : f32
+// CHECK: %[[INF_IM_SIGN:.*]] = math.copysign %[[INF]], %[[IM]] fastmath<nnan,contract> : f32
+// CHECK: %[[RESULT_IM_NAN:.*]] = arith.cmpf uno, %[[SQRT_ABS]], %[[SQRT_ABS]] : f32
+// CHECK: %[[RESULT_IM3:.*]] = arith.select %[[RESULT_IM_NAN]], %[[NAN]], %[[RESULT_IM2]] : f32
+// CHECK: %[[RESULT_IM_INF:.*]] = arith.ori %[[ABSIMINF]], %[[RENINF]] : i1
+// CHECK: %[[RESULT_IM4:.*]] = arith.select %[[RESULT_IM_INF]], %[[INF_IM_SIGN]], %[[RESULT_IM3]] : f32
+// CHECK: %[[RESULT_ZERO:.*]] = arith.cmpf oeq, %[[SQRT_ABS]], %[[ZERO]] fastmath<nnan,contract> : f32
+// CHECK: %[[RESULT_RE4:.*]] = arith.select %[[RESULT_ZERO]], %[[ZERO]], %[[RESULT_RE3]] : f32
+// CHECK: %[[RESULT_IM5:.*]] = arith.select %[[RESULT_ZERO]], %[[ZERO]], %[[RESULT_IM4]] : f32
+// CHECK: %[[RESULT:.*]] = complex.create %[[RESULT_RE4]], %[[RESULT_IM5]] : complex<f32>
+// CHECK: return %[[RESULT]] : complex<f32>
 
 // -----
 
@@ -1857,9 +1925,9 @@ func.func @complex_sign_with_fmf(%arg: complex<f32>) -> complex<f32> {
 // CHECK: %[[REAL_IS_ZERO:.*]] = arith.cmpf oeq, %[[REAL]], %[[ZERO]] : f32
 // CHECK: %[[IMAG_IS_ZERO:.*]] = arith.cmpf oeq, %[[IMAG]], %[[ZERO]] : f32
 // CHECK: %[[IS_ZERO:.*]] = arith.andi %[[REAL_IS_ZERO]], %[[IMAG_IS_ZERO]] : i1
-// CHECK: %[[ONE:.*]] = arith.constant 1.000000e+00 : f32
 // CHECK: %[[REAL2:.*]] = complex.re %[[ARG]] : complex<f32>
 // CHECK: %[[IMAG2:.*]] = complex.im %[[ARG]] : complex<f32>
+// CHECK: %[[ONE:.*]] = arith.constant 1.000000e+00 : f32
 // CHECK: %[[ABS_REAL:.*]] = math.absf %[[REAL2]] fastmath<nnan,contract> : f32
 // CHECK: %[[ABS_IMAG:.*]] = math.absf %[[IMAG2]] fastmath<nnan,contract> : f32
 // CHECK: %[[MAX:.*]] = arith.maximumf %[[ABS_REAL]], %[[ABS_IMAG]] fastmath<nnan,contract> : f32
diff --git a/mlir/test/Conversion/ComplexToStandard/full-conversion.mlir b/mlir/test/Conversion/ComplexToStandard/full-conversion.mlir
index 2649d004a76a..110a78631fb9 100644
--- a/mlir/test/Conversion/ComplexToStandard/full-conversion.mlir
+++ b/mlir/test/Conversion/ComplexToStandard/full-conversion.mlir
@@ -6,10 +6,10 @@ func.func @complex_abs(%arg: complex<f32>) -> f32 {
   %abs = complex.abs %arg: complex<f32>
   return %abs : f32
 }
-// CHECK: %[[ONE:.*]] = llvm.mlir.constant(1.000000e+00 : f32) : f32
 // CHECK: %[[REAL:.*]] = llvm.extractvalue %[[ARG]][0] : ![[C_TY]]
 // CHECK: %[[IMAG:.*]] = llvm.extractvalue %[[ARG]][1] : ![[C_TY]]
 
+// CHECK: %[[ONE:.*]] = llvm.mlir.constant(1.000000e+00 : f32) : f32
 // CHECK: %[[ABS_REAL:.*]] = llvm.intr.fabs(%[[REAL]]) : (f32) -> f32
 // CHECK: %[[ABS_IMAG:.*]] = llvm.intr.fabs(%[[IMAG]]) : (f32) -> f32
 // CHECK: %[[MAX:.*]] = llvm.intr.maximum(%[[ABS_REAL]], %[[ABS_IMAG]]) : (f32, f32) -> f32
-- 
GitLab


From 1e7763557bbace002b6bb811167942ebebf50f99 Mon Sep 17 00:00:00 2001
From: wanglei <wanglei@loongson.cn>
Date: Fri, 12 Apr 2024 16:15:02 +0800
Subject: [PATCH 037/464] [LoongArch] Add support for getNumberOfRegisters()
 (#88372)

The `TTI` hooks are used during vectorization for calculating register
pressure. The default implementation defined wrong value for register
number (all register class are 8 registers).

This patch also defines LoongArch's own register classes.
---
 .../LoongArchTargetTransformInfo.cpp          | 41 +++++++++++++++++
 .../LoongArch/LoongArchTargetTransformInfo.h  |  4 ++
 .../LoopVectorize/LoongArch/reg-usage.ll      | 44 +++++++++++++++++++
 3 files changed, 89 insertions(+)
 create mode 100644 llvm/test/Transforms/LoopVectorize/LoongArch/reg-usage.ll

diff --git a/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.cpp b/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.cpp
index d47dded9ea6e..add1c60d89d2 100644
--- a/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.cpp
@@ -40,4 +40,45 @@ TypeSize LoongArchTTIImpl::getRegisterBitWidth(
   llvm_unreachable("Unsupported register kind");
 }
 
+unsigned LoongArchTTIImpl::getNumberOfRegisters(unsigned ClassID) const {
+  switch (ClassID) {
+  case LoongArchRegisterClass::GPRRC:
+    // 30 = 32 GPRs - r0 (zero register) - r21 (non-allocatable)
+    return 30;
+  case LoongArchRegisterClass::FPRRC:
+    return ST->hasBasicF() ? 32 : 0;
+  case LoongArchRegisterClass::VRRC:
+    return ST->hasExtLSX() ? 32 : 0;
+  }
+  llvm_unreachable("unknown register class");
+}
+
+unsigned LoongArchTTIImpl::getRegisterClassForType(bool Vector,
+                                                   Type *Ty) const {
+  if (Vector)
+    return LoongArchRegisterClass::VRRC;
+  if (!Ty)
+    return LoongArchRegisterClass::GPRRC;
+
+  Type *ScalarTy = Ty->getScalarType();
+  if ((ScalarTy->isFloatTy() && ST->hasBasicF()) ||
+      (ScalarTy->isDoubleTy() && ST->hasBasicD())) {
+    return LoongArchRegisterClass::FPRRC;
+  }
+
+  return LoongArchRegisterClass::GPRRC;
+}
+
+const char *LoongArchTTIImpl::getRegisterClassName(unsigned ClassID) const {
+  switch (ClassID) {
+  case LoongArchRegisterClass::GPRRC:
+    return "LoongArch::GPRRC";
+  case LoongArchRegisterClass::FPRRC:
+    return "LoongArch::FPRRC";
+  case LoongArchRegisterClass::VRRC:
+    return "LoongArch::VRRC";
+  }
+  llvm_unreachable("unknown register class");
+}
+
 // TODO: Implement more hooks to provide TTI machinery for LoongArch.
diff --git a/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.h b/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.h
index d296c9ed576f..34c18163bbdb 100644
--- a/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.h
+++ b/llvm/lib/Target/LoongArch/LoongArchTargetTransformInfo.h
@@ -28,6 +28,7 @@ class LoongArchTTIImpl : public BasicTTIImplBase<LoongArchTTIImpl> {
   typedef TargetTransformInfo TTI;
   friend BaseT;
 
+  enum LoongArchRegisterClass { GPRRC, FPRRC, VRRC };
   const LoongArchSubtarget *ST;
   const LoongArchTargetLowering *TLI;
 
@@ -40,6 +41,9 @@ public:
         TLI(ST->getTargetLowering()) {}
 
   TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const;
+  unsigned getNumberOfRegisters(unsigned ClassID) const;
+  unsigned getRegisterClassForType(bool Vector, Type *Ty = nullptr) const;
+  const char *getRegisterClassName(unsigned ClassID) const;
 
   // TODO: Implement more hooks to provide TTI machinery for LoongArch.
 };
diff --git a/llvm/test/Transforms/LoopVectorize/LoongArch/reg-usage.ll b/llvm/test/Transforms/LoopVectorize/LoongArch/reg-usage.ll
new file mode 100644
index 000000000000..f45a2f0f5b7e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/LoongArch/reg-usage.ll
@@ -0,0 +1,44 @@
+; REQUIRES: asserts
+; RUN: opt --passes=loop-vectorize --mtriple loongarch64-linux-gnu \
+; RUN:   --mattr=+lsx -debug-only=loop-vectorize --force-vector-width=1 \
+; RUN:   -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK-SCALAR
+; RUN: opt --passes=loop-vectorize --mtriple loongarch64-linux-gnu \
+; RUN:   --mattr=+lsx -debug-only=loop-vectorize --force-vector-width=4 \
+; RUN:   -S < %s 2>&1 | FileCheck %s --check-prefix=CHECK-VECTOR
+
+define void @bar(ptr %A, i32 signext %n) {
+; CHECK-LABEL: bar
+; CHECK-SCALAR:      LV(REG): Found max usage: 2 item
+; CHECK-SCALAR-NEXT: LV(REG): RegisterClass: LoongArch::GPRRC, 2 registers
+; CHECK-SCALAR-NEXT: LV(REG): RegisterClass: LoongArch::FPRRC, 1 registers
+; CHECK-SCALAR-NEXT: LV(REG): Found invariant usage: 1 item
+; CHECK-SCALAR-NEXT: LV(REG): RegisterClass: LoongArch::GPRRC, 1 registers
+; CHECK-SCALAR-NEXT: LV: The target has 30 registers of LoongArch::GPRRC register class
+; CHECK-SCALAR-NEXT: LV: The target has 32 registers of LoongArch::FPRRC register class
+; CHECK-VECTOR:      LV(REG): Found max usage: 1 item
+; CHECK-VECTOR-NEXT: LV(REG): RegisterClass: LoongArch::VRRC, 3 registers
+; CHECK-VECTOR-NEXT: LV(REG): Found invariant usage: 1 item
+; CHECK-VECTOR-NEXT: LV(REG): RegisterClass: LoongArch::GPRRC, 1 registers
+; CHECK-VECTOR-NEXT: LV: The target has 32 registers of LoongArch::VRRC register class
+
+entry:
+  %cmp4 = icmp sgt i32 %n, 0
+  br i1 %cmp4, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:                               ; preds = %entry
+  %wide.trip.count = zext nneg i32 %n to i64
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body.preheader, %for.body
+  %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+  %0 = trunc i64 %indvars.iv to i32
+  %conv = sitofp i32 %0 to float
+  %arrayidx = getelementptr inbounds float, ptr %A, i64 %indvars.iv
+  store float %conv, ptr %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
-- 
GitLab


From a5ed14bc8e122fa5ac0aa81f8d8390931bd6b4e4 Mon Sep 17 00:00:00 2001
From: Phoebe Wang <phoebe.wang@intel.com>
Date: Fri, 12 Apr 2024 16:16:56 +0800
Subject: [PATCH 038/464] [X86][BF16] Fix a typo, NFCI

VR128X is not enabled under AVXNECONVERT.
---
 llvm/lib/Target/X86/X86InstrSSE.td | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/X86/X86InstrSSE.td b/llvm/lib/Target/X86/X86InstrSSE.td
index 2b391b60f2c9..5d799fc00df9 100644
--- a/llvm/lib/Target/X86/X86InstrSSE.td
+++ b/llvm/lib/Target/X86/X86InstrSSE.td
@@ -8314,7 +8314,7 @@ let Predicates = [HasAVXNECONVERT] in {
        f256mem>, T8;
   defm VCVTNEPS2BF16 : VCVTNEPS2BF16_BASE, VEX, T8, XS, ExplicitVEXPrefix;
 
-  def : Pat<(v8bf16 (X86cvtneps2bf16 (v4f32 VR128X:$src))),
+  def : Pat<(v8bf16 (X86cvtneps2bf16 (v4f32 VR128:$src))),
             (VCVTNEPS2BF16rr VR128:$src)>;
   def : Pat<(v8bf16 (X86cvtneps2bf16 (loadv4f32 addr:$src))),
             (VCVTNEPS2BF16rm addr:$src)>;
-- 
GitLab


From ad4e1aba3fd12f81de71ce3985ae66ff80773d90 Mon Sep 17 00:00:00 2001
From: jeanPerier <jperier@nvidia.com>
Date: Fri, 12 Apr 2024 10:29:01 +0200
Subject: [PATCH 039/464] [flang] Pass VALUE CHARACTER arg by register in
 BIND(C) calls (#87774)

Fortran mandates "CHARACTER(1), VALUE" be passed as a C "char" in calls
to BIND(C) procedures (F'2023 18.3.7 (4)). Lowering passed them by
memory instead. Update call interface lowering code to pass them by
register. Fix related test and update it to use HLFIR.
---
 flang/lib/Lower/CallInterface.cpp   |  24 ++++---
 flang/lib/Lower/ConvertCall.cpp     |  16 +++--
 flang/lib/Lower/ConvertVariable.cpp |   9 ++-
 flang/test/Lower/call-by-value.f90  | 106 +++++++++++++++++-----------
 4 files changed, 101 insertions(+), 54 deletions(-)

diff --git a/flang/lib/Lower/CallInterface.cpp b/flang/lib/Lower/CallInterface.cpp
index 29cdb3cff589..05a0c10c7097 100644
--- a/flang/lib/Lower/CallInterface.cpp
+++ b/flang/lib/Lower/CallInterface.cpp
@@ -1136,14 +1136,22 @@ private:
       addPassedArg(PassEntityBy::Box, entity, characteristics);
     } else if (dynamicType.category() ==
                Fortran::common::TypeCategory::Character) {
-      // Pass as fir.box_char
-      mlir::Type boxCharTy =
-          fir::BoxCharType::get(&mlirContext, dynamicType.kind());
-      addFirOperand(boxCharTy, nextPassedArgPosition(), Property::BoxChar,
-                    attrs);
-      addPassedArg(isValueAttr ? PassEntityBy::CharBoxValueAttribute
-                               : PassEntityBy::BoxChar,
-                   entity, characteristics);
+      if (isValueAttr && isBindC) {
+        // Pass as fir.char<1>
+        mlir::Type charTy =
+            fir::CharacterType::getSingleton(&mlirContext, dynamicType.kind());
+        addFirOperand(charTy, nextPassedArgPosition(), Property::Value, attrs);
+        addPassedArg(PassEntityBy::Value, entity, characteristics);
+      } else {
+        // Pass as fir.box_char
+        mlir::Type boxCharTy =
+            fir::BoxCharType::get(&mlirContext, dynamicType.kind());
+        addFirOperand(boxCharTy, nextPassedArgPosition(), Property::BoxChar,
+                      attrs);
+        addPassedArg(isValueAttr ? PassEntityBy::CharBoxValueAttribute
+                                 : PassEntityBy::BoxChar,
+                     entity, characteristics);
+      }
     } else {
       // Pass as fir.ref unless it's by VALUE and BIND(C). Also pass-by-value
       // for numerical/logical scalar without OPTIONAL so that the behavior is
diff --git a/flang/lib/Lower/ConvertCall.cpp b/flang/lib/Lower/ConvertCall.cpp
index 315a3f6736aa..c6f7d3410ad5 100644
--- a/flang/lib/Lower/ConvertCall.cpp
+++ b/flang/lib/Lower/ConvertCall.cpp
@@ -1494,11 +1494,19 @@ genUserCall(Fortran::lower::PreparedActualArguments &loweredActuals,
           value =
               hlfir::Entity{genRecordCPtrValueArg(builder, loc, value, eleTy)};
         }
-      } else if (fir::isa_derived(value.getFortranElementType())) {
-        // BIND(C), VALUE derived type. The derived type value must really
+      } else if (fir::isa_derived(value.getFortranElementType()) ||
+                 value.isCharacter()) {
+        // BIND(C), VALUE derived type or character. The value must really
         // be loaded here.
-        auto [derived, cleanup] = hlfir::convertToValue(loc, builder, value);
-        mlir::Value loadedValue = fir::getBase(derived);
+        auto [exv, cleanup] = hlfir::convertToValue(loc, builder, value);
+        mlir::Value loadedValue = fir::getBase(exv);
+        // Character actual arguments may have unknown length or a length longer
+        // than one. Cast the memory ref to the dummy type so that the load is
+        // valid and only loads what is needed.
+        if (mlir::Type baseTy = fir::dyn_cast_ptrEleTy(loadedValue.getType()))
+          if (fir::isa_char(baseTy))
+            loadedValue = builder.createConvert(
+                loc, fir::ReferenceType::get(argTy), loadedValue);
         if (fir::isa_ref_type(loadedValue.getType()))
           loadedValue = builder.create<fir::LoadOp>(loc, loadedValue);
         caller.placeInput(arg, loadedValue);
diff --git a/flang/lib/Lower/ConvertVariable.cpp b/flang/lib/Lower/ConvertVariable.cpp
index 01473f7f6cd7..2d2d9eba905b 100644
--- a/flang/lib/Lower/ConvertVariable.cpp
+++ b/flang/lib/Lower/ConvertVariable.cpp
@@ -2100,10 +2100,15 @@ void Fortran::lower::mapSymbolAttributes(
   if (ba.isChar()) {
     if (arg) {
       assert(!preAlloc && "dummy cannot be pre-allocated");
-      if (arg.getType().isa<fir::BoxCharType>())
+      if (mlir::isa<fir::BoxCharType>(arg.getType())) {
         std::tie(addr, len) = charHelp.createUnboxChar(arg);
-      else if (!addr)
+      } else if (mlir::isa<fir::CharacterType>(arg.getType())) {
+        // fir.char<1> passed by value (BIND(C) with VALUE attribute).
+        addr = builder.create<fir::AllocaOp>(loc, arg.getType());
+        builder.create<fir::StoreOp>(loc, arg, addr);
+      } else if (!addr) {
         addr = arg;
+      }
       // Ensure proper type is given to array/scalar that was transmitted as a
       // fir.boxchar arg or is a statement function actual argument with
       // a different length than the dummy.
diff --git a/flang/test/Lower/call-by-value.f90 b/flang/test/Lower/call-by-value.f90
index e489ea432305..d4694f4575d0 100644
--- a/flang/test/Lower/call-by-value.f90
+++ b/flang/test/Lower/call-by-value.f90
@@ -1,9 +1,10 @@
 ! Test for PassBy::Value
-! RUN: bbc -emit-fir -hlfir=false %s -o - | FileCheck %s
+! RUN: bbc -emit-fir %s -o - | FileCheck %s
 
 !CHECK-LABEL: func @_QQmain()
-!CHECK: %[[LOGICAL:.*]] = fir.alloca !fir.logical<1>
 !CHECK: %false = arith.constant false
+!CHECK: %[[LOGICAL_ALLOC:.*]] = fir.alloca !fir.logical<1>
+!CHECK: %[[LOGICAL:.*]] = fir.declare %[[LOGICAL_ALLOC]]
 !CHECK: %[[VALUE:.*]] = fir.convert %false : (i1) -> !fir.logical<1>
 !CHECK: fir.store %[[VALUE]] to %[[LOGICAL]]
 !CHECK: %[[LOAD:.*]] = fir.load %[[LOGICAL]]
@@ -23,39 +24,42 @@ program call_by_value
   call omp_set_nested(do_nested)
 end program call_by_value
 
-! CHECK-LABEL: func.func @test_integer_value(
-! CHECK-SAME:                                %[[VAL_0:.*]]: i32 {fir.bindc_name = "x"}) attributes {fir.bindc_name = "test_integer_value"} {
-! CHECK:         %[[VAL_1:.*]] = fir.alloca i32
-! CHECK:         fir.store %[[VAL_0]] to %[[VAL_1]] : !fir.ref<i32>
-! CHECK:         fir.call @_QPinternal_call(%[[VAL_1]]) {{.*}}: (!fir.ref<i32>) -> ()
-! CHECK:         return
-! CHECK:       }
+! CHECK-LABEL:   func.func @test_integer_value(
+! CHECK-SAME:                                  %[[VAL_0:.*]]: i32
+! CHECK:           %[[VAL_1:.*]] = fir.alloca i32
+! CHECK:           fir.store %[[VAL_0]] to %[[VAL_1]] : !fir.ref<i32>
+! CHECK:           %[[VAL_2:.*]] = fir.declare %[[VAL_1]]
+! CHECK:           fir.call @_QPinternal_call(%[[VAL_2]]) {{.*}}: (!fir.ref<i32>) -> ()
+! CHECK:           return
+! CHECK:         }
 
 subroutine test_integer_value(x) bind(c)
   integer, value :: x
   call internal_call(x)
 end
+! CHECK-LABEL:   func.func @test_real_value(
+! CHECK-SAME:                               %[[VAL_0:.*]]: f32
+! CHECK:           %[[VAL_1:.*]] = fir.alloca f32
+! CHECK:           fir.store %[[VAL_0]] to %[[VAL_1]] : !fir.ref<f32>
+! CHECK:           %[[VAL_2:.*]] = fir.declare %[[VAL_1]]
+! CHECK:           fir.call @_QPinternal_call2(%[[VAL_2]]) {{.*}}: (!fir.ref<f32>) -> ()
+! CHECK:           return
+! CHECK:         }
 
-! CHECK-LABEL: func.func @test_real_value(
-! CHECK-SAME:                             %[[VAL_0:.*]]: f32 {fir.bindc_name = "x"}) attributes {fir.bindc_name = "test_real_value"} {
-! CHECK:         %[[VAL_1:.*]] = fir.alloca f32
-! CHECK:         fir.store %[[VAL_0]] to %[[VAL_1]] : !fir.ref<f32>
-! CHECK:         fir.call @_QPinternal_call2(%[[VAL_1]]) {{.*}}: (!fir.ref<f32>) -> ()
-! CHECK:         return
-! CHECK:       }
 
 subroutine test_real_value(x) bind(c)
   real, value :: x
   call internal_call2(x)
 end
+! CHECK-LABEL:   func.func @test_complex_value(
+! CHECK-SAME:                                  %[[VAL_0:.*]]: !fir.complex<4>
+! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.complex<4>
+! CHECK:           fir.store %[[VAL_0]] to %[[VAL_1]] : !fir.ref<!fir.complex<4>>
+! CHECK:           %[[VAL_2:.*]] = fir.declare %[[VAL_1]]
+! CHECK:           fir.call @_QPinternal_call3(%[[VAL_2]]) {{.*}}: (!fir.ref<!fir.complex<4>>) -> ()
+! CHECK:           return
+! CHECK:         }
 
-! CHECK-LABEL: func.func @test_complex_value(
-! CHECK-SAME:                                %[[VAL_0:.*]]: !fir.complex<4> {fir.bindc_name = "x"}) attributes {fir.bindc_name = "test_complex_value"} {
-! CHECK:         %[[VAL_1:.*]] = fir.alloca !fir.complex<4>
-! CHECK:         fir.store %[[VAL_0]] to %[[VAL_1]] : !fir.ref<!fir.complex<4>>
-! CHECK:         fir.call @_QPinternal_call3(%[[VAL_1]]) {{.*}}: (!fir.ref<!fir.complex<4>>) -> ()
-! CHECK:         return
-! CHECK:       }
 
 subroutine test_complex_value(x) bind(c)
   complex, value :: x
@@ -63,12 +67,13 @@ subroutine test_complex_value(x) bind(c)
 end
 
 ! CHECK-LABEL:   func.func @test_char_value(
-! CHECK-SAME:                               %[[VAL_0:.*]]: !fir.boxchar<1> {fir.bindc_name = "x"}) attributes {fir.bindc_name = "test_char_value"} {
-! CHECK:           %[[VAL_1:.*]]:2 = fir.unboxchar %[[VAL_0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
-! CHECK:           %[[VAL_3:.*]] = fir.convert %[[VAL_1]]#0 : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1>>
-! CHECK:           %[[VAL_2:.*]] = arith.constant 1 : index
-! CHECK:           %[[VAL_5:.*]] = fir.emboxchar %[[VAL_3]], %[[VAL_2]] : (!fir.ref<!fir.char<1>>, index) -> !fir.boxchar<1>
-! CHECK:           fir.call @_QPinternal_call4(%[[VAL_5]]) fastmath<contract> : (!fir.boxchar<1>) -> ()
+! CHECK-SAME:                               %[[VAL_0:.*]]: !fir.char<1>
+! CHECK:           %[[VAL_1:.*]] = arith.constant 1 : index
+! CHECK:           %[[VAL_2:.*]] = fir.alloca !fir.char<1>
+! CHECK:           fir.store %[[VAL_0]] to %[[VAL_2]] : !fir.ref<!fir.char<1>>
+! CHECK:           %[[VAL_3:.*]] = fir.declare %[[VAL_2]] typeparams %[[VAL_1]]
+! CHECK:           %[[VAL_4:.*]] = fir.emboxchar %[[VAL_3]], %[[VAL_1]] : (!fir.ref<!fir.char<1>>, index) -> !fir.boxchar<1>
+! CHECK:           fir.call @_QPinternal_call4(%[[VAL_4]]) {{.*}}: (!fir.boxchar<1>) -> ()
 ! CHECK:           return
 ! CHECK:         }
 
@@ -77,19 +82,40 @@ subroutine test_char_value(x) bind(c)
   call internal_call4(x)
 end
 
-! CHECK-LABEL: func.func @_QPtest_cptr_value(
-! CHECK-SAME:                                %[[VAL_0:.*]]: !fir.ref<i64> {fir.bindc_name = "x"}) {
-! CHECK:         %[[VAL_1:.*]] = fir.alloca !fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>
-! CHECK:         %[[VAL_2:.*]] = fir.field_index __address, !fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>
-! CHECK:         %[[VAL_3:.*]] = fir.coordinate_of %[[VAL_1]], %[[VAL_2]] : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.field) -> !fir.ref<i64>
-! CHECK:         %[[VAL_4:.*]] = fir.convert %[[VAL_0]] : (!fir.ref<i64>) -> i64
-! CHECK:         fir.store %[[VAL_4]] to %[[VAL_3]] : !fir.ref<i64>
-! CHECK:         fir.call @_QPinternal_call5(%[[VAL_1]]) {{.*}}: (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>) -> ()
-! CHECK:         return
-! CHECK:       }
+! CHECK-LABEL:   func.func @_QPtest_call_char_value(
+! CHECK-SAME:                                       %[[VAL_0:.*]]: !fir.boxchar<1>
+! CHECK:           %[[VAL_1:.*]]:2 = fir.unboxchar %[[VAL_0]] : (!fir.boxchar<1>) -> (!fir.ref<!fir.char<1,?>>, index)
+! CHECK:           %[[VAL_2:.*]] = fir.declare %[[VAL_1]]#0 typeparams %[[VAL_1]]#1
+! CHECK:           %[[VAL_3:.*]] = fir.emboxchar %[[VAL_2]], %[[VAL_1]]#1 : (!fir.ref<!fir.char<1,?>>, index) -> !fir.boxchar<1>
+! CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<!fir.char<1,?>>) -> !fir.ref<!fir.char<1>>
+! CHECK:           %[[VAL_5:.*]] = fir.load %[[VAL_4]] : !fir.ref<!fir.char<1>>
+! CHECK:           fir.call @test_char_value(%[[VAL_5]]) {{.*}}: (!fir.char<1>) -> ()
+! CHECK:           return
+! CHECK:         }
+subroutine test_call_char_value(x)
+  character(*) :: x
+  interface
+    subroutine test_char_value(x) bind(c)
+      character(1), value :: x
+    end
+  end interface
+  call test_char_value(x)
+end subroutine
+
+! CHECK-LABEL:   func.func @_QPtest_cptr_value(
+! CHECK-SAME:                                  %[[VAL_0:.*]]: !fir.ref<i64>
+! CHECK:           %[[VAL_1:.*]] = fir.alloca !fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>
+! CHECK:           %[[VAL_2:.*]] = fir.field_index __address, !fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>
+! CHECK:           %[[VAL_3:.*]] = fir.coordinate_of %[[VAL_1]], %[[VAL_2]] : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, !fir.field) -> !fir.ref<i64>
+! CHECK:           %[[VAL_4:.*]] = fir.convert %[[VAL_0]] : (!fir.ref<i64>) -> i64
+! CHECK:           fir.store %[[VAL_4]] to %[[VAL_3]] : !fir.ref<i64>
+! CHECK:           %[[VAL_5:.*]] = fir.declare %[[VAL_1]]
+! CHECK:           fir.call @_QPinternal_call5(%[[VAL_5]]) fastmath<contract> : (!fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>) -> ()
+! CHECK:           return
+! CHECK:         }
 
 subroutine test_cptr_value(x)
-  use iso_c_binding
+  use iso_c_binding, only: c_ptr
   type(c_ptr), value :: x
   call internal_call5(x)
 end
-- 
GitLab


From d488b2225d484027eb2c21a06d27decf008e878b Mon Sep 17 00:00:00 2001
From: Beal Wang <colorfishes@outlook.com>
Date: Fri, 12 Apr 2024 16:37:50 +0800
Subject: [PATCH 040/464] [mlir][ods] Do not print default-valued properties
 when the value is equal to the default (#87970)

This diff causes the `tblgen`-erated printProperties() function to skip
printing a `DefaultValuedAttr` property when the value is equal to the
default.

Co-authored-by: Biao Wang <biaow@nvidia.com>
---
 .../include/mlir/Dialect/XeGPU/IR/XeGPUOps.td |  3 +-
 mlir/include/mlir/IR/OpDefinition.h           | 31 ++++++++++++-------
 mlir/lib/IR/Operation.cpp                     | 30 ++++++++++++++----
 mlir/test/IR/properties.mlir                  |  5 +++
 mlir/test/lib/Dialect/Test/TestOps.td         | 17 +++++++---
 mlir/tools/mlir-tblgen/OpFormatGen.cpp        | 23 +++++++++++++-
 6 files changed, 85 insertions(+), 24 deletions(-)

diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td
index 93c56ad05b43..b8ebd1a40c60 100644
--- a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td
+++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td
@@ -27,7 +27,8 @@ class XeGPU_Op<string mnemonic, list<Trait> traits = []>:
 
   code extraBaseClassDeclaration = [{
     void printProperties(::mlir::MLIRContext *ctx,
-            ::mlir::OpAsmPrinter &p, const Properties &prop) {
+            ::mlir::OpAsmPrinter &p, const Properties &prop,
+            ::mlir::ArrayRef<::llvm::StringRef> elidedProps) {
       Attribute propAttr = getPropertiesAsAttr(ctx, prop);
       if (propAttr)
         p << "<" << propAttr << ">";
diff --git a/mlir/include/mlir/IR/OpDefinition.h b/mlir/include/mlir/IR/OpDefinition.h
index 2d1dee2303e8..f4355c8ce26a 100644
--- a/mlir/include/mlir/IR/OpDefinition.h
+++ b/mlir/include/mlir/IR/OpDefinition.h
@@ -226,8 +226,10 @@ protected:
   static ParseResult genericParseProperties(OpAsmParser &parser,
                                             Attribute &result);
 
-  /// Print the properties as a Attribute.
-  static void genericPrintProperties(OpAsmPrinter &p, Attribute properties);
+  /// Print the properties as a Attribute with names not included within
+  /// 'elidedProps'
+  static void genericPrintProperties(OpAsmPrinter &p, Attribute properties,
+                                     ArrayRef<StringRef> elidedProps = {});
 
   /// Print an operation name, eliding the dialect prefix if necessary.
   static void printOpName(Operation *op, OpAsmPrinter &p,
@@ -1805,10 +1807,13 @@ private:
   template <typename T>
   using detect_has_print = llvm::is_detected<has_print, T>;
 
-  /// Trait to check if printProperties(OpAsmPrinter, T) exist
+  /// Trait to check if printProperties(OpAsmPrinter, T, ArrayRef<StringRef>)
+  /// exist
   template <typename T, typename... Args>
-  using has_print_properties = decltype(printProperties(
-      std::declval<OpAsmPrinter &>(), std::declval<T>()));
+  using has_print_properties =
+      decltype(printProperties(std::declval<OpAsmPrinter &>(),
+                               std::declval<T>(),
+                               std::declval<ArrayRef<StringRef>>()));
   template <typename T>
   using detect_has_print_properties =
       llvm::is_detected<has_print_properties, T>;
@@ -1974,16 +1979,18 @@ public:
   static void populateDefaultProperties(OperationName opName,
                                         InferredProperties<T> &properties) {}
 
-  /// Print the operation properties. Unless overridden, this method will try to
-  /// dispatch to a `printProperties` free-function if it exists, and otherwise
-  /// by converting the properties to an Attribute.
+  /// Print the operation properties with names not included within
+  /// 'elidedProps'. Unless overridden, this method will try to dispatch to a
+  /// `printProperties` free-function if it exists, and otherwise by converting
+  /// the properties to an Attribute.
   template <typename T>
   static void printProperties(MLIRContext *ctx, OpAsmPrinter &p,
-                              const T &properties) {
+                              const T &properties,
+                              ArrayRef<StringRef> elidedProps = {}) {
     if constexpr (detect_has_print_properties<T>::value)
-      return printProperties(p, properties);
-    genericPrintProperties(p,
-                           ConcreteType::getPropertiesAsAttr(ctx, properties));
+      return printProperties(p, properties, elidedProps);
+    genericPrintProperties(
+        p, ConcreteType::getPropertiesAsAttr(ctx, properties), elidedProps);
   }
 
   /// Parser the properties. Unless overridden, this method will print by
diff --git a/mlir/lib/IR/Operation.cpp b/mlir/lib/IR/Operation.cpp
index ca5ff9f72e3e..db903d540761 100644
--- a/mlir/lib/IR/Operation.cpp
+++ b/mlir/lib/IR/Operation.cpp
@@ -790,15 +790,33 @@ void OpState::printOpName(Operation *op, OpAsmPrinter &p,
 /// Parse properties as a Attribute.
 ParseResult OpState::genericParseProperties(OpAsmParser &parser,
                                             Attribute &result) {
-  if (parser.parseLess() || parser.parseAttribute(result) ||
-      parser.parseGreater())
-    return failure();
+  if (succeeded(parser.parseOptionalLess())) { // The less is optional.
+    if (parser.parseAttribute(result) || parser.parseGreater())
+      return failure();
+  }
   return success();
 }
 
-/// Print the properties as a Attribute.
-void OpState::genericPrintProperties(OpAsmPrinter &p, Attribute properties) {
-  p << "<" << properties << ">";
+/// Print the properties as a Attribute with names not included within
+/// 'elidedProps'
+void OpState::genericPrintProperties(OpAsmPrinter &p, Attribute properties,
+                                     ArrayRef<StringRef> elidedProps) {
+  auto dictAttr = dyn_cast_or_null<::mlir::DictionaryAttr>(properties);
+  if (dictAttr && !elidedProps.empty()) {
+    ArrayRef<NamedAttribute> attrs = dictAttr.getValue();
+    llvm::SmallDenseSet<StringRef> elidedAttrsSet(elidedProps.begin(),
+                                                  elidedProps.end());
+    bool atLeastOneAttr = llvm::any_of(attrs, [&](NamedAttribute attr) {
+      return !elidedAttrsSet.contains(attr.getName().strref());
+    });
+    if (atLeastOneAttr) {
+      p << "<";
+      p.printOptionalAttrDict(dictAttr.getValue(), elidedProps);
+      p << ">";
+    }
+  } else {
+    p << "<" << properties << ">";
+  }
 }
 
 /// Emit an error about fatal conditions with this operation, reporting up to
diff --git a/mlir/test/IR/properties.mlir b/mlir/test/IR/properties.mlir
index 3c4bd57859ef..1d22cb1940f2 100644
--- a/mlir/test/IR/properties.mlir
+++ b/mlir/test/IR/properties.mlir
@@ -33,3 +33,8 @@ test.using_property_in_custom [1, 4, 20]
 // GENERIC-SAME: second = 4
 // GENERIC-SAME: }>
 test.using_property_ref_in_custom 1 + 4 = 5
+
+// CHECK:   test.with_default_valued_properties {{$}}
+// GENERIC: "test.with_default_valued_properties"()
+// GENERIC-SAME:  <{a = 0 : i32}>
+test.with_default_valued_properties <{a = 0 : i32}>
diff --git a/mlir/test/lib/Dialect/Test/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td
index e6c3601d08da..edca05fde5a5 100644
--- a/mlir/test/lib/Dialect/Test/TestOps.td
+++ b/mlir/test/lib/Dialect/Test/TestOps.td
@@ -2909,7 +2909,8 @@ def TestOpWithNiceProperties : TEST_Op<"with_nice_properties"> {
   );
   let extraClassDeclaration = [{
     void printProperties(::mlir::MLIRContext *ctx, ::mlir::OpAsmPrinter &p,
-                         const Properties &prop);
+                         const Properties &prop,
+                         ::mlir::ArrayRef<::llvm::StringRef> elidedProps);
     static ::mlir::ParseResult  parseProperties(::mlir::OpAsmParser &parser,
                                      ::mlir::OperationState &result);
     static ::mlir::LogicalResult readFromMlirBytecode(
@@ -2938,7 +2939,8 @@ def TestOpWithNiceProperties : TEST_Op<"with_nice_properties"> {
       writer.writeVarInt(prop.value);
     }
     void TestOpWithNiceProperties::printProperties(::mlir::MLIRContext *ctx,
-            ::mlir::OpAsmPrinter &p, const Properties &prop) {
+            ::mlir::OpAsmPrinter &p, const Properties &prop,
+            ::mlir::ArrayRef<::llvm::StringRef> elidedProps) {
       customPrintProperties(p, prop.prop);
     }
     ::mlir::ParseResult TestOpWithNiceProperties::parseProperties(
@@ -2971,7 +2973,8 @@ def TestOpWithVersionedProperties : TEST_Op<"with_versioned_properties"> {
   );
   let extraClassDeclaration = [{
     void printProperties(::mlir::MLIRContext *ctx, ::mlir::OpAsmPrinter &p,
-                         const Properties &prop);
+                         const Properties &prop,
+                         ::mlir::ArrayRef<::llvm::StringRef> elidedProps);
     static ::mlir::ParseResult  parseProperties(::mlir::OpAsmParser &parser,
                                      ::mlir::OperationState &result);
     static ::mlir::LogicalResult readFromMlirBytecode(
@@ -2983,7 +2986,8 @@ def TestOpWithVersionedProperties : TEST_Op<"with_versioned_properties"> {
   }];
   let extraClassDefinition = [{
     void TestOpWithVersionedProperties::printProperties(::mlir::MLIRContext *ctx,
-            ::mlir::OpAsmPrinter &p, const Properties &prop) {
+            ::mlir::OpAsmPrinter &p, const Properties &prop,
+            ::mlir::ArrayRef<::llvm::StringRef> elidedProps) {
       customPrintProperties(p, prop.prop);
     }
     ::mlir::ParseResult TestOpWithVersionedProperties::parseProperties(
@@ -2997,6 +3001,11 @@ def TestOpWithVersionedProperties : TEST_Op<"with_versioned_properties"> {
   }];
 }
 
+def TestOpWithDefaultValuedProperties : TEST_Op<"with_default_valued_properties"> {
+  let assemblyFormat = "prop-dict attr-dict";
+  let arguments = (ins DefaultValuedAttr<I32Attr, "0">:$a);
+}
+
 //===----------------------------------------------------------------------===//
 // Test Dataflow
 //===----------------------------------------------------------------------===//
diff --git a/mlir/tools/mlir-tblgen/OpFormatGen.cpp b/mlir/tools/mlir-tblgen/OpFormatGen.cpp
index c8e0476d45b9..5963b5e689da 100644
--- a/mlir/tools/mlir-tblgen/OpFormatGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpFormatGen.cpp
@@ -1775,9 +1775,30 @@ const char *enumAttrBeginPrinterCode = R"(
 /// Generate the printer for the 'prop-dict' directive.
 static void genPropDictPrinter(OperationFormat &fmt, Operator &op,
                                MethodBody &body) {
+  body << "  ::llvm::SmallVector<::llvm::StringRef, 2> elidedProps;\n";
+  // Add code to check attributes for equality with the default value
+  // for attributes with the elidePrintingDefaultValue bit set.
+  for (const NamedAttribute &namedAttr : op.getAttributes()) {
+    const Attribute &attr = namedAttr.attr;
+    if (!attr.isDerivedAttr() && attr.hasDefaultValue()) {
+      const StringRef &name = namedAttr.name;
+      FmtContext fctx;
+      fctx.withBuilder("odsBuilder");
+      std::string defaultValue = std::string(
+          tgfmt(attr.getConstBuilderTemplate(), &fctx, attr.getDefaultValue()));
+      body << "  {\n";
+      body << "     ::mlir::Builder odsBuilder(getContext());\n";
+      body << "     ::mlir::Attribute attr = " << op.getGetterName(name)
+           << "Attr();\n";
+      body << "     if(attr && (attr == " << defaultValue << "))\n";
+      body << "       elidedProps.push_back(\"" << name << "\");\n";
+      body << "  }\n";
+    }
+  }
+
   body << "  _odsPrinter << \" \";\n"
        << "  printProperties(this->getContext(), _odsPrinter, "
-          "getProperties());\n";
+          "getProperties(), elidedProps);\n";
 }
 
 /// Generate the printer for the 'attr-dict' directive.
-- 
GitLab


From d019b9aaaaa60c858ee82fdbaf5de16e048e9db2 Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Fri, 12 Apr 2024 12:20:22 +0300
Subject: [PATCH 041/464] [clang][NFC] Refactor `CXXSpecialMember`

In preparation for `SemaCUDA`, which requires this enum to be forward-declarable.
---
 clang/include/clang/Sema/Sema.h               |  64 ++-
 clang/lib/Sema/SemaAccess.cpp                 |  14 +-
 clang/lib/Sema/SemaCUDA.cpp                   |   2 +-
 clang/lib/Sema/SemaDecl.cpp                   |  51 +-
 clang/lib/Sema/SemaDeclCXX.cpp                | 522 ++++++++++--------
 clang/lib/Sema/SemaInit.cpp                   |   6 +-
 clang/lib/Sema/SemaLookup.cpp                 |  78 +--
 clang/lib/Sema/SemaOverload.cpp               |  16 +-
 clang/lib/Sema/SemaTemplateInstantiate.cpp    |   7 +-
 .../lib/Sema/SemaTemplateInstantiateDecl.cpp  |   8 +-
 clang/lib/Sema/SemaType.cpp                   |   3 +-
 11 files changed, 424 insertions(+), 347 deletions(-)

diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index c4a603cc5a4a..90c916628270 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -60,6 +60,7 @@
 #include "clang/Sema/TypoCorrection.h"
 #include "clang/Sema/Weak.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLForwardCompat.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallBitVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -423,6 +424,17 @@ enum class TemplateDeductionResult {
   AlreadyDiagnosed
 };
 
+/// Kinds of C++ special members.
+enum class CXXSpecialMemberKind {
+  DefaultConstructor,
+  CopyConstructor,
+  MoveConstructor,
+  CopyAssignment,
+  MoveAssignment,
+  Destructor,
+  Invalid
+};
+
 /// Sema - This implements semantic analysis and AST building for C.
 /// \nosubgrouping
 class Sema final : public SemaBase {
@@ -4077,22 +4089,11 @@ public:
       SourceRange SpecificationRange, ArrayRef<ParsedType> DynamicExceptions,
       ArrayRef<SourceRange> DynamicExceptionRanges, Expr *NoexceptExpr);
 
-  /// Kinds of C++ special members.
-  enum CXXSpecialMember {
-    CXXDefaultConstructor,
-    CXXCopyConstructor,
-    CXXMoveConstructor,
-    CXXCopyAssignment,
-    CXXMoveAssignment,
-    CXXDestructor,
-    CXXInvalid
-  };
-
   class InheritedConstructorInfo;
 
   /// Determine if a special member function should have a deleted
   /// definition when it is defaulted.
-  bool ShouldDeleteSpecialMember(CXXMethodDecl *MD, CXXSpecialMember CSM,
+  bool ShouldDeleteSpecialMember(CXXMethodDecl *MD, CXXSpecialMemberKind CSM,
                                  InheritedConstructorInfo *ICI = nullptr,
                                  bool Diagnose = false);
 
@@ -4458,7 +4459,7 @@ public:
   void CheckExplicitlyDefaultedFunction(Scope *S, FunctionDecl *MD);
 
   bool CheckExplicitlyDefaultedSpecialMember(CXXMethodDecl *MD,
-                                             CXXSpecialMember CSM,
+                                             CXXSpecialMemberKind CSM,
                                              SourceLocation DefaultLoc);
   void CheckDelayedMemberExceptionSpecs();
 
@@ -4618,7 +4619,7 @@ public:
   void CheckCXXDefaultArguments(FunctionDecl *FD);
   void CheckExtraCXXDefaultArguments(Declarator &D);
 
-  CXXSpecialMember getSpecialMember(const CXXMethodDecl *MD) {
+  CXXSpecialMemberKind getSpecialMember(const CXXMethodDecl *MD) {
     return getDefaultedFunctionKind(MD).asSpecialMember();
   }
 
@@ -4645,7 +4646,8 @@ public:
                                    AccessSpecifier AS,
                                    const ParsedAttr &MSPropertyAttr);
 
-  void DiagnoseNontrivial(const CXXRecordDecl *Record, CXXSpecialMember CSM);
+  void DiagnoseNontrivial(const CXXRecordDecl *Record,
+                          CXXSpecialMemberKind CSM);
 
   enum TrivialABIHandling {
     /// The triviality of a method unaffected by "trivial_abi".
@@ -4655,26 +4657,31 @@ public:
     TAH_ConsiderTrivialABI
   };
 
-  bool SpecialMemberIsTrivial(CXXMethodDecl *MD, CXXSpecialMember CSM,
+  bool SpecialMemberIsTrivial(CXXMethodDecl *MD, CXXSpecialMemberKind CSM,
                               TrivialABIHandling TAH = TAH_IgnoreTrivialABI,
                               bool Diagnose = false);
 
   /// For a defaulted function, the kind of defaulted function that it is.
   class DefaultedFunctionKind {
+    LLVM_PREFERRED_TYPE(CXXSpecialMemberKind)
     unsigned SpecialMember : 8;
     unsigned Comparison : 8;
 
   public:
     DefaultedFunctionKind()
-        : SpecialMember(CXXInvalid),
+        : SpecialMember(llvm::to_underlying(CXXSpecialMemberKind::Invalid)),
           Comparison(llvm::to_underlying(DefaultedComparisonKind::None)) {}
-    DefaultedFunctionKind(CXXSpecialMember CSM)
-        : SpecialMember(CSM),
+    DefaultedFunctionKind(CXXSpecialMemberKind CSM)
+        : SpecialMember(llvm::to_underlying(CSM)),
           Comparison(llvm::to_underlying(DefaultedComparisonKind::None)) {}
     DefaultedFunctionKind(DefaultedComparisonKind Comp)
-        : SpecialMember(CXXInvalid), Comparison(llvm::to_underlying(Comp)) {}
+        : SpecialMember(llvm::to_underlying(CXXSpecialMemberKind::Invalid)),
+          Comparison(llvm::to_underlying(Comp)) {}
 
-    bool isSpecialMember() const { return SpecialMember != CXXInvalid; }
+    bool isSpecialMember() const {
+      return static_cast<CXXSpecialMemberKind>(SpecialMember) !=
+             CXXSpecialMemberKind::Invalid;
+    }
     bool isComparison() const {
       return static_cast<DefaultedComparisonKind>(Comparison) !=
              DefaultedComparisonKind::None;
@@ -4684,8 +4691,8 @@ public:
       return isSpecialMember() || isComparison();
     }
 
-    CXXSpecialMember asSpecialMember() const {
-      return static_cast<CXXSpecialMember>(SpecialMember);
+    CXXSpecialMemberKind asSpecialMember() const {
+      return static_cast<CXXSpecialMemberKind>(SpecialMember);
     }
     DefaultedComparisonKind asComparison() const {
       return static_cast<DefaultedComparisonKind>(Comparison);
@@ -4693,7 +4700,8 @@ public:
 
     /// Get the index of this function kind for use in diagnostics.
     unsigned getDiagnosticIndex() const {
-      static_assert(CXXInvalid > CXXDestructor,
+      static_assert(llvm::to_underlying(CXXSpecialMemberKind::Invalid) >
+                        llvm::to_underlying(CXXSpecialMemberKind::Destructor),
                     "invalid should have highest index");
       static_assert((unsigned)DefaultedComparisonKind::None == 0,
                     "none should be equal to zero");
@@ -4799,7 +4807,7 @@ public:
   /// definition in this translation unit.
   llvm::MapVector<NamedDecl *, SourceLocation> UndefinedButUsed;
 
-  typedef llvm::PointerIntPair<CXXRecordDecl *, 3, CXXSpecialMember>
+  typedef llvm::PointerIntPair<CXXRecordDecl *, 3, CXXSpecialMemberKind>
       SpecialMemberDecl;
 
   /// The C++ special members which we are currently in the process of
@@ -7493,7 +7501,7 @@ public:
   };
 
   SpecialMemberOverloadResult
-  LookupSpecialMember(CXXRecordDecl *D, CXXSpecialMember SM, bool ConstArg,
+  LookupSpecialMember(CXXRecordDecl *D, CXXSpecialMemberKind SM, bool ConstArg,
                       bool VolatileArg, bool RValueThis, bool ConstThis,
                       bool VolatileThis);
 
@@ -10014,7 +10022,7 @@ public:
       unsigned NumCallArgs;
 
       /// The special member being declared or defined.
-      CXXSpecialMember SpecialMember;
+      CXXSpecialMemberKind SpecialMember;
     };
 
     ArrayRef<TemplateArgument> template_arguments() const {
@@ -13109,7 +13117,7 @@ public:
   /// The result of this call is implicit CUDA target attribute(s) attached to
   /// the member declaration.
   bool inferCUDATargetForImplicitSpecialMember(CXXRecordDecl *ClassDecl,
-                                               CXXSpecialMember CSM,
+                                               CXXSpecialMemberKind CSM,
                                                CXXMethodDecl *MemberDecl,
                                                bool ConstRHS, bool Diagnose);
 
diff --git a/clang/lib/Sema/SemaAccess.cpp b/clang/lib/Sema/SemaAccess.cpp
index 4af3c0f30a8e..6a707eeb66d0 100644
--- a/clang/lib/Sema/SemaAccess.cpp
+++ b/clang/lib/Sema/SemaAccess.cpp
@@ -10,8 +10,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/Basic/Specifiers.h"
-#include "clang/Sema/SemaInternal.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/CXXInheritance.h"
 #include "clang/AST/DeclCXX.h"
@@ -19,9 +17,12 @@
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/DependentDiagnostic.h"
 #include "clang/AST/ExprCXX.h"
+#include "clang/Basic/Specifiers.h"
 #include "clang/Sema/DelayedDiagnostic.h"
 #include "clang/Sema/Initialization.h"
 #include "clang/Sema/Lookup.h"
+#include "clang/Sema/SemaInternal.h"
+#include "llvm/ADT/STLForwardCompat.h"
 
 using namespace clang;
 using namespace sema;
@@ -1658,21 +1659,24 @@ Sema::AccessResult Sema::CheckConstructorAccess(SourceLocation UseLoc,
   case InitializedEntity::EK_Base:
     PD = PDiag(diag::err_access_base_ctor);
     PD << Entity.isInheritedVirtualBase()
-       << Entity.getBaseSpecifier()->getType() << getSpecialMember(Constructor);
+       << Entity.getBaseSpecifier()->getType()
+       << llvm::to_underlying(getSpecialMember(Constructor));
     break;
 
   case InitializedEntity::EK_Member:
   case InitializedEntity::EK_ParenAggInitMember: {
     const FieldDecl *Field = cast<FieldDecl>(Entity.getDecl());
     PD = PDiag(diag::err_access_field_ctor);
-    PD << Field->getType() << getSpecialMember(Constructor);
+    PD << Field->getType()
+       << llvm::to_underlying(getSpecialMember(Constructor));
     break;
   }
 
   case InitializedEntity::EK_LambdaCapture: {
     StringRef VarName = Entity.getCapturedVarName();
     PD = PDiag(diag::err_access_lambda_capture);
-    PD << VarName << Entity.getType() << getSpecialMember(Constructor);
+    PD << VarName << Entity.getType()
+       << llvm::to_underlying(getSpecialMember(Constructor));
     break;
   }
 
diff --git a/clang/lib/Sema/SemaCUDA.cpp b/clang/lib/Sema/SemaCUDA.cpp
index 4d4f4b6a2d4d..1596222e3d1d 100644
--- a/clang/lib/Sema/SemaCUDA.cpp
+++ b/clang/lib/Sema/SemaCUDA.cpp
@@ -358,7 +358,7 @@ resolveCalleeCUDATargetConflict(Sema::CUDAFunctionTarget Target1,
 }
 
 bool Sema::inferCUDATargetForImplicitSpecialMember(CXXRecordDecl *ClassDecl,
-                                                   CXXSpecialMember CSM,
+                                                   CXXSpecialMemberKind CSM,
                                                    CXXMethodDecl *MemberDecl,
                                                    bool ConstRHS,
                                                    bool Diagnose) {
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index afd9e407946a..b448bacd49f3 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -48,6 +48,7 @@
 #include "clang/Sema/SemaHLSL.h"
 #include "clang/Sema/SemaInternal.h"
 #include "clang/Sema/Template.h"
+#include "llvm/ADT/STLForwardCompat.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/TargetParser/Triple.h"
@@ -4049,13 +4050,13 @@ bool Sema::MergeFunctionDecl(FunctionDecl *New, NamedDecl *&OldD, Scope *S,
         } else {
           Diag(NewMethod->getLocation(),
                diag::err_definition_of_implicitly_declared_member)
-            << New << getSpecialMember(OldMethod);
+              << New << llvm::to_underlying(getSpecialMember(OldMethod));
           return true;
         }
       } else if (OldMethod->getFirstDecl()->isExplicitlyDefaulted() && !isFriend) {
         Diag(NewMethod->getLocation(),
              diag::err_definition_of_explicitly_defaulted_member)
-          << getSpecialMember(OldMethod);
+            << llvm::to_underlying(getSpecialMember(OldMethod));
         return true;
       }
     }
@@ -18847,22 +18848,22 @@ bool Sema::CheckNontrivialField(FieldDecl *FD) {
       // because otherwise we'll never get complaints about
       // copy constructors.
 
-      CXXSpecialMember member = CXXInvalid;
+      CXXSpecialMemberKind member = CXXSpecialMemberKind::Invalid;
       // We're required to check for any non-trivial constructors. Since the
       // implicit default constructor is suppressed if there are any
       // user-declared constructors, we just need to check that there is a
       // trivial default constructor and a trivial copy constructor. (We don't
       // worry about move constructors here, since this is a C++98 check.)
       if (RDecl->hasNonTrivialCopyConstructor())
-        member = CXXCopyConstructor;
+        member = CXXSpecialMemberKind::CopyConstructor;
       else if (!RDecl->hasTrivialDefaultConstructor())
-        member = CXXDefaultConstructor;
+        member = CXXSpecialMemberKind::DefaultConstructor;
       else if (RDecl->hasNonTrivialCopyAssignment())
-        member = CXXCopyAssignment;
+        member = CXXSpecialMemberKind::CopyAssignment;
       else if (RDecl->hasNonTrivialDestructor())
-        member = CXXDestructor;
+        member = CXXSpecialMemberKind::Destructor;
 
-      if (member != CXXInvalid) {
+      if (member != CXXSpecialMemberKind::Invalid) {
         if (!getLangOpts().CPlusPlus11 &&
             getLangOpts().ObjCAutoRefCount && RDecl->hasObjectMember()) {
           // Objective-C++ ARC: it is an error to have a non-trivial field of
@@ -18879,10 +18880,13 @@ bool Sema::CheckNontrivialField(FieldDecl *FD) {
           }
         }
 
-        Diag(FD->getLocation(), getLangOpts().CPlusPlus11 ?
-               diag::warn_cxx98_compat_nontrivial_union_or_anon_struct_member :
-               diag::err_illegal_union_or_anon_struct_member)
-          << FD->getParent()->isUnion() << FD->getDeclName() << member;
+        Diag(
+            FD->getLocation(),
+            getLangOpts().CPlusPlus11
+                ? diag::warn_cxx98_compat_nontrivial_union_or_anon_struct_member
+                : diag::err_illegal_union_or_anon_struct_member)
+            << FD->getParent()->isUnion() << FD->getDeclName()
+            << llvm::to_underlying(member);
         DiagnoseNontrivial(RDecl, member);
         return !getLangOpts().CPlusPlus11;
       }
@@ -19132,10 +19136,10 @@ static void ComputeSelectedDestructor(Sema &S, CXXRecordDecl *Record) {
 static bool AreSpecialMemberFunctionsSameKind(ASTContext &Context,
                                               CXXMethodDecl *M1,
                                               CXXMethodDecl *M2,
-                                              Sema::CXXSpecialMember CSM) {
+                                              CXXSpecialMemberKind CSM) {
   // We don't want to compare templates to non-templates: See
   // https://github.com/llvm/llvm-project/issues/59206
-  if (CSM == Sema::CXXDefaultConstructor)
+  if (CSM == CXXSpecialMemberKind::DefaultConstructor)
     return bool(M1->getDescribedFunctionTemplate()) ==
            bool(M2->getDescribedFunctionTemplate());
   // FIXME: better resolve CWG
@@ -19158,7 +19162,7 @@ static bool AreSpecialMemberFunctionsSameKind(ASTContext &Context,
 ///   [CWG2595], if any, are satisfied is more constrained.
 static void SetEligibleMethods(Sema &S, CXXRecordDecl *Record,
                                ArrayRef<CXXMethodDecl *> Methods,
-                               Sema::CXXSpecialMember CSM) {
+                               CXXSpecialMemberKind CSM) {
   SmallVector<bool, 4> SatisfactionStatus;
 
   for (CXXMethodDecl *Method : Methods) {
@@ -19216,7 +19220,8 @@ static void SetEligibleMethods(Sema &S, CXXRecordDecl *Record,
     // DR1734 and DR1496.
     if (!AnotherMethodIsMoreConstrained) {
       Method->setIneligibleOrNotSelected(false);
-      Record->addedEligibleSpecialMemberFunction(Method, 1 << CSM);
+      Record->addedEligibleSpecialMemberFunction(Method,
+                                                 1 << llvm::to_underlying(CSM));
     }
   }
 }
@@ -19255,13 +19260,15 @@ static void ComputeSpecialMemberFunctionsEligiblity(Sema &S,
   }
 
   SetEligibleMethods(S, Record, DefaultConstructors,
-                     Sema::CXXDefaultConstructor);
-  SetEligibleMethods(S, Record, CopyConstructors, Sema::CXXCopyConstructor);
-  SetEligibleMethods(S, Record, MoveConstructors, Sema::CXXMoveConstructor);
+                     CXXSpecialMemberKind::DefaultConstructor);
+  SetEligibleMethods(S, Record, CopyConstructors,
+                     CXXSpecialMemberKind::CopyConstructor);
+  SetEligibleMethods(S, Record, MoveConstructors,
+                     CXXSpecialMemberKind::MoveConstructor);
   SetEligibleMethods(S, Record, CopyAssignmentOperators,
-                     Sema::CXXCopyAssignment);
+                     CXXSpecialMemberKind::CopyAssignment);
   SetEligibleMethods(S, Record, MoveAssignmentOperators,
-                     Sema::CXXMoveAssignment);
+                     CXXSpecialMemberKind::MoveAssignment);
 }
 
 void Sema::ActOnFields(Scope *S, SourceLocation RecLoc, Decl *EnclosingDecl,
@@ -19630,7 +19637,7 @@ void Sema::ActOnFields(Scope *S, SourceLocation RecLoc, Decl *EnclosingDecl,
     if (CXXRecord) {
       auto *Dtor = CXXRecord->getDestructor();
       if (Dtor && Dtor->isImplicit() &&
-          ShouldDeleteSpecialMember(Dtor, CXXDestructor)) {
+          ShouldDeleteSpecialMember(Dtor, CXXSpecialMemberKind::Destructor)) {
         CXXRecord->setImplicitDestructorIsDeleted();
         SetDeclDeleted(Dtor, CXXRecord->getLocation());
       }
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index 858951580ea4..51c14443d2d8 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -46,6 +46,7 @@
 #include "clang/Sema/Template.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/STLForwardCompat.h"
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
@@ -657,13 +658,13 @@ bool Sema::MergeCXXFunctionDecl(FunctionDecl *New, FunctionDecl *Old,
   // is ill-formed. This can only happen for constructors.
   if (isa<CXXConstructorDecl>(New) &&
       New->getMinRequiredArguments() < Old->getMinRequiredArguments()) {
-    CXXSpecialMember NewSM = getSpecialMember(cast<CXXMethodDecl>(New)),
-                     OldSM = getSpecialMember(cast<CXXMethodDecl>(Old));
+    CXXSpecialMemberKind NewSM = getSpecialMember(cast<CXXMethodDecl>(New)),
+                         OldSM = getSpecialMember(cast<CXXMethodDecl>(Old));
     if (NewSM != OldSM) {
       ParmVarDecl *NewParam = New->getParamDecl(New->getMinRequiredArguments());
       assert(NewParam->hasDefaultArg());
       Diag(NewParam->getLocation(), diag::err_default_arg_makes_ctor_special)
-        << NewParam->getDefaultArgRange() << NewSM;
+          << NewParam->getDefaultArgRange() << llvm::to_underlying(NewSM);
       Diag(Old->getLocation(), diag::note_previous_declaration);
     }
   }
@@ -6779,7 +6780,7 @@ void Sema::propagateDLLAttrToBaseClassTemplate(
 ///
 /// If the function is both a default constructor and a copy / move constructor
 /// (due to having a default argument for the first parameter), this picks
-/// CXXDefaultConstructor.
+/// CXXSpecialMemberKind::DefaultConstructor.
 ///
 /// FIXME: Check that case is properly handled by all callers.
 Sema::DefaultedFunctionKind
@@ -6787,23 +6788,23 @@ Sema::getDefaultedFunctionKind(const FunctionDecl *FD) {
   if (auto *MD = dyn_cast<CXXMethodDecl>(FD)) {
     if (const CXXConstructorDecl *Ctor = dyn_cast<CXXConstructorDecl>(FD)) {
       if (Ctor->isDefaultConstructor())
-        return Sema::CXXDefaultConstructor;
+        return CXXSpecialMemberKind::DefaultConstructor;
 
       if (Ctor->isCopyConstructor())
-        return Sema::CXXCopyConstructor;
+        return CXXSpecialMemberKind::CopyConstructor;
 
       if (Ctor->isMoveConstructor())
-        return Sema::CXXMoveConstructor;
+        return CXXSpecialMemberKind::MoveConstructor;
     }
 
     if (MD->isCopyAssignmentOperator())
-      return Sema::CXXCopyAssignment;
+      return CXXSpecialMemberKind::CopyAssignment;
 
     if (MD->isMoveAssignmentOperator())
-      return Sema::CXXMoveAssignment;
+      return CXXSpecialMemberKind::MoveAssignment;
 
     if (isa<CXXDestructorDecl>(FD))
-      return Sema::CXXDestructor;
+      return CXXSpecialMemberKind::Destructor;
   }
 
   switch (FD->getDeclName().getCXXOverloadedOperator()) {
@@ -6843,26 +6844,26 @@ static void DefineDefaultedFunction(Sema &S, FunctionDecl *FD,
     return S.DefineDefaultedComparison(DefaultLoc, FD, DFK.asComparison());
 
   switch (DFK.asSpecialMember()) {
-  case Sema::CXXDefaultConstructor:
+  case CXXSpecialMemberKind::DefaultConstructor:
     S.DefineImplicitDefaultConstructor(DefaultLoc,
                                        cast<CXXConstructorDecl>(FD));
     break;
-  case Sema::CXXCopyConstructor:
+  case CXXSpecialMemberKind::CopyConstructor:
     S.DefineImplicitCopyConstructor(DefaultLoc, cast<CXXConstructorDecl>(FD));
     break;
-  case Sema::CXXCopyAssignment:
+  case CXXSpecialMemberKind::CopyAssignment:
     S.DefineImplicitCopyAssignment(DefaultLoc, cast<CXXMethodDecl>(FD));
     break;
-  case Sema::CXXDestructor:
+  case CXXSpecialMemberKind::Destructor:
     S.DefineImplicitDestructor(DefaultLoc, cast<CXXDestructorDecl>(FD));
     break;
-  case Sema::CXXMoveConstructor:
+  case CXXSpecialMemberKind::MoveConstructor:
     S.DefineImplicitMoveConstructor(DefaultLoc, cast<CXXConstructorDecl>(FD));
     break;
-  case Sema::CXXMoveAssignment:
+  case CXXSpecialMemberKind::MoveAssignment:
     S.DefineImplicitMoveAssignment(DefaultLoc, cast<CXXMethodDecl>(FD));
     break;
-  case Sema::CXXInvalid:
+  case CXXSpecialMemberKind::Invalid:
     llvm_unreachable("Invalid special member.");
   }
 }
@@ -7182,9 +7183,9 @@ void Sema::CheckCompletedCXXClass(Scope *S, CXXRecordDecl *Record) {
 
     // For an explicitly defaulted or deleted special member, we defer
     // determining triviality until the class is complete. That time is now!
-    CXXSpecialMember CSM = getSpecialMember(M);
+    CXXSpecialMemberKind CSM = getSpecialMember(M);
     if (!M->isImplicit() && !M->isUserProvided()) {
-      if (CSM != CXXInvalid) {
+      if (CSM != CXXSpecialMemberKind::Invalid) {
         M->setTrivial(SpecialMemberIsTrivial(M, CSM));
         // Inform the class that we've finished declaring this member.
         Record->finishedDefaultedOrDeletedMember(M);
@@ -7197,8 +7198,10 @@ void Sema::CheckCompletedCXXClass(Scope *S, CXXRecordDecl *Record) {
 
     // Set triviality for the purpose of calls if this is a user-provided
     // copy/move constructor or destructor.
-    if ((CSM == CXXCopyConstructor || CSM == CXXMoveConstructor ||
-         CSM == CXXDestructor) && M->isUserProvided()) {
+    if ((CSM == CXXSpecialMemberKind::CopyConstructor ||
+         CSM == CXXSpecialMemberKind::MoveConstructor ||
+         CSM == CXXSpecialMemberKind::Destructor) &&
+        M->isUserProvided()) {
       M->setTrivialForCall(HasTrivialABI);
       Record->setTrivialForCallFlags(M);
     }
@@ -7207,8 +7210,9 @@ void Sema::CheckCompletedCXXClass(Scope *S, CXXRecordDecl *Record) {
         M->hasAttr<DLLExportAttr>()) {
       if (getLangOpts().isCompatibleWithMSVC(LangOptions::MSVC2015) &&
           M->isTrivial() &&
-          (CSM == CXXDefaultConstructor || CSM == CXXCopyConstructor ||
-           CSM == CXXDestructor))
+          (CSM == CXXSpecialMemberKind::DefaultConstructor ||
+           CSM == CXXSpecialMemberKind::CopyConstructor ||
+           CSM == CXXSpecialMemberKind::Destructor))
         M->dropAttr<DLLExportAttr>();
 
       if (M->hasAttr<DLLExportAttr>()) {
@@ -7220,8 +7224,8 @@ void Sema::CheckCompletedCXXClass(Scope *S, CXXRecordDecl *Record) {
     // Define defaulted constexpr virtual functions that override a base class
     // function right away.
     // FIXME: We can defer doing this until the vtable is marked as used.
-    if (CSM != CXXInvalid && !M->isDeleted() && M->isDefaulted() &&
-        M->isConstexpr() && M->size_overridden_methods())
+    if (CSM != CXXSpecialMemberKind::Invalid && !M->isDeleted() &&
+        M->isDefaulted() && M->isConstexpr() && M->size_overridden_methods())
       DefineDefaultedFunction(*this, M, M->getLocation());
 
     if (!Incomplete)
@@ -7343,15 +7347,18 @@ void Sema::CheckCompletedCXXClass(Scope *S, CXXRecordDecl *Record) {
 /// \param ConstRHS True if this is a copy operation with a const object
 ///        on its RHS, that is, if the argument to the outer special member
 ///        function is 'const' and this is not a field marked 'mutable'.
-static Sema::SpecialMemberOverloadResult lookupCallFromSpecialMember(
-    Sema &S, CXXRecordDecl *Class, Sema::CXXSpecialMember CSM,
-    unsigned FieldQuals, bool ConstRHS) {
+static Sema::SpecialMemberOverloadResult
+lookupCallFromSpecialMember(Sema &S, CXXRecordDecl *Class,
+                            CXXSpecialMemberKind CSM, unsigned FieldQuals,
+                            bool ConstRHS) {
   unsigned LHSQuals = 0;
-  if (CSM == Sema::CXXCopyAssignment || CSM == Sema::CXXMoveAssignment)
+  if (CSM == CXXSpecialMemberKind::CopyAssignment ||
+      CSM == CXXSpecialMemberKind::MoveAssignment)
     LHSQuals = FieldQuals;
 
   unsigned RHSQuals = FieldQuals;
-  if (CSM == Sema::CXXDefaultConstructor || CSM == Sema::CXXDestructor)
+  if (CSM == CXXSpecialMemberKind::DefaultConstructor ||
+      CSM == CXXSpecialMemberKind::Destructor)
     RHSQuals = 0;
   else if (ConstRHS)
     RHSQuals |= Qualifiers::Const;
@@ -7447,12 +7454,10 @@ public:
 
 /// Is the special member function which would be selected to perform the
 /// specified operation on the specified class type a constexpr constructor?
-static bool
-specialMemberIsConstexpr(Sema &S, CXXRecordDecl *ClassDecl,
-                         Sema::CXXSpecialMember CSM, unsigned Quals,
-                         bool ConstRHS,
-                         CXXConstructorDecl *InheritedCtor = nullptr,
-                         Sema::InheritedConstructorInfo *Inherited = nullptr) {
+static bool specialMemberIsConstexpr(
+    Sema &S, CXXRecordDecl *ClassDecl, CXXSpecialMemberKind CSM, unsigned Quals,
+    bool ConstRHS, CXXConstructorDecl *InheritedCtor = nullptr,
+    Sema::InheritedConstructorInfo *Inherited = nullptr) {
   // Suppress duplicate constraint checking here, in case a constraint check
   // caused us to decide to do this.  Any truely recursive checks will get
   // caught during these checks anyway.
@@ -7461,16 +7466,16 @@ specialMemberIsConstexpr(Sema &S, CXXRecordDecl *ClassDecl,
   // If we're inheriting a constructor, see if we need to call it for this base
   // class.
   if (InheritedCtor) {
-    assert(CSM == Sema::CXXDefaultConstructor);
+    assert(CSM == CXXSpecialMemberKind::DefaultConstructor);
     auto BaseCtor =
         Inherited->findConstructorForBase(ClassDecl, InheritedCtor).first;
     if (BaseCtor)
       return BaseCtor->isConstexpr();
   }
 
-  if (CSM == Sema::CXXDefaultConstructor)
+  if (CSM == CXXSpecialMemberKind::DefaultConstructor)
     return ClassDecl->hasConstexprDefaultConstructor();
-  if (CSM == Sema::CXXDestructor)
+  if (CSM == CXXSpecialMemberKind::Destructor)
     return ClassDecl->hasConstexprDestructor();
 
   Sema::SpecialMemberOverloadResult SMOR =
@@ -7485,8 +7490,8 @@ specialMemberIsConstexpr(Sema &S, CXXRecordDecl *ClassDecl,
 /// Determine whether the specified special member function would be constexpr
 /// if it were implicitly defined.
 static bool defaultedSpecialMemberIsConstexpr(
-    Sema &S, CXXRecordDecl *ClassDecl, Sema::CXXSpecialMember CSM,
-    bool ConstArg, CXXConstructorDecl *InheritedCtor = nullptr,
+    Sema &S, CXXRecordDecl *ClassDecl, CXXSpecialMemberKind CSM, bool ConstArg,
+    CXXConstructorDecl *InheritedCtor = nullptr,
     Sema::InheritedConstructorInfo *Inherited = nullptr) {
   if (!S.getLangOpts().CPlusPlus11)
     return false;
@@ -7495,7 +7500,7 @@ static bool defaultedSpecialMemberIsConstexpr(
   // In the definition of a constexpr constructor [...]
   bool Ctor = true;
   switch (CSM) {
-  case Sema::CXXDefaultConstructor:
+  case CXXSpecialMemberKind::DefaultConstructor:
     if (Inherited)
       break;
     // Since default constructor lookup is essentially trivial (and cannot
@@ -7506,23 +7511,23 @@ static bool defaultedSpecialMemberIsConstexpr(
     // constructor is constexpr to determine whether the type is a literal type.
     return ClassDecl->defaultedDefaultConstructorIsConstexpr();
 
-  case Sema::CXXCopyConstructor:
-  case Sema::CXXMoveConstructor:
+  case CXXSpecialMemberKind::CopyConstructor:
+  case CXXSpecialMemberKind::MoveConstructor:
     // For copy or move constructors, we need to perform overload resolution.
     break;
 
-  case Sema::CXXCopyAssignment:
-  case Sema::CXXMoveAssignment:
+  case CXXSpecialMemberKind::CopyAssignment:
+  case CXXSpecialMemberKind::MoveAssignment:
     if (!S.getLangOpts().CPlusPlus14)
       return false;
     // In C++1y, we need to perform overload resolution.
     Ctor = false;
     break;
 
-  case Sema::CXXDestructor:
+  case CXXSpecialMemberKind::Destructor:
     return ClassDecl->defaultedDestructorIsConstexpr();
 
-  case Sema::CXXInvalid:
+  case CXXSpecialMemberKind::Invalid:
     return false;
   }
 
@@ -7534,7 +7539,7 @@ static bool defaultedSpecialMemberIsConstexpr(
   // will be initialized (if the constructor isn't deleted), we just don't know
   // which one.
   if (Ctor && ClassDecl->isUnion())
-    return CSM == Sema::CXXDefaultConstructor
+    return CSM == CXXSpecialMemberKind::DefaultConstructor
                ? ClassDecl->hasInClassInitializer() ||
                      !ClassDecl->hasVariantMembers()
                : true;
@@ -7575,7 +7580,8 @@ static bool defaultedSpecialMemberIsConstexpr(
     for (const auto *F : ClassDecl->fields()) {
       if (F->isInvalidDecl())
         continue;
-      if (CSM == Sema::CXXDefaultConstructor && F->hasInClassInitializer())
+      if (CSM == CXXSpecialMemberKind::DefaultConstructor &&
+          F->hasInClassInitializer())
         continue;
       QualType BaseType = S.Context.getBaseElementType(F->getType());
       if (const RecordType *RecordTy = BaseType->getAs<RecordType>()) {
@@ -7584,7 +7590,7 @@ static bool defaultedSpecialMemberIsConstexpr(
                                       BaseType.getCVRQualifiers(),
                                       ConstArg && !F->isMutable()))
           return false;
-      } else if (CSM == Sema::CXXDefaultConstructor) {
+      } else if (CSM == CXXSpecialMemberKind::DefaultConstructor) {
         return false;
       }
     }
@@ -7615,9 +7621,10 @@ struct ComputingExceptionSpec {
 }
 
 static Sema::ImplicitExceptionSpecification
-ComputeDefaultedSpecialMemberExceptionSpec(
-    Sema &S, SourceLocation Loc, CXXMethodDecl *MD, Sema::CXXSpecialMember CSM,
-    Sema::InheritedConstructorInfo *ICI);
+ComputeDefaultedSpecialMemberExceptionSpec(Sema &S, SourceLocation Loc,
+                                           CXXMethodDecl *MD,
+                                           CXXSpecialMemberKind CSM,
+                                           Sema::InheritedConstructorInfo *ICI);
 
 static Sema::ImplicitExceptionSpecification
 ComputeDefaultedComparisonExceptionSpec(Sema &S, SourceLocation Loc,
@@ -7641,7 +7648,7 @@ computeImplicitExceptionSpec(Sema &S, SourceLocation Loc, FunctionDecl *FD) {
   Sema::InheritedConstructorInfo ICI(
       S, Loc, CD->getInheritedConstructor().getShadowDecl());
   return ComputeDefaultedSpecialMemberExceptionSpec(
-      S, Loc, CD, Sema::CXXDefaultConstructor, &ICI);
+      S, Loc, CD, CXXSpecialMemberKind::DefaultConstructor, &ICI);
 }
 
 static FunctionProtoType::ExtProtoInfo getImplicitMethodEPI(Sema &S,
@@ -7693,11 +7700,11 @@ void Sema::CheckExplicitlyDefaultedFunction(Scope *S, FunctionDecl *FD) {
 }
 
 bool Sema::CheckExplicitlyDefaultedSpecialMember(CXXMethodDecl *MD,
-                                                 CXXSpecialMember CSM,
+                                                 CXXSpecialMemberKind CSM,
                                                  SourceLocation DefaultLoc) {
   CXXRecordDecl *RD = MD->getParent();
 
-  assert(MD->isExplicitlyDefaulted() && CSM != CXXInvalid &&
+  assert(MD->isExplicitlyDefaulted() && CSM != CXXSpecialMemberKind::Invalid &&
          "not an explicitly-defaulted special member");
 
   // Defer all checking for special members of a dependent type.
@@ -7723,21 +7730,22 @@ bool Sema::CheckExplicitlyDefaultedSpecialMember(CXXMethodDecl *MD,
   bool DeleteOnTypeMismatch = getLangOpts().CPlusPlus20 && First;
   bool ShouldDeleteForTypeMismatch = false;
   unsigned ExpectedParams = 1;
-  if (CSM == CXXDefaultConstructor || CSM == CXXDestructor)
+  if (CSM == CXXSpecialMemberKind::DefaultConstructor ||
+      CSM == CXXSpecialMemberKind::Destructor)
     ExpectedParams = 0;
   if (MD->getNumExplicitParams() != ExpectedParams) {
     // This checks for default arguments: a copy or move constructor with a
     // default argument is classified as a default constructor, and assignment
     // operations and destructors can't have default arguments.
     Diag(MD->getLocation(), diag::err_defaulted_special_member_params)
-      << CSM << MD->getSourceRange();
+        << llvm::to_underlying(CSM) << MD->getSourceRange();
     HadError = true;
   } else if (MD->isVariadic()) {
     if (DeleteOnTypeMismatch)
       ShouldDeleteForTypeMismatch = true;
     else {
       Diag(MD->getLocation(), diag::err_defaulted_special_member_variadic)
-        << CSM << MD->getSourceRange();
+          << llvm::to_underlying(CSM) << MD->getSourceRange();
       HadError = true;
     }
   }
@@ -7745,13 +7753,14 @@ bool Sema::CheckExplicitlyDefaultedSpecialMember(CXXMethodDecl *MD,
   const FunctionProtoType *Type = MD->getType()->castAs<FunctionProtoType>();
 
   bool CanHaveConstParam = false;
-  if (CSM == CXXCopyConstructor)
+  if (CSM == CXXSpecialMemberKind::CopyConstructor)
     CanHaveConstParam = RD->implicitCopyConstructorHasConstParam();
-  else if (CSM == CXXCopyAssignment)
+  else if (CSM == CXXSpecialMemberKind::CopyAssignment)
     CanHaveConstParam = RD->implicitCopyAssignmentHasConstParam();
 
   QualType ReturnType = Context.VoidTy;
-  if (CSM == CXXCopyAssignment || CSM == CXXMoveAssignment) {
+  if (CSM == CXXSpecialMemberKind::CopyAssignment ||
+      CSM == CXXSpecialMemberKind::MoveAssignment) {
     // Check for return type matching.
     ReturnType = Type->getReturnType();
     QualType ThisType = MD->getFunctionObjectParameterType();
@@ -7765,7 +7774,8 @@ bool Sema::CheckExplicitlyDefaultedSpecialMember(CXXMethodDecl *MD,
 
     if (!Context.hasSameType(ReturnType, ExpectedReturnType)) {
       Diag(MD->getLocation(), diag::err_defaulted_special_member_return_type)
-        << (CSM == CXXMoveAssignment) << ExpectedReturnType;
+          << (CSM == CXXSpecialMemberKind::MoveAssignment)
+          << ExpectedReturnType;
       HadError = true;
     }
 
@@ -7775,7 +7785,8 @@ bool Sema::CheckExplicitlyDefaultedSpecialMember(CXXMethodDecl *MD,
         ShouldDeleteForTypeMismatch = true;
       else {
         Diag(MD->getLocation(), diag::err_defaulted_special_member_quals)
-          << (CSM == CXXMoveAssignment) << getLangOpts().CPlusPlus14;
+            << (CSM == CXXSpecialMemberKind::MoveAssignment)
+            << getLangOpts().CPlusPlus14;
         HadError = true;
       }
     }
@@ -7793,7 +7804,8 @@ bool Sema::CheckExplicitlyDefaultedSpecialMember(CXXMethodDecl *MD,
       else {
         Diag(MD->getLocation(),
              diag::err_defaulted_special_member_explicit_object_mismatch)
-            << (CSM == CXXMoveAssignment) << RD << MD->getSourceRange();
+            << (CSM == CXXSpecialMemberKind::MoveAssignment) << RD
+            << MD->getSourceRange();
         HadError = true;
       }
     }
@@ -7815,7 +7827,8 @@ bool Sema::CheckExplicitlyDefaultedSpecialMember(CXXMethodDecl *MD,
         ShouldDeleteForTypeMismatch = true;
       else {
         Diag(MD->getLocation(),
-             diag::err_defaulted_special_member_volatile_param) << CSM;
+             diag::err_defaulted_special_member_volatile_param)
+            << llvm::to_underlying(CSM);
         HadError = true;
       }
     }
@@ -7823,23 +7836,25 @@ bool Sema::CheckExplicitlyDefaultedSpecialMember(CXXMethodDecl *MD,
     if (HasConstParam && !CanHaveConstParam) {
       if (DeleteOnTypeMismatch)
         ShouldDeleteForTypeMismatch = true;
-      else if (CSM == CXXCopyConstructor || CSM == CXXCopyAssignment) {
+      else if (CSM == CXXSpecialMemberKind::CopyConstructor ||
+               CSM == CXXSpecialMemberKind::CopyAssignment) {
         Diag(MD->getLocation(),
              diag::err_defaulted_special_member_copy_const_param)
-          << (CSM == CXXCopyAssignment);
+            << (CSM == CXXSpecialMemberKind::CopyAssignment);
         // FIXME: Explain why this special member can't be const.
         HadError = true;
       } else {
         Diag(MD->getLocation(),
              diag::err_defaulted_special_member_move_const_param)
-          << (CSM == CXXMoveAssignment);
+            << (CSM == CXXSpecialMemberKind::MoveAssignment);
         HadError = true;
       }
     }
   } else if (ExpectedParams) {
     // A copy assignment operator can take its argument by value, but a
     // defaulted one cannot.
-    assert(CSM == CXXCopyAssignment && "unexpected non-ref argument");
+    assert(CSM == CXXSpecialMemberKind::CopyAssignment &&
+           "unexpected non-ref argument");
     Diag(MD->getLocation(), diag::err_defaulted_copy_assign_not_ref);
     HadError = true;
   }
@@ -7874,12 +7889,12 @@ bool Sema::CheckExplicitlyDefaultedSpecialMember(CXXMethodDecl *MD,
         if (!MD->isConsteval() && RD->getNumVBases()) {
           Diag(MD->getBeginLoc(),
                diag::err_incorrect_defaulted_constexpr_with_vb)
-              << CSM;
+              << llvm::to_underlying(CSM);
           for (const auto &I : RD->vbases())
             Diag(I.getBeginLoc(), diag::note_constexpr_virtual_base_here);
         } else {
           Diag(MD->getBeginLoc(), diag::err_incorrect_defaulted_constexpr)
-              << CSM << MD->isConsteval();
+              << llvm::to_underlying(CSM) << MD->isConsteval();
         }
         HadError = true;
         // FIXME: Explain why the special member can't be constexpr.
@@ -7912,9 +7927,11 @@ bool Sema::CheckExplicitlyDefaultedSpecialMember(CXXMethodDecl *MD,
     if (First) {
       SetDeclDeleted(MD, MD->getLocation());
       if (!inTemplateInstantiation() && !HadError) {
-        Diag(MD->getLocation(), diag::warn_defaulted_method_deleted) << CSM;
+        Diag(MD->getLocation(), diag::warn_defaulted_method_deleted)
+            << llvm::to_underlying(CSM);
         if (ShouldDeleteForTypeMismatch) {
-          Diag(MD->getLocation(), diag::note_deleted_type_mismatch) << CSM;
+          Diag(MD->getLocation(), diag::note_deleted_type_mismatch)
+              << llvm::to_underlying(CSM);
         } else if (ShouldDeleteSpecialMember(MD, CSM, nullptr,
                                              /*Diagnose*/ true) &&
                    DefaultLoc.isValid()) {
@@ -7924,13 +7941,15 @@ bool Sema::CheckExplicitlyDefaultedSpecialMember(CXXMethodDecl *MD,
       }
       if (ShouldDeleteForTypeMismatch && !HadError) {
         Diag(MD->getLocation(),
-             diag::warn_cxx17_compat_defaulted_method_type_mismatch) << CSM;
+             diag::warn_cxx17_compat_defaulted_method_type_mismatch)
+            << llvm::to_underlying(CSM);
       }
     } else {
       // C++11 [dcl.fct.def.default]p4:
       //   [For a] user-provided explicitly-defaulted function [...] if such a
       //   function is implicitly defined as deleted, the program is ill-formed.
-      Diag(MD->getLocation(), diag::err_out_of_line_default_deletes) << CSM;
+      Diag(MD->getLocation(), diag::err_out_of_line_default_deletes)
+          << llvm::to_underlying(CSM);
       assert(!ShouldDeleteForTypeMismatch && "deleted non-first decl");
       ShouldDeleteSpecialMember(MD, CSM, nullptr, /*Diagnose*/true);
       HadError = true;
@@ -9271,28 +9290,28 @@ template<typename Derived>
 struct SpecialMemberVisitor {
   Sema &S;
   CXXMethodDecl *MD;
-  Sema::CXXSpecialMember CSM;
+  CXXSpecialMemberKind CSM;
   Sema::InheritedConstructorInfo *ICI;
 
   // Properties of the special member, computed for convenience.
   bool IsConstructor = false, IsAssignment = false, ConstArg = false;
 
-  SpecialMemberVisitor(Sema &S, CXXMethodDecl *MD, Sema::CXXSpecialMember CSM,
+  SpecialMemberVisitor(Sema &S, CXXMethodDecl *MD, CXXSpecialMemberKind CSM,
                        Sema::InheritedConstructorInfo *ICI)
       : S(S), MD(MD), CSM(CSM), ICI(ICI) {
     switch (CSM) {
-    case Sema::CXXDefaultConstructor:
-    case Sema::CXXCopyConstructor:
-    case Sema::CXXMoveConstructor:
+    case CXXSpecialMemberKind::DefaultConstructor:
+    case CXXSpecialMemberKind::CopyConstructor:
+    case CXXSpecialMemberKind::MoveConstructor:
       IsConstructor = true;
       break;
-    case Sema::CXXCopyAssignment:
-    case Sema::CXXMoveAssignment:
+    case CXXSpecialMemberKind::CopyAssignment:
+    case CXXSpecialMemberKind::MoveAssignment:
       IsAssignment = true;
       break;
-    case Sema::CXXDestructor:
+    case CXXSpecialMemberKind::Destructor:
       break;
-    case Sema::CXXInvalid:
+    case CXXSpecialMemberKind::Invalid:
       llvm_unreachable("invalid special member kind");
     }
 
@@ -9307,7 +9326,8 @@ struct SpecialMemberVisitor {
 
   /// Is this a "move" special member?
   bool isMove() const {
-    return CSM == Sema::CXXMoveConstructor || CSM == Sema::CXXMoveAssignment;
+    return CSM == CXXSpecialMemberKind::MoveConstructor ||
+           CSM == CXXSpecialMemberKind::MoveAssignment;
   }
 
   /// Look up the corresponding special member in the given class.
@@ -9322,7 +9342,7 @@ struct SpecialMemberVisitor {
   Sema::SpecialMemberOverloadResult lookupInheritedCtor(CXXRecordDecl *Class) {
     if (!ICI)
       return {};
-    assert(CSM == Sema::CXXDefaultConstructor);
+    assert(CSM == CXXSpecialMemberKind::DefaultConstructor);
     auto *BaseCtor =
       cast<CXXConstructorDecl>(MD)->getInheritedConstructor().getConstructor();
     if (auto *MD = ICI->findConstructorForBase(Class, BaseCtor).first)
@@ -9392,15 +9412,15 @@ struct SpecialMemberDeletionInfo
   bool AllFieldsAreConst;
 
   SpecialMemberDeletionInfo(Sema &S, CXXMethodDecl *MD,
-                            Sema::CXXSpecialMember CSM,
+                            CXXSpecialMemberKind CSM,
                             Sema::InheritedConstructorInfo *ICI, bool Diagnose)
       : SpecialMemberVisitor(S, MD, CSM, ICI), Diagnose(Diagnose),
         Loc(MD->getLocation()), AllFieldsAreConst(true) {}
 
   bool inUnion() const { return MD->getParent()->isUnion(); }
 
-  Sema::CXXSpecialMember getEffectiveCSM() {
-    return ICI ? Sema::CXXInvalid : CSM;
+  CXXSpecialMemberKind getEffectiveCSM() {
+    return ICI ? CXXSpecialMemberKind::Invalid : CSM;
   }
 
   bool shouldDeleteForVariantObjCPtrMember(FieldDecl *FD, QualType FieldType);
@@ -9466,7 +9486,7 @@ bool SpecialMemberDeletionInfo::shouldDeleteForSubobjectCall(
     // must be accessible and non-deleted, but need not be trivial. Such a
     // destructor is never actually called, but is semantically checked as
     // if it were.
-    if (CSM == Sema::CXXDefaultConstructor) {
+    if (CSM == CXXSpecialMemberKind::DefaultConstructor) {
       // [class.default.ctor]p2:
       //   A defaulted default constructor for class X is defined as deleted if
       //   - X is a union that has a variant member with a non-trivial default
@@ -9487,15 +9507,16 @@ bool SpecialMemberDeletionInfo::shouldDeleteForSubobjectCall(
     if (Field) {
       S.Diag(Field->getLocation(),
              diag::note_deleted_special_member_class_subobject)
-        << getEffectiveCSM() << MD->getParent() << /*IsField*/true
-        << Field << DiagKind << IsDtorCallInCtor << /*IsObjCPtr*/false;
+          << llvm::to_underlying(getEffectiveCSM()) << MD->getParent()
+          << /*IsField*/ true << Field << DiagKind << IsDtorCallInCtor
+          << /*IsObjCPtr*/ false;
     } else {
       CXXBaseSpecifier *Base = Subobj.get<CXXBaseSpecifier*>();
       S.Diag(Base->getBeginLoc(),
              diag::note_deleted_special_member_class_subobject)
-          << getEffectiveCSM() << MD->getParent() << /*IsField*/ false
-          << Base->getType() << DiagKind << IsDtorCallInCtor
-          << /*IsObjCPtr*/false;
+          << llvm::to_underlying(getEffectiveCSM()) << MD->getParent()
+          << /*IsField*/ false << Base->getType() << DiagKind
+          << IsDtorCallInCtor << /*IsObjCPtr*/ false;
     }
 
     if (DiagKind == 1)
@@ -9527,8 +9548,8 @@ bool SpecialMemberDeletionInfo::shouldDeleteForClassSubobject(
   // C++11 [class.dtor]p5:
   // -- any direct or virtual base class [...] has a type with a destructor
   //    that is deleted or inaccessible
-  if (!(CSM == Sema::CXXDefaultConstructor &&
-        Field && Field->hasInClassInitializer()) &&
+  if (!(CSM == CXXSpecialMemberKind::DefaultConstructor && Field &&
+        Field->hasInClassInitializer()) &&
       shouldDeleteForSubobjectCall(Subobj, lookupIn(Class, Quals, IsMutable),
                                    false))
     return true;
@@ -9538,8 +9559,8 @@ bool SpecialMemberDeletionInfo::shouldDeleteForClassSubobject(
   //    type with a destructor that is deleted or inaccessible
   if (IsConstructor) {
     Sema::SpecialMemberOverloadResult SMOR =
-        S.LookupSpecialMember(Class, Sema::CXXDestructor,
-                              false, false, false, false, false);
+        S.LookupSpecialMember(Class, CXXSpecialMemberKind::Destructor, false,
+                              false, false, false, false);
     if (shouldDeleteForSubobjectCall(Subobj, SMOR, true))
       return true;
   }
@@ -9557,15 +9578,16 @@ bool SpecialMemberDeletionInfo::shouldDeleteForVariantObjCPtrMember(
 
   // Don't make the defaulted default constructor defined as deleted if the
   // member has an in-class initializer.
-  if (CSM == Sema::CXXDefaultConstructor && FD->hasInClassInitializer())
+  if (CSM == CXXSpecialMemberKind::DefaultConstructor &&
+      FD->hasInClassInitializer())
     return false;
 
   if (Diagnose) {
     auto *ParentClass = cast<CXXRecordDecl>(FD->getParent());
-    S.Diag(FD->getLocation(),
-           diag::note_deleted_special_member_class_subobject)
-        << getEffectiveCSM() << ParentClass << /*IsField*/true
-        << FD << 4 << /*IsDtorCallInCtor*/false << /*IsObjCPtr*/true;
+    S.Diag(FD->getLocation(), diag::note_deleted_special_member_class_subobject)
+        << llvm::to_underlying(getEffectiveCSM()) << ParentClass
+        << /*IsField*/ true << FD << 4 << /*IsDtorCallInCtor*/ false
+        << /*IsObjCPtr*/ true;
   }
 
   return true;
@@ -9590,9 +9612,9 @@ bool SpecialMemberDeletionInfo::shouldDeleteForBase(CXXBaseSpecifier *Base) {
     if (BaseCtor->isDeleted() && Diagnose) {
       S.Diag(Base->getBeginLoc(),
              diag::note_deleted_special_member_class_subobject)
-          << getEffectiveCSM() << MD->getParent() << /*IsField*/ false
-          << Base->getType() << /*Deleted*/ 1 << /*IsDtorCallInCtor*/ false
-          << /*IsObjCPtr*/false;
+          << llvm::to_underlying(getEffectiveCSM()) << MD->getParent()
+          << /*IsField*/ false << Base->getType() << /*Deleted*/ 1
+          << /*IsDtorCallInCtor*/ false << /*IsObjCPtr*/ false;
       S.NoteDeletedFunction(BaseCtor);
     }
     return BaseCtor->isDeleted();
@@ -9609,7 +9631,7 @@ bool SpecialMemberDeletionInfo::shouldDeleteForField(FieldDecl *FD) {
   if (inUnion() && shouldDeleteForVariantObjCPtrMember(FD, FieldType))
     return true;
 
-  if (CSM == Sema::CXXDefaultConstructor) {
+  if (CSM == CXXSpecialMemberKind::DefaultConstructor) {
     // For a default constructor, all references must be initialized in-class
     // and, if a union, it must have a non-const member.
     if (FieldType->isReferenceType() && !FD->hasInClassInitializer()) {
@@ -9632,7 +9654,7 @@ bool SpecialMemberDeletionInfo::shouldDeleteForField(FieldDecl *FD) {
 
     if (inUnion() && !FieldType.isConstQualified())
       AllFieldsAreConst = false;
-  } else if (CSM == Sema::CXXCopyConstructor) {
+  } else if (CSM == CXXSpecialMemberKind::CopyConstructor) {
     // For a copy constructor, data members must not be of rvalue reference
     // type.
     if (FieldType->isRValueReferenceType()) {
@@ -9683,8 +9705,8 @@ bool SpecialMemberDeletionInfo::shouldDeleteForField(FieldDecl *FD) {
       }
 
       // At least one member in each anonymous union must be non-const
-      if (CSM == Sema::CXXDefaultConstructor && AllVariantFieldsAreConst &&
-          !FieldRecord->field_empty()) {
+      if (CSM == CXXSpecialMemberKind::DefaultConstructor &&
+          AllVariantFieldsAreConst && !FieldRecord->field_empty()) {
         if (Diagnose)
           S.Diag(FieldRecord->getLocation(),
                  diag::note_deleted_default_ctor_all_const)
@@ -9712,7 +9734,8 @@ bool SpecialMemberDeletionInfo::shouldDeleteForField(FieldDecl *FD) {
 bool SpecialMemberDeletionInfo::shouldDeleteForAllConstMembers() {
   // This is a silly definition, because it gives an empty union a deleted
   // default constructor. Don't do that.
-  if (CSM == Sema::CXXDefaultConstructor && inUnion() && AllFieldsAreConst) {
+  if (CSM == CXXSpecialMemberKind::DefaultConstructor && inUnion() &&
+      AllFieldsAreConst) {
     bool AnyFields = false;
     for (auto *F : MD->getParent()->fields())
       if ((AnyFields = !F->isUnnamedBitfield()))
@@ -9731,7 +9754,8 @@ bool SpecialMemberDeletionInfo::shouldDeleteForAllConstMembers() {
 /// Determine whether a defaulted special member function should be defined as
 /// deleted, as specified in C++11 [class.ctor]p5, C++11 [class.copy]p11,
 /// C++11 [class.copy]p23, and C++11 [class.dtor]p5.
-bool Sema::ShouldDeleteSpecialMember(CXXMethodDecl *MD, CXXSpecialMember CSM,
+bool Sema::ShouldDeleteSpecialMember(CXXMethodDecl *MD,
+                                     CXXSpecialMemberKind CSM,
                                      InheritedConstructorInfo *ICI,
                                      bool Diagnose) {
   if (MD->isInvalidDecl())
@@ -9748,7 +9772,8 @@ bool Sema::ShouldDeleteSpecialMember(CXXMethodDecl *MD, CXXSpecialMember CSM,
   //   assignment operator.
   // C++2a adds back these operators if the lambda has no lambda-capture.
   if (RD->isLambda() && !RD->lambdaIsDefaultConstructibleAndAssignable() &&
-      (CSM == CXXDefaultConstructor || CSM == CXXCopyAssignment)) {
+      (CSM == CXXSpecialMemberKind::DefaultConstructor ||
+       CSM == CXXSpecialMemberKind::CopyAssignment)) {
     if (Diagnose)
       Diag(RD->getLocation(), diag::note_lambda_decl);
     return true;
@@ -9757,16 +9782,16 @@ bool Sema::ShouldDeleteSpecialMember(CXXMethodDecl *MD, CXXSpecialMember CSM,
   // For an anonymous struct or union, the copy and assignment special members
   // will never be used, so skip the check. For an anonymous union declared at
   // namespace scope, the constructor and destructor are used.
-  if (CSM != CXXDefaultConstructor && CSM != CXXDestructor &&
-      RD->isAnonymousStructOrUnion())
+  if (CSM != CXXSpecialMemberKind::DefaultConstructor &&
+      CSM != CXXSpecialMemberKind::Destructor && RD->isAnonymousStructOrUnion())
     return false;
 
   // C++11 [class.copy]p7, p18:
   //   If the class definition declares a move constructor or move assignment
   //   operator, an implicitly declared copy constructor or copy assignment
   //   operator is defined as deleted.
-  if (MD->isImplicit() &&
-      (CSM == CXXCopyConstructor || CSM == CXXCopyAssignment)) {
+  if (MD->isImplicit() && (CSM == CXXSpecialMemberKind::CopyConstructor ||
+                           CSM == CXXSpecialMemberKind::CopyAssignment)) {
     CXXMethodDecl *UserDeclaredMove = nullptr;
 
     // In Microsoft mode up to MSVC 2013, a user-declared move only causes the
@@ -9777,7 +9802,8 @@ bool Sema::ShouldDeleteSpecialMember(CXXMethodDecl *MD, CXXSpecialMember CSM,
         !getLangOpts().isCompatibleWithMSVC(LangOptions::MSVC2015);
 
     if (RD->hasUserDeclaredMoveConstructor() &&
-        (!DeletesOnlyMatchingCopy || CSM == CXXCopyConstructor)) {
+        (!DeletesOnlyMatchingCopy ||
+         CSM == CXXSpecialMemberKind::CopyConstructor)) {
       if (!Diagnose) return true;
 
       // Find any user-declared move constructor.
@@ -9789,7 +9815,8 @@ bool Sema::ShouldDeleteSpecialMember(CXXMethodDecl *MD, CXXSpecialMember CSM,
       }
       assert(UserDeclaredMove);
     } else if (RD->hasUserDeclaredMoveAssignment() &&
-               (!DeletesOnlyMatchingCopy || CSM == CXXCopyAssignment)) {
+               (!DeletesOnlyMatchingCopy ||
+                CSM == CXXSpecialMemberKind::CopyAssignment)) {
       if (!Diagnose) return true;
 
       // Find any user-declared move assignment operator.
@@ -9805,8 +9832,8 @@ bool Sema::ShouldDeleteSpecialMember(CXXMethodDecl *MD, CXXSpecialMember CSM,
     if (UserDeclaredMove) {
       Diag(UserDeclaredMove->getLocation(),
            diag::note_deleted_copy_user_declared_move)
-        << (CSM == CXXCopyAssignment) << RD
-        << UserDeclaredMove->isMoveAssignmentOperator();
+          << (CSM == CXXSpecialMemberKind::CopyAssignment) << RD
+          << UserDeclaredMove->isMoveAssignmentOperator();
       return true;
     }
   }
@@ -9817,7 +9844,7 @@ bool Sema::ShouldDeleteSpecialMember(CXXMethodDecl *MD, CXXSpecialMember CSM,
   // C++11 [class.dtor]p5:
   // -- for a virtual destructor, lookup of the non-array deallocation function
   //    results in an ambiguity or in a function that is deleted or inaccessible
-  if (CSM == CXXDestructor && MD->isVirtual()) {
+  if (CSM == CXXSpecialMemberKind::Destructor && MD->isVirtual()) {
     FunctionDecl *OperatorDelete = nullptr;
     DeclarationName Name =
       Context.DeclarationNames.getCXXOperatorName(OO_Delete);
@@ -9891,7 +9918,7 @@ void Sema::DiagnoseDeletedDefaultedFunction(FunctionDecl *FD) {
 /// If \p ForCall is true, look at CXXRecord::HasTrivialSpecialMembersForCall to
 /// determine whether the special member is trivial.
 static bool findTrivialSpecialMember(Sema &S, CXXRecordDecl *RD,
-                                     Sema::CXXSpecialMember CSM, unsigned Quals,
+                                     CXXSpecialMemberKind CSM, unsigned Quals,
                                      bool ConstRHS,
                                      Sema::TrivialABIHandling TAH,
                                      CXXMethodDecl **Selected) {
@@ -9899,10 +9926,10 @@ static bool findTrivialSpecialMember(Sema &S, CXXRecordDecl *RD,
     *Selected = nullptr;
 
   switch (CSM) {
-  case Sema::CXXInvalid:
+  case CXXSpecialMemberKind::Invalid:
     llvm_unreachable("not a special member");
 
-  case Sema::CXXDefaultConstructor:
+  case CXXSpecialMemberKind::DefaultConstructor:
     // C++11 [class.ctor]p5:
     //   A default constructor is trivial if:
     //    - all the [direct subobjects] have trivial default constructors
@@ -9931,7 +9958,7 @@ static bool findTrivialSpecialMember(Sema &S, CXXRecordDecl *RD,
 
     return false;
 
-  case Sema::CXXDestructor:
+  case CXXSpecialMemberKind::Destructor:
     // C++11 [class.dtor]p5:
     //   A destructor is trivial if:
     //    - all the direct [subobjects] have trivial destructors
@@ -9948,7 +9975,7 @@ static bool findTrivialSpecialMember(Sema &S, CXXRecordDecl *RD,
 
     return false;
 
-  case Sema::CXXCopyConstructor:
+  case CXXSpecialMemberKind::CopyConstructor:
     // C++11 [class.copy]p12:
     //   A copy constructor is trivial if:
     //    - the constructor selected to copy each direct [subobject] is trivial
@@ -9969,7 +9996,7 @@ static bool findTrivialSpecialMember(Sema &S, CXXRecordDecl *RD,
     //   struct B { mutable A a; };
     goto NeedOverloadResolution;
 
-  case Sema::CXXCopyAssignment:
+  case CXXSpecialMemberKind::CopyAssignment:
     // C++11 [class.copy]p25:
     //   A copy assignment operator is trivial if:
     //    - the assignment operator selected to copy each direct [subobject] is
@@ -9984,8 +10011,8 @@ static bool findTrivialSpecialMember(Sema &S, CXXRecordDecl *RD,
     // treat that as a language defect.
     goto NeedOverloadResolution;
 
-  case Sema::CXXMoveConstructor:
-  case Sema::CXXMoveAssignment:
+  case CXXSpecialMemberKind::MoveConstructor:
+  case CXXSpecialMemberKind::MoveAssignment:
   NeedOverloadResolution:
     Sema::SpecialMemberOverloadResult SMOR =
         lookupCallFromSpecialMember(S, RD, CSM, Quals, ConstRHS);
@@ -10009,7 +10036,8 @@ static bool findTrivialSpecialMember(Sema &S, CXXRecordDecl *RD,
       *Selected = SMOR.getMethod();
 
     if (TAH == Sema::TAH_ConsiderTrivialABI &&
-        (CSM == Sema::CXXCopyConstructor || CSM == Sema::CXXMoveConstructor))
+        (CSM == CXXSpecialMemberKind::CopyConstructor ||
+         CSM == CXXSpecialMemberKind::MoveConstructor))
       return SMOR.getMethod()->isTrivialForCall();
     return SMOR.getMethod()->isTrivial();
   }
@@ -10047,9 +10075,10 @@ enum TrivialSubobjectKind {
 /// Check whether the special member selected for a given type would be trivial.
 static bool checkTrivialSubobjectCall(Sema &S, SourceLocation SubobjLoc,
                                       QualType SubType, bool ConstRHS,
-                                      Sema::CXXSpecialMember CSM,
+                                      CXXSpecialMemberKind CSM,
                                       TrivialSubobjectKind Kind,
-                                      Sema::TrivialABIHandling TAH, bool Diagnose) {
+                                      Sema::TrivialABIHandling TAH,
+                                      bool Diagnose) {
   CXXRecordDecl *SubRD = SubType->getAsCXXRecordDecl();
   if (!SubRD)
     return true;
@@ -10063,27 +10092,28 @@ static bool checkTrivialSubobjectCall(Sema &S, SourceLocation SubobjLoc,
     if (ConstRHS)
       SubType.addConst();
 
-    if (!Selected && CSM == Sema::CXXDefaultConstructor) {
+    if (!Selected && CSM == CXXSpecialMemberKind::DefaultConstructor) {
       S.Diag(SubobjLoc, diag::note_nontrivial_no_def_ctor)
         << Kind << SubType.getUnqualifiedType();
       if (CXXConstructorDecl *CD = findUserDeclaredCtor(SubRD))
         S.Diag(CD->getLocation(), diag::note_user_declared_ctor);
     } else if (!Selected)
       S.Diag(SubobjLoc, diag::note_nontrivial_no_copy)
-        << Kind << SubType.getUnqualifiedType() << CSM << SubType;
+          << Kind << SubType.getUnqualifiedType() << llvm::to_underlying(CSM)
+          << SubType;
     else if (Selected->isUserProvided()) {
       if (Kind == TSK_CompleteObject)
         S.Diag(Selected->getLocation(), diag::note_nontrivial_user_provided)
-          << Kind << SubType.getUnqualifiedType() << CSM;
+            << Kind << SubType.getUnqualifiedType() << llvm::to_underlying(CSM);
       else {
         S.Diag(SubobjLoc, diag::note_nontrivial_user_provided)
-          << Kind << SubType.getUnqualifiedType() << CSM;
+            << Kind << SubType.getUnqualifiedType() << llvm::to_underlying(CSM);
         S.Diag(Selected->getLocation(), diag::note_declared_at);
       }
     } else {
       if (Kind != TSK_CompleteObject)
         S.Diag(SubobjLoc, diag::note_nontrivial_subobject)
-          << Kind << SubType.getUnqualifiedType() << CSM;
+            << Kind << SubType.getUnqualifiedType() << llvm::to_underlying(CSM);
 
       // Explain why the defaulted or deleted special member isn't trivial.
       S.SpecialMemberIsTrivial(Selected, CSM, Sema::TAH_IgnoreTrivialABI,
@@ -10097,8 +10127,7 @@ static bool checkTrivialSubobjectCall(Sema &S, SourceLocation SubobjLoc,
 /// Check whether the members of a class type allow a special member to be
 /// trivial.
 static bool checkTrivialClassMembers(Sema &S, CXXRecordDecl *RD,
-                                     Sema::CXXSpecialMember CSM,
-                                     bool ConstArg,
+                                     CXXSpecialMemberKind CSM, bool ConstArg,
                                      Sema::TrivialABIHandling TAH,
                                      bool Diagnose) {
   for (const auto *FI : RD->fields()) {
@@ -10119,7 +10148,8 @@ static bool checkTrivialClassMembers(Sema &S, CXXRecordDecl *RD,
     //   A default constructor is trivial if [...]
     //    -- no non-static data member of its class has a
     //       brace-or-equal-initializer
-    if (CSM == Sema::CXXDefaultConstructor && FI->hasInClassInitializer()) {
+    if (CSM == CXXSpecialMemberKind::DefaultConstructor &&
+        FI->hasInClassInitializer()) {
       if (Diagnose)
         S.Diag(FI->getLocation(), diag::note_nontrivial_default_member_init)
             << FI;
@@ -10148,10 +10178,12 @@ static bool checkTrivialClassMembers(Sema &S, CXXRecordDecl *RD,
 
 /// Diagnose why the specified class does not have a trivial special member of
 /// the given kind.
-void Sema::DiagnoseNontrivial(const CXXRecordDecl *RD, CXXSpecialMember CSM) {
+void Sema::DiagnoseNontrivial(const CXXRecordDecl *RD,
+                              CXXSpecialMemberKind CSM) {
   QualType Ty = Context.getRecordType(RD);
 
-  bool ConstArg = (CSM == CXXCopyConstructor || CSM == CXXCopyAssignment);
+  bool ConstArg = (CSM == CXXSpecialMemberKind::CopyConstructor ||
+                   CSM == CXXSpecialMemberKind::CopyAssignment);
   checkTrivialSubobjectCall(*this, RD->getLocation(), Ty, ConstArg, CSM,
                             TSK_CompleteObject, TAH_IgnoreTrivialABI,
                             /*Diagnose*/true);
@@ -10160,9 +10192,10 @@ void Sema::DiagnoseNontrivial(const CXXRecordDecl *RD, CXXSpecialMember CSM) {
 /// Determine whether a defaulted or deleted special member function is trivial,
 /// as specified in C++11 [class.ctor]p5, C++11 [class.copy]p12,
 /// C++11 [class.copy]p25, and C++11 [class.dtor]p5.
-bool Sema::SpecialMemberIsTrivial(CXXMethodDecl *MD, CXXSpecialMember CSM,
+bool Sema::SpecialMemberIsTrivial(CXXMethodDecl *MD, CXXSpecialMemberKind CSM,
                                   TrivialABIHandling TAH, bool Diagnose) {
-  assert(!MD->isUserProvided() && CSM != CXXInvalid && "not special enough");
+  assert(!MD->isUserProvided() && CSM != CXXSpecialMemberKind::Invalid &&
+         "not special enough");
 
   CXXRecordDecl *RD = MD->getParent();
 
@@ -10172,13 +10205,13 @@ bool Sema::SpecialMemberIsTrivial(CXXMethodDecl *MD, CXXSpecialMember CSM,
   //   A [special member] is trivial if [...] its parameter-type-list is
   //   equivalent to the parameter-type-list of an implicit declaration [...]
   switch (CSM) {
-  case CXXDefaultConstructor:
-  case CXXDestructor:
+  case CXXSpecialMemberKind::DefaultConstructor:
+  case CXXSpecialMemberKind::Destructor:
     // Trivial default constructors and destructors cannot have parameters.
     break;
 
-  case CXXCopyConstructor:
-  case CXXCopyAssignment: {
+  case CXXSpecialMemberKind::CopyConstructor:
+  case CXXSpecialMemberKind::CopyAssignment: {
     const ParmVarDecl *Param0 = MD->getNonObjectParameter(0);
     const ReferenceType *RT = Param0->getType()->getAs<ReferenceType>();
 
@@ -10207,8 +10240,8 @@ bool Sema::SpecialMemberIsTrivial(CXXMethodDecl *MD, CXXSpecialMember CSM,
     break;
   }
 
-  case CXXMoveConstructor:
-  case CXXMoveAssignment: {
+  case CXXSpecialMemberKind::MoveConstructor:
+  case CXXSpecialMemberKind::MoveAssignment: {
     // Trivial move operations always have non-cv-qualified parameters.
     const ParmVarDecl *Param0 = MD->getNonObjectParameter(0);
     const RValueReferenceType *RT =
@@ -10223,7 +10256,7 @@ bool Sema::SpecialMemberIsTrivial(CXXMethodDecl *MD, CXXSpecialMember CSM,
     break;
   }
 
-  case CXXInvalid:
+  case CXXSpecialMemberKind::Invalid:
     llvm_unreachable("not a special member");
   }
 
@@ -10272,7 +10305,7 @@ bool Sema::SpecialMemberIsTrivial(CXXMethodDecl *MD, CXXSpecialMember CSM,
   // C++11 [class.dtor]p5:
   //   A destructor is trivial if [...]
   //    -- the destructor is not virtual
-  if (CSM == CXXDestructor && MD->isVirtual()) {
+  if (CSM == CXXSpecialMemberKind::Destructor && MD->isVirtual()) {
     if (Diagnose)
       Diag(MD->getLocation(), diag::note_nontrivial_virtual_dtor) << RD;
     return false;
@@ -10281,7 +10314,8 @@ bool Sema::SpecialMemberIsTrivial(CXXMethodDecl *MD, CXXSpecialMember CSM,
   // C++11 [class.ctor]p5, C++11 [class.copy]p12, C++11 [class.copy]p25:
   //   A [special member] for class X is trivial if [...]
   //    -- class X has no virtual functions and no virtual base classes
-  if (CSM != CXXDestructor && MD->getParent()->isDynamicClass()) {
+  if (CSM != CXXSpecialMemberKind::Destructor &&
+      MD->getParent()->isDynamicClass()) {
     if (!Diagnose)
       return false;
 
@@ -13760,7 +13794,7 @@ struct SpecialMemberExceptionSpecInfo
   Sema::ImplicitExceptionSpecification ExceptSpec;
 
   SpecialMemberExceptionSpecInfo(Sema &S, CXXMethodDecl *MD,
-                                 Sema::CXXSpecialMember CSM,
+                                 CXXSpecialMemberKind CSM,
                                  Sema::InheritedConstructorInfo *ICI,
                                  SourceLocation Loc)
       : SpecialMemberVisitor(S, MD, CSM, ICI), Loc(Loc), ExceptSpec(S) {}
@@ -13793,7 +13827,8 @@ bool SpecialMemberExceptionSpecInfo::visitBase(CXXBaseSpecifier *Base) {
 }
 
 bool SpecialMemberExceptionSpecInfo::visitField(FieldDecl *FD) {
-  if (CSM == Sema::CXXDefaultConstructor && FD->hasInClassInitializer()) {
+  if (CSM == CXXSpecialMemberKind::DefaultConstructor &&
+      FD->hasInClassInitializer()) {
     Expr *E = FD->getInClassInitializer();
     if (!E)
       // FIXME: It's a little wasteful to build and throw away a
@@ -13852,7 +13887,7 @@ ExplicitSpecifier Sema::ActOnExplicitBoolSpecifier(Expr *ExplicitExpr) {
 
 static Sema::ImplicitExceptionSpecification
 ComputeDefaultedSpecialMemberExceptionSpec(
-    Sema &S, SourceLocation Loc, CXXMethodDecl *MD, Sema::CXXSpecialMember CSM,
+    Sema &S, SourceLocation Loc, CXXMethodDecl *MD, CXXSpecialMemberKind CSM,
     Sema::InheritedConstructorInfo *ICI) {
   ComputingExceptionSpec CES(S, MD, Loc);
 
@@ -13902,7 +13937,7 @@ struct DeclaringSpecialMember {
   Sema::ContextRAII SavedContext;
   bool WasAlreadyBeingDeclared;
 
-  DeclaringSpecialMember(Sema &S, CXXRecordDecl *RD, Sema::CXXSpecialMember CSM)
+  DeclaringSpecialMember(Sema &S, CXXRecordDecl *RD, CXXSpecialMemberKind CSM)
       : S(S), D(RD, CSM), SavedContext(S, RD) {
     WasAlreadyBeingDeclared = !S.SpecialMembersBeingDeclared.insert(D).second;
     if (WasAlreadyBeingDeclared)
@@ -13992,13 +14027,13 @@ CXXConstructorDecl *Sema::DeclareImplicitDefaultConstructor(
   assert(ClassDecl->needsImplicitDefaultConstructor() &&
          "Should not build implicit default constructor!");
 
-  DeclaringSpecialMember DSM(*this, ClassDecl, CXXDefaultConstructor);
+  DeclaringSpecialMember DSM(*this, ClassDecl,
+                             CXXSpecialMemberKind::DefaultConstructor);
   if (DSM.isAlreadyBeingDeclared())
     return nullptr;
 
-  bool Constexpr = defaultedSpecialMemberIsConstexpr(*this, ClassDecl,
-                                                     CXXDefaultConstructor,
-                                                     false);
+  bool Constexpr = defaultedSpecialMemberIsConstexpr(
+      *this, ClassDecl, CXXSpecialMemberKind::DefaultConstructor, false);
 
   // Create the actual constructor declaration.
   CanQualType ClassType
@@ -14020,10 +14055,10 @@ CXXConstructorDecl *Sema::DeclareImplicitDefaultConstructor(
   setupImplicitSpecialMemberType(DefaultCon, Context.VoidTy, std::nullopt);
 
   if (getLangOpts().CUDA)
-    inferCUDATargetForImplicitSpecialMember(ClassDecl, CXXDefaultConstructor,
-                                            DefaultCon,
-                                            /* ConstRHS */ false,
-                                            /* Diagnose */ false);
+    inferCUDATargetForImplicitSpecialMember(
+        ClassDecl, CXXSpecialMemberKind::DefaultConstructor, DefaultCon,
+        /* ConstRHS */ false,
+        /* Diagnose */ false);
 
   // We don't need to use SpecialMemberIsTrivial here; triviality for default
   // constructors is easy to compute.
@@ -14035,7 +14070,8 @@ CXXConstructorDecl *Sema::DeclareImplicitDefaultConstructor(
   Scope *S = getScopeForContext(ClassDecl);
   CheckImplicitSpecialMemberDeclaration(S, DefaultCon);
 
-  if (ShouldDeleteSpecialMember(DefaultCon, CXXDefaultConstructor))
+  if (ShouldDeleteSpecialMember(DefaultCon,
+                                CXXSpecialMemberKind::DefaultConstructor))
     SetDeclDeleted(DefaultCon, ClassLoc);
 
   if (S)
@@ -14127,10 +14163,10 @@ Sema::findInheritingConstructor(SourceLocation Loc,
   // from which it was inherited.
   InheritedConstructorInfo ICI(*this, Loc, Shadow);
 
-  bool Constexpr =
-      BaseCtor->isConstexpr() &&
-      defaultedSpecialMemberIsConstexpr(*this, Derived, CXXDefaultConstructor,
-                                        false, BaseCtor, &ICI);
+  bool Constexpr = BaseCtor->isConstexpr() &&
+                   defaultedSpecialMemberIsConstexpr(
+                       *this, Derived, CXXSpecialMemberKind::DefaultConstructor,
+                       false, BaseCtor, &ICI);
 
   CXXConstructorDecl *DerivedCtor = CXXConstructorDecl::Create(
       Context, Derived, UsingLoc, NameInfo, TInfo->getType(), TInfo,
@@ -14174,7 +14210,8 @@ Sema::findInheritingConstructor(SourceLocation Loc,
   DerivedCtor->setParams(ParamDecls);
   Derived->addDecl(DerivedCtor);
 
-  if (ShouldDeleteSpecialMember(DerivedCtor, CXXDefaultConstructor, &ICI))
+  if (ShouldDeleteSpecialMember(DerivedCtor,
+                                CXXSpecialMemberKind::DefaultConstructor, &ICI))
     SetDeclDeleted(DerivedCtor, UsingLoc);
 
   return DerivedCtor;
@@ -14183,8 +14220,9 @@ Sema::findInheritingConstructor(SourceLocation Loc,
 void Sema::NoteDeletedInheritingConstructor(CXXConstructorDecl *Ctor) {
   InheritedConstructorInfo ICI(*this, Ctor->getLocation(),
                                Ctor->getInheritedConstructor().getShadowDecl());
-  ShouldDeleteSpecialMember(Ctor, CXXDefaultConstructor, &ICI,
-                            /*Diagnose*/true);
+  ShouldDeleteSpecialMember(Ctor, CXXSpecialMemberKind::DefaultConstructor,
+                            &ICI,
+                            /*Diagnose*/ true);
 }
 
 void Sema::DefineInheritingConstructor(SourceLocation CurrentLocation,
@@ -14275,13 +14313,13 @@ CXXDestructorDecl *Sema::DeclareImplicitDestructor(CXXRecordDecl *ClassDecl) {
   //   inline public member of its class.
   assert(ClassDecl->needsImplicitDestructor());
 
-  DeclaringSpecialMember DSM(*this, ClassDecl, CXXDestructor);
+  DeclaringSpecialMember DSM(*this, ClassDecl,
+                             CXXSpecialMemberKind::Destructor);
   if (DSM.isAlreadyBeingDeclared())
     return nullptr;
 
-  bool Constexpr = defaultedSpecialMemberIsConstexpr(*this, ClassDecl,
-                                                     CXXDestructor,
-                                                     false);
+  bool Constexpr = defaultedSpecialMemberIsConstexpr(
+      *this, ClassDecl, CXXSpecialMemberKind::Destructor, false);
 
   // Create the actual destructor declaration.
   CanQualType ClassType
@@ -14303,10 +14341,10 @@ CXXDestructorDecl *Sema::DeclareImplicitDestructor(CXXRecordDecl *ClassDecl) {
   setupImplicitSpecialMemberType(Destructor, Context.VoidTy, std::nullopt);
 
   if (getLangOpts().CUDA)
-    inferCUDATargetForImplicitSpecialMember(ClassDecl, CXXDestructor,
-                                            Destructor,
-                                            /* ConstRHS */ false,
-                                            /* Diagnose */ false);
+    inferCUDATargetForImplicitSpecialMember(
+        ClassDecl, CXXSpecialMemberKind::Destructor, Destructor,
+        /* ConstRHS */ false,
+        /* Diagnose */ false);
 
   // We don't need to use SpecialMemberIsTrivial here; triviality for
   // destructors is easy to compute.
@@ -14324,7 +14362,7 @@ CXXDestructorDecl *Sema::DeclareImplicitDestructor(CXXRecordDecl *ClassDecl) {
   // the definition of the class, because its validity depends on the alignment
   // of the class. We'll check this from ActOnFields once the class is complete.
   if (ClassDecl->isCompleteDefinition() &&
-      ShouldDeleteSpecialMember(Destructor, CXXDestructor))
+      ShouldDeleteSpecialMember(Destructor, CXXSpecialMemberKind::Destructor))
     SetDeclDeleted(Destructor, ClassLoc);
 
   // Introduce this destructor into its scope.
@@ -14905,7 +14943,8 @@ CXXMethodDecl *Sema::DeclareImplicitCopyAssignment(CXXRecordDecl *ClassDecl) {
   // operators taking an object instead of a reference are allowed.
   assert(ClassDecl->needsImplicitCopyAssignment());
 
-  DeclaringSpecialMember DSM(*this, ClassDecl, CXXCopyAssignment);
+  DeclaringSpecialMember DSM(*this, ClassDecl,
+                             CXXSpecialMemberKind::CopyAssignment);
   if (DSM.isAlreadyBeingDeclared())
     return nullptr;
 
@@ -14922,9 +14961,8 @@ CXXMethodDecl *Sema::DeclareImplicitCopyAssignment(CXXRecordDecl *ClassDecl) {
 
   ArgType = Context.getLValueReferenceType(ArgType);
 
-  bool Constexpr = defaultedSpecialMemberIsConstexpr(*this, ClassDecl,
-                                                     CXXCopyAssignment,
-                                                     Const);
+  bool Constexpr = defaultedSpecialMemberIsConstexpr(
+      *this, ClassDecl, CXXSpecialMemberKind::CopyAssignment, Const);
 
   //   An implicitly-declared copy assignment operator is an inline public
   //   member of its class.
@@ -14945,10 +14983,10 @@ CXXMethodDecl *Sema::DeclareImplicitCopyAssignment(CXXRecordDecl *ClassDecl) {
   setupImplicitSpecialMemberType(CopyAssignment, RetType, ArgType);
 
   if (getLangOpts().CUDA)
-    inferCUDATargetForImplicitSpecialMember(ClassDecl, CXXCopyAssignment,
-                                            CopyAssignment,
-                                            /* ConstRHS */ Const,
-                                            /* Diagnose */ false);
+    inferCUDATargetForImplicitSpecialMember(
+        ClassDecl, CXXSpecialMemberKind::CopyAssignment, CopyAssignment,
+        /* ConstRHS */ Const,
+        /* Diagnose */ false);
 
   // Add the parameter to the operator.
   ParmVarDecl *FromParam = ParmVarDecl::Create(Context, CopyAssignment,
@@ -14959,9 +14997,10 @@ CXXMethodDecl *Sema::DeclareImplicitCopyAssignment(CXXRecordDecl *ClassDecl) {
   CopyAssignment->setParams(FromParam);
 
   CopyAssignment->setTrivial(
-    ClassDecl->needsOverloadResolutionForCopyAssignment()
-      ? SpecialMemberIsTrivial(CopyAssignment, CXXCopyAssignment)
-      : ClassDecl->hasTrivialCopyAssignment());
+      ClassDecl->needsOverloadResolutionForCopyAssignment()
+          ? SpecialMemberIsTrivial(CopyAssignment,
+                                   CXXSpecialMemberKind::CopyAssignment)
+          : ClassDecl->hasTrivialCopyAssignment());
 
   // Note that we have added this copy-assignment operator.
   ++getASTContext().NumImplicitCopyAssignmentOperatorsDeclared;
@@ -14969,7 +15008,8 @@ CXXMethodDecl *Sema::DeclareImplicitCopyAssignment(CXXRecordDecl *ClassDecl) {
   Scope *S = getScopeForContext(ClassDecl);
   CheckImplicitSpecialMemberDeclaration(S, CopyAssignment);
 
-  if (ShouldDeleteSpecialMember(CopyAssignment, CXXCopyAssignment)) {
+  if (ShouldDeleteSpecialMember(CopyAssignment,
+                                CXXSpecialMemberKind::CopyAssignment)) {
     ClassDecl->setImplicitCopyAssignmentIsDeleted();
     SetDeclDeleted(CopyAssignment, ClassLoc);
   }
@@ -15256,7 +15296,8 @@ void Sema::DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
 CXXMethodDecl *Sema::DeclareImplicitMoveAssignment(CXXRecordDecl *ClassDecl) {
   assert(ClassDecl->needsImplicitMoveAssignment());
 
-  DeclaringSpecialMember DSM(*this, ClassDecl, CXXMoveAssignment);
+  DeclaringSpecialMember DSM(*this, ClassDecl,
+                             CXXSpecialMemberKind::MoveAssignment);
   if (DSM.isAlreadyBeingDeclared())
     return nullptr;
 
@@ -15272,9 +15313,8 @@ CXXMethodDecl *Sema::DeclareImplicitMoveAssignment(CXXRecordDecl *ClassDecl) {
   QualType RetType = Context.getLValueReferenceType(ArgType);
   ArgType = Context.getRValueReferenceType(ArgType);
 
-  bool Constexpr = defaultedSpecialMemberIsConstexpr(*this, ClassDecl,
-                                                     CXXMoveAssignment,
-                                                     false);
+  bool Constexpr = defaultedSpecialMemberIsConstexpr(
+      *this, ClassDecl, CXXSpecialMemberKind::MoveAssignment, false);
 
   //   An implicitly-declared move assignment operator is an inline public
   //   member of its class.
@@ -15295,10 +15335,10 @@ CXXMethodDecl *Sema::DeclareImplicitMoveAssignment(CXXRecordDecl *ClassDecl) {
   setupImplicitSpecialMemberType(MoveAssignment, RetType, ArgType);
 
   if (getLangOpts().CUDA)
-    inferCUDATargetForImplicitSpecialMember(ClassDecl, CXXMoveAssignment,
-                                            MoveAssignment,
-                                            /* ConstRHS */ false,
-                                            /* Diagnose */ false);
+    inferCUDATargetForImplicitSpecialMember(
+        ClassDecl, CXXSpecialMemberKind::MoveAssignment, MoveAssignment,
+        /* ConstRHS */ false,
+        /* Diagnose */ false);
 
   // Add the parameter to the operator.
   ParmVarDecl *FromParam = ParmVarDecl::Create(Context, MoveAssignment,
@@ -15309,9 +15349,10 @@ CXXMethodDecl *Sema::DeclareImplicitMoveAssignment(CXXRecordDecl *ClassDecl) {
   MoveAssignment->setParams(FromParam);
 
   MoveAssignment->setTrivial(
-    ClassDecl->needsOverloadResolutionForMoveAssignment()
-      ? SpecialMemberIsTrivial(MoveAssignment, CXXMoveAssignment)
-      : ClassDecl->hasTrivialMoveAssignment());
+      ClassDecl->needsOverloadResolutionForMoveAssignment()
+          ? SpecialMemberIsTrivial(MoveAssignment,
+                                   CXXSpecialMemberKind::MoveAssignment)
+          : ClassDecl->hasTrivialMoveAssignment());
 
   // Note that we have added this copy-assignment operator.
   ++getASTContext().NumImplicitMoveAssignmentOperatorsDeclared;
@@ -15319,7 +15360,8 @@ CXXMethodDecl *Sema::DeclareImplicitMoveAssignment(CXXRecordDecl *ClassDecl) {
   Scope *S = getScopeForContext(ClassDecl);
   CheckImplicitSpecialMemberDeclaration(S, MoveAssignment);
 
-  if (ShouldDeleteSpecialMember(MoveAssignment, CXXMoveAssignment)) {
+  if (ShouldDeleteSpecialMember(MoveAssignment,
+                                CXXSpecialMemberKind::MoveAssignment)) {
     ClassDecl->setImplicitMoveAssignmentIsDeleted();
     SetDeclDeleted(MoveAssignment, ClassLoc);
   }
@@ -15368,10 +15410,10 @@ static void checkMoveAssignmentForRepeatedMove(Sema &S, CXXRecordDecl *Class,
       // If we're not actually going to call a move assignment for this base,
       // or the selected move assignment is trivial, skip it.
       Sema::SpecialMemberOverloadResult SMOR =
-        S.LookupSpecialMember(Base, Sema::CXXMoveAssignment,
-                              /*ConstArg*/false, /*VolatileArg*/false,
-                              /*RValueThis*/true, /*ConstThis*/false,
-                              /*VolatileThis*/false);
+          S.LookupSpecialMember(Base, CXXSpecialMemberKind::MoveAssignment,
+                                /*ConstArg*/ false, /*VolatileArg*/ false,
+                                /*RValueThis*/ true, /*ConstThis*/ false,
+                                /*VolatileThis*/ false);
       if (!SMOR.getMethod() || SMOR.getMethod()->isTrivial() ||
           !SMOR.getMethod()->isMoveAssignmentOperator())
         continue;
@@ -15648,7 +15690,8 @@ CXXConstructorDecl *Sema::DeclareImplicitCopyConstructor(
   //   constructor, one is declared implicitly.
   assert(ClassDecl->needsImplicitCopyConstructor());
 
-  DeclaringSpecialMember DSM(*this, ClassDecl, CXXCopyConstructor);
+  DeclaringSpecialMember DSM(*this, ClassDecl,
+                             CXXSpecialMemberKind::CopyConstructor);
   if (DSM.isAlreadyBeingDeclared())
     return nullptr;
 
@@ -15666,9 +15709,8 @@ CXXConstructorDecl *Sema::DeclareImplicitCopyConstructor(
 
   ArgType = Context.getLValueReferenceType(ArgType);
 
-  bool Constexpr = defaultedSpecialMemberIsConstexpr(*this, ClassDecl,
-                                                     CXXCopyConstructor,
-                                                     Const);
+  bool Constexpr = defaultedSpecialMemberIsConstexpr(
+      *this, ClassDecl, CXXSpecialMemberKind::CopyConstructor, Const);
 
   DeclarationName Name
     = Context.DeclarationNames.getCXXConstructorName(
@@ -15691,10 +15733,10 @@ CXXConstructorDecl *Sema::DeclareImplicitCopyConstructor(
   setupImplicitSpecialMemberType(CopyConstructor, Context.VoidTy, ArgType);
 
   if (getLangOpts().CUDA)
-    inferCUDATargetForImplicitSpecialMember(ClassDecl, CXXCopyConstructor,
-                                            CopyConstructor,
-                                            /* ConstRHS */ Const,
-                                            /* Diagnose */ false);
+    inferCUDATargetForImplicitSpecialMember(
+        ClassDecl, CXXSpecialMemberKind::CopyConstructor, CopyConstructor,
+        /* ConstRHS */ Const,
+        /* Diagnose */ false);
 
   // During template instantiation of special member functions we need a
   // reliable TypeSourceInfo for the parameter types in order to allow functions
@@ -15712,14 +15754,16 @@ CXXConstructorDecl *Sema::DeclareImplicitCopyConstructor(
 
   CopyConstructor->setTrivial(
       ClassDecl->needsOverloadResolutionForCopyConstructor()
-          ? SpecialMemberIsTrivial(CopyConstructor, CXXCopyConstructor)
+          ? SpecialMemberIsTrivial(CopyConstructor,
+                                   CXXSpecialMemberKind::CopyConstructor)
           : ClassDecl->hasTrivialCopyConstructor());
 
   CopyConstructor->setTrivialForCall(
       ClassDecl->hasAttr<TrivialABIAttr>() ||
       (ClassDecl->needsOverloadResolutionForCopyConstructor()
-           ? SpecialMemberIsTrivial(CopyConstructor, CXXCopyConstructor,
-             TAH_ConsiderTrivialABI)
+           ? SpecialMemberIsTrivial(CopyConstructor,
+                                    CXXSpecialMemberKind::CopyConstructor,
+                                    TAH_ConsiderTrivialABI)
            : ClassDecl->hasTrivialCopyConstructorForCall()));
 
   // Note that we have declared this constructor.
@@ -15728,7 +15772,8 @@ CXXConstructorDecl *Sema::DeclareImplicitCopyConstructor(
   Scope *S = getScopeForContext(ClassDecl);
   CheckImplicitSpecialMemberDeclaration(S, CopyConstructor);
 
-  if (ShouldDeleteSpecialMember(CopyConstructor, CXXCopyConstructor)) {
+  if (ShouldDeleteSpecialMember(CopyConstructor,
+                                CXXSpecialMemberKind::CopyConstructor)) {
     ClassDecl->setImplicitCopyConstructorIsDeleted();
     SetDeclDeleted(CopyConstructor, ClassLoc);
   }
@@ -15793,7 +15838,8 @@ CXXConstructorDecl *Sema::DeclareImplicitMoveConstructor(
                                                     CXXRecordDecl *ClassDecl) {
   assert(ClassDecl->needsImplicitMoveConstructor());
 
-  DeclaringSpecialMember DSM(*this, ClassDecl, CXXMoveConstructor);
+  DeclaringSpecialMember DSM(*this, ClassDecl,
+                             CXXSpecialMemberKind::MoveConstructor);
   if (DSM.isAlreadyBeingDeclared())
     return nullptr;
 
@@ -15807,9 +15853,8 @@ CXXConstructorDecl *Sema::DeclareImplicitMoveConstructor(
     ArgType = Context.getAddrSpaceQualType(ClassType, AS);
   ArgType = Context.getRValueReferenceType(ArgType);
 
-  bool Constexpr = defaultedSpecialMemberIsConstexpr(*this, ClassDecl,
-                                                     CXXMoveConstructor,
-                                                     false);
+  bool Constexpr = defaultedSpecialMemberIsConstexpr(
+      *this, ClassDecl, CXXSpecialMemberKind::MoveConstructor, false);
 
   DeclarationName Name
     = Context.DeclarationNames.getCXXConstructorName(
@@ -15833,10 +15878,10 @@ CXXConstructorDecl *Sema::DeclareImplicitMoveConstructor(
   setupImplicitSpecialMemberType(MoveConstructor, Context.VoidTy, ArgType);
 
   if (getLangOpts().CUDA)
-    inferCUDATargetForImplicitSpecialMember(ClassDecl, CXXMoveConstructor,
-                                            MoveConstructor,
-                                            /* ConstRHS */ false,
-                                            /* Diagnose */ false);
+    inferCUDATargetForImplicitSpecialMember(
+        ClassDecl, CXXSpecialMemberKind::MoveConstructor, MoveConstructor,
+        /* ConstRHS */ false,
+        /* Diagnose */ false);
 
   // Add the parameter to the constructor.
   ParmVarDecl *FromParam = ParmVarDecl::Create(Context, MoveConstructor,
@@ -15848,13 +15893,15 @@ CXXConstructorDecl *Sema::DeclareImplicitMoveConstructor(
 
   MoveConstructor->setTrivial(
       ClassDecl->needsOverloadResolutionForMoveConstructor()
-          ? SpecialMemberIsTrivial(MoveConstructor, CXXMoveConstructor)
+          ? SpecialMemberIsTrivial(MoveConstructor,
+                                   CXXSpecialMemberKind::MoveConstructor)
           : ClassDecl->hasTrivialMoveConstructor());
 
   MoveConstructor->setTrivialForCall(
       ClassDecl->hasAttr<TrivialABIAttr>() ||
       (ClassDecl->needsOverloadResolutionForMoveConstructor()
-           ? SpecialMemberIsTrivial(MoveConstructor, CXXMoveConstructor,
+           ? SpecialMemberIsTrivial(MoveConstructor,
+                                    CXXSpecialMemberKind::MoveConstructor,
                                     TAH_ConsiderTrivialABI)
            : ClassDecl->hasTrivialMoveConstructorForCall()));
 
@@ -15864,7 +15911,8 @@ CXXConstructorDecl *Sema::DeclareImplicitMoveConstructor(
   Scope *S = getScopeForContext(ClassDecl);
   CheckImplicitSpecialMemberDeclaration(S, MoveConstructor);
 
-  if (ShouldDeleteSpecialMember(MoveConstructor, CXXMoveConstructor)) {
+  if (ShouldDeleteSpecialMember(MoveConstructor,
+                                CXXSpecialMemberKind::MoveConstructor)) {
     ClassDecl->setImplicitMoveConstructorIsDeleted();
     SetDeclDeleted(MoveConstructor, ClassLoc);
   }
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index a75e9925a431..6b8ce0f633a3 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -31,6 +31,7 @@
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/FoldingSet.h"
 #include "llvm/ADT/PointerIntPair.h"
+#include "llvm/ADT/STLForwardCompat.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
@@ -10023,8 +10024,9 @@ bool InitializationSequence::Diagnose(Sema &S,
         // implicit.
         if (S.isImplicitlyDeleted(Best->Function))
           S.Diag(Kind.getLocation(), diag::err_ovl_deleted_special_init)
-            << S.getSpecialMember(cast<CXXMethodDecl>(Best->Function))
-            << DestType << ArgsRange;
+              << llvm::to_underlying(
+                     S.getSpecialMember(cast<CXXMethodDecl>(Best->Function)))
+              << DestType << ArgsRange;
         else
           S.Diag(Kind.getLocation(), diag::err_ovl_deleted_init)
               << DestType << ArgsRange;
diff --git a/clang/lib/Sema/SemaLookup.cpp b/clang/lib/Sema/SemaLookup.cpp
index 38237ee57807..d65f52b8efe8 100644
--- a/clang/lib/Sema/SemaLookup.cpp
+++ b/clang/lib/Sema/SemaLookup.cpp
@@ -37,6 +37,7 @@
 #include "clang/Sema/TemplateDeduction.h"
 #include "clang/Sema/TypoCorrection.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/STLForwardCompat.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/TinyPtrVector.h"
 #include "llvm/ADT/edit_distance.h"
@@ -3341,21 +3342,20 @@ void Sema::LookupOverloadedOperatorName(OverloadedOperatorKind Op, Scope *S,
   Functions.append(Operators.begin(), Operators.end());
 }
 
-Sema::SpecialMemberOverloadResult Sema::LookupSpecialMember(CXXRecordDecl *RD,
-                                                           CXXSpecialMember SM,
-                                                           bool ConstArg,
-                                                           bool VolatileArg,
-                                                           bool RValueThis,
-                                                           bool ConstThis,
-                                                           bool VolatileThis) {
+Sema::SpecialMemberOverloadResult
+Sema::LookupSpecialMember(CXXRecordDecl *RD, CXXSpecialMemberKind SM,
+                          bool ConstArg, bool VolatileArg, bool RValueThis,
+                          bool ConstThis, bool VolatileThis) {
   assert(CanDeclareSpecialMemberFunction(RD) &&
          "doing special member lookup into record that isn't fully complete");
   RD = RD->getDefinition();
   if (RValueThis || ConstThis || VolatileThis)
-    assert((SM == CXXCopyAssignment || SM == CXXMoveAssignment) &&
+    assert((SM == CXXSpecialMemberKind::CopyAssignment ||
+            SM == CXXSpecialMemberKind::MoveAssignment) &&
            "constructors and destructors always have unqualified lvalue this");
   if (ConstArg || VolatileArg)
-    assert((SM != CXXDefaultConstructor && SM != CXXDestructor) &&
+    assert((SM != CXXSpecialMemberKind::DefaultConstructor &&
+            SM != CXXSpecialMemberKind::Destructor) &&
            "parameter-less special members can't have qualified arguments");
 
   // FIXME: Get the caller to pass in a location for the lookup.
@@ -3363,7 +3363,7 @@ Sema::SpecialMemberOverloadResult Sema::LookupSpecialMember(CXXRecordDecl *RD,
 
   llvm::FoldingSetNodeID ID;
   ID.AddPointer(RD);
-  ID.AddInteger(SM);
+  ID.AddInteger(llvm::to_underlying(SM));
   ID.AddInteger(ConstArg);
   ID.AddInteger(VolatileArg);
   ID.AddInteger(RValueThis);
@@ -3382,7 +3382,7 @@ Sema::SpecialMemberOverloadResult Sema::LookupSpecialMember(CXXRecordDecl *RD,
   Result = new (Result) SpecialMemberOverloadResultEntry(ID);
   SpecialMemberCache.InsertNode(Result, InsertPoint);
 
-  if (SM == CXXDestructor) {
+  if (SM == CXXSpecialMemberKind::Destructor) {
     if (RD->needsImplicitDestructor()) {
       runWithSufficientStackSpace(RD->getLocation(), [&] {
         DeclareImplicitDestructor(RD);
@@ -3406,7 +3406,7 @@ Sema::SpecialMemberOverloadResult Sema::LookupSpecialMember(CXXRecordDecl *RD,
   QualType ArgType = CanTy;
   ExprValueKind VK = VK_LValue;
 
-  if (SM == CXXDefaultConstructor) {
+  if (SM == CXXSpecialMemberKind::DefaultConstructor) {
     Name = Context.DeclarationNames.getCXXConstructorName(CanTy);
     NumArgs = 0;
     if (RD->needsImplicitDefaultConstructor()) {
@@ -3415,7 +3415,8 @@ Sema::SpecialMemberOverloadResult Sema::LookupSpecialMember(CXXRecordDecl *RD,
       });
     }
   } else {
-    if (SM == CXXCopyConstructor || SM == CXXMoveConstructor) {
+    if (SM == CXXSpecialMemberKind::CopyConstructor ||
+        SM == CXXSpecialMemberKind::MoveConstructor) {
       Name = Context.DeclarationNames.getCXXConstructorName(CanTy);
       if (RD->needsImplicitCopyConstructor()) {
         runWithSufficientStackSpace(RD->getLocation(), [&] {
@@ -3453,7 +3454,8 @@ Sema::SpecialMemberOverloadResult Sema::LookupSpecialMember(CXXRecordDecl *RD,
     // Possibly an XValue is actually correct in the case of move, but
     // there is no semantic difference for class types in this restricted
     // case.
-    if (SM == CXXCopyConstructor || SM == CXXCopyAssignment)
+    if (SM == CXXSpecialMemberKind::CopyConstructor ||
+        SM == CXXSpecialMemberKind::CopyAssignment)
       VK = VK_LValue;
     else
       VK = VK_PRValue;
@@ -3461,7 +3463,7 @@ Sema::SpecialMemberOverloadResult Sema::LookupSpecialMember(CXXRecordDecl *RD,
 
   OpaqueValueExpr FakeArg(LookupLoc, ArgType, VK);
 
-  if (SM != CXXDefaultConstructor) {
+  if (SM != CXXSpecialMemberKind::DefaultConstructor) {
     NumArgs = 1;
     Arg = &FakeArg;
   }
@@ -3487,7 +3489,7 @@ Sema::SpecialMemberOverloadResult Sema::LookupSpecialMember(CXXRecordDecl *RD,
     // type, rather than because there's some other declared constructor.
     // Every class has a copy/move constructor, copy/move assignment, and
     // destructor.
-    assert(SM == CXXDefaultConstructor &&
+    assert(SM == CXXSpecialMemberKind::DefaultConstructor &&
            "lookup for a constructor or assignment operator was empty");
     Result->setMethod(nullptr);
     Result->setKind(SpecialMemberOverloadResult::NoMemberOrDeleted);
@@ -3505,7 +3507,8 @@ Sema::SpecialMemberOverloadResult Sema::LookupSpecialMember(CXXRecordDecl *RD,
     DeclAccessPair Cand = DeclAccessPair::make(CandDecl, AS_public);
     auto CtorInfo = getConstructorInfo(Cand);
     if (CXXMethodDecl *M = dyn_cast<CXXMethodDecl>(Cand->getUnderlyingDecl())) {
-      if (SM == CXXCopyAssignment || SM == CXXMoveAssignment)
+      if (SM == CXXSpecialMemberKind::CopyAssignment ||
+          SM == CXXSpecialMemberKind::MoveAssignment)
         AddMethodCandidate(M, Cand, RD, ThisTy, Classification,
                            llvm::ArrayRef(&Arg, NumArgs), OCS, true);
       else if (CtorInfo)
@@ -3517,7 +3520,8 @@ Sema::SpecialMemberOverloadResult Sema::LookupSpecialMember(CXXRecordDecl *RD,
                              /*SuppressUserConversions*/ true);
     } else if (FunctionTemplateDecl *Tmpl =
                  dyn_cast<FunctionTemplateDecl>(Cand->getUnderlyingDecl())) {
-      if (SM == CXXCopyAssignment || SM == CXXMoveAssignment)
+      if (SM == CXXSpecialMemberKind::CopyAssignment ||
+          SM == CXXSpecialMemberKind::MoveAssignment)
         AddMethodTemplateCandidate(Tmpl, Cand, RD, nullptr, ThisTy,
                                    Classification,
                                    llvm::ArrayRef(&Arg, NumArgs), OCS, true);
@@ -3563,8 +3567,8 @@ Sema::SpecialMemberOverloadResult Sema::LookupSpecialMember(CXXRecordDecl *RD,
 /// Look up the default constructor for the given class.
 CXXConstructorDecl *Sema::LookupDefaultConstructor(CXXRecordDecl *Class) {
   SpecialMemberOverloadResult Result =
-    LookupSpecialMember(Class, CXXDefaultConstructor, false, false, false,
-                        false, false);
+      LookupSpecialMember(Class, CXXSpecialMemberKind::DefaultConstructor,
+                          false, false, false, false, false);
 
   return cast_or_null<CXXConstructorDecl>(Result.getMethod());
 }
@@ -3574,9 +3578,9 @@ CXXConstructorDecl *Sema::LookupCopyingConstructor(CXXRecordDecl *Class,
                                                    unsigned Quals) {
   assert(!(Quals & ~(Qualifiers::Const | Qualifiers::Volatile)) &&
          "non-const, non-volatile qualifiers for copy ctor arg");
-  SpecialMemberOverloadResult Result =
-    LookupSpecialMember(Class, CXXCopyConstructor, Quals & Qualifiers::Const,
-                        Quals & Qualifiers::Volatile, false, false, false);
+  SpecialMemberOverloadResult Result = LookupSpecialMember(
+      Class, CXXSpecialMemberKind::CopyConstructor, Quals & Qualifiers::Const,
+      Quals & Qualifiers::Volatile, false, false, false);
 
   return cast_or_null<CXXConstructorDecl>(Result.getMethod());
 }
@@ -3584,9 +3588,9 @@ CXXConstructorDecl *Sema::LookupCopyingConstructor(CXXRecordDecl *Class,
 /// Look up the moving constructor for the given class.
 CXXConstructorDecl *Sema::LookupMovingConstructor(CXXRecordDecl *Class,
                                                   unsigned Quals) {
-  SpecialMemberOverloadResult Result =
-    LookupSpecialMember(Class, CXXMoveConstructor, Quals & Qualifiers::Const,
-                        Quals & Qualifiers::Volatile, false, false, false);
+  SpecialMemberOverloadResult Result = LookupSpecialMember(
+      Class, CXXSpecialMemberKind::MoveConstructor, Quals & Qualifiers::Const,
+      Quals & Qualifiers::Volatile, false, false, false);
 
   return cast_or_null<CXXConstructorDecl>(Result.getMethod());
 }
@@ -3618,11 +3622,10 @@ CXXMethodDecl *Sema::LookupCopyingAssignment(CXXRecordDecl *Class,
          "non-const, non-volatile qualifiers for copy assignment arg");
   assert(!(ThisQuals & ~(Qualifiers::Const | Qualifiers::Volatile)) &&
          "non-const, non-volatile qualifiers for copy assignment this");
-  SpecialMemberOverloadResult Result =
-    LookupSpecialMember(Class, CXXCopyAssignment, Quals & Qualifiers::Const,
-                        Quals & Qualifiers::Volatile, RValueThis,
-                        ThisQuals & Qualifiers::Const,
-                        ThisQuals & Qualifiers::Volatile);
+  SpecialMemberOverloadResult Result = LookupSpecialMember(
+      Class, CXXSpecialMemberKind::CopyAssignment, Quals & Qualifiers::Const,
+      Quals & Qualifiers::Volatile, RValueThis, ThisQuals & Qualifiers::Const,
+      ThisQuals & Qualifiers::Volatile);
 
   return Result.getMethod();
 }
@@ -3634,11 +3637,10 @@ CXXMethodDecl *Sema::LookupMovingAssignment(CXXRecordDecl *Class,
                                             unsigned ThisQuals) {
   assert(!(ThisQuals & ~(Qualifiers::Const | Qualifiers::Volatile)) &&
          "non-const, non-volatile qualifiers for copy assignment this");
-  SpecialMemberOverloadResult Result =
-    LookupSpecialMember(Class, CXXMoveAssignment, Quals & Qualifiers::Const,
-                        Quals & Qualifiers::Volatile, RValueThis,
-                        ThisQuals & Qualifiers::Const,
-                        ThisQuals & Qualifiers::Volatile);
+  SpecialMemberOverloadResult Result = LookupSpecialMember(
+      Class, CXXSpecialMemberKind::MoveAssignment, Quals & Qualifiers::Const,
+      Quals & Qualifiers::Volatile, RValueThis, ThisQuals & Qualifiers::Const,
+      ThisQuals & Qualifiers::Volatile);
 
   return Result.getMethod();
 }
@@ -3651,8 +3653,8 @@ CXXMethodDecl *Sema::LookupMovingAssignment(CXXRecordDecl *Class,
 /// \returns The destructor for this class.
 CXXDestructorDecl *Sema::LookupDestructor(CXXRecordDecl *Class) {
   return cast_or_null<CXXDestructorDecl>(
-      LookupSpecialMember(Class, CXXDestructor, false, false, false, false,
-                          false)
+      LookupSpecialMember(Class, CXXSpecialMemberKind::Destructor, false, false,
+                          false, false, false)
           .getMethod());
 }
 
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index 3808af37ff54..1f674dd4bb0f 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -36,6 +36,7 @@
 #include "clang/Sema/TemplateDeduction.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/STLForwardCompat.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
@@ -11955,25 +11956,25 @@ static void DiagnoseBadTarget(Sema &S, OverloadCandidate *Cand) {
   CXXMethodDecl *Meth = dyn_cast<CXXMethodDecl>(Callee);
   if (Meth != nullptr && Meth->isImplicit()) {
     CXXRecordDecl *ParentClass = Meth->getParent();
-    Sema::CXXSpecialMember CSM;
+    CXXSpecialMemberKind CSM;
 
     switch (FnKindPair.first) {
     default:
       return;
     case oc_implicit_default_constructor:
-      CSM = Sema::CXXDefaultConstructor;
+      CSM = CXXSpecialMemberKind::DefaultConstructor;
       break;
     case oc_implicit_copy_constructor:
-      CSM = Sema::CXXCopyConstructor;
+      CSM = CXXSpecialMemberKind::CopyConstructor;
       break;
     case oc_implicit_move_constructor:
-      CSM = Sema::CXXMoveConstructor;
+      CSM = CXXSpecialMemberKind::MoveConstructor;
       break;
     case oc_implicit_copy_assignment:
-      CSM = Sema::CXXCopyAssignment;
+      CSM = CXXSpecialMemberKind::CopyAssignment;
       break;
     case oc_implicit_move_assignment:
-      CSM = Sema::CXXMoveAssignment;
+      CSM = CXXSpecialMemberKind::MoveAssignment;
       break;
     };
 
@@ -15064,7 +15065,8 @@ ExprResult Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
         DefaultedFunctionKind DFK = getDefaultedFunctionKind(DeletedFD);
         if (DFK.isSpecialMember()) {
           Diag(OpLoc, diag::err_ovl_deleted_special_oper)
-            << Args[0]->getType() << DFK.asSpecialMember();
+              << Args[0]->getType()
+              << llvm::to_underlying(DFK.asSpecialMember());
         } else {
           assert(DFK.isComparison());
           Diag(OpLoc, diag::err_ovl_deleted_comparison)
diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp
index a265fd1c46a6..7cd428de0bb3 100644
--- a/clang/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp
@@ -36,6 +36,7 @@
 #include "clang/Sema/Template.h"
 #include "clang/Sema/TemplateDeduction.h"
 #include "clang/Sema/TemplateInstCallback.h"
+#include "llvm/ADT/STLForwardCompat.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/TimeProfiler.h"
@@ -1142,7 +1143,8 @@ void Sema::PrintInstantiationStack() {
     case CodeSynthesisContext::DeclaringSpecialMember:
       Diags.Report(Active->PointOfInstantiation,
                    diag::note_in_declaration_of_implicit_special_member)
-        << cast<CXXRecordDecl>(Active->Entity) << Active->SpecialMember;
+          << cast<CXXRecordDecl>(Active->Entity)
+          << llvm::to_underlying(Active->SpecialMember);
       break;
 
     case CodeSynthesisContext::DeclaringImplicitEqualityComparison:
@@ -1160,7 +1162,8 @@ void Sema::PrintInstantiationStack() {
         auto *MD = cast<CXXMethodDecl>(FD);
         Diags.Report(Active->PointOfInstantiation,
                      diag::note_member_synthesized_at)
-            << MD->isExplicitlyDefaulted() << DFK.asSpecialMember()
+            << MD->isExplicitlyDefaulted()
+            << llvm::to_underlying(DFK.asSpecialMember())
             << Context.getTagDeclType(MD->getParent());
       } else if (DFK.isComparison()) {
         QualType RecordType = FD->getParamDecl(0)
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index 707446e13209..7f510d34d671 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -5123,10 +5123,10 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
     assert(PatternDecl->isDefaulted() &&
            "Special member needs to be defaulted");
     auto PatternSM = getDefaultedFunctionKind(PatternDecl).asSpecialMember();
-    if (!(PatternSM == Sema::CXXCopyConstructor ||
-          PatternSM == Sema::CXXCopyAssignment ||
-          PatternSM == Sema::CXXMoveConstructor ||
-          PatternSM == Sema::CXXMoveAssignment))
+    if (!(PatternSM == CXXSpecialMemberKind::CopyConstructor ||
+          PatternSM == CXXSpecialMemberKind::CopyAssignment ||
+          PatternSM == CXXSpecialMemberKind::MoveConstructor ||
+          PatternSM == CXXSpecialMemberKind::MoveAssignment))
       return;
 
     auto *NewRec = dyn_cast<CXXRecordDecl>(Function->getDeclContext());
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 8762744396f4..b8a1518fbe00 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -9738,7 +9738,8 @@ bool Sema::RequireLiteralType(SourceLocation Loc, QualType T,
                                     : diag::note_non_literal_nontrivial_dtor)
           << RD;
       if (!Dtor->isUserProvided())
-        SpecialMemberIsTrivial(Dtor, CXXDestructor, TAH_IgnoreTrivialABI,
+        SpecialMemberIsTrivial(Dtor, CXXSpecialMemberKind::Destructor,
+                               TAH_IgnoreTrivialABI,
                                /*Diagnose*/ true);
     }
   }
-- 
GitLab


From 9250aedb5cabf0e44c694f0c8e68ae03e937be89 Mon Sep 17 00:00:00 2001
From: Andreas Jonson <andjo403@hotmail.com>
Date: Fri, 12 Apr 2024 11:21:48 +0200
Subject: [PATCH 042/464] [SCEV] Add range attribute handling (#88449)

---
 llvm/lib/Analysis/ScalarEvolution.cpp         |  9 ++-
 .../ScalarEvolution/range-attribute.ll        | 74 +++++++++++++++++++
 2 files changed, 82 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/Analysis/ScalarEvolution/range-attribute.ll

diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 9fcce797f559..95440dda3b4c 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -6382,9 +6382,16 @@ uint32_t ScalarEvolution::getMinTrailingZeros(const SCEV *S) {
 
 /// Helper method to assign a range to V from metadata present in the IR.
 static std::optional<ConstantRange> GetRangeFromMetadata(Value *V) {
-  if (Instruction *I = dyn_cast<Instruction>(V))
+  if (Instruction *I = dyn_cast<Instruction>(V)) {
     if (MDNode *MD = I->getMetadata(LLVMContext::MD_range))
       return getConstantRangeFromMetadata(*MD);
+    if (const auto *CB = dyn_cast<CallBase>(V))
+      if (std::optional<ConstantRange> Range = CB->getRange())
+        return Range;
+  }
+  if (auto *A = dyn_cast<Argument>(V))
+    if (std::optional<ConstantRange> Range = A->getRange())
+      return Range;
 
   return std::nullopt;
 }
diff --git a/llvm/test/Analysis/ScalarEvolution/range-attribute.ll b/llvm/test/Analysis/ScalarEvolution/range-attribute.ll
new file mode 100644
index 000000000000..ea22695e58d1
--- /dev/null
+++ b/llvm/test/Analysis/ScalarEvolution/range-attribute.ll
@@ -0,0 +1,74 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -disable-output "-passes=print<scalar-evolution>" -scalar-evolution-classify-expressions=0 < %s 2>&1 | FileCheck %s
+
+define i32 @slt_trip_count_with_range_attr(i32 range(i32 1, 100) %limit) {
+;
+; CHECK-LABEL: 'slt_trip_count_with_range_attr'
+; CHECK-NEXT:  Determining loop execution counts for: @slt_trip_count_with_range_attr
+; CHECK-NEXT:  Loop %loop: backedge-taken count is (-1 + %limit)<nsw>
+; CHECK-NEXT:  Loop %loop: constant max backedge-taken count is i32 98
+; CHECK-NEXT:  Loop %loop: symbolic max backedge-taken count is (-1 + %limit)<nsw>
+; CHECK-NEXT:  Loop %loop: Trip multiple is 1
+;
+ entry:
+  br label %loop
+
+ loop:
+  %index = phi i32 [ 0, %entry ], [ %index.inc, %loop ]
+  %index.inc = add i32 %index, 1
+  %continue = icmp slt i32 %index.inc, %limit
+  br i1 %continue, label %loop, label %loop.exit
+
+ loop.exit:
+  ret i32 0
+}
+
+declare i32 @get_i32()
+
+define i32 @slt_trip_count_with_range_call() {
+;
+; CHECK-LABEL: 'slt_trip_count_with_range_call'
+; CHECK-NEXT:  Determining loop execution counts for: @slt_trip_count_with_range_call
+; CHECK-NEXT:  Loop %loop: backedge-taken count is (-1 + %limit)<nsw>
+; CHECK-NEXT:  Loop %loop: constant max backedge-taken count is i32 98
+; CHECK-NEXT:  Loop %loop: symbolic max backedge-taken count is (-1 + %limit)<nsw>
+; CHECK-NEXT:  Loop %loop: Trip multiple is 1
+;
+ entry:
+  %limit = call range(i32 1, 100) i32 @get_i32()
+  br label %loop
+
+ loop:
+  %index = phi i32 [ 0, %entry ], [ %index.inc, %loop ]
+  %index.inc = add i32 %index, 1
+  %continue = icmp slt i32 %index.inc, %limit
+  br i1 %continue, label %loop, label %loop.exit
+
+ loop.exit:
+  ret i32 0
+}
+
+declare range(i32 1, 100) i32 @get_i32_in_range()
+
+define i32 @slt_trip_count_with_range_result() {
+;
+; CHECK-LABEL: 'slt_trip_count_with_range_result'
+; CHECK-NEXT:  Determining loop execution counts for: @slt_trip_count_with_range_result
+; CHECK-NEXT:  Loop %loop: backedge-taken count is (-1 + %limit)<nsw>
+; CHECK-NEXT:  Loop %loop: constant max backedge-taken count is i32 98
+; CHECK-NEXT:  Loop %loop: symbolic max backedge-taken count is (-1 + %limit)<nsw>
+; CHECK-NEXT:  Loop %loop: Trip multiple is 1
+;
+ entry:
+  %limit = call i32 @get_i32_in_range()
+  br label %loop
+
+ loop:
+  %index = phi i32 [ 0, %entry ], [ %index.inc, %loop ]
+  %index.inc = add i32 %index, 1
+  %continue = icmp slt i32 %index.inc, %limit
+  br i1 %continue, label %loop, label %loop.exit
+
+ loop.exit:
+  ret i32 0
+}
-- 
GitLab


From 5d6b00929bd026d1448a5dd3a8341ea99b3befe1 Mon Sep 17 00:00:00 2001
From: AtariDreams <gfunni234@gmail.com>
Date: Fri, 12 Apr 2024 05:24:03 -0400
Subject: [PATCH 043/464] [NFC] Replace m_Sub(m_Zero(), X) with m_Neg(X)
 (#88461)

---
 llvm/lib/Analysis/InstructionSimplify.cpp                | 2 +-
 llvm/lib/Analysis/ValueTracking.cpp                      | 4 ++--
 llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp           | 2 +-
 llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp | 7 +++----
 4 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index 3c943a09a9c2..1aaa4d658803 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -5052,7 +5052,7 @@ static Value *simplifyGEPInst(Type *SrcTy, Value *Ptr,
 
       // gep (gep V, C), (sub 0, V) -> C
       if (match(Indices.back(),
-                m_Sub(m_Zero(), m_PtrToInt(m_Specific(StrippedBasePtr)))) &&
+                m_Neg(m_PtrToInt(m_Specific(StrippedBasePtr)))) &&
           !BasePtrOffset.isZero()) {
         auto *CI = ConstantInt::get(GEPTy->getContext(), BasePtrOffset);
         return ConstantExpr::getIntToPtr(CI, GEPTy);
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index eb8925c53475..ca257cc6aefd 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -8016,12 +8016,12 @@ bool llvm::isKnownNegation(const Value *X, const Value *Y, bool NeedNSW) {
 
   // X = sub (0, Y) || X = sub nsw (0, Y)
   if ((!NeedNSW && match(X, m_Sub(m_ZeroInt(), m_Specific(Y)))) ||
-      (NeedNSW && match(X, m_NSWSub(m_ZeroInt(), m_Specific(Y)))))
+      (NeedNSW && match(X, m_NSWNeg(m_Specific(Y)))))
     return true;
 
   // Y = sub (0, X) || Y = sub nsw (0, X)
   if ((!NeedNSW && match(Y, m_Sub(m_ZeroInt(), m_Specific(X)))) ||
-      (NeedNSW && match(Y, m_NSWSub(m_ZeroInt(), m_Specific(X)))))
+      (NeedNSW && match(Y, m_NSWNeg(m_Specific(X)))))
     return true;
 
   // X = sub (A, B), Y = sub (B, A) || X = sub nsw (A, B), Y = sub nsw (B, A)
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 1dd0fa49a460..e002cf1d1de3 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -4308,7 +4308,7 @@ bool SelectionDAG::isKnownToBeAPowerOfTwo(SDValue Val, unsigned Depth) const {
   //    x & -x -> non-zero pow2
   // so if we find the pattern return whether we know `x` is non-zero.
   SDValue X;
-  if (sd_match(Val, m_And(m_Value(X), m_Sub(m_Zero(), m_Deferred(X)))))
+  if (sd_match(Val, m_And(m_Value(X), m_Neg(m_Deferred(X)))))
     return isKnownNeverZero(X, Depth);
 
   if (Val.getOpcode() == ISD::ZERO_EXTEND)
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
index 8c698e52b5a0..4dc1319f1c43 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
@@ -1572,8 +1572,7 @@ Instruction *InstCombinerImpl::visitSDiv(BinaryOperator &I) {
     // -X / C --> X / -C (if the negation doesn't overflow).
     // TODO: This could be enhanced to handle arbitrary vector constants by
     //       checking if all elements are not the min-signed-val.
-    if (!Op1C->isMinSignedValue() &&
-        match(Op0, m_NSWSub(m_Zero(), m_Value(X)))) {
+    if (!Op1C->isMinSignedValue() && match(Op0, m_NSWNeg(m_Value(X)))) {
       Constant *NegC = ConstantInt::get(Ty, -(*Op1C));
       Instruction *BO = BinaryOperator::CreateSDiv(X, NegC);
       BO->setIsExact(I.isExact());
@@ -1583,7 +1582,7 @@ Instruction *InstCombinerImpl::visitSDiv(BinaryOperator &I) {
 
   // -X / Y --> -(X / Y)
   Value *Y;
-  if (match(&I, m_SDiv(m_OneUse(m_NSWSub(m_Zero(), m_Value(X))), m_Value(Y))))
+  if (match(&I, m_SDiv(m_OneUse(m_NSWNeg(m_Value(X))), m_Value(Y))))
     return BinaryOperator::CreateNSWNeg(
         Builder.CreateSDiv(X, Y, I.getName(), I.isExact()));
 
@@ -2173,7 +2172,7 @@ Instruction *InstCombinerImpl::visitSRem(BinaryOperator &I) {
 
   // -X srem Y --> -(X srem Y)
   Value *X, *Y;
-  if (match(&I, m_SRem(m_OneUse(m_NSWSub(m_Zero(), m_Value(X))), m_Value(Y))))
+  if (match(&I, m_SRem(m_OneUse(m_NSWNeg(m_Value(X))), m_Value(Y))))
     return BinaryOperator::CreateNSWNeg(Builder.CreateSRem(X, Y));
 
   // If the sign bits of both operands are zero (i.e. we can prove they are
-- 
GitLab


From 0e6367236413e50599b49aff15b086088f152769 Mon Sep 17 00:00:00 2001
From: Sergio Afonso <safonsof@amd.com>
Date: Fri, 12 Apr 2024 10:35:48 +0100
Subject: [PATCH 044/464] [MLIR][OpenMP] Add omp.loop_nest operation (#87083)

This patch introduces an operation intended to hold loop information
associated to the `omp.distribute`, `omp.simdloop`, `omp.taskloop` and
`omp.wsloop` operations. This is a stopgap solution to unblock work on
transitioning these operations to becoming wrappers, as discussed in
[this
RFC](https://discourse.llvm.org/t/rfc-representing-combined-composite-constructs-in-the-openmp-dialect/76986).

Long-term, this operation will likely be replaced by
`omp.canonical_loop`, which is being designed to address missing support
for loop transformations, etc.
---
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td | 66 +++++++++++++++-
 mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp  | 71 +++++++++++++++++
 mlir/test/Dialect/OpenMP/invalid.mlir         | 37 +++++++++
 mlir/test/Dialect/OpenMP/ops.mlir             | 79 +++++++++++++++++++
 4 files changed, 252 insertions(+), 1 deletion(-)

diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index 2a8582be3833..a2eb39a8e607 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -524,6 +524,70 @@ def SingleOp : OpenMP_Op<"single", [AttrSizedOperandSegments]> {
   let hasVerifier = 1;
 }
 
+//===----------------------------------------------------------------------===//
+// Loop Nest
+//===----------------------------------------------------------------------===//
+
+def LoopNestOp : OpenMP_Op<"loop_nest", [SameVariadicOperandSize,
+                        AllTypesMatch<["lowerBound", "upperBound", "step"]>,
+                        ParentOneOf<["DistributeOp", "SimdLoopOp", "TaskloopOp",
+                                     "WsloopOp"]>,
+                        RecursiveMemoryEffects]> {
+  let summary = "rectangular loop nest";
+  let description = [{
+    This operation represents a collapsed rectangular loop nest. For each
+    rectangular loop of the nest represented by an instance of this operation,
+    lower and upper bounds, as well as a step variable, must be defined.
+
+    The lower and upper bounds specify a half-open range: the range includes the
+    lower bound but does not include the upper bound. If the `inclusive`
+    attribute is specified then the upper bound is also included.
+
+    The body region can contain any number of blocks. The region is terminated
+    by an `omp.yield` instruction without operands. The induction variables,
+    represented as entry block arguments to the loop nest operation's single
+    region, match the types of the `lowerBound`, `upperBound` and `step`
+    arguments.
+
+    ```mlir
+    omp.loop_nest (%i1, %i2) : i32 = (%c0, %c0) to (%c10, %c10) step (%c1, %c1) {
+      %a = load %arrA[%i1, %i2] : memref<?x?xf32>
+      %b = load %arrB[%i1, %i2] : memref<?x?xf32>
+      %sum = arith.addf %a, %b : f32
+      store %sum, %arrC[%i1, %i2] : memref<?x?xf32>
+      omp.yield
+    }
+    ```
+
+    This is a temporary simplified definition of a loop based on existing OpenMP
+    loop operations intended to serve as a stopgap solution until the long-term
+    representation of canonical loops is defined. Specifically, this operation
+    is intended to serve as a unique source for loop information during the
+    transition to making `omp.distribute`, `omp.simdloop`, `omp.taskloop` and
+    `omp.wsloop` wrapper operations. It is not intended to help with the
+    addition of support for loop transformations, non-rectangular loops and
+    non-perfectly nested loops.
+  }];
+
+  let arguments = (ins Variadic<IntLikeType>:$lowerBound,
+                       Variadic<IntLikeType>:$upperBound,
+                       Variadic<IntLikeType>:$step,
+                       UnitAttr:$inclusive);
+
+  let regions = (region AnyRegion:$region);
+
+  let extraClassDeclaration = [{
+    /// Returns the number of loops in the loop nest.
+    unsigned getNumLoops() { return getLowerBound().size(); }
+
+    /// Returns the induction variables of the loop nest.
+    ArrayRef<BlockArgument> getIVs() { return getRegion().getArguments(); }
+  }];
+
+  let hasCustomAssemblyFormat = 1;
+  let hasVerifier = 1;
+}
+
 //===----------------------------------------------------------------------===//
 // 2.9.2 Workshare Loop Construct
 //===----------------------------------------------------------------------===//
@@ -743,7 +807,7 @@ def SimdLoopOp : OpenMP_Op<"simdloop", [AttrSizedOperandSegments,
 
 def YieldOp : OpenMP_Op<"yield",
     [Pure, ReturnLike, Terminator,
-     ParentOneOf<["WsloopOp", "DeclareReductionOp",
+     ParentOneOf<["LoopNestOp", "WsloopOp", "DeclareReductionOp",
      "AtomicUpdateOp", "SimdLoopOp", "PrivateClauseOp"]>]> {
   let summary = "loop yield and termination operation";
   let description = [{
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index 2d5b2231d2dd..4ab78bf331f8 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -1862,6 +1862,77 @@ LogicalResult TaskloopOp::verify() {
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// LoopNestOp
+//===----------------------------------------------------------------------===//
+
+ParseResult LoopNestOp::parse(OpAsmParser &parser, OperationState &result) {
+  // Parse an opening `(` followed by induction variables followed by `)`
+  SmallVector<OpAsmParser::Argument> ivs;
+  SmallVector<OpAsmParser::UnresolvedOperand> lbs, ubs;
+  Type loopVarType;
+  if (parser.parseArgumentList(ivs, OpAsmParser::Delimiter::Paren) ||
+      parser.parseColonType(loopVarType) ||
+      // Parse loop bounds.
+      parser.parseEqual() ||
+      parser.parseOperandList(lbs, ivs.size(), OpAsmParser::Delimiter::Paren) ||
+      parser.parseKeyword("to") ||
+      parser.parseOperandList(ubs, ivs.size(), OpAsmParser::Delimiter::Paren))
+    return failure();
+
+  for (auto &iv : ivs)
+    iv.type = loopVarType;
+
+  // Parse "inclusive" flag.
+  if (succeeded(parser.parseOptionalKeyword("inclusive")))
+    result.addAttribute("inclusive",
+                        UnitAttr::get(parser.getBuilder().getContext()));
+
+  // Parse step values.
+  SmallVector<OpAsmParser::UnresolvedOperand> steps;
+  if (parser.parseKeyword("step") ||
+      parser.parseOperandList(steps, ivs.size(), OpAsmParser::Delimiter::Paren))
+    return failure();
+
+  // Parse the body.
+  Region *region = result.addRegion();
+  if (parser.parseRegion(*region, ivs))
+    return failure();
+
+  // Resolve operands.
+  if (parser.resolveOperands(lbs, loopVarType, result.operands) ||
+      parser.resolveOperands(ubs, loopVarType, result.operands) ||
+      parser.resolveOperands(steps, loopVarType, result.operands))
+    return failure();
+
+  // Parse the optional attribute list.
+  return parser.parseOptionalAttrDict(result.attributes);
+}
+
+void LoopNestOp::print(OpAsmPrinter &p) {
+  Region &region = getRegion();
+  auto args = region.getArguments();
+  p << " (" << args << ") : " << args[0].getType() << " = (" << getLowerBound()
+    << ") to (" << getUpperBound() << ") ";
+  if (getInclusive())
+    p << "inclusive ";
+  p << "step (" << getStep() << ") ";
+  p.printRegion(region, /*printEntryBlockArgs=*/false);
+}
+
+LogicalResult LoopNestOp::verify() {
+  if (getLowerBound().size() != getIVs().size())
+    return emitOpError() << "number of range arguments and IVs do not match";
+
+  for (auto [lb, iv] : llvm::zip_equal(getLowerBound(), getIVs())) {
+    if (lb.getType() != iv.getType())
+      return emitOpError()
+             << "range argument type does not match corresponding IV type";
+  }
+
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // WsloopOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir
index 27a440b3f97c..3802fbde534d 100644
--- a/mlir/test/Dialect/OpenMP/invalid.mlir
+++ b/mlir/test/Dialect/OpenMP/invalid.mlir
@@ -87,6 +87,43 @@ func.func @proc_bind_once() {
 
 // -----
 
+func.func @invalid_parent(%lb : index, %ub : index, %step : index) {
+  // expected-error@+1 {{op expects parent op to be one of 'omp.distribute, omp.simdloop, omp.taskloop, omp.wsloop'}}
+  omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
+    omp.yield
+  }
+}
+
+// -----
+
+func.func @type_mismatch(%lb : index, %ub : index, %step : index) {
+  // TODO Remove induction variables from omp.wsloop.
+  omp.wsloop for (%iv) : index = (%lb) to (%ub) step (%step) {
+    // expected-error@+1 {{range argument type does not match corresponding IV type}}
+    "omp.loop_nest" (%lb, %ub, %step) ({
+    ^bb0(%iv2: i32):
+      omp.yield
+    }) : (index, index, index) -> ()
+    omp.yield
+  }
+}
+
+// -----
+
+func.func @iv_number_mismatch(%lb : index, %ub : index, %step : index) {
+  // TODO Remove induction variables from omp.wsloop.
+  omp.wsloop for (%iv) : index = (%lb) to (%ub) step (%step) {
+    // expected-error@+1 {{number of range arguments and IVs do not match}}
+    "omp.loop_nest" (%lb, %ub, %step) ({
+    ^bb0(%iv1 : index, %iv2 : index):
+      omp.yield
+    }) : (index, index, index) -> ()
+    omp.yield
+  }
+}
+
+// -----
+
 func.func @inclusive_not_a_clause(%lb : index, %ub : index, %step : index) {
   // expected-error @below {{expected 'for'}}
   omp.wsloop nowait inclusive
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index ad5b74b84ac7..851d44ad984e 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -133,6 +133,85 @@ func.func @omp_parallel_pretty(%data_var : memref<i32>, %if_cond : i1, %num_thre
   return
 }
 
+// CHECK-LABEL: omp_loop_nest
+func.func @omp_loop_nest(%lb : index, %ub : index, %step : index) -> () {
+  // TODO Remove induction variables from omp.wsloop.
+  omp.wsloop for (%iv) : index = (%lb) to (%ub) step (%step) {
+    // CHECK: omp.loop_nest
+    // CHECK-SAME: (%{{.*}}) : index =
+    // CHECK-SAME: (%{{.*}}) to (%{{.*}}) step (%{{.*}})
+    "omp.loop_nest" (%lb, %ub, %step) ({
+    ^bb0(%iv2: index):
+      omp.yield
+    }) : (index, index, index) -> ()
+    omp.yield
+  }
+
+  // TODO Remove induction variables from omp.wsloop.
+  omp.wsloop for (%iv) : index = (%lb) to (%ub) step (%step) {
+    // CHECK: omp.loop_nest
+    // CHECK-SAME: (%{{.*}}) : index =
+    // CHECK-SAME: (%{{.*}}) to (%{{.*}}) inclusive step (%{{.*}})
+    "omp.loop_nest" (%lb, %ub, %step) ({
+    ^bb0(%iv2: index):
+      omp.yield
+    }) {inclusive} : (index, index, index) -> ()
+    omp.yield
+  }
+
+  // TODO Remove induction variables from omp.wsloop.
+  omp.wsloop for (%iv) : index = (%lb) to (%ub) step (%step) {
+    // CHECK: omp.loop_nest
+    // CHECK-SAME: (%{{.*}}, %{{.*}}) : index =
+    // CHECK-SAME: (%{{.*}}, %{{.*}}) to (%{{.*}}, %{{.*}}) step (%{{.*}}, %{{.*}})
+    "omp.loop_nest" (%lb, %lb, %ub, %ub, %step, %step) ({
+    ^bb0(%iv2: index, %iv3: index):
+      omp.yield
+    }) : (index, index, index, index, index, index) -> ()
+    omp.yield
+  }
+
+  return
+}
+
+// CHECK-LABEL: omp_loop_nest_pretty
+func.func @omp_loop_nest_pretty(%lb : index, %ub : index, %step : index) -> () {
+  // TODO Remove induction variables from omp.wsloop.
+  omp.wsloop for (%iv) : index = (%lb) to (%ub) step (%step) {
+    // CHECK: omp.loop_nest
+    // CHECK-SAME: (%{{.*}}) : index =
+    // CHECK-SAME: (%{{.*}}) to (%{{.*}}) step (%{{.*}})
+    omp.loop_nest (%iv2) : index = (%lb) to (%ub) step (%step) {
+      omp.yield
+    }
+    omp.yield
+  }
+
+  // TODO Remove induction variables from omp.wsloop.
+  omp.wsloop for (%iv) : index = (%lb) to (%ub) step (%step) {
+    // CHECK: omp.loop_nest
+    // CHECK-SAME: (%{{.*}}) : index =
+    // CHECK-SAME: (%{{.*}}) to (%{{.*}}) inclusive step (%{{.*}})
+    omp.loop_nest (%iv2) : index = (%lb) to (%ub) inclusive step (%step) {
+      omp.yield
+    }
+    omp.yield
+  }
+
+  // TODO Remove induction variables from omp.wsloop.
+  omp.wsloop for (%iv) : index = (%lb) to (%ub) step (%step) {
+    // CHECK: omp.loop_nest
+    // CHECK-SAME: (%{{.*}}) : index =
+    // CHECK-SAME: (%{{.*}}, %{{.*}}) to (%{{.*}}, %{{.*}}) step (%{{.*}}, %{{.*}})
+    omp.loop_nest (%iv2, %iv3) : index = (%lb, %lb) to (%ub, %ub) step (%step, %step) {
+      omp.yield
+    }
+    omp.yield
+  }
+
+  return
+}
+
 // CHECK-LABEL: omp_wsloop
 func.func @omp_wsloop(%lb : index, %ub : index, %step : index, %data_var : memref<i32>, %linear_var : i32, %chunk_var : i32) -> () {
 
-- 
GitLab


From f5b2d24b59f168eaeed08b5f45bf0c8dfcc1c292 Mon Sep 17 00:00:00 2001
From: Benjamin Kramer <benny.kra@googlemail.com>
Date: Fri, 12 Apr 2024 11:47:53 +0200
Subject: [PATCH 045/464] [NVPTX] Move roundeven on bf16 into the sm_90 has it
 but sm_80 doesn't bucket

---
 llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp  |  1 +
 llvm/test/CodeGen/NVPTX/bf16-instructions.ll | 12 ++++++++++++
 2 files changed, 13 insertions(+)

diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index e1cdbd6a4b4d..44b61a937d64 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -447,6 +447,7 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
       case ISD::FFLOOR:
       case ISD::FNEARBYINT:
       case ISD::FRINT:
+      case ISD::FROUNDEVEN:
       case ISD::FTRUNC:
         IsOpSupported = STI.getSmVersion() >= 90 && STI.getPTXVersion() >= 78;
         break;
diff --git a/llvm/test/CodeGen/NVPTX/bf16-instructions.ll b/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
index 884860712632..4cbab8bf2416 100644
--- a/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
@@ -330,3 +330,15 @@ define bfloat @test_uitofp_i64(i64 %a) {
   %r = uitofp i64 %a to bfloat
   ret bfloat %r
 }
+
+; CHECK-LABEL: test_roundeven(
+; CHECK:      ld.param.b16      [[A:%rs[0-9]+]], [test_roundeven_param_0];
+; SM80:       cvt.rni.f32.f32   [[F:%f[0-9]+]]
+; SM80:       cvt.rn.bf16.f32   [[R:%rs[0-9]+]], [[F]];
+; SM90:       cvt.rni.bf16.bf16 [[R:%rs[0-9]+]], [[A]];
+; CHECK:      st.param.b16      [func_retval0+0], [[R]];
+; CHECK:      ret;
+define bfloat @test_roundeven(bfloat %a) {
+  %r = call bfloat @llvm.roundeven.bf16(bfloat %a)
+  ret bfloat %r
+}
-- 
GitLab


From b24af43fdfa1b1242b7cb77540462212227c57c4 Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Fri, 12 Apr 2024 10:57:01 +0100
Subject: [PATCH 046/464] [AArch64] Improve scheduling latency into Bundles
 (#86310)

By default the scheduling info of instructions into a BUNDLE are given a
latency of 0 as they operate on the implicit register of the bundle.
This modifies that for AArch64 so that the latency is adjusted to use
the latency from the instruction in the bundle instead. This essentially
assumes that the bundled instructions are executed in a single cycle,
which for AArch64 is probably OK considering they are mostly used for
MOVPFX bundles, where this can help create slightly better scheduling
especially for in-order cores.
---
 .../llvm/CodeGen/TargetSubtargetInfo.h        |   4 +-
 llvm/lib/CodeGen/MachinePipeliner.cpp         |   3 +-
 llvm/lib/CodeGen/ScheduleDAGInstrs.cpp        |   8 +-
 .../SelectionDAG/ScheduleDAGSDNodes.cpp       |   2 +-
 llvm/lib/Target/AArch64/AArch64Subtarget.cpp  |  39 ++++++
 llvm/lib/Target/AArch64/AArch64Subtarget.h    |   3 +
 llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp    |   5 +-
 llvm/lib/Target/AMDGPU/GCNSubtarget.h         |   3 +-
 llvm/lib/Target/Hexagon/HexagonSubtarget.cpp  |   6 +-
 llvm/lib/Target/Hexagon/HexagonSubtarget.h    |   3 +-
 ...x-deinterleaving-add-mull-scalable-fast.ll |   6 +-
 .../CodeGen/AArch64/llvm-ir-to-intrinsic.ll   |   2 +-
 llvm/test/CodeGen/AArch64/misched-bundle.mir  |  36 +++---
 .../AArch64/sve-fixed-length-build-vector.ll  |   2 +-
 .../AArch64/sve-fixed-length-fp-arith.ll      |  24 ++--
 .../AArch64/sve-fixed-length-fp-fma.ll        |   6 +-
 .../AArch64/sve-fixed-length-fp-minmax.ll     |  24 ++--
 .../AArch64/sve-fixed-length-int-arith.ll     |   8 +-
 .../AArch64/sve-fixed-length-int-minmax.ll    |  32 ++---
 .../AArch64/sve-fixed-length-int-mulh.ll      |  16 +--
 .../AArch64/sve-fixed-length-int-rem.ll       |  96 +++++++-------
 .../AArch64/sve-fixed-length-int-shifts.ll    |  24 ++--
 llvm/test/CodeGen/AArch64/sve-fpext-load.ll   |   2 +-
 llvm/test/CodeGen/AArch64/sve-fptosi-sat.ll   |  70 +++++-----
 llvm/test/CodeGen/AArch64/sve-fptoui-sat.ll   | 122 +++++++++---------
 .../AArch64/sve-gather-scatter-addr-opts.ll   |  10 +-
 .../AArch64/sve-masked-gather-legalize.ll     |   4 +-
 llvm/test/CodeGen/AArch64/sve-smulo-sdnode.ll |  24 ++--
 llvm/test/CodeGen/AArch64/sve-split-fcvt.ll   |  34 ++---
 ...aming-mode-fixed-length-fp-extend-trunc.ll |   6 +-
 ...sve-streaming-mode-fixed-length-int-div.ll |   8 +-
 ...sve-streaming-mode-fixed-length-int-rem.ll |  32 ++---
 ...e-streaming-mode-fixed-length-int-to-fp.ll |  20 +--
 llvm/test/CodeGen/AArch64/sve-umulo-sdnode.ll |  16 +--
 .../test/CodeGen/AArch64/sve-vecreduce-dot.ll |   2 +-
 llvm/test/CodeGen/AArch64/sve2-xar.ll         |   2 +-
 36 files changed, 377 insertions(+), 327 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h b/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h
index 7f8ed5c50198..5023e29ce145 100644
--- a/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h
@@ -235,7 +235,9 @@ public:
   // and UseOpIdx are the indices of the operands in Def and Use, respectively.
   // Otherwise, either may be -1.
   virtual void adjustSchedDependency(SUnit *Def, int DefOpIdx, SUnit *Use,
-                                     int UseOpIdx, SDep &Dep) const {}
+                                     int UseOpIdx, SDep &Dep,
+                                     const TargetSchedModel *SchedModel) const {
+  }
 
   // For use with PostRAScheduling: get the anti-dependence breaking that should
   // be performed before post-RA scheduling.
diff --git a/llvm/lib/CodeGen/MachinePipeliner.cpp b/llvm/lib/CodeGen/MachinePipeliner.cpp
index 930504067832..4f7d9d070cee 100644
--- a/llvm/lib/CodeGen/MachinePipeliner.cpp
+++ b/llvm/lib/CodeGen/MachinePipeliner.cpp
@@ -919,7 +919,8 @@ void SwingSchedulerDAG::updatePhiDependences() {
           if (!MI->isPHI()) {
             SDep Dep(SU, SDep::Data, Reg);
             Dep.setLatency(0);
-            ST.adjustSchedDependency(SU, 0, &I, MO.getOperandNo(), Dep);
+            ST.adjustSchedDependency(SU, 0, &I, MO.getOperandNo(), Dep,
+                                     &SchedModel);
             I.addPred(Dep);
           } else {
             HasPhiUse = Reg;
diff --git a/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp b/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp
index 51ede7992af5..c848ce4af86c 100644
--- a/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp
+++ b/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp
@@ -282,7 +282,7 @@ void ScheduleDAGInstrs::addPhysRegDataDeps(SUnit *SU, unsigned OperIdx) {
       } else {
         Dep.setLatency(0);
       }
-      ST.adjustSchedDependency(SU, OperIdx, UseSU, UseOpIdx, Dep);
+      ST.adjustSchedDependency(SU, OperIdx, UseSU, UseOpIdx, Dep, &SchedModel);
       UseSU->addPred(Dep);
     }
   }
@@ -323,7 +323,8 @@ void ScheduleDAGInstrs::addPhysRegDeps(SUnit *SU, unsigned OperIdx) {
           Dep.setLatency(
               SchedModel.computeOutputLatency(MI, OperIdx, DefInstr));
         }
-        ST.adjustSchedDependency(SU, OperIdx, DefSU, I->OpIdx, Dep);
+        ST.adjustSchedDependency(SU, OperIdx, DefSU, I->OpIdx, Dep,
+                                 &SchedModel);
         DefSU->addPred(Dep);
       }
     }
@@ -453,7 +454,8 @@ void ScheduleDAGInstrs::addVRegDefDeps(SUnit *SU, unsigned OperIdx) {
         SDep Dep(SU, SDep::Data, Reg);
         Dep.setLatency(SchedModel.computeOperandLatency(MI, OperIdx, Use,
                                                         I->OperandIndex));
-        ST.adjustSchedDependency(SU, OperIdx, UseSU, I->OperandIndex, Dep);
+        ST.adjustSchedDependency(SU, OperIdx, UseSU, I->OperandIndex, Dep,
+                                 &SchedModel);
         UseSU->addPred(Dep);
       }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
index 862fce6959f1..379456e3a9b6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
@@ -512,7 +512,7 @@ void ScheduleDAGSDNodes::AddSchedEdges() {
         Dep.setLatency(OpLatency);
         if (!isChain && !UnitLatencies) {
           computeOperandLatency(OpN, N, i, Dep);
-          ST.adjustSchedDependency(OpSU, DefIdx, &SU, i, Dep);
+          ST.adjustSchedDependency(OpSU, DefIdx, &SU, i, Dep, nullptr);
         }
 
         if (!SU.addPred(Dep) && !Dep.isCtrl() && OpSU->NumRegDefsLeft > 1) {
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
index bb268b2ba926..9eb1ecbb0315 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
@@ -472,6 +472,45 @@ void AArch64Subtarget::overrideSchedPolicy(MachineSchedPolicy &Policy,
   Policy.DisableLatencyHeuristic = DisableLatencySchedHeuristic;
 }
 
+void AArch64Subtarget::adjustSchedDependency(
+    SUnit *Def, int DefOpIdx, SUnit *Use, int UseOpIdx, SDep &Dep,
+    const TargetSchedModel *SchedModel) const {
+  if (!SchedModel || Dep.getKind() != SDep::Kind::Data || !Dep.getReg() ||
+      !Def->isInstr() || !Use->isInstr() ||
+      (Def->getInstr()->getOpcode() != TargetOpcode::BUNDLE &&
+       Use->getInstr()->getOpcode() != TargetOpcode::BUNDLE))
+    return;
+
+  // If the Def is a BUNDLE, find the last instruction in the bundle that defs
+  // the register.
+  const MachineInstr *DefMI = Def->getInstr();
+  if (DefMI->getOpcode() == TargetOpcode::BUNDLE) {
+    Register Reg = DefMI->getOperand(DefOpIdx).getReg();
+    for (const auto &Op : const_mi_bundle_ops(*DefMI)) {
+      if (Op.isReg() && Op.isDef() && Op.getReg() == Reg) {
+        DefMI = Op.getParent();
+        DefOpIdx = Op.getOperandNo();
+      }
+    }
+  }
+
+  // If the Use is a BUNDLE, find the first instruction that uses the Reg.
+  const MachineInstr *UseMI = Use->getInstr();
+  if (UseMI->getOpcode() == TargetOpcode::BUNDLE) {
+    Register Reg = UseMI->getOperand(UseOpIdx).getReg();
+    for (const auto &Op : const_mi_bundle_ops(*UseMI)) {
+      if (Op.isReg() && Op.isUse() && Op.getReg() == Reg) {
+        UseMI = Op.getParent();
+        UseOpIdx = Op.getOperandNo();
+        break;
+      }
+    }
+  }
+
+  Dep.setLatency(
+      SchedModel->computeOperandLatency(DefMI, DefOpIdx, UseMI, UseOpIdx));
+}
+
 bool AArch64Subtarget::enableEarlyIfConversion() const {
   return EnableEarlyIfConvert;
 }
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.h b/llvm/lib/Target/AArch64/AArch64Subtarget.h
index 95bef7a76bca..c5ebd4c6cc61 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.h
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.h
@@ -354,6 +354,9 @@ public:
 
   void overrideSchedPolicy(MachineSchedPolicy &Policy,
                            unsigned NumRegionInstrs) const override;
+  void adjustSchedDependency(SUnit *Def, int DefOpIdx, SUnit *Use, int UseOpIdx,
+                             SDep &Dep,
+                             const TargetSchedModel *SchedModel) const override;
 
   bool enableEarlyIfConversion() const override;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
index 8f0eae362eca..36e453f04426 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
@@ -860,8 +860,9 @@ unsigned GCNSubtarget::getMaxNumVGPRs(const MachineFunction &MF) const {
   return getBaseMaxNumVGPRs(F, MFI.getWavesPerEU());
 }
 
-void GCNSubtarget::adjustSchedDependency(SUnit *Def, int DefOpIdx, SUnit *Use,
-                                         int UseOpIdx, SDep &Dep) const {
+void GCNSubtarget::adjustSchedDependency(
+    SUnit *Def, int DefOpIdx, SUnit *Use, int UseOpIdx, SDep &Dep,
+    const TargetSchedModel *SchedModel) const {
   if (Dep.getKind() != SDep::Kind::Data || !Dep.getReg() ||
       !Def->isInstr() || !Use->isInstr())
     return;
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index 04ff53a6647b..8a4a46ce50d1 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -1500,7 +1500,8 @@ public:
   }
 
   void adjustSchedDependency(SUnit *Def, int DefOpIdx, SUnit *Use, int UseOpIdx,
-                             SDep &Dep) const override;
+                             SDep &Dep,
+                             const TargetSchedModel *SchedModel) const override;
 
   // \returns true if it's beneficial on this subtarget for the scheduler to
   // cluster stores as well as loads.
diff --git a/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp b/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp
index 2d320e6b0cad..da8ab5c4b21b 100644
--- a/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonSubtarget.cpp
@@ -437,9 +437,9 @@ bool HexagonSubtarget::useAA() const {
 
 /// Perform target specific adjustments to the latency of a schedule
 /// dependency.
-void HexagonSubtarget::adjustSchedDependency(SUnit *Src, int SrcOpIdx,
-                                             SUnit *Dst, int DstOpIdx,
-                                             SDep &Dep) const {
+void HexagonSubtarget::adjustSchedDependency(
+    SUnit *Src, int SrcOpIdx, SUnit *Dst, int DstOpIdx, SDep &Dep,
+    const TargetSchedModel *SchedModel) const {
   if (!Src->isInstr() || !Dst->isInstr())
     return;
 
diff --git a/llvm/lib/Target/Hexagon/HexagonSubtarget.h b/llvm/lib/Target/Hexagon/HexagonSubtarget.h
index e56007ee21e7..da3f8f56ccc4 100644
--- a/llvm/lib/Target/Hexagon/HexagonSubtarget.h
+++ b/llvm/lib/Target/Hexagon/HexagonSubtarget.h
@@ -308,7 +308,8 @@ public:
   /// Perform target specific adjustments to the latency of a schedule
   /// dependency.
   void adjustSchedDependency(SUnit *Def, int DefOpIdx, SUnit *Use, int UseOpIdx,
-                             SDep &Dep) const override;
+                             SDep &Dep,
+                             const TargetSchedModel *SchedModel) const override;
 
   unsigned getVectorLength() const {
     assert(useHVXOps());
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-add-mull-scalable-fast.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-add-mull-scalable-fast.ll
index aac43f7b960f..1751f99b3e92 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-add-mull-scalable-fast.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-add-mull-scalable-fast.ll
@@ -190,8 +190,8 @@ define <vscale x 4 x double> @mul_add_rot_mull(<vscale x 4 x double> %a, <vscale
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    uzp1 z24.d, z2.d, z3.d
 ; CHECK-NEXT:    uzp2 z25.d, z0.d, z1.d
-; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    uzp2 z2.d, z2.d, z3.d
+; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    uzp1 z0.d, z0.d, z1.d
 ; CHECK-NEXT:    uzp1 z26.d, z6.d, z7.d
 ; CHECK-NEXT:    fmul z1.d, z24.d, z25.d
@@ -199,12 +199,12 @@ define <vscale x 4 x double> @mul_add_rot_mull(<vscale x 4 x double> %a, <vscale
 ; CHECK-NEXT:    uzp2 z25.d, z4.d, z5.d
 ; CHECK-NEXT:    uzp1 z4.d, z4.d, z5.d
 ; CHECK-NEXT:    uzp2 z5.d, z6.d, z7.d
-; CHECK-NEXT:    fmla z1.d, p0/m, z2.d, z0.d
 ; CHECK-NEXT:    fmla z3.d, p0/m, z26.d, z25.d
+; CHECK-NEXT:    fmla z1.d, p0/m, z2.d, z0.d
 ; CHECK-NEXT:    movprfx z2, z3
 ; CHECK-NEXT:    fmla z2.d, p0/m, z5.d, z4.d
-; CHECK-NEXT:    fnmls z2.d, p0/m, z24.d, z0.d
 ; CHECK-NEXT:    fmla z1.d, p0/m, z26.d, z4.d
+; CHECK-NEXT:    fnmls z2.d, p0/m, z24.d, z0.d
 ; CHECK-NEXT:    fmls z1.d, p0/m, z5.d, z25.d
 ; CHECK-NEXT:    zip1 z0.d, z2.d, z1.d
 ; CHECK-NEXT:    zip2 z1.d, z2.d, z1.d
diff --git a/llvm/test/CodeGen/AArch64/llvm-ir-to-intrinsic.ll b/llvm/test/CodeGen/AArch64/llvm-ir-to-intrinsic.ll
index f8cba4ce4b63..ab15bf564ec4 100644
--- a/llvm/test/CodeGen/AArch64/llvm-ir-to-intrinsic.ll
+++ b/llvm/test/CodeGen/AArch64/llvm-ir-to-intrinsic.ll
@@ -1145,8 +1145,8 @@ define <vscale x 4 x i64> @fshl_rot_illegal_i64(<vscale x 4 x i64> %a, <vscale x
 ; CHECK-NEXT:    subr z3.d, z3.d, #0 // =0x0
 ; CHECK-NEXT:    and z4.d, z4.d, #0x3f
 ; CHECK-NEXT:    and z2.d, z2.d, #0x3f
-; CHECK-NEXT:    and z3.d, z3.d, #0x3f
 ; CHECK-NEXT:    and z5.d, z5.d, #0x3f
+; CHECK-NEXT:    and z3.d, z3.d, #0x3f
 ; CHECK-NEXT:    lslr z4.d, p0/m, z4.d, z0.d
 ; CHECK-NEXT:    lsr z0.d, p0/m, z0.d, z2.d
 ; CHECK-NEXT:    movprfx z2, z1
diff --git a/llvm/test/CodeGen/AArch64/misched-bundle.mir b/llvm/test/CodeGen/AArch64/misched-bundle.mir
index a947c04a4229..9adcd2904a25 100644
--- a/llvm/test/CodeGen/AArch64/misched-bundle.mir
+++ b/llvm/test/CodeGen/AArch64/misched-bundle.mir
@@ -46,9 +46,9 @@
 # CHECK-NEXT:   # rdefs left       : 0
 # CHECK-NEXT:   Latency            : 3
 # CHECK-NEXT:   Depth              : 0
-# CHECK-NEXT:   Height             : 0
+# CHECK-NEXT:   Height             : 7
 # CHECK-NEXT:   Successors:
-# CHECK-NEXT:     SU(7): Data Latency=0 Reg=$z3
+# CHECK-NEXT:     SU(7): Data Latency=3 Reg=$z3
 # CHECK-NEXT:     SU(9): Ord  Latency=0 Memory
 # CHECK-NEXT:     SU(8): Ord  Latency=0 Memory
 # CHECK-NEXT:   Single Issue       : false;
@@ -58,9 +58,9 @@
 # CHECK-NEXT:   # rdefs left       : 0
 # CHECK-NEXT:   Latency            : 3
 # CHECK-NEXT:   Depth              : 0
-# CHECK-NEXT:   Height             : 0
+# CHECK-NEXT:   Height             : 7
 # CHECK-NEXT:   Successors:
-# CHECK-NEXT:     SU(7): Data Latency=0 Reg=$z4
+# CHECK-NEXT:     SU(7): Data Latency=3 Reg=$z4
 # CHECK-NEXT:     SU(9): Ord  Latency=0 Memory
 # CHECK-NEXT:     SU(8): Ord  Latency=0 Memory
 # CHECK-NEXT:   Single Issue       : false;
@@ -70,9 +70,9 @@
 # CHECK-NEXT:   # rdefs left       : 0
 # CHECK-NEXT:   Latency            : 3
 # CHECK-NEXT:   Depth              : 0
-# CHECK-NEXT:   Height             : 0
+# CHECK-NEXT:   Height             : 7
 # CHECK-NEXT:   Successors:
-# CHECK-NEXT:     SU(7): Data Latency=0 Reg=$z5
+# CHECK-NEXT:     SU(7): Data Latency=3 Reg=$z5
 # CHECK-NEXT:     SU(9): Ord  Latency=0 Memory
 # CHECK-NEXT:     SU(8): Ord  Latency=0 Memory
 # CHECK-NEXT:   Single Issue       : false;
@@ -98,15 +98,15 @@
 # CHECK-NEXT:   # rdefs left       : 0
 # CHECK-NEXT:   Latency            : 1
 # CHECK-NEXT:   Depth              : 3
-# CHECK-NEXT:   Height             : 0
+# CHECK-NEXT:   Height             : 4
 # CHECK-NEXT:   Predecessors:
 # CHECK-NEXT:     SU(6): Anti Latency=0
-# CHECK-NEXT:     SU(5): Data Latency=0 Reg=$z5
-# CHECK-NEXT:     SU(4): Data Latency=0 Reg=$z4
-# CHECK-NEXT:     SU(3): Data Latency=0 Reg=$z3
+# CHECK-NEXT:     SU(5): Data Latency=3 Reg=$z5
+# CHECK-NEXT:     SU(4): Data Latency=3 Reg=$z4
+# CHECK-NEXT:     SU(3): Data Latency=3 Reg=$z3
 # CHECK-NEXT:     SU(1): Out  Latency=1
 # CHECK-NEXT:   Successors:
-# CHECK-NEXT:     SU(9): Data Latency=0 Reg=$z1
+# CHECK-NEXT:     SU(9): Data Latency=4 Reg=$z1
 # CHECK-NEXT:   Single Issue       : false;
 # CHECK-NEXT: SU(8):   ST1H killed renamable $z0, renamable $p0, renamable $x0, renamable $x10 :: (store unknown-size, align 1)
 # CHECK-NEXT:   # preds left       : 7
@@ -135,7 +135,7 @@
 # CHECK-NEXT:   Height             : 0
 # CHECK-NEXT:   Predecessors:
 # CHECK-NEXT:     SU(8): Ord  Latency=0 Memory
-# CHECK-NEXT:     SU(7): Data Latency=0 Reg=$z1
+# CHECK-NEXT:     SU(7): Data Latency=4 Reg=$z1
 # CHECK-NEXT:     SU(5): Ord  Latency=0 Memory
 # CHECK-NEXT:     SU(4): Ord  Latency=0 Memory
 # CHECK-NEXT:     SU(3): Ord  Latency=0 Memory
@@ -159,24 +159,24 @@ body:             |
   bb.0.entry:
     liveins: $p0, $x0, $x1, $x2, $x10, $x11, $x12, $x13
 
+
     ; CHECK-LABEL: name: test
     ; CHECK: liveins: $p0, $x0, $x1, $x2, $x10, $x11, $x12, $x13
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: renamable $z0 = LD1H renamable $p0, renamable $x1, renamable $x10 :: (load unknown-size, align 1)
     ; CHECK-NEXT: renamable $z1 = LD1H renamable $p0, renamable $x2, renamable $x10 :: (load unknown-size, align 1)
     ; CHECK-NEXT: renamable $z2 = LD1H renamable $p0, renamable $x0, renamable $x10 :: (load unknown-size, align 1)
-    ; CHECK-NEXT: $z0 = FMAD_ZPmZZ_H renamable $p0, killed $z0, renamable $z1, killed renamable $z2
     ; CHECK-NEXT: renamable $z3 = LD1H renamable $p0, renamable $x11, renamable $x10 :: (load unknown-size, align 1)
     ; CHECK-NEXT: renamable $z4 = LD1H renamable $p0, renamable $x12, renamable $x10 :: (load unknown-size, align 1)
     ; CHECK-NEXT: renamable $z5 = LD1H renamable $p0, renamable $x13, renamable $x10 :: (load unknown-size, align 1)
-    ; CHECK-NEXT: ST1H killed renamable $z0, renamable $p0, renamable $x0, renamable $x10 :: (store unknown-size, align 1)
-    ; CHECK-NEXT: BUNDLE implicit-def $z1, implicit-def $q1, implicit-def $d1, implicit-def $s1, implicit-def $h1, implicit-def $b1, implicit $z5, implicit $p0, implicit $z4, implicit $z3 {
+    ; CHECK-NEXT: $z0 = FMAD_ZPmZZ_H renamable $p0, killed $z0, killed renamable $z1, killed renamable $z2
+    ; CHECK-NEXT: BUNDLE implicit-def $z1, implicit-def $q1, implicit-def $d1, implicit-def $s1, implicit-def $h1, implicit-def $b1, implicit $z5, implicit $p0, implicit killed $z4, implicit killed $z3 {
     ; CHECK-NEXT:   $z1 = MOVPRFX_ZZ $z5
-    ; CHECK-NEXT:   $z1 = FMLA_ZPmZZ_H renamable $p0, internal $z1, renamable $z4, renamable $z3
+    ; CHECK-NEXT:   $z1 = FMLA_ZPmZZ_H renamable $p0, internal killed $z1, killed renamable $z4, killed renamable $z3
     ; CHECK-NEXT: }
-    ; CHECK-NEXT: ST1H renamable $z1, renamable $p0, renamable $x13, renamable $x10 :: (store unknown-size, align 1)
+    ; CHECK-NEXT: ST1H killed renamable $z0, renamable $p0, renamable $x0, renamable $x10 :: (store unknown-size, align 1)
+    ; CHECK-NEXT: ST1H killed renamable $z1, renamable $p0, renamable $x13, renamable $x10 :: (store unknown-size, align 1)
     ; CHECK-NEXT: RET_ReallyLR
-
     renamable $z0 = LD1H renamable $p0, renamable $x1, renamable $x10 :: (load unknown-size)
     renamable $z1 = LD1H renamable $p0, renamable $x2, renamable $x10 :: (load unknown-size)
     renamable $z2 = LD1H renamable $p0, renamable $x0, renamable $x10 :: (load unknown-size)
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-build-vector.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-build-vector.ll
index 0e3307d16172..ad482118ec0b 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-build-vector.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-build-vector.ll
@@ -43,7 +43,7 @@ define void @build_vector_minus2_dec32_v4i64(ptr %a) #0 {
 ; VBITS_GE_256-LABEL: build_vector_minus2_dec32_v4i64:
 ; VBITS_GE_256:       // %bb.0:
 ; VBITS_GE_256-NEXT:    ptrue p0.d, vl4
-; VBITS_GE_256-NEXT:    mov x8, #-32
+; VBITS_GE_256-NEXT:    mov x8, #-32 // =0xffffffffffffffe0
 ; VBITS_GE_256-NEXT:    index z0.d, #-2, x8
 ; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-arith.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-arith.ll
index 64c4eea0ee5f..6fbae7edfec0 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-arith.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-arith.ll
@@ -57,8 +57,8 @@ define void @fadd_v32f16(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    fadd z0.h, p0/m, z0.h, z1.h
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    fadd z1.h, p0/m, z1.h, z3.h
-; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x0, x8, lsl #1]
+; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fadd_v32f16:
@@ -156,8 +156,8 @@ define void @fadd_v16f32(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    fadd z0.s, p0/m, z0.s, z1.s
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    fadd z1.s, p0/m, z1.s, z3.s
-; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fadd_v16f32:
@@ -255,8 +255,8 @@ define void @fadd_v8f64(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    fadd z0.d, p0/m, z0.d, z1.d
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    fadd z1.d, p0/m, z1.d, z3.d
-; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fadd_v8f64:
@@ -662,8 +662,8 @@ define void @fma_v32f16(ptr %a, ptr %b, ptr %c) #0 {
 ; VBITS_GE_256-NEXT:    fmad z0.h, p0/m, z1.h, z2.h
 ; VBITS_GE_256-NEXT:    movprfx z1, z5
 ; VBITS_GE_256-NEXT:    fmla z1.h, p0/m, z3.h, z4.h
-; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x0, x8, lsl #1]
+; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fma_v32f16:
@@ -773,8 +773,8 @@ define void @fma_v16f32(ptr %a, ptr %b, ptr %c) #0 {
 ; VBITS_GE_256-NEXT:    fmad z0.s, p0/m, z1.s, z2.s
 ; VBITS_GE_256-NEXT:    movprfx z1, z5
 ; VBITS_GE_256-NEXT:    fmla z1.s, p0/m, z3.s, z4.s
-; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fma_v16f32:
@@ -883,8 +883,8 @@ define void @fma_v8f64(ptr %a, ptr %b, ptr %c) #0 {
 ; VBITS_GE_256-NEXT:    fmad z0.d, p0/m, z1.d, z2.d
 ; VBITS_GE_256-NEXT:    movprfx z1, z5
 ; VBITS_GE_256-NEXT:    fmla z1.d, p0/m, z3.d, z4.d
-; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fma_v8f64:
@@ -992,8 +992,8 @@ define void @fmul_v32f16(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    fmul z0.h, p0/m, z0.h, z1.h
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    fmul z1.h, p0/m, z1.h, z3.h
-; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x0, x8, lsl #1]
+; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fmul_v32f16:
@@ -1091,8 +1091,8 @@ define void @fmul_v16f32(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    fmul z0.s, p0/m, z0.s, z1.s
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    fmul z1.s, p0/m, z1.s, z3.s
-; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fmul_v16f32:
@@ -1190,8 +1190,8 @@ define void @fmul_v8f64(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    fmul z0.d, p0/m, z0.d, z1.d
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    fmul z1.d, p0/m, z1.d, z3.d
-; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fmul_v8f64:
@@ -1829,8 +1829,8 @@ define void @fsub_v32f16(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    fsub z0.h, p0/m, z0.h, z1.h
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    fsub z1.h, p0/m, z1.h, z3.h
-; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x0, x8, lsl #1]
+; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fsub_v32f16:
@@ -1928,8 +1928,8 @@ define void @fsub_v16f32(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    fsub z0.s, p0/m, z0.s, z1.s
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    fsub z1.s, p0/m, z1.s, z3.s
-; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fsub_v16f32:
@@ -2027,8 +2027,8 @@ define void @fsub_v8f64(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    fsub z0.d, p0/m, z0.d, z1.d
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    fsub z1.d, p0/m, z1.d, z3.d
-; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fsub_v8f64:
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-fma.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-fma.ll
index d3713b989301..e1ec5ee5f613 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-fma.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-fma.ll
@@ -66,8 +66,8 @@ define void @fma_v32f16(ptr %a, ptr %b, ptr %c) #0 {
 ; VBITS_GE_256-NEXT:    fmad z0.h, p0/m, z1.h, z2.h
 ; VBITS_GE_256-NEXT:    movprfx z1, z5
 ; VBITS_GE_256-NEXT:    fmla z1.h, p0/m, z3.h, z4.h
-; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x0, x8, lsl #1]
+; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fma_v32f16:
@@ -183,8 +183,8 @@ define void @fma_v16f32(ptr %a, ptr %b, ptr %c) #0 {
 ; VBITS_GE_256-NEXT:    fmad z0.s, p0/m, z1.s, z2.s
 ; VBITS_GE_256-NEXT:    movprfx z1, z5
 ; VBITS_GE_256-NEXT:    fmla z1.s, p0/m, z3.s, z4.s
-; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fma_v16f32:
@@ -299,8 +299,8 @@ define void @fma_v8f64(ptr %a, ptr %b, ptr %c) #0 {
 ; VBITS_GE_256-NEXT:    fmad z0.d, p0/m, z1.d, z2.d
 ; VBITS_GE_256-NEXT:    movprfx z1, z5
 ; VBITS_GE_256-NEXT:    fmla z1.d, p0/m, z3.d, z4.d
-; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fma_v8f64:
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-minmax.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-minmax.ll
index d20f083066a8..de60deeafaf3 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-minmax.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-minmax.ll
@@ -57,8 +57,8 @@ define void @fmaxnm_v32f16(ptr %a, ptr %b) #0 {
 ; VBITS_EQ_256-NEXT:    fmaxnm z0.h, p0/m, z0.h, z1.h
 ; VBITS_EQ_256-NEXT:    movprfx z1, z2
 ; VBITS_EQ_256-NEXT:    fmaxnm z1.h, p0/m, z1.h, z3.h
-; VBITS_EQ_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_EQ_256-NEXT:    st1h { z0.h }, p0, [x0, x8, lsl #1]
+; VBITS_EQ_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_EQ_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fmaxnm_v32f16:
@@ -156,8 +156,8 @@ define void @fmaxnm_v16f32(ptr %a, ptr %b) #0 {
 ; VBITS_EQ_256-NEXT:    fmaxnm z0.s, p0/m, z0.s, z1.s
 ; VBITS_EQ_256-NEXT:    movprfx z1, z2
 ; VBITS_EQ_256-NEXT:    fmaxnm z1.s, p0/m, z1.s, z3.s
-; VBITS_EQ_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_EQ_256-NEXT:    st1w { z0.s }, p0, [x0, x8, lsl #2]
+; VBITS_EQ_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_EQ_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fmaxnm_v16f32:
@@ -255,8 +255,8 @@ define void @fmaxnm_v8f64(ptr %a, ptr %b) #0 {
 ; VBITS_EQ_256-NEXT:    fmaxnm z0.d, p0/m, z0.d, z1.d
 ; VBITS_EQ_256-NEXT:    movprfx z1, z2
 ; VBITS_EQ_256-NEXT:    fmaxnm z1.d, p0/m, z1.d, z3.d
-; VBITS_EQ_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_EQ_256-NEXT:    st1d { z0.d }, p0, [x0, x8, lsl #3]
+; VBITS_EQ_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_EQ_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fmaxnm_v8f64:
@@ -358,8 +358,8 @@ define void @fminnm_v32f16(ptr %a, ptr %b) #0 {
 ; VBITS_EQ_256-NEXT:    fminnm z0.h, p0/m, z0.h, z1.h
 ; VBITS_EQ_256-NEXT:    movprfx z1, z2
 ; VBITS_EQ_256-NEXT:    fminnm z1.h, p0/m, z1.h, z3.h
-; VBITS_EQ_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_EQ_256-NEXT:    st1h { z0.h }, p0, [x0, x8, lsl #1]
+; VBITS_EQ_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_EQ_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fminnm_v32f16:
@@ -457,8 +457,8 @@ define void @fminnm_v16f32(ptr %a, ptr %b) #0 {
 ; VBITS_EQ_256-NEXT:    fminnm z0.s, p0/m, z0.s, z1.s
 ; VBITS_EQ_256-NEXT:    movprfx z1, z2
 ; VBITS_EQ_256-NEXT:    fminnm z1.s, p0/m, z1.s, z3.s
-; VBITS_EQ_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_EQ_256-NEXT:    st1w { z0.s }, p0, [x0, x8, lsl #2]
+; VBITS_EQ_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_EQ_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fminnm_v16f32:
@@ -556,8 +556,8 @@ define void @fminnm_v8f64(ptr %a, ptr %b) #0 {
 ; VBITS_EQ_256-NEXT:    fminnm z0.d, p0/m, z0.d, z1.d
 ; VBITS_EQ_256-NEXT:    movprfx z1, z2
 ; VBITS_EQ_256-NEXT:    fminnm z1.d, p0/m, z1.d, z3.d
-; VBITS_EQ_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_EQ_256-NEXT:    st1d { z0.d }, p0, [x0, x8, lsl #3]
+; VBITS_EQ_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_EQ_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fminnm_v8f64:
@@ -659,8 +659,8 @@ define void @fmax_v32f16(ptr %a, ptr %b) #0 {
 ; VBITS_EQ_256-NEXT:    fmax z0.h, p0/m, z0.h, z1.h
 ; VBITS_EQ_256-NEXT:    movprfx z1, z2
 ; VBITS_EQ_256-NEXT:    fmax z1.h, p0/m, z1.h, z3.h
-; VBITS_EQ_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_EQ_256-NEXT:    st1h { z0.h }, p0, [x0, x8, lsl #1]
+; VBITS_EQ_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_EQ_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fmax_v32f16:
@@ -758,8 +758,8 @@ define void @fmax_v16f32(ptr %a, ptr %b) #0 {
 ; VBITS_EQ_256-NEXT:    fmax z0.s, p0/m, z0.s, z1.s
 ; VBITS_EQ_256-NEXT:    movprfx z1, z2
 ; VBITS_EQ_256-NEXT:    fmax z1.s, p0/m, z1.s, z3.s
-; VBITS_EQ_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_EQ_256-NEXT:    st1w { z0.s }, p0, [x0, x8, lsl #2]
+; VBITS_EQ_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_EQ_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fmax_v16f32:
@@ -857,8 +857,8 @@ define void @fmax_v8f64(ptr %a, ptr %b) #0 {
 ; VBITS_EQ_256-NEXT:    fmax z0.d, p0/m, z0.d, z1.d
 ; VBITS_EQ_256-NEXT:    movprfx z1, z2
 ; VBITS_EQ_256-NEXT:    fmax z1.d, p0/m, z1.d, z3.d
-; VBITS_EQ_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_EQ_256-NEXT:    st1d { z0.d }, p0, [x0, x8, lsl #3]
+; VBITS_EQ_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_EQ_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fmax_v8f64:
@@ -960,8 +960,8 @@ define void @fmin_v32f16(ptr %a, ptr %b) #0 {
 ; VBITS_EQ_256-NEXT:    fmin z0.h, p0/m, z0.h, z1.h
 ; VBITS_EQ_256-NEXT:    movprfx z1, z2
 ; VBITS_EQ_256-NEXT:    fmin z1.h, p0/m, z1.h, z3.h
-; VBITS_EQ_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_EQ_256-NEXT:    st1h { z0.h }, p0, [x0, x8, lsl #1]
+; VBITS_EQ_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_EQ_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fmin_v32f16:
@@ -1059,8 +1059,8 @@ define void @fmin_v16f32(ptr %a, ptr %b) #0 {
 ; VBITS_EQ_256-NEXT:    fmin z0.s, p0/m, z0.s, z1.s
 ; VBITS_EQ_256-NEXT:    movprfx z1, z2
 ; VBITS_EQ_256-NEXT:    fmin z1.s, p0/m, z1.s, z3.s
-; VBITS_EQ_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_EQ_256-NEXT:    st1w { z0.s }, p0, [x0, x8, lsl #2]
+; VBITS_EQ_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_EQ_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fmin_v16f32:
@@ -1158,8 +1158,8 @@ define void @fmin_v8f64(ptr %a, ptr %b) #0 {
 ; VBITS_EQ_256-NEXT:    fmin z0.d, p0/m, z0.d, z1.d
 ; VBITS_EQ_256-NEXT:    movprfx z1, z2
 ; VBITS_EQ_256-NEXT:    fmin z1.d, p0/m, z1.d, z3.d
-; VBITS_EQ_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_EQ_256-NEXT:    st1d { z0.d }, p0, [x0, x8, lsl #3]
+; VBITS_EQ_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_EQ_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: fmin_v8f64:
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-arith.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-arith.ll
index 8f2aeade48a0..b45a62f4e758 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-arith.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-arith.ll
@@ -458,8 +458,8 @@ define void @mul_v64i8(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    mul z0.b, p0/m, z0.b, z1.b
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    mul z1.b, p0/m, z1.b, z3.b
-; VBITS_GE_256-NEXT:    st1b { z1.b }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1b { z0.b }, p0, [x0, x8]
+; VBITS_GE_256-NEXT:    st1b { z1.b }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: mul_v64i8:
@@ -557,8 +557,8 @@ define void @mul_v32i16(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    mul z0.h, p0/m, z0.h, z1.h
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    mul z1.h, p0/m, z1.h, z3.h
-; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x0, x8, lsl #1]
+; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: mul_v32i16:
@@ -656,8 +656,8 @@ define void @mul_v16i32(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    mul z0.s, p0/m, z0.s, z1.s
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    mul z1.s, p0/m, z1.s, z3.s
-; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: mul_v16i32:
@@ -761,8 +761,8 @@ define void @mul_v8i64(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    mul z0.d, p0/m, z0.d, z1.d
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    mul z1.d, p0/m, z1.d, z3.d
-; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: mul_v8i64:
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-minmax.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-minmax.ll
index 296e4818b1cd..4926684ddc2d 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-minmax.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-minmax.ll
@@ -57,8 +57,8 @@ define void @smax_v64i8(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    smax z0.b, p0/m, z0.b, z1.b
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    smax z1.b, p0/m, z1.b, z3.b
-; VBITS_GE_256-NEXT:    st1b { z1.b }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1b { z0.b }, p0, [x0, x8]
+; VBITS_GE_256-NEXT:    st1b { z1.b }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: smax_v64i8:
@@ -156,8 +156,8 @@ define void @smax_v32i16(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    smax z0.h, p0/m, z0.h, z1.h
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    smax z1.h, p0/m, z1.h, z3.h
-; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x0, x8, lsl #1]
+; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: smax_v32i16:
@@ -255,8 +255,8 @@ define void @smax_v16i32(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    smax z0.s, p0/m, z0.s, z1.s
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    smax z1.s, p0/m, z1.s, z3.s
-; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: smax_v16i32:
@@ -362,8 +362,8 @@ define void @smax_v8i64(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    smax z0.d, p0/m, z0.d, z1.d
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    smax z1.d, p0/m, z1.d, z3.d
-; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: smax_v8i64:
@@ -465,8 +465,8 @@ define void @smin_v64i8(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    smin z0.b, p0/m, z0.b, z1.b
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    smin z1.b, p0/m, z1.b, z3.b
-; VBITS_GE_256-NEXT:    st1b { z1.b }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1b { z0.b }, p0, [x0, x8]
+; VBITS_GE_256-NEXT:    st1b { z1.b }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: smin_v64i8:
@@ -564,8 +564,8 @@ define void @smin_v32i16(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    smin z0.h, p0/m, z0.h, z1.h
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    smin z1.h, p0/m, z1.h, z3.h
-; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x0, x8, lsl #1]
+; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: smin_v32i16:
@@ -663,8 +663,8 @@ define void @smin_v16i32(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    smin z0.s, p0/m, z0.s, z1.s
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    smin z1.s, p0/m, z1.s, z3.s
-; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: smin_v16i32:
@@ -770,8 +770,8 @@ define void @smin_v8i64(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    smin z0.d, p0/m, z0.d, z1.d
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    smin z1.d, p0/m, z1.d, z3.d
-; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: smin_v8i64:
@@ -873,8 +873,8 @@ define void @umax_v64i8(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    umax z0.b, p0/m, z0.b, z1.b
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    umax z1.b, p0/m, z1.b, z3.b
-; VBITS_GE_256-NEXT:    st1b { z1.b }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1b { z0.b }, p0, [x0, x8]
+; VBITS_GE_256-NEXT:    st1b { z1.b }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: umax_v64i8:
@@ -972,8 +972,8 @@ define void @umax_v32i16(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    umax z0.h, p0/m, z0.h, z1.h
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    umax z1.h, p0/m, z1.h, z3.h
-; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x0, x8, lsl #1]
+; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: umax_v32i16:
@@ -1071,8 +1071,8 @@ define void @umax_v16i32(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    umax z0.s, p0/m, z0.s, z1.s
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    umax z1.s, p0/m, z1.s, z3.s
-; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: umax_v16i32:
@@ -1178,8 +1178,8 @@ define void @umax_v8i64(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    umax z0.d, p0/m, z0.d, z1.d
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    umax z1.d, p0/m, z1.d, z3.d
-; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: umax_v8i64:
@@ -1281,8 +1281,8 @@ define void @umin_v64i8(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    umin z0.b, p0/m, z0.b, z1.b
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    umin z1.b, p0/m, z1.b, z3.b
-; VBITS_GE_256-NEXT:    st1b { z1.b }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1b { z0.b }, p0, [x0, x8]
+; VBITS_GE_256-NEXT:    st1b { z1.b }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: umin_v64i8:
@@ -1380,8 +1380,8 @@ define void @umin_v32i16(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    umin z0.h, p0/m, z0.h, z1.h
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    umin z1.h, p0/m, z1.h, z3.h
-; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x0, x8, lsl #1]
+; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: umin_v32i16:
@@ -1479,8 +1479,8 @@ define void @umin_v16i32(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    umin z0.s, p0/m, z0.s, z1.s
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    umin z1.s, p0/m, z1.s, z3.s
-; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: umin_v16i32:
@@ -1586,8 +1586,8 @@ define void @umin_v8i64(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    umin z0.d, p0/m, z0.d, z1.d
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    umin z1.d, p0/m, z1.d, z3.d
-; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: umin_v8i64:
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-mulh.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-mulh.ll
index 995da2be0cc1..331aab17a4ce 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-mulh.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-mulh.ll
@@ -82,8 +82,8 @@ define void @smulh_v64i8(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    smulh z0.b, p0/m, z0.b, z1.b
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    smulh z1.b, p0/m, z1.b, z3.b
-; VBITS_GE_256-NEXT:    st1b { z1.b }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1b { z0.b }, p0, [x0, x8]
+; VBITS_GE_256-NEXT:    st1b { z1.b }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: smulh_v64i8:
@@ -215,8 +215,8 @@ define void @smulh_v32i16(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    smulh z0.h, p0/m, z0.h, z1.h
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    smulh z1.h, p0/m, z1.h, z3.h
-; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x0, x8, lsl #1]
+; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: smulh_v32i16:
@@ -346,8 +346,8 @@ define void @smulh_v16i32(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    smulh z0.s, p0/m, z0.s, z1.s
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    smulh z1.s, p0/m, z1.s, z3.s
-; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: smulh_v16i32:
@@ -479,8 +479,8 @@ define void @smulh_v8i64(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    smulh z0.d, p0/m, z0.d, z1.d
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    smulh z1.d, p0/m, z1.d, z3.d
-; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: smulh_v8i64:
@@ -615,8 +615,8 @@ define void @umulh_v64i8(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    umulh z0.b, p0/m, z0.b, z1.b
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    umulh z1.b, p0/m, z1.b, z3.b
-; VBITS_GE_256-NEXT:    st1b { z1.b }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1b { z0.b }, p0, [x0, x8]
+; VBITS_GE_256-NEXT:    st1b { z1.b }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: umulh_v64i8:
@@ -749,8 +749,8 @@ define void @umulh_v32i16(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    umulh z0.h, p0/m, z0.h, z1.h
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    umulh z1.h, p0/m, z1.h, z3.h
-; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x0, x8, lsl #1]
+; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: umulh_v32i16:
@@ -882,8 +882,8 @@ define void @umulh_v16i32(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    umulh z0.s, p0/m, z0.s, z1.s
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    umulh z1.s, p0/m, z1.s, z3.s
-; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: umulh_v16i32:
@@ -1013,8 +1013,8 @@ define void @umulh_v8i64(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    umulh z0.d, p0/m, z0.d, z1.d
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    umulh z1.d, p0/m, z1.d, z3.d
-; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: umulh_v8i64:
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-rem.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-rem.ll
index 45960617f1d9..38444d83c1d7 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-rem.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-rem.ll
@@ -533,9 +533,9 @@ define void @srem_v128i16(ptr %a, ptr %b) vscale_range(16,0) #0 {
 define <2 x i32> @srem_v2i32(<2 x i32> %op1, <2 x i32> %op2) vscale_range(1,0) #0 {
 ; CHECK-LABEL: srem_v2i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
-; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
 ; CHECK-NEXT:    ptrue p0.s, vl2
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    movprfx z2, z0
 ; CHECK-NEXT:    sdiv z2.s, p0/m, z2.s, z1.s
 ; CHECK-NEXT:    mls v0.2s, v2.2s, v1.2s
@@ -549,9 +549,9 @@ define <2 x i32> @srem_v2i32(<2 x i32> %op1, <2 x i32> %op2) vscale_range(1,0) #
 define <4 x i32> @srem_v4i32(<4 x i32> %op1, <4 x i32> %op2) vscale_range(1,0) #0 {
 ; CHECK-LABEL: srem_v4i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
-; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
 ; CHECK-NEXT:    ptrue p0.s, vl4
+; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    movprfx z2, z0
 ; CHECK-NEXT:    sdiv z2.s, p0/m, z2.s, z1.s
 ; CHECK-NEXT:    mls v0.4s, v2.4s, v1.4s
@@ -582,25 +582,25 @@ define void @srem_v8i32(ptr %a, ptr %b) vscale_range(2,0) #0 {
 define void @srem_v16i32(ptr %a, ptr %b) #0 {
 ; VBITS_GE_128-LABEL: srem_v16i32:
 ; VBITS_GE_128:       // %bb.0:
-; VBITS_GE_128-NEXT:    ldp q0, q3, [x1]
 ; VBITS_GE_128-NEXT:    ptrue p0.s, vl4
+; VBITS_GE_128-NEXT:    ldp q0, q3, [x1]
 ; VBITS_GE_128-NEXT:    ldp q1, q2, [x0]
-; VBITS_GE_128-NEXT:    movprfx z4, z1
-; VBITS_GE_128-NEXT:    sdiv z4.s, p0/m, z4.s, z0.s
 ; VBITS_GE_128-NEXT:    ldp q16, q5, [x0, #32]
-; VBITS_GE_128-NEXT:    movprfx z19, z2
-; VBITS_GE_128-NEXT:    sdiv z19.s, p0/m, z19.s, z3.s
 ; VBITS_GE_128-NEXT:    ldp q17, q6, [x1, #32]
+; VBITS_GE_128-NEXT:    movprfx z4, z1
+; VBITS_GE_128-NEXT:    sdiv z4.s, p0/m, z4.s, z0.s
 ; VBITS_GE_128-NEXT:    movprfx z7, z5
 ; VBITS_GE_128-NEXT:    sdiv z7.s, p0/m, z7.s, z6.s
 ; VBITS_GE_128-NEXT:    movprfx z18, z16
 ; VBITS_GE_128-NEXT:    sdiv z18.s, p0/m, z18.s, z17.s
-; VBITS_GE_128-NEXT:    mls v1.4s, v4.4s, v0.4s
-; VBITS_GE_128-NEXT:    mls v2.4s, v19.4s, v3.4s
+; VBITS_GE_128-NEXT:    movprfx z19, z2
+; VBITS_GE_128-NEXT:    sdiv z19.s, p0/m, z19.s, z3.s
 ; VBITS_GE_128-NEXT:    mls v16.4s, v18.4s, v17.4s
 ; VBITS_GE_128-NEXT:    mls v5.4s, v7.4s, v6.4s
-; VBITS_GE_128-NEXT:    stp q1, q2, [x0]
+; VBITS_GE_128-NEXT:    mls v1.4s, v4.4s, v0.4s
+; VBITS_GE_128-NEXT:    mls v2.4s, v19.4s, v3.4s
 ; VBITS_GE_128-NEXT:    stp q16, q5, [x0, #32]
+; VBITS_GE_128-NEXT:    stp q1, q2, [x0]
 ; VBITS_GE_128-NEXT:    ret
 ;
 ; VBITS_GE_256-LABEL: srem_v16i32:
@@ -609,17 +609,17 @@ define void @srem_v16i32(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    mov x8, #8 // =0x8
 ; VBITS_GE_256-NEXT:    ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
 ; VBITS_GE_256-NEXT:    ld1w { z1.s }, p0/z, [x1, x8, lsl #2]
-; VBITS_GE_256-NEXT:    movprfx z2, z0
-; VBITS_GE_256-NEXT:    sdiv z2.s, p0/m, z2.s, z1.s
 ; VBITS_GE_256-NEXT:    ld1w { z3.s }, p0/z, [x0]
 ; VBITS_GE_256-NEXT:    ld1w { z4.s }, p0/z, [x1]
+; VBITS_GE_256-NEXT:    movprfx z2, z0
+; VBITS_GE_256-NEXT:    sdiv z2.s, p0/m, z2.s, z1.s
 ; VBITS_GE_256-NEXT:    movprfx z5, z3
 ; VBITS_GE_256-NEXT:    sdiv z5.s, p0/m, z5.s, z4.s
 ; VBITS_GE_256-NEXT:    mls z0.s, p0/m, z2.s, z1.s
 ; VBITS_GE_256-NEXT:    movprfx z1, z3
 ; VBITS_GE_256-NEXT:    mls z1.s, p0/m, z5.s, z4.s
-; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: srem_v16i32:
@@ -681,8 +681,8 @@ define <1 x i64> @srem_v1i64(<1 x i64> %op1, <1 x i64> %op2) vscale_range(1,0) #
 ; CHECK-LABEL: srem_v1i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d, vl1
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    movprfx z2, z0
 ; CHECK-NEXT:    sdiv z2.d, p0/m, z2.d, z1.d
 ; CHECK-NEXT:    mls z0.d, p0/m, z2.d, z1.d
@@ -698,8 +698,8 @@ define <2 x i64> @srem_v2i64(<2 x i64> %op1, <2 x i64> %op2) vscale_range(1,0) #
 ; CHECK-LABEL: srem_v2i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d, vl2
-; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    movprfx z2, z0
 ; CHECK-NEXT:    sdiv z2.d, p0/m, z2.d, z1.d
 ; CHECK-NEXT:    mls z0.d, p0/m, z2.d, z1.d
@@ -733,16 +733,16 @@ define void @srem_v8i64(ptr %a, ptr %b) #0 {
 ; VBITS_GE_128-NEXT:    ptrue p0.d, vl2
 ; VBITS_GE_128-NEXT:    ldp q0, q3, [x1]
 ; VBITS_GE_128-NEXT:    ldp q1, q2, [x0]
-; VBITS_GE_128-NEXT:    movprfx z4, z1
-; VBITS_GE_128-NEXT:    sdiv z4.d, p0/m, z4.d, z0.d
 ; VBITS_GE_128-NEXT:    ldp q16, q5, [x0, #32]
-; VBITS_GE_128-NEXT:    movprfx z19, z2
-; VBITS_GE_128-NEXT:    sdiv z19.d, p0/m, z19.d, z3.d
 ; VBITS_GE_128-NEXT:    ldp q17, q6, [x1, #32]
+; VBITS_GE_128-NEXT:    movprfx z4, z1
+; VBITS_GE_128-NEXT:    sdiv z4.d, p0/m, z4.d, z0.d
 ; VBITS_GE_128-NEXT:    movprfx z7, z5
 ; VBITS_GE_128-NEXT:    sdiv z7.d, p0/m, z7.d, z6.d
 ; VBITS_GE_128-NEXT:    movprfx z18, z16
 ; VBITS_GE_128-NEXT:    sdiv z18.d, p0/m, z18.d, z17.d
+; VBITS_GE_128-NEXT:    movprfx z19, z2
+; VBITS_GE_128-NEXT:    sdiv z19.d, p0/m, z19.d, z3.d
 ; VBITS_GE_128-NEXT:    mls z16.d, p0/m, z18.d, z17.d
 ; VBITS_GE_128-NEXT:    mls z5.d, p0/m, z7.d, z6.d
 ; VBITS_GE_128-NEXT:    msb z0.d, p0/m, z4.d, z1.d
@@ -758,17 +758,17 @@ define void @srem_v8i64(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    mov x8, #4 // =0x4
 ; VBITS_GE_256-NEXT:    ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
 ; VBITS_GE_256-NEXT:    ld1d { z1.d }, p0/z, [x1, x8, lsl #3]
-; VBITS_GE_256-NEXT:    movprfx z2, z0
-; VBITS_GE_256-NEXT:    sdiv z2.d, p0/m, z2.d, z1.d
 ; VBITS_GE_256-NEXT:    ld1d { z3.d }, p0/z, [x0]
 ; VBITS_GE_256-NEXT:    ld1d { z4.d }, p0/z, [x1]
+; VBITS_GE_256-NEXT:    movprfx z2, z0
+; VBITS_GE_256-NEXT:    sdiv z2.d, p0/m, z2.d, z1.d
 ; VBITS_GE_256-NEXT:    movprfx z5, z3
 ; VBITS_GE_256-NEXT:    sdiv z5.d, p0/m, z5.d, z4.d
 ; VBITS_GE_256-NEXT:    mls z0.d, p0/m, z2.d, z1.d
 ; VBITS_GE_256-NEXT:    movprfx z1, z3
 ; VBITS_GE_256-NEXT:    mls z1.d, p0/m, z5.d, z4.d
-; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: srem_v8i64:
@@ -1351,9 +1351,9 @@ define void @urem_v128i16(ptr %a, ptr %b) vscale_range(16,0) #0 {
 define <2 x i32> @urem_v2i32(<2 x i32> %op1, <2 x i32> %op2) vscale_range(1,0) #0 {
 ; CHECK-LABEL: urem_v2i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
-; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
 ; CHECK-NEXT:    ptrue p0.s, vl2
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    movprfx z2, z0
 ; CHECK-NEXT:    udiv z2.s, p0/m, z2.s, z1.s
 ; CHECK-NEXT:    mls v0.2s, v2.2s, v1.2s
@@ -1367,9 +1367,9 @@ define <2 x i32> @urem_v2i32(<2 x i32> %op1, <2 x i32> %op2) vscale_range(1,0) #
 define <4 x i32> @urem_v4i32(<4 x i32> %op1, <4 x i32> %op2) vscale_range(1,0) #0 {
 ; CHECK-LABEL: urem_v4i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
-; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
 ; CHECK-NEXT:    ptrue p0.s, vl4
+; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    movprfx z2, z0
 ; CHECK-NEXT:    udiv z2.s, p0/m, z2.s, z1.s
 ; CHECK-NEXT:    mls v0.4s, v2.4s, v1.4s
@@ -1400,25 +1400,25 @@ define void @urem_v8i32(ptr %a, ptr %b) vscale_range(2,0) #0 {
 define void @urem_v16i32(ptr %a, ptr %b) #0 {
 ; VBITS_GE_128-LABEL: urem_v16i32:
 ; VBITS_GE_128:       // %bb.0:
-; VBITS_GE_128-NEXT:    ldp q0, q3, [x1]
 ; VBITS_GE_128-NEXT:    ptrue p0.s, vl4
+; VBITS_GE_128-NEXT:    ldp q0, q3, [x1]
 ; VBITS_GE_128-NEXT:    ldp q1, q2, [x0]
-; VBITS_GE_128-NEXT:    movprfx z4, z1
-; VBITS_GE_128-NEXT:    udiv z4.s, p0/m, z4.s, z0.s
 ; VBITS_GE_128-NEXT:    ldp q16, q5, [x0, #32]
-; VBITS_GE_128-NEXT:    movprfx z19, z2
-; VBITS_GE_128-NEXT:    udiv z19.s, p0/m, z19.s, z3.s
 ; VBITS_GE_128-NEXT:    ldp q17, q6, [x1, #32]
+; VBITS_GE_128-NEXT:    movprfx z4, z1
+; VBITS_GE_128-NEXT:    udiv z4.s, p0/m, z4.s, z0.s
 ; VBITS_GE_128-NEXT:    movprfx z7, z5
 ; VBITS_GE_128-NEXT:    udiv z7.s, p0/m, z7.s, z6.s
 ; VBITS_GE_128-NEXT:    movprfx z18, z16
 ; VBITS_GE_128-NEXT:    udiv z18.s, p0/m, z18.s, z17.s
-; VBITS_GE_128-NEXT:    mls v1.4s, v4.4s, v0.4s
-; VBITS_GE_128-NEXT:    mls v2.4s, v19.4s, v3.4s
+; VBITS_GE_128-NEXT:    movprfx z19, z2
+; VBITS_GE_128-NEXT:    udiv z19.s, p0/m, z19.s, z3.s
 ; VBITS_GE_128-NEXT:    mls v16.4s, v18.4s, v17.4s
 ; VBITS_GE_128-NEXT:    mls v5.4s, v7.4s, v6.4s
-; VBITS_GE_128-NEXT:    stp q1, q2, [x0]
+; VBITS_GE_128-NEXT:    mls v1.4s, v4.4s, v0.4s
+; VBITS_GE_128-NEXT:    mls v2.4s, v19.4s, v3.4s
 ; VBITS_GE_128-NEXT:    stp q16, q5, [x0, #32]
+; VBITS_GE_128-NEXT:    stp q1, q2, [x0]
 ; VBITS_GE_128-NEXT:    ret
 ;
 ; VBITS_GE_256-LABEL: urem_v16i32:
@@ -1427,17 +1427,17 @@ define void @urem_v16i32(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    mov x8, #8 // =0x8
 ; VBITS_GE_256-NEXT:    ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
 ; VBITS_GE_256-NEXT:    ld1w { z1.s }, p0/z, [x1, x8, lsl #2]
-; VBITS_GE_256-NEXT:    movprfx z2, z0
-; VBITS_GE_256-NEXT:    udiv z2.s, p0/m, z2.s, z1.s
 ; VBITS_GE_256-NEXT:    ld1w { z3.s }, p0/z, [x0]
 ; VBITS_GE_256-NEXT:    ld1w { z4.s }, p0/z, [x1]
+; VBITS_GE_256-NEXT:    movprfx z2, z0
+; VBITS_GE_256-NEXT:    udiv z2.s, p0/m, z2.s, z1.s
 ; VBITS_GE_256-NEXT:    movprfx z5, z3
 ; VBITS_GE_256-NEXT:    udiv z5.s, p0/m, z5.s, z4.s
 ; VBITS_GE_256-NEXT:    mls z0.s, p0/m, z2.s, z1.s
 ; VBITS_GE_256-NEXT:    movprfx z1, z3
 ; VBITS_GE_256-NEXT:    mls z1.s, p0/m, z5.s, z4.s
-; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: urem_v16i32:
@@ -1499,8 +1499,8 @@ define <1 x i64> @urem_v1i64(<1 x i64> %op1, <1 x i64> %op2) vscale_range(1,0) #
 ; CHECK-LABEL: urem_v1i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d, vl1
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    movprfx z2, z0
 ; CHECK-NEXT:    udiv z2.d, p0/m, z2.d, z1.d
 ; CHECK-NEXT:    mls z0.d, p0/m, z2.d, z1.d
@@ -1516,8 +1516,8 @@ define <2 x i64> @urem_v2i64(<2 x i64> %op1, <2 x i64> %op2) vscale_range(1,0) #
 ; CHECK-LABEL: urem_v2i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d, vl2
-; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    movprfx z2, z0
 ; CHECK-NEXT:    udiv z2.d, p0/m, z2.d, z1.d
 ; CHECK-NEXT:    mls z0.d, p0/m, z2.d, z1.d
@@ -1551,16 +1551,16 @@ define void @urem_v8i64(ptr %a, ptr %b) #0 {
 ; VBITS_GE_128-NEXT:    ptrue p0.d, vl2
 ; VBITS_GE_128-NEXT:    ldp q0, q3, [x1]
 ; VBITS_GE_128-NEXT:    ldp q1, q2, [x0]
-; VBITS_GE_128-NEXT:    movprfx z4, z1
-; VBITS_GE_128-NEXT:    udiv z4.d, p0/m, z4.d, z0.d
 ; VBITS_GE_128-NEXT:    ldp q16, q5, [x0, #32]
-; VBITS_GE_128-NEXT:    movprfx z19, z2
-; VBITS_GE_128-NEXT:    udiv z19.d, p0/m, z19.d, z3.d
 ; VBITS_GE_128-NEXT:    ldp q17, q6, [x1, #32]
+; VBITS_GE_128-NEXT:    movprfx z4, z1
+; VBITS_GE_128-NEXT:    udiv z4.d, p0/m, z4.d, z0.d
 ; VBITS_GE_128-NEXT:    movprfx z7, z5
 ; VBITS_GE_128-NEXT:    udiv z7.d, p0/m, z7.d, z6.d
 ; VBITS_GE_128-NEXT:    movprfx z18, z16
 ; VBITS_GE_128-NEXT:    udiv z18.d, p0/m, z18.d, z17.d
+; VBITS_GE_128-NEXT:    movprfx z19, z2
+; VBITS_GE_128-NEXT:    udiv z19.d, p0/m, z19.d, z3.d
 ; VBITS_GE_128-NEXT:    mls z16.d, p0/m, z18.d, z17.d
 ; VBITS_GE_128-NEXT:    mls z5.d, p0/m, z7.d, z6.d
 ; VBITS_GE_128-NEXT:    msb z0.d, p0/m, z4.d, z1.d
@@ -1576,17 +1576,17 @@ define void @urem_v8i64(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    mov x8, #4 // =0x4
 ; VBITS_GE_256-NEXT:    ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
 ; VBITS_GE_256-NEXT:    ld1d { z1.d }, p0/z, [x1, x8, lsl #3]
-; VBITS_GE_256-NEXT:    movprfx z2, z0
-; VBITS_GE_256-NEXT:    udiv z2.d, p0/m, z2.d, z1.d
 ; VBITS_GE_256-NEXT:    ld1d { z3.d }, p0/z, [x0]
 ; VBITS_GE_256-NEXT:    ld1d { z4.d }, p0/z, [x1]
+; VBITS_GE_256-NEXT:    movprfx z2, z0
+; VBITS_GE_256-NEXT:    udiv z2.d, p0/m, z2.d, z1.d
 ; VBITS_GE_256-NEXT:    movprfx z5, z3
 ; VBITS_GE_256-NEXT:    udiv z5.d, p0/m, z5.d, z4.d
 ; VBITS_GE_256-NEXT:    mls z0.d, p0/m, z2.d, z1.d
 ; VBITS_GE_256-NEXT:    movprfx z1, z3
 ; VBITS_GE_256-NEXT:    mls z1.d, p0/m, z5.d, z4.d
-; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: urem_v8i64:
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-shifts.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-shifts.ll
index 7184cc970597..0fa8c8f50e29 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-shifts.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-shifts.ll
@@ -59,8 +59,8 @@ define void @ashr_v64i8(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    asr z0.b, p0/m, z0.b, z1.b
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    asr z1.b, p0/m, z1.b, z3.b
-; VBITS_GE_256-NEXT:    st1b { z1.b }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1b { z0.b }, p0, [x0, x8]
+; VBITS_GE_256-NEXT:    st1b { z1.b }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: ashr_v64i8:
@@ -160,8 +160,8 @@ define void @ashr_v32i16(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    asr z0.h, p0/m, z0.h, z1.h
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    asr z1.h, p0/m, z1.h, z3.h
-; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x0, x8, lsl #1]
+; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: ashr_v32i16:
@@ -261,8 +261,8 @@ define void @ashr_v16i32(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    asr z0.s, p0/m, z0.s, z1.s
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    asr z1.s, p0/m, z1.s, z3.s
-; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: ashr_v16i32:
@@ -362,8 +362,8 @@ define void @ashr_v8i64(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    asr z0.d, p0/m, z0.d, z1.d
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    asr z1.d, p0/m, z1.d, z3.d
-; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: ashr_v8i64:
@@ -467,8 +467,8 @@ define void @lshr_v64i8(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    lsr z0.b, p0/m, z0.b, z1.b
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    lsr z1.b, p0/m, z1.b, z3.b
-; VBITS_GE_256-NEXT:    st1b { z1.b }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1b { z0.b }, p0, [x0, x8]
+; VBITS_GE_256-NEXT:    st1b { z1.b }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: lshr_v64i8:
@@ -568,8 +568,8 @@ define void @lshr_v32i16(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    lsr z0.h, p0/m, z0.h, z1.h
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    lsr z1.h, p0/m, z1.h, z3.h
-; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x0, x8, lsl #1]
+; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: lshr_v32i16:
@@ -669,8 +669,8 @@ define void @lshr_v16i32(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    lsr z0.s, p0/m, z0.s, z1.s
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    lsr z1.s, p0/m, z1.s, z3.s
-; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: lshr_v16i32:
@@ -770,8 +770,8 @@ define void @lshr_v8i64(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    lsr z0.d, p0/m, z0.d, z1.d
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    lsr z1.d, p0/m, z1.d, z3.d
-; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: lshr_v8i64:
@@ -873,8 +873,8 @@ define void @shl_v64i8(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    lsl z0.b, p0/m, z0.b, z1.b
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    lsl z1.b, p0/m, z1.b, z3.b
-; VBITS_GE_256-NEXT:    st1b { z1.b }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1b { z0.b }, p0, [x0, x8]
+; VBITS_GE_256-NEXT:    st1b { z1.b }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: shl_v64i8:
@@ -972,8 +972,8 @@ define void @shl_v32i16(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    lsl z0.h, p0/m, z0.h, z1.h
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    lsl z1.h, p0/m, z1.h, z3.h
-; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1h { z0.h }, p0, [x0, x8, lsl #1]
+; VBITS_GE_256-NEXT:    st1h { z1.h }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: shl_v32i16:
@@ -1071,8 +1071,8 @@ define void @shl_v16i32(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    lsl z0.s, p0/m, z0.s, z1.s
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    lsl z1.s, p0/m, z1.s, z3.s
-; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1w { z0.s }, p0, [x0, x8, lsl #2]
+; VBITS_GE_256-NEXT:    st1w { z1.s }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: shl_v16i32:
@@ -1170,8 +1170,8 @@ define void @shl_v8i64(ptr %a, ptr %b) #0 {
 ; VBITS_GE_256-NEXT:    lsl z0.d, p0/m, z0.d, z1.d
 ; VBITS_GE_256-NEXT:    movprfx z1, z2
 ; VBITS_GE_256-NEXT:    lsl z1.d, p0/m, z1.d, z3.d
-; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    st1d { z0.d }, p0, [x0, x8, lsl #3]
+; VBITS_GE_256-NEXT:    st1d { z1.d }, p0, [x0]
 ; VBITS_GE_256-NEXT:    ret
 ;
 ; VBITS_GE_512-LABEL: shl_v8i64:
diff --git a/llvm/test/CodeGen/AArch64/sve-fpext-load.ll b/llvm/test/CodeGen/AArch64/sve-fpext-load.ll
index 78decf5ed1e3..aec2150124c3 100644
--- a/llvm/test/CodeGen/AArch64/sve-fpext-load.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fpext-load.ll
@@ -46,13 +46,13 @@ define <vscale x 8 x double> @ext8_f16_f64(ptr %ptr, i64 %index) {
 ; CHECK-NEXT:    uunpkhi z1.d, z1.s
 ; CHECK-NEXT:    uunpklo z3.d, z0.s
 ; CHECK-NEXT:    uunpkhi z4.d, z0.s
+; CHECK-NEXT:    fcvt z1.d, p0/m, z1.h
 ; CHECK-NEXT:    movprfx z0, z2
 ; CHECK-NEXT:    fcvt z0.d, p0/m, z2.h
 ; CHECK-NEXT:    movprfx z2, z3
 ; CHECK-NEXT:    fcvt z2.d, p0/m, z3.h
 ; CHECK-NEXT:    movprfx z3, z4
 ; CHECK-NEXT:    fcvt z3.d, p0/m, z4.h
-; CHECK-NEXT:    fcvt z1.d, p0/m, z1.h
 ; CHECK-NEXT:    ret
   %load = load <vscale x 8 x half>, ptr %ptr, align 4
   %load.ext = fpext <vscale x 8 x half> %load to <vscale x 8 x double>
diff --git a/llvm/test/CodeGen/AArch64/sve-fptosi-sat.ll b/llvm/test/CodeGen/AArch64/sve-fptosi-sat.ll
index 43113df07c07..813f1601e809 100644
--- a/llvm/test/CodeGen/AArch64/sve-fptosi-sat.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fptosi-sat.ll
@@ -74,12 +74,12 @@ define <vscale x 8 x i32> @test_signed_v8f32_v8i32(<vscale x 8 x float> %f) {
 ; CHECK-NEXT:    mov w8, #1325400063 // =0x4effffff
 ; CHECK-NEXT:    mov z6.s, #0x7fffffff
 ; CHECK-NEXT:    mov z4.s, w8
-; CHECK-NEXT:    movprfx z5, z1
-; CHECK-NEXT:    fcvtzs z5.s, p0/m, z1.s
 ; CHECK-NEXT:    fcmge p1.s, p0/z, z0.s, z2.s
 ; CHECK-NEXT:    fcmge p2.s, p0/z, z1.s, z2.s
 ; CHECK-NEXT:    movprfx z2, z0
 ; CHECK-NEXT:    fcvtzs z2.s, p0/m, z0.s
+; CHECK-NEXT:    movprfx z5, z1
+; CHECK-NEXT:    fcvtzs z5.s, p0/m, z1.s
 ; CHECK-NEXT:    fcmgt p3.s, p0/z, z0.s, z4.s
 ; CHECK-NEXT:    fcmgt p4.s, p0/z, z1.s, z4.s
 ; CHECK-NEXT:    not p1.b, p0/z, p1.b
@@ -134,17 +134,17 @@ define <vscale x 8 x i16> @test_signed_v8f32_v8i16(<vscale x 8 x float> %f) {
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    mov w8, #-956301312 // =0xc7000000
-; CHECK-NEXT:    movprfx z4, z0
-; CHECK-NEXT:    fcvtzs z4.s, p0/m, z0.s
+; CHECK-NEXT:    mov z5.s, #32767 // =0x7fff
 ; CHECK-NEXT:    mov z2.s, w8
 ; CHECK-NEXT:    mov w8, #65024 // =0xfe00
-; CHECK-NEXT:    mov z5.s, #32767 // =0x7fff
 ; CHECK-NEXT:    movk w8, #18175, lsl #16
 ; CHECK-NEXT:    mov z3.s, w8
 ; CHECK-NEXT:    fcmge p1.s, p0/z, z1.s, z2.s
 ; CHECK-NEXT:    fcmge p2.s, p0/z, z0.s, z2.s
 ; CHECK-NEXT:    movprfx z2, z1
 ; CHECK-NEXT:    fcvtzs z2.s, p0/m, z1.s
+; CHECK-NEXT:    movprfx z4, z0
+; CHECK-NEXT:    fcvtzs z4.s, p0/m, z0.s
 ; CHECK-NEXT:    fcmgt p3.s, p0/z, z1.s, z3.s
 ; CHECK-NEXT:    fcmgt p4.s, p0/z, z0.s, z3.s
 ; CHECK-NEXT:    not p1.b, p0/z, p1.b
@@ -206,13 +206,13 @@ define <vscale x 4 x i64> @test_signed_v4f32_v4i64(<vscale x 4 x float> %f) {
 ; CHECK-NEXT:    mov w8, #1593835519 // =0x5effffff
 ; CHECK-NEXT:    mov z3.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z4.s, w8
-; CHECK-NEXT:    movprfx z5, z0
-; CHECK-NEXT:    fcvtzs z5.d, p0/m, z0.s
 ; CHECK-NEXT:    mov z6.d, #0x7fffffffffffffff
 ; CHECK-NEXT:    fcmge p1.s, p0/z, z1.s, z2.s
 ; CHECK-NEXT:    fcmge p2.s, p0/z, z0.s, z2.s
 ; CHECK-NEXT:    movprfx z2, z1
 ; CHECK-NEXT:    fcvtzs z2.d, p0/m, z1.s
+; CHECK-NEXT:    movprfx z5, z0
+; CHECK-NEXT:    fcvtzs z5.d, p0/m, z0.s
 ; CHECK-NEXT:    fcmgt p3.s, p0/z, z1.s, z4.s
 ; CHECK-NEXT:    fcmgt p4.s, p0/z, z0.s, z4.s
 ; CHECK-NEXT:    not p1.b, p0/z, p1.b
@@ -283,13 +283,13 @@ define <vscale x 4 x i32> @test_signed_v4f64_v4i32(<vscale x 4 x double> %f) {
 ; CHECK-NEXT:    mov x8, #281474972516352 // =0xffffffc00000
 ; CHECK-NEXT:    mov z6.d, #0x7fffffff
 ; CHECK-NEXT:    movk x8, #16863, lsl #48
-; CHECK-NEXT:    movprfx z5, z0
-; CHECK-NEXT:    fcvtzs z5.d, p0/m, z0.d
 ; CHECK-NEXT:    mov z4.d, x8
 ; CHECK-NEXT:    fcmge p1.d, p0/z, z1.d, z2.d
 ; CHECK-NEXT:    fcmge p2.d, p0/z, z0.d, z2.d
 ; CHECK-NEXT:    movprfx z2, z1
 ; CHECK-NEXT:    fcvtzs z2.d, p0/m, z1.d
+; CHECK-NEXT:    movprfx z5, z0
+; CHECK-NEXT:    fcvtzs z5.d, p0/m, z0.d
 ; CHECK-NEXT:    fcmgt p3.d, p0/z, z1.d, z4.d
 ; CHECK-NEXT:    fcmgt p4.d, p0/z, z0.d, z4.d
 ; CHECK-NEXT:    not p1.b, p0/z, p1.b
@@ -324,24 +324,24 @@ define <vscale x 8 x i32> @test_signed_v8f64_v8i32(<vscale x 8 x double> %f) {
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov x8, #-4476578029606273024 // =0xc1e0000000000000
-; CHECK-NEXT:    movprfx z6, z1
-; CHECK-NEXT:    fcvtzs z6.d, p0/m, z1.d
+; CHECK-NEXT:    mov z26.d, #0x7fffffff
 ; CHECK-NEXT:    mov z4.d, x8
 ; CHECK-NEXT:    mov x8, #281474972516352 // =0xffffffc00000
-; CHECK-NEXT:    movprfx z7, z0
-; CHECK-NEXT:    fcvtzs z7.d, p0/m, z0.d
 ; CHECK-NEXT:    movk x8, #16863, lsl #48
-; CHECK-NEXT:    mov z26.d, #0x7fffffff
-; CHECK-NEXT:    movprfx z24, z3
-; CHECK-NEXT:    fcvtzs z24.d, p0/m, z3.d
 ; CHECK-NEXT:    mov z5.d, x8
-; CHECK-NEXT:    movprfx z25, z2
-; CHECK-NEXT:    fcvtzs z25.d, p0/m, z2.d
 ; CHECK-NEXT:    fcmge p1.d, p0/z, z1.d, z4.d
 ; CHECK-NEXT:    fcmge p2.d, p0/z, z0.d, z4.d
 ; CHECK-NEXT:    fcmge p3.d, p0/z, z3.d, z4.d
 ; CHECK-NEXT:    fcmge p4.d, p0/z, z2.d, z4.d
 ; CHECK-NEXT:    mov z4.d, #0xffffffff80000000
+; CHECK-NEXT:    movprfx z6, z1
+; CHECK-NEXT:    fcvtzs z6.d, p0/m, z1.d
+; CHECK-NEXT:    movprfx z7, z0
+; CHECK-NEXT:    fcvtzs z7.d, p0/m, z0.d
+; CHECK-NEXT:    movprfx z24, z3
+; CHECK-NEXT:    fcvtzs z24.d, p0/m, z3.d
+; CHECK-NEXT:    movprfx z25, z2
+; CHECK-NEXT:    fcvtzs z25.d, p0/m, z2.d
 ; CHECK-NEXT:    fcmgt p5.d, p0/z, z1.d, z5.d
 ; CHECK-NEXT:    fcmgt p6.d, p0/z, z0.d, z5.d
 ; CHECK-NEXT:    fcmgt p7.d, p0/z, z3.d, z5.d
@@ -389,17 +389,17 @@ define <vscale x 4 x i16> @test_signed_v4f64_v4i16(<vscale x 4 x double> %f) {
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov x8, #-4548635623644200960 // =0xc0e0000000000000
-; CHECK-NEXT:    movprfx z4, z0
-; CHECK-NEXT:    fcvtzs z4.d, p0/m, z0.d
+; CHECK-NEXT:    mov z5.d, #32767 // =0x7fff
 ; CHECK-NEXT:    mov z2.d, x8
 ; CHECK-NEXT:    mov x8, #281200098803712 // =0xffc000000000
-; CHECK-NEXT:    mov z5.d, #32767 // =0x7fff
 ; CHECK-NEXT:    movk x8, #16607, lsl #48
 ; CHECK-NEXT:    mov z3.d, x8
 ; CHECK-NEXT:    fcmge p1.d, p0/z, z1.d, z2.d
 ; CHECK-NEXT:    fcmge p2.d, p0/z, z0.d, z2.d
 ; CHECK-NEXT:    movprfx z2, z1
 ; CHECK-NEXT:    fcvtzs z2.d, p0/m, z1.d
+; CHECK-NEXT:    movprfx z4, z0
+; CHECK-NEXT:    fcvtzs z4.d, p0/m, z0.d
 ; CHECK-NEXT:    fcmgt p3.d, p0/z, z1.d, z3.d
 ; CHECK-NEXT:    fcmgt p4.d, p0/z, z0.d, z3.d
 ; CHECK-NEXT:    not p1.b, p0/z, p1.b
@@ -434,23 +434,23 @@ define <vscale x 8 x i16> @test_signed_v8f64_v8i16(<vscale x 8 x double> %f) {
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov x8, #-4548635623644200960 // =0xc0e0000000000000
+; CHECK-NEXT:    mov z25.d, #32767 // =0x7fff
+; CHECK-NEXT:    mov z4.d, x8
+; CHECK-NEXT:    mov x8, #281200098803712 // =0xffc000000000
+; CHECK-NEXT:    movk x8, #16607, lsl #48
+; CHECK-NEXT:    fcmge p1.d, p0/z, z3.d, z4.d
+; CHECK-NEXT:    fcmge p2.d, p0/z, z2.d, z4.d
+; CHECK-NEXT:    fcmge p3.d, p0/z, z1.d, z4.d
+; CHECK-NEXT:    fcmge p4.d, p0/z, z0.d, z4.d
 ; CHECK-NEXT:    movprfx z5, z3
 ; CHECK-NEXT:    fcvtzs z5.d, p0/m, z3.d
 ; CHECK-NEXT:    mov z4.d, x8
-; CHECK-NEXT:    mov x8, #281200098803712 // =0xffc000000000
 ; CHECK-NEXT:    movprfx z6, z2
 ; CHECK-NEXT:    fcvtzs z6.d, p0/m, z2.d
-; CHECK-NEXT:    movk x8, #16607, lsl #48
 ; CHECK-NEXT:    movprfx z7, z1
 ; CHECK-NEXT:    fcvtzs z7.d, p0/m, z1.d
 ; CHECK-NEXT:    movprfx z24, z0
 ; CHECK-NEXT:    fcvtzs z24.d, p0/m, z0.d
-; CHECK-NEXT:    mov z25.d, #32767 // =0x7fff
-; CHECK-NEXT:    fcmge p1.d, p0/z, z3.d, z4.d
-; CHECK-NEXT:    fcmge p2.d, p0/z, z2.d, z4.d
-; CHECK-NEXT:    fcmge p3.d, p0/z, z1.d, z4.d
-; CHECK-NEXT:    fcmge p4.d, p0/z, z0.d, z4.d
-; CHECK-NEXT:    mov z4.d, x8
 ; CHECK-NEXT:    not p1.b, p0/z, p1.b
 ; CHECK-NEXT:    fcmgt p5.d, p0/z, z3.d, z4.d
 ; CHECK-NEXT:    fcmgt p6.d, p0/z, z2.d, z4.d
@@ -528,12 +528,12 @@ define <vscale x 4 x i64> @test_signed_v4f64_v4i64(<vscale x 4 x double> %f) {
 ; CHECK-NEXT:    mov x8, #4890909195324358655 // =0x43dfffffffffffff
 ; CHECK-NEXT:    mov z6.d, #0x7fffffffffffffff
 ; CHECK-NEXT:    mov z4.d, x8
-; CHECK-NEXT:    movprfx z5, z1
-; CHECK-NEXT:    fcvtzs z5.d, p0/m, z1.d
 ; CHECK-NEXT:    fcmge p1.d, p0/z, z0.d, z2.d
 ; CHECK-NEXT:    fcmge p2.d, p0/z, z1.d, z2.d
 ; CHECK-NEXT:    movprfx z2, z0
 ; CHECK-NEXT:    fcvtzs z2.d, p0/m, z0.d
+; CHECK-NEXT:    movprfx z5, z1
+; CHECK-NEXT:    fcvtzs z5.d, p0/m, z1.d
 ; CHECK-NEXT:    fcmgt p3.d, p0/z, z0.d, z4.d
 ; CHECK-NEXT:    fcmgt p4.d, p0/z, z1.d, z4.d
 ; CHECK-NEXT:    not p1.b, p0/z, p1.b
@@ -629,13 +629,13 @@ define <vscale x 8 x i32> @test_signed_v8f16_v8i32(<vscale x 8 x half> %f) {
 ; CHECK-NEXT:    mov w8, #31743 // =0x7bff
 ; CHECK-NEXT:    mov z3.s, #0x80000000
 ; CHECK-NEXT:    mov z4.h, w8
-; CHECK-NEXT:    movprfx z5, z0
-; CHECK-NEXT:    fcvtzs z5.s, p0/m, z0.h
 ; CHECK-NEXT:    mov z6.s, #0x7fffffff
 ; CHECK-NEXT:    fcmge p1.h, p0/z, z1.h, z2.h
 ; CHECK-NEXT:    fcmge p2.h, p0/z, z0.h, z2.h
 ; CHECK-NEXT:    movprfx z2, z1
 ; CHECK-NEXT:    fcvtzs z2.s, p0/m, z1.h
+; CHECK-NEXT:    movprfx z5, z0
+; CHECK-NEXT:    fcvtzs z5.s, p0/m, z0.h
 ; CHECK-NEXT:    fcmgt p3.h, p0/z, z1.h, z4.h
 ; CHECK-NEXT:    fcmgt p4.h, p0/z, z0.h, z4.h
 ; CHECK-NEXT:    not p1.b, p0/z, p1.b
@@ -742,13 +742,13 @@ define <vscale x 4 x i64> @test_signed_v4f16_v4i64(<vscale x 4 x half> %f) {
 ; CHECK-NEXT:    mov w8, #31743 // =0x7bff
 ; CHECK-NEXT:    mov z3.d, #0x8000000000000000
 ; CHECK-NEXT:    mov z4.h, w8
-; CHECK-NEXT:    movprfx z5, z0
-; CHECK-NEXT:    fcvtzs z5.d, p0/m, z0.h
 ; CHECK-NEXT:    mov z6.d, #0x7fffffffffffffff
 ; CHECK-NEXT:    fcmge p1.h, p0/z, z1.h, z2.h
 ; CHECK-NEXT:    fcmge p2.h, p0/z, z0.h, z2.h
 ; CHECK-NEXT:    movprfx z2, z1
 ; CHECK-NEXT:    fcvtzs z2.d, p0/m, z1.h
+; CHECK-NEXT:    movprfx z5, z0
+; CHECK-NEXT:    fcvtzs z5.d, p0/m, z0.h
 ; CHECK-NEXT:    fcmgt p3.h, p0/z, z1.h, z4.h
 ; CHECK-NEXT:    fcmgt p4.h, p0/z, z0.h, z4.h
 ; CHECK-NEXT:    not p1.b, p0/z, p1.b
diff --git a/llvm/test/CodeGen/AArch64/sve-fptoui-sat.ll b/llvm/test/CodeGen/AArch64/sve-fptoui-sat.ll
index c9e06fd9f341..c56c0b37888d 100644
--- a/llvm/test/CodeGen/AArch64/sve-fptoui-sat.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fptoui-sat.ll
@@ -16,10 +16,10 @@ define <vscale x 2 x i32> @test_signed_v2f32_v2i32(<vscale x 2 x float> %f) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov w8, #1333788671 // =0x4f7fffff
-; CHECK-NEXT:    movprfx z2, z0
-; CHECK-NEXT:    fcvtzu z2.d, p0/m, z0.s
 ; CHECK-NEXT:    mov z1.s, w8
 ; CHECK-NEXT:    fcmge p1.s, p0/z, z0.s, #0.0
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    fcvtzu z2.d, p0/m, z0.s
 ; CHECK-NEXT:    not p1.b, p0/z, p1.b
 ; CHECK-NEXT:    fcmgt p0.s, p0/z, z0.s, z1.s
 ; CHECK-NEXT:    mov z0.d, #0xffffffff
@@ -35,10 +35,10 @@ define <vscale x 4 x i32> @test_signed_v4f32_v4i32(<vscale x 4 x float> %f) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    mov w8, #1333788671 // =0x4f7fffff
-; CHECK-NEXT:    movprfx z1, z0
-; CHECK-NEXT:    fcvtzu z1.s, p0/m, z0.s
 ; CHECK-NEXT:    mov z2.s, w8
 ; CHECK-NEXT:    fcmge p1.s, p0/z, z0.s, #0.0
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    fcvtzu z1.s, p0/m, z0.s
 ; CHECK-NEXT:    not p1.b, p0/z, p1.b
 ; CHECK-NEXT:    fcmgt p0.s, p0/z, z0.s, z2.s
 ; CHECK-NEXT:    mov z1.s, p1/m, #0 // =0x0
@@ -54,13 +54,13 @@ define <vscale x 8 x i32> @test_signed_v8f32_v8i32(<vscale x 8 x float> %f) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    mov w8, #1333788671 // =0x4f7fffff
+; CHECK-NEXT:    mov z4.s, w8
+; CHECK-NEXT:    fcmge p1.s, p0/z, z0.s, #0.0
+; CHECK-NEXT:    fcmge p2.s, p0/z, z1.s, #0.0
 ; CHECK-NEXT:    movprfx z2, z0
 ; CHECK-NEXT:    fcvtzu z2.s, p0/m, z0.s
-; CHECK-NEXT:    mov z4.s, w8
 ; CHECK-NEXT:    movprfx z3, z1
 ; CHECK-NEXT:    fcvtzu z3.s, p0/m, z1.s
-; CHECK-NEXT:    fcmge p1.s, p0/z, z0.s, #0.0
-; CHECK-NEXT:    fcmge p2.s, p0/z, z1.s, #0.0
 ; CHECK-NEXT:    fcmgt p3.s, p0/z, z0.s, z4.s
 ; CHECK-NEXT:    not p1.b, p0/z, p1.b
 ; CHECK-NEXT:    not p2.b, p0/z, p2.b
@@ -81,11 +81,11 @@ define <vscale x 4 x i16> @test_signed_v4f32_v4i16(<vscale x 4 x float> %f) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    mov w8, #65280 // =0xff00
-; CHECK-NEXT:    movprfx z2, z0
-; CHECK-NEXT:    fcvtzu z2.s, p0/m, z0.s
 ; CHECK-NEXT:    movk w8, #18303, lsl #16
 ; CHECK-NEXT:    mov z1.s, w8
 ; CHECK-NEXT:    fcmge p1.s, p0/z, z0.s, #0.0
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    fcvtzu z2.s, p0/m, z0.s
 ; CHECK-NEXT:    not p1.b, p0/z, p1.b
 ; CHECK-NEXT:    fcmgt p0.s, p0/z, z0.s, z1.s
 ; CHECK-NEXT:    mov z0.s, #65535 // =0xffff
@@ -101,14 +101,14 @@ define <vscale x 8 x i16> @test_signed_v8f32_v8i16(<vscale x 8 x float> %f) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    mov w8, #65280 // =0xff00
-; CHECK-NEXT:    movprfx z3, z1
-; CHECK-NEXT:    fcvtzu z3.s, p0/m, z1.s
 ; CHECK-NEXT:    movk w8, #18303, lsl #16
-; CHECK-NEXT:    movprfx z4, z0
-; CHECK-NEXT:    fcvtzu z4.s, p0/m, z0.s
 ; CHECK-NEXT:    mov z2.s, w8
 ; CHECK-NEXT:    fcmge p1.s, p0/z, z1.s, #0.0
 ; CHECK-NEXT:    fcmge p2.s, p0/z, z0.s, #0.0
+; CHECK-NEXT:    movprfx z3, z1
+; CHECK-NEXT:    fcvtzu z3.s, p0/m, z1.s
+; CHECK-NEXT:    movprfx z4, z0
+; CHECK-NEXT:    fcvtzu z4.s, p0/m, z0.s
 ; CHECK-NEXT:    fcmgt p3.s, p0/z, z1.s, z2.s
 ; CHECK-NEXT:    not p1.b, p0/z, p1.b
 ; CHECK-NEXT:    not p2.b, p0/z, p2.b
@@ -129,10 +129,10 @@ define <vscale x 2 x i64> @test_signed_v2f32_v2i64(<vscale x 2 x float> %f) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov w8, #1602224127 // =0x5f7fffff
-; CHECK-NEXT:    movprfx z1, z0
-; CHECK-NEXT:    fcvtzu z1.d, p0/m, z0.s
 ; CHECK-NEXT:    mov z2.s, w8
 ; CHECK-NEXT:    fcmge p1.s, p0/z, z0.s, #0.0
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    fcvtzu z1.d, p0/m, z0.s
 ; CHECK-NEXT:    not p1.b, p0/z, p1.b
 ; CHECK-NEXT:    fcmgt p0.s, p0/z, z0.s, z2.s
 ; CHECK-NEXT:    mov z1.d, p1/m, #0 // =0x0
@@ -150,13 +150,13 @@ define <vscale x 4 x i64> @test_signed_v4f32_v4i64(<vscale x 4 x float> %f) {
 ; CHECK-NEXT:    uunpklo z2.d, z0.s
 ; CHECK-NEXT:    uunpkhi z3.d, z0.s
 ; CHECK-NEXT:    mov w8, #1602224127 // =0x5f7fffff
+; CHECK-NEXT:    mov z4.s, w8
+; CHECK-NEXT:    fcmge p1.s, p0/z, z2.s, #0.0
+; CHECK-NEXT:    fcmge p2.s, p0/z, z3.s, #0.0
 ; CHECK-NEXT:    movprfx z0, z2
 ; CHECK-NEXT:    fcvtzu z0.d, p0/m, z2.s
 ; CHECK-NEXT:    movprfx z1, z3
 ; CHECK-NEXT:    fcvtzu z1.d, p0/m, z3.s
-; CHECK-NEXT:    mov z4.s, w8
-; CHECK-NEXT:    fcmge p1.s, p0/z, z2.s, #0.0
-; CHECK-NEXT:    fcmge p2.s, p0/z, z3.s, #0.0
 ; CHECK-NEXT:    fcmgt p3.s, p0/z, z2.s, z4.s
 ; CHECK-NEXT:    not p1.b, p0/z, p1.b
 ; CHECK-NEXT:    not p2.b, p0/z, p2.b
@@ -185,11 +185,11 @@ define <vscale x 2 x i32> @test_signed_v2f64_v2i32(<vscale x 2 x double> %f) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov x8, #281474974613504 // =0xffffffe00000
-; CHECK-NEXT:    movprfx z2, z0
-; CHECK-NEXT:    fcvtzu z2.d, p0/m, z0.d
 ; CHECK-NEXT:    movk x8, #16879, lsl #48
 ; CHECK-NEXT:    mov z1.d, x8
 ; CHECK-NEXT:    fcmge p1.d, p0/z, z0.d, #0.0
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    fcvtzu z2.d, p0/m, z0.d
 ; CHECK-NEXT:    not p1.b, p0/z, p1.b
 ; CHECK-NEXT:    fcmgt p0.d, p0/z, z0.d, z1.d
 ; CHECK-NEXT:    mov z0.d, #0xffffffff
@@ -205,14 +205,14 @@ define <vscale x 4 x i32> @test_signed_v4f64_v4i32(<vscale x 4 x double> %f) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov x8, #281474974613504 // =0xffffffe00000
-; CHECK-NEXT:    movprfx z3, z1
-; CHECK-NEXT:    fcvtzu z3.d, p0/m, z1.d
 ; CHECK-NEXT:    movk x8, #16879, lsl #48
-; CHECK-NEXT:    movprfx z4, z0
-; CHECK-NEXT:    fcvtzu z4.d, p0/m, z0.d
 ; CHECK-NEXT:    mov z2.d, x8
 ; CHECK-NEXT:    fcmge p1.d, p0/z, z1.d, #0.0
 ; CHECK-NEXT:    fcmge p2.d, p0/z, z0.d, #0.0
+; CHECK-NEXT:    movprfx z3, z1
+; CHECK-NEXT:    fcvtzu z3.d, p0/m, z1.d
+; CHECK-NEXT:    movprfx z4, z0
+; CHECK-NEXT:    fcvtzu z4.d, p0/m, z0.d
 ; CHECK-NEXT:    fcmgt p3.d, p0/z, z1.d, z2.d
 ; CHECK-NEXT:    not p1.b, p0/z, p1.b
 ; CHECK-NEXT:    not p2.b, p0/z, p2.b
@@ -240,20 +240,20 @@ define <vscale x 8 x i32> @test_signed_v8f64_v8i32(<vscale x 8 x double> %f) {
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov x8, #281474974613504 // =0xffffffe00000
+; CHECK-NEXT:    movk x8, #16879, lsl #48
+; CHECK-NEXT:    mov z4.d, x8
+; CHECK-NEXT:    fcmge p1.d, p0/z, z1.d, #0.0
+; CHECK-NEXT:    fcmge p2.d, p0/z, z0.d, #0.0
+; CHECK-NEXT:    fcmge p3.d, p0/z, z3.d, #0.0
+; CHECK-NEXT:    fcmge p4.d, p0/z, z2.d, #0.0
 ; CHECK-NEXT:    movprfx z5, z1
 ; CHECK-NEXT:    fcvtzu z5.d, p0/m, z1.d
-; CHECK-NEXT:    movk x8, #16879, lsl #48
 ; CHECK-NEXT:    movprfx z6, z0
 ; CHECK-NEXT:    fcvtzu z6.d, p0/m, z0.d
 ; CHECK-NEXT:    movprfx z7, z3
 ; CHECK-NEXT:    fcvtzu z7.d, p0/m, z3.d
-; CHECK-NEXT:    mov z4.d, x8
 ; CHECK-NEXT:    movprfx z24, z2
 ; CHECK-NEXT:    fcvtzu z24.d, p0/m, z2.d
-; CHECK-NEXT:    fcmge p1.d, p0/z, z1.d, #0.0
-; CHECK-NEXT:    fcmge p2.d, p0/z, z0.d, #0.0
-; CHECK-NEXT:    fcmge p3.d, p0/z, z3.d, #0.0
-; CHECK-NEXT:    fcmge p4.d, p0/z, z2.d, #0.0
 ; CHECK-NEXT:    fcmgt p5.d, p0/z, z1.d, z4.d
 ; CHECK-NEXT:    fcmgt p6.d, p0/z, z0.d, z4.d
 ; CHECK-NEXT:    mov z0.d, #0xffffffff
@@ -288,14 +288,14 @@ define <vscale x 4 x i16> @test_signed_v4f64_v4i16(<vscale x 4 x double> %f) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov x8, #281337537757184 // =0xffe000000000
-; CHECK-NEXT:    movprfx z3, z1
-; CHECK-NEXT:    fcvtzu z3.d, p0/m, z1.d
 ; CHECK-NEXT:    movk x8, #16623, lsl #48
-; CHECK-NEXT:    movprfx z4, z0
-; CHECK-NEXT:    fcvtzu z4.d, p0/m, z0.d
 ; CHECK-NEXT:    mov z2.d, x8
 ; CHECK-NEXT:    fcmge p1.d, p0/z, z1.d, #0.0
 ; CHECK-NEXT:    fcmge p2.d, p0/z, z0.d, #0.0
+; CHECK-NEXT:    movprfx z3, z1
+; CHECK-NEXT:    fcvtzu z3.d, p0/m, z1.d
+; CHECK-NEXT:    movprfx z4, z0
+; CHECK-NEXT:    fcvtzu z4.d, p0/m, z0.d
 ; CHECK-NEXT:    fcmgt p3.d, p0/z, z1.d, z2.d
 ; CHECK-NEXT:    not p1.b, p0/z, p1.b
 ; CHECK-NEXT:    not p2.b, p0/z, p2.b
@@ -323,20 +323,20 @@ define <vscale x 8 x i16> @test_signed_v8f64_v8i16(<vscale x 8 x double> %f) {
 ; CHECK-NEXT:    .cfi_offset w29, -16
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov x8, #281337537757184 // =0xffe000000000
+; CHECK-NEXT:    movk x8, #16623, lsl #48
+; CHECK-NEXT:    mov z4.d, x8
+; CHECK-NEXT:    fcmge p1.d, p0/z, z3.d, #0.0
+; CHECK-NEXT:    fcmge p2.d, p0/z, z2.d, #0.0
+; CHECK-NEXT:    fcmge p3.d, p0/z, z1.d, #0.0
+; CHECK-NEXT:    fcmge p4.d, p0/z, z0.d, #0.0
 ; CHECK-NEXT:    movprfx z5, z3
 ; CHECK-NEXT:    fcvtzu z5.d, p0/m, z3.d
-; CHECK-NEXT:    movk x8, #16623, lsl #48
 ; CHECK-NEXT:    movprfx z6, z2
 ; CHECK-NEXT:    fcvtzu z6.d, p0/m, z2.d
 ; CHECK-NEXT:    movprfx z7, z1
 ; CHECK-NEXT:    fcvtzu z7.d, p0/m, z1.d
-; CHECK-NEXT:    mov z4.d, x8
 ; CHECK-NEXT:    movprfx z24, z0
 ; CHECK-NEXT:    fcvtzu z24.d, p0/m, z0.d
-; CHECK-NEXT:    fcmge p1.d, p0/z, z3.d, #0.0
-; CHECK-NEXT:    fcmge p2.d, p0/z, z2.d, #0.0
-; CHECK-NEXT:    fcmge p3.d, p0/z, z1.d, #0.0
-; CHECK-NEXT:    fcmge p4.d, p0/z, z0.d, #0.0
 ; CHECK-NEXT:    fcmgt p5.d, p0/z, z3.d, z4.d
 ; CHECK-NEXT:    fcmgt p6.d, p0/z, z2.d, z4.d
 ; CHECK-NEXT:    mov z2.d, #65535 // =0xffff
@@ -372,10 +372,10 @@ define <vscale x 2 x i64> @test_signed_v2f64_v2i64(<vscale x 2 x double> %f) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov x8, #4895412794951729151 // =0x43efffffffffffff
-; CHECK-NEXT:    movprfx z1, z0
-; CHECK-NEXT:    fcvtzu z1.d, p0/m, z0.d
 ; CHECK-NEXT:    mov z2.d, x8
 ; CHECK-NEXT:    fcmge p1.d, p0/z, z0.d, #0.0
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    fcvtzu z1.d, p0/m, z0.d
 ; CHECK-NEXT:    not p1.b, p0/z, p1.b
 ; CHECK-NEXT:    fcmgt p0.d, p0/z, z0.d, z2.d
 ; CHECK-NEXT:    mov z1.d, p1/m, #0 // =0x0
@@ -391,13 +391,13 @@ define <vscale x 4 x i64> @test_signed_v4f64_v4i64(<vscale x 4 x double> %f) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov x8, #4895412794951729151 // =0x43efffffffffffff
+; CHECK-NEXT:    mov z4.d, x8
+; CHECK-NEXT:    fcmge p1.d, p0/z, z0.d, #0.0
+; CHECK-NEXT:    fcmge p2.d, p0/z, z1.d, #0.0
 ; CHECK-NEXT:    movprfx z2, z0
 ; CHECK-NEXT:    fcvtzu z2.d, p0/m, z0.d
-; CHECK-NEXT:    mov z4.d, x8
 ; CHECK-NEXT:    movprfx z3, z1
 ; CHECK-NEXT:    fcvtzu z3.d, p0/m, z1.d
-; CHECK-NEXT:    fcmge p1.d, p0/z, z0.d, #0.0
-; CHECK-NEXT:    fcmge p2.d, p0/z, z1.d, #0.0
 ; CHECK-NEXT:    fcmgt p3.d, p0/z, z0.d, z4.d
 ; CHECK-NEXT:    not p1.b, p0/z, p1.b
 ; CHECK-NEXT:    not p2.b, p0/z, p2.b
@@ -429,10 +429,10 @@ define <vscale x 2 x i32> @test_signed_v2f16_v2i32(<vscale x 2 x half> %f) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov w8, #31743 // =0x7bff
-; CHECK-NEXT:    movprfx z2, z0
-; CHECK-NEXT:    fcvtzu z2.d, p0/m, z0.h
 ; CHECK-NEXT:    mov z1.h, w8
 ; CHECK-NEXT:    fcmge p1.h, p0/z, z0.h, #0.0
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    fcvtzu z2.d, p0/m, z0.h
 ; CHECK-NEXT:    not p1.b, p0/z, p1.b
 ; CHECK-NEXT:    fcmgt p0.h, p0/z, z0.h, z1.h
 ; CHECK-NEXT:    mov z0.d, #0xffffffff
@@ -448,10 +448,10 @@ define <vscale x 4 x i32> @test_signed_v4f16_v4i32(<vscale x 4 x half> %f) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    mov w8, #31743 // =0x7bff
-; CHECK-NEXT:    movprfx z1, z0
-; CHECK-NEXT:    fcvtzu z1.s, p0/m, z0.h
 ; CHECK-NEXT:    mov z2.h, w8
 ; CHECK-NEXT:    fcmge p1.h, p0/z, z0.h, #0.0
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    fcvtzu z1.s, p0/m, z0.h
 ; CHECK-NEXT:    not p1.b, p0/z, p1.b
 ; CHECK-NEXT:    fcmgt p0.h, p0/z, z0.h, z2.h
 ; CHECK-NEXT:    mov z1.s, p1/m, #0 // =0x0
@@ -469,13 +469,13 @@ define <vscale x 8 x i32> @test_signed_v8f16_v8i32(<vscale x 8 x half> %f) {
 ; CHECK-NEXT:    uunpklo z2.s, z0.h
 ; CHECK-NEXT:    uunpkhi z3.s, z0.h
 ; CHECK-NEXT:    mov w8, #31743 // =0x7bff
+; CHECK-NEXT:    mov z4.h, w8
+; CHECK-NEXT:    fcmge p1.h, p0/z, z2.h, #0.0
+; CHECK-NEXT:    fcmge p2.h, p0/z, z3.h, #0.0
 ; CHECK-NEXT:    movprfx z0, z2
 ; CHECK-NEXT:    fcvtzu z0.s, p0/m, z2.h
 ; CHECK-NEXT:    movprfx z1, z3
 ; CHECK-NEXT:    fcvtzu z1.s, p0/m, z3.h
-; CHECK-NEXT:    mov z4.h, w8
-; CHECK-NEXT:    fcmge p1.h, p0/z, z2.h, #0.0
-; CHECK-NEXT:    fcmge p2.h, p0/z, z3.h, #0.0
 ; CHECK-NEXT:    fcmgt p3.h, p0/z, z2.h, z4.h
 ; CHECK-NEXT:    not p1.b, p0/z, p1.b
 ; CHECK-NEXT:    not p2.b, p0/z, p2.b
@@ -494,10 +494,10 @@ define <vscale x 4 x i16> @test_signed_v4f16_v4i16(<vscale x 4 x half> %f) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    mov w8, #31743 // =0x7bff
-; CHECK-NEXT:    movprfx z2, z0
-; CHECK-NEXT:    fcvtzu z2.s, p0/m, z0.h
 ; CHECK-NEXT:    mov z1.h, w8
 ; CHECK-NEXT:    fcmge p1.h, p0/z, z0.h, #0.0
+; CHECK-NEXT:    movprfx z2, z0
+; CHECK-NEXT:    fcvtzu z2.s, p0/m, z0.h
 ; CHECK-NEXT:    not p1.b, p0/z, p1.b
 ; CHECK-NEXT:    fcmgt p0.h, p0/z, z0.h, z1.h
 ; CHECK-NEXT:    mov z0.s, #65535 // =0xffff
@@ -513,10 +513,10 @@ define <vscale x 8 x i16> @test_signed_v8f16_v8i16(<vscale x 8 x half> %f) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    mov w8, #31743 // =0x7bff
-; CHECK-NEXT:    movprfx z1, z0
-; CHECK-NEXT:    fcvtzu z1.h, p0/m, z0.h
 ; CHECK-NEXT:    mov z2.h, w8
 ; CHECK-NEXT:    fcmge p1.h, p0/z, z0.h, #0.0
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    fcvtzu z1.h, p0/m, z0.h
 ; CHECK-NEXT:    not p1.b, p0/z, p1.b
 ; CHECK-NEXT:    fcmgt p0.h, p0/z, z0.h, z2.h
 ; CHECK-NEXT:    mov z1.h, p1/m, #0 // =0x0
@@ -532,10 +532,10 @@ define <vscale x 2 x i64> @test_signed_v2f16_v2i64(<vscale x 2 x half> %f) {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    mov w8, #31743 // =0x7bff
-; CHECK-NEXT:    movprfx z1, z0
-; CHECK-NEXT:    fcvtzu z1.d, p0/m, z0.h
 ; CHECK-NEXT:    mov z2.h, w8
 ; CHECK-NEXT:    fcmge p1.h, p0/z, z0.h, #0.0
+; CHECK-NEXT:    movprfx z1, z0
+; CHECK-NEXT:    fcvtzu z1.d, p0/m, z0.h
 ; CHECK-NEXT:    not p1.b, p0/z, p1.b
 ; CHECK-NEXT:    fcmgt p0.h, p0/z, z0.h, z2.h
 ; CHECK-NEXT:    mov z1.d, p1/m, #0 // =0x0
@@ -553,13 +553,13 @@ define <vscale x 4 x i64> @test_signed_v4f16_v4i64(<vscale x 4 x half> %f) {
 ; CHECK-NEXT:    uunpklo z2.d, z0.s
 ; CHECK-NEXT:    uunpkhi z3.d, z0.s
 ; CHECK-NEXT:    mov w8, #31743 // =0x7bff
+; CHECK-NEXT:    mov z4.h, w8
+; CHECK-NEXT:    fcmge p1.h, p0/z, z2.h, #0.0
+; CHECK-NEXT:    fcmge p2.h, p0/z, z3.h, #0.0
 ; CHECK-NEXT:    movprfx z0, z2
 ; CHECK-NEXT:    fcvtzu z0.d, p0/m, z2.h
 ; CHECK-NEXT:    movprfx z1, z3
 ; CHECK-NEXT:    fcvtzu z1.d, p0/m, z3.h
-; CHECK-NEXT:    mov z4.h, w8
-; CHECK-NEXT:    fcmge p1.h, p0/z, z2.h, #0.0
-; CHECK-NEXT:    fcmge p2.h, p0/z, z3.h, #0.0
 ; CHECK-NEXT:    fcmgt p3.h, p0/z, z2.h, z4.h
 ; CHECK-NEXT:    not p1.b, p0/z, p1.b
 ; CHECK-NEXT:    not p2.b, p0/z, p2.b
diff --git a/llvm/test/CodeGen/AArch64/sve-gather-scatter-addr-opts.ll b/llvm/test/CodeGen/AArch64/sve-gather-scatter-addr-opts.ll
index 1a6e4814cee6..5c4c9463528b 100644
--- a/llvm/test/CodeGen/AArch64/sve-gather-scatter-addr-opts.ll
+++ b/llvm/test/CodeGen/AArch64/sve-gather-scatter-addr-opts.ll
@@ -71,18 +71,18 @@ define <vscale x 4 x i8> @gather_i8_index_offset_8(ptr %base, i64 %offset, <vsca
 define void @scatter_f16_index_offset_var(ptr %base, i64 %offset, i64 %scale, <vscale x 4 x i1> %pg, <vscale x 4 x half> %data) #0 {
 ; CHECK-LABEL: scatter_f16_index_offset_var:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    index z1.d, #0, #1
 ; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    index z1.d, #0, #1
 ; CHECK-NEXT:    mov z2.d, x1
-; CHECK-NEXT:    movprfx z4, z2
-; CHECK-NEXT:    mla z4.d, p1/m, z1.d, z2.d
 ; CHECK-NEXT:    punpklo p2.h, p0.b
 ; CHECK-NEXT:    uunpklo z3.d, z0.s
-; CHECK-NEXT:    punpkhi p0.h, p0.b
 ; CHECK-NEXT:    uunpkhi z0.d, z0.s
+; CHECK-NEXT:    punpkhi p0.h, p0.b
+; CHECK-NEXT:    movprfx z4, z2
+; CHECK-NEXT:    mla z4.d, p1/m, z1.d, z2.d
 ; CHECK-NEXT:    incd z1.d
-; CHECK-NEXT:    st1h { z3.d }, p2, [x0, z4.d, lsl #1]
 ; CHECK-NEXT:    mad z1.d, p1/m, z2.d, z2.d
+; CHECK-NEXT:    st1h { z3.d }, p2, [x0, z4.d, lsl #1]
 ; CHECK-NEXT:    st1h { z0.d }, p0, [x0, z1.d, lsl #1]
 ; CHECK-NEXT:    ret
   %t0 = insertelement <vscale x 4 x i64> undef, i64 %offset, i32 0
diff --git a/llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll b/llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll
index 3ecfa7e6c0b1..e40d65efb158 100644
--- a/llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll
+++ b/llvm/test/CodeGen/AArch64/sve-masked-gather-legalize.ll
@@ -145,10 +145,10 @@ define <vscale x 4 x double> @masked_gather_nxv4f64(ptr %base, <vscale x 4 x i16
 ; CHECK-NEXT:    ptrue p1.s
 ; CHECK-NEXT:    movprfx z1, z0
 ; CHECK-NEXT:    sxth z1.s, p1/m, z0.s
-; CHECK-NEXT:    sunpklo z0.d, z1.s
 ; CHECK-NEXT:    punpklo p1.h, p0.b
-; CHECK-NEXT:    sunpkhi z1.d, z1.s
 ; CHECK-NEXT:    punpkhi p0.h, p0.b
+; CHECK-NEXT:    sunpklo z0.d, z1.s
+; CHECK-NEXT:    sunpkhi z1.d, z1.s
 ; CHECK-NEXT:    ld1d { z0.d }, p1/z, [x0, z0.d, lsl #3]
 ; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x0, z1.d, lsl #3]
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/sve-smulo-sdnode.ll b/llvm/test/CodeGen/AArch64/sve-smulo-sdnode.ll
index 871c74a1de14..a2429d4975b7 100644
--- a/llvm/test/CodeGen/AArch64/sve-smulo-sdnode.ll
+++ b/llvm/test/CodeGen/AArch64/sve-smulo-sdnode.ll
@@ -129,9 +129,9 @@ define <vscale x 64 x i8> @smulo_nxv64i8(<vscale x 64 x i8> %x, <vscale x 64 x i
 ; CHECK-NEXT:    mul z27.b, p0/m, z27.b, z5.b
 ; CHECK-NEXT:    smulh z3.b, p0/m, z3.b, z7.b
 ; CHECK-NEXT:    smulh z0.b, p0/m, z0.b, z4.b
-; CHECK-NEXT:    asr z4.b, z25.b, #7
 ; CHECK-NEXT:    smulh z2.b, p0/m, z2.b, z6.b
 ; CHECK-NEXT:    smulh z1.b, p0/m, z1.b, z5.b
+; CHECK-NEXT:    asr z4.b, z25.b, #7
 ; CHECK-NEXT:    asr z5.b, z24.b, #7
 ; CHECK-NEXT:    asr z6.b, z26.b, #7
 ; CHECK-NEXT:    asr z7.b, z27.b, #7
@@ -140,13 +140,13 @@ define <vscale x 64 x i8> @smulo_nxv64i8(<vscale x 64 x i8> %x, <vscale x 64 x i
 ; CHECK-NEXT:    cmpne p3.b, p0/z, z2.b, z6.b
 ; CHECK-NEXT:    cmpne p0.b, p0/z, z1.b, z7.b
 ; CHECK-NEXT:    mov z25.b, p1/m, #0 // =0x0
+; CHECK-NEXT:    mov z26.b, p3/m, #0 // =0x0
 ; CHECK-NEXT:    mov z24.b, p2/m, #0 // =0x0
 ; CHECK-NEXT:    mov z27.b, p0/m, #0 // =0x0
-; CHECK-NEXT:    mov z26.b, p3/m, #0 // =0x0
 ; CHECK-NEXT:    mov z0.d, z25.d
+; CHECK-NEXT:    mov z2.d, z26.d
 ; CHECK-NEXT:    mov z3.d, z24.d
 ; CHECK-NEXT:    mov z1.d, z27.d
-; CHECK-NEXT:    mov z2.d, z26.d
 ; CHECK-NEXT:    ret
   %a = call { <vscale x 64 x i8>, <vscale x 64 x i1> } @llvm.smul.with.overflow.nxv64i8(<vscale x 64 x i8> %x, <vscale x 64 x i8> %y)
   %b = extractvalue { <vscale x 64 x i8>, <vscale x 64 x i1> } %a, 0
@@ -262,9 +262,9 @@ define <vscale x 32 x i16> @smulo_nxv32i16(<vscale x 32 x i16> %x, <vscale x 32
 ; CHECK-NEXT:    mul z27.h, p0/m, z27.h, z5.h
 ; CHECK-NEXT:    smulh z3.h, p0/m, z3.h, z7.h
 ; CHECK-NEXT:    smulh z0.h, p0/m, z0.h, z4.h
-; CHECK-NEXT:    asr z4.h, z25.h, #15
 ; CHECK-NEXT:    smulh z2.h, p0/m, z2.h, z6.h
 ; CHECK-NEXT:    smulh z1.h, p0/m, z1.h, z5.h
+; CHECK-NEXT:    asr z4.h, z25.h, #15
 ; CHECK-NEXT:    asr z5.h, z24.h, #15
 ; CHECK-NEXT:    asr z6.h, z26.h, #15
 ; CHECK-NEXT:    asr z7.h, z27.h, #15
@@ -273,13 +273,13 @@ define <vscale x 32 x i16> @smulo_nxv32i16(<vscale x 32 x i16> %x, <vscale x 32
 ; CHECK-NEXT:    cmpne p3.h, p0/z, z2.h, z6.h
 ; CHECK-NEXT:    cmpne p0.h, p0/z, z1.h, z7.h
 ; CHECK-NEXT:    mov z25.h, p1/m, #0 // =0x0
+; CHECK-NEXT:    mov z26.h, p3/m, #0 // =0x0
 ; CHECK-NEXT:    mov z24.h, p2/m, #0 // =0x0
 ; CHECK-NEXT:    mov z27.h, p0/m, #0 // =0x0
-; CHECK-NEXT:    mov z26.h, p3/m, #0 // =0x0
 ; CHECK-NEXT:    mov z0.d, z25.d
+; CHECK-NEXT:    mov z2.d, z26.d
 ; CHECK-NEXT:    mov z3.d, z24.d
 ; CHECK-NEXT:    mov z1.d, z27.d
-; CHECK-NEXT:    mov z2.d, z26.d
 ; CHECK-NEXT:    ret
   %a = call { <vscale x 32 x i16>, <vscale x 32 x i1> } @llvm.smul.with.overflow.nxv32i16(<vscale x 32 x i16> %x, <vscale x 32 x i16> %y)
   %b = extractvalue { <vscale x 32 x i16>, <vscale x 32 x i1> } %a, 0
@@ -374,9 +374,9 @@ define <vscale x 16 x i32> @smulo_nxv16i32(<vscale x 16 x i32> %x, <vscale x 16
 ; CHECK-NEXT:    mul z27.s, p0/m, z27.s, z5.s
 ; CHECK-NEXT:    smulh z3.s, p0/m, z3.s, z7.s
 ; CHECK-NEXT:    smulh z0.s, p0/m, z0.s, z4.s
-; CHECK-NEXT:    asr z4.s, z25.s, #31
 ; CHECK-NEXT:    smulh z2.s, p0/m, z2.s, z6.s
 ; CHECK-NEXT:    smulh z1.s, p0/m, z1.s, z5.s
+; CHECK-NEXT:    asr z4.s, z25.s, #31
 ; CHECK-NEXT:    asr z5.s, z24.s, #31
 ; CHECK-NEXT:    asr z6.s, z26.s, #31
 ; CHECK-NEXT:    asr z7.s, z27.s, #31
@@ -385,13 +385,13 @@ define <vscale x 16 x i32> @smulo_nxv16i32(<vscale x 16 x i32> %x, <vscale x 16
 ; CHECK-NEXT:    cmpne p3.s, p0/z, z2.s, z6.s
 ; CHECK-NEXT:    cmpne p0.s, p0/z, z1.s, z7.s
 ; CHECK-NEXT:    mov z25.s, p1/m, #0 // =0x0
+; CHECK-NEXT:    mov z26.s, p3/m, #0 // =0x0
 ; CHECK-NEXT:    mov z24.s, p2/m, #0 // =0x0
 ; CHECK-NEXT:    mov z27.s, p0/m, #0 // =0x0
-; CHECK-NEXT:    mov z26.s, p3/m, #0 // =0x0
 ; CHECK-NEXT:    mov z0.d, z25.d
+; CHECK-NEXT:    mov z2.d, z26.d
 ; CHECK-NEXT:    mov z3.d, z24.d
 ; CHECK-NEXT:    mov z1.d, z27.d
-; CHECK-NEXT:    mov z2.d, z26.d
 ; CHECK-NEXT:    ret
   %a = call { <vscale x 16 x i32>, <vscale x 16 x i1> } @llvm.smul.with.overflow.nxv16i32(<vscale x 16 x i32> %x, <vscale x 16 x i32> %y)
   %b = extractvalue { <vscale x 16 x i32>, <vscale x 16 x i1> } %a, 0
@@ -465,9 +465,9 @@ define <vscale x 8 x i64> @smulo_nxv8i64(<vscale x 8 x i64> %x, <vscale x 8 x i6
 ; CHECK-NEXT:    mul z27.d, p0/m, z27.d, z5.d
 ; CHECK-NEXT:    smulh z3.d, p0/m, z3.d, z7.d
 ; CHECK-NEXT:    smulh z0.d, p0/m, z0.d, z4.d
-; CHECK-NEXT:    asr z4.d, z25.d, #63
 ; CHECK-NEXT:    smulh z2.d, p0/m, z2.d, z6.d
 ; CHECK-NEXT:    smulh z1.d, p0/m, z1.d, z5.d
+; CHECK-NEXT:    asr z4.d, z25.d, #63
 ; CHECK-NEXT:    asr z5.d, z24.d, #63
 ; CHECK-NEXT:    asr z6.d, z26.d, #63
 ; CHECK-NEXT:    asr z7.d, z27.d, #63
@@ -476,13 +476,13 @@ define <vscale x 8 x i64> @smulo_nxv8i64(<vscale x 8 x i64> %x, <vscale x 8 x i6
 ; CHECK-NEXT:    cmpne p3.d, p0/z, z2.d, z6.d
 ; CHECK-NEXT:    cmpne p0.d, p0/z, z1.d, z7.d
 ; CHECK-NEXT:    mov z25.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    mov z26.d, p3/m, #0 // =0x0
 ; CHECK-NEXT:    mov z24.d, p2/m, #0 // =0x0
 ; CHECK-NEXT:    mov z27.d, p0/m, #0 // =0x0
-; CHECK-NEXT:    mov z26.d, p3/m, #0 // =0x0
 ; CHECK-NEXT:    mov z0.d, z25.d
+; CHECK-NEXT:    mov z2.d, z26.d
 ; CHECK-NEXT:    mov z3.d, z24.d
 ; CHECK-NEXT:    mov z1.d, z27.d
-; CHECK-NEXT:    mov z2.d, z26.d
 ; CHECK-NEXT:    ret
   %a = call { <vscale x 8 x i64>, <vscale x 8 x i1> } @llvm.smul.with.overflow.nxv8i64(<vscale x 8 x i64> %x, <vscale x 8 x i64> %y)
   %b = extractvalue { <vscale x 8 x i64>, <vscale x 8 x i1> } %a, 0
diff --git a/llvm/test/CodeGen/AArch64/sve-split-fcvt.ll b/llvm/test/CodeGen/AArch64/sve-split-fcvt.ll
index 44796606e7a1..3997409172d0 100644
--- a/llvm/test/CodeGen/AArch64/sve-split-fcvt.ll
+++ b/llvm/test/CodeGen/AArch64/sve-split-fcvt.ll
@@ -6,8 +6,8 @@
 define <vscale x 8 x float> @fcvts_nxv8f16(<vscale x 8 x half> %a) {
 ; CHECK-LABEL: fcvts_nxv8f16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    uunpklo z1.s, z0.h
 ; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    uunpklo z1.s, z0.h
 ; CHECK-NEXT:    uunpkhi z2.s, z0.h
 ; CHECK-NEXT:    movprfx z0, z1
 ; CHECK-NEXT:    fcvt z0.s, p0/m, z1.h
@@ -21,8 +21,8 @@ define <vscale x 8 x float> @fcvts_nxv8f16(<vscale x 8 x half> %a) {
 define <vscale x 4 x double> @fcvtd_nxv4f16(<vscale x 4 x half> %a) {
 ; CHECK-LABEL: fcvtd_nxv4f16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    uunpklo z1.d, z0.s
 ; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    uunpklo z1.d, z0.s
 ; CHECK-NEXT:    uunpkhi z2.d, z0.s
 ; CHECK-NEXT:    movprfx z0, z1
 ; CHECK-NEXT:    fcvt z0.d, p0/m, z1.h
@@ -43,13 +43,13 @@ define <vscale x 8 x double> @fcvtd_nxv8f16(<vscale x 8 x half> %a) {
 ; CHECK-NEXT:    uunpkhi z1.d, z1.s
 ; CHECK-NEXT:    uunpklo z3.d, z0.s
 ; CHECK-NEXT:    uunpkhi z4.d, z0.s
+; CHECK-NEXT:    fcvt z1.d, p0/m, z1.h
 ; CHECK-NEXT:    movprfx z0, z2
 ; CHECK-NEXT:    fcvt z0.d, p0/m, z2.h
 ; CHECK-NEXT:    movprfx z2, z3
 ; CHECK-NEXT:    fcvt z2.d, p0/m, z3.h
 ; CHECK-NEXT:    movprfx z3, z4
 ; CHECK-NEXT:    fcvt z3.d, p0/m, z4.h
-; CHECK-NEXT:    fcvt z1.d, p0/m, z1.h
 ; CHECK-NEXT:    ret
   %res = fpext <vscale x 8 x half> %a to <vscale x 8 x double>
   ret <vscale x 8 x double> %res
@@ -58,8 +58,8 @@ define <vscale x 8 x double> @fcvtd_nxv8f16(<vscale x 8 x half> %a) {
 define <vscale x 4 x double> @fcvtd_nxv4f32(<vscale x 4 x float> %a) {
 ; CHECK-LABEL: fcvtd_nxv4f32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    uunpklo z1.d, z0.s
 ; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    uunpklo z1.d, z0.s
 ; CHECK-NEXT:    uunpkhi z2.d, z0.s
 ; CHECK-NEXT:    movprfx z0, z1
 ; CHECK-NEXT:    fcvt z0.d, p0/m, z1.s
@@ -73,17 +73,17 @@ define <vscale x 4 x double> @fcvtd_nxv4f32(<vscale x 4 x float> %a) {
 define <vscale x 8 x double> @fcvtd_nxv8f32(<vscale x 8 x float> %a) {
 ; CHECK-LABEL: fcvtd_nxv8f32:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    uunpklo z2.d, z0.s
 ; CHECK-NEXT:    uunpkhi z3.d, z0.s
-; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    uunpklo z4.d, z1.s
+; CHECK-NEXT:    uunpkhi z5.d, z1.s
 ; CHECK-NEXT:    movprfx z0, z2
 ; CHECK-NEXT:    fcvt z0.d, p0/m, z2.s
-; CHECK-NEXT:    movprfx z2, z4
-; CHECK-NEXT:    fcvt z2.d, p0/m, z4.s
-; CHECK-NEXT:    uunpkhi z5.d, z1.s
 ; CHECK-NEXT:    movprfx z1, z3
 ; CHECK-NEXT:    fcvt z1.d, p0/m, z3.s
+; CHECK-NEXT:    movprfx z2, z4
+; CHECK-NEXT:    fcvt z2.d, p0/m, z4.s
 ; CHECK-NEXT:    movprfx z3, z5
 ; CHECK-NEXT:    fcvt z3.d, p0/m, z5.s
 ; CHECK-NEXT:    ret
@@ -195,8 +195,8 @@ define <vscale x 8 x i16> @fcvtzs_h_nxv8f64(<vscale x 8 x double> %a) {
 define <vscale x 4 x i64> @fcvtzs_d_nxv4f32(<vscale x 4 x float> %a) {
 ; CHECK-LABEL: fcvtzs_d_nxv4f32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    uunpklo z1.d, z0.s
 ; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    uunpklo z1.d, z0.s
 ; CHECK-NEXT:    uunpkhi z2.d, z0.s
 ; CHECK-NEXT:    movprfx z0, z1
 ; CHECK-NEXT:    fcvtzs z0.d, p0/m, z1.s
@@ -210,17 +210,17 @@ define <vscale x 4 x i64> @fcvtzs_d_nxv4f32(<vscale x 4 x float> %a) {
 define <vscale x 16 x i32> @fcvtzs_s_nxv16f16(<vscale x 16 x half> %a) {
 ; CHECK-LABEL: fcvtzs_s_nxv16f16:
 ; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    uunpklo z2.s, z0.h
 ; CHECK-NEXT:    uunpkhi z3.s, z0.h
-; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    uunpklo z4.s, z1.h
+; CHECK-NEXT:    uunpkhi z5.s, z1.h
 ; CHECK-NEXT:    movprfx z0, z2
 ; CHECK-NEXT:    fcvtzs z0.s, p0/m, z2.h
-; CHECK-NEXT:    movprfx z2, z4
-; CHECK-NEXT:    fcvtzs z2.s, p0/m, z4.h
-; CHECK-NEXT:    uunpkhi z5.s, z1.h
 ; CHECK-NEXT:    movprfx z1, z3
 ; CHECK-NEXT:    fcvtzs z1.s, p0/m, z3.h
+; CHECK-NEXT:    movprfx z2, z4
+; CHECK-NEXT:    fcvtzs z2.s, p0/m, z4.h
 ; CHECK-NEXT:    movprfx z3, z5
 ; CHECK-NEXT:    fcvtzs z3.s, p0/m, z5.h
 ; CHECK-NEXT:    ret
@@ -247,8 +247,8 @@ define <vscale x 4 x i32> @fcvtzu_s_nxv4f64(<vscale x 4 x double> %a) {
 define <vscale x 4 x i64> @fcvtzu_d_nxv4f32(<vscale x 4 x float> %a) {
 ; CHECK-LABEL: fcvtzu_d_nxv4f32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    uunpklo z1.d, z0.s
 ; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    uunpklo z1.d, z0.s
 ; CHECK-NEXT:    uunpkhi z2.d, z0.s
 ; CHECK-NEXT:    movprfx z0, z1
 ; CHECK-NEXT:    fcvtzu z0.d, p0/m, z1.s
@@ -301,13 +301,13 @@ define <vscale x 16 x float> @scvtf_s_nxv16i8(<vscale x 16 x i8> %a) {
 ; CHECK-NEXT:    sunpkhi z1.s, z1.h
 ; CHECK-NEXT:    sunpklo z3.s, z0.h
 ; CHECK-NEXT:    sunpkhi z4.s, z0.h
+; CHECK-NEXT:    scvtf z1.s, p0/m, z1.s
 ; CHECK-NEXT:    movprfx z0, z2
 ; CHECK-NEXT:    scvtf z0.s, p0/m, z2.s
 ; CHECK-NEXT:    movprfx z2, z3
 ; CHECK-NEXT:    scvtf z2.s, p0/m, z3.s
 ; CHECK-NEXT:    movprfx z3, z4
 ; CHECK-NEXT:    scvtf z3.s, p0/m, z4.s
-; CHECK-NEXT:    scvtf z1.s, p0/m, z1.s
 ; CHECK-NEXT:    ret
   %res = sitofp <vscale x 16 x i8> %a to <vscale x 16 x float>
   ret <vscale x 16 x float> %res
@@ -316,8 +316,8 @@ define <vscale x 16 x float> @scvtf_s_nxv16i8(<vscale x 16 x i8> %a) {
 define <vscale x 4 x double> @scvtf_d_nxv4i32(<vscale x 4 x i32> %a) {
 ; CHECK-LABEL: scvtf_d_nxv4i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sunpklo z1.d, z0.s
 ; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    sunpklo z1.d, z0.s
 ; CHECK-NEXT:    sunpkhi z2.d, z0.s
 ; CHECK-NEXT:    movprfx z0, z1
 ; CHECK-NEXT:    scvtf z0.d, p0/m, z1.d
@@ -378,8 +378,8 @@ define <vscale x 8 x half> @ucvtf_h_nxv8i64(<vscale x 8 x i64> %a) {
 define <vscale x 4 x double> @ucvtf_d_nxv4i32(<vscale x 4 x i32> %a) {
 ; CHECK-LABEL: ucvtf_d_nxv4i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    uunpklo z1.d, z0.s
 ; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    uunpklo z1.d, z0.s
 ; CHECK-NEXT:    uunpkhi z2.d, z0.s
 ; CHECK-NEXT:    movprfx z0, z1
 ; CHECK-NEXT:    ucvtf z0.d, p0/m, z1.d
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-extend-trunc.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-extend-trunc.ll
index 37cc79011a98..28e02da53af4 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-extend-trunc.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-fp-extend-trunc.ll
@@ -239,10 +239,10 @@ define void @fcvt_v16f16_v16f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    mov x8, #4 // =0x4
 ; CHECK-NEXT:    fcvt z0.d, p0/m, z0.h
 ; CHECK-NEXT:    ld1h { z4.d }, p0/z, [x0, x8, lsl #1]
-; CHECK-NEXT:    fcvt z1.d, p0/m, z1.h
 ; CHECK-NEXT:    mov x8, #6 // =0x6
-; CHECK-NEXT:    fcvt z2.d, p0/m, z2.h
+; CHECK-NEXT:    fcvt z1.d, p0/m, z1.h
 ; CHECK-NEXT:    ld1h { z5.d }, p0/z, [x0, x8, lsl #1]
+; CHECK-NEXT:    fcvt z2.d, p0/m, z2.h
 ; CHECK-NEXT:    mov x8, #2 // =0x2
 ; CHECK-NEXT:    fcvt z3.d, p0/m, z3.h
 ; CHECK-NEXT:    ld1h { z7.d }, p0/z, [x0, x8, lsl #1]
@@ -255,8 +255,8 @@ define void @fcvt_v16f16_v16f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    stp q2, q3, [x1, #64]
 ; CHECK-NEXT:    movprfx z2, z7
 ; CHECK-NEXT:    fcvt z2.d, p0/m, z7.h
-; CHECK-NEXT:    stp q1, q2, [x1]
 ; CHECK-NEXT:    stp q4, q0, [x1, #32]
+; CHECK-NEXT:    stp q1, q2, [x1]
 ; CHECK-NEXT:    ret
   %op1 = load <16 x half>, ptr %a
   %res = fpext <16 x half> %op1 to <16 x double>
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-div.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-div.ll
index 5e3ce0ddebe2..fcf4f21c6ea8 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-div.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-div.ll
@@ -750,14 +750,14 @@ define void @udiv_constantsplat_v8i32(ptr %a)  {
 ; SVE-NEXT:    mov z0.s, w8
 ; SVE-NEXT:    movprfx z3, z1
 ; SVE-NEXT:    umulh z3.s, p0/m, z3.s, z0.s
-; SVE-NEXT:    sub z1.s, z1.s, z3.s
 ; SVE-NEXT:    umulh z0.s, p0/m, z0.s, z2.s
-; SVE-NEXT:    lsr z1.s, z1.s, #1
+; SVE-NEXT:    sub z1.s, z1.s, z3.s
 ; SVE-NEXT:    sub z2.s, z2.s, z0.s
-; SVE-NEXT:    add z1.s, z1.s, z3.s
+; SVE-NEXT:    lsr z1.s, z1.s, #1
 ; SVE-NEXT:    lsr z2.s, z2.s, #1
-; SVE-NEXT:    lsr z1.s, z1.s, #6
+; SVE-NEXT:    add z1.s, z1.s, z3.s
 ; SVE-NEXT:    add z0.s, z2.s, z0.s
+; SVE-NEXT:    lsr z1.s, z1.s, #6
 ; SVE-NEXT:    lsr z0.s, z0.s, #6
 ; SVE-NEXT:    stp q1, q0, [x0]
 ; SVE-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-rem.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-rem.ll
index eb95a410209b..4a1209b942f4 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-rem.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-rem.ll
@@ -139,7 +139,6 @@ define void @srem_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    sdiv z5.s, p0/m, z5.s, z4.s
 ; CHECK-NEXT:    ldr q3, [x0]
 ; CHECK-NEXT:    ldr q4, [x1]
-; CHECK-NEXT:    uzp1 z5.h, z5.h, z5.h
 ; CHECK-NEXT:    mov z18.d, z3.d
 ; CHECK-NEXT:    mov z17.d, z4.d
 ; CHECK-NEXT:    uzp1 z6.h, z6.h, z6.h
@@ -154,6 +153,7 @@ define void @srem_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    sdivr z7.s, p0/m, z7.s, z16.s
 ; CHECK-NEXT:    sunpklo z18.s, z18.h
 ; CHECK-NEXT:    uzp1 z2.h, z2.h, z2.h
+; CHECK-NEXT:    uzp1 z5.h, z5.h, z5.h
 ; CHECK-NEXT:    sunpklo z17.s, z17.h
 ; CHECK-NEXT:    sdivr z19.s, p0/m, z19.s, z20.s
 ; CHECK-NEXT:    sunpklo z20.h, z3.b
@@ -172,21 +172,21 @@ define void @srem_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    sdivr z18.s, p0/m, z18.s, z20.s
 ; CHECK-NEXT:    ptrue p0.h, vl4
 ; CHECK-NEXT:    uzp1 z19.h, z21.h, z21.h
+; CHECK-NEXT:    splice z16.h, p0, z16.h, z17.h
 ; CHECK-NEXT:    splice z2.h, p0, z2.h, z5.h
 ; CHECK-NEXT:    splice z6.h, p0, z6.h, z7.h
-; CHECK-NEXT:    splice z16.h, p0, z16.h, z17.h
+; CHECK-NEXT:    uzp1 z5.b, z16.b, z16.b
 ; CHECK-NEXT:    uzp1 z2.b, z2.b, z2.b
 ; CHECK-NEXT:    uzp1 z6.b, z6.b, z6.b
-; CHECK-NEXT:    uzp1 z5.b, z16.b, z16.b
 ; CHECK-NEXT:    uzp1 z18.h, z18.h, z18.h
 ; CHECK-NEXT:    splice z19.h, p0, z19.h, z18.h
 ; CHECK-NEXT:    ptrue p0.b, vl8
 ; CHECK-NEXT:    uzp1 z7.b, z19.b, z19.b
 ; CHECK-NEXT:    splice z6.b, p0, z6.b, z2.b
 ; CHECK-NEXT:    splice z7.b, p0, z7.b, z5.b
+; CHECK-NEXT:    mls z0.b, p1/m, z6.b, z1.b
 ; CHECK-NEXT:    movprfx z2, z3
 ; CHECK-NEXT:    mls z2.b, p1/m, z7.b, z4.b
-; CHECK-NEXT:    mls z0.b, p1/m, z6.b, z1.b
 ; CHECK-NEXT:    stp q2, q0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
@@ -293,8 +293,8 @@ define <2 x i32> @srem_v2i32(<2 x i32> %op1, <2 x i32> %op2) {
 ; CHECK-LABEL: srem_v2i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s, vl2
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    movprfx z2, z0
 ; CHECK-NEXT:    sdiv z2.s, p0/m, z2.s, z1.s
 ; CHECK-NEXT:    mls z0.s, p0/m, z2.s, z1.s
@@ -308,8 +308,8 @@ define <4 x i32> @srem_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
 ; CHECK-LABEL: srem_v4i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s, vl4
-; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    movprfx z2, z0
 ; CHECK-NEXT:    sdiv z2.s, p0/m, z2.s, z1.s
 ; CHECK-NEXT:    mls z0.s, p0/m, z2.s, z1.s
@@ -345,8 +345,8 @@ define <1 x i64> @srem_v1i64(<1 x i64> %op1, <1 x i64> %op2) {
 ; CHECK-LABEL: srem_v1i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d, vl1
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    movprfx z2, z0
 ; CHECK-NEXT:    sdiv z2.d, p0/m, z2.d, z1.d
 ; CHECK-NEXT:    mls z0.d, p0/m, z2.d, z1.d
@@ -360,8 +360,8 @@ define <2 x i64> @srem_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
 ; CHECK-LABEL: srem_v2i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d, vl2
-; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    movprfx z2, z0
 ; CHECK-NEXT:    sdiv z2.d, p0/m, z2.d, z1.d
 ; CHECK-NEXT:    mls z0.d, p0/m, z2.d, z1.d
@@ -528,7 +528,6 @@ define void @urem_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    udiv z5.s, p0/m, z5.s, z4.s
 ; CHECK-NEXT:    ldr q3, [x0]
 ; CHECK-NEXT:    ldr q4, [x1]
-; CHECK-NEXT:    uzp1 z5.h, z5.h, z5.h
 ; CHECK-NEXT:    mov z18.d, z3.d
 ; CHECK-NEXT:    mov z17.d, z4.d
 ; CHECK-NEXT:    uzp1 z6.h, z6.h, z6.h
@@ -543,6 +542,7 @@ define void @urem_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    udivr z7.s, p0/m, z7.s, z16.s
 ; CHECK-NEXT:    uunpklo z18.s, z18.h
 ; CHECK-NEXT:    uzp1 z2.h, z2.h, z2.h
+; CHECK-NEXT:    uzp1 z5.h, z5.h, z5.h
 ; CHECK-NEXT:    uunpklo z17.s, z17.h
 ; CHECK-NEXT:    udivr z19.s, p0/m, z19.s, z20.s
 ; CHECK-NEXT:    uunpklo z20.h, z3.b
@@ -561,21 +561,21 @@ define void @urem_v32i8(ptr %a, ptr %b) {
 ; CHECK-NEXT:    udivr z18.s, p0/m, z18.s, z20.s
 ; CHECK-NEXT:    ptrue p0.h, vl4
 ; CHECK-NEXT:    uzp1 z19.h, z21.h, z21.h
+; CHECK-NEXT:    splice z16.h, p0, z16.h, z17.h
 ; CHECK-NEXT:    splice z2.h, p0, z2.h, z5.h
 ; CHECK-NEXT:    splice z6.h, p0, z6.h, z7.h
-; CHECK-NEXT:    splice z16.h, p0, z16.h, z17.h
+; CHECK-NEXT:    uzp1 z5.b, z16.b, z16.b
 ; CHECK-NEXT:    uzp1 z2.b, z2.b, z2.b
 ; CHECK-NEXT:    uzp1 z6.b, z6.b, z6.b
-; CHECK-NEXT:    uzp1 z5.b, z16.b, z16.b
 ; CHECK-NEXT:    uzp1 z18.h, z18.h, z18.h
 ; CHECK-NEXT:    splice z19.h, p0, z19.h, z18.h
 ; CHECK-NEXT:    ptrue p0.b, vl8
 ; CHECK-NEXT:    uzp1 z7.b, z19.b, z19.b
 ; CHECK-NEXT:    splice z6.b, p0, z6.b, z2.b
 ; CHECK-NEXT:    splice z7.b, p0, z7.b, z5.b
+; CHECK-NEXT:    mls z0.b, p1/m, z6.b, z1.b
 ; CHECK-NEXT:    movprfx z2, z3
 ; CHECK-NEXT:    mls z2.b, p1/m, z7.b, z4.b
-; CHECK-NEXT:    mls z0.b, p1/m, z6.b, z1.b
 ; CHECK-NEXT:    stp q2, q0, [x0]
 ; CHECK-NEXT:    ret
   %op1 = load <32 x i8>, ptr %a
@@ -682,8 +682,8 @@ define <2 x i32> @urem_v2i32(<2 x i32> %op1, <2 x i32> %op2) {
 ; CHECK-LABEL: urem_v2i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s, vl2
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    movprfx z2, z0
 ; CHECK-NEXT:    udiv z2.s, p0/m, z2.s, z1.s
 ; CHECK-NEXT:    mls z0.s, p0/m, z2.s, z1.s
@@ -697,8 +697,8 @@ define <4 x i32> @urem_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
 ; CHECK-LABEL: urem_v4i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.s, vl4
-; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    movprfx z2, z0
 ; CHECK-NEXT:    udiv z2.s, p0/m, z2.s, z1.s
 ; CHECK-NEXT:    mls z0.s, p0/m, z2.s, z1.s
@@ -734,8 +734,8 @@ define <1 x i64> @urem_v1i64(<1 x i64> %op1, <1 x i64> %op2) {
 ; CHECK-LABEL: urem_v1i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d, vl1
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    // kill: def $d1 killed $d1 def $z1
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    movprfx z2, z0
 ; CHECK-NEXT:    udiv z2.d, p0/m, z2.d, z1.d
 ; CHECK-NEXT:    mls z0.d, p0/m, z2.d, z1.d
@@ -749,8 +749,8 @@ define <2 x i64> @urem_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
 ; CHECK-LABEL: urem_v2i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ptrue p0.d, vl2
-; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    // kill: def $q1 killed $q1 def $z1
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
 ; CHECK-NEXT:    movprfx z2, z0
 ; CHECK-NEXT:    udiv z2.d, p0/m, z2.d, z1.d
 ; CHECK-NEXT:    mls z0.d, p0/m, z2.d, z1.d
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-to-fp.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-to-fp.ll
index c110e89326cc..5c5cf68135bf 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-to-fp.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-to-fp.ll
@@ -221,21 +221,21 @@ define void @ucvtf_v16i16_v16f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ext z6.b, z6.b, z1.b, #8
 ; CHECK-NEXT:    uunpklo z1.d, z1.s
 ; CHECK-NEXT:    ucvtf z2.d, p0/m, z2.d
-; CHECK-NEXT:    ucvtf z3.d, p0/m, z3.d
 ; CHECK-NEXT:    uunpklo z7.d, z7.s
+; CHECK-NEXT:    ucvtf z3.d, p0/m, z3.d
 ; CHECK-NEXT:    uunpklo z5.d, z5.s
+; CHECK-NEXT:    uunpklo z6.d, z6.s
 ; CHECK-NEXT:    ucvtf z4.d, p0/m, z4.d
 ; CHECK-NEXT:    ucvtf z0.d, p0/m, z0.d
-; CHECK-NEXT:    uunpklo z6.d, z6.s
 ; CHECK-NEXT:    ucvtf z1.d, p0/m, z1.d
 ; CHECK-NEXT:    ucvtf z5.d, p0/m, z5.d
 ; CHECK-NEXT:    stp q2, q4, [x1, #64]
 ; CHECK-NEXT:    movprfx z2, z6
 ; CHECK-NEXT:    ucvtf z2.d, p0/m, z6.d
-; CHECK-NEXT:    stp q1, q2, [x1, #32]
 ; CHECK-NEXT:    stp q0, q5, [x1, #96]
 ; CHECK-NEXT:    movprfx z0, z7
 ; CHECK-NEXT:    ucvtf z0.d, p0/m, z7.d
+; CHECK-NEXT:    stp q1, q2, [x1, #32]
 ; CHECK-NEXT:    stp q3, q0, [x1]
 ; CHECK-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
@@ -481,10 +481,10 @@ define <8 x half> @ucvtf_v8i64_v8f16(ptr %a) {
 ; CHECK-NEXT:    uzp1 z3.s, z3.s, z3.s
 ; CHECK-NEXT:    uzp1 z2.s, z2.s, z2.s
 ; CHECK-NEXT:    splice z1.s, p0, z1.s, z0.s
-; CHECK-NEXT:    movprfx z0, z1
-; CHECK-NEXT:    fcvt z0.h, p1/m, z1.s
 ; CHECK-NEXT:    splice z2.s, p0, z2.s, z3.s
 ; CHECK-NEXT:    ptrue p0.h, vl4
+; CHECK-NEXT:    movprfx z0, z1
+; CHECK-NEXT:    fcvt z0.h, p1/m, z1.s
 ; CHECK-NEXT:    movprfx z1, z2
 ; CHECK-NEXT:    fcvt z1.h, p1/m, z2.s
 ; CHECK-NEXT:    uzp1 z2.h, z0.h, z0.h
@@ -794,21 +794,21 @@ define void @scvtf_v16i16_v16f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    ext z6.b, z6.b, z1.b, #8
 ; CHECK-NEXT:    sunpklo z1.d, z1.s
 ; CHECK-NEXT:    scvtf z2.d, p0/m, z2.d
-; CHECK-NEXT:    scvtf z3.d, p0/m, z3.d
 ; CHECK-NEXT:    sunpklo z7.d, z7.s
+; CHECK-NEXT:    scvtf z3.d, p0/m, z3.d
 ; CHECK-NEXT:    sunpklo z5.d, z5.s
+; CHECK-NEXT:    sunpklo z6.d, z6.s
 ; CHECK-NEXT:    scvtf z4.d, p0/m, z4.d
 ; CHECK-NEXT:    scvtf z0.d, p0/m, z0.d
-; CHECK-NEXT:    sunpklo z6.d, z6.s
 ; CHECK-NEXT:    scvtf z1.d, p0/m, z1.d
 ; CHECK-NEXT:    scvtf z5.d, p0/m, z5.d
 ; CHECK-NEXT:    stp q2, q4, [x1, #64]
 ; CHECK-NEXT:    movprfx z2, z6
 ; CHECK-NEXT:    scvtf z2.d, p0/m, z6.d
-; CHECK-NEXT:    stp q1, q2, [x1, #32]
 ; CHECK-NEXT:    stp q0, q5, [x1, #96]
 ; CHECK-NEXT:    movprfx z0, z7
 ; CHECK-NEXT:    scvtf z0.d, p0/m, z7.d
+; CHECK-NEXT:    stp q1, q2, [x1, #32]
 ; CHECK-NEXT:    stp q3, q0, [x1]
 ; CHECK-NEXT:    ret
   %op1 = load <16 x i16>, ptr %a
@@ -982,16 +982,16 @@ define void @scvtf_v16i32_v16f64(ptr %a, ptr %b) {
 ; CHECK-NEXT:    sunpklo z0.d, z0.s
 ; CHECK-NEXT:    sunpklo z1.d, z1.s
 ; CHECK-NEXT:    ext z6.b, z6.b, z4.b, #8
-; CHECK-NEXT:    sunpklo z4.d, z4.s
 ; CHECK-NEXT:    ext z7.b, z7.b, z5.b, #8
+; CHECK-NEXT:    sunpklo z4.d, z4.s
 ; CHECK-NEXT:    sunpklo z5.d, z5.s
 ; CHECK-NEXT:    sunpklo z2.d, z2.s
 ; CHECK-NEXT:    sunpklo z3.d, z3.s
 ; CHECK-NEXT:    scvtf z0.d, p0/m, z0.d
 ; CHECK-NEXT:    sunpklo z6.d, z6.s
+; CHECK-NEXT:    sunpklo z7.d, z7.s
 ; CHECK-NEXT:    scvtf z1.d, p0/m, z1.d
 ; CHECK-NEXT:    scvtf z4.d, p0/m, z4.d
-; CHECK-NEXT:    sunpklo z7.d, z7.s
 ; CHECK-NEXT:    scvtf z2.d, p0/m, z2.d
 ; CHECK-NEXT:    scvtf z3.d, p0/m, z3.d
 ; CHECK-NEXT:    stp q1, q3, [x1, #64]
diff --git a/llvm/test/CodeGen/AArch64/sve-umulo-sdnode.ll b/llvm/test/CodeGen/AArch64/sve-umulo-sdnode.ll
index 45d796f68c42..36d64725742e 100644
--- a/llvm/test/CodeGen/AArch64/sve-umulo-sdnode.ll
+++ b/llvm/test/CodeGen/AArch64/sve-umulo-sdnode.ll
@@ -120,16 +120,16 @@ define <vscale x 64 x i8> @umulo_nxv64i8(<vscale x 64 x i8> %x, <vscale x 64 x i
 ; CHECK-NEXT:    umulh z27.b, p0/m, z27.b, z5.b
 ; CHECK-NEXT:    mul z3.b, p0/m, z3.b, z7.b
 ; CHECK-NEXT:    mul z0.b, p0/m, z0.b, z4.b
-; CHECK-NEXT:    cmpne p1.b, p0/z, z25.b, #0
 ; CHECK-NEXT:    mul z2.b, p0/m, z2.b, z6.b
 ; CHECK-NEXT:    mul z1.b, p0/m, z1.b, z5.b
+; CHECK-NEXT:    cmpne p1.b, p0/z, z25.b, #0
 ; CHECK-NEXT:    cmpne p2.b, p0/z, z24.b, #0
 ; CHECK-NEXT:    cmpne p3.b, p0/z, z26.b, #0
 ; CHECK-NEXT:    cmpne p0.b, p0/z, z27.b, #0
 ; CHECK-NEXT:    mov z0.b, p1/m, #0 // =0x0
+; CHECK-NEXT:    mov z2.b, p3/m, #0 // =0x0
 ; CHECK-NEXT:    mov z3.b, p2/m, #0 // =0x0
 ; CHECK-NEXT:    mov z1.b, p0/m, #0 // =0x0
-; CHECK-NEXT:    mov z2.b, p3/m, #0 // =0x0
 ; CHECK-NEXT:    ret
   %a = call { <vscale x 64 x i8>, <vscale x 64 x i1> } @llvm.umul.with.overflow.nxv64i8(<vscale x 64 x i8> %x, <vscale x 64 x i8> %y)
   %b = extractvalue { <vscale x 64 x i8>, <vscale x 64 x i1> } %a, 0
@@ -237,16 +237,16 @@ define <vscale x 32 x i16> @umulo_nxv32i16(<vscale x 32 x i16> %x, <vscale x 32
 ; CHECK-NEXT:    umulh z27.h, p0/m, z27.h, z5.h
 ; CHECK-NEXT:    mul z3.h, p0/m, z3.h, z7.h
 ; CHECK-NEXT:    mul z0.h, p0/m, z0.h, z4.h
-; CHECK-NEXT:    cmpne p1.h, p0/z, z25.h, #0
 ; CHECK-NEXT:    mul z2.h, p0/m, z2.h, z6.h
 ; CHECK-NEXT:    mul z1.h, p0/m, z1.h, z5.h
+; CHECK-NEXT:    cmpne p1.h, p0/z, z25.h, #0
 ; CHECK-NEXT:    cmpne p2.h, p0/z, z24.h, #0
 ; CHECK-NEXT:    cmpne p3.h, p0/z, z26.h, #0
 ; CHECK-NEXT:    cmpne p0.h, p0/z, z27.h, #0
 ; CHECK-NEXT:    mov z0.h, p1/m, #0 // =0x0
+; CHECK-NEXT:    mov z2.h, p3/m, #0 // =0x0
 ; CHECK-NEXT:    mov z3.h, p2/m, #0 // =0x0
 ; CHECK-NEXT:    mov z1.h, p0/m, #0 // =0x0
-; CHECK-NEXT:    mov z2.h, p3/m, #0 // =0x0
 ; CHECK-NEXT:    ret
   %a = call { <vscale x 32 x i16>, <vscale x 32 x i1> } @llvm.umul.with.overflow.nxv32i16(<vscale x 32 x i16> %x, <vscale x 32 x i16> %y)
   %b = extractvalue { <vscale x 32 x i16>, <vscale x 32 x i1> } %a, 0
@@ -334,16 +334,16 @@ define <vscale x 16 x i32> @umulo_nxv16i32(<vscale x 16 x i32> %x, <vscale x 16
 ; CHECK-NEXT:    umulh z27.s, p0/m, z27.s, z5.s
 ; CHECK-NEXT:    mul z3.s, p0/m, z3.s, z7.s
 ; CHECK-NEXT:    mul z0.s, p0/m, z0.s, z4.s
-; CHECK-NEXT:    cmpne p1.s, p0/z, z25.s, #0
 ; CHECK-NEXT:    mul z2.s, p0/m, z2.s, z6.s
 ; CHECK-NEXT:    mul z1.s, p0/m, z1.s, z5.s
+; CHECK-NEXT:    cmpne p1.s, p0/z, z25.s, #0
 ; CHECK-NEXT:    cmpne p2.s, p0/z, z24.s, #0
 ; CHECK-NEXT:    cmpne p3.s, p0/z, z26.s, #0
 ; CHECK-NEXT:    cmpne p0.s, p0/z, z27.s, #0
 ; CHECK-NEXT:    mov z0.s, p1/m, #0 // =0x0
+; CHECK-NEXT:    mov z2.s, p3/m, #0 // =0x0
 ; CHECK-NEXT:    mov z3.s, p2/m, #0 // =0x0
 ; CHECK-NEXT:    mov z1.s, p0/m, #0 // =0x0
-; CHECK-NEXT:    mov z2.s, p3/m, #0 // =0x0
 ; CHECK-NEXT:    ret
   %a = call { <vscale x 16 x i32>, <vscale x 16 x i1> } @llvm.umul.with.overflow.nxv16i32(<vscale x 16 x i32> %x, <vscale x 16 x i32> %y)
   %b = extractvalue { <vscale x 16 x i32>, <vscale x 16 x i1> } %a, 0
@@ -411,16 +411,16 @@ define <vscale x 8 x i64> @umulo_nxv8i64(<vscale x 8 x i64> %x, <vscale x 8 x i6
 ; CHECK-NEXT:    umulh z27.d, p0/m, z27.d, z5.d
 ; CHECK-NEXT:    mul z3.d, p0/m, z3.d, z7.d
 ; CHECK-NEXT:    mul z0.d, p0/m, z0.d, z4.d
-; CHECK-NEXT:    cmpne p1.d, p0/z, z25.d, #0
 ; CHECK-NEXT:    mul z2.d, p0/m, z2.d, z6.d
 ; CHECK-NEXT:    mul z1.d, p0/m, z1.d, z5.d
+; CHECK-NEXT:    cmpne p1.d, p0/z, z25.d, #0
 ; CHECK-NEXT:    cmpne p2.d, p0/z, z24.d, #0
 ; CHECK-NEXT:    cmpne p3.d, p0/z, z26.d, #0
 ; CHECK-NEXT:    cmpne p0.d, p0/z, z27.d, #0
 ; CHECK-NEXT:    mov z0.d, p1/m, #0 // =0x0
+; CHECK-NEXT:    mov z2.d, p3/m, #0 // =0x0
 ; CHECK-NEXT:    mov z3.d, p2/m, #0 // =0x0
 ; CHECK-NEXT:    mov z1.d, p0/m, #0 // =0x0
-; CHECK-NEXT:    mov z2.d, p3/m, #0 // =0x0
 ; CHECK-NEXT:    ret
   %a = call { <vscale x 8 x i64>, <vscale x 8 x i1> } @llvm.umul.with.overflow.nxv8i64(<vscale x 8 x i64> %x, <vscale x 8 x i64> %y)
   %b = extractvalue { <vscale x 8 x i64>, <vscale x 8 x i1> } %a, 0
diff --git a/llvm/test/CodeGen/AArch64/sve-vecreduce-dot.ll b/llvm/test/CodeGen/AArch64/sve-vecreduce-dot.ll
index 4d0d2a991d0e..194d1071301d 100644
--- a/llvm/test/CodeGen/AArch64/sve-vecreduce-dot.ll
+++ b/llvm/test/CodeGen/AArch64/sve-vecreduce-dot.ll
@@ -31,8 +31,8 @@ define i32 @test(<vscale x 32 x i8> %bin.rdx, <vscale x 32 x i8> %bin.rdx2)  {
 ; CHECK-NEXT:    sunpkhi z24.s, z3.h
 ; CHECK-NEXT:    mul z0.s, p0/m, z0.s, z2.s
 ; CHECK-NEXT:    sunpkhi z2.s, z7.h
-; CHECK-NEXT:    sunpklo z3.s, z3.h
 ; CHECK-NEXT:    sunpklo z7.s, z7.h
+; CHECK-NEXT:    sunpklo z3.s, z3.h
 ; CHECK-NEXT:    mla z0.s, p0/m, z25.s, z24.s
 ; CHECK-NEXT:    mad z2.s, p0/m, z6.s, z4.s
 ; CHECK-NEXT:    mad z1.s, p0/m, z3.s, z26.s
diff --git a/llvm/test/CodeGen/AArch64/sve2-xar.ll b/llvm/test/CodeGen/AArch64/sve2-xar.ll
index 62680522bab9..e297ade6b9ae 100644
--- a/llvm/test/CodeGen/AArch64/sve2-xar.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-xar.ll
@@ -155,8 +155,8 @@ define <vscale x 2 x i64> @xar_nxv2i64_l_neg1(<vscale x 2 x i64> %x, <vscale x 2
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    subr z2.d, z2.d, #0 // =0x0
 ; CHECK-NEXT:    eor z0.d, z0.d, z1.d
-; CHECK-NEXT:    and z2.d, z2.d, #0x3f
 ; CHECK-NEXT:    and z3.d, z3.d, #0x3f
+; CHECK-NEXT:    and z2.d, z2.d, #0x3f
 ; CHECK-NEXT:    movprfx z1, z0
 ; CHECK-NEXT:    lsl z1.d, p0/m, z1.d, z3.d
 ; CHECK-NEXT:    lsr z0.d, p0/m, z0.d, z2.d
-- 
GitLab


From 5fc8a190b37906cba3f175440abd79dabd8acc73 Mon Sep 17 00:00:00 2001
From: wanglei <wanglei@loongson.cn>
Date: Fri, 12 Apr 2024 16:49:52 +0800
Subject: [PATCH 047/464] [LoongArch] Pre commit test for #88371. NFC

---
 .../CodeGenPrepare/LoongArch/lit.local.cfg    |  2 ++
 .../CodeGenPrepare/LoongArch/splitgep.ll      | 34 +++++++++++++++++++
 2 files changed, 36 insertions(+)
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/LoongArch/lit.local.cfg
 create mode 100644 llvm/test/Transforms/CodeGenPrepare/LoongArch/splitgep.ll

diff --git a/llvm/test/Transforms/CodeGenPrepare/LoongArch/lit.local.cfg b/llvm/test/Transforms/CodeGenPrepare/LoongArch/lit.local.cfg
new file mode 100644
index 000000000000..cc24278acbb4
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/LoongArch/lit.local.cfg
@@ -0,0 +1,2 @@
+if not "LoongArch" in config.root.targets:
+    config.unsupported = True
diff --git a/llvm/test/Transforms/CodeGenPrepare/LoongArch/splitgep.ll b/llvm/test/Transforms/CodeGenPrepare/LoongArch/splitgep.ll
new file mode 100644
index 000000000000..ca2a16a0ca7e
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/LoongArch/splitgep.ll
@@ -0,0 +1,34 @@
+; RUN: opt --mtriple=loongarch64 -S --passes='require<profile-summary>,function(codegenprepare)' %s | FileCheck %s
+
+; Check that we have deterministic output
+define void @test(ptr %sp, ptr %t, i32 %n) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %s = load ptr, ptr %sp, align 8
+; CHECK-NEXT:    br label %while_cond
+
+entry:
+  %s = load ptr, ptr %sp
+  br label %while_cond
+
+while_cond:
+  %phi = phi i32 [ 0, %entry ], [ %i, %while_body ]
+  %gep0 = getelementptr [65536 x i32], ptr %s, i64 0, i64 20000
+  %gep1 = getelementptr [65536 x i32], ptr %s, i64 0, i64 20001
+  %gep2 = getelementptr [65536 x i32], ptr %t, i64 0, i64 20000
+  %gep3 = getelementptr [65536 x i32], ptr %t, i64 0, i64 20001
+  %cmp = icmp slt i32 %phi, %n
+  br i1 %cmp, label %while_body, label %while_end
+
+while_body:
+  %i = add i32 %phi, 1
+  store i32 %i, ptr %gep0
+  store i32 %phi, ptr %gep1
+  store i32 %i, ptr %gep2
+  store i32 %phi, ptr %gep3
+  br label %while_cond
+
+while_end:
+  ret void
+}
+
-- 
GitLab


From c39df496d73621238ba87de235d9a61e2b2203b9 Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Fri, 12 Apr 2024 13:05:49 +0300
Subject: [PATCH 048/464] [clang][NFC] Refactor `CUDAFunctionTarget`

Refactor `CUDAFunctionTarget` into a scoped enum at namespace scope, so that it can be forward declared. This is done in preparation for `SemaCUDA`.
---
 clang/include/clang/Sema/Sema.h |  29 ++++---
 clang/lib/Sema/SemaCUDA.cpp     | 146 ++++++++++++++++++--------------
 clang/lib/Sema/SemaDecl.cpp     |   8 +-
 clang/lib/Sema/SemaDeclAttr.cpp |  15 ++--
 clang/lib/Sema/SemaExpr.cpp     |  16 ++--
 clang/lib/Sema/SemaExprCXX.cpp  |   3 +-
 clang/lib/Sema/SemaOverload.cpp |  14 +--
 clang/lib/Sema/SemaStmt.cpp     |   3 +-
 clang/lib/Sema/SemaType.cpp     |  32 +++----
 9 files changed, 145 insertions(+), 121 deletions(-)

diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 90c916628270..00888b7f7a73 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -435,6 +435,14 @@ enum class CXXSpecialMemberKind {
   Invalid
 };
 
+enum class CUDAFunctionTarget {
+  Device,
+  Global,
+  Host,
+  HostDevice,
+  InvalidTarget
+};
+
 /// Sema - This implements semantic analysis and AST building for C.
 /// \nosubgrouping
 class Sema final : public SemaBase {
@@ -3663,20 +3671,12 @@ public:
   InternalLinkageAttr *mergeInternalLinkageAttr(Decl *D,
                                                 const InternalLinkageAttr &AL);
 
-  enum CUDAFunctionTarget {
-    CFT_Device,
-    CFT_Global,
-    CFT_Host,
-    CFT_HostDevice,
-    CFT_InvalidTarget
-  };
-
   /// Check validaty of calling convention attribute \p attr. If \p FD
   /// is not null pointer, use \p FD to determine the CUDA/HIP host/device
   /// target. Otherwise, it is specified by \p CFT.
-  bool CheckCallingConvAttr(const ParsedAttr &attr, CallingConv &CC,
-                            const FunctionDecl *FD = nullptr,
-                            CUDAFunctionTarget CFT = CFT_InvalidTarget);
+  bool CheckCallingConvAttr(
+      const ParsedAttr &attr, CallingConv &CC, const FunctionDecl *FD = nullptr,
+      CUDAFunctionTarget CFT = CUDAFunctionTarget::InvalidTarget);
 
   void AddParameterABIAttr(Decl *D, const AttributeCommonInfo &CI,
                            ParameterABI ABI);
@@ -12967,7 +12967,8 @@ public:
   /// Example usage:
   ///
   ///  // Variable-length arrays are not allowed in CUDA device code.
-  ///  if (CUDADiagIfDeviceCode(Loc, diag::err_cuda_vla) << CurrentCUDATarget())
+  ///  if (CUDADiagIfDeviceCode(Loc, diag::err_cuda_vla)
+  ///     << llvm::to_underlying(CurrentCUDATarget()))
   ///    return ExprError();
   ///  // Otherwise, continue parsing as normal.
   SemaDiagnosticBuilder CUDADiagIfDeviceCode(SourceLocation Loc,
@@ -12983,7 +12984,7 @@ public:
   /// function.
   ///
   /// Use this rather than examining the function's attributes yourself -- you
-  /// will get it wrong.  Returns CFT_Host if D is null.
+  /// will get it wrong.  Returns CUDAFunctionTarget::Host if D is null.
   CUDAFunctionTarget IdentifyCUDATarget(const FunctionDecl *D,
                                         bool IgnoreImplicitHDAttr = false);
   CUDAFunctionTarget IdentifyCUDATarget(const ParsedAttributesView &Attrs);
@@ -13008,7 +13009,7 @@ public:
   /// Define the current global CUDA host/device context where a function may be
   /// called. Only used when a function is called outside of any functions.
   struct CUDATargetContext {
-    CUDAFunctionTarget Target = CFT_HostDevice;
+    CUDAFunctionTarget Target = CUDAFunctionTarget::HostDevice;
     CUDATargetContextKind Kind = CTCK_Unknown;
     Decl *D = nullptr;
   } CurCUDATargetCtx;
diff --git a/clang/lib/Sema/SemaCUDA.cpp b/clang/lib/Sema/SemaCUDA.cpp
index 1596222e3d1d..9d6d709e262a 100644
--- a/clang/lib/Sema/SemaCUDA.cpp
+++ b/clang/lib/Sema/SemaCUDA.cpp
@@ -22,6 +22,7 @@
 #include "clang/Sema/SemaDiagnostic.h"
 #include "clang/Sema/SemaInternal.h"
 #include "clang/Sema/Template.h"
+#include "llvm/ADT/STLForwardCompat.h"
 #include "llvm/ADT/SmallVector.h"
 #include <optional>
 using namespace clang;
@@ -64,8 +65,7 @@ ExprResult Sema::ActOnCUDAExecConfigExpr(Scope *S, SourceLocation LLLLoc,
                        /*IsExecConfig=*/true);
 }
 
-Sema::CUDAFunctionTarget
-Sema::IdentifyCUDATarget(const ParsedAttributesView &Attrs) {
+CUDAFunctionTarget Sema::IdentifyCUDATarget(const ParsedAttributesView &Attrs) {
   bool HasHostAttr = false;
   bool HasDeviceAttr = false;
   bool HasGlobalAttr = false;
@@ -90,18 +90,18 @@ Sema::IdentifyCUDATarget(const ParsedAttributesView &Attrs) {
   }
 
   if (HasInvalidTargetAttr)
-    return CFT_InvalidTarget;
+    return CUDAFunctionTarget::InvalidTarget;
 
   if (HasGlobalAttr)
-    return CFT_Global;
+    return CUDAFunctionTarget::Global;
 
   if (HasHostAttr && HasDeviceAttr)
-    return CFT_HostDevice;
+    return CUDAFunctionTarget::HostDevice;
 
   if (HasDeviceAttr)
-    return CFT_Device;
+    return CUDAFunctionTarget::Device;
 
-  return CFT_Host;
+  return CUDAFunctionTarget::Host;
 }
 
 template <typename A>
@@ -120,43 +120,43 @@ Sema::CUDATargetContextRAII::CUDATargetContextRAII(Sema &S_,
   assert(K == CTCK_InitGlobalVar);
   auto *VD = dyn_cast_or_null<VarDecl>(D);
   if (VD && VD->hasGlobalStorage() && !VD->isStaticLocal()) {
-    auto Target = CFT_Host;
+    auto Target = CUDAFunctionTarget::Host;
     if ((hasAttr<CUDADeviceAttr>(VD, /*IgnoreImplicit=*/true) &&
          !hasAttr<CUDAHostAttr>(VD, /*IgnoreImplicit=*/true)) ||
         hasAttr<CUDASharedAttr>(VD, /*IgnoreImplicit=*/true) ||
         hasAttr<CUDAConstantAttr>(VD, /*IgnoreImplicit=*/true))
-      Target = CFT_Device;
+      Target = CUDAFunctionTarget::Device;
     S.CurCUDATargetCtx = {Target, K, VD};
   }
 }
 
 /// IdentifyCUDATarget - Determine the CUDA compilation target for this function
-Sema::CUDAFunctionTarget Sema::IdentifyCUDATarget(const FunctionDecl *D,
-                                                  bool IgnoreImplicitHDAttr) {
+CUDAFunctionTarget Sema::IdentifyCUDATarget(const FunctionDecl *D,
+                                            bool IgnoreImplicitHDAttr) {
   // Code that lives outside a function gets the target from CurCUDATargetCtx.
   if (D == nullptr)
     return CurCUDATargetCtx.Target;
 
   if (D->hasAttr<CUDAInvalidTargetAttr>())
-    return CFT_InvalidTarget;
+    return CUDAFunctionTarget::InvalidTarget;
 
   if (D->hasAttr<CUDAGlobalAttr>())
-    return CFT_Global;
+    return CUDAFunctionTarget::Global;
 
   if (hasAttr<CUDADeviceAttr>(D, IgnoreImplicitHDAttr)) {
     if (hasAttr<CUDAHostAttr>(D, IgnoreImplicitHDAttr))
-      return CFT_HostDevice;
-    return CFT_Device;
+      return CUDAFunctionTarget::HostDevice;
+    return CUDAFunctionTarget::Device;
   } else if (hasAttr<CUDAHostAttr>(D, IgnoreImplicitHDAttr)) {
-    return CFT_Host;
+    return CUDAFunctionTarget::Host;
   } else if ((D->isImplicit() || !D->isUserProvided()) &&
              !IgnoreImplicitHDAttr) {
     // Some implicit declarations (like intrinsic functions) are not marked.
     // Set the most lenient target on them for maximal flexibility.
-    return CFT_HostDevice;
+    return CUDAFunctionTarget::HostDevice;
   }
 
-  return CFT_Host;
+  return CUDAFunctionTarget::Host;
 }
 
 /// IdentifyTarget - Determine the CUDA compilation target for this variable.
@@ -181,10 +181,10 @@ Sema::CUDAVariableTarget Sema::IdentifyCUDATarget(const VarDecl *Var) {
   //  - on device side in device or global functions
   if (auto *FD = dyn_cast<FunctionDecl>(Var->getDeclContext())) {
     switch (IdentifyCUDATarget(FD)) {
-    case CFT_HostDevice:
+    case CUDAFunctionTarget::HostDevice:
       return CVT_Both;
-    case CFT_Device:
-    case CFT_Global:
+    case CUDAFunctionTarget::Device:
+    case CUDAFunctionTarget::Global:
       return CVT_Device;
     default:
       return CVT_Host;
@@ -230,7 +230,7 @@ Sema::IdentifyCUDAPreference(const FunctionDecl *Caller,
   // trivial ctor/dtor without device attr to be used. Non-trivial ctor/dtor
   // will be diagnosed by checkAllowedCUDAInitializer.
   if (Caller == nullptr && CurCUDATargetCtx.Kind == CTCK_InitGlobalVar &&
-      CurCUDATargetCtx.Target == CFT_Device &&
+      CurCUDATargetCtx.Target == CUDAFunctionTarget::Device &&
       (isa<CXXConstructorDecl>(Callee) || isa<CXXDestructorDecl>(Callee)))
     return CFP_HostDevice;
 
@@ -239,40 +239,47 @@ Sema::IdentifyCUDAPreference(const FunctionDecl *Caller,
 
   // If one of the targets is invalid, the check always fails, no matter what
   // the other target is.
-  if (CallerTarget == CFT_InvalidTarget || CalleeTarget == CFT_InvalidTarget)
+  if (CallerTarget == CUDAFunctionTarget::InvalidTarget ||
+      CalleeTarget == CUDAFunctionTarget::InvalidTarget)
     return CFP_Never;
 
   // (a) Can't call global from some contexts until we support CUDA's
   // dynamic parallelism.
-  if (CalleeTarget == CFT_Global &&
-      (CallerTarget == CFT_Global || CallerTarget == CFT_Device))
+  if (CalleeTarget == CUDAFunctionTarget::Global &&
+      (CallerTarget == CUDAFunctionTarget::Global ||
+       CallerTarget == CUDAFunctionTarget::Device))
     return CFP_Never;
 
   // (b) Calling HostDevice is OK for everyone.
-  if (CalleeTarget == CFT_HostDevice)
+  if (CalleeTarget == CUDAFunctionTarget::HostDevice)
     return CFP_HostDevice;
 
   // (c) Best case scenarios
   if (CalleeTarget == CallerTarget ||
-      (CallerTarget == CFT_Host && CalleeTarget == CFT_Global) ||
-      (CallerTarget == CFT_Global && CalleeTarget == CFT_Device))
+      (CallerTarget == CUDAFunctionTarget::Host &&
+       CalleeTarget == CUDAFunctionTarget::Global) ||
+      (CallerTarget == CUDAFunctionTarget::Global &&
+       CalleeTarget == CUDAFunctionTarget::Device))
     return CFP_Native;
 
   // HipStdPar mode is special, in that assessing whether a device side call to
   // a host target is deferred to a subsequent pass, and cannot unambiguously be
   // adjudicated in the AST, hence we optimistically allow them to pass here.
   if (getLangOpts().HIPStdPar &&
-      (CallerTarget == CFT_Global || CallerTarget == CFT_Device ||
-       CallerTarget == CFT_HostDevice) &&
-      CalleeTarget == CFT_Host)
+      (CallerTarget == CUDAFunctionTarget::Global ||
+       CallerTarget == CUDAFunctionTarget::Device ||
+       CallerTarget == CUDAFunctionTarget::HostDevice) &&
+      CalleeTarget == CUDAFunctionTarget::Host)
     return CFP_HostDevice;
 
   // (d) HostDevice behavior depends on compilation mode.
-  if (CallerTarget == CFT_HostDevice) {
+  if (CallerTarget == CUDAFunctionTarget::HostDevice) {
     // It's OK to call a compilation-mode matching function from an HD one.
-    if ((getLangOpts().CUDAIsDevice && CalleeTarget == CFT_Device) ||
+    if ((getLangOpts().CUDAIsDevice &&
+         CalleeTarget == CUDAFunctionTarget::Device) ||
         (!getLangOpts().CUDAIsDevice &&
-         (CalleeTarget == CFT_Host || CalleeTarget == CFT_Global)))
+         (CalleeTarget == CUDAFunctionTarget::Host ||
+          CalleeTarget == CUDAFunctionTarget::Global)))
       return CFP_SameSide;
 
     // Calls from HD to non-mode-matching functions (i.e., to host functions
@@ -283,9 +290,12 @@ Sema::IdentifyCUDAPreference(const FunctionDecl *Caller,
   }
 
   // (e) Calling across device/host boundary is not something you should do.
-  if ((CallerTarget == CFT_Host && CalleeTarget == CFT_Device) ||
-      (CallerTarget == CFT_Device && CalleeTarget == CFT_Host) ||
-      (CallerTarget == CFT_Global && CalleeTarget == CFT_Host))
+  if ((CallerTarget == CUDAFunctionTarget::Host &&
+       CalleeTarget == CUDAFunctionTarget::Device) ||
+      (CallerTarget == CUDAFunctionTarget::Device &&
+       CalleeTarget == CUDAFunctionTarget::Host) ||
+      (CallerTarget == CUDAFunctionTarget::Global &&
+       CalleeTarget == CUDAFunctionTarget::Host))
     return CFP_Never;
 
   llvm_unreachable("All cases should've been handled by now.");
@@ -337,16 +347,16 @@ void Sema::EraseUnwantedCUDAMatches(
 /// \param ResolvedTarget with a target that resolves for both calls.
 /// \return true if there's a conflict, false otherwise.
 static bool
-resolveCalleeCUDATargetConflict(Sema::CUDAFunctionTarget Target1,
-                                Sema::CUDAFunctionTarget Target2,
-                                Sema::CUDAFunctionTarget *ResolvedTarget) {
+resolveCalleeCUDATargetConflict(CUDAFunctionTarget Target1,
+                                CUDAFunctionTarget Target2,
+                                CUDAFunctionTarget *ResolvedTarget) {
   // Only free functions and static member functions may be global.
-  assert(Target1 != Sema::CFT_Global);
-  assert(Target2 != Sema::CFT_Global);
+  assert(Target1 != CUDAFunctionTarget::Global);
+  assert(Target2 != CUDAFunctionTarget::Global);
 
-  if (Target1 == Sema::CFT_HostDevice) {
+  if (Target1 == CUDAFunctionTarget::HostDevice) {
     *ResolvedTarget = Target2;
-  } else if (Target2 == Sema::CFT_HostDevice) {
+  } else if (Target2 == CUDAFunctionTarget::HostDevice) {
     *ResolvedTarget = Target1;
   } else if (Target1 != Target2) {
     return true;
@@ -422,7 +432,8 @@ bool Sema::inferCUDATargetForImplicitSpecialMember(CXXRecordDecl *ClassDecl,
         if (Diagnose) {
           Diag(ClassDecl->getLocation(),
                diag::note_implicit_member_target_infer_collision)
-              << (unsigned)CSM << *InferredTarget << BaseMethodTarget;
+              << (unsigned)CSM << llvm::to_underlying(*InferredTarget)
+              << llvm::to_underlying(BaseMethodTarget);
         }
         MemberDecl->addAttr(CUDAInvalidTargetAttr::CreateImplicit(Context));
         return true;
@@ -465,7 +476,8 @@ bool Sema::inferCUDATargetForImplicitSpecialMember(CXXRecordDecl *ClassDecl,
         if (Diagnose) {
           Diag(ClassDecl->getLocation(),
                diag::note_implicit_member_target_infer_collision)
-              << (unsigned)CSM << *InferredTarget << FieldMethodTarget;
+              << (unsigned)CSM << llvm::to_underlying(*InferredTarget)
+              << llvm::to_underlying(FieldMethodTarget);
         }
         MemberDecl->addAttr(CUDAInvalidTargetAttr::CreateImplicit(Context));
         return true;
@@ -478,9 +490,9 @@ bool Sema::inferCUDATargetForImplicitSpecialMember(CXXRecordDecl *ClassDecl,
   // it's the least restrictive option that can be invoked from any target.
   bool NeedsH = true, NeedsD = true;
   if (InferredTarget) {
-    if (*InferredTarget == CFT_Device)
+    if (*InferredTarget == CUDAFunctionTarget::Device)
       NeedsH = false;
-    else if (*InferredTarget == CFT_Host)
+    else if (*InferredTarget == CUDAFunctionTarget::Host)
       NeedsD = false;
   }
 
@@ -677,9 +689,10 @@ void Sema::checkAllowedCUDAInitializer(VarDecl *VD) {
     }
     if (InitFn) {
       CUDAFunctionTarget InitFnTarget = IdentifyCUDATarget(InitFn);
-      if (InitFnTarget != CFT_Host && InitFnTarget != CFT_HostDevice) {
+      if (InitFnTarget != CUDAFunctionTarget::Host &&
+          InitFnTarget != CUDAFunctionTarget::HostDevice) {
         Diag(VD->getLocation(), diag::err_ref_bad_target_global_initializer)
-            << InitFnTarget << InitFn;
+            << llvm::to_underlying(InitFnTarget) << InitFn;
         Diag(InitFn->getLocation(), diag::note_previous_decl) << InitFn;
         VD->setInvalidDecl();
       }
@@ -699,8 +712,9 @@ void Sema::CUDARecordImplicitHostDeviceFuncUsedByDevice(
   CUDAFunctionTarget CallerTarget = IdentifyCUDATarget(Caller);
 
   // Record whether an implicit host device function is used on device side.
-  if (CallerTarget != CFT_Device && CallerTarget != CFT_Global &&
-      (CallerTarget != CFT_HostDevice ||
+  if (CallerTarget != CUDAFunctionTarget::Device &&
+      CallerTarget != CUDAFunctionTarget::Global &&
+      (CallerTarget != CUDAFunctionTarget::HostDevice ||
        (isCUDAImplicitHostDeviceFunction(Caller) &&
         !getASTContext().CUDAImplicitHostDeviceFunUsedByDevice.count(Caller))))
     return;
@@ -806,10 +820,10 @@ Sema::SemaDiagnosticBuilder Sema::CUDADiagIfDeviceCode(SourceLocation Loc,
     if (!CurFunContext)
       return SemaDiagnosticBuilder::K_Nop;
     switch (CurrentCUDATarget()) {
-    case CFT_Global:
-    case CFT_Device:
+    case CUDAFunctionTarget::Global:
+    case CUDAFunctionTarget::Device:
       return SemaDiagnosticBuilder::K_Immediate;
-    case CFT_HostDevice:
+    case CUDAFunctionTarget::HostDevice:
       // An HD function counts as host code if we're compiling for host, and
       // device code if we're compiling for device.  Defer any errors in device
       // mode until the function is known-emitted.
@@ -836,9 +850,9 @@ Sema::SemaDiagnosticBuilder Sema::CUDADiagIfHostCode(SourceLocation Loc,
     if (!CurFunContext)
       return SemaDiagnosticBuilder::K_Nop;
     switch (CurrentCUDATarget()) {
-    case CFT_Host:
+    case CUDAFunctionTarget::Host:
       return SemaDiagnosticBuilder::K_Immediate;
-    case CFT_HostDevice:
+    case CUDAFunctionTarget::HostDevice:
       // An HD function counts as host code if we're compiling for host, and
       // device code if we're compiling for device.  Defer any errors in device
       // mode until the function is known-emitted.
@@ -911,8 +925,8 @@ bool Sema::CheckCUDACall(SourceLocation Loc, FunctionDecl *Callee) {
     return true;
 
   SemaDiagnosticBuilder(DiagKind, Loc, diag::err_ref_bad_target, Caller, *this)
-      << IdentifyCUDATarget(Callee) << /*function*/ 0 << Callee
-      << IdentifyCUDATarget(Caller);
+      << llvm::to_underlying(IdentifyCUDATarget(Callee)) << /*function*/ 0
+      << Callee << llvm::to_underlying(IdentifyCUDATarget(Caller));
   if (!Callee->getBuiltinID())
     SemaDiagnosticBuilder(DiagKind, Callee->getLocation(),
                           diag::note_previous_decl, Caller, *this)
@@ -995,19 +1009,21 @@ void Sema::checkCUDATargetOverload(FunctionDecl *NewFD,
     // HD/global functions "exist" in some sense on both the host and device, so
     // should have the same implementation on both sides.
     if (NewTarget != OldTarget &&
-        ((NewTarget == CFT_HostDevice &&
+        ((NewTarget == CUDAFunctionTarget::HostDevice &&
           !(LangOpts.OffloadImplicitHostDeviceTemplates &&
             isCUDAImplicitHostDeviceFunction(NewFD) &&
-            OldTarget == CFT_Device)) ||
-         (OldTarget == CFT_HostDevice &&
+            OldTarget == CUDAFunctionTarget::Device)) ||
+         (OldTarget == CUDAFunctionTarget::HostDevice &&
           !(LangOpts.OffloadImplicitHostDeviceTemplates &&
             isCUDAImplicitHostDeviceFunction(OldFD) &&
-            NewTarget == CFT_Device)) ||
-         (NewTarget == CFT_Global) || (OldTarget == CFT_Global)) &&
+            NewTarget == CUDAFunctionTarget::Device)) ||
+         (NewTarget == CUDAFunctionTarget::Global) ||
+         (OldTarget == CUDAFunctionTarget::Global)) &&
         !IsOverload(NewFD, OldFD, /* UseMemberUsingDeclRules = */ false,
                     /* ConsiderCudaAttrs = */ false)) {
       Diag(NewFD->getLocation(), diag::err_cuda_ovl_target)
-          << NewTarget << NewFD->getDeclName() << OldTarget << OldFD;
+          << llvm::to_underlying(NewTarget) << NewFD->getDeclName()
+          << llvm::to_underlying(OldTarget) << OldFD;
       Diag(OldFD->getLocation(), diag::note_previous_declaration);
       NewFD->setInvalidDecl();
       break;
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index b448bacd49f3..720e56692359 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -20666,11 +20666,11 @@ Sema::FunctionEmissionStatus Sema::getEmissionStatus(const FunctionDecl *FD,
     // when compiling for host, device and global functions are never emitted.
     // (Technically, we do emit a host-side stub for global functions, but this
     // doesn't count for our purposes here.)
-    Sema::CUDAFunctionTarget T = IdentifyCUDATarget(FD);
-    if (LangOpts.CUDAIsDevice && T == Sema::CFT_Host)
+    CUDAFunctionTarget T = IdentifyCUDATarget(FD);
+    if (LangOpts.CUDAIsDevice && T == CUDAFunctionTarget::Host)
       return FunctionEmissionStatus::CUDADiscarded;
     if (!LangOpts.CUDAIsDevice &&
-        (T == Sema::CFT_Device || T == Sema::CFT_Global))
+        (T == CUDAFunctionTarget::Device || T == CUDAFunctionTarget::Global))
       return FunctionEmissionStatus::CUDADiscarded;
 
     if (IsEmittedForExternalSymbol())
@@ -20691,5 +20691,5 @@ bool Sema::shouldIgnoreInHostDeviceCheck(FunctionDecl *Callee) {
   // for host, only HD functions actually called from the host get marked as
   // known-emitted.
   return LangOpts.CUDA && !LangOpts.CUDAIsDevice &&
-         IdentifyCUDATarget(Callee) == CFT_Global;
+         IdentifyCUDATarget(Callee) == CUDAFunctionTarget::Global;
 }
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index b91064e28e41..56c9d90c9b52 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -42,6 +42,7 @@
 #include "clang/Sema/SemaHLSL.h"
 #include "clang/Sema/SemaInternal.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/STLForwardCompat.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/IR/Assumptions.h"
 #include "llvm/MC/MCSectionMachO.h"
@@ -5099,7 +5100,7 @@ static void handleSharedAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   }
   if (S.getLangOpts().CUDA && VD->hasLocalStorage() &&
       S.CUDADiagIfHostCode(AL.getLoc(), diag::err_cuda_host_shared)
-          << S.CurrentCUDATarget())
+          << llvm::to_underlying(S.CurrentCUDATarget()))
     return;
   D->addAttr(::new (S.Context) CUDASharedAttr(S.Context, AL));
 }
@@ -5493,22 +5494,22 @@ bool Sema::CheckCallingConvAttr(const ParsedAttr &Attrs, CallingConv &CC,
   // on their host/device attributes.
   if (LangOpts.CUDA) {
     auto *Aux = Context.getAuxTargetInfo();
-    assert(FD || CFT != CFT_InvalidTarget);
+    assert(FD || CFT != CUDAFunctionTarget::InvalidTarget);
     auto CudaTarget = FD ? IdentifyCUDATarget(FD) : CFT;
     bool CheckHost = false, CheckDevice = false;
     switch (CudaTarget) {
-    case CFT_HostDevice:
+    case CUDAFunctionTarget::HostDevice:
       CheckHost = true;
       CheckDevice = true;
       break;
-    case CFT_Host:
+    case CUDAFunctionTarget::Host:
       CheckHost = true;
       break;
-    case CFT_Device:
-    case CFT_Global:
+    case CUDAFunctionTarget::Device:
+    case CUDAFunctionTarget::Global:
       CheckDevice = true;
       break;
-    case CFT_InvalidTarget:
+    case CUDAFunctionTarget::InvalidTarget:
       llvm_unreachable("unexpected cuda target");
     }
     auto *HostTI = LangOpts.CUDAIsDevice ? Aux : &TI;
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index ec5ca2b9352e..18284e0c3e98 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -53,6 +53,7 @@
 #include "clang/Sema/SemaInternal.h"
 #include "clang/Sema/Template.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/STLForwardCompat.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/ConvertUTF.h"
@@ -17307,7 +17308,8 @@ ExprResult Sema::BuildVAArgExpr(SourceLocation BuiltinLoc,
   if (getLangOpts().CUDA && getLangOpts().CUDAIsDevice) {
     if (const FunctionDecl *F = dyn_cast<FunctionDecl>(CurContext)) {
       CUDAFunctionTarget T = IdentifyCUDATarget(F);
-      if (T == CFT_Global || T == CFT_Device || T == CFT_HostDevice)
+      if (T == CUDAFunctionTarget::Global || T == CUDAFunctionTarget::Device ||
+          T == CUDAFunctionTarget::HostDevice)
         return ExprError(Diag(E->getBeginLoc(), diag::err_va_arg_in_device));
     }
   }
@@ -19182,14 +19184,16 @@ MarkVarDeclODRUsed(ValueDecl *V, SourceLocation Loc, Sema &SemaRef,
     auto VarTarget = SemaRef.IdentifyCUDATarget(Var);
     auto UserTarget = SemaRef.IdentifyCUDATarget(FD);
     if (VarTarget == Sema::CVT_Host &&
-        (UserTarget == Sema::CFT_Device || UserTarget == Sema::CFT_HostDevice ||
-         UserTarget == Sema::CFT_Global)) {
+        (UserTarget == CUDAFunctionTarget::Device ||
+         UserTarget == CUDAFunctionTarget::HostDevice ||
+         UserTarget == CUDAFunctionTarget::Global)) {
       // Diagnose ODR-use of host global variables in device functions.
       // Reference of device global variables in host functions is allowed
       // through shadow variables therefore it is not diagnosed.
       if (SemaRef.LangOpts.CUDAIsDevice && !SemaRef.LangOpts.HIPStdPar) {
         SemaRef.targetDiag(Loc, diag::err_ref_bad_target)
-            << /*host*/ 2 << /*variable*/ 1 << Var << UserTarget;
+            << /*host*/ 2 << /*variable*/ 1 << Var
+            << llvm::to_underlying(UserTarget);
         SemaRef.targetDiag(Var->getLocation(),
                            Var->getType().isConstQualified()
                                ? diag::note_cuda_const_var_unpromoted
@@ -19197,8 +19201,8 @@ MarkVarDeclODRUsed(ValueDecl *V, SourceLocation Loc, Sema &SemaRef,
       }
     } else if (VarTarget == Sema::CVT_Device &&
                !Var->hasAttr<CUDASharedAttr>() &&
-               (UserTarget == Sema::CFT_Host ||
-                UserTarget == Sema::CFT_HostDevice)) {
+               (UserTarget == CUDAFunctionTarget::Host ||
+                UserTarget == CUDAFunctionTarget::HostDevice)) {
       // Record a CUDA/HIP device side variable if it is ODR-used
       // by host code. This is done conservatively, when the variable is
       // referenced in any of the following contexts:
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index ac39cb99fc93..ce9d5c26e218 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -44,6 +44,7 @@
 #include "clang/Sema/TemplateDeduction.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/STLForwardCompat.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/TypeSize.h"
@@ -884,7 +885,7 @@ ExprResult Sema::BuildCXXThrow(SourceLocation OpLoc, Expr *Ex,
   // Exceptions aren't allowed in CUDA device code.
   if (getLangOpts().CUDA)
     CUDADiagIfDeviceCode(OpLoc, diag::err_cuda_device_exceptions)
-        << "throw" << CurrentCUDATarget();
+        << "throw" << llvm::to_underlying(CurrentCUDATarget());
 
   if (getCurScope() && getCurScope()->isOpenMPSimdDirectiveScope())
     Diag(OpLoc, diag::err_omp_simd_region_cannot_use_stmt) << "throw";
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index 1f674dd4bb0f..e1155dc2d5d2 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -1549,10 +1549,10 @@ static bool IsOverloadOrOverrideImpl(Sema &SemaRef, FunctionDecl *New,
     // Don't allow overloading of destructors.  (In theory we could, but it
     // would be a giant change to clang.)
     if (!isa<CXXDestructorDecl>(New)) {
-      Sema::CUDAFunctionTarget NewTarget = SemaRef.IdentifyCUDATarget(New),
-                               OldTarget = SemaRef.IdentifyCUDATarget(Old);
-      if (NewTarget != Sema::CFT_InvalidTarget) {
-        assert((OldTarget != Sema::CFT_InvalidTarget) &&
+      CUDAFunctionTarget NewTarget = SemaRef.IdentifyCUDATarget(New),
+                         OldTarget = SemaRef.IdentifyCUDATarget(Old);
+      if (NewTarget != CUDAFunctionTarget::InvalidTarget) {
+        assert((OldTarget != CUDAFunctionTarget::InvalidTarget) &&
                "Unexpected invalid target.");
 
         // Allow overloading of functions with same signature and different CUDA
@@ -11938,8 +11938,8 @@ static void DiagnoseBadTarget(Sema &S, OverloadCandidate *Cand) {
   FunctionDecl *Caller = S.getCurFunctionDecl(/*AllowLambda=*/true);
   FunctionDecl *Callee = Cand->Function;
 
-  Sema::CUDAFunctionTarget CallerTarget = S.IdentifyCUDATarget(Caller),
-                           CalleeTarget = S.IdentifyCUDATarget(Callee);
+  CUDAFunctionTarget CallerTarget = S.IdentifyCUDATarget(Caller),
+                     CalleeTarget = S.IdentifyCUDATarget(Callee);
 
   std::string FnDesc;
   std::pair<OverloadCandidateKind, OverloadCandidateSelect> FnKindPair =
@@ -11949,7 +11949,7 @@ static void DiagnoseBadTarget(Sema &S, OverloadCandidate *Cand) {
   S.Diag(Callee->getLocation(), diag::note_ovl_candidate_bad_target)
       << (unsigned)FnKindPair.first << (unsigned)ocs_non_template
       << FnDesc /* Ignored */
-      << CalleeTarget << CallerTarget;
+      << llvm::to_underlying(CalleeTarget) << llvm::to_underlying(CallerTarget);
 
   // This could be an implicit constructor for which we could not infer the
   // target due to a collsion. Diagnose that case.
diff --git a/clang/lib/Sema/SemaStmt.cpp b/clang/lib/Sema/SemaStmt.cpp
index e53c76e65b03..1c2f6120f621 100644
--- a/clang/lib/Sema/SemaStmt.cpp
+++ b/clang/lib/Sema/SemaStmt.cpp
@@ -37,6 +37,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/STLForwardCompat.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
@@ -4574,7 +4575,7 @@ StmtResult Sema::ActOnCXXTryBlock(SourceLocation TryLoc, Stmt *TryBlock,
   // Exceptions aren't allowed in CUDA device code.
   if (getLangOpts().CUDA)
     CUDADiagIfDeviceCode(TryLoc, diag::err_cuda_device_exceptions)
-        << "try" << CurrentCUDATarget();
+        << "try" << llvm::to_underlying(CurrentCUDATarget());
 
   if (getCurScope() && getCurScope()->isOpenMPSimdDirectiveScope())
     Diag(TryLoc, diag::err_omp_simd_region_cannot_use_stmt) << "try";
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index b8a1518fbe00..b3f6078952f6 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -37,6 +37,7 @@
 #include "clang/Sema/Template.h"
 #include "clang/Sema/TemplateInstCallback.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLForwardCompat.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
@@ -376,11 +377,10 @@ enum TypeAttrLocation {
 static void
 processTypeAttrs(TypeProcessingState &state, QualType &type,
                  TypeAttrLocation TAL, const ParsedAttributesView &attrs,
-                 Sema::CUDAFunctionTarget CFT = Sema::CFT_HostDevice);
+                 CUDAFunctionTarget CFT = CUDAFunctionTarget::HostDevice);
 
 static bool handleFunctionTypeAttr(TypeProcessingState &state, ParsedAttr &attr,
-                                   QualType &type,
-                                   Sema::CUDAFunctionTarget CFT);
+                                   QualType &type, CUDAFunctionTarget CFT);
 
 static bool handleMSPointerTypeQualifierAttr(TypeProcessingState &state,
                                              ParsedAttr &attr, QualType &type);
@@ -627,7 +627,7 @@ static void distributeFunctionTypeAttr(TypeProcessingState &state,
 static bool distributeFunctionTypeAttrToInnermost(
     TypeProcessingState &state, ParsedAttr &attr,
     ParsedAttributesView &attrList, QualType &declSpecType,
-    Sema::CUDAFunctionTarget CFT) {
+    CUDAFunctionTarget CFT) {
   Declarator &declarator = state.getDeclarator();
 
   // Put it on the innermost function chunk, if there is one.
@@ -644,10 +644,10 @@ static bool distributeFunctionTypeAttrToInnermost(
 
 /// A function type attribute was written in the decl spec.  Try to
 /// apply it somewhere.
-static void
-distributeFunctionTypeAttrFromDeclSpec(TypeProcessingState &state,
-                                       ParsedAttr &attr, QualType &declSpecType,
-                                       Sema::CUDAFunctionTarget CFT) {
+static void distributeFunctionTypeAttrFromDeclSpec(TypeProcessingState &state,
+                                                   ParsedAttr &attr,
+                                                   QualType &declSpecType,
+                                                   CUDAFunctionTarget CFT) {
   state.saveDeclSpecAttrs();
 
   // Try to distribute to the innermost.
@@ -664,9 +664,10 @@ distributeFunctionTypeAttrFromDeclSpec(TypeProcessingState &state,
 /// Try to apply it somewhere.
 /// `Attrs` is the attribute list containing the declaration (either of the
 /// declarator or the declaration).
-static void distributeFunctionTypeAttrFromDeclarator(
-    TypeProcessingState &state, ParsedAttr &attr, QualType &declSpecType,
-    Sema::CUDAFunctionTarget CFT) {
+static void distributeFunctionTypeAttrFromDeclarator(TypeProcessingState &state,
+                                                     ParsedAttr &attr,
+                                                     QualType &declSpecType,
+                                                     CUDAFunctionTarget CFT) {
   Declarator &declarator = state.getDeclarator();
 
   // Try to distribute to the innermost.
@@ -694,7 +695,7 @@ static void distributeFunctionTypeAttrFromDeclarator(
 /// declarator or the declaration).
 static void distributeTypeAttrsFromDeclarator(TypeProcessingState &state,
                                               QualType &declSpecType,
-                                              Sema::CUDAFunctionTarget CFT) {
+                                              CUDAFunctionTarget CFT) {
   // The called functions in this loop actually remove things from the current
   // list, so iterating over the existing list isn't possible.  Instead, make a
   // non-owning copy and iterate over that.
@@ -2734,7 +2735,7 @@ QualType Sema::BuildArrayType(QualType T, ArraySizeModifier ASM,
       bool IsCUDADevice = (getLangOpts().CUDA && getLangOpts().CUDAIsDevice);
       targetDiag(Loc,
                  IsCUDADevice ? diag::err_cuda_vla : diag::err_vla_unsupported)
-          << (IsCUDADevice ? CurrentCUDATarget() : 0);
+          << (IsCUDADevice ? llvm::to_underlying(CurrentCUDATarget()) : 0);
     } else if (sema::FunctionScopeInfo *FSI = getCurFunction()) {
       // VLAs are supported on this target, but we may need to do delayed
       // checking that the VLA is not being used within a coroutine.
@@ -8028,8 +8029,7 @@ static bool handleArmStateAttribute(Sema &S,
 /// Process an individual function attribute.  Returns true to
 /// indicate that the attribute was handled, false if it wasn't.
 static bool handleFunctionTypeAttr(TypeProcessingState &state, ParsedAttr &attr,
-                                   QualType &type,
-                                   Sema::CUDAFunctionTarget CFT) {
+                                   QualType &type, CUDAFunctionTarget CFT) {
   Sema &S = state.getSema();
 
   FunctionTypeUnwrapper unwrapped(S, type);
@@ -8863,7 +8863,7 @@ static void HandleHLSLParamModifierAttr(QualType &CurType,
 static void processTypeAttrs(TypeProcessingState &state, QualType &type,
                              TypeAttrLocation TAL,
                              const ParsedAttributesView &attrs,
-                             Sema::CUDAFunctionTarget CFT) {
+                             CUDAFunctionTarget CFT) {
 
   state.setParsedNoDeref(false);
   if (attrs.empty())
-- 
GitLab


From 3bb147e05365680d0a954d823a7c014f357448e7 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Fri, 12 Apr 2024 10:54:35 +0100
Subject: [PATCH 049/464] [X86] combine-sse41-intrinsics.ll - add AVX2 test
 coverage

---
 .../CodeGen/X86/combine-sse41-intrinsics.ll   | 52 +++++++++++++------
 1 file changed, 36 insertions(+), 16 deletions(-)

diff --git a/llvm/test/CodeGen/X86/combine-sse41-intrinsics.ll b/llvm/test/CodeGen/X86/combine-sse41-intrinsics.ll
index cbb5bd09c239..6034a24099e4 100644
--- a/llvm/test/CodeGen/X86/combine-sse41-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/combine-sse41-intrinsics.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=sse4.1 | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx    | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx    | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx2   | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
 
 
 define <2 x double> @test_x86_sse41_blend_pd(<2 x double> %a0, <2 x double> %a1) {
@@ -146,12 +147,18 @@ define <16 x i8> @demandedelts_pblendvb(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>
 ; SSE-NEXT:    movdqa %xmm3, %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: demandedelts_pblendvb:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX-NEXT:    vpshufb %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    retq
+; AVX1-LABEL: demandedelts_pblendvb:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpshufb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: demandedelts_pblendvb:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpbroadcastb %xmm0, %xmm0
+; AVX2-NEXT:    retq
   %1 = shufflevector <16 x i8> %a0, <16 x i8> undef, <16 x i32> zeroinitializer
   %2 = shufflevector <16 x i8> %a1, <16 x i8> undef, <16 x i32> zeroinitializer
   %3 = shufflevector <16 x i8> %a2, <16 x i8> undef, <16 x i32> zeroinitializer
@@ -193,15 +200,28 @@ define <4 x float> @demandedbits_uitofp_blendvps(<4 x float> %a0, <4 x float> %a
 ; SSE-NEXT:    movaps %xmm3, %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: demandedbits_uitofp_blendvps:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpblendw {{.*#+}} xmm3 = xmm2[0],mem[1],xmm2[2],mem[3],xmm2[4],mem[5],xmm2[6],mem[7]
-; AVX-NEXT:    vpsrld $16, %xmm2, %xmm2
-; AVX-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],mem[1],xmm2[2],mem[3],xmm2[4],mem[5],xmm2[6],mem[7]
-; AVX-NEXT:    vsubps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
-; AVX-NEXT:    vaddps %xmm2, %xmm3, %xmm2
-; AVX-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    retq
+; AVX1-LABEL: demandedbits_uitofp_blendvps:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm2[0],mem[1],xmm2[2],mem[3],xmm2[4],mem[5],xmm2[6],mem[7]
+; AVX1-NEXT:    vpsrld $16, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],mem[1],xmm2[2],mem[3],xmm2[4],mem[5],xmm2[6],mem[7]
+; AVX1-NEXT:    vsubps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2
+; AVX1-NEXT:    vaddps %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: demandedbits_uitofp_blendvps:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [1258291200,1258291200,1258291200,1258291200]
+; AVX2-NEXT:    vpblendw {{.*#+}} xmm3 = xmm2[0],xmm3[1],xmm2[2],xmm3[3],xmm2[4],xmm3[5],xmm2[6],xmm3[7]
+; AVX2-NEXT:    vpsrld $16, %xmm2, %xmm2
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [1392508928,1392508928,1392508928,1392508928]
+; AVX2-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm4[1],xmm2[2],xmm4[3],xmm2[4],xmm4[5],xmm2[6],xmm4[7]
+; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm4 = [5.49764202E+11,5.49764202E+11,5.49764202E+11,5.49764202E+11]
+; AVX2-NEXT:    vsubps %xmm4, %xmm2, %xmm2
+; AVX2-NEXT:    vaddps %xmm2, %xmm3, %xmm2
+; AVX2-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
   %cvt = uitofp <4 x i32> %a2 to <4 x float>
   %sel = tail call noundef <4 x float> @llvm.x86.sse41.blendvps(<4 x float> %a0, <4 x float> %a1, <4 x float> %cvt)
   ret <4 x float> %sel
-- 
GitLab


From 8c0f52e9d5a99bf96bb64ac23b5893482c292527 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Fri, 12 Apr 2024 11:23:51 +0100
Subject: [PATCH 050/464] [DAG] Fold
 extract_subvector(insert_subvector(x,y,c1),c2) --> extract_subvector(y,c2-c1)
 (#87925)

If the extract_subvector is cheap, attempt to extract directly from an inserted subvector
---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp |   17 +
 .../any_extend_vector_inreg_of_broadcast.ll   |   42 +-
 llvm/test/CodeGen/X86/dpbusd.ll               |    2 +-
 llvm/test/CodeGen/X86/dpbusd_i4.ll            |    2 +-
 .../vector-interleaved-load-i16-stride-3.ll   | 1300 ++++++++---------
 .../vector-interleaved-store-i8-stride-7.ll   |  602 ++++----
 .../zero_extend_vector_inreg_of_broadcast.ll  |   28 +-
 7 files changed, 987 insertions(+), 1006 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 24f50b87c4cf..d9c6c28d5dac 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -24463,6 +24463,23 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode *N) {
       if (!LegalOperations || TLI.isOperationLegal(ISD::SPLAT_VECTOR, NVT))
         return DAG.getSplatVector(NVT, DL, V.getOperand(0));
 
+  // extract_subvector(insert_subvector(x,y,c1),c2)
+  //  --> extract_subvector(y,c2-c1)
+  // iff we're just extracting from the inserted subvector.
+  if (V.getOpcode() == ISD::INSERT_SUBVECTOR) {
+    SDValue InsSub = V.getOperand(1);
+    EVT InsSubVT = InsSub.getValueType();
+    unsigned NumInsElts = InsSubVT.getVectorMinNumElements();
+    unsigned InsIdx = V.getConstantOperandVal(2);
+    unsigned NumSubElts = NVT.getVectorMinNumElements();
+    if (InsIdx <= ExtIdx && (ExtIdx + NumSubElts) <= (InsIdx + NumInsElts) &&
+        TLI.isExtractSubvectorCheap(NVT, InsSubVT, ExtIdx - InsIdx)) {
+      SDLoc DL(N);
+      return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NVT, InsSub,
+                         DAG.getVectorIdxConstant(ExtIdx - InsIdx, DL));
+    }
+  }
+
   // Try to move vector bitcast after extract_subv by scaling extraction index:
   // extract_subv (bitcast X), Index --> bitcast (extract_subv X, Index')
   if (V.getOpcode() == ISD::BITCAST &&
diff --git a/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll b/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll
index 4242d8483e72..39c7ce1413d1 100644
--- a/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll
+++ b/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll
@@ -314,8 +314,8 @@ define void @vec64_i16_widen_to_i32_factor2_broadcast_to_v2i32_factor2(ptr %in.v
 ;
 ; AVX512F-LABEL: vec64_i16_widen_to_i32_factor2_broadcast_to_v2i32_factor2:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,10,11,0,1,14,15,u,u,u,u,u,u,u,u]
 ; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)
@@ -324,8 +324,8 @@ define void @vec64_i16_widen_to_i32_factor2_broadcast_to_v2i32_factor2(ptr %in.v
 ;
 ; AVX512DQ-LABEL: vec64_i16_widen_to_i32_factor2_broadcast_to_v2i32_factor2:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512DQ-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,10,11,0,1,14,15,u,u,u,u,u,u,u,u]
 ; AVX512DQ-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, (%rcx)
@@ -981,7 +981,7 @@ define void @vec128_i32_widen_to_i64_factor2_broadcast_to_v2i64_factor2(ptr %in.
 ; AVX512F-NEXT:    vpmovsxbd {{.*#+}} xmm0 = [0,5,0,7]
 ; AVX512F-NEXT:    vmovdqa (%rdi), %ymm1
 ; AVX512F-NEXT:    vpaddb (%rsi), %ymm1, %ymm1
-; AVX512F-NEXT:    vpermd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpermd %ymm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)
 ; AVX512F-NEXT:    vzeroupper
@@ -992,7 +992,7 @@ define void @vec128_i32_widen_to_i64_factor2_broadcast_to_v2i64_factor2(ptr %in.
 ; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} xmm0 = [0,5,0,7]
 ; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm1
 ; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpermd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpermd %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, (%rcx)
 ; AVX512DQ-NEXT:    vzeroupper
@@ -3507,13 +3507,12 @@ define void @vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12(ptr %i
 ;
 ; AVX512F-LABEL: vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX512F-NEXT:    vmovdqa 48(%rdi), %xmm1
 ; AVX512F-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX512F-NEXT:    vpbroadcastw %xmm0, %ymm2
-; AVX512F-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2],ymm1[3],ymm2[4],ymm1[5],ymm2[6],ymm1[7],ymm2[8],ymm1[9],ymm2[10],ymm1[11],ymm2[12],ymm1[13],ymm2[14],ymm1[15]
+; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512F-NEXT:    vpbroadcastw %xmm0, %ymm0
+; AVX512F-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7],ymm0[8],ymm1[9],ymm0[10],ymm1[11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
 ; AVX512F-NEXT:    vpaddb (%rdx), %ymm1, %ymm1
 ; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm0, 32(%rcx)
@@ -3523,13 +3522,12 @@ define void @vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12(ptr %i
 ;
 ; AVX512DQ-LABEL: vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX512DQ-NEXT:    vmovdqa 48(%rdi), %xmm1
 ; AVX512DQ-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX512DQ-NEXT:    vpbroadcastw %xmm0, %ymm2
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2],ymm1[3],ymm2[4],ymm1[5],ymm2[6],ymm1[7],ymm2[8],ymm1[9],ymm2[10],ymm1[11],ymm2[12],ymm1[13],ymm2[14],ymm1[15]
+; AVX512DQ-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpbroadcastw %xmm0, %ymm0
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7],ymm0[8],ymm1[9],ymm0[10],ymm1[11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
 ; AVX512DQ-NEXT:    vpaddb (%rdx), %ymm1, %ymm1
 ; AVX512DQ-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, 32(%rcx)
@@ -3768,10 +3766,10 @@ define void @vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6(ptr %in.
 ;
 ; AVX512F-LABEL: vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX512F-NEXT:    vmovdqa 48(%rdi), %xmm1
 ; AVX512F-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
+; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512F-NEXT:    vpbroadcastq %xmm0, %ymm2
 ; AVX512F-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1,2,3],ymm2[4],ymm1[5,6,7],ymm2[8],ymm1[9,10,11],ymm2[12],ymm1[13,14,15]
 ; AVX512F-NEXT:    vpbroadcastw %xmm0, %ymm0
@@ -3784,10 +3782,10 @@ define void @vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6(ptr %in.
 ;
 ; AVX512DQ-LABEL: vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX512DQ-NEXT:    vmovdqa 48(%rdi), %xmm1
 ; AVX512DQ-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpbroadcastq %xmm0, %ymm2
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1,2,3],ymm2[4],ymm1[5,6,7],ymm2[8],ymm1[9,10,11],ymm2[12],ymm1[13,14,15]
 ; AVX512DQ-NEXT:    vpbroadcastw %xmm0, %ymm0
@@ -4147,9 +4145,9 @@ define void @vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2(ptr %
 ;
 ; AVX512F-LABEL: vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX512F-NEXT:    vmovdqa 48(%rdi), %xmm1
+; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512F-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
 ; AVX512F-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]
 ; AVX512F-NEXT:    vpbroadcastw %xmm0, %xmm0
@@ -4161,9 +4159,9 @@ define void @vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2(ptr %
 ;
 ; AVX512DQ-LABEL: vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX512DQ-NEXT:    vmovdqa 48(%rdi), %xmm1
+; AVX512DQ-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]
 ; AVX512DQ-NEXT:    vpbroadcastw %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/dpbusd.ll b/llvm/test/CodeGen/X86/dpbusd.ll
index fbea08eb1e55..04d7a9691b64 100644
--- a/llvm/test/CodeGen/X86/dpbusd.ll
+++ b/llvm/test/CodeGen/X86/dpbusd.ll
@@ -26,7 +26,7 @@ define i32 @no_dpbusd(ptr%a, ptr%b, i32 %c, i32 %n) {
 ; AVX512-NEXT:    vpmovzxbw {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
 ; AVX512-NEXT:    vpmaddwd %ymm0, %ymm1, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
diff --git a/llvm/test/CodeGen/X86/dpbusd_i4.ll b/llvm/test/CodeGen/X86/dpbusd_i4.ll
index 906fead7f8db..a212f99680ef 100644
--- a/llvm/test/CodeGen/X86/dpbusd_i4.ll
+++ b/llvm/test/CodeGen/X86/dpbusd_i4.ll
@@ -86,7 +86,7 @@ define i32 @mul_sext_i4i4(<16 x i4> %a, <16 x i4> %b, i32 %c) {
 ; CHECK-NEXT:    vpsraw $12, %ymm0, %ymm0
 ; CHECK-NEXT:    vpmaddwd %ymm1, %ymm0, %ymm0
 ; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; CHECK-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-3.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-3.ll
index 1436922f9dd1..6d5fc9ed0ab5 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-3.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-3.ll
@@ -1828,22 +1828,22 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX512-LABEL: load_i16_stride3_vf32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm1 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
 ; AVX512-NEXT:    vmovdqa 128(%rdi), %ymm5
 ; AVX512-NEXT:    vmovdqa 160(%rdi), %ymm6
-; AVX512-NEXT:    vmovdqa %ymm1, %ymm0
-; AVX512-NEXT:    vpternlogq $202, %ymm5, %ymm6, %ymm0
-; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2,3],ymm2[4],ymm0[5,6],ymm2[7],ymm0[8],ymm2[9],ymm0[10,11],ymm2[12],ymm0[13,14],ymm2[15]
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm3 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
-; AVX512-NEXT:    vmovdqa 112(%rdi), %xmm0
+; AVX512-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512-NEXT:    vpternlogq $202, %ymm5, %ymm6, %ymm1
+; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
+; AVX512-NEXT:    vmovdqa 112(%rdi), %xmm1
 ; AVX512-NEXT:    vmovdqa 96(%rdi), %xmm2
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm2[0],xmm0[1],xmm2[2,3],xmm0[4],xmm2[5,6],xmm0[7]
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[0,1,6,7,12,13,2,3,8,9,14,15,u,u,u,u]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm7 = ymm4[0,1,2],ymm3[3,4,5,6,7]
 ; AVX512-NEXT:    vmovdqa (%rdi), %ymm8
 ; AVX512-NEXT:    vmovdqa 32(%rdi), %ymm9
-; AVX512-NEXT:    vmovdqa %ymm1, %ymm3
+; AVX512-NEXT:    vmovdqa %ymm0, %ymm3
 ; AVX512-NEXT:    vpternlogq $202, %ymm9, %ymm8, %ymm3
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm3[2,3,0,1]
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7],ymm3[8],ymm4[9],ymm3[10,11],ymm4[12],ymm3[13,14],ymm4[15]
@@ -1857,14 +1857,14 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,6,5,4,7]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm11[4,5,6,7]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm10, %zmm7
-; AVX512-NEXT:    vmovdqa %ymm1, %ymm10
+; AVX512-NEXT:    vmovdqa %ymm0, %ymm10
 ; AVX512-NEXT:    vpternlogq $202, %ymm6, %ymm5, %ymm10
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm11 = ymm10[2,3,0,1]
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0,1],ymm11[2],ymm10[3,4],ymm11[5],ymm10[6,7,8,9],ymm11[10],ymm10[11,12],ymm11[13],ymm10[14,15]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm11 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
 ; AVX512-NEXT:    vpshufb %ymm11, %ymm10, %ymm10
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]
-; AVX512-NEXT:    vpshufb %xmm11, %xmm12, %xmm12
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm12 = xmm12[2,3,8,9,14,15,4,5,10,11,10,11,10,11,10,11]
 ; AVX512-NEXT:    vpblendw {{.*#+}} xmm12 = xmm12[0,1,2,3,4],xmm10[5,6,7]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm10 = ymm12[0,1,2,3],ymm10[4,5,6,7]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm12 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
@@ -1885,21 +1885,19 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1,2],ymm5[3],ymm12[4,5],ymm5[6],ymm12[7],ymm5[8],ymm12[9,10],ymm5[11],ymm12[12,13],ymm5[14],ymm12[15]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm6 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
 ; AVX512-NEXT:    vpshufb %ymm6, %ymm5, %ymm5
-; AVX512-NEXT:    vpternlogq $202, %ymm8, %ymm9, %ymm1
-; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm1[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm8[0],ymm1[1,2],ymm8[3],ymm1[4,5],ymm8[6],ymm1[7],ymm8[8],ymm1[9,10],ymm8[11],ymm1[12,13],ymm8[14],ymm1[15]
-; AVX512-NEXT:    vpshufb %ymm6, %ymm1, %ymm1
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]
-; AVX512-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
-; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5,6,7]
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2],xmm0[3,4],xmm2[5],xmm0[6,7]
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; AVX512-NEXT:    vextracti32x4 $2, %zmm0, %xmm0
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm5[5,6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3,4],xmm2[5],xmm1[6,7]
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm5[5,6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm5[4,5,6,7]
+; AVX512-NEXT:    vpternlogq $202, %ymm8, %ymm9, %ymm0
+; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1,2],ymm2[3],ymm0[4,5],ymm2[6],ymm0[7],ymm2[8],ymm0[9,10],ymm2[11],ymm0[12,13],ymm2[14],ymm0[15]
+; AVX512-NEXT:    vpshufb %ymm6, %ymm0, %ymm0
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]
+; AVX512-NEXT:    vpshufb %xmm6, %xmm2, %xmm2
+; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5,6,7]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vmovdqa64 %zmm7, (%rsi)
 ; AVX512-NEXT:    vmovdqa64 %zmm10, (%rdx)
 ; AVX512-NEXT:    vmovdqa64 %zmm0, (%rcx)
@@ -1908,22 +1906,22 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX512-FCP-LABEL: load_i16_stride3_vf32:
 ; AVX512-FCP:       # %bb.0:
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm1 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
 ; AVX512-FCP-NEXT:    vmovdqa 128(%rdi), %ymm5
 ; AVX512-FCP-NEXT:    vmovdqa 160(%rdi), %ymm6
-; AVX512-FCP-NEXT:    vmovdqa %ymm1, %ymm0
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm5, %ymm6, %ymm0
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2,3],ymm2[4],ymm0[5,6],ymm2[7],ymm0[8],ymm2[9],ymm0[10,11],ymm2[12],ymm0[13,14],ymm2[15]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
-; AVX512-FCP-NEXT:    vmovdqa 112(%rdi), %xmm0
+; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm5, %ymm6, %ymm1
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
+; AVX512-FCP-NEXT:    vmovdqa 112(%rdi), %xmm1
 ; AVX512-FCP-NEXT:    vmovdqa 96(%rdi), %xmm2
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm2[0],xmm0[1],xmm2[2,3],xmm0[4],xmm2[5,6],xmm0[7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[0,1,6,7,12,13,2,3,8,9,14,15,u,u,u,u]
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm7 = ymm4[0,1,2],ymm3[3,4,5,6,7]
 ; AVX512-FCP-NEXT:    vmovdqa (%rdi), %ymm8
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %ymm9
-; AVX512-FCP-NEXT:    vmovdqa %ymm1, %ymm3
+; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm3
 ; AVX512-FCP-NEXT:    vpternlogq $202, %ymm9, %ymm8, %ymm3
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm3[2,3,0,1]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7],ymm3[8],ymm4[9],ymm3[10,11],ymm4[12],ymm3[13,14],ymm4[15]
@@ -1937,14 +1935,14 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,6,5,4,7]
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm11[4,5,6,7]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm10, %zmm7
-; AVX512-FCP-NEXT:    vmovdqa %ymm1, %ymm10
+; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm10
 ; AVX512-FCP-NEXT:    vpternlogq $202, %ymm6, %ymm5, %ymm10
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm11 = ymm10[2,3,0,1]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0,1],ymm11[2],ymm10[3,4],ymm11[5],ymm10[6,7,8,9],ymm11[10],ymm10[11,12],ymm11[13],ymm10[14,15]
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm11 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
 ; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm10, %ymm10
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]
-; AVX512-FCP-NEXT:    vpshufb %xmm11, %xmm12, %xmm12
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm12 = xmm12[2,3,8,9,14,15,4,5,10,11,10,11,10,11,10,11]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm12 = xmm12[0,1,2,3,4],xmm10[5,6,7]
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm10 = ymm12[0,1,2,3],ymm10[4,5,6,7]
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm12 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
@@ -1965,21 +1963,19 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1,2],ymm5[3],ymm12[4,5],ymm5[6],ymm12[7],ymm5[8],ymm12[9,10],ymm5[11],ymm12[12,13],ymm5[14],ymm12[15]
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm6 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
 ; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm5, %ymm5
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm8, %ymm9, %ymm1
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm1[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm8[0],ymm1[1,2],ymm8[3],ymm1[4,5],ymm8[6],ymm1[7],ymm8[8],ymm1[9,10],ymm8[11],ymm1[12,13],ymm8[14],ymm1[15]
-; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm1, %ymm1
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]
-; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
-; AVX512-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5,6,7]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2],xmm0[3,4],xmm2[5],xmm0[6,7]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; AVX512-FCP-NEXT:    vextracti32x4 $2, %zmm0, %xmm0
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm5[5,6,7]
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3,4],xmm2[5],xmm1[6,7]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm5[5,6,7]
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm5[4,5,6,7]
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm8, %ymm9, %ymm0
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1,2],ymm2[3],ymm0[4,5],ymm2[6],ymm0[7],ymm2[8],ymm0[9,10],ymm2[11],ymm0[12,13],ymm2[14],ymm0[15]
+; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm0, %ymm0
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]
+; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm2, %xmm2
+; AVX512-FCP-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5,6,7]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm7, (%rsi)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm10, (%rdx)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, (%rcx)
@@ -1988,22 +1984,22 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX512DQ-LABEL: load_i16_stride3_vf32:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm1 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
 ; AVX512DQ-NEXT:    vmovdqa 128(%rdi), %ymm5
 ; AVX512DQ-NEXT:    vmovdqa 160(%rdi), %ymm6
-; AVX512DQ-NEXT:    vmovdqa %ymm1, %ymm0
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm5, %ymm6, %ymm0
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2,3],ymm2[4],ymm0[5,6],ymm2[7],ymm0[8],ymm2[9],ymm0[10,11],ymm2[12],ymm0[13,14],ymm2[15]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm3 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
-; AVX512DQ-NEXT:    vmovdqa 112(%rdi), %xmm0
+; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm5, %ymm6, %ymm1
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
+; AVX512DQ-NEXT:    vmovdqa 112(%rdi), %xmm1
 ; AVX512DQ-NEXT:    vmovdqa 96(%rdi), %xmm2
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm2[0],xmm0[1],xmm2[2,3],xmm0[4],xmm2[5,6],xmm0[7]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[0,1,6,7,12,13,2,3,8,9,14,15,u,u,u,u]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm7 = ymm4[0,1,2],ymm3[3,4,5,6,7]
 ; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm8
 ; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm9
-; AVX512DQ-NEXT:    vmovdqa %ymm1, %ymm3
+; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm3
 ; AVX512DQ-NEXT:    vpternlogq $202, %ymm9, %ymm8, %ymm3
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm3[2,3,0,1]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7],ymm3[8],ymm4[9],ymm3[10,11],ymm4[12],ymm3[13,14],ymm4[15]
@@ -2017,14 +2013,14 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,6,5,4,7]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm11[4,5,6,7]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm10, %zmm7
-; AVX512DQ-NEXT:    vmovdqa %ymm1, %ymm10
+; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm10
 ; AVX512DQ-NEXT:    vpternlogq $202, %ymm6, %ymm5, %ymm10
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm11 = ymm10[2,3,0,1]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0,1],ymm11[2],ymm10[3,4],ymm11[5],ymm10[6,7,8,9],ymm11[10],ymm10[11,12],ymm11[13],ymm10[14,15]
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm11 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
 ; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm10, %ymm10
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]
-; AVX512DQ-NEXT:    vpshufb %xmm11, %xmm12, %xmm12
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm12 = xmm12[2,3,8,9,14,15,4,5,10,11,10,11,10,11,10,11]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm12 = xmm12[0,1,2,3,4],xmm10[5,6,7]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm10 = ymm12[0,1,2,3],ymm10[4,5,6,7]
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm12 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
@@ -2045,21 +2041,19 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1,2],ymm5[3],ymm12[4,5],ymm5[6],ymm12[7],ymm5[8],ymm12[9,10],ymm5[11],ymm12[12,13],ymm5[14],ymm12[15]
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm6 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
 ; AVX512DQ-NEXT:    vpshufb %ymm6, %ymm5, %ymm5
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm8, %ymm9, %ymm1
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm1[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm8[0],ymm1[1,2],ymm8[3],ymm1[4,5],ymm8[6],ymm1[7],ymm8[8],ymm1[9,10],ymm8[11],ymm1[12,13],ymm8[14],ymm1[15]
-; AVX512DQ-NEXT:    vpshufb %ymm6, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]
-; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5,6,7]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2],xmm0[3,4],xmm2[5],xmm0[6,7]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; AVX512DQ-NEXT:    vextracti32x4 $2, %zmm0, %xmm0
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm5[5,6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3,4],xmm2[5],xmm1[6,7]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm5[5,6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm5[4,5,6,7]
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm8, %ymm9, %ymm0
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1,2],ymm2[3],ymm0[4,5],ymm2[6],ymm0[7],ymm2[8],ymm0[9,10],ymm2[11],ymm0[12,13],ymm2[14],ymm0[15]
+; AVX512DQ-NEXT:    vpshufb %ymm6, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]
+; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm2, %xmm2
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5,6,7]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm7, (%rsi)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm10, (%rdx)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm0, (%rcx)
@@ -2068,22 +2062,22 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX512DQ-FCP-LABEL: load_i16_stride3_vf32:
 ; AVX512DQ-FCP:       # %bb.0:
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm1 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
 ; AVX512DQ-FCP-NEXT:    vmovdqa 128(%rdi), %ymm5
 ; AVX512DQ-FCP-NEXT:    vmovdqa 160(%rdi), %ymm6
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm1, %ymm0
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm5, %ymm6, %ymm0
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2,3],ymm2[4],ymm0[5,6],ymm2[7],ymm0[8],ymm2[9],ymm0[10,11],ymm2[12],ymm0[13,14],ymm2[15]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
-; AVX512DQ-FCP-NEXT:    vmovdqa 112(%rdi), %xmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm5, %ymm6, %ymm1
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
+; AVX512DQ-FCP-NEXT:    vmovdqa 112(%rdi), %xmm1
 ; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdi), %xmm2
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm2[0],xmm0[1],xmm2[2,3],xmm0[4],xmm2[5,6],xmm0[7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[0,1,6,7,12,13,2,3,8,9,14,15,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm7 = ymm4[0,1,2],ymm3[3,4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %ymm8
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %ymm9
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm1, %ymm3
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm3
 ; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm9, %ymm8, %ymm3
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm3[2,3,0,1]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7],ymm3[8],ymm4[9],ymm3[10,11],ymm4[12],ymm3[13,14],ymm4[15]
@@ -2097,14 +2091,14 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,6,5,4,7]
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm11[4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm10, %zmm7
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm1, %ymm10
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm10
 ; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm6, %ymm5, %ymm10
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm11 = ymm10[2,3,0,1]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0,1],ymm11[2],ymm10[3,4],ymm11[5],ymm10[6,7,8,9],ymm11[10],ymm10[11,12],ymm11[13],ymm10[14,15]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm11 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm10, %ymm10
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm11, %xmm12, %xmm12
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm12 = xmm12[2,3,8,9,14,15,4,5,10,11,10,11,10,11,10,11]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm12 = xmm12[0,1,2,3,4],xmm10[5,6,7]
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm10 = ymm12[0,1,2,3],ymm10[4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm12 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
@@ -2125,21 +2119,19 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1,2],ymm5[3],ymm12[4,5],ymm5[6],ymm12[7],ymm5[8],ymm12[9,10],ymm5[11],ymm12[12,13],ymm5[14],ymm12[15]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm6 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm5, %ymm5
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm8, %ymm9, %ymm1
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm1[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm8[0],ymm1[1,2],ymm8[3],ymm1[4,5],ymm8[6],ymm1[7],ymm8[8],ymm1[9,10],ymm8[11],ymm1[12,13],ymm8[14],ymm1[15]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm1, %ymm1
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
-; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5,6,7]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2],xmm0[3,4],xmm2[5],xmm0[6,7]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; AVX512DQ-FCP-NEXT:    vextracti32x4 $2, %zmm0, %xmm0
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm5[5,6,7]
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3,4],xmm2[5],xmm1[6,7]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm5[5,6,7]
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm5[4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm8, %ymm9, %ymm0
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1,2],ymm2[3],ymm0[4,5],ymm2[6],ymm0[7],ymm2[8],ymm0[9,10],ymm2[11],ymm0[12,13],ymm2[14],ymm0[15]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm0, %ymm0
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm2, %xmm2
+; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5,6,7]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm7, (%rsi)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm10, (%rdx)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, (%rcx)
@@ -3500,688 +3492,668 @@ define void @load_i16_stride3_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-LABEL: load_i16_stride3_vf64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
-; AVX512-NEXT:    vmovdqa64 224(%rdi), %ymm20
-; AVX512-NEXT:    vmovdqa64 192(%rdi), %ymm21
+; AVX512-NEXT:    vmovdqa64 224(%rdi), %ymm18
+; AVX512-NEXT:    vmovdqa64 192(%rdi), %ymm20
 ; AVX512-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512-NEXT:    vpternlogq $202, %ymm20, %ymm21, %ymm1
-; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm1[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm3[1],ymm1[2,3],ymm3[4],ymm1[5,6],ymm3[7],ymm1[8],ymm3[9],ymm1[10,11],ymm3[12],ymm1[13,14],ymm3[15]
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]
-; AVX512-NEXT:    vpshufb %ymm3, %ymm1, %ymm5
-; AVX512-NEXT:    vmovdqa 272(%rdi), %xmm8
+; AVX512-NEXT:    vpternlogq $202, %ymm18, %ymm20, %ymm1
+; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm7 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]
+; AVX512-NEXT:    vpshufb %ymm7, %ymm2, %ymm5
+; AVX512-NEXT:    vmovdqa 272(%rdi), %xmm1
 ; AVX512-NEXT:    vmovdqa 256(%rdi), %xmm2
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm6 = xmm2[0,1],xmm8[2],xmm2[3,4],xmm8[5],xmm2[6,7]
-; AVX512-NEXT:    vmovdqa %xmm2, %xmm14
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm9 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]
-; AVX512-NEXT:    vpshufb %xmm9, %xmm6, %xmm6
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm6 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]
+; AVX512-NEXT:    vmovdqa %xmm2, %xmm3
+; AVX512-NEXT:    vmovdqa64 %xmm1, %xmm19
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm13 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]
+; AVX512-NEXT:    vpshufb %xmm13, %xmm6, %xmm6
 ; AVX512-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm6 = ymm5[0,1,2],ymm6[3,4,5,6,7],ymm5[8,9,10],ymm6[11,12,13,14,15]
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,6,5,4,7]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4,5,6,7]
-; AVX512-NEXT:    vmovdqa64 320(%rdi), %ymm22
-; AVX512-NEXT:    vmovdqa64 352(%rdi), %ymm23
-; AVX512-NEXT:    vmovdqa %ymm0, %ymm6
-; AVX512-NEXT:    vpternlogq $202, %ymm22, %ymm23, %ymm6
-; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0],ymm7[1],ymm6[2,3],ymm7[4],ymm6[5,6],ymm7[7],ymm6[8],ymm7[9],ymm6[10,11],ymm7[12],ymm6[13,14],ymm7[15]
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm11 = [0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
-; AVX512-NEXT:    vpshufb %ymm11, %ymm6, %ymm12
+; AVX512-NEXT:    vmovdqa64 320(%rdi), %ymm21
+; AVX512-NEXT:    vmovdqa64 352(%rdi), %ymm22
+; AVX512-NEXT:    vmovdqa %ymm0, %ymm8
+; AVX512-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm8
+; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm8[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm8 = ymm8[0],ymm9[1],ymm8[2,3],ymm9[4],ymm8[5,6],ymm9[7],ymm8[8],ymm9[9],ymm8[10,11],ymm9[12],ymm8[13,14],ymm9[15]
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm10 = [0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
+; AVX512-NEXT:    vpshufb %ymm10, %ymm8, %ymm11
 ; AVX512-NEXT:    vmovdqa 304(%rdi), %xmm1
 ; AVX512-NEXT:    vmovdqa 288(%rdi), %xmm2
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm13 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
 ; AVX512-NEXT:    vmovdqa %xmm2, %xmm4
-; AVX512-NEXT:    vmovdqa %xmm1, %xmm6
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm15 = [0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]
-; AVX512-NEXT:    vpshufb %xmm15, %xmm13, %xmm13
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm12 = ymm13[0,1,2],ymm12[3,4,5,6,7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm12, %zmm5, %zmm16
-; AVX512-NEXT:    vmovdqa64 128(%rdi), %ymm24
-; AVX512-NEXT:    vmovdqa 160(%rdi), %ymm13
+; AVX512-NEXT:    vmovdqa %xmm1, %xmm8
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm14 = [0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]
+; AVX512-NEXT:    vpshufb %xmm14, %xmm12, %xmm12
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm11 = ymm12[0,1,2],ymm11[3,4,5,6,7]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm11, %zmm5, %zmm16
+; AVX512-NEXT:    vmovdqa64 128(%rdi), %ymm23
+; AVX512-NEXT:    vmovdqa 160(%rdi), %ymm11
 ; AVX512-NEXT:    vmovdqa %ymm0, %ymm5
-; AVX512-NEXT:    vpternlogq $202, %ymm24, %ymm13, %ymm5
+; AVX512-NEXT:    vpternlogq $202, %ymm23, %ymm11, %ymm5
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm12 = ymm5[2,3,0,1]
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1],ymm5[2,3],ymm12[4],ymm5[5,6],ymm12[7],ymm5[8],ymm12[9],ymm5[10,11],ymm12[12],ymm5[13,14],ymm12[15]
-; AVX512-NEXT:    vpshufb %ymm11, %ymm5, %ymm5
-; AVX512-NEXT:    vmovdqa 112(%rdi), %xmm11
-; AVX512-NEXT:    vmovdqa 96(%rdi), %xmm12
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm10 = xmm12[0],xmm11[1],xmm12[2,3],xmm11[4],xmm12[5,6],xmm11[7]
-; AVX512-NEXT:    vpshufb %xmm15, %xmm10, %xmm10
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm10[0,1,2],ymm5[3,4,5,6,7]
-; AVX512-NEXT:    vmovdqa64 (%rdi), %ymm17
-; AVX512-NEXT:    vmovdqa 32(%rdi), %ymm5
+; AVX512-NEXT:    vpshufb %ymm10, %ymm5, %ymm10
+; AVX512-NEXT:    vmovdqa 112(%rdi), %xmm15
+; AVX512-NEXT:    vmovdqa 96(%rdi), %xmm5
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm12 = xmm5[0],xmm15[1],xmm5[2,3],xmm15[4],xmm5[5,6],xmm15[7]
+; AVX512-NEXT:    vpshufb %xmm14, %xmm12, %xmm12
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm6 = ymm12[0,1,2],ymm10[3,4,5,6,7]
+; AVX512-NEXT:    vmovdqa64 (%rdi), %ymm24
+; AVX512-NEXT:    vmovdqa 32(%rdi), %ymm12
 ; AVX512-NEXT:    vmovdqa %ymm0, %ymm10
-; AVX512-NEXT:    vpternlogq $202, %ymm5, %ymm17, %ymm10
-; AVX512-NEXT:    vpermq {{.*#+}} ymm15 = ymm10[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0],ymm15[1],ymm10[2,3],ymm15[4],ymm10[5,6],ymm15[7],ymm10[8],ymm15[9],ymm10[10,11],ymm15[12],ymm10[13,14],ymm15[15]
-; AVX512-NEXT:    vpshufb %ymm3, %ymm10, %ymm2
+; AVX512-NEXT:    vpternlogq $202, %ymm12, %ymm24, %ymm10
+; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm10[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm10[0],ymm1[1],ymm10[2,3],ymm1[4],ymm10[5,6],ymm1[7],ymm10[8],ymm1[9],ymm10[10,11],ymm1[12],ymm10[13,14],ymm1[15]
+; AVX512-NEXT:    vpshufb %ymm7, %ymm1, %ymm7
 ; AVX512-NEXT:    vmovdqa 80(%rdi), %xmm10
-; AVX512-NEXT:    vmovdqa 64(%rdi), %xmm15
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm3 = xmm15[0,1],xmm10[2],xmm15[3,4],xmm10[5],xmm15[6,7]
-; AVX512-NEXT:    vpshufb %xmm9, %xmm3, %xmm3
-; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm2[0,1,2],ymm3[3,4,5,6,7],ymm2[8,9,10],ymm3[11,12,13,14,15]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,4,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm18
-; AVX512-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512-NEXT:    vpternlogq $202, %ymm23, %ymm22, %ymm1
-; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7,8,9],ymm2[10],ymm1[11,12],ymm2[13],ymm1[14,15]
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm2 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
-; AVX512-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
-; AVX512-NEXT:    vmovdqa64 %ymm2, %ymm28
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1],xmm6[2],xmm4[3,4],xmm6[5],xmm4[6,7]
-; AVX512-NEXT:    vmovdqa64 %xmm6, %xmm25
+; AVX512-NEXT:    vmovdqa 64(%rdi), %xmm1
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm2 = xmm1[0,1],xmm10[2],xmm1[3,4],xmm10[5],xmm1[6,7]
+; AVX512-NEXT:    vpshufb %xmm13, %xmm2, %xmm2
+; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm7[0,1,2],ymm2[3,4,5,6,7],ymm7[8,9,10],ymm2[11,12,13,14,15]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,6,5,4,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm2, %zmm17
+; AVX512-NEXT:    vmovdqa %ymm0, %ymm2
+; AVX512-NEXT:    vpternlogq $202, %ymm22, %ymm21, %ymm2
+; AVX512-NEXT:    vpermq {{.*#+}} ymm6 = ymm2[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm6[2],ymm2[3,4],ymm6[5],ymm2[6,7,8,9],ymm6[10],ymm2[11,12],ymm6[13],ymm2[14,15]
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm9 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
+; AVX512-NEXT:    vpshufb %ymm9, %ymm2, %ymm2
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm7 = xmm4[0,1],xmm8[2],xmm4[3,4],xmm8[5],xmm4[6,7]
+; AVX512-NEXT:    vmovdqa64 %xmm8, %xmm25
 ; AVX512-NEXT:    vmovdqa64 %xmm4, %xmm26
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm6 = [2,3,8,9,14,15,4,5,10,11,10,11,10,11,10,11]
-; AVX512-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm1[5,6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm1[4,5,6,7]
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm9 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
-; AVX512-NEXT:    vmovdqa %ymm9, %ymm1
-; AVX512-NEXT:    vpternlogq $202, %ymm21, %ymm20, %ymm1
-; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm1[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm4[2],ymm1[3,4],ymm4[5],ymm1[6,7,8,9],ymm4[10],ymm1[11,12],ymm4[13],ymm1[14,15]
+; AVX512-NEXT:    vpshufb %xmm6, %xmm7, %xmm7
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm7 = xmm7[0,1,2,3,4],xmm2[5,6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm13 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
+; AVX512-NEXT:    vmovdqa %ymm13, %ymm2
+; AVX512-NEXT:    vpternlogq $202, %ymm20, %ymm18, %ymm2
+; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm2[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm4[2],ymm2[3,4],ymm4[5],ymm2[6,7,8,9],ymm4[10],ymm2[11,12],ymm4[13],ymm2[14,15]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm4 = [2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]
-; AVX512-NEXT:    vpshufb %ymm4, %ymm1, %ymm1
-; AVX512-NEXT:    vmovdqa %xmm14, %xmm7
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm14 = xmm8[0,1],xmm14[2],xmm8[3,4],xmm14[5],xmm8[6,7]
-; AVX512-NEXT:    vmovdqa64 %xmm8, %xmm27
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]
-; AVX512-NEXT:    vpshufb %xmm2, %xmm14, %xmm14
-; AVX512-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm14 = ymm1[0,1,2],ymm14[3,4,5,6,7],ymm1[8,9,10],ymm14[11,12,13,14,15]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,6,7,4]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm14[4,5,6,7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm19
-; AVX512-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512-NEXT:    vpternlogq $202, %ymm13, %ymm24, %ymm1
-; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm1[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm3[2],ymm1[3,4],ymm3[5],ymm1[6,7,8,9],ymm3[10],ymm1[11,12],ymm3[13],ymm1[14,15]
-; AVX512-NEXT:    vmovdqa64 %ymm28, %ymm3
-; AVX512-NEXT:    vpshufb %ymm3, %ymm1, %ymm1
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm3 = xmm12[0,1],xmm11[2],xmm12[3,4],xmm11[5],xmm12[6,7]
-; AVX512-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm1[5,6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
-; AVX512-NEXT:    vmovdqa %ymm9, %ymm3
-; AVX512-NEXT:    vpternlogq $202, %ymm17, %ymm5, %ymm3
-; AVX512-NEXT:    vpermq {{.*#+}} ymm6 = ymm3[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1],ymm6[2],ymm3[3,4],ymm6[5],ymm3[6,7,8,9],ymm6[10],ymm3[11,12],ymm6[13],ymm3[14,15]
-; AVX512-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm10[0,1],xmm15[2],xmm10[3,4],xmm15[5],xmm10[6,7]
-; AVX512-NEXT:    vpshufb %xmm2, %xmm4, %xmm2
-; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3,4,5,6,7],ymm3[8,9,10],ymm2[11,12,13,14,15]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,6,7,4]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX512-NEXT:    vpternlogq $226, %ymm24, %ymm9, %ymm13
-; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm13[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm13[1,2],ymm2[3],ymm13[4,5],ymm2[6],ymm13[7],ymm2[8],ymm13[9,10],ymm2[11],ymm13[12,13],ymm2[14],ymm13[15]
-; AVX512-NEXT:    vpternlogq $226, %ymm17, %ymm0, %ymm5
-; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm5[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm5[1,2],ymm3[3],ymm5[4,5],ymm3[6],ymm5[7],ymm3[8],ymm5[9,10],ymm3[11],ymm5[12,13],ymm3[14],ymm5[15]
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm4 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
-; AVX512-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm5 = xmm15[0],xmm10[1],xmm15[2,3],xmm10[4],xmm15[5,6],xmm10[7]
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm6 = [0,1,2,3,0,1,6,7,12,13,2,3,8,9,14,15]
-; AVX512-NEXT:    vpshufb %xmm6, %xmm5, %xmm5
-; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm5[5,6,7]
 ; AVX512-NEXT:    vpshufb %ymm4, %ymm2, %ymm2
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm5 = xmm11[0,1],xmm12[2],xmm11[3,4],xmm12[5],xmm11[6,7]
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm8 = [4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
-; AVX512-NEXT:    vpshufb %xmm8, %xmm5, %xmm5
-; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm3, %zmm5
-; AVX512-NEXT:    vextracti32x4 $2, %zmm5, %xmm5
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4],xmm2[5,6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-NEXT:    vmovdqa64 %xmm19, %xmm8
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm14 = xmm8[0,1],xmm3[2],xmm8[3,4],xmm3[5],xmm8[6,7]
+; AVX512-NEXT:    vmovdqa64 %xmm3, %xmm27
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]
+; AVX512-NEXT:    vpshufb %xmm3, %xmm14, %xmm14
+; AVX512-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm14 = ymm2[0,1,2],ymm14[3,4,5,6,7],ymm2[8,9,10],ymm14[11,12,13,14,15]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,6,7,4]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm14[4,5,6,7]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm2, %zmm19
+; AVX512-NEXT:    vmovdqa %ymm0, %ymm2
+; AVX512-NEXT:    vpternlogq $202, %ymm11, %ymm23, %ymm2
+; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm2[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm7[2],ymm2[3,4],ymm7[5],ymm2[6,7,8,9],ymm7[10],ymm2[11,12],ymm7[13],ymm2[14,15]
+; AVX512-NEXT:    vpshufb %ymm9, %ymm2, %ymm2
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm7 = xmm5[0,1],xmm15[2],xmm5[3,4],xmm15[5],xmm5[6,7]
+; AVX512-NEXT:    vpshufb %xmm6, %xmm7, %xmm6
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4],xmm2[5,6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-NEXT:    vmovdqa %ymm13, %ymm6
+; AVX512-NEXT:    vpternlogq $202, %ymm24, %ymm12, %ymm6
+; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0,1],ymm7[2],ymm6[3,4],ymm7[5],ymm6[6,7,8,9],ymm7[10],ymm6[11,12],ymm7[13],ymm6[14,15]
+; AVX512-NEXT:    vpshufb %ymm4, %ymm6, %ymm4
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm6 = xmm10[0,1],xmm1[2],xmm10[3,4],xmm1[5],xmm10[6,7]
+; AVX512-NEXT:    vpshufb %xmm3, %xmm6, %xmm3
+; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3,4,5,6,7],ymm4[8,9,10],ymm3[11,12,13,14,15]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,6,7,4]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
-; AVX512-NEXT:    vpternlogq $202, %ymm22, %ymm23, %ymm9
-; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm9[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm9[1,2],ymm3[3],ymm9[4,5],ymm3[6],ymm9[7],ymm3[8],ymm9[9,10],ymm3[11],ymm9[12,13],ymm3[14],ymm9[15]
-; AVX512-NEXT:    vpternlogq $202, %ymm21, %ymm20, %ymm0
-; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm0[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm5[0],ymm0[1,2],ymm5[3],ymm0[4,5],ymm5[6],ymm0[7],ymm5[8],ymm0[9,10],ymm5[11],ymm0[12,13],ymm5[14],ymm0[15]
-; AVX512-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
-; AVX512-NEXT:    vpshufb %ymm4, %ymm0, %ymm0
+; AVX512-NEXT:    vpternlogq $226, %ymm23, %ymm13, %ymm11
+; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm11[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm11[1,2],ymm3[3],ymm11[4,5],ymm3[6],ymm11[7],ymm3[8],ymm11[9,10],ymm3[11],ymm11[12,13],ymm3[14],ymm11[15]
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm11 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
+; AVX512-NEXT:    vpshufb %ymm11, %ymm3, %ymm3
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm15[0,1],xmm5[2],xmm15[3,4],xmm5[5],xmm15[6,7]
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm5 = [4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
+; AVX512-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512-NEXT:    vpternlogq $226, %ymm24, %ymm0, %ymm12
+; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm12[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0],ymm12[1,2],ymm4[3],ymm12[4,5],ymm4[6],ymm12[7],ymm4[8],ymm12[9,10],ymm4[11],ymm12[12,13],ymm4[14],ymm12[15]
+; AVX512-NEXT:    vpshufb %ymm11, %ymm4, %ymm4
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm10[1],xmm1[2,3],xmm10[4],xmm1[5,6],xmm10[7]
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm6 = [0,1,2,3,0,1,6,7,12,13,2,3,8,9,14,15]
+; AVX512-NEXT:    vpshufb %xmm6, %xmm1, %xmm1
+; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4],ymm1[5,6,7]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
+; AVX512-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm13
+; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm13[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm13[1,2],ymm3[3],ymm13[4,5],ymm3[6],ymm13[7],ymm3[8],ymm13[9,10],ymm3[11],ymm13[12,13],ymm3[14],ymm13[15]
+; AVX512-NEXT:    vmovdqa64 %xmm25, %xmm4
+; AVX512-NEXT:    vmovdqa64 %xmm26, %xmm7
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm7[2],xmm4[3,4],xmm7[5],xmm4[6,7]
+; AVX512-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
+; AVX512-NEXT:    vpshufb %ymm11, %ymm3, %ymm3
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512-NEXT:    vpternlogq $202, %ymm20, %ymm18, %ymm0
+; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm0[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm4[0],ymm0[1,2],ymm4[3],ymm0[4,5],ymm4[6],ymm0[7],ymm4[8],ymm0[9,10],ymm4[11],ymm0[12,13],ymm4[14],ymm0[15]
+; AVX512-NEXT:    vpshufb %ymm11, %ymm0, %ymm0
 ; AVX512-NEXT:    vmovdqa64 %xmm27, %xmm4
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm7[0],xmm4[1],xmm7[2,3],xmm4[4],xmm7[5,6],xmm4[7]
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0],xmm8[1],xmm4[2,3],xmm8[4],xmm4[5,6],xmm8[7]
 ; AVX512-NEXT:    vpshufb %xmm6, %xmm4, %xmm4
 ; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm4[5,6,7]
-; AVX512-NEXT:    vmovdqa64 %xmm25, %xmm4
-; AVX512-NEXT:    vmovdqa64 %xmm26, %xmm5
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2],xmm4[3,4],xmm5[5],xmm4[6,7]
-; AVX512-NEXT:    vpshufb %xmm8, %xmm4, %xmm4
-; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm4
-; AVX512-NEXT:    vextracti32x4 $2, %zmm4, %xmm4
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
-; AVX512-NEXT:    vmovdqa64 %zmm18, (%rsi)
+; AVX512-NEXT:    vmovdqa64 %zmm17, (%rsi)
 ; AVX512-NEXT:    vmovdqa64 %zmm16, 64(%rsi)
 ; AVX512-NEXT:    vmovdqa64 %zmm19, 64(%rdx)
-; AVX512-NEXT:    vmovdqa64 %zmm1, (%rdx)
+; AVX512-NEXT:    vmovdqa64 %zmm2, (%rdx)
 ; AVX512-NEXT:    vmovdqa64 %zmm0, 64(%rcx)
-; AVX512-NEXT:    vmovdqa64 %zmm2, (%rcx)
+; AVX512-NEXT:    vmovdqa64 %zmm1, (%rcx)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
 ; AVX512-FCP-LABEL: load_i16_stride3_vf64:
 ; AVX512-FCP:       # %bb.0:
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
-; AVX512-FCP-NEXT:    vmovdqa64 224(%rdi), %ymm20
-; AVX512-FCP-NEXT:    vmovdqa64 192(%rdi), %ymm21
+; AVX512-FCP-NEXT:    vmovdqa64 224(%rdi), %ymm18
+; AVX512-FCP-NEXT:    vmovdqa64 192(%rdi), %ymm20
 ; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm20, %ymm21, %ymm1
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm1[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm3[1],ymm1[2,3],ymm3[4],ymm1[5,6],ymm3[7],ymm1[8],ymm3[9],ymm1[10,11],ymm3[12],ymm1[13,14],ymm3[15]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]
-; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm1, %ymm5
-; AVX512-FCP-NEXT:    vmovdqa 272(%rdi), %xmm8
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm18, %ymm20, %ymm1
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm7 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]
+; AVX512-FCP-NEXT:    vpshufb %ymm7, %ymm2, %ymm5
+; AVX512-FCP-NEXT:    vmovdqa 272(%rdi), %xmm1
 ; AVX512-FCP-NEXT:    vmovdqa 256(%rdi), %xmm2
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm6 = xmm2[0,1],xmm8[2],xmm2[3,4],xmm8[5],xmm2[6,7]
-; AVX512-FCP-NEXT:    vmovdqa %xmm2, %xmm14
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm9 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]
-; AVX512-FCP-NEXT:    vpshufb %xmm9, %xmm6, %xmm6
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm6 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]
+; AVX512-FCP-NEXT:    vmovdqa %xmm2, %xmm3
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm1, %xmm19
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm13 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]
+; AVX512-FCP-NEXT:    vpshufb %xmm13, %xmm6, %xmm6
 ; AVX512-FCP-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm5[0,1,2],ymm6[3,4,5,6,7],ymm5[8,9,10],ymm6[11,12,13,14,15]
 ; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,6,5,4,7]
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4,5,6,7]
-; AVX512-FCP-NEXT:    vmovdqa64 320(%rdi), %ymm22
-; AVX512-FCP-NEXT:    vmovdqa64 352(%rdi), %ymm23
-; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm6
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm22, %ymm23, %ymm6
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0],ymm7[1],ymm6[2,3],ymm7[4],ymm6[5,6],ymm7[7],ymm6[8],ymm7[9],ymm6[10,11],ymm7[12],ymm6[13,14],ymm7[15]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm11 = [0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
-; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm6, %ymm12
+; AVX512-FCP-NEXT:    vmovdqa64 320(%rdi), %ymm21
+; AVX512-FCP-NEXT:    vmovdqa64 352(%rdi), %ymm22
+; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm8
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm8
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm8[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm8 = ymm8[0],ymm9[1],ymm8[2,3],ymm9[4],ymm8[5,6],ymm9[7],ymm8[8],ymm9[9],ymm8[10,11],ymm9[12],ymm8[13,14],ymm9[15]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm10 = [0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
+; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm8, %ymm11
 ; AVX512-FCP-NEXT:    vmovdqa 304(%rdi), %xmm1
 ; AVX512-FCP-NEXT:    vmovdqa 288(%rdi), %xmm2
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm13 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
 ; AVX512-FCP-NEXT:    vmovdqa %xmm2, %xmm4
-; AVX512-FCP-NEXT:    vmovdqa %xmm1, %xmm6
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm15 = [0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]
-; AVX512-FCP-NEXT:    vpshufb %xmm15, %xmm13, %xmm13
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm12 = ymm13[0,1,2],ymm12[3,4,5,6,7]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm5, %zmm16
-; AVX512-FCP-NEXT:    vmovdqa64 128(%rdi), %ymm24
-; AVX512-FCP-NEXT:    vmovdqa 160(%rdi), %ymm13
+; AVX512-FCP-NEXT:    vmovdqa %xmm1, %xmm8
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm14 = [0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]
+; AVX512-FCP-NEXT:    vpshufb %xmm14, %xmm12, %xmm12
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm11 = ymm12[0,1,2],ymm11[3,4,5,6,7]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm5, %zmm16
+; AVX512-FCP-NEXT:    vmovdqa64 128(%rdi), %ymm23
+; AVX512-FCP-NEXT:    vmovdqa 160(%rdi), %ymm11
 ; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm5
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm24, %ymm13, %ymm5
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm23, %ymm11, %ymm5
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm12 = ymm5[2,3,0,1]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1],ymm5[2,3],ymm12[4],ymm5[5,6],ymm12[7],ymm5[8],ymm12[9],ymm5[10,11],ymm12[12],ymm5[13,14],ymm12[15]
-; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm5, %ymm5
-; AVX512-FCP-NEXT:    vmovdqa 112(%rdi), %xmm11
-; AVX512-FCP-NEXT:    vmovdqa 96(%rdi), %xmm12
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm10 = xmm12[0],xmm11[1],xmm12[2,3],xmm11[4],xmm12[5,6],xmm11[7]
-; AVX512-FCP-NEXT:    vpshufb %xmm15, %xmm10, %xmm10
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm10[0,1,2],ymm5[3,4,5,6,7]
-; AVX512-FCP-NEXT:    vmovdqa64 (%rdi), %ymm17
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %ymm5
+; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm5, %ymm10
+; AVX512-FCP-NEXT:    vmovdqa 112(%rdi), %xmm15
+; AVX512-FCP-NEXT:    vmovdqa 96(%rdi), %xmm5
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm12 = xmm5[0],xmm15[1],xmm5[2,3],xmm15[4],xmm5[5,6],xmm15[7]
+; AVX512-FCP-NEXT:    vpshufb %xmm14, %xmm12, %xmm12
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm6 = ymm12[0,1,2],ymm10[3,4,5,6,7]
+; AVX512-FCP-NEXT:    vmovdqa64 (%rdi), %ymm24
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %ymm12
 ; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm10
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm5, %ymm17, %ymm10
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm15 = ymm10[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0],ymm15[1],ymm10[2,3],ymm15[4],ymm10[5,6],ymm15[7],ymm10[8],ymm15[9],ymm10[10,11],ymm15[12],ymm10[13,14],ymm15[15]
-; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm10, %ymm2
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm12, %ymm24, %ymm10
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm10[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm10[0],ymm1[1],ymm10[2,3],ymm1[4],ymm10[5,6],ymm1[7],ymm10[8],ymm1[9],ymm10[10,11],ymm1[12],ymm10[13,14],ymm1[15]
+; AVX512-FCP-NEXT:    vpshufb %ymm7, %ymm1, %ymm7
 ; AVX512-FCP-NEXT:    vmovdqa 80(%rdi), %xmm10
-; AVX512-FCP-NEXT:    vmovdqa 64(%rdi), %xmm15
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm15[0,1],xmm10[2],xmm15[3,4],xmm10[5],xmm15[6,7]
-; AVX512-FCP-NEXT:    vpshufb %xmm9, %xmm3, %xmm3
-; AVX512-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm2[0,1,2],ymm3[3,4,5,6,7],ymm2[8,9,10],ymm3[11,12,13,14,15]
-; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,4,7]
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm18
-; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm23, %ymm22, %ymm1
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7,8,9],ymm2[10],ymm1[11,12],ymm2[13],ymm1[14,15]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm2 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
-; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm2, %ymm28
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1],xmm6[2],xmm4[3,4],xmm6[5],xmm4[6,7]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm6, %xmm25
+; AVX512-FCP-NEXT:    vmovdqa 64(%rdi), %xmm1
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm2 = xmm1[0,1],xmm10[2],xmm1[3,4],xmm10[5],xmm1[6,7]
+; AVX512-FCP-NEXT:    vpshufb %xmm13, %xmm2, %xmm2
+; AVX512-FCP-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm7[0,1,2],ymm2[3,4,5,6,7],ymm7[8,9,10],ymm2[11,12,13,14,15]
+; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,6,5,4,7]
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm2, %zmm17
+; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm2
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm22, %ymm21, %ymm2
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm2[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm6[2],ymm2[3,4],ymm6[5],ymm2[6,7,8,9],ymm6[10],ymm2[11,12],ymm6[13],ymm2[14,15]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm9 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
+; AVX512-FCP-NEXT:    vpshufb %ymm9, %ymm2, %ymm2
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm7 = xmm4[0,1],xmm8[2],xmm4[3,4],xmm8[5],xmm4[6,7]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm8, %xmm25
 ; AVX512-FCP-NEXT:    vmovdqa64 %xmm4, %xmm26
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm6 = [2,3,8,9,14,15,4,5,10,11,10,11,10,11,10,11]
-; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm1[5,6,7]
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm1[4,5,6,7]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm9 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
-; AVX512-FCP-NEXT:    vmovdqa %ymm9, %ymm1
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm21, %ymm20, %ymm1
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm1[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm4[2],ymm1[3,4],ymm4[5],ymm1[6,7,8,9],ymm4[10],ymm1[11,12],ymm4[13],ymm1[14,15]
+; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm7, %xmm7
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm7 = xmm7[0,1,2,3,4],xmm2[5,6,7]
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm13 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
+; AVX512-FCP-NEXT:    vmovdqa %ymm13, %ymm2
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm20, %ymm18, %ymm2
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm2[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm4[2],ymm2[3,4],ymm4[5],ymm2[6,7,8,9],ymm4[10],ymm2[11,12],ymm4[13],ymm2[14,15]
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm4 = [2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]
-; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm1, %ymm1
-; AVX512-FCP-NEXT:    vmovdqa %xmm14, %xmm7
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm14 = xmm8[0,1],xmm14[2],xmm8[3,4],xmm14[5],xmm8[6,7]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm8, %xmm27
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]
-; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm14, %xmm14
-; AVX512-FCP-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm14 = ymm1[0,1,2],ymm14[3,4,5,6,7],ymm1[8,9,10],ymm14[11,12,13,14,15]
-; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,6,7,4]
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm14[4,5,6,7]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm19
-; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm13, %ymm24, %ymm1
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm1[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm3[2],ymm1[3,4],ymm3[5],ymm1[6,7,8,9],ymm3[10],ymm1[11,12],ymm3[13],ymm1[14,15]
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm28, %ymm3
-; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm1, %ymm1
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm12[0,1],xmm11[2],xmm12[3,4],xmm11[5],xmm12[6,7]
-; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm1[5,6,7]
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
-; AVX512-FCP-NEXT:    vmovdqa %ymm9, %ymm3
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm17, %ymm5, %ymm3
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm3[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1],ymm6[2],ymm3[3,4],ymm6[5],ymm3[6,7,8,9],ymm6[10],ymm3[11,12],ymm6[13],ymm3[14,15]
-; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm10[0,1],xmm15[2],xmm10[3,4],xmm15[5],xmm10[6,7]
-; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm4, %xmm2
-; AVX512-FCP-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3,4,5,6,7],ymm3[8,9,10],ymm2[11,12,13,14,15]
-; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,6,7,4]
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX512-FCP-NEXT:    vpternlogq $226, %ymm24, %ymm9, %ymm13
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm13[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm13[1,2],ymm2[3],ymm13[4,5],ymm2[6],ymm13[7],ymm2[8],ymm13[9,10],ymm2[11],ymm13[12,13],ymm2[14],ymm13[15]
-; AVX512-FCP-NEXT:    vpternlogq $226, %ymm17, %ymm0, %ymm5
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm5[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm5[1,2],ymm3[3],ymm5[4,5],ymm3[6],ymm5[7],ymm3[8],ymm5[9,10],ymm3[11],ymm5[12,13],ymm3[14],ymm5[15]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm4 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
-; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm5 = xmm15[0],xmm10[1],xmm15[2,3],xmm10[4],xmm15[5,6],xmm10[7]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm6 = [0,1,2,3,0,1,6,7,12,13,2,3,8,9,14,15]
-; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm5, %xmm5
-; AVX512-FCP-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm5[5,6,7]
 ; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm2, %ymm2
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm5 = xmm11[0,1],xmm12[2],xmm11[3,4],xmm12[5],xmm11[6,7]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm8 = [4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
-; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm5, %xmm5
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm3, %zmm5
-; AVX512-FCP-NEXT:    vextracti32x4 $2, %zmm5, %xmm5
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4],xmm2[5,6,7]
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm19, %xmm8
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm14 = xmm8[0,1],xmm3[2],xmm8[3,4],xmm3[5],xmm8[6,7]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm3, %xmm27
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]
+; AVX512-FCP-NEXT:    vpshufb %xmm3, %xmm14, %xmm14
+; AVX512-FCP-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm14 = ymm2[0,1,2],ymm14[3,4,5,6,7],ymm2[8,9,10],ymm14[11,12,13,14,15]
+; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,6,7,4]
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm14[4,5,6,7]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm2, %zmm19
+; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm2
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm11, %ymm23, %ymm2
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm2[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm7[2],ymm2[3,4],ymm7[5],ymm2[6,7,8,9],ymm7[10],ymm2[11,12],ymm7[13],ymm2[14,15]
+; AVX512-FCP-NEXT:    vpshufb %ymm9, %ymm2, %ymm2
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm7 = xmm5[0,1],xmm15[2],xmm5[3,4],xmm15[5],xmm5[6,7]
+; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm7, %xmm6
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4],xmm2[5,6,7]
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-FCP-NEXT:    vmovdqa %ymm13, %ymm6
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm24, %ymm12, %ymm6
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0,1],ymm7[2],ymm6[3,4],ymm7[5],ymm6[6,7,8,9],ymm7[10],ymm6[11,12],ymm7[13],ymm6[14,15]
+; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm6, %ymm4
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm6 = xmm10[0,1],xmm1[2],xmm10[3,4],xmm1[5],xmm10[6,7]
+; AVX512-FCP-NEXT:    vpshufb %xmm3, %xmm6, %xmm3
+; AVX512-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3,4,5,6,7],ymm4[8,9,10],ymm3[11,12,13,14,15]
+; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,6,7,4]
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm22, %ymm23, %ymm9
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm9[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm9[1,2],ymm3[3],ymm9[4,5],ymm3[6],ymm9[7],ymm3[8],ymm9[9,10],ymm3[11],ymm9[12,13],ymm3[14],ymm9[15]
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm21, %ymm20, %ymm0
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm5 = ymm0[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm5[0],ymm0[1,2],ymm5[3],ymm0[4,5],ymm5[6],ymm0[7],ymm5[8],ymm0[9,10],ymm5[11],ymm0[12,13],ymm5[14],ymm0[15]
-; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
-; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm0, %ymm0
+; AVX512-FCP-NEXT:    vpternlogq $226, %ymm23, %ymm13, %ymm11
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm11[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm11[1,2],ymm3[3],ymm11[4,5],ymm3[6],ymm11[7],ymm3[8],ymm11[9,10],ymm3[11],ymm11[12,13],ymm3[14],ymm11[15]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm11 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
+; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm3, %ymm3
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm15[0,1],xmm5[2],xmm15[3,4],xmm5[5],xmm15[6,7]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm5 = [4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
+; AVX512-FCP-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512-FCP-NEXT:    vpternlogq $226, %ymm24, %ymm0, %ymm12
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm12[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0],ymm12[1,2],ymm4[3],ymm12[4,5],ymm4[6],ymm12[7],ymm4[8],ymm12[9,10],ymm4[11],ymm12[12,13],ymm4[14],ymm12[15]
+; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm4, %ymm4
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm10[1],xmm1[2,3],xmm10[4],xmm1[5,6],xmm10[7]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm6 = [0,1,2,3,0,1,6,7,12,13,2,3,8,9,14,15]
+; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm1, %xmm1
+; AVX512-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4],ymm1[5,6,7]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm13
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm13[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm13[1,2],ymm3[3],ymm13[4,5],ymm3[6],ymm13[7],ymm3[8],ymm13[9,10],ymm3[11],ymm13[12,13],ymm3[14],ymm13[15]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm25, %xmm4
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm26, %xmm7
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm7[2],xmm4[3,4],xmm7[5],xmm4[6,7]
+; AVX512-FCP-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
+; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm3, %ymm3
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm20, %ymm18, %ymm0
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm0[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm4[0],ymm0[1,2],ymm4[3],ymm0[4,5],ymm4[6],ymm0[7],ymm4[8],ymm0[9,10],ymm4[11],ymm0[12,13],ymm4[14],ymm0[15]
+; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm0, %ymm0
 ; AVX512-FCP-NEXT:    vmovdqa64 %xmm27, %xmm4
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm7[0],xmm4[1],xmm7[2,3],xmm4[4],xmm7[5,6],xmm4[7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0],xmm8[1],xmm4[2,3],xmm8[4],xmm4[5,6],xmm8[7]
 ; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm4, %xmm4
 ; AVX512-FCP-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm4[5,6,7]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm25, %xmm4
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm26, %xmm5
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2],xmm4[3,4],xmm5[5],xmm4[6,7]
-; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm4, %xmm4
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm4
-; AVX512-FCP-NEXT:    vextracti32x4 $2, %zmm4, %xmm4
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm18, (%rsi)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm17, (%rsi)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm16, 64(%rsi)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm19, 64(%rdx)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, (%rdx)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, (%rdx)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, 64(%rcx)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, (%rcx)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, (%rcx)
 ; AVX512-FCP-NEXT:    vzeroupper
 ; AVX512-FCP-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: load_i16_stride3_vf64:
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
-; AVX512DQ-NEXT:    vmovdqa64 224(%rdi), %ymm20
-; AVX512DQ-NEXT:    vmovdqa64 192(%rdi), %ymm21
+; AVX512DQ-NEXT:    vmovdqa64 224(%rdi), %ymm18
+; AVX512DQ-NEXT:    vmovdqa64 192(%rdi), %ymm20
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm20, %ymm21, %ymm1
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm1[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm3[1],ymm1[2,3],ymm3[4],ymm1[5,6],ymm3[7],ymm1[8],ymm3[9],ymm1[10,11],ymm3[12],ymm1[13,14],ymm3[15]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]
-; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm1, %ymm5
-; AVX512DQ-NEXT:    vmovdqa 272(%rdi), %xmm8
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm18, %ymm20, %ymm1
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm7 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]
+; AVX512DQ-NEXT:    vpshufb %ymm7, %ymm2, %ymm5
+; AVX512DQ-NEXT:    vmovdqa 272(%rdi), %xmm1
 ; AVX512DQ-NEXT:    vmovdqa 256(%rdi), %xmm2
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm6 = xmm2[0,1],xmm8[2],xmm2[3,4],xmm8[5],xmm2[6,7]
-; AVX512DQ-NEXT:    vmovdqa %xmm2, %xmm14
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm9 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]
-; AVX512DQ-NEXT:    vpshufb %xmm9, %xmm6, %xmm6
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm6 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]
+; AVX512DQ-NEXT:    vmovdqa %xmm2, %xmm3
+; AVX512DQ-NEXT:    vmovdqa64 %xmm1, %xmm19
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm13 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]
+; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm6, %xmm6
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm6 = ymm5[0,1,2],ymm6[3,4,5,6,7],ymm5[8,9,10],ymm6[11,12,13,14,15]
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,6,5,4,7]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa64 320(%rdi), %ymm22
-; AVX512DQ-NEXT:    vmovdqa64 352(%rdi), %ymm23
-; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm6
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm22, %ymm23, %ymm6
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0],ymm7[1],ymm6[2,3],ymm7[4],ymm6[5,6],ymm7[7],ymm6[8],ymm7[9],ymm6[10,11],ymm7[12],ymm6[13,14],ymm7[15]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm11 = [0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
-; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm6, %ymm12
+; AVX512DQ-NEXT:    vmovdqa64 320(%rdi), %ymm21
+; AVX512DQ-NEXT:    vmovdqa64 352(%rdi), %ymm22
+; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm8
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm8
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm8[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm8 = ymm8[0],ymm9[1],ymm8[2,3],ymm9[4],ymm8[5,6],ymm9[7],ymm8[8],ymm9[9],ymm8[10,11],ymm9[12],ymm8[13,14],ymm9[15]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm10 = [0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
+; AVX512DQ-NEXT:    vpshufb %ymm10, %ymm8, %ymm11
 ; AVX512DQ-NEXT:    vmovdqa 304(%rdi), %xmm1
 ; AVX512DQ-NEXT:    vmovdqa 288(%rdi), %xmm2
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm13 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
 ; AVX512DQ-NEXT:    vmovdqa %xmm2, %xmm4
-; AVX512DQ-NEXT:    vmovdqa %xmm1, %xmm6
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm15 = [0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]
-; AVX512DQ-NEXT:    vpshufb %xmm15, %xmm13, %xmm13
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm12 = ymm13[0,1,2],ymm12[3,4,5,6,7]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm12, %zmm5, %zmm16
-; AVX512DQ-NEXT:    vmovdqa64 128(%rdi), %ymm24
-; AVX512DQ-NEXT:    vmovdqa 160(%rdi), %ymm13
+; AVX512DQ-NEXT:    vmovdqa %xmm1, %xmm8
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm14 = [0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm12, %xmm12
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm11 = ymm12[0,1,2],ymm11[3,4,5,6,7]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm11, %zmm5, %zmm16
+; AVX512DQ-NEXT:    vmovdqa64 128(%rdi), %ymm23
+; AVX512DQ-NEXT:    vmovdqa 160(%rdi), %ymm11
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm5
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm24, %ymm13, %ymm5
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm23, %ymm11, %ymm5
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm12 = ymm5[2,3,0,1]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1],ymm5[2,3],ymm12[4],ymm5[5,6],ymm12[7],ymm5[8],ymm12[9],ymm5[10,11],ymm12[12],ymm5[13,14],ymm12[15]
-; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm5, %ymm5
-; AVX512DQ-NEXT:    vmovdqa 112(%rdi), %xmm11
-; AVX512DQ-NEXT:    vmovdqa 96(%rdi), %xmm12
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm10 = xmm12[0],xmm11[1],xmm12[2,3],xmm11[4],xmm12[5,6],xmm11[7]
-; AVX512DQ-NEXT:    vpshufb %xmm15, %xmm10, %xmm10
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm10[0,1,2],ymm5[3,4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa64 (%rdi), %ymm17
-; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm5
+; AVX512DQ-NEXT:    vpshufb %ymm10, %ymm5, %ymm10
+; AVX512DQ-NEXT:    vmovdqa 112(%rdi), %xmm15
+; AVX512DQ-NEXT:    vmovdqa 96(%rdi), %xmm5
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm12 = xmm5[0],xmm15[1],xmm5[2,3],xmm15[4],xmm5[5,6],xmm15[7]
+; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm12, %xmm12
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm6 = ymm12[0,1,2],ymm10[3,4,5,6,7]
+; AVX512DQ-NEXT:    vmovdqa64 (%rdi), %ymm24
+; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm12
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm10
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm5, %ymm17, %ymm10
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm15 = ymm10[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0],ymm15[1],ymm10[2,3],ymm15[4],ymm10[5,6],ymm15[7],ymm10[8],ymm15[9],ymm10[10,11],ymm15[12],ymm10[13,14],ymm15[15]
-; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm10, %ymm2
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm12, %ymm24, %ymm10
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm10[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm10[0],ymm1[1],ymm10[2,3],ymm1[4],ymm10[5,6],ymm1[7],ymm10[8],ymm1[9],ymm10[10,11],ymm1[12],ymm10[13,14],ymm1[15]
+; AVX512DQ-NEXT:    vpshufb %ymm7, %ymm1, %ymm7
 ; AVX512DQ-NEXT:    vmovdqa 80(%rdi), %xmm10
-; AVX512DQ-NEXT:    vmovdqa 64(%rdi), %xmm15
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm3 = xmm15[0,1],xmm10[2],xmm15[3,4],xmm10[5],xmm15[6,7]
-; AVX512DQ-NEXT:    vpshufb %xmm9, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm2[0,1,2],ymm3[3,4,5,6,7],ymm2[8,9,10],ymm3[11,12,13,14,15]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,4,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm18
-; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm23, %ymm22, %ymm1
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7,8,9],ymm2[10],ymm1[11,12],ymm2[13],ymm1[14,15]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
-; AVX512DQ-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vmovdqa64 %ymm2, %ymm28
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1],xmm6[2],xmm4[3,4],xmm6[5],xmm4[6,7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm6, %xmm25
+; AVX512DQ-NEXT:    vmovdqa 64(%rdi), %xmm1
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm2 = xmm1[0,1],xmm10[2],xmm1[3,4],xmm10[5],xmm1[6,7]
+; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm2, %xmm2
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm7[0,1,2],ymm2[3,4,5,6,7],ymm7[8,9,10],ymm2[11,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,6,5,4,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm6, %zmm2, %zmm17
+; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm2
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm22, %ymm21, %ymm2
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm6 = ymm2[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm6[2],ymm2[3,4],ymm6[5],ymm2[6,7,8,9],ymm6[10],ymm2[11,12],ymm6[13],ymm2[14,15]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm9 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
+; AVX512DQ-NEXT:    vpshufb %ymm9, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm7 = xmm4[0,1],xmm8[2],xmm4[3,4],xmm8[5],xmm4[6,7]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm8, %xmm25
 ; AVX512DQ-NEXT:    vmovdqa64 %xmm4, %xmm26
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm6 = [2,3,8,9,14,15,4,5,10,11,10,11,10,11,10,11]
-; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm1[5,6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm9 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
-; AVX512DQ-NEXT:    vmovdqa %ymm9, %ymm1
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm21, %ymm20, %ymm1
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm1[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm4[2],ymm1[3,4],ymm4[5],ymm1[6,7,8,9],ymm4[10],ymm1[11,12],ymm4[13],ymm1[14,15]
+; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm7, %xmm7
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm7 = xmm7[0,1,2,3,4],xmm2[5,6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm13 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
+; AVX512DQ-NEXT:    vmovdqa %ymm13, %ymm2
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm20, %ymm18, %ymm2
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm2[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm4[2],ymm2[3,4],ymm4[5],ymm2[6,7,8,9],ymm4[10],ymm2[11,12],ymm4[13],ymm2[14,15]
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm4 = [2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]
-; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vmovdqa %xmm14, %xmm7
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm14 = xmm8[0,1],xmm14[2],xmm8[3,4],xmm14[5],xmm8[6,7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm8, %xmm27
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm2 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm14, %xmm14
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm14 = ymm1[0,1,2],ymm14[3,4,5,6,7],ymm1[8,9,10],ymm14[11,12,13,14,15]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,6,7,4]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm14[4,5,6,7]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm19
-; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm13, %ymm24, %ymm1
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm1[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm3[2],ymm1[3,4],ymm3[5],ymm1[6,7,8,9],ymm3[10],ymm1[11,12],ymm3[13],ymm1[14,15]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm28, %ymm3
-; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm3 = xmm12[0,1],xmm11[2],xmm12[3,4],xmm11[5],xmm12[6,7]
-; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm1[5,6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa %ymm9, %ymm3
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm17, %ymm5, %ymm3
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm6 = ymm3[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1],ymm6[2],ymm3[3,4],ymm6[5],ymm3[6,7,8,9],ymm6[10],ymm3[11,12],ymm6[13],ymm3[14,15]
-; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm10[0,1],xmm15[2],xmm10[3,4],xmm15[5],xmm10[6,7]
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm4, %xmm2
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3,4,5,6,7],ymm3[8,9,10],ymm2[11,12,13,14,15]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,6,7,4]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX512DQ-NEXT:    vpternlogq $226, %ymm24, %ymm9, %ymm13
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm13[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm13[1,2],ymm2[3],ymm13[4,5],ymm2[6],ymm13[7],ymm2[8],ymm13[9,10],ymm2[11],ymm13[12,13],ymm2[14],ymm13[15]
-; AVX512DQ-NEXT:    vpternlogq $226, %ymm17, %ymm0, %ymm5
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm5[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm5[1,2],ymm3[3],ymm5[4,5],ymm3[6],ymm5[7],ymm3[8],ymm5[9,10],ymm3[11],ymm5[12,13],ymm3[14],ymm5[15]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm4 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
-; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm5 = xmm15[0],xmm10[1],xmm15[2,3],xmm10[4],xmm15[5,6],xmm10[7]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm6 = [0,1,2,3,0,1,6,7,12,13,2,3,8,9,14,15]
-; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm5, %xmm5
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm5[5,6,7]
 ; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm5 = xmm11[0,1],xmm12[2],xmm11[3,4],xmm12[5],xmm11[6,7]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm8 = [4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
-; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm5, %xmm5
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm3, %zmm5
-; AVX512DQ-NEXT:    vextracti32x4 $2, %zmm5, %xmm5
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4],xmm2[5,6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm19, %xmm8
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm14 = xmm8[0,1],xmm3[2],xmm8[3,4],xmm3[5],xmm8[6,7]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm3, %xmm27
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm3 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]
+; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm14, %xmm14
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm14 = ymm2[0,1,2],ymm14[3,4,5,6,7],ymm2[8,9,10],ymm14[11,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,6,7,4]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm14[4,5,6,7]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm2, %zmm19
+; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm2
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm11, %ymm23, %ymm2
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm2[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm7[2],ymm2[3,4],ymm7[5],ymm2[6,7,8,9],ymm7[10],ymm2[11,12],ymm7[13],ymm2[14,15]
+; AVX512DQ-NEXT:    vpshufb %ymm9, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm7 = xmm5[0,1],xmm15[2],xmm5[3,4],xmm15[5],xmm5[6,7]
+; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm7, %xmm6
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4],xmm2[5,6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-NEXT:    vmovdqa %ymm13, %ymm6
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm24, %ymm12, %ymm6
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0,1],ymm7[2],ymm6[3,4],ymm7[5],ymm6[6,7,8,9],ymm7[10],ymm6[11,12],ymm7[13],ymm6[14,15]
+; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm6, %ymm4
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm6 = xmm10[0,1],xmm1[2],xmm10[3,4],xmm1[5],xmm10[6,7]
+; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm6, %xmm3
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3,4,5,6,7],ymm4[8,9,10],ymm3[11,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,6,7,4]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm22, %ymm23, %ymm9
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm9[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm9[1,2],ymm3[3],ymm9[4,5],ymm3[6],ymm9[7],ymm3[8],ymm9[9,10],ymm3[11],ymm9[12,13],ymm3[14],ymm9[15]
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm21, %ymm20, %ymm0
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm5 = ymm0[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm5[0],ymm0[1,2],ymm5[3],ymm0[4,5],ymm5[6],ymm0[7],ymm5[8],ymm0[9,10],ymm5[11],ymm0[12,13],ymm5[14],ymm0[15]
-; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
-; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpternlogq $226, %ymm23, %ymm13, %ymm11
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm11[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm11[1,2],ymm3[3],ymm11[4,5],ymm3[6],ymm11[7],ymm3[8],ymm11[9,10],ymm3[11],ymm11[12,13],ymm3[14],ymm11[15]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm11 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
+; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm15[0,1],xmm5[2],xmm15[3,4],xmm5[5],xmm15[6,7]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm5 = [4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
+; AVX512DQ-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512DQ-NEXT:    vpternlogq $226, %ymm24, %ymm0, %ymm12
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm12[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0],ymm12[1,2],ymm4[3],ymm12[4,5],ymm4[6],ymm12[7],ymm4[8],ymm12[9,10],ymm4[11],ymm12[12,13],ymm4[14],ymm12[15]
+; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm4, %ymm4
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm10[1],xmm1[2,3],xmm10[4],xmm1[5,6],xmm10[7]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm6 = [0,1,2,3,0,1,6,7,12,13,2,3,8,9,14,15]
+; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm1, %xmm1
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4],ymm1[5,6,7]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm13
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm13[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm13[1,2],ymm3[3],ymm13[4,5],ymm3[6],ymm13[7],ymm3[8],ymm13[9,10],ymm3[11],ymm13[12,13],ymm3[14],ymm13[15]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm25, %xmm4
+; AVX512DQ-NEXT:    vmovdqa64 %xmm26, %xmm7
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm7[2],xmm4[3,4],xmm7[5],xmm4[6,7]
+; AVX512DQ-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
+; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm20, %ymm18, %ymm0
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm0[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm4[0],ymm0[1,2],ymm4[3],ymm0[4,5],ymm4[6],ymm0[7],ymm4[8],ymm0[9,10],ymm4[11],ymm0[12,13],ymm4[14],ymm0[15]
+; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa64 %xmm27, %xmm4
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm7[0],xmm4[1],xmm7[2,3],xmm4[4],xmm7[5,6],xmm4[7]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0],xmm8[1],xmm4[2,3],xmm8[4],xmm4[5,6],xmm8[7]
 ; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm4, %xmm4
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm4[5,6,7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm25, %xmm4
-; AVX512DQ-NEXT:    vmovdqa64 %xmm26, %xmm5
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2],xmm4[3,4],xmm5[5],xmm4[6,7]
-; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm4, %xmm4
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm4
-; AVX512DQ-NEXT:    vextracti32x4 $2, %zmm4, %xmm4
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vmovdqa64 %zmm18, (%rsi)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm17, (%rsi)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm16, 64(%rsi)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm19, 64(%rdx)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm1, (%rdx)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm2, (%rdx)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm0, 64(%rcx)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm2, (%rcx)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm1, (%rcx)
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512DQ-FCP-LABEL: load_i16_stride3_vf64:
 ; AVX512DQ-FCP:       # %bb.0:
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 224(%rdi), %ymm20
-; AVX512DQ-FCP-NEXT:    vmovdqa64 192(%rdi), %ymm21
+; AVX512DQ-FCP-NEXT:    vmovdqa64 224(%rdi), %ymm18
+; AVX512DQ-FCP-NEXT:    vmovdqa64 192(%rdi), %ymm20
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm20, %ymm21, %ymm1
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm1[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm3[1],ymm1[2,3],ymm3[4],ymm1[5,6],ymm3[7],ymm1[8],ymm3[9],ymm1[10,11],ymm3[12],ymm1[13,14],ymm3[15]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm1, %ymm5
-; AVX512DQ-FCP-NEXT:    vmovdqa 272(%rdi), %xmm8
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm18, %ymm20, %ymm1
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm7 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm7, %ymm2, %ymm5
+; AVX512DQ-FCP-NEXT:    vmovdqa 272(%rdi), %xmm1
 ; AVX512DQ-FCP-NEXT:    vmovdqa 256(%rdi), %xmm2
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm6 = xmm2[0,1],xmm8[2],xmm2[3,4],xmm8[5],xmm2[6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm2, %xmm14
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm9 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm9, %xmm6, %xmm6
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm6 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm2, %xmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm1, %xmm19
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm13 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm13, %xmm6, %xmm6
 ; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm5[0,1,2],ymm6[3,4,5,6,7],ymm5[8,9,10],ymm6[11,12,13,14,15]
 ; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,6,5,4,7]
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 320(%rdi), %ymm22
-; AVX512DQ-FCP-NEXT:    vmovdqa64 352(%rdi), %ymm23
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm6
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm22, %ymm23, %ymm6
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0],ymm7[1],ymm6[2,3],ymm7[4],ymm6[5,6],ymm7[7],ymm6[8],ymm7[9],ymm6[10,11],ymm7[12],ymm6[13,14],ymm7[15]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm11 = [0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm6, %ymm12
+; AVX512DQ-FCP-NEXT:    vmovdqa64 320(%rdi), %ymm21
+; AVX512DQ-FCP-NEXT:    vmovdqa64 352(%rdi), %ymm22
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm8
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm8
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm8[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm8 = ymm8[0],ymm9[1],ymm8[2,3],ymm9[4],ymm8[5,6],ymm9[7],ymm8[8],ymm9[9],ymm8[10,11],ymm9[12],ymm8[13,14],ymm9[15]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm10 = [0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm8, %ymm11
 ; AVX512DQ-FCP-NEXT:    vmovdqa 304(%rdi), %xmm1
 ; AVX512DQ-FCP-NEXT:    vmovdqa 288(%rdi), %xmm2
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm13 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm2, %xmm4
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, %xmm6
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm15 = [0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm15, %xmm13, %xmm13
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm12 = ymm13[0,1,2],ymm12[3,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm5, %zmm16
-; AVX512DQ-FCP-NEXT:    vmovdqa64 128(%rdi), %ymm24
-; AVX512DQ-FCP-NEXT:    vmovdqa 160(%rdi), %ymm13
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, %xmm8
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm14 = [0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm14, %xmm12, %xmm12
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm11 = ymm12[0,1,2],ymm11[3,4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm5, %zmm16
+; AVX512DQ-FCP-NEXT:    vmovdqa64 128(%rdi), %ymm23
+; AVX512DQ-FCP-NEXT:    vmovdqa 160(%rdi), %ymm11
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm5
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm24, %ymm13, %ymm5
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm23, %ymm11, %ymm5
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm12 = ymm5[2,3,0,1]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1],ymm5[2,3],ymm12[4],ymm5[5,6],ymm12[7],ymm5[8],ymm12[9],ymm5[10,11],ymm12[12],ymm5[13,14],ymm12[15]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm5, %ymm5
-; AVX512DQ-FCP-NEXT:    vmovdqa 112(%rdi), %xmm11
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdi), %xmm12
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm10 = xmm12[0],xmm11[1],xmm12[2,3],xmm11[4],xmm12[5,6],xmm11[7]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm15, %xmm10, %xmm10
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm10[0,1,2],ymm5[3,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 (%rdi), %ymm17
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %ymm5
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm5, %ymm10
+; AVX512DQ-FCP-NEXT:    vmovdqa 112(%rdi), %xmm15
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdi), %xmm5
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm12 = xmm5[0],xmm15[1],xmm5[2,3],xmm15[4],xmm5[5,6],xmm15[7]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm14, %xmm12, %xmm12
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm6 = ymm12[0,1,2],ymm10[3,4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 (%rdi), %ymm24
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %ymm12
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm10
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm5, %ymm17, %ymm10
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm15 = ymm10[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0],ymm15[1],ymm10[2,3],ymm15[4],ymm10[5,6],ymm15[7],ymm10[8],ymm15[9],ymm10[10,11],ymm15[12],ymm10[13,14],ymm15[15]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm10, %ymm2
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm12, %ymm24, %ymm10
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm10[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm10[0],ymm1[1],ymm10[2,3],ymm1[4],ymm10[5,6],ymm1[7],ymm10[8],ymm1[9],ymm10[10,11],ymm1[12],ymm10[13,14],ymm1[15]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm7, %ymm1, %ymm7
 ; AVX512DQ-FCP-NEXT:    vmovdqa 80(%rdi), %xmm10
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdi), %xmm15
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm15[0,1],xmm10[2],xmm15[3,4],xmm10[5],xmm15[6,7]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm9, %xmm3, %xmm3
-; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm2[0,1,2],ymm3[3,4,5,6,7],ymm2[8,9,10],ymm3[11,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,4,7]
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm18
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm23, %ymm22, %ymm1
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7,8,9],ymm2[10],ymm1[11,12],ymm2[13],ymm1[14,15]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm2 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm2, %ymm28
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1],xmm6[2],xmm4[3,4],xmm6[5],xmm4[6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm6, %xmm25
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdi), %xmm1
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm2 = xmm1[0,1],xmm10[2],xmm1[3,4],xmm10[5],xmm1[6,7]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm13, %xmm2, %xmm2
+; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm7[0,1,2],ymm2[3,4,5,6,7],ymm7[8,9,10],ymm2[11,12,13,14,15]
+; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,6,5,4,7]
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm2, %zmm17
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm2
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm22, %ymm21, %ymm2
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm2[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm6[2],ymm2[3,4],ymm6[5],ymm2[6,7,8,9],ymm6[10],ymm2[11,12],ymm6[13],ymm2[14,15]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm9 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm9, %ymm2, %ymm2
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm7 = xmm4[0,1],xmm8[2],xmm4[3,4],xmm8[5],xmm4[6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm8, %xmm25
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm4, %xmm26
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm6 = [2,3,8,9,14,15,4,5,10,11,10,11,10,11,10,11]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm1[5,6,7]
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm9 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm9, %ymm1
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm21, %ymm20, %ymm1
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm1[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm4[2],ymm1[3,4],ymm4[5],ymm1[6,7,8,9],ymm4[10],ymm1[11,12],ymm4[13],ymm1[14,15]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm7, %xmm7
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm7 = xmm7[0,1,2,3,4],xmm2[5,6,7]
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm13 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm13, %ymm2
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm20, %ymm18, %ymm2
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm2[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm4[2],ymm2[3,4],ymm4[5],ymm2[6,7,8,9],ymm4[10],ymm2[11,12],ymm4[13],ymm2[14,15]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm4 = [2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm1, %ymm1
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm14, %xmm7
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm14 = xmm8[0,1],xmm14[2],xmm8[3,4],xmm14[5],xmm8[6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm8, %xmm27
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm2, %xmm14, %xmm14
-; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm14 = ymm1[0,1,2],ymm14[3,4,5,6,7],ymm1[8,9,10],ymm14[11,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,6,7,4]
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm14[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm19
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm13, %ymm24, %ymm1
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm1[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm3[2],ymm1[3,4],ymm3[5],ymm1[6,7,8,9],ymm3[10],ymm1[11,12],ymm3[13],ymm1[14,15]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm28, %ymm3
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm1, %ymm1
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm12[0,1],xmm11[2],xmm12[3,4],xmm11[5],xmm12[6,7]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm1[5,6,7]
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm9, %ymm3
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm17, %ymm5, %ymm3
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm3[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1],ymm6[2],ymm3[3,4],ymm6[5],ymm3[6,7,8,9],ymm6[10],ymm3[11,12],ymm6[13],ymm3[14,15]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm10[0,1],xmm15[2],xmm10[3,4],xmm15[5],xmm10[6,7]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm2, %xmm4, %xmm2
-; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3,4,5,6,7],ymm3[8,9,10],ymm2[11,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,6,7,4]
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %ymm24, %ymm9, %ymm13
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm13[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm13[1,2],ymm2[3],ymm13[4,5],ymm2[6],ymm13[7],ymm2[8],ymm13[9,10],ymm2[11],ymm13[12,13],ymm2[14],ymm13[15]
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %ymm17, %ymm0, %ymm5
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm5[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm5[1,2],ymm3[3],ymm5[4,5],ymm3[6],ymm5[7],ymm3[8],ymm5[9,10],ymm3[11],ymm5[12,13],ymm3[14],ymm5[15]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm4 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm5 = xmm15[0],xmm10[1],xmm15[2,3],xmm10[4],xmm15[5,6],xmm10[7]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm6 = [0,1,2,3,0,1,6,7,12,13,2,3,8,9,14,15]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm5, %xmm5
-; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm5[5,6,7]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm2, %ymm2
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm5 = xmm11[0,1],xmm12[2],xmm11[3,4],xmm12[5],xmm11[6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm8 = [4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm5, %xmm5
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm3, %zmm5
-; AVX512DQ-FCP-NEXT:    vextracti32x4 $2, %zmm5, %xmm5
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4],xmm2[5,6,7]
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm19, %xmm8
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm14 = xmm8[0,1],xmm3[2],xmm8[3,4],xmm3[5],xmm8[6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm3, %xmm27
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm3, %xmm14, %xmm14
+; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm14 = ymm2[0,1,2],ymm14[3,4,5,6,7],ymm2[8,9,10],ymm14[11,12,13,14,15]
+; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,6,7,4]
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm14[4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm2, %zmm19
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm2
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm11, %ymm23, %ymm2
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm2[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm7[2],ymm2[3,4],ymm7[5],ymm2[6,7,8,9],ymm7[10],ymm2[11,12],ymm7[13],ymm2[14,15]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm9, %ymm2, %ymm2
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm7 = xmm5[0,1],xmm15[2],xmm5[3,4],xmm15[5],xmm5[6,7]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm7, %xmm6
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4],xmm2[5,6,7]
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm13, %ymm6
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm24, %ymm12, %ymm6
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0,1],ymm7[2],ymm6[3,4],ymm7[5],ymm6[6,7,8,9],ymm7[10],ymm6[11,12],ymm7[13],ymm6[14,15]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm6, %ymm4
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm6 = xmm10[0,1],xmm1[2],xmm10[3,4],xmm1[5],xmm10[6,7]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm3, %xmm6, %xmm3
+; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3,4,5,6,7],ymm4[8,9,10],ymm3[11,12,13,14,15]
+; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,6,7,4]
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm22, %ymm23, %ymm9
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm9[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm9[1,2],ymm3[3],ymm9[4,5],ymm3[6],ymm9[7],ymm3[8],ymm9[9,10],ymm3[11],ymm9[12,13],ymm3[14],ymm9[15]
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm21, %ymm20, %ymm0
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm5 = ymm0[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm5[0],ymm0[1,2],ymm5[3],ymm0[4,5],ymm5[6],ymm0[7],ymm5[8],ymm0[9,10],ymm5[11],ymm0[12,13],ymm5[14],ymm0[15]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm0, %ymm0
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %ymm23, %ymm13, %ymm11
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm11[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm11[1,2],ymm3[3],ymm11[4,5],ymm3[6],ymm11[7],ymm3[8],ymm11[9,10],ymm3[11],ymm11[12,13],ymm3[14],ymm11[15]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm11 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm3, %ymm3
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm15[0,1],xmm5[2],xmm15[3,4],xmm5[5],xmm15[6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm5 = [4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %ymm24, %ymm0, %ymm12
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm12[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0],ymm12[1,2],ymm4[3],ymm12[4,5],ymm4[6],ymm12[7],ymm4[8],ymm12[9,10],ymm4[11],ymm12[12,13],ymm4[14],ymm12[15]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm4, %ymm4
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm10[1],xmm1[2,3],xmm10[4],xmm1[5,6],xmm10[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm6 = [0,1,2,3,0,1,6,7,12,13,2,3,8,9,14,15]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm1, %xmm1
+; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4],ymm1[5,6,7]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm13
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm13[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm13[1,2],ymm3[3],ymm13[4,5],ymm3[6],ymm13[7],ymm3[8],ymm13[9,10],ymm3[11],ymm13[12,13],ymm3[14],ymm13[15]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm25, %xmm4
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm26, %xmm7
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm7[2],xmm4[3,4],xmm7[5],xmm4[6,7]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm3, %ymm3
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm20, %ymm18, %ymm0
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm0[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm4[0],ymm0[1,2],ymm4[3],ymm0[4,5],ymm4[6],ymm0[7],ymm4[8],ymm0[9,10],ymm4[11],ymm0[12,13],ymm4[14],ymm0[15]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm0, %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm27, %xmm4
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm7[0],xmm4[1],xmm7[2,3],xmm4[4],xmm7[5,6],xmm4[7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0],xmm8[1],xmm4[2,3],xmm8[4],xmm4[5,6],xmm8[7]
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm4, %xmm4
 ; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm4[5,6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm25, %xmm4
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm26, %xmm5
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2],xmm4[3,4],xmm5[5],xmm4[6,7]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm4, %xmm4
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm4
-; AVX512DQ-FCP-NEXT:    vextracti32x4 $2, %zmm4, %xmm4
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm18, (%rsi)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm17, (%rsi)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm16, 64(%rsi)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm19, 64(%rdx)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, (%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm2, (%rdx)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, 64(%rcx)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm2, (%rcx)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, (%rcx)
 ; AVX512DQ-FCP-NEXT:    vzeroupper
 ; AVX512DQ-FCP-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
index 8b6ba51506ab..8091afbbfd70 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
@@ -1246,29 +1246,28 @@ define void @store_i8_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecp
 ; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
 ; AVX512BW-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm1 = [0,2,4,0]
 ; AVX512BW-FCP-NEXT:    vpermi2q %ymm3, %ymm0, %ymm1
-; AVX512BW-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [1,3,5,7,1,3,5,7]
-; AVX512BW-FCP-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX512BW-FCP-NEXT:    vpermd %ymm2, %ymm0, %ymm0
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm0[1,5,9,13],zero,zero,zero,ymm0[2,6,10,14],zero,zero,zero,ymm0[19,23,27,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm3 = [1,3,5,0,5,1,3,0]
-; AVX512BW-FCP-NEXT:    vpermd %ymm1, %ymm3, %ymm3
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[0,4,8],zero,zero,zero,zero,ymm3[1,5,9],zero,zero,zero,zero,ymm3[2,6,18],zero,zero,zero,zero,ymm3[23,27,19],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm2[0,8],zero,zero,zero,zero,zero,ymm2[1,9],zero,zero,zero,zero,zero,ymm2[2,10,18,26],zero,zero,zero,zero,zero,ymm2[19,27],zero,zero,zero,zero,zero,ymm2[20,28]
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm2[2,3,0,1]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,ymm3[0,8],zero,zero,zero,zero,zero,ymm3[1,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[19,27],zero,zero,zero,zero,zero,ymm3[20,28],zero,zero
 ; AVX512BW-FCP-NEXT:    vpor %ymm0, %ymm3, %ymm0
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm2[0,8],zero,zero,zero,zero,zero,ymm2[1,9],zero,zero,zero,zero,zero,ymm2[2,10,18,26],zero,zero,zero,zero,zero,ymm2[19,27],zero,zero,zero,zero,zero,ymm2[20,28]
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,3,0,1]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,ymm2[0,8],zero,zero,zero,zero,zero,ymm2[1,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[19,27],zero,zero,zero,zero,zero,ymm2[20,28],zero,zero
-; AVX512BW-FCP-NEXT:    vpor %ymm3, %ymm2, %ymm2
 ; AVX512BW-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [0,2,4,6,0,2,4,6]
 ; AVX512BW-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
-; AVX512BW-FCP-NEXT:    vpermd %ymm1, %ymm3, %ymm1
+; AVX512BW-FCP-NEXT:    vpermd %ymm1, %ymm3, %ymm3
 ; AVX512BW-FCP-NEXT:    movl $236730480, %ecx # imm = 0xE1C3870
 ; AVX512BW-FCP-NEXT:    kmovd %ecx, %k1
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm2 {%k1} = ymm1[u,u,u,u,0,4,8,u,u,u,u,1,5,9,u,u,u,u,18,22,26,u,u,u,u,19,23,27,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm0
-; AVX512BW-FCP-NEXT:    vmovdqa %ymm2, (%rax)
-; AVX512BW-FCP-NEXT:    vextracti32x4 $2, %zmm0, 32(%rax)
-; AVX512BW-FCP-NEXT:    vextracti32x4 $3, %zmm0, %xmm0
-; AVX512BW-FCP-NEXT:    vmovq %xmm0, 48(%rax)
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm0 {%k1} = ymm3[u,u,u,u,0,4,8,u,u,u,u,1,5,9,u,u,u,u,18,22,26,u,u,u,u,19,23,27,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [1,3,5,7,1,3,5,7]
+; AVX512BW-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
+; AVX512BW-FCP-NEXT:    vpermd %ymm2, %ymm3, %ymm2
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,zero,ymm2[1,5,9,13],zero,zero,zero,ymm2[2,6,10,14],zero,zero,zero,ymm2[19,23,27,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm3 = [1,3,5,0,5,1,3,0]
+; AVX512BW-FCP-NEXT:    vpermd %ymm1, %ymm3, %ymm1
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[0,4,8],zero,zero,zero,zero,ymm1[1,5,9],zero,zero,zero,zero,ymm1[2,6,18],zero,zero,zero,zero,ymm1[23,27,19],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-FCP-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX512BW-FCP-NEXT:    vmovq %xmm2, 48(%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa %xmm1, 32(%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa %ymm0, (%rax)
 ; AVX512BW-FCP-NEXT:    vzeroupper
 ; AVX512BW-FCP-NEXT:    retq
 ;
@@ -1326,29 +1325,28 @@ define void @store_i8_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecp
 ; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
 ; AVX512DQ-BW-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm1 = [0,2,4,0]
 ; AVX512DQ-BW-FCP-NEXT:    vpermi2q %ymm3, %ymm0, %ymm1
-; AVX512DQ-BW-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [1,3,5,7,1,3,5,7]
-; AVX512DQ-BW-FCP-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm2, %ymm0, %ymm0
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm0[1,5,9,13],zero,zero,zero,ymm0[2,6,10,14],zero,zero,zero,ymm0[19,23,27,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm3 = [1,3,5,0,5,1,3,0]
-; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm1, %ymm3, %ymm3
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[0,4,8],zero,zero,zero,zero,ymm3[1,5,9],zero,zero,zero,zero,ymm3[2,6,18],zero,zero,zero,zero,ymm3[23,27,19],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm2[0,8],zero,zero,zero,zero,zero,ymm2[1,9],zero,zero,zero,zero,zero,ymm2[2,10,18,26],zero,zero,zero,zero,zero,ymm2[19,27],zero,zero,zero,zero,zero,ymm2[20,28]
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm2[2,3,0,1]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,ymm3[0,8],zero,zero,zero,zero,zero,ymm3[1,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[19,27],zero,zero,zero,zero,zero,ymm3[20,28],zero,zero
 ; AVX512DQ-BW-FCP-NEXT:    vpor %ymm0, %ymm3, %ymm0
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm2[0,8],zero,zero,zero,zero,zero,ymm2[1,9],zero,zero,zero,zero,zero,ymm2[2,10,18,26],zero,zero,zero,zero,zero,ymm2[19,27],zero,zero,zero,zero,zero,ymm2[20,28]
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,3,0,1]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,ymm2[0,8],zero,zero,zero,zero,zero,ymm2[1,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[19,27],zero,zero,zero,zero,zero,ymm2[20,28],zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vpor %ymm3, %ymm2, %ymm2
 ; AVX512DQ-BW-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [0,2,4,6,0,2,4,6]
 ; AVX512DQ-BW-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
-; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm1, %ymm3, %ymm1
+; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm1, %ymm3, %ymm3
 ; AVX512DQ-BW-FCP-NEXT:    movl $236730480, %ecx # imm = 0xE1C3870
 ; AVX512DQ-BW-FCP-NEXT:    kmovd %ecx, %k1
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm2 {%k1} = ymm1[u,u,u,u,0,4,8,u,u,u,u,1,5,9,u,u,u,u,18,22,26,u,u,u,u,19,23,27,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm0
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa %ymm2, (%rax)
-; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $2, %zmm0, 32(%rax)
-; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $3, %zmm0, %xmm0
-; AVX512DQ-BW-FCP-NEXT:    vmovq %xmm0, 48(%rax)
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm0 {%k1} = ymm3[u,u,u,u,0,4,8,u,u,u,u,1,5,9,u,u,u,u,18,22,26,u,u,u,u,19,23,27,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [1,3,5,7,1,3,5,7]
+; AVX512DQ-BW-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
+; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm2, %ymm3, %ymm2
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,zero,ymm2[1,5,9,13],zero,zero,zero,ymm2[2,6,10,14],zero,zero,zero,ymm2[19,23,27,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm3 = [1,3,5,0,5,1,3,0]
+; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm1, %ymm3, %ymm1
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[0,4,8],zero,zero,zero,zero,ymm1[1,5,9],zero,zero,zero,zero,ymm1[2,6,18],zero,zero,zero,zero,ymm1[23,27,19],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX512DQ-BW-FCP-NEXT:    vmovq %xmm2, 48(%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa %xmm1, 32(%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa %ymm0, (%rax)
 ; AVX512DQ-BW-FCP-NEXT:    vzeroupper
 ; AVX512DQ-BW-FCP-NEXT:    retq
   %in.vec0 = load <8 x i8>, ptr %in.vecptr0, align 64
@@ -2053,77 +2051,76 @@ define void @store_i8_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512-NEXT:    vmovdqa (%rdi), %xmm4
-; AVX512-NEXT:    vmovdqa (%rsi), %xmm5
-; AVX512-NEXT:    vmovdqa (%rdx), %xmm6
-; AVX512-NEXT:    vmovdqa (%rcx), %xmm7
-; AVX512-NEXT:    vmovdqa (%r8), %xmm0
-; AVX512-NEXT:    vmovdqa (%r10), %xmm1
-; AVX512-NEXT:    vinserti128 $1, %xmm7, %ymm6, %ymm3
-; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm4, %ymm2
-; AVX512-NEXT:    vinserti128 $1, (%r9), %ymm0, %ymm0
-; AVX512-NEXT:    vinserti32x4 $2, %xmm1, %zmm0, %zmm0
-; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm6[8],xmm7[8],xmm6[9],xmm7[9],xmm6[10],xmm7[10],xmm6[11],xmm7[11],xmm6[12],xmm7[12],xmm6[13],xmm7[13],xmm6[14],xmm7[14],xmm6[15],xmm7[15]
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[u,u],zero,zero,xmm6[12,13,u,u,u],zero,zero,xmm6[14,15,u,u,u]
-; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm4[8],xmm5[8],xmm4[9],xmm5[9],xmm4[10],xmm5[10],xmm4[11],xmm5[11],xmm4[12],xmm5[12],xmm4[13],xmm5[13],xmm4[14],xmm5[14],xmm4[15],xmm5[15]
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,12,13],zero,zero,xmm4[u,u,u,14,15],zero,zero,xmm4[u,u,u]
-; AVX512-NEXT:    vpor %xmm6, %xmm4, %xmm4
-; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm5
-; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[10],zero,xmm5[u,u,u,u,13,12],zero,xmm5[u,u,u,u,15,14],zero
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm6 = zero,xmm1[13,u,u,u,u],zero,zero,xmm1[14,u,u,u,u],zero,zero,xmm1[15]
-; AVX512-NEXT:    vpor %xmm6, %xmm5, %xmm5
-; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4, %xmm5
-; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm2[3,1,1,3]
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm4 = ymm4[1],zero,zero,ymm4[u,u,u,10,2],zero,zero,ymm4[u,u,u,11,3],zero,zero,ymm4[u,u,u,20,28],zero,zero,ymm4[u,u,u,21,29],zero,zero,ymm4[u]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm6 = ymm3[1,3,3,1]
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm6 = zero,ymm6[1,9,u,u,u],zero,zero,ymm6[2,10,u,u,u],zero,zero,ymm6[3,19,u,u,u],zero,zero,ymm6[28,20,u,u,u],zero,zero,ymm6[29,21,u]
-; AVX512-NEXT:    vpor %ymm4, %ymm6, %ymm4
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm6 = xmm1[0,1,2,3,4,5,5,6]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,2,3,3]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,1,0,1]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm0[1,3,1,3]
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vmovdqa (%rsi), %xmm1
+; AVX512-NEXT:    vmovdqa (%rdx), %xmm5
+; AVX512-NEXT:    vmovdqa (%rcx), %xmm6
+; AVX512-NEXT:    vmovdqa (%r8), %xmm3
+; AVX512-NEXT:    vmovdqa (%r9), %xmm4
+; AVX512-NEXT:    vmovdqa (%r10), %xmm2
+; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm8
+; AVX512-NEXT:    vinserti128 $1, %xmm6, %ymm5, %ymm9
+; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm7
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm10 = ymm9[u,u,u,u,u,5],zero,ymm9[u,u,u,u,u,6],zero,ymm9[u,u,u,u,u],zero,ymm9[23,u,u,u,u,u],zero,ymm9[24,u,u,u,u]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm11 = ymm9[2,3,0,1]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[u,u,u,u,u],zero,ymm11[5,u,u,u,u,u],zero,ymm11[6,u,u,u,u,u,23],zero,ymm11[u,u,u,u,u,24],zero,ymm11[u,u,u,u]
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm12 = [255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255]
+; AVX512-NEXT:    vpternlogq $50, %ymm10, %ymm12, %ymm11
+; AVX512-NEXT:    vpermq {{.*#+}} ymm10 = ymm9[0,2,0,2]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm10 = zero,zero,ymm10[0,8,u,u,u],zero,zero,ymm10[1,9,u,u,u],zero,zero,ymm10[18,26,u,u,u],zero,zero,ymm10[19,27,u,u,u],zero,zero,ymm10[20,28]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm11, %zmm10, %zmm10
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm11 = ymm8[u,u,u,5],zero,ymm8[u,u,u,u,u,6],zero,ymm8[u,u,u,u,u],zero,ymm8[23,u,u,u,u,u],zero,ymm8[24,u,u,u,u,u],zero
+; AVX512-NEXT:    vpermq {{.*#+}} ymm13 = ymm8[2,3,0,1]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm13 = ymm13[u,u,u],zero,ymm13[5,u,u,u,u,u],zero,ymm13[6,u,u,u,u,u,23],zero,ymm13[u,u,u,u,u,24],zero,ymm13[u,u,u,u,u,25]
+; AVX512-NEXT:    vpternlogq $200, %ymm11, %ymm12, %ymm13
+; AVX512-NEXT:    vpermq {{.*#+}} ymm11 = ymm8[0,2,0,2]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[0,8],zero,zero,ymm11[u,u,u,1,9],zero,zero,ymm11[u,u,u,2,10],zero,zero,ymm11[u,u,u,19,27],zero,zero,ymm11[u,u,u,20,28],zero,zero
+; AVX512-NEXT:    vinserti64x4 $1, %ymm13, %zmm11, %zmm11
+; AVX512-NEXT:    vporq %zmm10, %zmm11, %zmm10
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm11 = ymm7[4],zero,ymm7[u,u,u,u,u,5],zero,ymm7[u,u,u,u,u,6],zero,ymm7[u,u,u,u,u],zero,ymm7[23,u,u,u,u,u],zero,ymm7[24,u,u]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm12 = ymm7[2,3,0,1]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm12 = zero,ymm12[4,u,u,u,u,u],zero,ymm12[5,u,u,u,u,u],zero,ymm12[6,u,u,u,u,u,23],zero,ymm12[u,u,u,u,u,24],zero,ymm12[u,u]
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm13 = [255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255]
+; AVX512-NEXT:    vpternlogq $200, %ymm11, %ymm13, %ymm12
+; AVX512-NEXT:    vpermq {{.*#+}} ymm11 = ymm7[0,2,0,2]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[u,u,u,u,0,8],zero,ymm11[u,u,u,u,1,9],zero,ymm11[u,u,u,u,18,26],zero,ymm11[u,u,u,u,19,27],zero,ymm11[u,u,u,u]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm12, %zmm11, %zmm11
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm12 = xmm2[4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,0,1,0]
+; AVX512-NEXT:    vpandn %ymm12, %ymm13, %ymm12
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm13 = xmm2[1,1,0,0,4,5,6,7]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm13 = xmm13[0,1,2,0]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,0,1,0]
+; AVX512-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm13, %ymm13
+; AVX512-NEXT:    vinserti64x4 $1, %ymm12, %zmm13, %zmm12
+; AVX512-NEXT:    vporq %zmm12, %zmm11, %zmm11
+; AVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm11
+; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[3,1,1,3]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm8 = ymm8[1],zero,zero,ymm8[u,u,u,10,2],zero,zero,ymm8[u,u,u,11,3],zero,zero,ymm8[u,u,u,20,28],zero,zero,ymm8[u,u,u,21,29],zero,zero,ymm8[u]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[1,3,3,1]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm9 = zero,ymm9[1,9,u,u,u],zero,zero,ymm9[2,10,u,u,u],zero,zero,ymm9[3,19,u,u,u],zero,zero,ymm9[28,20,u,u,u],zero,zero,ymm9[29,21,u]
+; AVX512-NEXT:    vpor %ymm8, %ymm9, %ymm8
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm9 = xmm2[0,1,2,3,4,5,5,6]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm9 = xmm9[2,2,3,3]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,1,0,1]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[1,3,1,3]
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,1,9],zero,ymm7[u,u,u,u,2,10],zero,ymm7[u,u,u,u,19,27],zero,ymm7[u,u,u,u,20,28],zero,ymm7[u,u,u,u,21]
-; AVX512-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm6, %ymm7
-; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm7
-; AVX512-NEXT:    vinserti32x4 $2, %xmm5, %zmm7, %zmm4
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm6 = ymm3[u,u,u,u,u,5],zero,ymm3[u,u,u,u,u,6],zero,ymm3[u,u,u,u,u],zero,ymm3[23,u,u,u,u,u],zero,ymm3[24,u,u,u,u]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm3[2,3,0,1]
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,u,u],zero,ymm7[5,u,u,u,u,u],zero,ymm7[6,u,u,u,u,u,23],zero,ymm7[u,u,u,u,u,24],zero,ymm7[u,u,u,u]
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm8 = [255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255]
-; AVX512-NEXT:    vpternlogq $50, %ymm6, %ymm8, %ymm7
-; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,2,0,2]
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,ymm3[0,8,u,u,u],zero,zero,ymm3[1,9,u,u,u],zero,zero,ymm3[18,26,u,u,u],zero,zero,ymm3[19,27,u,u,u],zero,zero,ymm3[20,28]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm3, %zmm3
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm6 = ymm2[u,u,u,5],zero,ymm2[u,u,u,u,u,6],zero,ymm2[u,u,u,u,u],zero,ymm2[23,u,u,u,u,u],zero,ymm2[24,u,u,u,u,u],zero
-; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm2[2,3,0,1]
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u],zero,ymm7[5,u,u,u,u,u],zero,ymm7[6,u,u,u,u,u,23],zero,ymm7[u,u,u,u,u,24],zero,ymm7[u,u,u,u,u,25]
-; AVX512-NEXT:    vpternlogq $200, %ymm6, %ymm8, %ymm7
-; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,0,2]
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[0,8],zero,zero,ymm2[u,u,u,1,9],zero,zero,ymm2[u,u,u,2,10],zero,zero,ymm2[u,u,u,19,27],zero,zero,ymm2[u,u,u,20,28],zero,zero
-; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm2, %zmm2
-; AVX512-NEXT:    vporq %zmm3, %zmm2, %zmm2
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm3 = xmm1[4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,1,0]
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm6 = [255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255]
-; AVX512-NEXT:    vpandn %ymm3, %ymm6, %ymm3
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[1,1,0,0,4,5,6,7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,1,0]
-; AVX512-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm3 = ymm0[4],zero,ymm0[u,u,u,u,u,5],zero,ymm0[u,u,u,u,u,6],zero,ymm0[u,u,u,u,u],zero,ymm0[23,u,u,u,u,u],zero,ymm0[24,u,u]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm0[2,3,0,1]
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm7 = zero,ymm7[4,u,u,u,u,u],zero,ymm7[5,u,u,u,u,u],zero,ymm7[6,u,u,u,u,u,23],zero,ymm7[u,u,u,u,u,24],zero,ymm7[u,u]
-; AVX512-NEXT:    vpternlogq $200, %ymm3, %ymm6, %ymm7
-; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,0,2]
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,0,8],zero,ymm0[u,u,u,u,1,9],zero,ymm0[u,u,u,u,18,26],zero,ymm0[u,u,u,u,19,27],zero,ymm0[u,u,u,u]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm0, %zmm0
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0
-; AVX512-NEXT:    vmovdqa %xmm5, 96(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm0, (%rax)
-; AVX512-NEXT:    vmovdqa %ymm4, 64(%rax)
+; AVX512-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm9, %ymm7
+; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm8, %ymm7
+; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm5[8],xmm6[8],xmm5[9],xmm6[9],xmm5[10],xmm6[10],xmm5[11],xmm6[11],xmm5[12],xmm6[12],xmm5[13],xmm6[13],xmm5[14],xmm6[14],xmm5[15],xmm6[15]
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u],zero,zero,xmm5[12,13,u,u,u],zero,zero,xmm5[14,15,u,u,u]
+; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,12,13],zero,zero,xmm0[u,u,u,14,15],zero,zero,xmm0[u,u,u]
+; AVX512-NEXT:    vpor %xmm5, %xmm0, %xmm0
+; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm4[8],xmm3[8],xmm4[9],xmm3[9],xmm4[10],xmm3[10],xmm4[11],xmm3[11],xmm4[12],xmm3[12],xmm4[13],xmm3[13],xmm4[14],xmm3[14],xmm4[15],xmm3[15]
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[10],zero,xmm1[u,u,u,u,13,12],zero,xmm1[u,u,u,u,15,14],zero
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm2 = zero,xmm2[13,u,u,u,u],zero,zero,xmm2[14,u,u,u,u],zero,zero,xmm2[15]
+; AVX512-NEXT:    vpor %xmm2, %xmm1, %xmm1
+; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
+; AVX512-NEXT:    vinserti32x4 $2, %xmm1, %zmm7, %zmm0
+; AVX512-NEXT:    vmovdqa %xmm1, 96(%rax)
+; AVX512-NEXT:    vmovdqa %ymm0, 64(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm11, (%rax)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
@@ -2131,70 +2128,69 @@ define void @store_i8_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP:       # %bb.0:
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm2
-; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm3
-; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm4
-; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm5
-; AVX512-FCP-NEXT:    vmovdqa (%r8), %xmm1
-; AVX512-FCP-NEXT:    vmovdqa (%r10), %xmm0
-; AVX512-FCP-NEXT:    vinserti128 $1, %xmm5, %ymm4, %ymm6
-; AVX512-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm7
-; AVX512-FCP-NEXT:    vinserti128 $1, (%r9), %ymm1, %ymm1
-; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm1
-; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm4[8],xmm5[8],xmm4[9],xmm5[9],xmm4[10],xmm5[10],xmm4[11],xmm5[11],xmm4[12],xmm5[12],xmm4[13],xmm5[13],xmm4[14],xmm5[14],xmm4[15],xmm5[15]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u],zero,zero,xmm4[12,13,u,u,u],zero,zero,xmm4[14,15,u,u,u]
-; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,12,13],zero,zero,xmm2[u,u,u,14,15],zero,zero,xmm2[u,u,u]
-; AVX512-FCP-NEXT:    vpor %xmm4, %xmm2, %xmm2
-; AVX512-FCP-NEXT:    vextracti128 $1, %ymm1, %xmm3
-; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[10],zero,xmm3[u,u,u,u,13,12],zero,xmm3[u,u,u,u,15,14],zero
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = zero,xmm0[13,u,u,u,u],zero,zero,xmm0[14,u,u,u,u],zero,zero,xmm0[15]
-; AVX512-FCP-NEXT:    vpor %xmm4, %xmm3, %xmm3
-; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm3
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm7[3,1,1,3]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[1],zero,zero,ymm2[u,u,u,10,2],zero,zero,ymm2[u,u,u,11,3],zero,zero,ymm2[u,u,u,20,28],zero,zero,ymm2[u,u,u,21,29],zero,zero,ymm2[u]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm6[1,3,3,1]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = zero,ymm4[1,9,u,u,u],zero,zero,ymm4[2,10,u,u,u],zero,zero,ymm4[3,19,u,u,u],zero,zero,ymm4[28,20,u,u,u],zero,zero,ymm4[29,21,u]
-; AVX512-FCP-NEXT:    vpor %ymm2, %ymm4, %ymm2
-; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm0[0,1,2,3,4,5,5,6]
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [2,2,3,3,2,2,3,3]
-; AVX512-FCP-NEXT:    # ymm5 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpermd %ymm4, %ymm5, %ymm4
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm5 = ymm1[1,3,1,3]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm5 = ymm5[u,u,u,1,9],zero,ymm5[u,u,u,u,2,10],zero,ymm5[u,u,u,u,19,27],zero,ymm5[u,u,u,u,20,28],zero,ymm5[u,u,u,u,21]
-; AVX512-FCP-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm5
-; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm5
-; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm3, %zmm5, %zmm2
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm6[0,2,0,2]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = zero,zero,ymm4[0,8,u,u,u],zero,zero,ymm4[1,9,u,u,u],zero,zero,ymm4[18,26,u,u,u],zero,zero,ymm4[19,27,u,u,u],zero,zero,ymm4[20,28]
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [1,5,2,6,1,5,2,6]
-; AVX512-FCP-NEXT:    # ymm5 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpermd %ymm6, %ymm5, %ymm6
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[u,u,u],zero,zero,ymm6[1,5,u,u,u],zero,zero,ymm6[2,6,u,u,u],zero,zero,ymm6[19,23,u,u,u],zero,zero,ymm6[24,28,u,u,u],zero
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm4, %zmm4
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm7[0,2,0,2]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[0,8],zero,zero,ymm6[u,u,u,1,9],zero,zero,ymm6[u,u,u,2,10],zero,zero,ymm6[u,u,u,19,27],zero,zero,ymm6[u,u,u,20,28],zero,zero
-; AVX512-FCP-NEXT:    vpermd %ymm7, %ymm5, %ymm7
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,1,5],zero,zero,ymm7[u,u,u,2,6],zero,zero,ymm7[u,u,u,19,23],zero,zero,ymm7[u,u,u,24,28],zero,zero,ymm7[u,u,u,25]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm6, %zmm6
-; AVX512-FCP-NEXT:    vporq %zmm4, %zmm6, %zmm4
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm0[1,1,0,0,4,5,6,7]
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm7 = [0,1,0,1,0,0,0,0]
-; AVX512-FCP-NEXT:    vpermd %ymm6, %ymm7, %ymm6
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm6, %zmm0
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm1[0,2,0,2]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[u,u,u,u,0,8],zero,ymm6[u,u,u,u,1,9],zero,ymm6[u,u,u,u,18,26],zero,ymm6[u,u,u,u,19,27],zero,ymm6[u,u,u,u]
-; AVX512-FCP-NEXT:    vpermd %ymm1, %ymm5, %ymm1
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[0,4],zero,ymm1[u,u,u,u,1,5],zero,ymm1[u,u,u,u,2,6],zero,ymm1[u,u,u,u,19,23],zero,ymm1[u,u,u,u,24,28],zero,ymm1[u]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm6, %zmm1
-; AVX512-FCP-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1
-; AVX512-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm1
-; AVX512-FCP-NEXT:    vmovdqa %xmm3, 96(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, (%rax)
-; AVX512-FCP-NEXT:    vmovdqa %ymm2, 64(%rax)
+; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm1
+; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm5
+; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm6
+; AVX512-FCP-NEXT:    vmovdqa (%r8), %xmm3
+; AVX512-FCP-NEXT:    vmovdqa (%r9), %xmm4
+; AVX512-FCP-NEXT:    vmovdqa (%r10), %xmm2
+; AVX512-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm7
+; AVX512-FCP-NEXT:    vinserti128 $1, %xmm6, %ymm5, %ymm8
+; AVX512-FCP-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm9
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm8[0,2,0,2]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm10 = zero,zero,ymm10[0,8,u,u,u],zero,zero,ymm10[1,9,u,u,u],zero,zero,ymm10[18,26,u,u,u],zero,zero,ymm10[19,27,u,u,u],zero,zero,ymm10[20,28]
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm11 = [1,5,2,6,1,5,2,6]
+; AVX512-FCP-NEXT:    # ymm11 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vpermd %ymm8, %ymm11, %ymm12
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm12 = ymm12[u,u,u],zero,zero,ymm12[1,5,u,u,u],zero,zero,ymm12[2,6,u,u,u],zero,zero,ymm12[19,23,u,u,u],zero,zero,ymm12[24,28,u,u,u],zero
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm10, %zmm10
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm12 = ymm7[0,2,0,2]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm12 = ymm12[0,8],zero,zero,ymm12[u,u,u,1,9],zero,zero,ymm12[u,u,u,2,10],zero,zero,ymm12[u,u,u,19,27],zero,zero,ymm12[u,u,u,20,28],zero,zero
+; AVX512-FCP-NEXT:    vpermd %ymm7, %ymm11, %ymm13
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm13 = ymm13[u,u,u,1,5],zero,zero,ymm13[u,u,u,2,6],zero,zero,ymm13[u,u,u,19,23],zero,zero,ymm13[u,u,u,24,28],zero,zero,ymm13[u,u,u,25]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm12, %zmm12
+; AVX512-FCP-NEXT:    vporq %zmm10, %zmm12, %zmm10
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm12 = xmm2[1,1,0,0,4,5,6,7]
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm13 = [0,1,0,1,0,0,0,0]
+; AVX512-FCP-NEXT:    vpermd %ymm12, %ymm13, %ymm12
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm13 = xmm2[4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,0,1,0]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm12, %zmm12
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm13 = ymm9[0,2,0,2]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm13 = ymm13[u,u,u,u,0,8],zero,ymm13[u,u,u,u,1,9],zero,ymm13[u,u,u,u,18,26],zero,ymm13[u,u,u,u,19,27],zero,ymm13[u,u,u,u]
+; AVX512-FCP-NEXT:    vpermd %ymm9, %ymm11, %ymm11
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[0,4],zero,ymm11[u,u,u,u,1,5],zero,ymm11[u,u,u,u,2,6],zero,ymm11[u,u,u,u,19,23],zero,ymm11[u,u,u,u,24,28],zero,ymm11[u]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm13, %zmm11
+; AVX512-FCP-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm12, %zmm11
+; AVX512-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm11
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[3,1,1,3]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[1],zero,zero,ymm7[u,u,u,10,2],zero,zero,ymm7[u,u,u,11,3],zero,zero,ymm7[u,u,u,20,28],zero,zero,ymm7[u,u,u,21,29],zero,zero,ymm7[u]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[1,3,3,1]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm8 = zero,ymm8[1,9,u,u,u],zero,zero,ymm8[2,10,u,u,u],zero,zero,ymm8[3,19,u,u,u],zero,zero,ymm8[28,20,u,u,u],zero,zero,ymm8[29,21,u]
+; AVX512-FCP-NEXT:    vpor %ymm7, %ymm8, %ymm7
+; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm8 = xmm2[0,1,2,3,4,5,5,6]
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [2,2,3,3,2,2,3,3]
+; AVX512-FCP-NEXT:    # ymm10 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vpermd %ymm8, %ymm10, %ymm8
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[1,3,1,3]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm9 = ymm9[u,u,u,1,9],zero,ymm9[u,u,u,u,2,10],zero,ymm9[u,u,u,u,19,27],zero,ymm9[u,u,u,u,20,28],zero,ymm9[u,u,u,u,21]
+; AVX512-FCP-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm8, %ymm9
+; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm7, %ymm9
+; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm5[8],xmm6[8],xmm5[9],xmm6[9],xmm5[10],xmm6[10],xmm5[11],xmm6[11],xmm5[12],xmm6[12],xmm5[13],xmm6[13],xmm5[14],xmm6[14],xmm5[15],xmm6[15]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u],zero,zero,xmm5[12,13,u,u,u],zero,zero,xmm5[14,15,u,u,u]
+; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,12,13],zero,zero,xmm0[u,u,u,14,15],zero,zero,xmm0[u,u,u]
+; AVX512-FCP-NEXT:    vpor %xmm5, %xmm0, %xmm0
+; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm4[8],xmm3[8],xmm4[9],xmm3[9],xmm4[10],xmm3[10],xmm4[11],xmm3[11],xmm4[12],xmm3[12],xmm4[13],xmm3[13],xmm4[14],xmm3[14],xmm4[15],xmm3[15]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[10],zero,xmm1[u,u,u,u,13,12],zero,xmm1[u,u,u,u,15,14],zero
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = zero,xmm2[13,u,u,u,u],zero,zero,xmm2[14,u,u,u,u],zero,zero,xmm2[15]
+; AVX512-FCP-NEXT:    vpor %xmm2, %xmm1, %xmm1
+; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
+; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm1, %zmm9, %zmm0
+; AVX512-FCP-NEXT:    vmovdqa %xmm1, 96(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm11, (%rax)
+; AVX512-FCP-NEXT:    vmovdqa %ymm0, 64(%rax)
 ; AVX512-FCP-NEXT:    vzeroupper
 ; AVX512-FCP-NEXT:    retq
 ;
@@ -2202,77 +2198,76 @@ define void @store_i8_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm4
-; AVX512DQ-NEXT:    vmovdqa (%rsi), %xmm5
-; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm6
-; AVX512DQ-NEXT:    vmovdqa (%rcx), %xmm7
-; AVX512DQ-NEXT:    vmovdqa (%r8), %xmm0
-; AVX512DQ-NEXT:    vmovdqa (%r10), %xmm1
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm7, %ymm6, %ymm3
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm5, %ymm4, %ymm2
-; AVX512DQ-NEXT:    vinserti128 $1, (%r9), %ymm0, %ymm0
-; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm1, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm6[8],xmm7[8],xmm6[9],xmm7[9],xmm6[10],xmm7[10],xmm6[11],xmm7[11],xmm6[12],xmm7[12],xmm6[13],xmm7[13],xmm6[14],xmm7[14],xmm6[15],xmm7[15]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[u,u],zero,zero,xmm6[12,13,u,u,u],zero,zero,xmm6[14,15,u,u,u]
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm4[8],xmm5[8],xmm4[9],xmm5[9],xmm4[10],xmm5[10],xmm4[11],xmm5[11],xmm4[12],xmm5[12],xmm4[13],xmm5[13],xmm4[14],xmm5[14],xmm4[15],xmm5[15]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,12,13],zero,zero,xmm4[u,u,u,14,15],zero,zero,xmm4[u,u,u]
-; AVX512DQ-NEXT:    vpor %xmm6, %xmm4, %xmm4
-; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm5
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[10],zero,xmm5[u,u,u,u,13,12],zero,xmm5[u,u,u,u,15,14],zero
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm6 = zero,xmm1[13,u,u,u,u],zero,zero,xmm1[14,u,u,u,u],zero,zero,xmm1[15]
-; AVX512DQ-NEXT:    vpor %xmm6, %xmm5, %xmm5
-; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4, %xmm5
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm2[3,1,1,3]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm4 = ymm4[1],zero,zero,ymm4[u,u,u,10,2],zero,zero,ymm4[u,u,u,11,3],zero,zero,ymm4[u,u,u,20,28],zero,zero,ymm4[u,u,u,21,29],zero,zero,ymm4[u]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm6 = ymm3[1,3,3,1]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm6 = zero,ymm6[1,9,u,u,u],zero,zero,ymm6[2,10,u,u,u],zero,zero,ymm6[3,19,u,u,u],zero,zero,ymm6[28,20,u,u,u],zero,zero,ymm6[29,21,u]
-; AVX512DQ-NEXT:    vpor %ymm4, %ymm6, %ymm4
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm6 = xmm1[0,1,2,3,4,5,5,6]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,2,3,3]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,1,0,1]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm0[1,3,1,3]
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512DQ-NEXT:    vmovdqa (%rsi), %xmm1
+; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm5
+; AVX512DQ-NEXT:    vmovdqa (%rcx), %xmm6
+; AVX512DQ-NEXT:    vmovdqa (%r8), %xmm3
+; AVX512DQ-NEXT:    vmovdqa (%r9), %xmm4
+; AVX512DQ-NEXT:    vmovdqa (%r10), %xmm2
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm8
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm6, %ymm5, %ymm9
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm7
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm10 = ymm9[u,u,u,u,u,5],zero,ymm9[u,u,u,u,u,6],zero,ymm9[u,u,u,u,u],zero,ymm9[23,u,u,u,u,u],zero,ymm9[24,u,u,u,u]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm11 = ymm9[2,3,0,1]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[u,u,u,u,u],zero,ymm11[5,u,u,u,u,u],zero,ymm11[6,u,u,u,u,u,23],zero,ymm11[u,u,u,u,u,24],zero,ymm11[u,u,u,u]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm12 = [255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255]
+; AVX512DQ-NEXT:    vpternlogq $50, %ymm10, %ymm12, %ymm11
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm10 = ymm9[0,2,0,2]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm10 = zero,zero,ymm10[0,8,u,u,u],zero,zero,ymm10[1,9,u,u,u],zero,zero,ymm10[18,26,u,u,u],zero,zero,ymm10[19,27,u,u,u],zero,zero,ymm10[20,28]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm11, %zmm10, %zmm10
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm11 = ymm8[u,u,u,5],zero,ymm8[u,u,u,u,u,6],zero,ymm8[u,u,u,u,u],zero,ymm8[23,u,u,u,u,u],zero,ymm8[24,u,u,u,u,u],zero
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm13 = ymm8[2,3,0,1]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm13 = ymm13[u,u,u],zero,ymm13[5,u,u,u,u,u],zero,ymm13[6,u,u,u,u,u,23],zero,ymm13[u,u,u,u,u,24],zero,ymm13[u,u,u,u,u,25]
+; AVX512DQ-NEXT:    vpternlogq $200, %ymm11, %ymm12, %ymm13
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm11 = ymm8[0,2,0,2]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[0,8],zero,zero,ymm11[u,u,u,1,9],zero,zero,ymm11[u,u,u,2,10],zero,zero,ymm11[u,u,u,19,27],zero,zero,ymm11[u,u,u,20,28],zero,zero
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm13, %zmm11, %zmm11
+; AVX512DQ-NEXT:    vporq %zmm10, %zmm11, %zmm10
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm11 = ymm7[4],zero,ymm7[u,u,u,u,u,5],zero,ymm7[u,u,u,u,u,6],zero,ymm7[u,u,u,u,u],zero,ymm7[23,u,u,u,u,u],zero,ymm7[24,u,u]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm12 = ymm7[2,3,0,1]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm12 = zero,ymm12[4,u,u,u,u,u],zero,ymm12[5,u,u,u,u,u],zero,ymm12[6,u,u,u,u,u,23],zero,ymm12[u,u,u,u,u,24],zero,ymm12[u,u]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm13 = [255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255]
+; AVX512DQ-NEXT:    vpternlogq $200, %ymm11, %ymm13, %ymm12
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm11 = ymm7[0,2,0,2]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[u,u,u,u,0,8],zero,ymm11[u,u,u,u,1,9],zero,ymm11[u,u,u,u,18,26],zero,ymm11[u,u,u,u,19,27],zero,ymm11[u,u,u,u]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm12, %zmm11, %zmm11
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm12 = xmm2[4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,0,1,0]
+; AVX512DQ-NEXT:    vpandn %ymm12, %ymm13, %ymm12
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm13 = xmm2[1,1,0,0,4,5,6,7]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm13 = xmm13[0,1,2,0]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,0,1,0]
+; AVX512DQ-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm13, %ymm13
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm12, %zmm13, %zmm12
+; AVX512DQ-NEXT:    vporq %zmm12, %zmm11, %zmm11
+; AVX512DQ-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm11
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[3,1,1,3]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm8 = ymm8[1],zero,zero,ymm8[u,u,u,10,2],zero,zero,ymm8[u,u,u,11,3],zero,zero,ymm8[u,u,u,20,28],zero,zero,ymm8[u,u,u,21,29],zero,zero,ymm8[u]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[1,3,3,1]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm9 = zero,ymm9[1,9,u,u,u],zero,zero,ymm9[2,10,u,u,u],zero,zero,ymm9[3,19,u,u,u],zero,zero,ymm9[28,20,u,u,u],zero,zero,ymm9[29,21,u]
+; AVX512DQ-NEXT:    vpor %ymm8, %ymm9, %ymm8
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm9 = xmm2[0,1,2,3,4,5,5,6]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm9 = xmm9[2,2,3,3]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,1,0,1]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[1,3,1,3]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,1,9],zero,ymm7[u,u,u,u,2,10],zero,ymm7[u,u,u,u,19,27],zero,ymm7[u,u,u,u,20,28],zero,ymm7[u,u,u,u,21]
-; AVX512DQ-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm6, %ymm7
-; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm7
-; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm5, %zmm7, %zmm4
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm6 = ymm3[u,u,u,u,u,5],zero,ymm3[u,u,u,u,u,6],zero,ymm3[u,u,u,u,u],zero,ymm3[23,u,u,u,u,u],zero,ymm3[24,u,u,u,u]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm3[2,3,0,1]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,u,u],zero,ymm7[5,u,u,u,u,u],zero,ymm7[6,u,u,u,u,u,23],zero,ymm7[u,u,u,u,u,24],zero,ymm7[u,u,u,u]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm8 = [255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255]
-; AVX512DQ-NEXT:    vpternlogq $50, %ymm6, %ymm8, %ymm7
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,2,0,2]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,ymm3[0,8,u,u,u],zero,zero,ymm3[1,9,u,u,u],zero,zero,ymm3[18,26,u,u,u],zero,zero,ymm3[19,27,u,u,u],zero,zero,ymm3[20,28]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm3, %zmm3
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm6 = ymm2[u,u,u,5],zero,ymm2[u,u,u,u,u,6],zero,ymm2[u,u,u,u,u],zero,ymm2[23,u,u,u,u,u],zero,ymm2[24,u,u,u,u,u],zero
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm2[2,3,0,1]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u],zero,ymm7[5,u,u,u,u,u],zero,ymm7[6,u,u,u,u,u,23],zero,ymm7[u,u,u,u,u,24],zero,ymm7[u,u,u,u,u,25]
-; AVX512DQ-NEXT:    vpternlogq $200, %ymm6, %ymm8, %ymm7
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,0,2]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[0,8],zero,zero,ymm2[u,u,u,1,9],zero,zero,ymm2[u,u,u,2,10],zero,zero,ymm2[u,u,u,19,27],zero,zero,ymm2[u,u,u,20,28],zero,zero
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm2, %zmm2
-; AVX512DQ-NEXT:    vporq %zmm3, %zmm2, %zmm2
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm3 = xmm1[4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,1,0]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm6 = [255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255]
-; AVX512DQ-NEXT:    vpandn %ymm3, %ymm6, %ymm3
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[1,1,0,0,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,1,0]
-; AVX512DQ-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm3 = ymm0[4],zero,ymm0[u,u,u,u,u,5],zero,ymm0[u,u,u,u,u,6],zero,ymm0[u,u,u,u,u],zero,ymm0[23,u,u,u,u,u],zero,ymm0[24,u,u]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm0[2,3,0,1]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm7 = zero,ymm7[4,u,u,u,u,u],zero,ymm7[5,u,u,u,u,u],zero,ymm7[6,u,u,u,u,u,23],zero,ymm7[u,u,u,u,u,24],zero,ymm7[u,u]
-; AVX512DQ-NEXT:    vpternlogq $200, %ymm3, %ymm6, %ymm7
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,0,2]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,0,8],zero,ymm0[u,u,u,u,1,9],zero,ymm0[u,u,u,u,18,26],zero,ymm0[u,u,u,u,19,27],zero,ymm0[u,u,u,u]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vporq %zmm1, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0
-; AVX512DQ-NEXT:    vmovdqa %xmm5, 96(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm0, (%rax)
-; AVX512DQ-NEXT:    vmovdqa %ymm4, 64(%rax)
+; AVX512DQ-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm9, %ymm7
+; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm8, %ymm7
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm5[8],xmm6[8],xmm5[9],xmm6[9],xmm5[10],xmm6[10],xmm5[11],xmm6[11],xmm5[12],xmm6[12],xmm5[13],xmm6[13],xmm5[14],xmm6[14],xmm5[15],xmm6[15]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u],zero,zero,xmm5[12,13,u,u,u],zero,zero,xmm5[14,15,u,u,u]
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,12,13],zero,zero,xmm0[u,u,u,14,15],zero,zero,xmm0[u,u,u]
+; AVX512DQ-NEXT:    vpor %xmm5, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm4[8],xmm3[8],xmm4[9],xmm3[9],xmm4[10],xmm3[10],xmm4[11],xmm3[11],xmm4[12],xmm3[12],xmm4[13],xmm3[13],xmm4[14],xmm3[14],xmm4[15],xmm3[15]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[10],zero,xmm1[u,u,u,u,13,12],zero,xmm1[u,u,u,u,15,14],zero
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm2 = zero,xmm2[13,u,u,u,u],zero,zero,xmm2[14,u,u,u,u],zero,zero,xmm2[15]
+; AVX512DQ-NEXT:    vpor %xmm2, %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
+; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm1, %zmm7, %zmm0
+; AVX512DQ-NEXT:    vmovdqa %xmm1, 96(%rax)
+; AVX512DQ-NEXT:    vmovdqa %ymm0, 64(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm11, (%rax)
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
@@ -2280,70 +2275,69 @@ define void @store_i8_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP:       # %bb.0:
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm2
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm4
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm5
-; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %xmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa (%r10), %xmm0
-; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm5, %ymm4, %ymm6
-; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm7
-; AVX512DQ-FCP-NEXT:    vinserti128 $1, (%r9), %ymm1, %ymm1
-; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm1
-; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm4[8],xmm5[8],xmm4[9],xmm5[9],xmm4[10],xmm5[10],xmm4[11],xmm5[11],xmm4[12],xmm5[12],xmm4[13],xmm5[13],xmm4[14],xmm5[14],xmm4[15],xmm5[15]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u],zero,zero,xmm4[12,13,u,u,u],zero,zero,xmm4[14,15,u,u,u]
-; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,12,13],zero,zero,xmm2[u,u,u,14,15],zero,zero,xmm2[u,u,u]
-; AVX512DQ-FCP-NEXT:    vpor %xmm4, %xmm2, %xmm2
-; AVX512DQ-FCP-NEXT:    vextracti128 $1, %ymm1, %xmm3
-; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[10],zero,xmm3[u,u,u,u,13,12],zero,xmm3[u,u,u,u,15,14],zero
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = zero,xmm0[13,u,u,u,u],zero,zero,xmm0[14,u,u,u,u],zero,zero,xmm0[15]
-; AVX512DQ-FCP-NEXT:    vpor %xmm4, %xmm3, %xmm3
-; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm3
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm7[3,1,1,3]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[1],zero,zero,ymm2[u,u,u,10,2],zero,zero,ymm2[u,u,u,11,3],zero,zero,ymm2[u,u,u,20,28],zero,zero,ymm2[u,u,u,21,29],zero,zero,ymm2[u]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm6[1,3,3,1]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = zero,ymm4[1,9,u,u,u],zero,zero,ymm4[2,10,u,u,u],zero,zero,ymm4[3,19,u,u,u],zero,zero,ymm4[28,20,u,u,u],zero,zero,ymm4[29,21,u]
-; AVX512DQ-FCP-NEXT:    vpor %ymm2, %ymm4, %ymm2
-; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm0[0,1,2,3,4,5,5,6]
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [2,2,3,3,2,2,3,3]
-; AVX512DQ-FCP-NEXT:    # ymm5 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm4, %ymm5, %ymm4
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm5 = ymm1[1,3,1,3]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm5 = ymm5[u,u,u,1,9],zero,ymm5[u,u,u,u,2,10],zero,ymm5[u,u,u,u,19,27],zero,ymm5[u,u,u,u,20,28],zero,ymm5[u,u,u,u,21]
-; AVX512DQ-FCP-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm5
-; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm5
-; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm3, %zmm5, %zmm2
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm6[0,2,0,2]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = zero,zero,ymm4[0,8,u,u,u],zero,zero,ymm4[1,9,u,u,u],zero,zero,ymm4[18,26,u,u,u],zero,zero,ymm4[19,27,u,u,u],zero,zero,ymm4[20,28]
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [1,5,2,6,1,5,2,6]
-; AVX512DQ-FCP-NEXT:    # ymm5 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm6, %ymm5, %ymm6
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[u,u,u],zero,zero,ymm6[1,5,u,u,u],zero,zero,ymm6[2,6,u,u,u],zero,zero,ymm6[19,23,u,u,u],zero,zero,ymm6[24,28,u,u,u],zero
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm4, %zmm4
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm7[0,2,0,2]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[0,8],zero,zero,ymm6[u,u,u,1,9],zero,zero,ymm6[u,u,u,2,10],zero,zero,ymm6[u,u,u,19,27],zero,zero,ymm6[u,u,u,20,28],zero,zero
-; AVX512DQ-FCP-NEXT:    vpermd %ymm7, %ymm5, %ymm7
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,1,5],zero,zero,ymm7[u,u,u,2,6],zero,zero,ymm7[u,u,u,19,23],zero,zero,ymm7[u,u,u,24,28],zero,zero,ymm7[u,u,u,25]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm6, %zmm6
-; AVX512DQ-FCP-NEXT:    vporq %zmm4, %zmm6, %zmm4
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm0[1,1,0,0,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm7 = [0,1,0,1,0,0,0,0]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm6, %ymm7, %ymm6
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm6, %zmm0
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm1[0,2,0,2]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[u,u,u,u,0,8],zero,ymm6[u,u,u,u,1,9],zero,ymm6[u,u,u,u,18,26],zero,ymm6[u,u,u,u,19,27],zero,ymm6[u,u,u,u]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm1, %ymm5, %ymm1
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[0,4],zero,ymm1[u,u,u,u,1,5],zero,ymm1[u,u,u,u,2,6],zero,ymm1[u,u,u,u,19,23],zero,ymm1[u,u,u,u,24,28],zero,ymm1[u]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm6, %zmm1
-; AVX512DQ-FCP-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1
-; AVX512DQ-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm3, 96(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, (%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm2, 64(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm1
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm5
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm6
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %xmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %xmm4
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r10), %xmm2
+; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm7
+; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm6, %ymm5, %ymm8
+; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm9
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm8[0,2,0,2]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm10 = zero,zero,ymm10[0,8,u,u,u],zero,zero,ymm10[1,9,u,u,u],zero,zero,ymm10[18,26,u,u,u],zero,zero,ymm10[19,27,u,u,u],zero,zero,ymm10[20,28]
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm11 = [1,5,2,6,1,5,2,6]
+; AVX512DQ-FCP-NEXT:    # ymm11 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm8, %ymm11, %ymm12
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm12 = ymm12[u,u,u],zero,zero,ymm12[1,5,u,u,u],zero,zero,ymm12[2,6,u,u,u],zero,zero,ymm12[19,23,u,u,u],zero,zero,ymm12[24,28,u,u,u],zero
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm10, %zmm10
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm12 = ymm7[0,2,0,2]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm12 = ymm12[0,8],zero,zero,ymm12[u,u,u,1,9],zero,zero,ymm12[u,u,u,2,10],zero,zero,ymm12[u,u,u,19,27],zero,zero,ymm12[u,u,u,20,28],zero,zero
+; AVX512DQ-FCP-NEXT:    vpermd %ymm7, %ymm11, %ymm13
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm13 = ymm13[u,u,u,1,5],zero,zero,ymm13[u,u,u,2,6],zero,zero,ymm13[u,u,u,19,23],zero,zero,ymm13[u,u,u,24,28],zero,zero,ymm13[u,u,u,25]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm12, %zmm12
+; AVX512DQ-FCP-NEXT:    vporq %zmm10, %zmm12, %zmm10
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm12 = xmm2[1,1,0,0,4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm13 = [0,1,0,1,0,0,0,0]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm12, %ymm13, %ymm12
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm13 = xmm2[4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,0,1,0]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm12, %zmm12
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm13 = ymm9[0,2,0,2]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm13 = ymm13[u,u,u,u,0,8],zero,ymm13[u,u,u,u,1,9],zero,ymm13[u,u,u,u,18,26],zero,ymm13[u,u,u,u,19,27],zero,ymm13[u,u,u,u]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm9, %ymm11, %ymm11
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[0,4],zero,ymm11[u,u,u,u,1,5],zero,ymm11[u,u,u,u,2,6],zero,ymm11[u,u,u,u,19,23],zero,ymm11[u,u,u,u,24,28],zero,ymm11[u]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm13, %zmm11
+; AVX512DQ-FCP-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm12, %zmm11
+; AVX512DQ-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm11
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[3,1,1,3]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[1],zero,zero,ymm7[u,u,u,10,2],zero,zero,ymm7[u,u,u,11,3],zero,zero,ymm7[u,u,u,20,28],zero,zero,ymm7[u,u,u,21,29],zero,zero,ymm7[u]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[1,3,3,1]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm8 = zero,ymm8[1,9,u,u,u],zero,zero,ymm8[2,10,u,u,u],zero,zero,ymm8[3,19,u,u,u],zero,zero,ymm8[28,20,u,u,u],zero,zero,ymm8[29,21,u]
+; AVX512DQ-FCP-NEXT:    vpor %ymm7, %ymm8, %ymm7
+; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm8 = xmm2[0,1,2,3,4,5,5,6]
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [2,2,3,3,2,2,3,3]
+; AVX512DQ-FCP-NEXT:    # ymm10 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm8, %ymm10, %ymm8
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[1,3,1,3]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm9 = ymm9[u,u,u,1,9],zero,ymm9[u,u,u,u,2,10],zero,ymm9[u,u,u,u,19,27],zero,ymm9[u,u,u,u,20,28],zero,ymm9[u,u,u,u,21]
+; AVX512DQ-FCP-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm8, %ymm9
+; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm7, %ymm9
+; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm5[8],xmm6[8],xmm5[9],xmm6[9],xmm5[10],xmm6[10],xmm5[11],xmm6[11],xmm5[12],xmm6[12],xmm5[13],xmm6[13],xmm5[14],xmm6[14],xmm5[15],xmm6[15]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u],zero,zero,xmm5[12,13,u,u,u],zero,zero,xmm5[14,15,u,u,u]
+; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,12,13],zero,zero,xmm0[u,u,u,14,15],zero,zero,xmm0[u,u,u]
+; AVX512DQ-FCP-NEXT:    vpor %xmm5, %xmm0, %xmm0
+; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm4[8],xmm3[8],xmm4[9],xmm3[9],xmm4[10],xmm3[10],xmm4[11],xmm3[11],xmm4[12],xmm3[12],xmm4[13],xmm3[13],xmm4[14],xmm3[14],xmm4[15],xmm3[15]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[10],zero,xmm1[u,u,u,u,13,12],zero,xmm1[u,u,u,u,15,14],zero
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = zero,xmm2[13,u,u,u,u],zero,zero,xmm2[14,u,u,u,u],zero,zero,xmm2[15]
+; AVX512DQ-FCP-NEXT:    vpor %xmm2, %xmm1, %xmm1
+; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
+; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm1, %zmm9, %zmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, 96(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm11, (%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, 64(%rax)
 ; AVX512DQ-FCP-NEXT:    vzeroupper
 ; AVX512DQ-FCP-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll b/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
index 11f422d67154..99e8cdb179c8 100644
--- a/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
+++ b/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
@@ -314,8 +314,8 @@ define void @vec64_i16_widen_to_i32_factor2_broadcast_to_v2i32_factor2(ptr %in.v
 ;
 ; AVX512F-LABEL: vec64_i16_widen_to_i32_factor2_broadcast_to_v2i32_factor2:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,10,11,0,1,14,15,u,u,u,u,u,u,u,u]
 ; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)
@@ -324,8 +324,8 @@ define void @vec64_i16_widen_to_i32_factor2_broadcast_to_v2i32_factor2(ptr %in.v
 ;
 ; AVX512DQ-LABEL: vec64_i16_widen_to_i32_factor2_broadcast_to_v2i32_factor2:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512DQ-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,10,11,0,1,14,15,u,u,u,u,u,u,u,u]
 ; AVX512DQ-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, (%rcx)
@@ -981,7 +981,7 @@ define void @vec128_i32_widen_to_i64_factor2_broadcast_to_v2i64_factor2(ptr %in.
 ; AVX512F-NEXT:    vpmovsxbd {{.*#+}} xmm0 = [0,5,0,7]
 ; AVX512F-NEXT:    vmovdqa (%rdi), %ymm1
 ; AVX512F-NEXT:    vpaddb (%rsi), %ymm1, %ymm1
-; AVX512F-NEXT:    vpermd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpermd %ymm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)
 ; AVX512F-NEXT:    vzeroupper
@@ -992,7 +992,7 @@ define void @vec128_i32_widen_to_i64_factor2_broadcast_to_v2i64_factor2(ptr %in.
 ; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} xmm0 = [0,5,0,7]
 ; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm1
 ; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpermd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpermd %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, (%rcx)
 ; AVX512DQ-NEXT:    vzeroupper
@@ -4026,10 +4026,10 @@ define void @vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6(ptr %in.
 ;
 ; AVX512F-FAST-LABEL: vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6:
 ; AVX512F-FAST:       # %bb.0:
-; AVX512F-FAST-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512F-FAST-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX512F-FAST-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX512F-FAST-NEXT:    vmovdqa 48(%rdi), %xmm1
 ; AVX512F-FAST-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
+; AVX512F-FAST-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512F-FAST-NEXT:    vpbroadcastq %xmm0, %ymm2
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1,2,3],ymm2[4],ymm1[5,6,7],ymm2[8],ymm1[9,10,11],ymm2[12],ymm1[13,14,15]
 ; AVX512F-FAST-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
@@ -4062,10 +4062,10 @@ define void @vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6(ptr %in.
 ;
 ; AVX512DQ-FAST-LABEL: vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6:
 ; AVX512DQ-FAST:       # %bb.0:
-; AVX512DQ-FAST-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512DQ-FAST-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX512DQ-FAST-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX512DQ-FAST-NEXT:    vmovdqa 48(%rdi), %xmm1
 ; AVX512DQ-FAST-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
+; AVX512DQ-FAST-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512DQ-FAST-NEXT:    vpbroadcastq %xmm0, %ymm2
 ; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1,2,3],ymm2[4],ymm1[5,6,7],ymm2[8],ymm1[9,10,11],ymm2[12],ymm1[13,14,15]
 ; AVX512DQ-FAST-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
@@ -4541,9 +4541,9 @@ define void @vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2(ptr %
 ;
 ; AVX512F-LABEL: vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX512F-NEXT:    vmovdqa 48(%rdi), %xmm1
+; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512F-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
 ; AVX512F-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]
 ; AVX512F-NEXT:    vpbroadcastw %xmm0, %ymm0
@@ -4559,9 +4559,9 @@ define void @vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2(ptr %
 ;
 ; AVX512DQ-LABEL: vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX512DQ-NEXT:    vmovdqa 48(%rdi), %xmm1
+; AVX512DQ-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]
 ; AVX512DQ-NEXT:    vpbroadcastw %xmm0, %ymm0
-- 
GitLab


From 78eac466095c205988ef1e2380033d042a169a3b Mon Sep 17 00:00:00 2001
From: Sergio Afonso <safonsof@amd.com>
Date: Fri, 12 Apr 2024 12:42:41 +0100
Subject: [PATCH 051/464] [Flang][OpenMP][Lower] Use clause operand structures
 (#86802)

This patch updates Flang lowering to use the new set of OpenMP clause
operand structures and their groupings into directive-specific sets of
clause operands.

It simplifies the passing of information from the clause processor and
the creation of operations.

The `DataSharingProcessor` is slightly modified to not hold delayed
privatization state. Instead, optional arguments are added to
`processStep1` which are only passed when delayed privatization is used.
This enables using the clause operand structure for `private` and
removes the need for the ad-hoc `DelayedPrivatizationInfo` structure.

The processing of the `schedule` clause is updated to process the
`chunk` modifier rather than requiring two separate calls to the
`ClauseProcessor`.

Lowering of a block-associated `ordered` construct is updated to emit a
TODO error if the `simd` clause is specified, since it is not currently
supported by the `ClauseProcessor` or later compilation stages.

Removed processing of `schedule` from `omp.simdloop`, as it doesn't
apply to `simd` constructs.
---
 flang/lib/Lower/OpenMP/ClauseProcessor.cpp    | 273 +++++----
 flang/lib/Lower/OpenMP/ClauseProcessor.h      | 109 ++--
 .../lib/Lower/OpenMP/DataSharingProcessor.cpp |  38 +-
 flang/lib/Lower/OpenMP/DataSharingProcessor.h |  45 +-
 flang/lib/Lower/OpenMP/OpenMP.cpp             | 536 +++++++-----------
 .../Dialect/OpenMP/OpenMPClauseOperands.h     |  10 +-
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td |   4 +
 mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp  |   6 +
 8 files changed, 459 insertions(+), 562 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index fb24c8d1fe3e..ae0d8bd37228 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -162,14 +162,13 @@ getIfClauseOperand(Fortran::lower::AbstractConverter &converter,
                                     ifVal);
 }
 
-static void
-addUseDeviceClause(Fortran::lower::AbstractConverter &converter,
-                   const omp::ObjectList &objects,
-                   llvm::SmallVectorImpl<mlir::Value> &operands,
-                   llvm::SmallVectorImpl<mlir::Type> &useDeviceTypes,
-                   llvm::SmallVectorImpl<mlir::Location> &useDeviceLocs,
-                   llvm::SmallVectorImpl<const Fortran::semantics::Symbol *>
-                       &useDeviceSymbols) {
+static void addUseDeviceClause(
+    Fortran::lower::AbstractConverter &converter,
+    const omp::ObjectList &objects,
+    llvm::SmallVectorImpl<mlir::Value> &operands,
+    llvm::SmallVectorImpl<mlir::Type> &useDeviceTypes,
+    llvm::SmallVectorImpl<mlir::Location> &useDeviceLocs,
+    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &useDeviceSyms) {
   genObjectList(objects, converter, operands);
   for (mlir::Value &operand : operands) {
     checkMapType(operand.getLoc(), operand.getType());
@@ -177,25 +176,24 @@ addUseDeviceClause(Fortran::lower::AbstractConverter &converter,
     useDeviceLocs.push_back(operand.getLoc());
   }
   for (const omp::Object &object : objects)
-    useDeviceSymbols.push_back(object.id());
+    useDeviceSyms.push_back(object.id());
 }
 
 static void convertLoopBounds(Fortran::lower::AbstractConverter &converter,
                               mlir::Location loc,
-                              llvm::SmallVectorImpl<mlir::Value> &lowerBound,
-                              llvm::SmallVectorImpl<mlir::Value> &upperBound,
-                              llvm::SmallVectorImpl<mlir::Value> &step,
+                              mlir::omp::CollapseClauseOps &result,
                               std::size_t loopVarTypeSize) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   // The types of lower bound, upper bound, and step are converted into the
   // type of the loop variable if necessary.
   mlir::Type loopVarType = getLoopVarType(converter, loopVarTypeSize);
-  for (unsigned it = 0; it < (unsigned)lowerBound.size(); it++) {
-    lowerBound[it] =
-        firOpBuilder.createConvert(loc, loopVarType, lowerBound[it]);
-    upperBound[it] =
-        firOpBuilder.createConvert(loc, loopVarType, upperBound[it]);
-    step[it] = firOpBuilder.createConvert(loc, loopVarType, step[it]);
+  for (unsigned it = 0; it < (unsigned)result.loopLBVar.size(); it++) {
+    result.loopLBVar[it] =
+        firOpBuilder.createConvert(loc, loopVarType, result.loopLBVar[it]);
+    result.loopUBVar[it] =
+        firOpBuilder.createConvert(loc, loopVarType, result.loopUBVar[it]);
+    result.loopStepVar[it] =
+        firOpBuilder.createConvert(loc, loopVarType, result.loopStepVar[it]);
   }
 }
 
@@ -205,9 +203,7 @@ static void convertLoopBounds(Fortran::lower::AbstractConverter &converter,
 
 bool ClauseProcessor::processCollapse(
     mlir::Location currentLocation, Fortran::lower::pft::Evaluation &eval,
-    llvm::SmallVectorImpl<mlir::Value> &lowerBound,
-    llvm::SmallVectorImpl<mlir::Value> &upperBound,
-    llvm::SmallVectorImpl<mlir::Value> &step,
+    mlir::omp::CollapseClauseOps &result,
     llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &iv) const {
   bool found = false;
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
@@ -238,15 +234,15 @@ bool ClauseProcessor::processCollapse(
         std::get_if<Fortran::parser::LoopControl::Bounds>(&loopControl->u);
     assert(bounds && "Expected bounds for worksharing do loop");
     Fortran::lower::StatementContext stmtCtx;
-    lowerBound.push_back(fir::getBase(converter.genExprValue(
+    result.loopLBVar.push_back(fir::getBase(converter.genExprValue(
         *Fortran::semantics::GetExpr(bounds->lower), stmtCtx)));
-    upperBound.push_back(fir::getBase(converter.genExprValue(
+    result.loopUBVar.push_back(fir::getBase(converter.genExprValue(
         *Fortran::semantics::GetExpr(bounds->upper), stmtCtx)));
     if (bounds->step) {
-      step.push_back(fir::getBase(converter.genExprValue(
+      result.loopStepVar.push_back(fir::getBase(converter.genExprValue(
           *Fortran::semantics::GetExpr(bounds->step), stmtCtx)));
     } else { // If `step` is not present, assume it as `1`.
-      step.push_back(firOpBuilder.createIntegerConstant(
+      result.loopStepVar.push_back(firOpBuilder.createIntegerConstant(
           currentLocation, firOpBuilder.getIntegerType(32), 1));
     }
     iv.push_back(bounds->name.thing.symbol);
@@ -257,8 +253,7 @@ bool ClauseProcessor::processCollapse(
         &*std::next(doConstructEval->getNestedEvaluations().begin());
   } while (collapseValue > 0);
 
-  convertLoopBounds(converter, currentLocation, lowerBound, upperBound, step,
-                    loopVarTypeSize);
+  convertLoopBounds(converter, currentLocation, result, loopVarTypeSize);
 
   return found;
 }
@@ -286,7 +281,7 @@ bool ClauseProcessor::processDefault() const {
 }
 
 bool ClauseProcessor::processDevice(Fortran::lower::StatementContext &stmtCtx,
-                                    mlir::Value &result) const {
+                                    mlir::omp::DeviceClauseOps &result) const {
   const Fortran::parser::CharBlock *source = nullptr;
   if (auto *clause = findUniqueClause<omp::clause::Device>(&source)) {
     mlir::Location clauseLocation = converter.genLocation(*source);
@@ -298,25 +293,26 @@ bool ClauseProcessor::processDevice(Fortran::lower::StatementContext &stmtCtx,
       }
     }
     const auto &deviceExpr = std::get<omp::SomeExpr>(clause->t);
-    result = fir::getBase(converter.genExprValue(deviceExpr, stmtCtx));
+    result.deviceVar =
+        fir::getBase(converter.genExprValue(deviceExpr, stmtCtx));
     return true;
   }
   return false;
 }
 
 bool ClauseProcessor::processDeviceType(
-    mlir::omp::DeclareTargetDeviceType &result) const {
+    mlir::omp::DeviceTypeClauseOps &result) const {
   if (auto *clause = findUniqueClause<omp::clause::DeviceType>()) {
     // Case: declare target ... device_type(any | host | nohost)
     switch (clause->v) {
     case omp::clause::DeviceType::DeviceTypeDescription::Nohost:
-      result = mlir::omp::DeclareTargetDeviceType::nohost;
+      result.deviceType = mlir::omp::DeclareTargetDeviceType::nohost;
       break;
     case omp::clause::DeviceType::DeviceTypeDescription::Host:
-      result = mlir::omp::DeclareTargetDeviceType::host;
+      result.deviceType = mlir::omp::DeclareTargetDeviceType::host;
       break;
     case omp::clause::DeviceType::DeviceTypeDescription::Any:
-      result = mlir::omp::DeclareTargetDeviceType::any;
+      result.deviceType = mlir::omp::DeclareTargetDeviceType::any;
       break;
     }
     return true;
@@ -325,7 +321,7 @@ bool ClauseProcessor::processDeviceType(
 }
 
 bool ClauseProcessor::processFinal(Fortran::lower::StatementContext &stmtCtx,
-                                   mlir::Value &result) const {
+                                   mlir::omp::FinalClauseOps &result) const {
   const Fortran::parser::CharBlock *source = nullptr;
   if (auto *clause = findUniqueClause<omp::clause::Final>(&source)) {
     fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
@@ -333,100 +329,108 @@ bool ClauseProcessor::processFinal(Fortran::lower::StatementContext &stmtCtx,
 
     mlir::Value finalVal =
         fir::getBase(converter.genExprValue(clause->v, stmtCtx));
-    result = firOpBuilder.createConvert(clauseLocation,
-                                        firOpBuilder.getI1Type(), finalVal);
+    result.finalVar = firOpBuilder.createConvert(
+        clauseLocation, firOpBuilder.getI1Type(), finalVal);
     return true;
   }
   return false;
 }
 
-bool ClauseProcessor::processHint(mlir::IntegerAttr &result) const {
+bool ClauseProcessor::processHint(mlir::omp::HintClauseOps &result) const {
   if (auto *clause = findUniqueClause<omp::clause::Hint>()) {
     fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
     int64_t hintValue = *Fortran::evaluate::ToInt64(clause->v);
-    result = firOpBuilder.getI64IntegerAttr(hintValue);
+    result.hintAttr = firOpBuilder.getI64IntegerAttr(hintValue);
     return true;
   }
   return false;
 }
 
-bool ClauseProcessor::processMergeable(mlir::UnitAttr &result) const {
-  return markClauseOccurrence<omp::clause::Mergeable>(result);
+bool ClauseProcessor::processMergeable(
+    mlir::omp::MergeableClauseOps &result) const {
+  return markClauseOccurrence<omp::clause::Mergeable>(result.mergeableAttr);
 }
 
-bool ClauseProcessor::processNowait(mlir::UnitAttr &result) const {
-  return markClauseOccurrence<omp::clause::Nowait>(result);
+bool ClauseProcessor::processNowait(mlir::omp::NowaitClauseOps &result) const {
+  return markClauseOccurrence<omp::clause::Nowait>(result.nowaitAttr);
 }
 
-bool ClauseProcessor::processNumTeams(Fortran::lower::StatementContext &stmtCtx,
-                                      mlir::Value &result) const {
+bool ClauseProcessor::processNumTeams(
+    Fortran::lower::StatementContext &stmtCtx,
+    mlir::omp::NumTeamsClauseOps &result) const {
   // TODO Get lower and upper bounds for num_teams when parser is updated to
   // accept both.
   if (auto *clause = findUniqueClause<omp::clause::NumTeams>()) {
     // auto lowerBound = std::get<std::optional<ExprTy>>(clause->t);
     auto &upperBound = std::get<ExprTy>(clause->t);
-    result = fir::getBase(converter.genExprValue(upperBound, stmtCtx));
+    result.numTeamsUpperVar =
+        fir::getBase(converter.genExprValue(upperBound, stmtCtx));
     return true;
   }
   return false;
 }
 
 bool ClauseProcessor::processNumThreads(
-    Fortran::lower::StatementContext &stmtCtx, mlir::Value &result) const {
+    Fortran::lower::StatementContext &stmtCtx,
+    mlir::omp::NumThreadsClauseOps &result) const {
   if (auto *clause = findUniqueClause<omp::clause::NumThreads>()) {
     // OMPIRBuilder expects `NUM_THREADS` clause as a `Value`.
-    result = fir::getBase(converter.genExprValue(clause->v, stmtCtx));
+    result.numThreadsVar =
+        fir::getBase(converter.genExprValue(clause->v, stmtCtx));
     return true;
   }
   return false;
 }
 
-bool ClauseProcessor::processOrdered(mlir::IntegerAttr &result) const {
+bool ClauseProcessor::processOrdered(
+    mlir::omp::OrderedClauseOps &result) const {
   if (auto *clause = findUniqueClause<omp::clause::Ordered>()) {
     fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
     int64_t orderedClauseValue = 0l;
     if (clause->v.has_value())
       orderedClauseValue = *Fortran::evaluate::ToInt64(*clause->v);
-    result = firOpBuilder.getI64IntegerAttr(orderedClauseValue);
+    result.orderedAttr = firOpBuilder.getI64IntegerAttr(orderedClauseValue);
     return true;
   }
   return false;
 }
 
-bool ClauseProcessor::processPriority(Fortran::lower::StatementContext &stmtCtx,
-                                      mlir::Value &result) const {
+bool ClauseProcessor::processPriority(
+    Fortran::lower::StatementContext &stmtCtx,
+    mlir::omp::PriorityClauseOps &result) const {
   if (auto *clause = findUniqueClause<omp::clause::Priority>()) {
-    result = fir::getBase(converter.genExprValue(clause->v, stmtCtx));
+    result.priorityVar =
+        fir::getBase(converter.genExprValue(clause->v, stmtCtx));
     return true;
   }
   return false;
 }
 
 bool ClauseProcessor::processProcBind(
-    mlir::omp::ClauseProcBindKindAttr &result) const {
+    mlir::omp::ProcBindClauseOps &result) const {
   if (auto *clause = findUniqueClause<omp::clause::ProcBind>()) {
     fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-    result = genProcBindKindAttr(firOpBuilder, *clause);
+    result.procBindKindAttr = genProcBindKindAttr(firOpBuilder, *clause);
     return true;
   }
   return false;
 }
 
-bool ClauseProcessor::processSafelen(mlir::IntegerAttr &result) const {
+bool ClauseProcessor::processSafelen(
+    mlir::omp::SafelenClauseOps &result) const {
   if (auto *clause = findUniqueClause<omp::clause::Safelen>()) {
     fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
     const std::optional<std::int64_t> safelenVal =
         Fortran::evaluate::ToInt64(clause->v);
-    result = firOpBuilder.getI64IntegerAttr(*safelenVal);
+    result.safelenAttr = firOpBuilder.getI64IntegerAttr(*safelenVal);
     return true;
   }
   return false;
 }
 
 bool ClauseProcessor::processSchedule(
-    mlir::omp::ClauseScheduleKindAttr &valAttr,
-    mlir::omp::ScheduleModifierAttr &modifierAttr,
-    mlir::UnitAttr &simdModifierAttr) const {
+    Fortran::lower::StatementContext &stmtCtx,
+    mlir::omp::ScheduleClauseOps &result) const {
   if (auto *clause = findUniqueClause<omp::clause::Schedule>()) {
     fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
     mlir::MLIRContext *context = firOpBuilder.getContext();
@@ -451,53 +455,51 @@ bool ClauseProcessor::processSchedule(
       break;
     }
 
-    mlir::omp::ScheduleModifier scheduleModifier = getScheduleModifier(*clause);
+    result.scheduleValAttr =
+        mlir::omp::ClauseScheduleKindAttr::get(context, scheduleKind);
 
+    mlir::omp::ScheduleModifier scheduleModifier = getScheduleModifier(*clause);
     if (scheduleModifier != mlir::omp::ScheduleModifier::none)
-      modifierAttr =
+      result.scheduleModAttr =
           mlir::omp::ScheduleModifierAttr::get(context, scheduleModifier);
 
     if (getSimdModifier(*clause) != mlir::omp::ScheduleModifier::none)
-      simdModifierAttr = firOpBuilder.getUnitAttr();
+      result.scheduleSimdAttr = firOpBuilder.getUnitAttr();
 
-    valAttr = mlir::omp::ClauseScheduleKindAttr::get(context, scheduleKind);
-    return true;
-  }
-  return false;
-}
-
-bool ClauseProcessor::processScheduleChunk(
-    Fortran::lower::StatementContext &stmtCtx, mlir::Value &result) const {
-  if (auto *clause = findUniqueClause<omp::clause::Schedule>()) {
     if (const auto &chunkExpr = std::get<omp::MaybeExpr>(clause->t))
-      result = fir::getBase(converter.genExprValue(*chunkExpr, stmtCtx));
+      result.scheduleChunkVar =
+          fir::getBase(converter.genExprValue(*chunkExpr, stmtCtx));
+
     return true;
   }
   return false;
 }
 
-bool ClauseProcessor::processSimdlen(mlir::IntegerAttr &result) const {
+bool ClauseProcessor::processSimdlen(
+    mlir::omp::SimdlenClauseOps &result) const {
   if (auto *clause = findUniqueClause<omp::clause::Simdlen>()) {
     fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
     const std::optional<std::int64_t> simdlenVal =
         Fortran::evaluate::ToInt64(clause->v);
-    result = firOpBuilder.getI64IntegerAttr(*simdlenVal);
+    result.simdlenAttr = firOpBuilder.getI64IntegerAttr(*simdlenVal);
     return true;
   }
   return false;
 }
 
 bool ClauseProcessor::processThreadLimit(
-    Fortran::lower::StatementContext &stmtCtx, mlir::Value &result) const {
+    Fortran::lower::StatementContext &stmtCtx,
+    mlir::omp::ThreadLimitClauseOps &result) const {
   if (auto *clause = findUniqueClause<omp::clause::ThreadLimit>()) {
-    result = fir::getBase(converter.genExprValue(clause->v, stmtCtx));
+    result.threadLimitVar =
+        fir::getBase(converter.genExprValue(clause->v, stmtCtx));
     return true;
   }
   return false;
 }
 
-bool ClauseProcessor::processUntied(mlir::UnitAttr &result) const {
-  return markClauseOccurrence<omp::clause::Untied>(result);
+bool ClauseProcessor::processUntied(mlir::omp::UntiedClauseOps &result) const {
+  return markClauseOccurrence<omp::clause::Untied>(result.untiedAttr);
 }
 
 //===----------------------------------------------------------------------===//
@@ -505,13 +507,12 @@ bool ClauseProcessor::processUntied(mlir::UnitAttr &result) const {
 //===----------------------------------------------------------------------===//
 
 bool ClauseProcessor::processAllocate(
-    llvm::SmallVectorImpl<mlir::Value> &allocatorOperands,
-    llvm::SmallVectorImpl<mlir::Value> &allocateOperands) const {
+    mlir::omp::AllocateClauseOps &result) const {
   return findRepeatableClause<omp::clause::Allocate>(
       [&](const omp::clause::Allocate &clause,
           const Fortran::parser::CharBlock &) {
-        genAllocateClause(converter, clause, allocatorOperands,
-                          allocateOperands);
+        genAllocateClause(converter, clause, result.allocatorVars,
+                          result.allocateVars);
       });
 }
 
@@ -660,10 +661,9 @@ createCopyFunc(mlir::Location loc, Fortran::lower::AbstractConverter &converter,
   return funcOp;
 }
 
-bool ClauseProcessor::processCopyPrivate(
+bool ClauseProcessor::processCopyprivate(
     mlir::Location currentLocation,
-    llvm::SmallVectorImpl<mlir::Value> &copyPrivateVars,
-    llvm::SmallVectorImpl<mlir::Attribute> &copyPrivateFuncs) const {
+    mlir::omp::CopyprivateClauseOps &result) const {
   auto addCopyPrivateVar = [&](Fortran::semantics::Symbol *sym) {
     mlir::Value symVal = converter.getSymbolAddress(*sym);
     auto declOp = symVal.getDefiningOp<hlfir::DeclareOp>();
@@ -690,10 +690,10 @@ bool ClauseProcessor::processCopyPrivate(
       cpVar = alloca;
     }
 
-    copyPrivateVars.push_back(cpVar);
+    result.copyprivateVars.push_back(cpVar);
     mlir::func::FuncOp funcOp =
         createCopyFunc(currentLocation, converter, cpVar.getType(), attrs);
-    copyPrivateFuncs.push_back(mlir::SymbolRefAttr::get(funcOp));
+    result.copyprivateFuncs.push_back(mlir::SymbolRefAttr::get(funcOp));
   };
 
   bool hasCopyPrivate = findRepeatableClause<clause::Copyprivate>(
@@ -714,9 +714,7 @@ bool ClauseProcessor::processCopyPrivate(
   return hasCopyPrivate;
 }
 
-bool ClauseProcessor::processDepend(
-    llvm::SmallVectorImpl<mlir::Attribute> &dependTypeOperands,
-    llvm::SmallVectorImpl<mlir::Value> &dependOperands) const {
+bool ClauseProcessor::processDepend(mlir::omp::DependClauseOps &result) const {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
 
   return findRepeatableClause<omp::clause::Depend>(
@@ -731,7 +729,7 @@ bool ClauseProcessor::processDepend(
 
         mlir::omp::ClauseTaskDependAttr dependTypeOperand =
             genDependKindAttr(firOpBuilder, kind);
-        dependTypeOperands.append(objects.size(), dependTypeOperand);
+        result.dependTypeAttrs.append(objects.size(), dependTypeOperand);
 
         for (const omp::Object &object : objects) {
           assert(object.ref() && "Expecting designator");
@@ -746,13 +744,13 @@ bool ClauseProcessor::processDepend(
 
           Fortran::semantics::Symbol *sym = object.id();
           const mlir::Value variable = converter.getSymbolAddress(*sym);
-          dependOperands.push_back(variable);
+          result.dependVars.push_back(variable);
         }
       });
 }
 
 bool ClauseProcessor::processHasDeviceAddr(
-    llvm::SmallVectorImpl<mlir::Value> &operands,
+    mlir::omp::HasDeviceAddrClauseOps &result,
     llvm::SmallVectorImpl<mlir::Type> &isDeviceTypes,
     llvm::SmallVectorImpl<mlir::Location> &isDeviceLocs,
     llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &isDeviceSymbols)
@@ -760,14 +758,14 @@ bool ClauseProcessor::processHasDeviceAddr(
   return findRepeatableClause<omp::clause::HasDeviceAddr>(
       [&](const omp::clause::HasDeviceAddr &devAddrClause,
           const Fortran::parser::CharBlock &) {
-        addUseDeviceClause(converter, devAddrClause.v, operands, isDeviceTypes,
-                           isDeviceLocs, isDeviceSymbols);
+        addUseDeviceClause(converter, devAddrClause.v, result.hasDeviceAddrVars,
+                           isDeviceTypes, isDeviceLocs, isDeviceSymbols);
       });
 }
 
 bool ClauseProcessor::processIf(
     omp::clause::If::DirectiveNameModifier directiveName,
-    mlir::Value &result) const {
+    mlir::omp::IfClauseOps &result) const {
   bool found = false;
   findRepeatableClause<omp::clause::If>(
       [&](const omp::clause::If &clause,
@@ -778,7 +776,7 @@ bool ClauseProcessor::processIf(
         // Assume that, at most, a single 'if' clause will be applicable to the
         // given directive.
         if (operand) {
-          result = operand;
+          result.ifVar = operand;
           found = true;
         }
       });
@@ -786,7 +784,7 @@ bool ClauseProcessor::processIf(
 }
 
 bool ClauseProcessor::processIsDevicePtr(
-    llvm::SmallVectorImpl<mlir::Value> &operands,
+    mlir::omp::IsDevicePtrClauseOps &result,
     llvm::SmallVectorImpl<mlir::Type> &isDeviceTypes,
     llvm::SmallVectorImpl<mlir::Location> &isDeviceLocs,
     llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &isDeviceSymbols)
@@ -794,8 +792,8 @@ bool ClauseProcessor::processIsDevicePtr(
   return findRepeatableClause<omp::clause::IsDevicePtr>(
       [&](const omp::clause::IsDevicePtr &devPtrClause,
           const Fortran::parser::CharBlock &) {
-        addUseDeviceClause(converter, devPtrClause.v, operands, isDeviceTypes,
-                           isDeviceLocs, isDeviceSymbols);
+        addUseDeviceClause(converter, devPtrClause.v, result.isDevicePtrVars,
+                           isDeviceTypes, isDeviceLocs, isDeviceSymbols);
       });
 }
 
@@ -835,12 +833,10 @@ createMapInfoOp(fir::FirOpBuilder &builder, mlir::Location loc,
 
 bool ClauseProcessor::processMap(
     mlir::Location currentLocation, const llvm::omp::Directive &directive,
-    Fortran::lower::StatementContext &stmtCtx,
-    llvm::SmallVectorImpl<mlir::Value> &mapOperands,
-    llvm::SmallVectorImpl<mlir::Type> *mapSymTypes,
+    Fortran::lower::StatementContext &stmtCtx, mlir::omp::MapClauseOps &result,
+    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *mapSyms,
     llvm::SmallVectorImpl<mlir::Location> *mapSymLocs,
-    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *mapSymbols)
-    const {
+    llvm::SmallVectorImpl<mlir::Type> *mapSymTypes) const {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   return findRepeatableClause<omp::clause::Map>(
       [&](const omp::clause::Map &clause,
@@ -915,25 +911,23 @@ bool ClauseProcessor::processMap(
                   mapTypeBits),
               mlir::omp::VariableCaptureKind::ByRef, symAddr.getType());
 
-          mapOperands.push_back(mapOp);
-          if (mapSymTypes)
-            mapSymTypes->push_back(symAddr.getType());
+          result.mapVars.push_back(mapOp);
+
+          if (mapSyms)
+            mapSyms->push_back(object.id());
           if (mapSymLocs)
             mapSymLocs->push_back(symAddr.getLoc());
-
-          if (mapSymbols)
-            mapSymbols->push_back(object.id());
+          if (mapSymTypes)
+            mapSymTypes->push_back(symAddr.getType());
         }
       });
 }
 
 bool ClauseProcessor::processReduction(
-    mlir::Location currentLocation,
-    llvm::SmallVectorImpl<mlir::Value> &outReductionVars,
-    llvm::SmallVectorImpl<mlir::Type> &outReductionTypes,
-    llvm::SmallVectorImpl<mlir::Attribute> &outReductionDeclSymbols,
-    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *>
-        *outReductionSymbols) const {
+    mlir::Location currentLocation, mlir::omp::ReductionClauseOps &result,
+    llvm::SmallVectorImpl<mlir::Type> *outReductionTypes,
+    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *outReductionSyms)
+    const {
   return findRepeatableClause<omp::clause::Reduction>(
       [&](const omp::clause::Reduction &clause,
           const Fortran::parser::CharBlock &) {
@@ -943,30 +937,31 @@ bool ClauseProcessor::processReduction(
         // whether to do the reduction byref.
         llvm::SmallVector<mlir::Value> reductionVars;
         llvm::SmallVector<mlir::Attribute> reductionDeclSymbols;
-        llvm::SmallVector<const Fortran::semantics::Symbol *> reductionSymbols;
+        llvm::SmallVector<const Fortran::semantics::Symbol *> reductionSyms;
         ReductionProcessor rp;
         rp.addDeclareReduction(currentLocation, converter, clause,
                                reductionVars, reductionDeclSymbols,
-                               outReductionSymbols ? &reductionSymbols
-                                                   : nullptr);
+                               outReductionSyms ? &reductionSyms : nullptr);
 
         // Copy local lists into the output.
-        llvm::copy(reductionVars, std::back_inserter(outReductionVars));
+        llvm::copy(reductionVars, std::back_inserter(result.reductionVars));
         llvm::copy(reductionDeclSymbols,
-                   std::back_inserter(outReductionDeclSymbols));
-        if (outReductionSymbols)
-          llvm::copy(reductionSymbols,
-                     std::back_inserter(*outReductionSymbols));
-
-        outReductionTypes.reserve(outReductionTypes.size() +
-                                  reductionVars.size());
-        llvm::transform(reductionVars, std::back_inserter(outReductionTypes),
-                        [](mlir::Value v) { return v.getType(); });
+                   std::back_inserter(result.reductionDeclSymbols));
+
+        if (outReductionTypes) {
+          outReductionTypes->reserve(outReductionTypes->size() +
+                                     reductionVars.size());
+          llvm::transform(reductionVars, std::back_inserter(*outReductionTypes),
+                          [](mlir::Value v) { return v.getType(); });
+        }
+
+        if (outReductionSyms)
+          llvm::copy(reductionSyms, std::back_inserter(*outReductionSyms));
       });
 }
 
 bool ClauseProcessor::processSectionsReduction(
-    mlir::Location currentLocation) const {
+    mlir::Location currentLocation, mlir::omp::ReductionClauseOps &) const {
   return findRepeatableClause<omp::clause::Reduction>(
       [&](const omp::clause::Reduction &, const Fortran::parser::CharBlock &) {
         TODO(currentLocation, "OMPC_Reduction");
@@ -995,30 +990,30 @@ bool ClauseProcessor::processEnter(
 }
 
 bool ClauseProcessor::processUseDeviceAddr(
-    llvm::SmallVectorImpl<mlir::Value> &operands,
+    mlir::omp::UseDeviceClauseOps &result,
     llvm::SmallVectorImpl<mlir::Type> &useDeviceTypes,
     llvm::SmallVectorImpl<mlir::Location> &useDeviceLocs,
-    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &useDeviceSymbols)
+    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &useDeviceSyms)
     const {
   return findRepeatableClause<omp::clause::UseDeviceAddr>(
       [&](const omp::clause::UseDeviceAddr &clause,
           const Fortran::parser::CharBlock &) {
-        addUseDeviceClause(converter, clause.v, operands, useDeviceTypes,
-                           useDeviceLocs, useDeviceSymbols);
+        addUseDeviceClause(converter, clause.v, result.useDeviceAddrVars,
+                           useDeviceTypes, useDeviceLocs, useDeviceSyms);
       });
 }
 
 bool ClauseProcessor::processUseDevicePtr(
-    llvm::SmallVectorImpl<mlir::Value> &operands,
+    mlir::omp::UseDeviceClauseOps &result,
     llvm::SmallVectorImpl<mlir::Type> &useDeviceTypes,
     llvm::SmallVectorImpl<mlir::Location> &useDeviceLocs,
-    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &useDeviceSymbols)
+    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &useDeviceSyms)
     const {
   return findRepeatableClause<omp::clause::UseDevicePtr>(
       [&](const omp::clause::UseDevicePtr &clause,
           const Fortran::parser::CharBlock &) {
-        addUseDeviceClause(converter, clause.v, operands, useDeviceTypes,
-                           useDeviceLocs, useDeviceSymbols);
+        addUseDeviceClause(converter, clause.v, result.useDevicePtrVars,
+                           useDeviceTypes, useDeviceLocs, useDeviceSyms);
       });
 }
 
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.h b/flang/lib/Lower/OpenMP/ClauseProcessor.h
index df8f4f5310fc..aa2c14b61e75 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.h
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.h
@@ -37,7 +37,7 @@ namespace omp {
 /// corresponding clause if it is present in the clause list. Otherwise, they
 /// will return `false` to signal that the clause was not found.
 ///
-/// The intended use is of this class is to move clause processing outside of
+/// The intended use of this class is to move clause processing outside of
 /// construct processing, since the same clauses can appear attached to
 /// different constructs and constructs can be combined, so that code
 /// duplication is minimized.
@@ -56,61 +56,51 @@ public:
   // 'Unique' clauses: They can appear at most once in the clause list.
   bool processCollapse(
       mlir::Location currentLocation, Fortran::lower::pft::Evaluation &eval,
-      llvm::SmallVectorImpl<mlir::Value> &lowerBound,
-      llvm::SmallVectorImpl<mlir::Value> &upperBound,
-      llvm::SmallVectorImpl<mlir::Value> &step,
+      mlir::omp::CollapseClauseOps &result,
       llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &iv) const;
   bool processDefault() const;
   bool processDevice(Fortran::lower::StatementContext &stmtCtx,
-                     mlir::Value &result) const;
-  bool processDeviceType(mlir::omp::DeclareTargetDeviceType &result) const;
+                     mlir::omp::DeviceClauseOps &result) const;
+  bool processDeviceType(mlir::omp::DeviceTypeClauseOps &result) const;
   bool processFinal(Fortran::lower::StatementContext &stmtCtx,
-                    mlir::Value &result) const;
+                    mlir::omp::FinalClauseOps &result) const;
   bool
-  processHasDeviceAddr(llvm::SmallVectorImpl<mlir::Value> &operands,
+  processHasDeviceAddr(mlir::omp::HasDeviceAddrClauseOps &result,
                        llvm::SmallVectorImpl<mlir::Type> &isDeviceTypes,
                        llvm::SmallVectorImpl<mlir::Location> &isDeviceLocs,
                        llvm::SmallVectorImpl<const Fortran::semantics::Symbol *>
                            &isDeviceSymbols) const;
-  bool processHint(mlir::IntegerAttr &result) const;
-  bool processMergeable(mlir::UnitAttr &result) const;
-  bool processNowait(mlir::UnitAttr &result) const;
+  bool processHint(mlir::omp::HintClauseOps &result) const;
+  bool processMergeable(mlir::omp::MergeableClauseOps &result) const;
+  bool processNowait(mlir::omp::NowaitClauseOps &result) const;
   bool processNumTeams(Fortran::lower::StatementContext &stmtCtx,
-                       mlir::Value &result) const;
+                       mlir::omp::NumTeamsClauseOps &result) const;
   bool processNumThreads(Fortran::lower::StatementContext &stmtCtx,
-                         mlir::Value &result) const;
-  bool processOrdered(mlir::IntegerAttr &result) const;
+                         mlir::omp::NumThreadsClauseOps &result) const;
+  bool processOrdered(mlir::omp::OrderedClauseOps &result) const;
   bool processPriority(Fortran::lower::StatementContext &stmtCtx,
-                       mlir::Value &result) const;
-  bool processProcBind(mlir::omp::ClauseProcBindKindAttr &result) const;
-  bool processSafelen(mlir::IntegerAttr &result) const;
-  bool processSchedule(mlir::omp::ClauseScheduleKindAttr &valAttr,
-                       mlir::omp::ScheduleModifierAttr &modifierAttr,
-                       mlir::UnitAttr &simdModifierAttr) const;
-  bool processScheduleChunk(Fortran::lower::StatementContext &stmtCtx,
-                            mlir::Value &result) const;
-  bool processSimdlen(mlir::IntegerAttr &result) const;
+                       mlir::omp::PriorityClauseOps &result) const;
+  bool processProcBind(mlir::omp::ProcBindClauseOps &result) const;
+  bool processSafelen(mlir::omp::SafelenClauseOps &result) const;
+  bool processSchedule(Fortran::lower::StatementContext &stmtCtx,
+                       mlir::omp::ScheduleClauseOps &result) const;
+  bool processSimdlen(mlir::omp::SimdlenClauseOps &result) const;
   bool processThreadLimit(Fortran::lower::StatementContext &stmtCtx,
-                          mlir::Value &result) const;
-  bool processUntied(mlir::UnitAttr &result) const;
+                          mlir::omp::ThreadLimitClauseOps &result) const;
+  bool processUntied(mlir::omp::UntiedClauseOps &result) const;
 
   // 'Repeatable' clauses: They can appear multiple times in the clause list.
-  bool
-  processAllocate(llvm::SmallVectorImpl<mlir::Value> &allocatorOperands,
-                  llvm::SmallVectorImpl<mlir::Value> &allocateOperands) const;
+  bool processAllocate(mlir::omp::AllocateClauseOps &result) const;
   bool processCopyin() const;
-  bool processCopyPrivate(
-      mlir::Location currentLocation,
-      llvm::SmallVectorImpl<mlir::Value> &copyPrivateVars,
-      llvm::SmallVectorImpl<mlir::Attribute> &copyPrivateFuncs) const;
-  bool processDepend(llvm::SmallVectorImpl<mlir::Attribute> &dependTypeOperands,
-                     llvm::SmallVectorImpl<mlir::Value> &dependOperands) const;
+  bool processCopyprivate(mlir::Location currentLocation,
+                          mlir::omp::CopyprivateClauseOps &result) const;
+  bool processDepend(mlir::omp::DependClauseOps &result) const;
   bool
   processEnter(llvm::SmallVectorImpl<DeclareTargetCapturePair> &result) const;
   bool processIf(omp::clause::If::DirectiveNameModifier directiveName,
-                 mlir::Value &result) const;
+                 mlir::omp::IfClauseOps &result) const;
   bool
-  processIsDevicePtr(llvm::SmallVectorImpl<mlir::Value> &operands,
+  processIsDevicePtr(mlir::omp::IsDevicePtrClauseOps &result,
                      llvm::SmallVectorImpl<mlir::Type> &isDeviceTypes,
                      llvm::SmallVectorImpl<mlir::Location> &isDeviceLocs,
                      llvm::SmallVectorImpl<const Fortran::semantics::Symbol *>
@@ -119,43 +109,42 @@ public:
   processLink(llvm::SmallVectorImpl<DeclareTargetCapturePair> &result) const;
 
   // This method is used to process a map clause.
-  // The optional parameters - mapSymTypes, mapSymLocs & mapSymbols are used to
+  // The optional parameters - mapSymTypes, mapSymLocs & mapSyms are used to
   // store the original type, location and Fortran symbol for the map operands.
   // They may be used later on to create the block_arguments for some of the
   // target directives that require it.
-  bool processMap(mlir::Location currentLocation,
-                  const llvm::omp::Directive &directive,
-                  Fortran::lower::StatementContext &stmtCtx,
-                  llvm::SmallVectorImpl<mlir::Value> &mapOperands,
-                  llvm::SmallVectorImpl<mlir::Type> *mapSymTypes = nullptr,
-                  llvm::SmallVectorImpl<mlir::Location> *mapSymLocs = nullptr,
-                  llvm::SmallVectorImpl<const Fortran::semantics::Symbol *>
-                      *mapSymbols = nullptr) const;
-  bool
-  processReduction(mlir::Location currentLocation,
-                   llvm::SmallVectorImpl<mlir::Value> &reductionVars,
-                   llvm::SmallVectorImpl<mlir::Type> &reductionTypes,
-                   llvm::SmallVectorImpl<mlir::Attribute> &reductionDeclSymbols,
-                   llvm::SmallVectorImpl<const Fortran::semantics::Symbol *>
-                       *reductionSymbols = nullptr) const;
-  bool processSectionsReduction(mlir::Location currentLocation) const;
+  bool processMap(
+      mlir::Location currentLocation, const llvm::omp::Directive &directive,
+      Fortran::lower::StatementContext &stmtCtx,
+      mlir::omp::MapClauseOps &result,
+      llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *mapSyms =
+          nullptr,
+      llvm::SmallVectorImpl<mlir::Location> *mapSymLocs = nullptr,
+      llvm::SmallVectorImpl<mlir::Type> *mapSymTypes = nullptr) const;
+  bool processReduction(
+      mlir::Location currentLocation, mlir::omp::ReductionClauseOps &result,
+      llvm::SmallVectorImpl<mlir::Type> *reductionTypes = nullptr,
+      llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *reductionSyms =
+          nullptr) const;
+  bool processSectionsReduction(mlir::Location currentLocation,
+                                mlir::omp::ReductionClauseOps &result) const;
   bool processTo(llvm::SmallVectorImpl<DeclareTargetCapturePair> &result) const;
   bool
-  processUseDeviceAddr(llvm::SmallVectorImpl<mlir::Value> &operands,
+  processUseDeviceAddr(mlir::omp::UseDeviceClauseOps &result,
                        llvm::SmallVectorImpl<mlir::Type> &useDeviceTypes,
                        llvm::SmallVectorImpl<mlir::Location> &useDeviceLocs,
                        llvm::SmallVectorImpl<const Fortran::semantics::Symbol *>
-                           &useDeviceSymbols) const;
+                           &useDeviceSyms) const;
   bool
-  processUseDevicePtr(llvm::SmallVectorImpl<mlir::Value> &operands,
+  processUseDevicePtr(mlir::omp::UseDeviceClauseOps &result,
                       llvm::SmallVectorImpl<mlir::Type> &useDeviceTypes,
                       llvm::SmallVectorImpl<mlir::Location> &useDeviceLocs,
                       llvm::SmallVectorImpl<const Fortran::semantics::Symbol *>
-                          &useDeviceSymbols) const;
+                          &useDeviceSyms) const;
 
   template <typename T>
   bool processMotionClauses(Fortran::lower::StatementContext &stmtCtx,
-                            llvm::SmallVectorImpl<mlir::Value> &mapOperands);
+                            mlir::omp::MapClauseOps &result);
 
   // Call this method for these clauses that should be supported but are not
   // implemented yet. It triggers a compilation error if any of the given
@@ -197,7 +186,7 @@ private:
 template <typename T>
 bool ClauseProcessor::processMotionClauses(
     Fortran::lower::StatementContext &stmtCtx,
-    llvm::SmallVectorImpl<mlir::Value> &mapOperands) {
+    mlir::omp::MapClauseOps &result) {
   return findRepeatableClause<T>(
       [&](const T &clause, const Fortran::parser::CharBlock &source) {
         mlir::Location clauseLocation = converter.genLocation(source);
@@ -239,7 +228,7 @@ bool ClauseProcessor::processMotionClauses(
                   mapTypeBits),
               mlir::omp::VariableCaptureKind::ByRef, symAddr.getType());
 
-          mapOperands.push_back(mapOp);
+          result.mapVars.push_back(mapOp);
         }
       });
 }
diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
index e114ab9f4548..5a42e6a6aa41 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
@@ -23,11 +23,13 @@ namespace Fortran {
 namespace lower {
 namespace omp {
 
-void DataSharingProcessor::processStep1() {
+void DataSharingProcessor::processStep1(
+    mlir::omp::PrivateClauseOps *clauseOps,
+    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *privateSyms) {
   collectSymbolsForPrivatization();
   collectDefaultSymbols();
-  privatize();
-  defaultPrivatize();
+  privatize(clauseOps, privateSyms);
+  defaultPrivatize(clauseOps, privateSyms);
   insertBarrier();
 }
 
@@ -299,14 +301,16 @@ void DataSharingProcessor::collectDefaultSymbols() {
   }
 }
 
-void DataSharingProcessor::privatize() {
+void DataSharingProcessor::privatize(
+    mlir::omp::PrivateClauseOps *clauseOps,
+    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *privateSyms) {
   for (const Fortran::semantics::Symbol *sym : privatizedSymbols) {
     if (const auto *commonDet =
             sym->detailsIf<Fortran::semantics::CommonBlockDetails>()) {
       for (const auto &mem : commonDet->objects())
-        doPrivatize(&*mem);
+        doPrivatize(&*mem, clauseOps, privateSyms);
     } else
-      doPrivatize(sym);
+      doPrivatize(sym, clauseOps, privateSyms);
   }
 }
 
@@ -323,7 +327,9 @@ void DataSharingProcessor::copyLastPrivatize(mlir::Operation *op) {
     }
 }
 
-void DataSharingProcessor::defaultPrivatize() {
+void DataSharingProcessor::defaultPrivatize(
+    mlir::omp::PrivateClauseOps *clauseOps,
+    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *privateSyms) {
   for (const Fortran::semantics::Symbol *sym : defaultSymbols) {
     if (!Fortran::semantics::IsProcedure(*sym) &&
         !sym->GetUltimate().has<Fortran::semantics::DerivedTypeDetails>() &&
@@ -331,11 +337,14 @@ void DataSharingProcessor::defaultPrivatize() {
         !symbolsInNestedRegions.contains(sym) &&
         !symbolsInParentRegions.contains(sym) &&
         !privatizedSymbols.contains(sym))
-      doPrivatize(sym);
+      doPrivatize(sym, clauseOps, privateSyms);
   }
 }
 
-void DataSharingProcessor::doPrivatize(const Fortran::semantics::Symbol *sym) {
+void DataSharingProcessor::doPrivatize(
+    const Fortran::semantics::Symbol *sym,
+    mlir::omp::PrivateClauseOps *clauseOps,
+    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *privateSyms) {
   if (!useDelayedPrivatization) {
     cloneSymbol(sym);
     copyFirstPrivateSymbol(sym);
@@ -419,10 +428,13 @@ void DataSharingProcessor::doPrivatize(const Fortran::semantics::Symbol *sym) {
     return result;
   }();
 
-  delayedPrivatizationInfo.privatizers.push_back(
-      mlir::SymbolRefAttr::get(privatizerOp));
-  delayedPrivatizationInfo.originalAddresses.push_back(hsb.getAddr());
-  delayedPrivatizationInfo.symbols.push_back(sym);
+  if (clauseOps) {
+    clauseOps->privatizers.push_back(mlir::SymbolRefAttr::get(privatizerOp));
+    clauseOps->privateVars.push_back(hsb.getAddr());
+  }
+
+  if (privateSyms)
+    privateSyms->push_back(sym);
 }
 
 } // namespace omp
diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.h b/flang/lib/Lower/OpenMP/DataSharingProcessor.h
index 1cbc825fd5e1..c11ee299c5d0 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.h
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.h
@@ -19,28 +19,17 @@
 #include "flang/Parser/parse-tree.h"
 #include "flang/Semantics/symbol.h"
 
+namespace mlir {
+namespace omp {
+struct PrivateClauseOps;
+} // namespace omp
+} // namespace mlir
+
 namespace Fortran {
 namespace lower {
 namespace omp {
 
 class DataSharingProcessor {
-public:
-  /// Collects all the information needed for delayed privatization. This can be
-  /// used by ops with data-sharing clauses to properly generate their regions
-  /// (e.g. add region arguments) and map the original SSA values to their
-  /// corresponding OMP region operands.
-  struct DelayedPrivatizationInfo {
-    // The list of symbols referring to delayed privatizer ops (i.e.
-    // `omp.private` ops).
-    llvm::SmallVector<mlir::SymbolRefAttr> privatizers;
-    // SSA values that correspond to "original" values being privatized.
-    // "Original" here means the SSA value outside the OpenMP region from which
-    // a clone is created inside the region.
-    llvm::SmallVector<mlir::Value> originalAddresses;
-    // Fortran symbols corresponding to the above SSA values.
-    llvm::SmallVector<const Fortran::semantics::Symbol *> symbols;
-  };
-
 private:
   bool hasLastPrivateOp;
   mlir::OpBuilder::InsertPoint lastPrivIP;
@@ -57,7 +46,6 @@ private:
   Fortran::lower::pft::Evaluation &eval;
   bool useDelayedPrivatization;
   Fortran::lower::SymMap *symTable;
-  DelayedPrivatizationInfo delayedPrivatizationInfo;
 
   bool needBarrier();
   void collectSymbols(Fortran::semantics::Symbol::Flag flag);
@@ -67,9 +55,16 @@ private:
   void collectSymbolsForPrivatization();
   void insertBarrier();
   void collectDefaultSymbols();
-  void privatize();
-  void defaultPrivatize();
-  void doPrivatize(const Fortran::semantics::Symbol *sym);
+  void privatize(
+      mlir::omp::PrivateClauseOps *clauseOps,
+      llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *privateSyms);
+  void defaultPrivatize(
+      mlir::omp::PrivateClauseOps *clauseOps,
+      llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *privateSyms);
+  void doPrivatize(
+      const Fortran::semantics::Symbol *sym,
+      mlir::omp::PrivateClauseOps *clauseOps,
+      llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *privateSyms);
   void copyLastPrivatize(mlir::Operation *op);
   void insertLastPrivateCompare(mlir::Operation *op);
   void cloneSymbol(const Fortran::semantics::Symbol *sym);
@@ -103,17 +98,15 @@ public:
   // Step2 performs the copying for lastprivates and requires knowledge of the
   // MLIR operation to insert the last private update. Step2 adds
   // dealocation code as well.
-  void processStep1();
+  void processStep1(mlir::omp::PrivateClauseOps *clauseOps = nullptr,
+                    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *>
+                        *privateSyms = nullptr);
   void processStep2(mlir::Operation *op, bool isLoop);
 
   void setLoopIV(mlir::Value iv) {
     assert(!loopIV && "Loop iteration variable already set");
     loopIV = iv;
   }
-
-  const DelayedPrivatizationInfo &getDelayedPrivatizationInfo() const {
-    return delayedPrivatizationInfo;
-  }
 };
 
 } // namespace omp
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 50ad889052ab..3dcfe0fd775d 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -730,19 +730,25 @@ genMasterOp(Fortran::lower::AbstractConverter &converter,
             mlir::Location currentLocation) {
   return genOpWithBody<mlir::omp::MasterOp>(
       OpWithBodyGenInfo(converter, semaCtx, currentLocation, eval)
-          .setGenNested(genNested),
-      /*resultTypes=*/mlir::TypeRange());
+          .setGenNested(genNested));
 }
 
 static mlir::omp::OrderedRegionOp
 genOrderedRegionOp(Fortran::lower::AbstractConverter &converter,
                    Fortran::semantics::SemanticsContext &semaCtx,
                    Fortran::lower::pft::Evaluation &eval, bool genNested,
-                   mlir::Location currentLocation) {
+                   mlir::Location currentLocation,
+                   const Fortran::parser::OmpClauseList &clauseList) {
+  mlir::omp::OrderedRegionClauseOps clauseOps;
+
+  ClauseProcessor cp(converter, semaCtx, clauseList);
+  cp.processTODO<clause::Simd>(currentLocation,
+                               llvm::omp::Directive::OMPD_ordered);
+
   return genOpWithBody<mlir::omp::OrderedRegionOp>(
       OpWithBodyGenInfo(converter, semaCtx, currentLocation, eval)
           .setGenNested(genNested),
-      /*simd=*/false);
+      clauseOps);
 }
 
 static mlir::omp::ParallelOp
@@ -753,77 +759,62 @@ genParallelOp(Fortran::lower::AbstractConverter &converter,
               mlir::Location currentLocation,
               const Fortran::parser::OmpClauseList &clauseList,
               bool outerCombined = false) {
+  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   Fortran::lower::StatementContext stmtCtx;
-  mlir::Value ifClauseOperand, numThreadsClauseOperand;
-  mlir::omp::ClauseProcBindKindAttr procBindKindAttr;
-  llvm::SmallVector<mlir::Value> allocateOperands, allocatorOperands,
-      reductionVars;
+  mlir::omp::ParallelClauseOps clauseOps;
+  llvm::SmallVector<const Fortran::semantics::Symbol *> privateSyms;
   llvm::SmallVector<mlir::Type> reductionTypes;
-  llvm::SmallVector<mlir::Attribute> reductionDeclSymbols;
-  llvm::SmallVector<const Fortran::semantics::Symbol *> reductionSymbols;
+  llvm::SmallVector<const Fortran::semantics::Symbol *> reductionSyms;
 
   ClauseProcessor cp(converter, semaCtx, clauseList);
-  cp.processIf(llvm::omp::Directive::OMPD_parallel, ifClauseOperand);
-  cp.processNumThreads(stmtCtx, numThreadsClauseOperand);
-  cp.processProcBind(procBindKindAttr);
+  cp.processIf(llvm::omp::Directive::OMPD_parallel, clauseOps);
+  cp.processNumThreads(stmtCtx, clauseOps);
+  cp.processProcBind(clauseOps);
   cp.processDefault();
-  cp.processAllocate(allocatorOperands, allocateOperands);
+  cp.processAllocate(clauseOps);
+
   if (!outerCombined)
-    cp.processReduction(currentLocation, reductionVars, reductionTypes,
-                        reductionDeclSymbols, &reductionSymbols);
+    cp.processReduction(currentLocation, clauseOps, &reductionTypes,
+                        &reductionSyms);
+
+  if (ReductionProcessor::doReductionByRef(clauseOps.reductionVars))
+    clauseOps.reductionByRefAttr = firOpBuilder.getUnitAttr();
 
   auto reductionCallback = [&](mlir::Operation *op) {
-    llvm::SmallVector<mlir::Location> locs(reductionVars.size(),
+    llvm::SmallVector<mlir::Location> locs(clauseOps.reductionVars.size(),
                                            currentLocation);
-    auto *block = converter.getFirOpBuilder().createBlock(&op->getRegion(0), {},
-                                                          reductionTypes, locs);
+    auto *block =
+        firOpBuilder.createBlock(&op->getRegion(0), {}, reductionTypes, locs);
     for (auto [arg, prv] :
-         llvm::zip_equal(reductionSymbols, block->getArguments())) {
+         llvm::zip_equal(reductionSyms, block->getArguments())) {
       converter.bindSymbol(*arg, prv);
     }
-    return reductionSymbols;
+    return reductionSyms;
   };
 
-  mlir::UnitAttr byrefAttr;
-  if (ReductionProcessor::doReductionByRef(reductionVars))
-    byrefAttr = converter.getFirOpBuilder().getUnitAttr();
-
   OpWithBodyGenInfo genInfo =
       OpWithBodyGenInfo(converter, semaCtx, currentLocation, eval)
           .setGenNested(genNested)
           .setOuterCombined(outerCombined)
           .setClauses(&clauseList)
-          .setReductions(&reductionSymbols, &reductionTypes)
+          .setReductions(&reductionSyms, &reductionTypes)
           .setGenRegionEntryCb(reductionCallback);
 
-  if (!enableDelayedPrivatization) {
-    return genOpWithBody<mlir::omp::ParallelOp>(
-        genInfo,
-        /*resultTypes=*/mlir::TypeRange(), ifClauseOperand,
-        numThreadsClauseOperand, allocateOperands, allocatorOperands,
-        reductionVars,
-        reductionDeclSymbols.empty()
-            ? nullptr
-            : mlir::ArrayAttr::get(converter.getFirOpBuilder().getContext(),
-                                   reductionDeclSymbols),
-        procBindKindAttr, /*private_vars=*/llvm::SmallVector<mlir::Value>{},
-        /*privatizers=*/nullptr, byrefAttr);
-  }
+  if (!enableDelayedPrivatization)
+    return genOpWithBody<mlir::omp::ParallelOp>(genInfo, clauseOps);
 
   bool privatize = !outerCombined;
   DataSharingProcessor dsp(converter, semaCtx, clauseList, eval,
                            /*useDelayedPrivatization=*/true, &symTable);
 
   if (privatize)
-    dsp.processStep1();
-
-  const auto &delayedPrivatizationInfo = dsp.getDelayedPrivatizationInfo();
+    dsp.processStep1(&clauseOps, &privateSyms);
 
   auto genRegionEntryCB = [&](mlir::Operation *op) {
     auto parallelOp = llvm::cast<mlir::omp::ParallelOp>(op);
 
-    llvm::SmallVector<mlir::Location> reductionLocs(reductionVars.size(),
-                                                    currentLocation);
+    llvm::SmallVector<mlir::Location> reductionLocs(
+        clauseOps.reductionVars.size(), currentLocation);
 
     mlir::OperandRange privateVars = parallelOp.getPrivateVars();
     mlir::Region &region = parallelOp.getRegion();
@@ -838,12 +829,12 @@ genParallelOp(Fortran::lower::AbstractConverter &converter,
     llvm::transform(privateVars, std::back_inserter(privateVarLocs),
                     [](mlir::Value v) { return v.getLoc(); });
 
-    converter.getFirOpBuilder().createBlock(&region, /*insertPt=*/{},
-                                            privateVarTypes, privateVarLocs);
+    firOpBuilder.createBlock(&region, /*insertPt=*/{}, privateVarTypes,
+                             privateVarLocs);
 
     llvm::SmallVector<const Fortran::semantics::Symbol *> allSymbols =
-        reductionSymbols;
-    allSymbols.append(delayedPrivatizationInfo.symbols);
+        reductionSyms;
+    allSymbols.append(privateSyms);
     for (auto [arg, prv] : llvm::zip_equal(allSymbols, region.getArguments())) {
       converter.bindSymbol(*arg, prv);
     }
@@ -853,26 +844,7 @@ genParallelOp(Fortran::lower::AbstractConverter &converter,
 
   // TODO Merge with the reduction CB.
   genInfo.setGenRegionEntryCb(genRegionEntryCB).setDataSharingProcessor(&dsp);
-
-  llvm::SmallVector<mlir::Attribute> privatizers(
-      delayedPrivatizationInfo.privatizers.begin(),
-      delayedPrivatizationInfo.privatizers.end());
-
-  return genOpWithBody<mlir::omp::ParallelOp>(
-      genInfo,
-      /*resultTypes=*/mlir::TypeRange(), ifClauseOperand,
-      numThreadsClauseOperand, allocateOperands, allocatorOperands,
-      reductionVars,
-      reductionDeclSymbols.empty()
-          ? nullptr
-          : mlir::ArrayAttr::get(converter.getFirOpBuilder().getContext(),
-                                 reductionDeclSymbols),
-      procBindKindAttr, delayedPrivatizationInfo.originalAddresses,
-      delayedPrivatizationInfo.privatizers.empty()
-          ? nullptr
-          : mlir::ArrayAttr::get(converter.getFirOpBuilder().getContext(),
-                                 privatizers),
-      byrefAttr);
+  return genOpWithBody<mlir::omp::ParallelOp>(genInfo, clauseOps);
 }
 
 static mlir::omp::SectionOp
@@ -896,28 +868,21 @@ genSingleOp(Fortran::lower::AbstractConverter &converter,
             mlir::Location currentLocation,
             const Fortran::parser::OmpClauseList &beginClauseList,
             const Fortran::parser::OmpClauseList &endClauseList) {
-  llvm::SmallVector<mlir::Value> allocateOperands, allocatorOperands;
-  llvm::SmallVector<mlir::Value> copyPrivateVars;
-  llvm::SmallVector<mlir::Attribute> copyPrivateFuncs;
-  mlir::UnitAttr nowaitAttr;
+  mlir::omp::SingleClauseOps clauseOps;
 
   ClauseProcessor cp(converter, semaCtx, beginClauseList);
-  cp.processAllocate(allocatorOperands, allocateOperands);
+  cp.processAllocate(clauseOps);
+  // TODO Support delayed privatization.
 
   ClauseProcessor ecp(converter, semaCtx, endClauseList);
-  ecp.processNowait(nowaitAttr);
-  ecp.processCopyPrivate(currentLocation, copyPrivateVars, copyPrivateFuncs);
+  ecp.processNowait(clauseOps);
+  ecp.processCopyprivate(currentLocation, clauseOps);
 
   return genOpWithBody<mlir::omp::SingleOp>(
       OpWithBodyGenInfo(converter, semaCtx, currentLocation, eval)
           .setGenNested(genNested)
           .setClauses(&beginClauseList),
-      allocateOperands, allocatorOperands, copyPrivateVars,
-      copyPrivateFuncs.empty()
-          ? nullptr
-          : mlir::ArrayAttr::get(converter.getFirOpBuilder().getContext(),
-                                 copyPrivateFuncs),
-      nowaitAttr);
+      clauseOps);
 }
 
 static mlir::omp::TaskOp
@@ -927,21 +892,19 @@ genTaskOp(Fortran::lower::AbstractConverter &converter,
           mlir::Location currentLocation,
           const Fortran::parser::OmpClauseList &clauseList) {
   Fortran::lower::StatementContext stmtCtx;
-  mlir::Value ifClauseOperand, finalClauseOperand, priorityClauseOperand;
-  mlir::UnitAttr untiedAttr, mergeableAttr;
-  llvm::SmallVector<mlir::Attribute> dependTypeOperands;
-  llvm::SmallVector<mlir::Value> allocateOperands, allocatorOperands,
-      dependOperands;
+  mlir::omp::TaskClauseOps clauseOps;
 
   ClauseProcessor cp(converter, semaCtx, clauseList);
-  cp.processIf(llvm::omp::Directive::OMPD_task, ifClauseOperand);
-  cp.processAllocate(allocatorOperands, allocateOperands);
+  cp.processIf(llvm::omp::Directive::OMPD_task, clauseOps);
+  cp.processAllocate(clauseOps);
   cp.processDefault();
-  cp.processFinal(stmtCtx, finalClauseOperand);
-  cp.processUntied(untiedAttr);
-  cp.processMergeable(mergeableAttr);
-  cp.processPriority(stmtCtx, priorityClauseOperand);
-  cp.processDepend(dependTypeOperands, dependOperands);
+  cp.processFinal(stmtCtx, clauseOps);
+  cp.processUntied(clauseOps);
+  cp.processMergeable(clauseOps);
+  cp.processPriority(stmtCtx, clauseOps);
+  cp.processDepend(clauseOps);
+  // TODO Support delayed privatization.
+
   cp.processTODO<clause::InReduction, clause::Detach, clause::Affinity>(
       currentLocation, llvm::omp::Directive::OMPD_task);
 
@@ -949,14 +912,7 @@ genTaskOp(Fortran::lower::AbstractConverter &converter,
       OpWithBodyGenInfo(converter, semaCtx, currentLocation, eval)
           .setGenNested(genNested)
           .setClauses(&clauseList),
-      ifClauseOperand, finalClauseOperand, untiedAttr, mergeableAttr,
-      /*in_reduction_vars=*/mlir::ValueRange(),
-      /*in_reductions=*/nullptr, priorityClauseOperand,
-      dependTypeOperands.empty()
-          ? nullptr
-          : mlir::ArrayAttr::get(converter.getFirOpBuilder().getContext(),
-                                 dependTypeOperands),
-      dependOperands, allocateOperands, allocatorOperands);
+      clauseOps);
 }
 
 static mlir::omp::TaskgroupOp
@@ -965,17 +921,18 @@ genTaskgroupOp(Fortran::lower::AbstractConverter &converter,
                Fortran::lower::pft::Evaluation &eval, bool genNested,
                mlir::Location currentLocation,
                const Fortran::parser::OmpClauseList &clauseList) {
-  llvm::SmallVector<mlir::Value> allocateOperands, allocatorOperands;
+  mlir::omp::TaskgroupClauseOps clauseOps;
+
   ClauseProcessor cp(converter, semaCtx, clauseList);
-  cp.processAllocate(allocatorOperands, allocateOperands);
+  cp.processAllocate(clauseOps);
   cp.processTODO<clause::TaskReduction>(currentLocation,
                                         llvm::omp::Directive::OMPD_taskgroup);
+
   return genOpWithBody<mlir::omp::TaskgroupOp>(
       OpWithBodyGenInfo(converter, semaCtx, currentLocation, eval)
           .setGenNested(genNested)
           .setClauses(&clauseList),
-      /*task_reduction_vars=*/mlir::ValueRange(),
-      /*task_reductions=*/nullptr, allocateOperands, allocatorOperands);
+      clauseOps);
 }
 
 // This helper function implements the functionality of "promoting"
@@ -996,8 +953,7 @@ genTaskgroupOp(Fortran::lower::AbstractConverter &converter,
 //  clause. Support for such list items in a use_device_ptr clause
 //  is deprecated."
 static void promoteNonCPtrUseDevicePtrArgsToUseDeviceAddr(
-    llvm::SmallVectorImpl<mlir::Value> &devicePtrOperands,
-    llvm::SmallVectorImpl<mlir::Value> &deviceAddrOperands,
+    mlir::omp::UseDeviceClauseOps &clauseOps,
     llvm::SmallVectorImpl<mlir::Type> &useDeviceTypes,
     llvm::SmallVectorImpl<mlir::Location> &useDeviceLocs,
     llvm::SmallVectorImpl<const Fortran::semantics::Symbol *>
@@ -1010,9 +966,10 @@ static void promoteNonCPtrUseDevicePtrArgsToUseDeviceAddr(
 
   // Iterate over our use_device_ptr list and shift all non-cptr arguments into
   // use_device_addr.
-  for (auto *it = devicePtrOperands.begin(); it != devicePtrOperands.end();) {
+  for (auto *it = clauseOps.useDevicePtrVars.begin();
+       it != clauseOps.useDevicePtrVars.end();) {
     if (!fir::isa_builtin_cptr_type(fir::unwrapRefType(it->getType()))) {
-      deviceAddrOperands.push_back(*it);
+      clauseOps.useDeviceAddrVars.push_back(*it);
       // We have to shuffle the symbols around as well, to maintain
       // the correct Input -> BlockArg for use_device_ptr/use_device_addr.
       // NOTE: However, as map's do not seem to be included currently
@@ -1020,11 +977,11 @@ static void promoteNonCPtrUseDevicePtrArgsToUseDeviceAddr(
       // future alterations. I believe the reason they are not currently
       // is that the BlockArg assign/lowering needs to be extended
       // to a greater set of types.
-      auto idx = std::distance(devicePtrOperands.begin(), it);
+      auto idx = std::distance(clauseOps.useDevicePtrVars.begin(), it);
       moveElementToBack(idx, useDeviceTypes);
       moveElementToBack(idx, useDeviceLocs);
       moveElementToBack(idx, useDeviceSymbols);
-      it = devicePtrOperands.erase(it);
+      it = clauseOps.useDevicePtrVars.erase(it);
       continue;
     }
     ++it;
@@ -1038,20 +995,19 @@ genTargetDataOp(Fortran::lower::AbstractConverter &converter,
                 mlir::Location currentLocation,
                 const Fortran::parser::OmpClauseList &clauseList) {
   Fortran::lower::StatementContext stmtCtx;
-  mlir::Value ifClauseOperand, deviceOperand;
-  llvm::SmallVector<mlir::Value> mapOperands, devicePtrOperands,
-      deviceAddrOperands;
+  mlir::omp::TargetDataClauseOps clauseOps;
   llvm::SmallVector<mlir::Type> useDeviceTypes;
   llvm::SmallVector<mlir::Location> useDeviceLocs;
-  llvm::SmallVector<const Fortran::semantics::Symbol *> useDeviceSymbols;
+  llvm::SmallVector<const Fortran::semantics::Symbol *> useDeviceSyms;
 
   ClauseProcessor cp(converter, semaCtx, clauseList);
-  cp.processIf(llvm::omp::Directive::OMPD_target_data, ifClauseOperand);
-  cp.processDevice(stmtCtx, deviceOperand);
-  cp.processUseDevicePtr(devicePtrOperands, useDeviceTypes, useDeviceLocs,
-                         useDeviceSymbols);
-  cp.processUseDeviceAddr(deviceAddrOperands, useDeviceTypes, useDeviceLocs,
-                          useDeviceSymbols);
+  cp.processIf(llvm::omp::Directive::OMPD_target_data, clauseOps);
+  cp.processDevice(stmtCtx, clauseOps);
+  cp.processUseDevicePtr(clauseOps, useDeviceTypes, useDeviceLocs,
+                         useDeviceSyms);
+  cp.processUseDeviceAddr(clauseOps, useDeviceTypes, useDeviceLocs,
+                          useDeviceSyms);
+
   // This function implements the deprecated functionality of use_device_ptr
   // that allows users to provide non-CPTR arguments to it with the caveat
   // that the compiler will treat them as use_device_addr. A lot of legacy
@@ -1063,17 +1019,16 @@ genTargetDataOp(Fortran::lower::AbstractConverter &converter,
   // ordering.
   // TODO: Perhaps create a user provideable compiler option that will
   // re-introduce a hard-error rather than a warning in these cases.
-  promoteNonCPtrUseDevicePtrArgsToUseDeviceAddr(
-      devicePtrOperands, deviceAddrOperands, useDeviceTypes, useDeviceLocs,
-      useDeviceSymbols);
+  promoteNonCPtrUseDevicePtrArgsToUseDeviceAddr(clauseOps, useDeviceTypes,
+                                                useDeviceLocs, useDeviceSyms);
   cp.processMap(currentLocation, llvm::omp::Directive::OMPD_target_data,
-                stmtCtx, mapOperands);
+                stmtCtx, clauseOps);
 
   auto dataOp = converter.getFirOpBuilder().create<mlir::omp::TargetDataOp>(
-      currentLocation, ifClauseOperand, deviceOperand, devicePtrOperands,
-      deviceAddrOperands, mapOperands);
+      currentLocation, clauseOps);
+
   genBodyOfTargetDataOp(converter, semaCtx, eval, genNested, dataOp,
-                        useDeviceTypes, useDeviceLocs, useDeviceSymbols,
+                        useDeviceTypes, useDeviceLocs, useDeviceSyms,
                         currentLocation);
   return dataOp;
 }
@@ -1086,10 +1041,7 @@ static OpTy genTargetEnterExitDataUpdateOp(
     const Fortran::parser::OmpClauseList &clauseList) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   Fortran::lower::StatementContext stmtCtx;
-  mlir::Value ifClauseOperand, deviceOperand;
-  mlir::UnitAttr nowaitAttr;
-  llvm::SmallVector<mlir::Value> mapOperands, dependOperands;
-  llvm::SmallVector<mlir::Attribute> dependTypeOperands;
+  mlir::omp::TargetEnterExitUpdateDataClauseOps clauseOps;
 
   // GCC 9.3.0 emits a (probably) bogus warning about an unused variable.
   [[maybe_unused]] llvm::omp::Directive directive;
@@ -1104,25 +1056,19 @@ static OpTy genTargetEnterExitDataUpdateOp(
   }
 
   ClauseProcessor cp(converter, semaCtx, clauseList);
-  cp.processIf(directive, ifClauseOperand);
-  cp.processDevice(stmtCtx, deviceOperand);
-  cp.processDepend(dependTypeOperands, dependOperands);
-  cp.processNowait(nowaitAttr);
+  cp.processIf(directive, clauseOps);
+  cp.processDevice(stmtCtx, clauseOps);
+  cp.processDepend(clauseOps);
+  cp.processNowait(clauseOps);
 
   if constexpr (std::is_same_v<OpTy, mlir::omp::TargetUpdateOp>) {
-    cp.processMotionClauses<clause::To>(stmtCtx, mapOperands);
-    cp.processMotionClauses<clause::From>(stmtCtx, mapOperands);
+    cp.processMotionClauses<clause::To>(stmtCtx, clauseOps);
+    cp.processMotionClauses<clause::From>(stmtCtx, clauseOps);
   } else {
-    cp.processMap(currentLocation, directive, stmtCtx, mapOperands);
+    cp.processMap(currentLocation, directive, stmtCtx, clauseOps);
   }
 
-  return firOpBuilder.create<OpTy>(
-      currentLocation, ifClauseOperand, deviceOperand,
-      dependTypeOperands.empty()
-          ? nullptr
-          : mlir::ArrayAttr::get(converter.getFirOpBuilder().getContext(),
-                                 dependTypeOperands),
-      dependOperands, nowaitAttr, mapOperands);
+  return firOpBuilder.create<OpTy>(currentLocation, clauseOps);
 }
 
 // This functions creates a block for the body of the targetOp's region. It adds
@@ -1132,9 +1078,9 @@ genBodyOfTargetOp(Fortran::lower::AbstractConverter &converter,
                   Fortran::semantics::SemanticsContext &semaCtx,
                   Fortran::lower::pft::Evaluation &eval, bool genNested,
                   mlir::omp::TargetOp &targetOp,
-                  llvm::ArrayRef<mlir::Type> mapSymTypes,
+                  llvm::ArrayRef<const Fortran::semantics::Symbol *> mapSyms,
                   llvm::ArrayRef<mlir::Location> mapSymLocs,
-                  llvm::ArrayRef<const Fortran::semantics::Symbol *> mapSymbols,
+                  llvm::ArrayRef<mlir::Type> mapSymTypes,
                   const mlir::Location &currentLocation) {
   assert(mapSymTypes.size() == mapSymLocs.size());
 
@@ -1163,7 +1109,7 @@ genBodyOfTargetOp(Fortran::lower::AbstractConverter &converter,
   };
 
   // Bind the symbols to their corresponding block arguments.
-  for (auto [argIndex, argSymbol] : llvm::enumerate(mapSymbols)) {
+  for (auto [argIndex, argSymbol] : llvm::enumerate(mapSyms)) {
     const mlir::BlockArgument &arg = region.getArgument(argIndex);
     // Avoid capture of a reference to a structured binding.
     const Fortran::semantics::Symbol *sym = argSymbol;
@@ -1287,31 +1233,25 @@ genTargetOp(Fortran::lower::AbstractConverter &converter,
             const Fortran::parser::OmpClauseList &clauseList,
             llvm::omp::Directive directive, bool outerCombined = false) {
   Fortran::lower::StatementContext stmtCtx;
-  mlir::Value ifClauseOperand, deviceOperand, threadLimitOperand;
-  mlir::UnitAttr nowaitAttr;
-  llvm::SmallVector<mlir::Attribute> dependTypeOperands;
-  llvm::SmallVector<mlir::Value> mapOperands, dependOperands;
-  llvm::SmallVector<mlir::Type> mapSymTypes;
-  llvm::SmallVector<mlir::Location> mapSymLocs;
-  llvm::SmallVector<const Fortran::semantics::Symbol *> mapSymbols;
-  llvm::SmallVector<mlir::Value> devicePtrOperands, deviceAddrOperands;
-  llvm::SmallVector<mlir::Type> devicePtrTypes, deviceAddrTypes;
-  llvm::SmallVector<mlir::Location> devicePtrLocs, deviceAddrLocs;
-  llvm::SmallVector<const Fortran::semantics::Symbol *> devicePtrSymbols,
-      deviceAddrSymbols;
+  mlir::omp::TargetClauseOps clauseOps;
+  llvm::SmallVector<mlir::Type> mapTypes, devicePtrTypes, deviceAddrTypes;
+  llvm::SmallVector<mlir::Location> mapLocs, devicePtrLocs, deviceAddrLocs;
+  llvm::SmallVector<const Fortran::semantics::Symbol *> mapSyms, devicePtrSyms,
+      deviceAddrSyms;
 
   ClauseProcessor cp(converter, semaCtx, clauseList);
-  cp.processIf(llvm::omp::Directive::OMPD_target, ifClauseOperand);
-  cp.processDevice(stmtCtx, deviceOperand);
-  cp.processThreadLimit(stmtCtx, threadLimitOperand);
-  cp.processDepend(dependTypeOperands, dependOperands);
-  cp.processNowait(nowaitAttr);
-  cp.processMap(currentLocation, directive, stmtCtx, mapOperands, &mapSymTypes,
-                &mapSymLocs, &mapSymbols);
-  cp.processIsDevicePtr(devicePtrOperands, devicePtrTypes, devicePtrLocs,
-                        devicePtrSymbols);
-  cp.processHasDeviceAddr(deviceAddrOperands, deviceAddrTypes, deviceAddrLocs,
-                          deviceAddrSymbols);
+  cp.processIf(llvm::omp::Directive::OMPD_target, clauseOps);
+  cp.processDevice(stmtCtx, clauseOps);
+  cp.processThreadLimit(stmtCtx, clauseOps);
+  cp.processDepend(clauseOps);
+  cp.processNowait(clauseOps);
+  cp.processMap(currentLocation, directive, stmtCtx, clauseOps, &mapSyms,
+                &mapLocs, &mapTypes);
+  cp.processIsDevicePtr(clauseOps, devicePtrTypes, devicePtrLocs,
+                        devicePtrSyms);
+  cp.processHasDeviceAddr(clauseOps, deviceAddrTypes, deviceAddrLocs,
+                          deviceAddrSyms);
+  // TODO Support delayed privatization.
 
   cp.processTODO<clause::Private, clause::Firstprivate, clause::Reduction,
                  clause::InReduction, clause::Allocate, clause::UsesAllocators,
@@ -1323,7 +1263,7 @@ genTargetOp(Fortran::lower::AbstractConverter &converter,
   // symbols used inside the region that have not been explicitly mapped using
   // the map clause.
   auto captureImplicitMap = [&](const Fortran::semantics::Symbol &sym) {
-    if (llvm::find(mapSymbols, &sym) == mapSymbols.end()) {
+    if (llvm::find(mapSyms, &sym) == mapSyms.end()) {
       mlir::Value baseOp = converter.getSymbolAddress(sym);
       if (!baseOp)
         if (const auto *details = sym.template detailsIf<
@@ -1394,26 +1334,20 @@ genTargetOp(Fortran::lower::AbstractConverter &converter,
                 mapFlag),
             captureKind, baseOp.getType());
 
-        mapOperands.push_back(mapOp);
-        mapSymTypes.push_back(baseOp.getType());
-        mapSymLocs.push_back(baseOp.getLoc());
-        mapSymbols.push_back(&sym);
+        clauseOps.mapVars.push_back(mapOp);
+        mapSyms.push_back(&sym);
+        mapLocs.push_back(baseOp.getLoc());
+        mapTypes.push_back(baseOp.getType());
       }
     }
   };
   Fortran::lower::pft::visitAllSymbols(eval, captureImplicitMap);
 
   auto targetOp = converter.getFirOpBuilder().create<mlir::omp::TargetOp>(
-      currentLocation, ifClauseOperand, deviceOperand, threadLimitOperand,
-      dependTypeOperands.empty()
-          ? nullptr
-          : mlir::ArrayAttr::get(converter.getFirOpBuilder().getContext(),
-                                 dependTypeOperands),
-      dependOperands, nowaitAttr, devicePtrOperands, deviceAddrOperands,
-      mapOperands);
+      currentLocation, clauseOps);
 
-  genBodyOfTargetOp(converter, semaCtx, eval, genNested, targetOp, mapSymTypes,
-                    mapSymLocs, mapSymbols, currentLocation);
+  genBodyOfTargetOp(converter, semaCtx, eval, genNested, targetOp, mapSyms,
+                    mapLocs, mapTypes, currentLocation);
 
   return targetOp;
 }
@@ -1426,17 +1360,16 @@ genTeamsOp(Fortran::lower::AbstractConverter &converter,
            const Fortran::parser::OmpClauseList &clauseList,
            bool outerCombined = false) {
   Fortran::lower::StatementContext stmtCtx;
-  mlir::Value numTeamsClauseOperand, ifClauseOperand, threadLimitClauseOperand;
-  llvm::SmallVector<mlir::Value> allocateOperands, allocatorOperands,
-      reductionVars;
-  llvm::SmallVector<mlir::Attribute> reductionDeclSymbols;
+  mlir::omp::TeamsClauseOps clauseOps;
 
   ClauseProcessor cp(converter, semaCtx, clauseList);
-  cp.processIf(llvm::omp::Directive::OMPD_teams, ifClauseOperand);
-  cp.processAllocate(allocatorOperands, allocateOperands);
+  cp.processIf(llvm::omp::Directive::OMPD_teams, clauseOps);
+  cp.processAllocate(clauseOps);
   cp.processDefault();
-  cp.processNumTeams(stmtCtx, numTeamsClauseOperand);
-  cp.processThreadLimit(stmtCtx, threadLimitClauseOperand);
+  cp.processNumTeams(stmtCtx, clauseOps);
+  cp.processThreadLimit(stmtCtx, clauseOps);
+  // TODO Support delayed privatization.
+
   cp.processTODO<clause::Reduction>(currentLocation,
                                     llvm::omp::Directive::OMPD_teams);
 
@@ -1445,30 +1378,20 @@ genTeamsOp(Fortran::lower::AbstractConverter &converter,
           .setGenNested(genNested)
           .setOuterCombined(outerCombined)
           .setClauses(&clauseList),
-      /*num_teams_lower=*/nullptr, numTeamsClauseOperand, ifClauseOperand,
-      threadLimitClauseOperand, allocateOperands, allocatorOperands,
-      reductionVars,
-      reductionDeclSymbols.empty()
-          ? nullptr
-          : mlir::ArrayAttr::get(converter.getFirOpBuilder().getContext(),
-                                 reductionDeclSymbols));
+      clauseOps);
 }
 
 /// Extract the list of function and variable symbols affected by the given
 /// 'declare target' directive and return the intended device type for them.
-static mlir::omp::DeclareTargetDeviceType getDeclareTargetInfo(
+static void getDeclareTargetInfo(
     Fortran::lower::AbstractConverter &converter,
     Fortran::semantics::SemanticsContext &semaCtx,
     Fortran::lower::pft::Evaluation &eval,
     const Fortran::parser::OpenMPDeclareTargetConstruct &declareTargetConstruct,
+    mlir::omp::DeclareTargetClauseOps &clauseOps,
     llvm::SmallVectorImpl<DeclareTargetCapturePair> &symbolAndClause) {
-
-  // The default capture type
-  mlir::omp::DeclareTargetDeviceType deviceType =
-      mlir::omp::DeclareTargetDeviceType::any;
   const auto &spec = std::get<Fortran::parser::OmpDeclareTargetSpecifier>(
       declareTargetConstruct.t);
-
   if (const auto *objectList{
           Fortran::parser::Unwrap<Fortran::parser::OmpObjectList>(spec.u)}) {
     ObjectList objects{makeObjects(*objectList, semaCtx)};
@@ -1489,12 +1412,10 @@ static mlir::omp::DeclareTargetDeviceType getDeclareTargetInfo(
     cp.processTo(symbolAndClause);
     cp.processEnter(symbolAndClause);
     cp.processLink(symbolAndClause);
-    cp.processDeviceType(deviceType);
+    cp.processDeviceType(clauseOps);
     cp.processTODO<clause::Indirect>(converter.getCurrentLocation(),
                                      llvm::omp::Directive::OMPD_declare_target);
   }
-
-  return deviceType;
 }
 
 static void collectDeferredDeclareTargets(
@@ -1504,9 +1425,10 @@ static void collectDeferredDeclareTargets(
     const Fortran::parser::OpenMPDeclareTargetConstruct &declareTargetConstruct,
     llvm::SmallVectorImpl<Fortran::lower::OMPDeferredDeclareTargetInfo>
         &deferredDeclareTarget) {
+  mlir::omp::DeclareTargetClauseOps clauseOps;
   llvm::SmallVector<DeclareTargetCapturePair> symbolAndClause;
-  mlir::omp::DeclareTargetDeviceType devType = getDeclareTargetInfo(
-      converter, semaCtx, eval, declareTargetConstruct, symbolAndClause);
+  getDeclareTargetInfo(converter, semaCtx, eval, declareTargetConstruct,
+                       clauseOps, symbolAndClause);
   // Return the device type only if at least one of the targets for the
   // directive is a function or subroutine
   mlir::ModuleOp mod = converter.getFirOpBuilder().getModule();
@@ -1516,8 +1438,9 @@ static void collectDeferredDeclareTargets(
         std::get<const Fortran::semantics::Symbol &>(symClause)));
 
     if (!op) {
-      deferredDeclareTarget.push_back(
-          {std::get<0>(symClause), devType, std::get<1>(symClause)});
+      deferredDeclareTarget.push_back({std::get<0>(symClause),
+                                       clauseOps.deviceType,
+                                       std::get<1>(symClause)});
     }
   }
 }
@@ -1529,9 +1452,10 @@ getDeclareTargetFunctionDevice(
     Fortran::lower::pft::Evaluation &eval,
     const Fortran::parser::OpenMPDeclareTargetConstruct
         &declareTargetConstruct) {
+  mlir::omp::DeclareTargetClauseOps clauseOps;
   llvm::SmallVector<DeclareTargetCapturePair> symbolAndClause;
-  mlir::omp::DeclareTargetDeviceType deviceType = getDeclareTargetInfo(
-      converter, semaCtx, eval, declareTargetConstruct, symbolAndClause);
+  getDeclareTargetInfo(converter, semaCtx, eval, declareTargetConstruct,
+                       clauseOps, symbolAndClause);
 
   // Return the device type only if at least one of the targets for the
   // directive is a function or subroutine
@@ -1541,7 +1465,7 @@ getDeclareTargetFunctionDevice(
         std::get<const Fortran::semantics::Symbol &>(symClause)));
 
     if (mlir::isa_and_nonnull<mlir::func::FuncOp>(op))
-      return deviceType;
+      return clauseOps.deviceType;
   }
 
   return std::nullopt;
@@ -1571,12 +1495,14 @@ genOmpSimpleStandalone(Fortran::lower::AbstractConverter &converter,
   case llvm::omp::Directive::OMPD_barrier:
     firOpBuilder.create<mlir::omp::BarrierOp>(currentLocation);
     break;
-  case llvm::omp::Directive::OMPD_taskwait:
-    ClauseProcessor(converter, semaCtx, opClauseList)
-        .processTODO<clause::Depend, clause::Nowait>(
-            currentLocation, llvm::omp::Directive::OMPD_taskwait);
-    firOpBuilder.create<mlir::omp::TaskwaitOp>(currentLocation);
+  case llvm::omp::Directive::OMPD_taskwait: {
+    mlir::omp::TaskwaitClauseOps clauseOps;
+    ClauseProcessor cp(converter, semaCtx, opClauseList);
+    cp.processTODO<clause::Depend, clause::Nowait>(
+        currentLocation, llvm::omp::Directive::OMPD_taskwait);
+    firOpBuilder.create<mlir::omp::TaskwaitOp>(currentLocation, clauseOps);
     break;
+  }
   case llvm::omp::Directive::OMPD_taskyield:
     firOpBuilder.create<mlir::omp::TaskyieldOp>(currentLocation);
     break;
@@ -1711,32 +1637,21 @@ createSimdLoop(Fortran::lower::AbstractConverter &converter,
   dsp.processStep1();
 
   Fortran::lower::StatementContext stmtCtx;
-  mlir::Value scheduleChunkClauseOperand, ifClauseOperand;
-  llvm::SmallVector<mlir::Value> lowerBound, upperBound, step, reductionVars;
-  llvm::SmallVector<mlir::Value> alignedVars, nontemporalVars;
+  mlir::omp::SimdLoopClauseOps clauseOps;
   llvm::SmallVector<const Fortran::semantics::Symbol *> iv;
-  llvm::SmallVector<mlir::Type> reductionTypes;
-  llvm::SmallVector<mlir::Attribute> reductionDeclSymbols;
-  mlir::omp::ClauseOrderKindAttr orderClauseOperand;
-  mlir::IntegerAttr simdlenClauseOperand, safelenClauseOperand;
 
   ClauseProcessor cp(converter, semaCtx, loopOpClauseList);
-  cp.processCollapse(loc, eval, lowerBound, upperBound, step, iv);
-  cp.processScheduleChunk(stmtCtx, scheduleChunkClauseOperand);
-  cp.processReduction(loc, reductionVars, reductionTypes, reductionDeclSymbols);
-  cp.processIf(llvm::omp::Directive::OMPD_simd, ifClauseOperand);
-  cp.processSimdlen(simdlenClauseOperand);
-  cp.processSafelen(safelenClauseOperand);
+  cp.processCollapse(loc, eval, clauseOps, iv);
+  cp.processReduction(loc, clauseOps);
+  cp.processIf(llvm::omp::Directive::OMPD_simd, clauseOps);
+  cp.processSimdlen(clauseOps);
+  cp.processSafelen(clauseOps);
+  clauseOps.loopInclusiveAttr = firOpBuilder.getUnitAttr();
+  // TODO Support delayed privatization.
+
   cp.processTODO<clause::Aligned, clause::Allocate, clause::Linear,
                  clause::Nontemporal, clause::Order>(loc, ompDirective);
 
-  mlir::TypeRange resultType;
-  auto simdLoopOp = firOpBuilder.create<mlir::omp::SimdLoopOp>(
-      loc, resultType, lowerBound, upperBound, step, alignedVars,
-      /*alignment_values=*/nullptr, ifClauseOperand, nontemporalVars,
-      orderClauseOperand, simdlenClauseOperand, safelenClauseOperand,
-      /*inclusive=*/firOpBuilder.getUnitAttr());
-
   auto *nestedEval = getCollapsedLoopEval(
       eval, Fortran::lower::getCollapseValue(loopOpClauseList));
 
@@ -1744,11 +1659,12 @@ createSimdLoop(Fortran::lower::AbstractConverter &converter,
     return genLoopVars(op, converter, loc, iv);
   };
 
-  createBodyOfOp<mlir::omp::SimdLoopOp>(
-      simdLoopOp, OpWithBodyGenInfo(converter, semaCtx, loc, *nestedEval)
-                      .setClauses(&loopOpClauseList)
-                      .setDataSharingProcessor(&dsp)
-                      .setGenRegionEntryCb(ivCallback));
+  genOpWithBody<mlir::omp::SimdLoopOp>(
+      OpWithBodyGenInfo(converter, semaCtx, loc, *nestedEval)
+          .setClauses(&loopOpClauseList)
+          .setDataSharingProcessor(&dsp)
+          .setGenRegionEntryCb(ivCallback),
+      clauseOps);
 }
 
 static void createWsloop(Fortran::lower::AbstractConverter &converter,
@@ -1763,77 +1679,50 @@ static void createWsloop(Fortran::lower::AbstractConverter &converter,
   dsp.processStep1();
 
   Fortran::lower::StatementContext stmtCtx;
-  mlir::Value scheduleChunkClauseOperand;
-  llvm::SmallVector<mlir::Value> lowerBound, upperBound, step, reductionVars;
-  llvm::SmallVector<mlir::Value> linearVars, linearStepVars;
+  mlir::omp::WsloopClauseOps clauseOps;
   llvm::SmallVector<const Fortran::semantics::Symbol *> iv;
   llvm::SmallVector<mlir::Type> reductionTypes;
-  llvm::SmallVector<mlir::Attribute> reductionDeclSymbols;
-  llvm::SmallVector<const Fortran::semantics::Symbol *> reductionSymbols;
-  mlir::omp::ClauseOrderKindAttr orderClauseOperand;
-  mlir::omp::ClauseScheduleKindAttr scheduleValClauseOperand;
-  mlir::UnitAttr nowaitClauseOperand, byrefOperand, scheduleSimdClauseOperand;
-  mlir::IntegerAttr orderedClauseOperand;
-  mlir::omp::ScheduleModifierAttr scheduleModClauseOperand;
+  llvm::SmallVector<const Fortran::semantics::Symbol *> reductionSyms;
 
   ClauseProcessor cp(converter, semaCtx, beginClauseList);
-  cp.processCollapse(loc, eval, lowerBound, upperBound, step, iv);
-  cp.processScheduleChunk(stmtCtx, scheduleChunkClauseOperand);
-  cp.processReduction(loc, reductionVars, reductionTypes, reductionDeclSymbols,
-                      &reductionSymbols);
-  cp.processTODO<clause::Linear, clause::Order>(loc, ompDirective);
-
-  if (ReductionProcessor::doReductionByRef(reductionVars))
-    byrefOperand = firOpBuilder.getUnitAttr();
-
-  auto wsLoopOp = firOpBuilder.create<mlir::omp::WsloopOp>(
-      loc, lowerBound, upperBound, step, linearVars, linearStepVars,
-      reductionVars,
-      reductionDeclSymbols.empty()
-          ? nullptr
-          : mlir::ArrayAttr::get(firOpBuilder.getContext(),
-                                 reductionDeclSymbols),
-      scheduleValClauseOperand, scheduleChunkClauseOperand,
-      /*schedule_modifiers=*/nullptr,
-      /*simd_modifier=*/nullptr, nowaitClauseOperand, byrefOperand,
-      orderedClauseOperand, orderClauseOperand,
-      /*inclusive=*/firOpBuilder.getUnitAttr());
-
-  // Handle attribute based clauses.
-  if (cp.processOrdered(orderedClauseOperand))
-    wsLoopOp.setOrderedValAttr(orderedClauseOperand);
-
-  if (cp.processSchedule(scheduleValClauseOperand, scheduleModClauseOperand,
-                         scheduleSimdClauseOperand)) {
-    wsLoopOp.setScheduleValAttr(scheduleValClauseOperand);
-    wsLoopOp.setScheduleModifierAttr(scheduleModClauseOperand);
-    wsLoopOp.setSimdModifierAttr(scheduleSimdClauseOperand);
-  }
+  cp.processCollapse(loc, eval, clauseOps, iv);
+  cp.processSchedule(stmtCtx, clauseOps);
+  cp.processReduction(loc, clauseOps, &reductionTypes, &reductionSyms);
+  cp.processOrdered(clauseOps);
+  clauseOps.loopInclusiveAttr = firOpBuilder.getUnitAttr();
+  // TODO Support delayed privatization.
+
+  if (ReductionProcessor::doReductionByRef(clauseOps.reductionVars))
+    clauseOps.reductionByRefAttr = firOpBuilder.getUnitAttr();
+
+  cp.processTODO<clause::Allocate, clause::Linear, clause::Order>(loc,
+                                                                  ompDirective);
+
   // In FORTRAN `nowait` clause occur at the end of `omp do` directive.
   // i.e
   // !$omp do
   // <...>
   // !$omp end do nowait
   if (endClauseList) {
-    if (ClauseProcessor(converter, semaCtx, *endClauseList)
-            .processNowait(nowaitClauseOperand))
-      wsLoopOp.setNowaitAttr(nowaitClauseOperand);
+    ClauseProcessor ecp(converter, semaCtx, *endClauseList);
+    ecp.processNowait(clauseOps);
   }
 
   auto *nestedEval = getCollapsedLoopEval(
       eval, Fortran::lower::getCollapseValue(beginClauseList));
 
   auto ivCallback = [&](mlir::Operation *op) {
-    return genLoopAndReductionVars(op, converter, loc, iv, reductionSymbols,
+    return genLoopAndReductionVars(op, converter, loc, iv, reductionSyms,
                                    reductionTypes);
   };
 
-  createBodyOfOp<mlir::omp::WsloopOp>(
-      wsLoopOp, OpWithBodyGenInfo(converter, semaCtx, loc, *nestedEval)
-                    .setClauses(&beginClauseList)
-                    .setDataSharingProcessor(&dsp)
-                    .setReductions(&reductionSymbols, &reductionTypes)
-                    .setGenRegionEntryCb(ivCallback));
+  genOpWithBody<mlir::omp::WsloopOp>(
+      OpWithBodyGenInfo(converter, semaCtx, loc, *nestedEval)
+          .setClauses(&beginClauseList)
+          .setDataSharingProcessor(&dsp)
+          .setReductions(&reductionSyms, &reductionTypes)
+          .setGenRegionEntryCb(ivCallback),
+      clauseOps);
 }
 
 static void createSimdWsloop(
@@ -1921,10 +1810,11 @@ static void genOMP(Fortran::lower::AbstractConverter &converter,
                    Fortran::lower::pft::Evaluation &eval,
                    const Fortran::parser::OpenMPDeclareTargetConstruct
                        &declareTargetConstruct) {
+  mlir::omp::DeclareTargetClauseOps clauseOps;
   llvm::SmallVector<DeclareTargetCapturePair> symbolAndClause;
   mlir::ModuleOp mod = converter.getFirOpBuilder().getModule();
-  mlir::omp::DeclareTargetDeviceType deviceType = getDeclareTargetInfo(
-      converter, semaCtx, eval, declareTargetConstruct, symbolAndClause);
+  getDeclareTargetInfo(converter, semaCtx, eval, declareTargetConstruct,
+                       clauseOps, symbolAndClause);
 
   for (const DeclareTargetCapturePair &symClause : symbolAndClause) {
     mlir::Operation *op = mod.lookupSymbol(converter.mangleName(
@@ -1938,7 +1828,8 @@ static void genOMP(Fortran::lower::AbstractConverter &converter,
 
     markDeclareTarget(
         op, converter,
-        std::get<mlir::omp::DeclareTargetCaptureClause>(symClause), deviceType);
+        std::get<mlir::omp::DeclareTargetCaptureClause>(symClause),
+        clauseOps.deviceType);
   }
 }
 
@@ -2072,7 +1963,8 @@ genOMP(Fortran::lower::AbstractConverter &converter,
         !std::get_if<Fortran::parser::OmpClause::IsDevicePtr>(&clause.u) &&
         !std::get_if<Fortran::parser::OmpClause::HasDeviceAddr>(&clause.u) &&
         !std::get_if<Fortran::parser::OmpClause::ThreadLimit>(&clause.u) &&
-        !std::get_if<Fortran::parser::OmpClause::NumTeams>(&clause.u)) {
+        !std::get_if<Fortran::parser::OmpClause::NumTeams>(&clause.u) &&
+        !std::get_if<Fortran::parser::OmpClause::Simd>(&clause.u)) {
       TODO(clauseLocation, "OpenMP Block construct clause");
     }
   }
@@ -2092,7 +1984,7 @@ genOMP(Fortran::lower::AbstractConverter &converter,
     break;
   case llvm::omp::Directive::OMPD_ordered:
     genOrderedRegionOp(converter, semaCtx, eval, /*genNested=*/true,
-                       currentLocation);
+                       currentLocation, beginClauseList);
     break;
   case llvm::omp::Directive::OMPD_parallel:
     genParallelOp(converter, symTable, semaCtx, eval, /*genNested=*/true,
@@ -2183,7 +2075,6 @@ genOMP(Fortran::lower::AbstractConverter &converter,
        const Fortran::parser::OpenMPCriticalConstruct &criticalConstruct) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   mlir::Location currentLocation = converter.getCurrentLocation();
-  mlir::IntegerAttr hintClauseOp;
   std::string name;
   const Fortran::parser::OmpCriticalDirective &cd =
       std::get<Fortran::parser::OmpCriticalDirective>(criticalConstruct.t);
@@ -2192,21 +2083,28 @@ genOMP(Fortran::lower::AbstractConverter &converter,
         std::get<std::optional<Fortran::parser::Name>>(cd.t).value().ToString();
   }
 
-  const auto &clauseList = std::get<Fortran::parser::OmpClauseList>(cd.t);
-  ClauseProcessor(converter, semaCtx, clauseList).processHint(hintClauseOp);
-
   mlir::omp::CriticalOp criticalOp = [&]() {
     if (name.empty()) {
       return firOpBuilder.create<mlir::omp::CriticalOp>(
           currentLocation, mlir::FlatSymbolRefAttr());
     }
+
     mlir::ModuleOp module = firOpBuilder.getModule();
     mlir::OpBuilder modBuilder(module.getBodyRegion());
     auto global = module.lookupSymbol<mlir::omp::CriticalDeclareOp>(name);
-    if (!global)
-      global = modBuilder.create<mlir::omp::CriticalDeclareOp>(
-          currentLocation,
-          mlir::StringAttr::get(firOpBuilder.getContext(), name), hintClauseOp);
+    if (!global) {
+      mlir::omp::CriticalClauseOps clauseOps;
+      const auto &clauseList = std::get<Fortran::parser::OmpClauseList>(cd.t);
+
+      ClauseProcessor cp(converter, semaCtx, clauseList);
+      cp.processHint(clauseOps);
+      clauseOps.nameAttr =
+          mlir::StringAttr::get(firOpBuilder.getContext(), name);
+
+      global = modBuilder.create<mlir::omp::CriticalDeclareOp>(currentLocation,
+                                                               clauseOps);
+    }
+
     return firOpBuilder.create<mlir::omp::CriticalOp>(
         currentLocation, mlir::FlatSymbolRefAttr::get(firOpBuilder.getContext(),
                                                       global.getSymName()));
@@ -2323,8 +2221,7 @@ genOMP(Fortran::lower::AbstractConverter &converter,
        Fortran::lower::pft::Evaluation &eval,
        const Fortran::parser::OpenMPSectionsConstruct &sectionsConstruct) {
   mlir::Location currentLocation = converter.getCurrentLocation();
-  llvm::SmallVector<mlir::Value> allocateOperands, allocatorOperands;
-  mlir::UnitAttr nowaitClauseOperand;
+  mlir::omp::SectionsClauseOps clauseOps;
   const auto &beginSectionsDirective =
       std::get<Fortran::parser::OmpBeginSectionsDirective>(sectionsConstruct.t);
   const auto &sectionsClauseList =
@@ -2333,8 +2230,9 @@ genOMP(Fortran::lower::AbstractConverter &converter,
   // Process clauses before optional omp.parallel, so that new variables are
   // allocated outside of the parallel region
   ClauseProcessor cp(converter, semaCtx, sectionsClauseList);
-  cp.processSectionsReduction(currentLocation);
-  cp.processAllocate(allocatorOperands, allocateOperands);
+  cp.processSectionsReduction(currentLocation, clauseOps);
+  cp.processAllocate(clauseOps);
+  // TODO Support delayed privatization.
 
   llvm::omp::Directive dir =
       std::get<Fortran::parser::OmpSectionsDirective>(beginSectionsDirective.t)
@@ -2351,16 +2249,14 @@ genOMP(Fortran::lower::AbstractConverter &converter,
     const auto &endSectionsClauseList =
         std::get<Fortran::parser::OmpClauseList>(endSectionsDirective.t);
     ClauseProcessor(converter, semaCtx, endSectionsClauseList)
-        .processNowait(nowaitClauseOperand);
+        .processNowait(clauseOps);
   }
 
   // SECTIONS construct
   genOpWithBody<mlir::omp::SectionsOp>(
       OpWithBodyGenInfo(converter, semaCtx, currentLocation, eval)
           .setGenNested(false),
-      /*reduction_vars=*/mlir::ValueRange(),
-      /*reductions=*/nullptr, allocateOperands, allocatorOperands,
-      nowaitClauseOperand);
+      clauseOps);
 
   const auto &sectionBlocks =
       std::get<Fortran::parser::OmpSectionBlocks>(sectionsConstruct.t);
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPClauseOperands.h b/mlir/include/mlir/Dialect/OpenMP/OpenMPClauseOperands.h
index 4ce7e47da046..304a9740d91e 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPClauseOperands.h
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPClauseOperands.h
@@ -81,7 +81,7 @@ struct GrainsizeClauseOps {
   Value grainsizeVar;
 };
 
-struct HasDeviceAddrOps {
+struct HasDeviceAddrClauseOps {
   llvm::SmallVector<Value> hasDeviceAddrVars;
 };
 struct HintClauseOps {
@@ -97,7 +97,7 @@ struct InReductionClauseOps {
   llvm::SmallVector<Attribute> inReductionDeclSymbols;
 };
 
-struct IsDevicePtrOps {
+struct IsDevicePtrClauseOps {
   llvm::SmallVector<Value> isDevicePtrVars;
 };
 
@@ -234,6 +234,8 @@ using DistributeClauseOps =
     detail::Clauses<AllocateClauseOps, DistScheduleClauseOps, OrderClauseOps,
                     PrivateClauseOps>;
 
+using LoopNestClauseOps = detail::Clauses<CollapseClauseOps, LoopRelatedOps>;
+
 // TODO `filter` clause.
 using MaskedClauseOps = detail::Clauses<>;
 
@@ -261,8 +263,8 @@ using SingleClauseOps = detail::Clauses<AllocateClauseOps, CopyprivateClauseOps,
 // TODO `defaultmap`, `uses_allocators` clauses.
 using TargetClauseOps =
     detail::Clauses<AllocateClauseOps, DependClauseOps, DeviceClauseOps,
-                    HasDeviceAddrOps, IfClauseOps, InReductionClauseOps,
-                    IsDevicePtrOps, MapClauseOps, NowaitClauseOps,
+                    HasDeviceAddrClauseOps, IfClauseOps, InReductionClauseOps,
+                    IsDevicePtrClauseOps, MapClauseOps, NowaitClauseOps,
                     PrivateClauseOps, ReductionClauseOps, ThreadLimitClauseOps>;
 
 using TargetDataClauseOps = detail::Clauses<DeviceClauseOps, IfClauseOps,
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index a2eb39a8e607..596b2f5e4444 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -574,6 +574,10 @@ def LoopNestOp : OpenMP_Op<"loop_nest", [SameVariadicOperandSize,
                        Variadic<IntLikeType>:$step,
                        UnitAttr:$inclusive);
 
+  let builders = [
+    OpBuilder<(ins CArg<"const LoopNestClauseOps &">:$clauses)>
+  ];
+
   let regions = (region AnyRegion:$region);
 
   let extraClassDeclaration = [{
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index 4ab78bf331f8..35fb174046a3 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -1920,6 +1920,12 @@ void LoopNestOp::print(OpAsmPrinter &p) {
   p.printRegion(region, /*printEntryBlockArgs=*/false);
 }
 
+void LoopNestOp::build(OpBuilder &builder, OperationState &state,
+                       const LoopNestClauseOps &clauses) {
+  LoopNestOp::build(builder, state, clauses.loopLBVar, clauses.loopUBVar,
+                    clauses.loopStepVar, clauses.loopInclusiveAttr);
+}
+
 LogicalResult LoopNestOp::verify() {
   if (getLowerBound().size() != getIVs().size())
     return emitOpError() << "number of range arguments and IVs do not match";
-- 
GitLab


From 33779b861d748ed18b8d4f9cbad3a84deaa95e9f Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron@aaronballman.com>
Date: Fri, 12 Apr 2024 07:41:43 -0400
Subject: [PATCH 052/464] Reword a diagnostic; NFC

This diagnostic used to talk about complex integer types but is issued
for use of the increment or decrement operators on any complex type,
not just integral ones.

The diagnostic also had zero test coverage, so new coverage is added
along with the rewording.
---
 .../clang/Basic/DiagnosticSemaKinds.td        |  4 ++--
 clang/lib/Sema/SemaExpr.cpp                   |  4 ++--
 clang/test/Sema/complex-inc-dec.c             | 24 +++++++++++++++++++
 3 files changed, 28 insertions(+), 4 deletions(-)
 create mode 100644 clang/test/Sema/complex-inc-dec.c

diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 180e913155d6..774d2b53a382 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -7588,8 +7588,8 @@ def ext_gnu_ptr_func_arith : Extension<
   InGroup<GNUPointerArith>;
 def err_readonly_message_assignment : Error<
   "assigning to 'readonly' return result of an Objective-C message not allowed">;
-def ext_integer_increment_complex : Extension<
-  "ISO C does not support '++'/'--' on complex integer type %0">;
+def ext_increment_complex : Extension<
+  "'%select{--|++}0' on an object of complex type is a Clang extension">;
 def ext_integer_complement_complex : Extension<
   "ISO C does not support '~' for complex conjugation of %0">;
 def err_nosetter_property_assignment : Error<
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 18284e0c3e98..b294d2bd9f53 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -14859,8 +14859,8 @@ static QualType CheckIncrementDecrementOperand(Sema &S, Expr *Op,
       return QualType();
   } else if (ResType->isAnyComplexType()) {
     // C99 does not support ++/-- on complex types, we allow as an extension.
-    S.Diag(OpLoc, diag::ext_integer_increment_complex)
-      << ResType << Op->getSourceRange();
+    S.Diag(OpLoc, diag::ext_increment_complex)
+      << IsInc << Op->getSourceRange();
   } else if (ResType->isPlaceholderType()) {
     ExprResult PR = S.CheckPlaceholderExpr(Op);
     if (PR.isInvalid()) return QualType();
diff --git a/clang/test/Sema/complex-inc-dec.c b/clang/test/Sema/complex-inc-dec.c
new file mode 100644
index 000000000000..ebc23c36f40c
--- /dev/null
+++ b/clang/test/Sema/complex-inc-dec.c
@@ -0,0 +1,24 @@
+// RUN: %clang_cc1 -verify -pedantic -std=c99 %s
+
+void func(void) {
+  _Complex float cf;
+  _Complex double cd;
+  _Complex long double cld;
+
+  ++cf;  // expected-warning {{'++' on an object of complex type is a Clang extension}}
+  ++cd;  // expected-warning {{'++' on an object of complex type is a Clang extension}}
+  ++cld; // expected-warning {{'++' on an object of complex type is a Clang extension}}
+
+  --cf;  // expected-warning {{'--' on an object of complex type is a Clang extension}}
+  --cd;  // expected-warning {{'--' on an object of complex type is a Clang extension}}
+  --cld; // expected-warning {{'--' on an object of complex type is a Clang extension}}
+
+  cf++;  // expected-warning {{'++' on an object of complex type is a Clang extension}}
+  cd++;  // expected-warning {{'++' on an object of complex type is a Clang extension}}
+  cld++; // expected-warning {{'++' on an object of complex type is a Clang extension}}
+
+  cf--;  // expected-warning {{'--' on an object of complex type is a Clang extension}}
+  cd--;  // expected-warning {{'--' on an object of complex type is a Clang extension}}
+  cld--; // expected-warning {{'--' on an object of complex type is a Clang extension}}
+}
+
-- 
GitLab


From 5ebeaf211551f9410e17b1134126daadd8b163ac Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Fri, 12 Apr 2024 16:02:57 +0400
Subject: [PATCH 053/464] [clang] Add tests for some CWG 5xx issues (#87909)

This patch covers
[CWG393](https://cplusplus.github.io/CWG/issues/393.html) "Pointer to
array of unknown bound in template argument list in parameter",
[CWG528](https://cplusplus.github.io/CWG/issues/528.html) "Why are
incomplete class types not allowed with `typeid`?",
[CWG550](https://cplusplus.github.io/CWG/issues/550.html) "Pointer to
array of unknown bound in parameter declarations",
[CWG553](https://cplusplus.github.io/CWG/issues/553.html) "Problems with
friend allocation and deallocation functions",
[CWG555](https://cplusplus.github.io/CWG/issues/555.html)
"Pseudo-destructor name lookup",
[CWG560](https://cplusplus.github.io/CWG/issues/560.html) "Use of the
`typename` keyword in return types".

CWG393 is on this list, because CWG550 is marked as a duplicate of
CWG393.

Test for CWG553 has been already written, but it was missing a status
comment. As a drive-by fix, I'm adding missing status comments to
CWG1584 and CWG1903 as well.

CWG555 used CWG466 test, and also a variation of that test to test
references. CWG466 is now testing non-reference non-pointer case.

CWG560 showcases again that converting warnings to errors in DR tests
via `-pedantic-errors` doesn't make things more clear. By default that
test is accepted with an extension warning since we implemented
[P0634R3](https://wg21.link/p0634r3) "Down with `typename`!" in Clang
16.
---
 clang/test/CXX/drs/dr15xx.cpp |  2 +-
 clang/test/CXX/drs/dr19xx.cpp |  2 +-
 clang/test/CXX/drs/dr3xx.cpp  | 19 ++++++++
 clang/test/CXX/drs/dr4xx.cpp  | 60 +++++++++++------------
 clang/test/CXX/drs/dr5xx.cpp  | 89 +++++++++++++++++++++++++++++++++++
 clang/www/cxx_dr_status.html  | 16 +++----
 6 files changed, 149 insertions(+), 39 deletions(-)

diff --git a/clang/test/CXX/drs/dr15xx.cpp b/clang/test/CXX/drs/dr15xx.cpp
index 195c0fa610d5..6e3ad41c748f 100644
--- a/clang/test/CXX/drs/dr15xx.cpp
+++ b/clang/test/CXX/drs/dr15xx.cpp
@@ -555,7 +555,7 @@ auto DR1579_lambda_invalid = []() -> GenericMoveOnly<char> {
 #endif
 } // end namespace dr1579
 
-namespace dr1584 {
+namespace dr1584 { // dr1584: 7 drafting 2015-05
 #if __cplusplus >= 201103L
   // Deducing function types from cv-qualified types
   template<typename T> void f(const T *); // #dr1584-f
diff --git a/clang/test/CXX/drs/dr19xx.cpp b/clang/test/CXX/drs/dr19xx.cpp
index 716b1476831e..f8c1581f0854 100644
--- a/clang/test/CXX/drs/dr19xx.cpp
+++ b/clang/test/CXX/drs/dr19xx.cpp
@@ -34,7 +34,7 @@ namespace dr1902 { // dr1902: 3.7
 #endif
 }
 
-namespace dr1903 {
+namespace dr1903 { // dr1903: 2.7
   namespace A {
     struct a {};
     int a;
diff --git a/clang/test/CXX/drs/dr3xx.cpp b/clang/test/CXX/drs/dr3xx.cpp
index 483ebf7a08aa..d5cfebccf813 100644
--- a/clang/test/CXX/drs/dr3xx.cpp
+++ b/clang/test/CXX/drs/dr3xx.cpp
@@ -1568,6 +1568,25 @@ namespace dr391 { // dr391: 2.8 c++11
 }
 
 // dr392 is in dr392.cpp
+
+namespace dr393 { // dr393: 2.7
+template <typename T>
+struct S {};
+
+void f1(S<int (*)[]>);
+void f2(S<int (&)[]>);
+void g(int(*S<int>::*)[]);
+
+template<typename T>
+void sp_assert_convertible( T* ) {}
+
+template<typename T, typename U>
+void h() {
+  T (*p) [] = (U(*)[])0;
+  sp_assert_convertible<T[]>( (U(*)[])0 );
+}
+} // namespace dr393
+
 // dr394: na
 
 namespace dr395 { // dr395: 3.0
diff --git a/clang/test/CXX/drs/dr4xx.cpp b/clang/test/CXX/drs/dr4xx.cpp
index 34dd638c1d9b..9b7a9777202f 100644
--- a/clang/test/CXX/drs/dr4xx.cpp
+++ b/clang/test/CXX/drs/dr4xx.cpp
@@ -2,8 +2,9 @@
 // RUN: env ASAN_OPTIONS=detect_stack_use_after_return=0 %clang_cc1 -std=c++11 %s -verify=expected,cxx98-14,cxx98-17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: env ASAN_OPTIONS=detect_stack_use_after_return=0 %clang_cc1 -std=c++14 %s -verify=expected,cxx98-14,cxx98-17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: env ASAN_OPTIONS=detect_stack_use_after_return=0 %clang_cc1 -std=c++17 %s -verify=expected,since-cxx17,cxx98-17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors
-// RUN: env ASAN_OPTIONS=detect_stack_use_after_return=0 %clang_cc1 -std=c++20 %s -verify=expected,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors
-// RUN: env ASAN_OPTIONS=detect_stack_use_after_return=0 %clang_cc1 -std=c++23 %s -verify=expected,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors
+// RUN: env ASAN_OPTIONS=detect_stack_use_after_return=0 %clang_cc1 -std=c++20 %s -verify=expected,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors
+// RUN: env ASAN_OPTIONS=detect_stack_use_after_return=0 %clang_cc1 -std=c++23 %s -verify=expected,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors
+// RUN: env ASAN_OPTIONS=detect_stack_use_after_return=0 %clang_cc1 -std=c++2c %s -verify=expected,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors
 
 // FIXME: __SIZE_TYPE__ expands to 'long long' on some targets.
 __extension__ typedef __SIZE_TYPE__ size_t;
@@ -949,33 +950,34 @@ namespace dr460 { // dr460: yes
 // dr465: na
 
 namespace dr466 { // dr466: 2.8
-  typedef int I;
-  typedef const int CI;
-  typedef volatile int VI;
-  void f(int *a, CI *b, VI *c) {
-    a->~I();
-    a->~CI();
-    a->~VI();
-    a->I::~I();
-    a->CI::~CI();
-    a->VI::~VI();
-
-    a->CI::~VI(); // allowed by changes to [expr.id.prim.qual]/2 introduced in P1131R2
-
-    b->~I();
-    b->~CI();
-    b->~VI();
-    b->I::~I();
-    b->CI::~CI();
-    b->VI::~VI();
-
-    c->~I();
-    c->~CI();
-    c->~VI();
-    c->I::~I();
-    c->CI::~CI();
-    c->VI::~VI();
-  }
+typedef int I;
+typedef const int CI;
+typedef volatile int VI;
+void g(int a, CI b, VI c) {
+// since-cxx20-warning@-1 {{volatile-qualified parameter type 'VI' (aka 'volatile int') is deprecated}}
+  a.~I();
+  a.~CI();
+  a.~VI();
+  a.I::~I();
+  a.CI::~CI();
+  a.VI::~VI();
+
+  a.CI::~VI(); // allowed by changes to [expr.id.prim.qual]/2 introduced in P1131R2
+
+  b.~I();
+  b.~CI();
+  b.~VI();
+  b.I::~I();
+  b.CI::~CI();
+  b.VI::~VI();
+
+  c.~I();
+  c.~CI();
+  c.~VI();
+  c.I::~I();
+  c.CI::~CI();
+  c.VI::~VI();
+}
 }
 
 namespace dr467 { // dr467: yes
diff --git a/clang/test/CXX/drs/dr5xx.cpp b/clang/test/CXX/drs/dr5xx.cpp
index 0ea306a04116..d473f4827455 100644
--- a/clang/test/CXX/drs/dr5xx.cpp
+++ b/clang/test/CXX/drs/dr5xx.cpp
@@ -18,6 +18,10 @@ namespace std {
 void *operator new(size_t, std::align_val_t); // #dr5xx-global-operator-new-aligned
 #endif
 
+namespace std {
+  struct type_info;
+}
+
 namespace dr500 { // dr500: dup 372
   class D;
   class A {
@@ -265,6 +269,18 @@ namespace dr527 { // dr527: na
   int ax = a.x, bx = b.x, cx = c.x, dx = d.x, ex = E::e->x, fx = F::f->x;
 }
 
+namespace dr528 { // dr528: 2.7
+
+struct S; // #dr528-S
+
+void f() {
+  typeid(S);
+  // expected-error@-1 {{'typeid' of incomplete type 'S'}}
+  //   expected-note@#dr528-S {{forward declaration of 'dr528::S'}}
+}
+
+} // namespace dr528
+
 namespace dr530 { // dr530: yes
   template<int*> struct S { enum { N = 1 }; };
   template<void(*)()> struct T { enum { N = 1 }; };
@@ -618,6 +634,8 @@ namespace dr548 { // dr548: dup 482
   template void dr548::f<int>();
 }
 
+// dr550: dup 393
+
 namespace dr551 { // dr551: yes c++11
   // FIXME: This obviously should apply in C++98 mode too.
   template<typename T> void f() {}
@@ -641,6 +659,7 @@ namespace dr552 { // dr552: yes
   X<Y, 0> x;
 }
 
+// dr553: 2.7
 struct dr553_class {
   friend void *operator new(size_t, dr553_class);
 };
@@ -661,6 +680,62 @@ namespace dr553 {
 }
 
 // dr554: na
+
+namespace dr555 { // dr555: 2.8
+typedef int I;
+typedef const int CI;
+typedef volatile int VI;
+void f(int *a, CI *b, VI *c) {
+  a->~I();
+  a->~CI();
+  a->~VI();
+  a->I::~I();
+  a->CI::~CI();
+  a->VI::~VI();
+
+  a->CI::~VI(); // allowed by changes to [expr.id.prim.qual]/2 introduced in P1131R2
+
+  b->~I();
+  b->~CI();
+  b->~VI();
+  b->I::~I();
+  b->CI::~CI();
+  b->VI::~VI();
+
+  c->~I();
+  c->~CI();
+  c->~VI();
+  c->I::~I();
+  c->CI::~CI();
+  c->VI::~VI();
+}
+
+void g(int &a, CI &b, VI &c) {
+  a.~I();
+  a.~CI();
+  a.~VI();
+  a.I::~I();
+  a.CI::~CI();
+  a.VI::~VI();
+
+  a.CI::~VI(); // allowed by changes to [expr.id.prim.qual]/2 introduced in P1131R2
+
+  b.~I();
+  b.~CI();
+  b.~VI();
+  b.I::~I();
+  b.CI::~CI();
+  b.VI::~VI();
+
+  c.~I();
+  c.~CI();
+  c.~VI();
+  c.I::~I();
+  c.CI::~CI();
+  c.VI::~VI();
+}
+} // namespace dr555
+
 // dr556: na
 
 namespace dr557 { // dr557: 3.1
@@ -689,6 +764,20 @@ namespace dr558 { // dr558: 2.9
 
 template<typename> struct dr559 { typedef int T; dr559::T u; }; // dr559: yes
 
+namespace dr560 { // dr560: 16
+
+template <class T>
+struct Outer {
+  struct Inner {
+    Inner* self();
+  };
+};
+template <class T>
+Outer<T>::Inner* Outer<T>::Inner::self() { return this; }
+// cxx98-17-error@-1 {{missing 'typename' prior to dependent type name Outer<T>::Inner; implicit 'typename' is a C++20 extension}}
+
+} // namespace dr560
+
 namespace dr561 { // dr561: yes
   template<typename T> void f(int);
   template<typename T> void g(T t) {
diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html
index 6daa5ad2d131..83b71e7c122d 100755
--- a/clang/www/cxx_dr_status.html
+++ b/clang/www/cxx_dr_status.html
@@ -2398,7 +2398,7 @@ of class templates</td>
     <td><a href="https://cplusplus.github.io/CWG/issues/393.html">393</a></td>
     <td>CD4</td>
     <td>Pointer to array of unknown bound in template argument list in parameter</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="full" align="center">Clang 2.7</td>
   </tr>
   <tr id="394">
     <td><a href="https://cplusplus.github.io/CWG/issues/394.html">394</a></td>
@@ -3208,7 +3208,7 @@ of class templates</td>
     <td><a href="https://cplusplus.github.io/CWG/issues/528.html">528</a></td>
     <td>NAD</td>
     <td>Why are incomplete class types not allowed with <TT>typeid</TT>?</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="full" align="center">Clang 2.7</td>
   </tr>
   <tr class="open" id="529">
     <td><a href="https://cplusplus.github.io/CWG/issues/529.html">529</a></td>
@@ -3342,7 +3342,7 @@ and <I>POD class</I></td>
     <td><a href="https://cplusplus.github.io/CWG/issues/550.html">550</a></td>
     <td>dup</td>
     <td>Pointer to array of unknown bound in parameter declarations</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="full" align="center">Duplicate of <a href="#393">393</a></td>
   </tr>
   <tr id="551">
     <td><a href="https://cplusplus.github.io/CWG/issues/551.html">551</a></td>
@@ -3360,7 +3360,7 @@ and <I>POD class</I></td>
     <td><a href="https://cplusplus.github.io/CWG/issues/553.html">553</a></td>
     <td>NAD</td>
     <td>Problems with friend allocation and deallocation functions</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="full" align="center">Clang 2.7</td>
   </tr>
   <tr id="554">
     <td><a href="https://cplusplus.github.io/CWG/issues/554.html">554</a></td>
@@ -3372,7 +3372,7 @@ and <I>POD class</I></td>
     <td><a href="https://cplusplus.github.io/CWG/issues/555.html">555</a></td>
     <td>CD5</td>
     <td>Pseudo-destructor name lookup</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="full" align="center">Clang 2.8</td>
   </tr>
   <tr id="556">
     <td><a href="https://cplusplus.github.io/CWG/issues/556.html">556</a></td>
@@ -3402,7 +3402,7 @@ and <I>POD class</I></td>
     <td><a href="https://cplusplus.github.io/CWG/issues/560.html">560</a></td>
     <td>NAD</td>
     <td>Use of the <TT>typename</TT> keyword in return types</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="full" align="center">Clang 16</td>
   </tr>
   <tr id="561">
     <td><a href="https://cplusplus.github.io/CWG/issues/561.html">561</a></td>
@@ -9312,7 +9312,7 @@ and <I>POD class</I></td>
     <td><a href="https://cplusplus.github.io/CWG/issues/1584.html">1584</a></td>
     <td>drafting</td>
     <td>Deducing function types from cv-qualified types</td>
-    <td align="center">Not resolved</td>
+    <td title="Clang 7 implements 2015-05 resolution" align="center">Not Resolved*</td>
   </tr>
   <tr id="1585">
     <td><a href="https://cplusplus.github.io/CWG/issues/1585.html">1585</a></td>
@@ -11226,7 +11226,7 @@ and <I>POD class</I></td>
     <td><a href="https://cplusplus.github.io/CWG/issues/1903.html">1903</a></td>
     <td>CD4</td>
     <td>What declarations are introduced by a non-member <I>using-declaration</I>?</td>
-    <td class="unknown" align="center">Unknown</td>
+    <td class="full" align="center">Clang 2.7</td>
   </tr>
   <tr id="1904">
     <td><a href="https://cplusplus.github.io/CWG/issues/1904.html">1904</a></td>
-- 
GitLab


From ab037c4ff3452a680efb758ccacb6ee210ce333b Mon Sep 17 00:00:00 2001
From: SahilPatidar <patidarsahil2001@gmail.com>
Date: Fri, 12 Apr 2024 17:34:54 +0530
Subject: [PATCH 054/464] [DAG] computeKnownBits - add ISD::ABDU/ISD::ABDS
 handling #84905 (#88253)

Resolve #84905
---
 .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp |  12 ++
 llvm/test/CodeGen/AArch64/abd-combine.ll      |  12 +-
 .../CodeGen/AArch64/arm64-neon-aba-abd.ll     | 157 ++++++++++++++++++
 3 files changed, 174 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index e002cf1d1de3..ddb12a101e02 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -3397,6 +3397,18 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
     Known = KnownBits::mulhs(Known, Known2);
     break;
   }
+  case ISD::ABDU: {
+    Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
+    Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
+    Known = KnownBits::abdu(Known, Known2);
+    break;
+  }
+  case ISD::ABDS: {
+    Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
+    Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
+    Known = KnownBits::abds(Known, Known2);
+    break;
+  }
   case ISD::UMUL_LOHI: {
     assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
     Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
diff --git a/llvm/test/CodeGen/AArch64/abd-combine.ll b/llvm/test/CodeGen/AArch64/abd-combine.ll
index cc29a59288c7..e48680f4be98 100644
--- a/llvm/test/CodeGen/AArch64/abd-combine.ll
+++ b/llvm/test/CodeGen/AArch64/abd-combine.ll
@@ -530,13 +530,11 @@ define <8 x i16> @abds_i_reassoc(<8 x i16> %src1) {
 define <1 x i64> @recursive() {
 ; CHECK-LABEL: recursive:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v0.2d, #0xffffffffffffffff
-; CHECK-NEXT:    movi v1.8b, #1
-; CHECK-NEXT:    uabd v2.8b, v1.8b, v0.8b
-; CHECK-NEXT:    uabdl v0.8h, v1.8b, v0.8b
-; CHECK-NEXT:    dup v1.8b, v2.b[0]
-; CHECK-NEXT:    saddlp v0.1d, v0.2s
-; CHECK-NEXT:    orr v0.8b, v1.8b, v0.8b
+; CHECK-NEXT:    movi v0.8b, #254
+; CHECK-NEXT:    ushll v1.8h, v0.8b, #0
+; CHECK-NEXT:    dup v0.8b, v0.b[0]
+; CHECK-NEXT:    saddlp v1.1d, v1.2s
+; CHECK-NEXT:    orr v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    ret
   %1 = tail call <8 x i8> @llvm.aarch64.neon.umax.v8i8(<8 x i8> zeroinitializer, <8 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>)
   %2 = tail call <8 x i8> @llvm.aarch64.neon.uabd.v8i8(<8 x i8> %1, <8 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>)
diff --git a/llvm/test/CodeGen/AArch64/arm64-neon-aba-abd.ll b/llvm/test/CodeGen/AArch64/arm64-neon-aba-abd.ll
index a13eac9b0a5e..cc9732b83e10 100644
--- a/llvm/test/CodeGen/AArch64/arm64-neon-aba-abd.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-neon-aba-abd.ll
@@ -291,3 +291,160 @@ define <2 x double> @test_fabd_v2f64(<2 x double> %lhs, <2 x double> %rhs) {
   %abd = call <2 x double> @llvm.aarch64.neon.fabd.v2f64(<2 x double> %lhs, <2 x double> %rhs)
   ret <2 x double> %abd
 }
+
+define <8 x i16> @test_uabd_knownbits_vec8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
+; CHECK-LABEL: test_uabd_knownbits_vec8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v2.8h, #15
+; CHECK-NEXT:    and v0.16b, v0.16b, v2.16b
+; CHECK-NEXT:    and v1.16b, v1.16b, v2.16b
+; CHECK-NEXT:    uabd v0.8h, v0.8h, v1.8h
+; CHECK-NEXT:    rev64 v0.8h, v0.8h
+; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    ret
+  %and1 = and <8 x i16> %lhs, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+  %and2 = and <8 x i16> %rhs, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+  %uabd = call <8 x i16> @llvm.aarch64.neon.uabd.v8i16(<8 x i16> %and1, <8 x i16> %and2)
+  %suff = shufflevector <8 x i16> %uabd, <8 x i16> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  %res = and <8 x i16> %suff, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+  ret <8 x i16> %res
+}
+
+define <4 x i32> @knownbits_uabd_mask_and_shuffle_lshr(<4 x i32> %a0, <4 x i32> %a1) {
+; CHECK-LABEL: knownbits_uabd_mask_and_shuffle_lshr:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    ushr v0.4s, v0.4s, #17
+; CHECK-NEXT:    ret
+  %1 = and <4 x i32> %a0, <i32 65535, i32 65535, i32 65535, i32 65535>
+  %2 = and <4 x i32> %a1, <i32 65535, i32 65535, i32 65535, i32 65535>
+  %3 = call <4 x i32> @llvm.aarch64.neon.uabd.v4i32(<4 x i32> %1, <4 x i32> %2)
+  %4 = shufflevector <4 x i32> %3, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %5 = lshr <4 x i32> %4, <i32 17, i32 17, i32 17, i32 17>
+  ret <4 x i32> %5
+}
+
+define <4 x i32> @knownbits_mask_and_shuffle_lshr(<4 x i32> %a0, <4 x i32> %a1) {
+; CHECK-LABEL: knownbits_mask_and_shuffle_lshr:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    ret
+  %1 = and <4 x i32> %a0, <i32 32767, i32 32767, i32 32767, i32 32767>
+  %2 = and <4 x i32> %a1, <i32 32767, i32 32767, i32 32767, i32 32767>
+  %3 = call <4 x i32> @llvm.aarch64.neon.uabd.v4i32(<4 x i32> %1, <4 x i32> %2)
+  %4 = shufflevector <4 x i32> %3, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %5 = lshr <4 x i32> %4, <i32 17, i32 17, i32 17, i32 17>
+  ret <4 x i32> %5
+}
+
+define <4 x i32> @test_sabd_knownbits_vec4i32(<4 x i32> %lhs, <4 x i32> %rhs) {
+; CHECK-LABEL: test_sabd_knownbits_vec4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI31_0
+; CHECK-NEXT:    adrp x9, .LCPI31_1
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI31_0]
+; CHECK-NEXT:    ldr q3, [x9, :lo12:.LCPI31_1]
+; CHECK-NEXT:    and v0.16b, v0.16b, v2.16b
+; CHECK-NEXT:    and v1.16b, v1.16b, v3.16b
+; CHECK-NEXT:    sabd v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    movi v1.2d, #0x0000ff000000ff
+; CHECK-NEXT:    mov v0.s[1], v0.s[0]
+; CHECK-NEXT:    trn2 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ret
+  %and1 = and <4 x i32> %lhs, <i32 255, i32 -1, i32 -1, i32 255>
+  %and2 = and <4 x i32> %rhs, <i32 255, i32 255, i32 -1, i32 -1>
+  %abd = call <4 x i32> @llvm.aarch64.neon.sabd.v4i32(<4 x i32> %and1, <4 x i32> %and2)
+  %s = shufflevector <4 x i32> %abd, <4 x i32> undef, <4 x i32> <i32 0, i32 0, i32 3, i32 3>
+  %4 = and <4 x i32> %s, <i32 255, i32 255, i32 255, i32 255>
+  ret <4 x i32> %4
+}
+
+define <4 x i32> @knownbits_sabd_and_mask(<4 x i32> %a0, <4 x i32> %a1) {
+; CHECK-LABEL: knownbits_sabd_and_mask:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI32_0
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI32_0]
+; CHECK-NEXT:    and v0.16b, v0.16b, v2.16b
+; CHECK-NEXT:    and v1.16b, v1.16b, v2.16b
+; CHECK-NEXT:    sabd v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    zip2 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT:    ret
+  %1 = and <4 x i32> %a0, <i32 -1, i32 -1, i32 255, i32 4085>
+  %2 = and <4 x i32> %a1, <i32 -1, i32 -1, i32 255, i32 4085>
+  %3 = call <4 x i32> @llvm.aarch64.neon.sabd.v4i32(<4 x i32> %1, <4 x i32> %2)
+  %4 = shufflevector <4 x i32> %3, <4 x i32> undef, <4 x i32> <i32 2, i32 2, i32 3, i32 3>
+  ret <4 x i32> %4
+}
+
+define <4 x i32> @knownbits_sabd_and_or_mask(<4 x i32> %a0, <4 x i32> %a1) {
+; CHECK-LABEL: knownbits_sabd_and_or_mask:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    ret
+  %1 = and <4 x i32> %a0, <i32 -1, i32 -1, i32 255, i32 4085>
+  %2 = or <4 x i32> %1, <i32 65535, i32 65535, i32 65535, i32 65535>
+  %3 = and <4 x i32> %a1, <i32 -1, i32 -1, i32 255, i32 4085>
+  %4 = or <4 x i32> %3, <i32 65535, i32 65535, i32 65535, i32 65535>
+  %5 = call <4 x i32> @llvm.aarch64.neon.uabd.v4i32(<4 x i32> %2, <4 x i32> %4)
+  %6 = shufflevector <4 x i32> %5, <4 x i32> undef, <4 x i32> <i32 2, i32 2, i32 3, i32 3>
+  ret <4 x i32> %6
+}
+
+define <4 x i32> @knownbits_sabd_and_xor_mask(<4 x i32> %a0, <4 x i32> %a1) {
+; CHECK-LABEL: knownbits_sabd_and_xor_mask:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI34_0
+; CHECK-NEXT:    movi v3.2d, #0x00ffff0000ffff
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI34_0]
+; CHECK-NEXT:    and v0.16b, v0.16b, v2.16b
+; CHECK-NEXT:    and v1.16b, v1.16b, v2.16b
+; CHECK-NEXT:    eor v0.16b, v0.16b, v3.16b
+; CHECK-NEXT:    eor v1.16b, v1.16b, v3.16b
+; CHECK-NEXT:    sabd v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    zip2 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT:    ret
+  %1 = and <4 x i32> %a0, <i32 -1, i32 -1, i32 255, i32 4085>
+  %2 = xor <4 x i32> %1, <i32 65535, i32 65535, i32 65535, i32 65535>
+  %3 = and <4 x i32> %a1, <i32 -1, i32 -1, i32 255, i32 4085>
+  %4 = xor <4 x i32> %3, <i32 65535, i32 65535, i32 65535, i32 65535>
+  %5 = call <4 x i32> @llvm.aarch64.neon.sabd.v4i32(<4 x i32> %2, <4 x i32> %4)
+  %6 = shufflevector <4 x i32> %5, <4 x i32> undef, <4 x i32> <i32 2, i32 2, i32 3, i32 3>
+  ret <4 x i32> %6
+}
+
+define <4 x i32> @knownbits_sabd_and_shl_mask(<4 x i32> %a0, <4 x i32> %a1) {
+; CHECK-LABEL: knownbits_sabd_and_shl_mask:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    ret
+  %1 = and <4 x i32> %a0, <i32 -65536, i32 -7, i32 -7, i32 -65536>
+  %2 = shl <4 x i32> %1, <i32 17, i32 17, i32 17, i32 17>
+  %3 = and <4 x i32> %a1, <i32 -65536, i32 -7, i32 -7, i32 -65536>
+  %4 = shl <4 x i32> %3, <i32 17, i32 17, i32 17, i32 17>
+  %5 = call <4 x i32> @llvm.aarch64.neon.sabd.v4i32(<4 x i32> %2, <4 x i32> %4)
+  %6 = shufflevector <4 x i32> %5, <4 x i32> undef, <4 x i32> <i32 0, i32 0, i32 3, i32 3>
+  ret <4 x i32> %6
+}
+
+define <4 x i32> @knownbits_sabd_and_mul_mask(<4 x i32> %a0, <4 x i32> %a1) {
+; CHECK-LABEL: knownbits_sabd_and_mul_mask:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adrp x8, .LCPI36_0
+; CHECK-NEXT:    ldr q2, [x8, :lo12:.LCPI36_0]
+; CHECK-NEXT:    and v3.16b, v0.16b, v2.16b
+; CHECK-NEXT:    and v2.16b, v1.16b, v2.16b
+; CHECK-NEXT:    mul v0.4s, v0.4s, v3.4s
+; CHECK-NEXT:    mul v1.4s, v1.4s, v2.4s
+; CHECK-NEXT:    sabd v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    mov v0.s[1], v0.s[0]
+; CHECK-NEXT:    trn2 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT:    ret
+  %1 = and <4 x i32> %a0, <i32 -65536, i32 -7, i32 -7, i32 -65536>
+  %2 = mul <4 x i32> %a0, %1
+  %3 = and <4 x i32> %a1, <i32 -65536, i32 -7, i32 -7, i32 -65536>
+  %4 = mul <4 x i32> %a1, %3
+  %5 = call <4 x i32> @llvm.aarch64.neon.sabd.v4i32(<4 x i32> %2, <4 x i32> %4)
+  %6 = shufflevector <4 x i32> %5, <4 x i32> undef, <4 x i32> <i32 0, i32 0, i32 3, i32 3>
+  ret <4 x i32> %6
+}
-- 
GitLab


From 6ec467297db272d5ae8f76e695296bd4945dedc8 Mon Sep 17 00:00:00 2001
From: Nathan Sidwell <nathan@acm.org>
Date: Fri, 12 Apr 2024 08:34:43 -0400
Subject: [PATCH 055/464] [BOLT][NFC] Adjust misleading comment & formatting
 (#88409)

This originally dealt with tbss, but now handles any bss-like section.
So the comment is inaccurate. Also, the `{}` on the messaging seem
unnecessary.
---
 bolt/lib/Rewrite/RewriteInstance.cpp | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index eea66454b289..cbbe7d00d35d 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -4300,7 +4300,7 @@ RewriteInstance::getOutputSections(ELFObjectFile<ELFT> *File,
   for (auto &SectionKV : OutputSections) {
     ELFShdrTy &Section = SectionKV.second;
 
-    // Ignore TLS sections as they don't take any space in the file.
+    // Ignore NOBITS sections as they don't take any space in the file.
     if (Section.sh_type == ELF::SHT_NOBITS)
       continue;
 
@@ -4308,10 +4308,9 @@ RewriteInstance::getOutputSections(ELFObjectFile<ELFT> *File,
     // placed in different loadable segments.
     if (PrevSection &&
         PrevSection->sh_offset + PrevSection->sh_size > Section.sh_offset) {
-      if (opts::Verbosity > 1) {
+      if (opts::Verbosity > 1)
         BC->outs() << "BOLT-INFO: adjusting size for section "
                    << PrevBinSec->getOutputName() << '\n';
-      }
       PrevSection->sh_size = Section.sh_offset - PrevSection->sh_offset;
     }
 
-- 
GitLab


From f220d26eb1ab5b588215acf538ca82bb3c8798cc Mon Sep 17 00:00:00 2001
From: Tom Eccles <tom.eccles@arm.com>
Date: Fri, 12 Apr 2024 13:43:48 +0100
Subject: [PATCH 056/464] [flang][runtime] Add ACCESS library procedure
 (#88395)

This is a GNU extension:
https://gcc.gnu.org/onlinedocs/gfortran/ACCESS.html

Used in SALMON:
https://salmon-tddft.jp/download.html

Unfortunately the intrinsic takes a file path to operate on so there
isn't an easy way to make the test robust. The unit test expects to be
able to create, set read write and execute permissions, and delete files
called
  `std::filesystem::temp_directory_path() / <test_name>.<pid>`

The test will fail if a file already exists with that name.

I have not implemented the intrinsic on Windows because this is wrapping
a POSIX system call and Windows doesn't support all of the permission
bits tested by the intrinsic. I don't have a Windows machine easily
available to check if Gfortran implements this intrinsic on Windows.
---
 flang/docs/Intrinsics.md                 |   8 +
 flang/include/flang/Runtime/extensions.h |   7 +
 flang/runtime/extensions.cpp             |  73 +++++
 flang/unittests/Runtime/AccessTest.cpp   | 390 +++++++++++++++++++++++
 flang/unittests/Runtime/CMakeLists.txt   |   1 +
 5 files changed, 479 insertions(+)
 create mode 100644 flang/unittests/Runtime/AccessTest.cpp

diff --git a/flang/docs/Intrinsics.md b/flang/docs/Intrinsics.md
index ccb93e104dab..848619cb65d9 100644
--- a/flang/docs/Intrinsics.md
+++ b/flang/docs/Intrinsics.md
@@ -657,6 +657,14 @@ CALL CO_REDUCE
 CALL CO_SUM
 ```
 
+### Inquiry Functions
+ACCESS (GNU extension) is not supported on Windows. Otherwise:
+```
+CHARACTER(LEN=*) :: path = 'path/to/file'
+IF (ACCESS(path, 'rwx')) &
+  ...
+```
+
 ## Non-standard intrinsics
 ### PGI
 ```
diff --git a/flang/include/flang/Runtime/extensions.h b/flang/include/flang/Runtime/extensions.h
index 7d0952206fc1..fef651f3b2ee 100644
--- a/flang/include/flang/Runtime/extensions.h
+++ b/flang/include/flang/Runtime/extensions.h
@@ -44,5 +44,12 @@ std::int64_t RTNAME(Signal)(std::int64_t number, void (*handler)(int));
 // GNU extension subroutine SLEEP(SECONDS)
 void RTNAME(Sleep)(std::int64_t seconds);
 
+// GNU extension function ACCESS(NAME, MODE)
+// TODO: not supported on Windows
+#ifndef _WIN32
+std::int64_t FORTRAN_PROCEDURE_NAME(access)(const char *name,
+    std::int64_t nameLength, const char *mode, std::int64_t modeLength);
+#endif
+
 } // extern "C"
 #endif // FORTRAN_RUNTIME_EXTENSIONS_H_
diff --git a/flang/runtime/extensions.cpp b/flang/runtime/extensions.cpp
index 3ac98000335d..12498b502ae1 100644
--- a/flang/runtime/extensions.cpp
+++ b/flang/runtime/extensions.cpp
@@ -17,6 +17,7 @@
 #include "flang/Runtime/entry-names.h"
 #include "flang/Runtime/io-api.h"
 #include <chrono>
+#include <cstring>
 #include <ctime>
 #include <signal.h>
 #include <thread>
@@ -138,5 +139,77 @@ void RTNAME(Sleep)(std::int64_t seconds) {
   std::this_thread::sleep_for(std::chrono::seconds(seconds));
 }
 
+// TODO: not supported on Windows
+#ifndef _WIN32
+std::int64_t FORTRAN_PROCEDURE_NAME(access)(const char *name,
+    std::int64_t nameLength, const char *mode, std::int64_t modeLength) {
+  std::int64_t ret{-1};
+  if (nameLength <= 0 || modeLength <= 0 || !name || !mode) {
+    return ret;
+  }
+
+  // ensure name is null terminated
+  char *newName{nullptr};
+  if (name[nameLength - 1] != '\0') {
+    newName = static_cast<char *>(std::malloc(nameLength + 1));
+    std::memcpy(newName, name, nameLength);
+    newName[nameLength] = '\0';
+    name = newName;
+  }
+
+  // calculate mode
+  bool read{false};
+  bool write{false};
+  bool execute{false};
+  bool exists{false};
+  int imode{0};
+
+  for (std::int64_t i = 0; i < modeLength; ++i) {
+    switch (mode[i]) {
+    case 'r':
+      read = true;
+      break;
+    case 'w':
+      write = true;
+      break;
+    case 'x':
+      execute = true;
+      break;
+    case ' ':
+      exists = true;
+      break;
+    default:
+      // invalid mode
+      goto cleanup;
+    }
+  }
+  if (!read && !write && !execute && !exists) {
+    // invalid mode
+    goto cleanup;
+  }
+
+  if (!read && !write && !execute) {
+    imode = F_OK;
+  } else {
+    if (read) {
+      imode |= R_OK;
+    }
+    if (write) {
+      imode |= W_OK;
+    }
+    if (execute) {
+      imode |= X_OK;
+    }
+  }
+  ret = access(name, imode);
+
+cleanup:
+  if (newName) {
+    free(newName);
+  }
+  return ret;
+}
+#endif
+
 } // namespace Fortran::runtime
 } // extern "C"
diff --git a/flang/unittests/Runtime/AccessTest.cpp b/flang/unittests/Runtime/AccessTest.cpp
new file mode 100644
index 000000000000..5c8a634192d9
--- /dev/null
+++ b/flang/unittests/Runtime/AccessTest.cpp
@@ -0,0 +1,390 @@
+//===-- flang/unittests/Runtime/AccessTest.cpp ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// TODO: ACCESS is not yet implemented on Windows
+#ifndef _WIN32
+
+#include "CrashHandlerFixture.h"
+#include "gtest/gtest.h"
+#include "flang/Runtime/extensions.h"
+
+#include <fcntl.h>
+#include <filesystem>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+namespace {
+
+struct AccessTests : public CrashHandlerFixture {};
+
+struct AccessType {
+  bool read{false};
+  bool write{false};
+  bool execute{false};
+  bool exists{false};
+};
+
+} // namespace
+
+static std::string addPIDSuffix(const char *name) {
+  std::stringstream ss;
+  ss << name;
+  ss << '.';
+
+  ss << getpid();
+
+  return ss.str();
+}
+
+static std::filesystem::path createTemporaryFile(
+    const char *name, const AccessType &accessType) {
+  std::filesystem::path path{
+      std::filesystem::temp_directory_path() / addPIDSuffix(name)};
+
+  // O_CREAT | O_EXCL enforces that this file is newly created by this call.
+  // This feels risky. If we don't have permission to create files in the
+  // temporary directory or if the files already exist, the test will fail.
+  // But we can't use std::tmpfile() because we need a path to the file and
+  // to control the filesystem permissions
+  mode_t mode{0};
+  if (accessType.read) {
+    mode |= S_IRUSR;
+  }
+  if (accessType.write) {
+    mode |= S_IWUSR;
+  }
+  if (accessType.execute) {
+    mode |= S_IXUSR;
+  }
+
+  int file = open(path.c_str(), O_CREAT | O_EXCL, mode);
+  if (file == -1) {
+    return {};
+  }
+
+  close(file);
+
+  return path;
+}
+
+static std::int64_t callAccess(
+    const std::filesystem::path &path, const AccessType &accessType) {
+  const char *cpath{path.c_str()};
+  std::int64_t pathlen = std::strlen(cpath);
+
+  std::string mode;
+  if (accessType.read) {
+    mode += 'r';
+  }
+  if (accessType.write) {
+    mode += 'w';
+  }
+  if (accessType.execute) {
+    mode += 'x';
+  }
+  if (accessType.exists) {
+    mode += ' ';
+  }
+
+  const char *cmode = mode.c_str();
+  std::int64_t modelen = std::strlen(cmode);
+
+  return FORTRAN_PROCEDURE_NAME(access)(cpath, pathlen, cmode, modelen);
+}
+
+TEST(AccessTests, TestExists) {
+  AccessType accessType;
+  accessType.exists = true;
+
+  std::filesystem::path path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  std::int64_t res = callAccess(path, accessType);
+
+  std::filesystem::remove(path);
+
+  ASSERT_EQ(res, 0);
+}
+
+TEST(AccessTests, TestNotExists) {
+  std::filesystem::path nonExistant{addPIDSuffix(__func__)};
+  ASSERT_FALSE(std::filesystem::exists(nonExistant));
+
+  AccessType accessType;
+  accessType.exists = true;
+  std::int64_t res = callAccess(nonExistant, accessType);
+
+  ASSERT_NE(res, 0);
+}
+
+TEST(AccessTests, TestRead) {
+  AccessType accessType;
+  accessType.read = true;
+
+  std::filesystem::path path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  std::int64_t res = callAccess(path, accessType);
+
+  std::filesystem::remove(path);
+
+  ASSERT_EQ(res, 0);
+}
+
+TEST(AccessTests, TestNotRead) {
+  AccessType accessType;
+  accessType.read = false;
+
+  std::filesystem::path path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  accessType.read = true;
+  std::int64_t res = callAccess(path, accessType);
+
+  std::filesystem::remove(path);
+
+  ASSERT_NE(res, 0);
+}
+
+TEST(AccessTests, TestWrite) {
+  AccessType accessType;
+  accessType.write = true;
+
+  std::filesystem::path path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  std::int64_t res = callAccess(path, accessType);
+
+  std::filesystem::remove(path);
+
+  ASSERT_EQ(res, 0);
+}
+
+TEST(AccessTests, TestNotWrite) {
+  AccessType accessType;
+  accessType.write = false;
+
+  std::filesystem::path path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  accessType.write = true;
+  std::int64_t res = callAccess(path, accessType);
+
+  std::filesystem::remove(path);
+
+  ASSERT_NE(res, 0);
+}
+
+TEST(AccessTests, TestReadWrite) {
+  AccessType accessType;
+  accessType.read = true;
+  accessType.write = true;
+
+  std::filesystem::path path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  std::int64_t res = callAccess(path, accessType);
+
+  std::filesystem::remove(path);
+
+  ASSERT_EQ(res, 0);
+}
+
+TEST(AccessTests, TestNotReadWrite0) {
+  AccessType accessType;
+  accessType.read = false;
+  accessType.write = false;
+
+  std::filesystem::path path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  accessType.read = true;
+  accessType.write = true;
+  std::int64_t res = callAccess(path, accessType);
+
+  std::filesystem::remove(path);
+
+  ASSERT_NE(res, 0);
+}
+
+TEST(AccessTests, TestNotReadWrite1) {
+  AccessType accessType;
+  accessType.read = true;
+  accessType.write = false;
+
+  std::filesystem::path path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  accessType.read = true;
+  accessType.write = true;
+  std::int64_t res = callAccess(path, accessType);
+
+  std::filesystem::remove(path);
+
+  ASSERT_NE(res, 0);
+}
+
+TEST(AccessTests, TestNotReadWrite2) {
+  AccessType accessType;
+  accessType.read = false;
+  accessType.write = true;
+
+  std::filesystem::path path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  accessType.read = true;
+  accessType.write = true;
+  std::int64_t res = callAccess(path, accessType);
+
+  std::filesystem::remove(path);
+
+  ASSERT_NE(res, 0);
+}
+
+TEST(AccessTests, TestExecute) {
+  AccessType accessType;
+  accessType.execute = true;
+
+  std::filesystem::path path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  std::int64_t res = callAccess(path, accessType);
+
+  std::filesystem::remove(path);
+
+  ASSERT_EQ(res, 0);
+}
+
+TEST(AccessTests, TestNotExecute) {
+  AccessType accessType;
+  accessType.execute = false;
+
+  std::filesystem::path path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  accessType.execute = true;
+  std::int64_t res = callAccess(path, accessType);
+
+  std::filesystem::remove(path);
+
+  ASSERT_NE(res, 0);
+}
+
+TEST(AccessTests, TestRWX) {
+  AccessType accessType;
+  accessType.read = true;
+  accessType.write = true;
+  accessType.execute = true;
+
+  std::filesystem::path path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  std::int64_t res = callAccess(path, accessType);
+
+  std::filesystem::remove(path);
+
+  ASSERT_EQ(res, 0);
+}
+
+TEST(AccessTests, TestNotRWX0) {
+  AccessType accessType;
+  accessType.read = false;
+  accessType.write = false;
+  accessType.execute = false;
+
+  std::filesystem::path path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  accessType.read = true;
+  accessType.write = true;
+  accessType.execute = true;
+  std::int64_t res = callAccess(path, accessType);
+
+  std::filesystem::remove(path);
+
+  ASSERT_NE(res, 0);
+}
+
+TEST(AccessTests, TestNotRWX1) {
+  AccessType accessType;
+  accessType.read = true;
+  accessType.write = false;
+  accessType.execute = false;
+
+  std::filesystem::path path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  accessType.read = true;
+  accessType.write = true;
+  accessType.execute = true;
+  std::int64_t res = callAccess(path, accessType);
+
+  std::filesystem::remove(path);
+
+  ASSERT_NE(res, 0);
+}
+
+TEST(AccessTests, TestNotRWX2) {
+  AccessType accessType;
+  accessType.read = true;
+  accessType.write = true;
+  accessType.execute = false;
+
+  std::filesystem::path path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  accessType.read = true;
+  accessType.write = true;
+  accessType.execute = true;
+  std::int64_t res = callAccess(path, accessType);
+
+  std::filesystem::remove(path);
+
+  ASSERT_NE(res, 0);
+}
+
+TEST(AccessTests, TestNotRWX3) {
+  AccessType accessType;
+  accessType.read = true;
+  accessType.write = false;
+  accessType.execute = true;
+
+  std::filesystem::path path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  accessType.read = true;
+  accessType.write = true;
+  accessType.execute = true;
+  std::int64_t res = callAccess(path, accessType);
+
+  std::filesystem::remove(path);
+
+  ASSERT_NE(res, 0);
+}
+
+TEST(AccessTests, TestNotRWX4) {
+  AccessType accessType;
+  accessType.read = false;
+  accessType.write = true;
+  accessType.execute = true;
+
+  std::filesystem::path path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  accessType.read = true;
+  accessType.write = true;
+  accessType.execute = true;
+  std::int64_t res = callAccess(path, accessType);
+
+  std::filesystem::remove(path);
+
+  ASSERT_NE(res, 0);
+}
+
+#endif // !_WIN32
diff --git a/flang/unittests/Runtime/CMakeLists.txt b/flang/unittests/Runtime/CMakeLists.txt
index 23f02aa75124..f7caacad3a59 100644
--- a/flang/unittests/Runtime/CMakeLists.txt
+++ b/flang/unittests/Runtime/CMakeLists.txt
@@ -1,4 +1,5 @@
 add_flang_unittest(FlangRuntimeTests
+  AccessTest.cpp
   Allocatable.cpp
   ArrayConstructor.cpp
   BufferTest.cpp
-- 
GitLab


From 7e7468c9ed4e29fe8327525923fd8ab0169b1c07 Mon Sep 17 00:00:00 2001
From: Tom Eccles <tom.eccles@arm.com>
Date: Fri, 12 Apr 2024 13:50:39 +0100
Subject: [PATCH 057/464] Revert "[flang][runtime] Add ACCESS library
 procedure" (#88507)

Reverts llvm/llvm-project#88395

This broke the powerpc buildbot. That build doesn't support using
`std::filesystem` in flang unit tests.
---
 flang/docs/Intrinsics.md                 |   8 -
 flang/include/flang/Runtime/extensions.h |   7 -
 flang/runtime/extensions.cpp             |  73 -----
 flang/unittests/Runtime/AccessTest.cpp   | 390 -----------------------
 flang/unittests/Runtime/CMakeLists.txt   |   1 -
 5 files changed, 479 deletions(-)
 delete mode 100644 flang/unittests/Runtime/AccessTest.cpp

diff --git a/flang/docs/Intrinsics.md b/flang/docs/Intrinsics.md
index 848619cb65d9..ccb93e104dab 100644
--- a/flang/docs/Intrinsics.md
+++ b/flang/docs/Intrinsics.md
@@ -657,14 +657,6 @@ CALL CO_REDUCE
 CALL CO_SUM
 ```
 
-### Inquiry Functions
-ACCESS (GNU extension) is not supported on Windows. Otherwise:
-```
-CHARACTER(LEN=*) :: path = 'path/to/file'
-IF (ACCESS(path, 'rwx')) &
-  ...
-```
-
 ## Non-standard intrinsics
 ### PGI
 ```
diff --git a/flang/include/flang/Runtime/extensions.h b/flang/include/flang/Runtime/extensions.h
index fef651f3b2ee..7d0952206fc1 100644
--- a/flang/include/flang/Runtime/extensions.h
+++ b/flang/include/flang/Runtime/extensions.h
@@ -44,12 +44,5 @@ std::int64_t RTNAME(Signal)(std::int64_t number, void (*handler)(int));
 // GNU extension subroutine SLEEP(SECONDS)
 void RTNAME(Sleep)(std::int64_t seconds);
 
-// GNU extension function ACCESS(NAME, MODE)
-// TODO: not supported on Windows
-#ifndef _WIN32
-std::int64_t FORTRAN_PROCEDURE_NAME(access)(const char *name,
-    std::int64_t nameLength, const char *mode, std::int64_t modeLength);
-#endif
-
 } // extern "C"
 #endif // FORTRAN_RUNTIME_EXTENSIONS_H_
diff --git a/flang/runtime/extensions.cpp b/flang/runtime/extensions.cpp
index 12498b502ae1..3ac98000335d 100644
--- a/flang/runtime/extensions.cpp
+++ b/flang/runtime/extensions.cpp
@@ -17,7 +17,6 @@
 #include "flang/Runtime/entry-names.h"
 #include "flang/Runtime/io-api.h"
 #include <chrono>
-#include <cstring>
 #include <ctime>
 #include <signal.h>
 #include <thread>
@@ -139,77 +138,5 @@ void RTNAME(Sleep)(std::int64_t seconds) {
   std::this_thread::sleep_for(std::chrono::seconds(seconds));
 }
 
-// TODO: not supported on Windows
-#ifndef _WIN32
-std::int64_t FORTRAN_PROCEDURE_NAME(access)(const char *name,
-    std::int64_t nameLength, const char *mode, std::int64_t modeLength) {
-  std::int64_t ret{-1};
-  if (nameLength <= 0 || modeLength <= 0 || !name || !mode) {
-    return ret;
-  }
-
-  // ensure name is null terminated
-  char *newName{nullptr};
-  if (name[nameLength - 1] != '\0') {
-    newName = static_cast<char *>(std::malloc(nameLength + 1));
-    std::memcpy(newName, name, nameLength);
-    newName[nameLength] = '\0';
-    name = newName;
-  }
-
-  // calculate mode
-  bool read{false};
-  bool write{false};
-  bool execute{false};
-  bool exists{false};
-  int imode{0};
-
-  for (std::int64_t i = 0; i < modeLength; ++i) {
-    switch (mode[i]) {
-    case 'r':
-      read = true;
-      break;
-    case 'w':
-      write = true;
-      break;
-    case 'x':
-      execute = true;
-      break;
-    case ' ':
-      exists = true;
-      break;
-    default:
-      // invalid mode
-      goto cleanup;
-    }
-  }
-  if (!read && !write && !execute && !exists) {
-    // invalid mode
-    goto cleanup;
-  }
-
-  if (!read && !write && !execute) {
-    imode = F_OK;
-  } else {
-    if (read) {
-      imode |= R_OK;
-    }
-    if (write) {
-      imode |= W_OK;
-    }
-    if (execute) {
-      imode |= X_OK;
-    }
-  }
-  ret = access(name, imode);
-
-cleanup:
-  if (newName) {
-    free(newName);
-  }
-  return ret;
-}
-#endif
-
 } // namespace Fortran::runtime
 } // extern "C"
diff --git a/flang/unittests/Runtime/AccessTest.cpp b/flang/unittests/Runtime/AccessTest.cpp
deleted file mode 100644
index 5c8a634192d9..000000000000
--- a/flang/unittests/Runtime/AccessTest.cpp
+++ /dev/null
@@ -1,390 +0,0 @@
-//===-- flang/unittests/Runtime/AccessTest.cpp ----------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// TODO: ACCESS is not yet implemented on Windows
-#ifndef _WIN32
-
-#include "CrashHandlerFixture.h"
-#include "gtest/gtest.h"
-#include "flang/Runtime/extensions.h"
-
-#include <fcntl.h>
-#include <filesystem>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <unistd.h>
-
-namespace {
-
-struct AccessTests : public CrashHandlerFixture {};
-
-struct AccessType {
-  bool read{false};
-  bool write{false};
-  bool execute{false};
-  bool exists{false};
-};
-
-} // namespace
-
-static std::string addPIDSuffix(const char *name) {
-  std::stringstream ss;
-  ss << name;
-  ss << '.';
-
-  ss << getpid();
-
-  return ss.str();
-}
-
-static std::filesystem::path createTemporaryFile(
-    const char *name, const AccessType &accessType) {
-  std::filesystem::path path{
-      std::filesystem::temp_directory_path() / addPIDSuffix(name)};
-
-  // O_CREAT | O_EXCL enforces that this file is newly created by this call.
-  // This feels risky. If we don't have permission to create files in the
-  // temporary directory or if the files already exist, the test will fail.
-  // But we can't use std::tmpfile() because we need a path to the file and
-  // to control the filesystem permissions
-  mode_t mode{0};
-  if (accessType.read) {
-    mode |= S_IRUSR;
-  }
-  if (accessType.write) {
-    mode |= S_IWUSR;
-  }
-  if (accessType.execute) {
-    mode |= S_IXUSR;
-  }
-
-  int file = open(path.c_str(), O_CREAT | O_EXCL, mode);
-  if (file == -1) {
-    return {};
-  }
-
-  close(file);
-
-  return path;
-}
-
-static std::int64_t callAccess(
-    const std::filesystem::path &path, const AccessType &accessType) {
-  const char *cpath{path.c_str()};
-  std::int64_t pathlen = std::strlen(cpath);
-
-  std::string mode;
-  if (accessType.read) {
-    mode += 'r';
-  }
-  if (accessType.write) {
-    mode += 'w';
-  }
-  if (accessType.execute) {
-    mode += 'x';
-  }
-  if (accessType.exists) {
-    mode += ' ';
-  }
-
-  const char *cmode = mode.c_str();
-  std::int64_t modelen = std::strlen(cmode);
-
-  return FORTRAN_PROCEDURE_NAME(access)(cpath, pathlen, cmode, modelen);
-}
-
-TEST(AccessTests, TestExists) {
-  AccessType accessType;
-  accessType.exists = true;
-
-  std::filesystem::path path = createTemporaryFile(__func__, accessType);
-  ASSERT_FALSE(path.empty());
-
-  std::int64_t res = callAccess(path, accessType);
-
-  std::filesystem::remove(path);
-
-  ASSERT_EQ(res, 0);
-}
-
-TEST(AccessTests, TestNotExists) {
-  std::filesystem::path nonExistant{addPIDSuffix(__func__)};
-  ASSERT_FALSE(std::filesystem::exists(nonExistant));
-
-  AccessType accessType;
-  accessType.exists = true;
-  std::int64_t res = callAccess(nonExistant, accessType);
-
-  ASSERT_NE(res, 0);
-}
-
-TEST(AccessTests, TestRead) {
-  AccessType accessType;
-  accessType.read = true;
-
-  std::filesystem::path path = createTemporaryFile(__func__, accessType);
-  ASSERT_FALSE(path.empty());
-
-  std::int64_t res = callAccess(path, accessType);
-
-  std::filesystem::remove(path);
-
-  ASSERT_EQ(res, 0);
-}
-
-TEST(AccessTests, TestNotRead) {
-  AccessType accessType;
-  accessType.read = false;
-
-  std::filesystem::path path = createTemporaryFile(__func__, accessType);
-  ASSERT_FALSE(path.empty());
-
-  accessType.read = true;
-  std::int64_t res = callAccess(path, accessType);
-
-  std::filesystem::remove(path);
-
-  ASSERT_NE(res, 0);
-}
-
-TEST(AccessTests, TestWrite) {
-  AccessType accessType;
-  accessType.write = true;
-
-  std::filesystem::path path = createTemporaryFile(__func__, accessType);
-  ASSERT_FALSE(path.empty());
-
-  std::int64_t res = callAccess(path, accessType);
-
-  std::filesystem::remove(path);
-
-  ASSERT_EQ(res, 0);
-}
-
-TEST(AccessTests, TestNotWrite) {
-  AccessType accessType;
-  accessType.write = false;
-
-  std::filesystem::path path = createTemporaryFile(__func__, accessType);
-  ASSERT_FALSE(path.empty());
-
-  accessType.write = true;
-  std::int64_t res = callAccess(path, accessType);
-
-  std::filesystem::remove(path);
-
-  ASSERT_NE(res, 0);
-}
-
-TEST(AccessTests, TestReadWrite) {
-  AccessType accessType;
-  accessType.read = true;
-  accessType.write = true;
-
-  std::filesystem::path path = createTemporaryFile(__func__, accessType);
-  ASSERT_FALSE(path.empty());
-
-  std::int64_t res = callAccess(path, accessType);
-
-  std::filesystem::remove(path);
-
-  ASSERT_EQ(res, 0);
-}
-
-TEST(AccessTests, TestNotReadWrite0) {
-  AccessType accessType;
-  accessType.read = false;
-  accessType.write = false;
-
-  std::filesystem::path path = createTemporaryFile(__func__, accessType);
-  ASSERT_FALSE(path.empty());
-
-  accessType.read = true;
-  accessType.write = true;
-  std::int64_t res = callAccess(path, accessType);
-
-  std::filesystem::remove(path);
-
-  ASSERT_NE(res, 0);
-}
-
-TEST(AccessTests, TestNotReadWrite1) {
-  AccessType accessType;
-  accessType.read = true;
-  accessType.write = false;
-
-  std::filesystem::path path = createTemporaryFile(__func__, accessType);
-  ASSERT_FALSE(path.empty());
-
-  accessType.read = true;
-  accessType.write = true;
-  std::int64_t res = callAccess(path, accessType);
-
-  std::filesystem::remove(path);
-
-  ASSERT_NE(res, 0);
-}
-
-TEST(AccessTests, TestNotReadWrite2) {
-  AccessType accessType;
-  accessType.read = false;
-  accessType.write = true;
-
-  std::filesystem::path path = createTemporaryFile(__func__, accessType);
-  ASSERT_FALSE(path.empty());
-
-  accessType.read = true;
-  accessType.write = true;
-  std::int64_t res = callAccess(path, accessType);
-
-  std::filesystem::remove(path);
-
-  ASSERT_NE(res, 0);
-}
-
-TEST(AccessTests, TestExecute) {
-  AccessType accessType;
-  accessType.execute = true;
-
-  std::filesystem::path path = createTemporaryFile(__func__, accessType);
-  ASSERT_FALSE(path.empty());
-
-  std::int64_t res = callAccess(path, accessType);
-
-  std::filesystem::remove(path);
-
-  ASSERT_EQ(res, 0);
-}
-
-TEST(AccessTests, TestNotExecute) {
-  AccessType accessType;
-  accessType.execute = false;
-
-  std::filesystem::path path = createTemporaryFile(__func__, accessType);
-  ASSERT_FALSE(path.empty());
-
-  accessType.execute = true;
-  std::int64_t res = callAccess(path, accessType);
-
-  std::filesystem::remove(path);
-
-  ASSERT_NE(res, 0);
-}
-
-TEST(AccessTests, TestRWX) {
-  AccessType accessType;
-  accessType.read = true;
-  accessType.write = true;
-  accessType.execute = true;
-
-  std::filesystem::path path = createTemporaryFile(__func__, accessType);
-  ASSERT_FALSE(path.empty());
-
-  std::int64_t res = callAccess(path, accessType);
-
-  std::filesystem::remove(path);
-
-  ASSERT_EQ(res, 0);
-}
-
-TEST(AccessTests, TestNotRWX0) {
-  AccessType accessType;
-  accessType.read = false;
-  accessType.write = false;
-  accessType.execute = false;
-
-  std::filesystem::path path = createTemporaryFile(__func__, accessType);
-  ASSERT_FALSE(path.empty());
-
-  accessType.read = true;
-  accessType.write = true;
-  accessType.execute = true;
-  std::int64_t res = callAccess(path, accessType);
-
-  std::filesystem::remove(path);
-
-  ASSERT_NE(res, 0);
-}
-
-TEST(AccessTests, TestNotRWX1) {
-  AccessType accessType;
-  accessType.read = true;
-  accessType.write = false;
-  accessType.execute = false;
-
-  std::filesystem::path path = createTemporaryFile(__func__, accessType);
-  ASSERT_FALSE(path.empty());
-
-  accessType.read = true;
-  accessType.write = true;
-  accessType.execute = true;
-  std::int64_t res = callAccess(path, accessType);
-
-  std::filesystem::remove(path);
-
-  ASSERT_NE(res, 0);
-}
-
-TEST(AccessTests, TestNotRWX2) {
-  AccessType accessType;
-  accessType.read = true;
-  accessType.write = true;
-  accessType.execute = false;
-
-  std::filesystem::path path = createTemporaryFile(__func__, accessType);
-  ASSERT_FALSE(path.empty());
-
-  accessType.read = true;
-  accessType.write = true;
-  accessType.execute = true;
-  std::int64_t res = callAccess(path, accessType);
-
-  std::filesystem::remove(path);
-
-  ASSERT_NE(res, 0);
-}
-
-TEST(AccessTests, TestNotRWX3) {
-  AccessType accessType;
-  accessType.read = true;
-  accessType.write = false;
-  accessType.execute = true;
-
-  std::filesystem::path path = createTemporaryFile(__func__, accessType);
-  ASSERT_FALSE(path.empty());
-
-  accessType.read = true;
-  accessType.write = true;
-  accessType.execute = true;
-  std::int64_t res = callAccess(path, accessType);
-
-  std::filesystem::remove(path);
-
-  ASSERT_NE(res, 0);
-}
-
-TEST(AccessTests, TestNotRWX4) {
-  AccessType accessType;
-  accessType.read = false;
-  accessType.write = true;
-  accessType.execute = true;
-
-  std::filesystem::path path = createTemporaryFile(__func__, accessType);
-  ASSERT_FALSE(path.empty());
-
-  accessType.read = true;
-  accessType.write = true;
-  accessType.execute = true;
-  std::int64_t res = callAccess(path, accessType);
-
-  std::filesystem::remove(path);
-
-  ASSERT_NE(res, 0);
-}
-
-#endif // !_WIN32
diff --git a/flang/unittests/Runtime/CMakeLists.txt b/flang/unittests/Runtime/CMakeLists.txt
index f7caacad3a59..23f02aa75124 100644
--- a/flang/unittests/Runtime/CMakeLists.txt
+++ b/flang/unittests/Runtime/CMakeLists.txt
@@ -1,5 +1,4 @@
 add_flang_unittest(FlangRuntimeTests
-  AccessTest.cpp
   Allocatable.cpp
   ArrayConstructor.cpp
   BufferTest.cpp
-- 
GitLab


From ea3d0db130b9a6c4678c11dd8bc48e5630624b62 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Fri, 12 Apr 2024 13:51:41 +0100
Subject: [PATCH 058/464] [VectorCombine] foldShuffleOfCastops - ensure we can
 scale shuffle masks between bitcasted vector types

Don't just assert that the src/dst vector element counts are multiples of one another - in general IR this can actually happen.

Reported by @mikaelholmen
---
 llvm/lib/Transforms/Vectorize/VectorCombine.cpp   |  6 ++++++
 .../VectorCombine/X86/shuffle-of-casts.ll         | 15 +++++++++++++++
 2 files changed, 21 insertions(+)

diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 44cba60013af..653e0ff7a3ec 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -1483,6 +1483,12 @@ bool VectorCombine::foldShuffleOfCastops(Instruction &I) {
   assert((NumDstElts == NumSrcElts || Opcode == Instruction::BitCast) &&
          "Only bitcasts expected to alter src/dst element counts");
 
+  // Check for bitcasting of unscalable vector types.
+  // e.g. <32 x i40> -> <40 x i32>
+  if (NumDstElts != NumSrcElts && (NumSrcElts % NumDstElts) != 0 &&
+      (NumDstElts % NumSrcElts) != 0)
+    return false;
+
   SmallVector<int, 16> NewMask;
   if (NumSrcElts >= NumDstElts) {
     // The bitcast is from wide to narrow/equal elements. The shuffle mask can
diff --git a/llvm/test/Transforms/VectorCombine/X86/shuffle-of-casts.ll b/llvm/test/Transforms/VectorCombine/X86/shuffle-of-casts.ll
index 4f8cfea146ea..7d9f7e390b9c 100644
--- a/llvm/test/Transforms/VectorCombine/X86/shuffle-of-casts.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/shuffle-of-casts.ll
@@ -239,6 +239,21 @@ define <16 x i16> @revpair_bitcast_v4i32_v16i16(<4 x i32> %a0, <4 x i32> %a1) {
   ret <16 x i16> %r
 }
 
+; negative - bitcasts (unscalable element counts)
+
+define <4 x i32> @shuffle_bitcast_v32i40_v4i32(<32 x i40> %a0, <32 x i40> %a1) {
+; CHECK-LABEL: @shuffle_bitcast_v32i40_v4i32(
+; CHECK-NEXT:    [[X0:%.*]] = bitcast <32 x i40> [[A0:%.*]] to <40 x i32>
+; CHECK-NEXT:    [[X1:%.*]] = bitcast <32 x i40> [[A1:%.*]] to <40 x i32>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <40 x i32> [[X0]], <40 x i32> [[X1]], <4 x i32> <i32 0, i32 42, i32 poison, i32 poison>
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %x0 = bitcast <32 x i40> %a0 to <40 x i32>
+  %x1 = bitcast <32 x i40> %a1 to <40 x i32>
+  %r = shufflevector <40 x i32> %x0, <40 x i32> %x1, <4 x i32> <i32 0, i32 42, i32 poison, i32 poison>
+  ret <4 x i32> %r
+}
+
 ; negative - src type mismatch
 
 define <8 x i32> @concat_sext_v4i8_v4i16_v8i32(<4 x i8> %a0, <4 x i16> %a1) {
-- 
GitLab


From 3fa830804e1b90d38d0a62084b5b65e70c83c345 Mon Sep 17 00:00:00 2001
From: Brandon Wu <brandon.wu@sifive.com>
Date: Fri, 12 Apr 2024 21:11:45 +0800
Subject: [PATCH 059/464] Revert "[RISCV] RISCV vector calling convention (2/2)
 (#79096)" (#88511)

This reverts commit 29e8bfc13c6078ed07e6474e8c9634c42aa2f6f4.
This patch didn't handle vector return type correctly.
---
 .../Target/RISCV/GISel/RISCVCallLowering.cpp  |  55 +++---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   | 180 ++++--------------
 llvm/lib/Target/RISCV/RISCVISelLowering.h     |  56 +-----
 llvm/test/CodeGen/RISCV/rvv/calling-conv.ll   |  87 ---------
 .../RISCV/rvv/vector-deinterleave-load.ll     |   6 +-
 .../CodeGen/RISCV/rvv/vector-deinterleave.ll  |  19 +-
 6 files changed, 83 insertions(+), 320 deletions(-)

diff --git a/llvm/lib/Target/RISCV/GISel/RISCVCallLowering.cpp b/llvm/lib/Target/RISCV/GISel/RISCVCallLowering.cpp
index 8af4bc658409..45e19cdea300 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVCallLowering.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVCallLowering.cpp
@@ -34,15 +34,14 @@ private:
   // Whether this is assigning args for a return.
   bool IsRet;
 
-  RVVArgDispatcher &RVVDispatcher;
+  // true if assignArg has been called for a mask argument, false otherwise.
+  bool AssignedFirstMaskArg = false;
 
 public:
   RISCVOutgoingValueAssigner(
-      RISCVTargetLowering::RISCVCCAssignFn *RISCVAssignFn_, bool IsRet,
-      RVVArgDispatcher &RVVDispatcher)
+      RISCVTargetLowering::RISCVCCAssignFn *RISCVAssignFn_, bool IsRet)
       : CallLowering::OutgoingValueAssigner(nullptr),
-        RISCVAssignFn(RISCVAssignFn_), IsRet(IsRet),
-        RVVDispatcher(RVVDispatcher) {}
+        RISCVAssignFn(RISCVAssignFn_), IsRet(IsRet) {}
 
   bool assignArg(unsigned ValNo, EVT OrigVT, MVT ValVT, MVT LocVT,
                  CCValAssign::LocInfo LocInfo,
@@ -52,9 +51,16 @@ public:
     const DataLayout &DL = MF.getDataLayout();
     const RISCVSubtarget &Subtarget = MF.getSubtarget<RISCVSubtarget>();
 
+    std::optional<unsigned> FirstMaskArgument;
+    if (Subtarget.hasVInstructions() && !AssignedFirstMaskArg &&
+        ValVT.isVector() && ValVT.getVectorElementType() == MVT::i1) {
+      FirstMaskArgument = ValNo;
+      AssignedFirstMaskArg = true;
+    }
+
     if (RISCVAssignFn(DL, Subtarget.getTargetABI(), ValNo, ValVT, LocVT,
                       LocInfo, Flags, State, Info.IsFixed, IsRet, Info.Ty,
-                      *Subtarget.getTargetLowering(), RVVDispatcher))
+                      *Subtarget.getTargetLowering(), FirstMaskArgument))
       return true;
 
     StackSize = State.getStackSize();
@@ -175,15 +181,14 @@ private:
   // Whether this is assigning args from a return.
   bool IsRet;
 
-  RVVArgDispatcher &RVVDispatcher;
+  // true if assignArg has been called for a mask argument, false otherwise.
+  bool AssignedFirstMaskArg = false;
 
 public:
   RISCVIncomingValueAssigner(
-      RISCVTargetLowering::RISCVCCAssignFn *RISCVAssignFn_, bool IsRet,
-      RVVArgDispatcher &RVVDispatcher)
+      RISCVTargetLowering::RISCVCCAssignFn *RISCVAssignFn_, bool IsRet)
       : CallLowering::IncomingValueAssigner(nullptr),
-        RISCVAssignFn(RISCVAssignFn_), IsRet(IsRet),
-        RVVDispatcher(RVVDispatcher) {}
+        RISCVAssignFn(RISCVAssignFn_), IsRet(IsRet) {}
 
   bool assignArg(unsigned ValNo, EVT OrigVT, MVT ValVT, MVT LocVT,
                  CCValAssign::LocInfo LocInfo,
@@ -196,9 +201,16 @@ public:
     if (LocVT.isScalableVector())
       MF.getInfo<RISCVMachineFunctionInfo>()->setIsVectorCall();
 
+    std::optional<unsigned> FirstMaskArgument;
+    if (Subtarget.hasVInstructions() && !AssignedFirstMaskArg &&
+        ValVT.isVector() && ValVT.getVectorElementType() == MVT::i1) {
+      FirstMaskArgument = ValNo;
+      AssignedFirstMaskArg = true;
+    }
+
     if (RISCVAssignFn(DL, Subtarget.getTargetABI(), ValNo, ValVT, LocVT,
                       LocInfo, Flags, State, /*IsFixed=*/true, IsRet, Info.Ty,
-                      *Subtarget.getTargetLowering(), RVVDispatcher))
+                      *Subtarget.getTargetLowering(), FirstMaskArgument))
       return true;
 
     StackSize = State.getStackSize();
@@ -408,11 +420,9 @@ bool RISCVCallLowering::lowerReturnVal(MachineIRBuilder &MIRBuilder,
   SmallVector<ArgInfo, 4> SplitRetInfos;
   splitToValueTypes(OrigRetInfo, SplitRetInfos, DL, CC);
 
-  RVVArgDispatcher Dispatcher{&MF, getTLI<RISCVTargetLowering>(),
-                              F.getReturnType()};
   RISCVOutgoingValueAssigner Assigner(
       CC == CallingConv::Fast ? RISCV::CC_RISCV_FastCC : RISCV::CC_RISCV,
-      /*IsRet=*/true, Dispatcher);
+      /*IsRet=*/true);
   RISCVOutgoingValueHandler Handler(MIRBuilder, MF.getRegInfo(), Ret);
   return determineAndHandleAssignments(Handler, Assigner, SplitRetInfos,
                                        MIRBuilder, CC, F.isVarArg());
@@ -521,7 +531,6 @@ bool RISCVCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
   CallingConv::ID CC = F.getCallingConv();
 
   SmallVector<ArgInfo, 32> SplitArgInfos;
-  SmallVector<Type *, 4> TypeList;
   unsigned Index = 0;
   for (auto &Arg : F.args()) {
     // Construct the ArgInfo object from destination register and argument type.
@@ -533,15 +542,12 @@ bool RISCVCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
     // correspondingly and appended to SplitArgInfos.
     splitToValueTypes(AInfo, SplitArgInfos, DL, CC);
 
-    TypeList.push_back(Arg.getType());
-
     ++Index;
   }
 
-  RVVArgDispatcher Dispatcher{&MF, getTLI<RISCVTargetLowering>(), TypeList};
   RISCVIncomingValueAssigner Assigner(
       CC == CallingConv::Fast ? RISCV::CC_RISCV_FastCC : RISCV::CC_RISCV,
-      /*IsRet=*/false, Dispatcher);
+      /*IsRet=*/false);
   RISCVFormalArgHandler Handler(MIRBuilder, MF.getRegInfo());
 
   SmallVector<CCValAssign, 16> ArgLocs;
@@ -579,13 +585,11 @@ bool RISCVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
 
   SmallVector<ArgInfo, 32> SplitArgInfos;
   SmallVector<ISD::OutputArg, 8> Outs;
-  SmallVector<Type *, 4> TypeList;
   for (auto &AInfo : Info.OrigArgs) {
     // Handle any required unmerging of split value types from a given VReg into
     // physical registers. ArgInfo objects are constructed correspondingly and
     // appended to SplitArgInfos.
     splitToValueTypes(AInfo, SplitArgInfos, DL, CC);
-    TypeList.push_back(AInfo.Ty);
   }
 
   // TODO: Support tail calls.
@@ -603,10 +607,9 @@ bool RISCVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
   const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
   Call.addRegMask(TRI->getCallPreservedMask(MF, Info.CallConv));
 
-  RVVArgDispatcher ArgDispatcher{&MF, getTLI<RISCVTargetLowering>(), TypeList};
   RISCVOutgoingValueAssigner ArgAssigner(
       CC == CallingConv::Fast ? RISCV::CC_RISCV_FastCC : RISCV::CC_RISCV,
-      /*IsRet=*/false, ArgDispatcher);
+      /*IsRet=*/false);
   RISCVOutgoingValueHandler ArgHandler(MIRBuilder, MF.getRegInfo(), Call);
   if (!determineAndHandleAssignments(ArgHandler, ArgAssigner, SplitArgInfos,
                                      MIRBuilder, CC, Info.IsVarArg))
@@ -634,11 +637,9 @@ bool RISCVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
   SmallVector<ArgInfo, 4> SplitRetInfos;
   splitToValueTypes(Info.OrigRet, SplitRetInfos, DL, CC);
 
-  RVVArgDispatcher RetDispatcher{&MF, getTLI<RISCVTargetLowering>(),
-                                 F.getReturnType()};
   RISCVIncomingValueAssigner RetAssigner(
       CC == CallingConv::Fast ? RISCV::CC_RISCV_FastCC : RISCV::CC_RISCV,
-      /*IsRet=*/true, RetDispatcher);
+      /*IsRet=*/true);
   RISCVCallReturnHandler RetHandler(MIRBuilder, MF.getRegInfo(), Call);
   if (!determineAndHandleAssignments(RetHandler, RetAssigner, SplitRetInfos,
                                      MIRBuilder, CC, Info.IsVarArg))
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 3e7bc8c2367d..5a572002091f 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -22,7 +22,6 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/MemoryLocation.h"
 #include "llvm/Analysis/VectorUtils.h"
-#include "llvm/CodeGen/Analysis.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
@@ -18079,12 +18078,33 @@ static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1,
   return false;
 }
 
+static unsigned allocateRVVReg(MVT ValVT, unsigned ValNo,
+                               std::optional<unsigned> FirstMaskArgument,
+                               CCState &State, const RISCVTargetLowering &TLI) {
+  const TargetRegisterClass *RC = TLI.getRegClassFor(ValVT);
+  if (RC == &RISCV::VRRegClass) {
+    // Assign the first mask argument to V0.
+    // This is an interim calling convention and it may be changed in the
+    // future.
+    if (FirstMaskArgument && ValNo == *FirstMaskArgument)
+      return State.AllocateReg(RISCV::V0);
+    return State.AllocateReg(ArgVRs);
+  }
+  if (RC == &RISCV::VRM2RegClass)
+    return State.AllocateReg(ArgVRM2s);
+  if (RC == &RISCV::VRM4RegClass)
+    return State.AllocateReg(ArgVRM4s);
+  if (RC == &RISCV::VRM8RegClass)
+    return State.AllocateReg(ArgVRM8s);
+  llvm_unreachable("Unhandled register class for ValueType");
+}
+
 // Implements the RISC-V calling convention. Returns true upon failure.
 bool RISCV::CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo,
                      MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo,
                      ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed,
                      bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI,
-                     RVVArgDispatcher &RVVDispatcher) {
+                     std::optional<unsigned> FirstMaskArgument) {
   unsigned XLen = DL.getLargestLegalIntTypeSizeInBits();
   assert(XLen == 32 || XLen == 64);
   MVT XLenVT = XLen == 32 ? MVT::i32 : MVT::i64;
@@ -18253,7 +18273,7 @@ bool RISCV::CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo,
   else if (ValVT == MVT::f64 && !UseGPRForF64)
     Reg = State.AllocateReg(ArgFPR64s);
   else if (ValVT.isVector()) {
-    Reg = RVVDispatcher.getNextPhysReg();
+    Reg = allocateRVVReg(ValVT, ValNo, FirstMaskArgument, State, TLI);
     if (!Reg) {
       // For return values, the vector must be passed fully via registers or
       // via the stack.
@@ -18339,13 +18359,9 @@ void RISCVTargetLowering::analyzeInputArgs(
   unsigned NumArgs = Ins.size();
   FunctionType *FType = MF.getFunction().getFunctionType();
 
-  SmallVector<Type *, 4> TypeList;
-  if (IsRet)
-    TypeList.push_back(MF.getFunction().getReturnType());
-  else
-    for (const Argument &Arg : MF.getFunction().args())
-      TypeList.push_back(Arg.getType());
-  RVVArgDispatcher Dispatcher{&MF, this, TypeList};
+  std::optional<unsigned> FirstMaskArgument;
+  if (Subtarget.hasVInstructions())
+    FirstMaskArgument = preAssignMask(Ins);
 
   for (unsigned i = 0; i != NumArgs; ++i) {
     MVT ArgVT = Ins[i].VT;
@@ -18360,7 +18376,7 @@ void RISCVTargetLowering::analyzeInputArgs(
     RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
     if (Fn(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full,
            ArgFlags, CCInfo, /*IsFixed=*/true, IsRet, ArgTy, *this,
-           Dispatcher)) {
+           FirstMaskArgument)) {
       LLVM_DEBUG(dbgs() << "InputArg #" << i << " has unhandled type "
                         << ArgVT << '\n');
       llvm_unreachable(nullptr);
@@ -18374,13 +18390,9 @@ void RISCVTargetLowering::analyzeOutputArgs(
     CallLoweringInfo *CLI, RISCVCCAssignFn Fn) const {
   unsigned NumArgs = Outs.size();
 
-  SmallVector<Type *, 4> TypeList;
-  if (IsRet)
-    TypeList.push_back(MF.getFunction().getReturnType());
-  else if (CLI)
-    for (const TargetLowering::ArgListEntry &Arg : CLI->getArgs())
-      TypeList.push_back(Arg.Ty);
-  RVVArgDispatcher Dispatcher{&MF, this, TypeList};
+  std::optional<unsigned> FirstMaskArgument;
+  if (Subtarget.hasVInstructions())
+    FirstMaskArgument = preAssignMask(Outs);
 
   for (unsigned i = 0; i != NumArgs; i++) {
     MVT ArgVT = Outs[i].VT;
@@ -18390,7 +18402,7 @@ void RISCVTargetLowering::analyzeOutputArgs(
     RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
     if (Fn(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full,
            ArgFlags, CCInfo, Outs[i].IsFixed, IsRet, OrigTy, *this,
-           Dispatcher)) {
+           FirstMaskArgument)) {
       LLVM_DEBUG(dbgs() << "OutputArg #" << i << " has unhandled type "
                         << ArgVT << "\n");
       llvm_unreachable(nullptr);
@@ -18571,7 +18583,7 @@ bool RISCV::CC_RISCV_FastCC(const DataLayout &DL, RISCVABI::ABI ABI,
                             ISD::ArgFlagsTy ArgFlags, CCState &State,
                             bool IsFixed, bool IsRet, Type *OrigTy,
                             const RISCVTargetLowering &TLI,
-                            RVVArgDispatcher &RVVDispatcher) {
+                            std::optional<unsigned> FirstMaskArgument) {
   if (LocVT == MVT::i32 || LocVT == MVT::i64) {
     if (unsigned Reg = State.AllocateReg(getFastCCArgGPRs(ABI))) {
       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
@@ -18649,14 +18661,13 @@ bool RISCV::CC_RISCV_FastCC(const DataLayout &DL, RISCVABI::ABI ABI,
   }
 
   if (LocVT.isVector()) {
-    MCPhysReg AllocatedVReg = RVVDispatcher.getNextPhysReg();
-    if (AllocatedVReg) {
+    if (unsigned Reg =
+            allocateRVVReg(ValVT, ValNo, FirstMaskArgument, State, TLI)) {
       // Fixed-length vectors are located in the corresponding scalable-vector
       // container types.
       if (ValVT.isFixedLengthVector())
         LocVT = TLI.getContainerForFixedLengthVector(LocVT);
-      State.addLoc(
-          CCValAssign::getReg(ValNo, ValVT, AllocatedVReg, LocVT, LocInfo));
+      State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
     } else {
       // Try and pass the address via a "fast" GPR.
       if (unsigned GPRReg = State.AllocateReg(getFastCCArgGPRs(ABI))) {
@@ -19284,15 +19295,17 @@ bool RISCVTargetLowering::CanLowerReturn(
   SmallVector<CCValAssign, 16> RVLocs;
   CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
 
-  RVVArgDispatcher Dispatcher{&MF, this, MF.getFunction().getReturnType()};
+  std::optional<unsigned> FirstMaskArgument;
+  if (Subtarget.hasVInstructions())
+    FirstMaskArgument = preAssignMask(Outs);
 
   for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
     MVT VT = Outs[i].VT;
     ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
     RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
     if (RISCV::CC_RISCV(MF.getDataLayout(), ABI, i, VT, VT, CCValAssign::Full,
-                        ArgFlags, CCInfo, /*IsFixed=*/true, /*IsRet=*/true,
-                        nullptr, *this, Dispatcher))
+                 ArgFlags, CCInfo, /*IsFixed=*/true, /*IsRet=*/true, nullptr,
+                 *this, FirstMaskArgument))
       return false;
   }
   return true;
@@ -21089,119 +21102,6 @@ unsigned RISCVTargetLowering::getMinimumJumpTableEntries() const {
   return Subtarget.getMinimumJumpTableEntries();
 }
 
-void RVVArgDispatcher::constructArgInfos(ArrayRef<Type *> TypeList) {
-  const DataLayout &DL = MF->getDataLayout();
-  const Function &F = MF->getFunction();
-  LLVMContext &Context = F.getContext();
-
-  bool FirstVMaskAssigned = false;
-  for (Type *Ty : TypeList) {
-    StructType *STy = dyn_cast<StructType>(Ty);
-    if (STy && STy->containsHomogeneousScalableVectorTypes()) {
-      Type *ElemTy = STy->getTypeAtIndex(0U);
-      EVT VT = TLI->getValueType(DL, ElemTy);
-      MVT RegisterVT =
-          TLI->getRegisterTypeForCallingConv(Context, F.getCallingConv(), VT);
-
-      RVVArgInfos.push_back({STy->getNumElements(), RegisterVT, false});
-    } else {
-      SmallVector<EVT, 4> ValueVTs;
-      ComputeValueVTs(*TLI, DL, Ty, ValueVTs);
-
-      for (unsigned Value = 0, NumValues = ValueVTs.size(); Value != NumValues;
-           ++Value) {
-        EVT VT = ValueVTs[Value];
-        MVT RegisterVT =
-            TLI->getRegisterTypeForCallingConv(Context, F.getCallingConv(), VT);
-        unsigned NumRegs =
-            TLI->getNumRegistersForCallingConv(Context, F.getCallingConv(), VT);
-
-        // Skip non-RVV register type
-        if (!RegisterVT.isVector())
-          continue;
-
-        if (RegisterVT.isFixedLengthVector())
-          RegisterVT = TLI->getContainerForFixedLengthVector(RegisterVT);
-
-        if (!FirstVMaskAssigned &&
-            RegisterVT.getVectorElementType() == MVT::i1) {
-          RVVArgInfos.push_back({1, RegisterVT, true});
-          FirstVMaskAssigned = true;
-          --NumRegs;
-        }
-
-        RVVArgInfos.insert(RVVArgInfos.end(), NumRegs, {1, RegisterVT, false});
-      }
-    }
-  }
-}
-
-void RVVArgDispatcher::allocatePhysReg(unsigned NF, unsigned LMul,
-                                       unsigned StartReg) {
-  assert((StartReg % LMul) == 0 &&
-         "Start register number should be multiple of lmul");
-  const MCPhysReg *VRArrays;
-  switch (LMul) {
-  default:
-    report_fatal_error("Invalid lmul");
-  case 1:
-    VRArrays = ArgVRs;
-    break;
-  case 2:
-    VRArrays = ArgVRM2s;
-    break;
-  case 4:
-    VRArrays = ArgVRM4s;
-    break;
-  case 8:
-    VRArrays = ArgVRM8s;
-    break;
-  }
-
-  for (unsigned i = 0; i < NF; ++i)
-    if (StartReg)
-      AllocatedPhysRegs.push_back(VRArrays[(StartReg - 8) / LMul + i]);
-    else
-      AllocatedPhysRegs.push_back(MCPhysReg());
-}
-
-/// This function determines if each RVV argument is passed by register, if the
-/// argument can be assigned to a VR, then give it a specific register.
-/// Otherwise, assign the argument to 0 which is a invalid MCPhysReg.
-void RVVArgDispatcher::compute() {
-  uint32_t AssignedMap = 0;
-  auto allocate = [&](const RVVArgInfo &ArgInfo) {
-    // Allocate first vector mask argument to V0.
-    if (ArgInfo.FirstVMask) {
-      AllocatedPhysRegs.push_back(RISCV::V0);
-      return;
-    }
-
-    unsigned RegsNeeded = divideCeil(
-        ArgInfo.VT.getSizeInBits().getKnownMinValue(), RISCV::RVVBitsPerBlock);
-    unsigned TotalRegsNeeded = ArgInfo.NF * RegsNeeded;
-    for (unsigned StartReg = 0; StartReg + TotalRegsNeeded <= NumArgVRs;
-         StartReg += RegsNeeded) {
-      uint32_t Map = ((1 << TotalRegsNeeded) - 1) << StartReg;
-      if ((AssignedMap & Map) == 0) {
-        allocatePhysReg(ArgInfo.NF, RegsNeeded, StartReg + 8);
-        AssignedMap |= Map;
-        return;
-      }
-    }
-
-    allocatePhysReg(ArgInfo.NF, RegsNeeded, 0);
-  };
-
-  for (unsigned i = 0; i < RVVArgInfos.size(); ++i)
-    allocate(RVVArgInfos[i]);
-}
-
-MCPhysReg RVVArgDispatcher::getNextPhysReg() {
-  assert(CurIdx < AllocatedPhysRegs.size() && "Index out of range");
-  return AllocatedPhysRegs[CurIdx++];
-}
-
 namespace llvm::RISCVVIntrinsicsTable {
 
 #define GET_RISCVVIntrinsicsTable_IMPL
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index c28552354bf4..ace5b3fd2b95 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -24,7 +24,6 @@ namespace llvm {
 class InstructionCost;
 class RISCVSubtarget;
 struct RISCVRegisterInfo;
-class RVVArgDispatcher;
 
 namespace RISCVISD {
 // clang-format off
@@ -876,7 +875,7 @@ public:
                                ISD::ArgFlagsTy ArgFlags, CCState &State,
                                bool IsFixed, bool IsRet, Type *OrigTy,
                                const RISCVTargetLowering &TLI,
-                               RVVArgDispatcher &RVVDispatcher);
+                               std::optional<unsigned> FirstMaskArgument);
 
 private:
   void analyzeInputArgs(MachineFunction &MF, CCState &CCInfo,
@@ -1016,68 +1015,19 @@ private:
   unsigned getMinimumJumpTableEntries() const override;
 };
 
-/// As per the spec, the rules for passing vector arguments are as follows:
-///
-/// 1. For the first vector mask argument, use v0 to pass it.
-/// 2. For vector data arguments or rest vector mask arguments, starting from
-/// the v8 register, if a vector register group between v8-v23 that has not been
-/// allocated can be found and the first register number is a multiple of LMUL,
-/// then allocate this vector register group to the argument and mark these
-/// registers as allocated. Otherwise, pass it by reference and are replaced in
-/// the argument list with the address.
-/// 3. For tuple vector data arguments, starting from the v8 register, if
-/// NFIELDS consecutive vector register groups between v8-v23 that have not been
-/// allocated can be found and the first register number is a multiple of LMUL,
-/// then allocate these vector register groups to the argument and mark these
-/// registers as allocated. Otherwise, pass it by reference and are replaced in
-/// the argument list with the address.
-class RVVArgDispatcher {
-public:
-  static constexpr unsigned NumArgVRs = 16;
-
-  struct RVVArgInfo {
-    unsigned NF;
-    MVT VT;
-    bool FirstVMask = false;
-  };
-
-  RVVArgDispatcher(const MachineFunction *MF, const RISCVTargetLowering *TLI,
-                   ArrayRef<Type *> TypeList)
-      : MF(MF), TLI(TLI) {
-    constructArgInfos(TypeList);
-    compute();
-  }
-
-  MCPhysReg getNextPhysReg();
-
-private:
-  SmallVector<RVVArgInfo, 4> RVVArgInfos;
-  SmallVector<MCPhysReg, 4> AllocatedPhysRegs;
-
-  const MachineFunction *MF = nullptr;
-  const RISCVTargetLowering *TLI = nullptr;
-
-  unsigned CurIdx = 0;
-
-  void constructArgInfos(ArrayRef<Type *> TypeList);
-  void compute();
-  void allocatePhysReg(unsigned NF = 1, unsigned LMul = 1,
-                       unsigned StartReg = 0);
-};
-
 namespace RISCV {
 
 bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo,
               MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo,
               ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed,
               bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI,
-              RVVArgDispatcher &RVVDispatcher);
+              std::optional<unsigned> FirstMaskArgument);
 
 bool CC_RISCV_FastCC(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo,
                      MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo,
                      ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed,
                      bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI,
-                     RVVArgDispatcher &RVVDispatcher);
+                     std::optional<unsigned> FirstMaskArgument);
 
 bool CC_RISCV_GHC(unsigned ValNo, MVT ValVT, MVT LocVT,
                   CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
diff --git a/llvm/test/CodeGen/RISCV/rvv/calling-conv.ll b/llvm/test/CodeGen/RISCV/rvv/calling-conv.ll
index 90edb994ce82..78e8700a9fef 100644
--- a/llvm/test/CodeGen/RISCV/rvv/calling-conv.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/calling-conv.ll
@@ -162,90 +162,3 @@ define void @caller_tuple_argument({<vscale x 4 x i32>, <vscale x 4 x i32>} %x)
 }
 
 declare void @callee_tuple_argument({<vscale x 4 x i32>, <vscale x 4 x i32>})
-
-; %0 -> v8
-; %1 -> v9
-define <vscale x 1 x i64> @case1(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1) {
-; CHECK-LABEL: case1:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
-; CHECK-NEXT:    vadd.vv v8, v8, v9
-; CHECK-NEXT:    ret
-  %a = add <vscale x 1 x i64> %0, %1
-  ret <vscale x 1 x i64> %a
-}
-
-; %0 -> v8
-; %1 -> v10-v11
-; %2 -> v9
-define <vscale x 1 x i64> @case2_1(<vscale x 1 x i64> %0, <vscale x 2 x i64> %1, <vscale x 1 x i64> %2) {
-; CHECK-LABEL: case2_1:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
-; CHECK-NEXT:    vadd.vv v8, v8, v9
-; CHECK-NEXT:    ret
-  %a = add <vscale x 1 x i64> %0, %2
-  ret <vscale x 1 x i64> %a
-}
-define <vscale x 2 x i64> @case2_2(<vscale x 1 x i64> %0, <vscale x 2 x i64> %1, <vscale x 1 x i64> %2) {
-; CHECK-LABEL: case2_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64, m2, ta, ma
-; CHECK-NEXT:    vadd.vv v8, v10, v10
-; CHECK-NEXT:    ret
-  %a = add <vscale x 2 x i64> %1, %1
-  ret <vscale x 2 x i64> %a
-}
-
-; %0 -> v8
-; %1 -> {v10-v11, v12-v13}
-; %2 -> v9
-define <vscale x 1 x i64> @case3_1(<vscale x 1 x i64> %0, {<vscale x 2 x i64>, <vscale x 2 x i64>} %1, <vscale x 1 x i64> %2) {
-; CHECK-LABEL: case3_1:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
-; CHECK-NEXT:    vadd.vv v8, v8, v9
-; CHECK-NEXT:    ret
-  %add = add <vscale x 1 x i64> %0, %2
-  ret <vscale x 1 x i64> %add
-}
-define <vscale x 2 x i64> @case3_2(<vscale x 1 x i64> %0, {<vscale x 2 x i64>, <vscale x 2 x i64>} %1, <vscale x 1 x i64> %2) {
-; CHECK-LABEL: case3_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64, m2, ta, ma
-; CHECK-NEXT:    vadd.vv v8, v10, v12
-; CHECK-NEXT:    ret
-  %a = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } %1, 0
-  %b = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } %1, 1
-  %add = add <vscale x 2 x i64> %a, %b
-  ret <vscale x 2 x i64> %add
-}
-
-; %0 -> v8
-; %1 -> {by-ref, by-ref}
-; %2 -> v9
-define <vscale x 8 x i64> @case4_1(<vscale x 1 x i64> %0, {<vscale x 8 x i64>, <vscale x 8 x i64>} %1, <vscale x 1 x i64> %2) {
-; CHECK-LABEL: case4_1:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    slli a1, a1, 3
-; CHECK-NEXT:    add a1, a0, a1
-; CHECK-NEXT:    vl8re64.v v8, (a1)
-; CHECK-NEXT:    vl8re64.v v16, (a0)
-; CHECK-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
-; CHECK-NEXT:    vadd.vv v8, v16, v8
-; CHECK-NEXT:    ret
-  %a = extractvalue { <vscale x 8 x i64>, <vscale x 8 x i64> } %1, 0
-  %b = extractvalue { <vscale x 8 x i64>, <vscale x 8 x i64> } %1, 1
-  %add = add <vscale x 8 x i64> %a, %b
-  ret <vscale x 8 x i64> %add
-}
-define <vscale x 1 x i64> @case4_2(<vscale x 1 x i64> %0, {<vscale x 8 x i64>, <vscale x 8 x i64>} %1, <vscale x 1 x i64> %2) {
-; CHECK-LABEL: case4_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
-; CHECK-NEXT:    vadd.vv v8, v8, v9
-; CHECK-NEXT:    ret
-  %add = add <vscale x 1 x i64> %0, %2
-  ret <vscale x 1 x i64> %add
-}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll
index 6a712080fda7..a320aecc6fce 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll
@@ -18,10 +18,10 @@ define {<vscale x 16 x i1>, <vscale x 16 x i1>} @vector_deinterleave_load_nxv16i
 ; CHECK-NEXT:    vmerge.vim v14, v10, 1, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    vmerge.vim v12, v10, 1, v0
-; CHECK-NEXT:    vnsrl.wi v10, v12, 0
-; CHECK-NEXT:    vmsne.vi v8, v10, 0
+; CHECK-NEXT:    vnsrl.wi v8, v12, 0
+; CHECK-NEXT:    vmsne.vi v0, v8, 0
 ; CHECK-NEXT:    vnsrl.wi v10, v12, 8
-; CHECK-NEXT:    vmsne.vi v9, v10, 0
+; CHECK-NEXT:    vmsne.vi v8, v10, 0
 ; CHECK-NEXT:    ret
   %vec = load <vscale x 32 x i1>, ptr %p
   %retval = call {<vscale x 16 x i1>, <vscale x 16 x i1>} @llvm.experimental.vector.deinterleave2.nxv32i1(<vscale x 32 x i1> %vec)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll
index d98597fabcd9..ef4baf34d23f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll
@@ -8,18 +8,18 @@ define {<vscale x 16 x i1>, <vscale x 16 x i1>} @vector_deinterleave_nxv16i1_nxv
 ; CHECK-LABEL: vector_deinterleave_nxv16i1_nxv32i1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
-; CHECK-NEXT:    vmv.v.i v8, 0
-; CHECK-NEXT:    vmerge.vim v12, v8, 1, v0
+; CHECK-NEXT:    vmv.v.i v10, 0
+; CHECK-NEXT:    vmerge.vim v8, v10, 1, v0
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 2
 ; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a0
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
-; CHECK-NEXT:    vmerge.vim v14, v8, 1, v0
-; CHECK-NEXT:    vnsrl.wi v10, v12, 0
-; CHECK-NEXT:    vmsne.vi v8, v10, 0
-; CHECK-NEXT:    vnsrl.wi v10, v12, 8
-; CHECK-NEXT:    vmsne.vi v9, v10, 0
+; CHECK-NEXT:    vmerge.vim v10, v10, 1, v0
+; CHECK-NEXT:    vnsrl.wi v12, v8, 0
+; CHECK-NEXT:    vmsne.vi v0, v12, 0
+; CHECK-NEXT:    vnsrl.wi v12, v8, 8
+; CHECK-NEXT:    vmsne.vi v8, v12, 0
 ; CHECK-NEXT:    ret
 %retval = call {<vscale x 16 x i1>, <vscale x 16 x i1>} @llvm.experimental.vector.deinterleave2.nxv32i1(<vscale x 32 x i1> %vec)
 ret {<vscale x 16 x i1>, <vscale x 16 x i1>} %retval
@@ -102,13 +102,12 @@ define {<vscale x 64 x i1>, <vscale x 64 x i1>} @vector_deinterleave_nxv64i1_nxv
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m4, ta, ma
 ; CHECK-NEXT:    vnsrl.wi v28, v8, 0
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
-; CHECK-NEXT:    vmsne.vi v7, v24, 0
+; CHECK-NEXT:    vmsne.vi v0, v24, 0
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m4, ta, ma
 ; CHECK-NEXT:    vnsrl.wi v24, v16, 8
 ; CHECK-NEXT:    vnsrl.wi v28, v8, 8
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
-; CHECK-NEXT:    vmsne.vi v9, v24, 0
-; CHECK-NEXT:    vmv1r.v v8, v7
+; CHECK-NEXT:    vmsne.vi v8, v24, 0
 ; CHECK-NEXT:    ret
 %retval = call {<vscale x 64 x i1>, <vscale x 64 x i1>} @llvm.experimental.vector.deinterleave2.nxv128i1(<vscale x 128 x i1> %vec)
 ret {<vscale x 64 x i1>, <vscale x 64 x i1>} %retval
-- 
GitLab


From b4df0da9e8421c8026bd09980b8a6acd7a6ce8c9 Mon Sep 17 00:00:00 2001
From: Shilei Tian <i@tianshilei.me>
Date: Fri, 12 Apr 2024 09:34:46 -0400
Subject: [PATCH 060/464] [AMDGPU] Fix a potential wrong return value
 indicating whether a pass modifies a function (#88197)

When the alloca is too big for vectorization, the function could have
already
been modified in previous iteration of the `for` loop.
---
 llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp    |  2 +-
 llvm/test/CodeGen/AMDGPU/half-alloca-promotion.ll | 11 +++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/half-alloca-promotion.ll

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
index 6f3cdf54dcee..c0846b123d18 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
@@ -336,7 +336,7 @@ bool AMDGPUPromoteAllocaImpl::run(Function &F, bool PromoteToLDS) {
     if (AllocaCost > VectorizationBudget) {
       LLVM_DEBUG(dbgs() << "  Alloca too big for vectorization: " << *AI
                         << "\n");
-      return false;
+      return Changed;
     }
 
     if (tryPromoteAllocaToVector(*AI)) {
diff --git a/llvm/test/CodeGen/AMDGPU/half-alloca-promotion.ll b/llvm/test/CodeGen/AMDGPU/half-alloca-promotion.ll
new file mode 100644
index 000000000000..cfec49f3652f
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/half-alloca-promotion.ll
@@ -0,0 +1,11 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes="amdgpu-promote-alloca-to-vector" -o - %s
+; We don't really need to check anything here because with expensive check, this
+; test case crashes. The correctness of the pass is beyond the scope.
+
+define fastcc void @foo() {
+entry:
+  %det = alloca [4 x i32], align 16, addrspace(5)
+  %trkltPosTmpYZ = alloca [2 x float], align 4, addrspace(5)
+  %trkltCovTmp = alloca [3 x float], align 4, addrspace(5)
+  ret void
+}
-- 
GitLab


From 2a5ba4fb895931b7dc86f4304b0e64153a1175d4 Mon Sep 17 00:00:00 2001
From: Xiaoyang Liu <siujoeng.lau@gmail.com>
Date: Fri, 12 Apr 2024 10:04:21 -0400
Subject: [PATCH 061/464] [libc++] LWG3643: Missing `constexpr` in
 `std::counted_iterator` (#87901)

This pull request implements LWG3643: Missing constexpr in
std::counted_iterator. Specifically, one overload of
std::counted_operator::operator++ was not marked as constexpr,
despite being eligible for it after the introduction of try-block
support in constexpr functions in C++20.
---
 libcxx/docs/Status/Cxx23Issues.csv            |  2 +-
 libcxx/include/__iterator/counted_iterator.h  |  2 +-
 .../counted.iterator/increment.pass.cpp       | 45 +++++++++----------
 3 files changed, 24 insertions(+), 25 deletions(-)

diff --git a/libcxx/docs/Status/Cxx23Issues.csv b/libcxx/docs/Status/Cxx23Issues.csv
index a212d56685c0..fa82e67ac7db 100644
--- a/libcxx/docs/Status/Cxx23Issues.csv
+++ b/libcxx/docs/Status/Cxx23Issues.csv
@@ -150,7 +150,7 @@
 "`3619 <https://wg21.link/LWG3619>`__","Specification of ``vformat_to`` contains ill-formed ``formatted_size`` calls","February 2022","|Nothing to do|","","|format|"
 "`3621 <https://wg21.link/LWG3621>`__","Remove feature-test macro ``__cpp_lib_monadic_optional`` ","February 2022","|Complete|","15.0"
 "`3632 <https://wg21.link/LWG3632>`__","``unique_ptr`` ""Mandates: This constructor is not selected by class template argument deduction""","February 2022","|Nothing to do|",""
-"`3643 <https://wg21.link/LWG3643>`__","Missing ``constexpr`` in ``std::counted_iterator`` ","February 2022","","","|ranges|"
+"`3643 <https://wg21.link/LWG3643>`__","Missing ``constexpr`` in ``std::counted_iterator`` ","February 2022","|Complete|","19.0","|ranges|"
 "`3648 <https://wg21.link/LWG3648>`__","``format`` should not print ``bool`` with ``'c'`` ","February 2022","|Complete|","15.0","|format|"
 "`3649 <https://wg21.link/LWG3649>`__","[fund.ts.v2] Reinstate and bump ``__cpp_lib_experimental_memory_resource`` feature test macro","February 2022","",""
 "`3650 <https://wg21.link/LWG3650>`__","Are ``std::basic_string`` 's ``iterator`` and ``const_iterator`` constexpr iterators?","February 2022","|Nothing to do|",""
diff --git a/libcxx/include/__iterator/counted_iterator.h b/libcxx/include/__iterator/counted_iterator.h
index 008c52fa87ce..ea2832e3b978 100644
--- a/libcxx/include/__iterator/counted_iterator.h
+++ b/libcxx/include/__iterator/counted_iterator.h
@@ -129,7 +129,7 @@ public:
     return *this;
   }
 
-  _LIBCPP_HIDE_FROM_ABI decltype(auto) operator++(int) {
+  _LIBCPP_HIDE_FROM_ABI constexpr decltype(auto) operator++(int) {
     _LIBCPP_ASSERT_UNCATEGORIZED(__count_ > 0, "Iterator already at or past end.");
     --__count_;
 #  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
diff --git a/libcxx/test/std/iterators/predef.iterators/counted.iterator/increment.pass.cpp b/libcxx/test/std/iterators/predef.iterators/counted.iterator/increment.pass.cpp
index 66d90a418744..2fea2561ebf7 100644
--- a/libcxx/test/std/iterators/predef.iterators/counted.iterator/increment.pass.cpp
+++ b/libcxx/test/std/iterators/predef.iterators/counted.iterator/increment.pass.cpp
@@ -9,7 +9,7 @@
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 
 // constexpr counted_iterator& operator++();
-// decltype(auto) operator++(int);
+// constexpr decltype(auto) operator++(int);
 // constexpr counted_iterator operator++(int)
 //   requires forward_iterator<I>;
 
@@ -62,6 +62,26 @@ concept PlusEnabled = requires(Iter& iter) {
 constexpr bool test() {
   int buffer[8] = {1, 2, 3, 4, 5, 6, 7, 8};
 
+  {
+    using Counted = std::counted_iterator<InputOrOutputArchetype>;
+    std::counted_iterator iter(InputOrOutputArchetype{buffer}, 8);
+
+    iter++;
+    assert((++iter).base().ptr == buffer + 2);
+
+    ASSERT_SAME_TYPE(decltype(iter++), void);
+    ASSERT_SAME_TYPE(decltype(++iter), Counted&);
+  }
+  {
+    using Counted = std::counted_iterator<cpp20_input_iterator<int*>>;
+    std::counted_iterator iter(cpp20_input_iterator<int*>{buffer}, 8);
+
+    iter++;
+    assert(++iter == Counted(cpp20_input_iterator<int*>{buffer + 2}, 6));
+
+    ASSERT_SAME_TYPE(decltype(iter++), void);
+    ASSERT_SAME_TYPE(decltype(++iter), Counted&);
+  }
   {
     using Counted = std::counted_iterator<forward_iterator<int*>>;
     std::counted_iterator iter(forward_iterator<int*>{buffer}, 8);
@@ -95,29 +115,8 @@ int main(int, char**) {
   test();
   static_assert(test());
 
-  int buffer[8] = {1, 2, 3, 4, 5, 6, 7, 8};
-
-  {
-    using Counted = std::counted_iterator<InputOrOutputArchetype>;
-    std::counted_iterator iter(InputOrOutputArchetype{buffer}, 8);
-
-    iter++;
-    assert((++iter).base().ptr == buffer + 2);
-
-    ASSERT_SAME_TYPE(decltype(iter++), void);
-    ASSERT_SAME_TYPE(decltype(++iter), Counted&);
-  }
-  {
-    using Counted = std::counted_iterator<cpp20_input_iterator<int*>>;
-    std::counted_iterator iter(cpp20_input_iterator<int*>{buffer}, 8);
-
-    iter++;
-    assert(++iter == Counted(cpp20_input_iterator<int*>{buffer + 2}, 6));
-
-    ASSERT_SAME_TYPE(decltype(iter++), void);
-    ASSERT_SAME_TYPE(decltype(++iter), Counted&);
-  }
 #ifndef TEST_HAS_NO_EXCEPTIONS
+  int buffer[8] = {1, 2, 3, 4, 5, 6, 7, 8};
   {
     using Counted = std::counted_iterator<ThrowsOnInc<int*>>;
     std::counted_iterator iter(ThrowsOnInc<int*>{buffer}, 8);
-- 
GitLab


From 5fa58e28f34f4b5b41dd2869212ac917fb42b2e3 Mon Sep 17 00:00:00 2001
From: Michael Maitland <michaeltmaitland@gmail.com>
Date: Thu, 4 Apr 2024 06:55:47 -0700
Subject: [PATCH 062/464] [RISCV] Split PseudoVFADD, PseudoVFSUB, and
 PseudoVFRSUB by SEW

Co-authored-by: Wang Pengcheng <wangpengcheng.pp@bytedance.com>
---
 .../Target/RISCV/RISCVInstrInfoVPseudos.td    | 26 ++++++++++---------
 .../Target/RISCV/RISCVInstrInfoVSDPatterns.td |  6 ++---
 .../Target/RISCV/RISCVInstrInfoVVLPatterns.td |  6 ++---
 llvm/lib/Target/RISCV/RISCVSchedSiFive7.td    | 16 +++++++++---
 llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td | 16 +++++++++---
 llvm/lib/Target/RISCV/RISCVScheduleV.td       | 16 ++++++------
 6 files changed, 52 insertions(+), 34 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index cf9a31c23a06..ebe8a29fdfa8 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -2917,16 +2917,17 @@ multiclass VPseudoVMAX_VV_VF {
 
 multiclass VPseudoVALU_VV_VF_RM {
   foreach m = MxListF in {
-    defm "" : VPseudoBinaryFV_VV_RM<m>,
-              SchedBinary<"WriteVFALUV", "ReadVFALUV", "ReadVFALUV", m.MX,
-                          forceMergeOpRead=true>;
+    foreach e = SchedSEWSet<m.MX, isF=1>.val in
+      defm "" : VPseudoBinaryFV_VV_RM<m, "", sew=e>,
+                SchedBinary<"WriteVFALUV", "ReadVFALUV", "ReadVFALUV", m.MX, e,
+                            forceMergeOpRead=true>;
   }
 
   foreach f = FPList in {
     foreach m = f.MxList in {
-      defm "" : VPseudoBinaryV_VF_RM<m, f>,
+      defm "" : VPseudoBinaryV_VF_RM<m, f, "", sew=f.SEW>,
                 SchedBinary<"WriteVFALUF", "ReadVFALUV", "ReadVFALUF", m.MX,
-                            forceMergeOpRead=true>;
+                            f.SEW, forceMergeOpRead=true>;
     }
   }
 }
@@ -2934,9 +2935,9 @@ multiclass VPseudoVALU_VV_VF_RM {
 multiclass VPseudoVALU_VF_RM {
   foreach f = FPList in {
     foreach m = f.MxList in {
-      defm "" : VPseudoBinaryV_VF_RM<m, f>,
+      defm "" : VPseudoBinaryV_VF_RM<m, f, "", sew=f.SEW>,
                 SchedBinary<"WriteVFALUF", "ReadVFALUV", "ReadVFALUF", m.MX,
-                            forceMergeOpRead=true>;
+                            f.SEW, forceMergeOpRead=true>;
     }
   }
 }
@@ -7061,11 +7062,12 @@ defm : VPatBinaryV_WV_WX_WI_RM<"int_riscv_vnclip", "PseudoVNCLIP",
 //===----------------------------------------------------------------------===//
 // 13.2. Vector Single-Width Floating-Point Add/Subtract Instructions
 //===----------------------------------------------------------------------===//
-defm : VPatBinaryV_VV_VX_RM<"int_riscv_vfadd", "PseudoVFADD",
-                            AllFloatVectors>;
-defm : VPatBinaryV_VV_VX_RM<"int_riscv_vfsub", "PseudoVFSUB",
-                            AllFloatVectors>;
-defm : VPatBinaryV_VX_RM<"int_riscv_vfrsub", "PseudoVFRSUB", AllFloatVectors>;
+defm : VPatBinaryV_VV_VX_RM<"int_riscv_vfadd", "PseudoVFADD", AllFloatVectors,
+                            isSEWAware = 1>;
+defm : VPatBinaryV_VV_VX_RM<"int_riscv_vfsub", "PseudoVFSUB", AllFloatVectors,
+                            isSEWAware = 1>;
+defm : VPatBinaryV_VX_RM<"int_riscv_vfrsub", "PseudoVFRSUB", AllFloatVectors,
+                         isSEWAware = 1>;
 
 //===----------------------------------------------------------------------===//
 // 13.3. Vector Widening Floating-Point Add/Subtract Instructions
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
index da761ae85670..0d5cd93984af 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
@@ -1204,9 +1204,9 @@ foreach mti = AllMasks in {
 // 13. Vector Floating-Point Instructions
 
 // 13.2. Vector Single-Width Floating-Point Add/Subtract Instructions
-defm : VPatBinaryFPSDNode_VV_VF_RM<any_fadd, "PseudoVFADD">;
-defm : VPatBinaryFPSDNode_VV_VF_RM<any_fsub, "PseudoVFSUB">;
-defm : VPatBinaryFPSDNode_R_VF_RM<any_fsub, "PseudoVFRSUB">;
+defm : VPatBinaryFPSDNode_VV_VF_RM<any_fadd, "PseudoVFADD", isSEWAware=1>;
+defm : VPatBinaryFPSDNode_VV_VF_RM<any_fsub, "PseudoVFSUB", isSEWAware=1>;
+defm : VPatBinaryFPSDNode_R_VF_RM<any_fsub, "PseudoVFRSUB", isSEWAware=1>;
 
 // 13.3. Vector Widening Floating-Point Add/Subtract Instructions
 defm : VPatWidenBinaryFPSDNode_VV_VF_WV_WF_RM<fadd, "PseudoVFWADD">;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
index b721dcd98988..befe8d07e575 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
@@ -2425,9 +2425,9 @@ foreach vtiToWti = AllWidenableIntVectors in
 // 13. Vector Floating-Point Instructions
 
 // 13.2. Vector Single-Width Floating-Point Add/Subtract Instructions
-defm : VPatBinaryFPVL_VV_VF_RM<any_riscv_fadd_vl, "PseudoVFADD">;
-defm : VPatBinaryFPVL_VV_VF_RM<any_riscv_fsub_vl, "PseudoVFSUB">;
-defm : VPatBinaryFPVL_R_VF_RM<any_riscv_fsub_vl, "PseudoVFRSUB">;
+defm : VPatBinaryFPVL_VV_VF_RM<any_riscv_fadd_vl, "PseudoVFADD", isSEWAware=1>;
+defm : VPatBinaryFPVL_VV_VF_RM<any_riscv_fsub_vl, "PseudoVFSUB", isSEWAware=1>;
+defm : VPatBinaryFPVL_R_VF_RM<any_riscv_fsub_vl, "PseudoVFRSUB", isSEWAware=1>;
 
 // 13.3. Vector Widening Floating-Point Add/Subtract Instructions
 defm : VPatBinaryFPWVL_VV_VF_WV_WF_RM<riscv_vfwadd_vl, riscv_vfwadd_w_vl, "PseudoVFWADD">;
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
index 54a13889ee69..f14766c98439 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
@@ -732,12 +732,20 @@ foreach mx = SchedMxListW in {
 }
 
 // 13. Vector Floating-Point Instructions
+foreach mx = SchedMxListF in {
+  foreach sew = SchedSEWSet<mx, isF=1>.val in {
+    defvar Cycles = SiFive7GetCyclesDefault<mx>.c;
+    defvar IsWorstCase = SiFive7IsWorstCaseMXSEW<mx, sew, SchedMxListF, isF=1>.c;
+    let Latency = 8, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in {
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFALUV",  [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFALUF",  [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
+    }
+  }
+}
 foreach mx = SchedMxList in {
   defvar Cycles = SiFive7GetCyclesDefault<mx>.c;
   defvar IsWorstCase = SiFive7IsWorstCaseMX<mx, SchedMxList>.c;
   let Latency = 8, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in {
-    defm "" : LMULWriteResMX<"WriteVFALUV",      [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVFALUF",      [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFMulV",      [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFMulF",      [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFMulAddV",   [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
@@ -1137,8 +1145,8 @@ defm "" : LMULReadAdvanceW<"ReadVNClipV", 0>;
 defm "" : LMULReadAdvanceW<"ReadVNClipX", 0>;
 
 // 14. Vector Floating-Point Instructions
-defm "" : LMULReadAdvance<"ReadVFALUV", 0>;
-defm "" : LMULReadAdvance<"ReadVFALUF", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFALUV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFALUF", 0>;
 defm "" : LMULReadAdvanceFW<"ReadVFWALUV", 0>;
 defm "" : LMULReadAdvanceFW<"ReadVFWALUF", 0>;
 defm "" : LMULReadAdvance<"ReadVFMulV", 0>;
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
index 54016959d348..4d81bafdf88b 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
@@ -486,12 +486,20 @@ foreach mx = SchedMxList in {
 }
 
 // 13. Vector Floating-Point Instructions
+foreach mx = SchedMxListF in {
+  foreach sew = SchedSEWSet<mx, isF=1>.val in {
+    defvar LMulLat = SiFiveP600GetLMulCycles<mx>.c;
+    defvar IsWorstCase = SiFiveP600IsWorstCaseMXSEW<mx, sew, SchedMxListF, isF=1>.c;
+    let Latency = 6, ReleaseAtCycles = [LMulLat] in {
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFALUV",  [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFALUF",  [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
+    }
+  }
+}
 foreach mx = SchedMxList in {
   defvar LMulLat = SiFiveP600GetLMulCycles<mx>.c;
   defvar IsWorstCase = SiFiveP600IsWorstCaseMX<mx, SchedMxList>.c;
   let Latency = 6, ReleaseAtCycles = [LMulLat] in {
-    defm "" : LMULWriteResMX<"WriteVFALUV",    [SiFiveP600VectorArith], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVFALUF",    [SiFiveP600VectorArith], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFMulV",    [SiFiveP600VectorArith], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFMulF",    [SiFiveP600VectorArith], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFMulAddV", [SiFiveP600VectorArith], mx, IsWorstCase>;
@@ -925,8 +933,8 @@ defm "" : LMULReadAdvanceW<"ReadVNClipV", 0>;
 defm "" : LMULReadAdvanceW<"ReadVNClipX", 0>;
 
 // 14. Vector Floating-Point Instructions
-defm "" : LMULReadAdvance<"ReadVFALUV", 0>;
-defm "" : LMULReadAdvance<"ReadVFALUF", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFALUV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFALUF", 0>;
 defm "" : LMULReadAdvanceFW<"ReadVFWALUV", 0>;
 defm "" : LMULReadAdvanceFW<"ReadVFWALUF", 0>;
 defm "" : LMULReadAdvance<"ReadVFMulV", 0>;
diff --git a/llvm/lib/Target/RISCV/RISCVScheduleV.td b/llvm/lib/Target/RISCV/RISCVScheduleV.td
index 379622d4ca83..9b9c72e0767a 100644
--- a/llvm/lib/Target/RISCV/RISCVScheduleV.td
+++ b/llvm/lib/Target/RISCV/RISCVScheduleV.td
@@ -397,8 +397,8 @@ defm "" : LMULSchedWritesW<"WriteVNClipI">;
 
 // 13. Vector Floating-Point Instructions
 // 13.2. Vector Single-Width Floating-Point Add/Subtract Instructions
-defm "" : LMULSchedWrites<"WriteVFALUV">;
-defm "" : LMULSchedWrites<"WriteVFALUF">;
+defm "" : LMULSEWSchedWritesF<"WriteVFALUV">;
+defm "" : LMULSEWSchedWritesF<"WriteVFALUF">;
 // 13.3. Vector Widening Floating-Point Add/Subtract Instructions
 defm "" : LMULSchedWritesFW<"WriteVFWALUV">;
 defm "" : LMULSchedWritesFW<"WriteVFWALUF">;
@@ -622,8 +622,8 @@ defm "" : LMULSchedReadsW<"ReadVNClipX">;
 
 // 13. Vector Floating-Point Instructions
 // 13.2. Vector Single-Width Floating-Point Add/Subtract Instructions
-defm "" : LMULSchedReads<"ReadVFALUV">;
-defm "" : LMULSchedReads<"ReadVFALUF">;
+defm "" : LMULSEWSchedReadsF<"ReadVFALUV">;
+defm "" : LMULSEWSchedReadsF<"ReadVFALUF">;
 // 13.3. Vector Widening Floating-Point Add/Subtract Instructions
 defm "" : LMULSchedReadsFW<"ReadVFWALUV">;
 defm "" : LMULSchedReadsFW<"ReadVFWALUF">;
@@ -868,8 +868,8 @@ defm "" : LMULWriteResW<"WriteVNClipX", []>;
 defm "" : LMULWriteResW<"WriteVNClipI", []>;
 
 // 13. Vector Floating-Point Instructions
-defm "" : LMULWriteRes<"WriteVFALUV", []>;
-defm "" : LMULWriteRes<"WriteVFALUF", []>;
+defm "" : LMULSEWWriteResF<"WriteVFALUV", []>;
+defm "" : LMULSEWWriteResF<"WriteVFALUF", []>;
 defm "" : LMULWriteResFW<"WriteVFWALUV", []>;
 defm "" : LMULWriteResFW<"WriteVFWALUF", []>;
 defm "" : LMULWriteRes<"WriteVFMulV", []>;
@@ -1024,8 +1024,8 @@ defm "" : LMULReadAdvanceW<"ReadVNClipV", 0>;
 defm "" : LMULReadAdvanceW<"ReadVNClipX", 0>;
 
 // 13. Vector Floating-Point Instructions
-defm "" : LMULReadAdvance<"ReadVFALUV", 0>;
-defm "" : LMULReadAdvance<"ReadVFALUF", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFALUV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFALUF", 0>;
 defm "" : LMULReadAdvanceFW<"ReadVFWALUV", 0>;
 defm "" : LMULReadAdvanceFW<"ReadVFWALUF", 0>;
 defm "" : LMULReadAdvance<"ReadVFMulV", 0>;
-- 
GitLab


From aece68269c8cb28e16dc5c4ec9aabecb807a0438 Mon Sep 17 00:00:00 2001
From: Michael Maitland <michaeltmaitland@gmail.com>
Date: Thu, 4 Apr 2024 08:38:32 -0700
Subject: [PATCH 063/464] [RISCV] Split PseudoVFWADD, PseudoVFWSUB, and
 PseudoVFWMUL by SEW

Co-authored-by: Wang Pengcheng <wangpengcheng.pp@bytedance.com>
---
 llvm/lib/Target/RISCV/RISCVInstrInfo.cpp      |  46 ++++++-
 .../Target/RISCV/RISCVInstrInfoVPseudos.td    | 121 ++++++++++--------
 .../Target/RISCV/RISCVInstrInfoVSDPatterns.td |  12 +-
 .../Target/RISCV/RISCVInstrInfoVVLPatterns.td |  43 ++++---
 llvm/lib/Target/RISCV/RISCVSchedSiFive7.td    |  24 ++--
 llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td |  24 ++--
 llvm/lib/Target/RISCV/RISCVScheduleV.td       |  44 ++++---
 7 files changed, 203 insertions(+), 111 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index d78f5bd9dedf..3b76e7158ffd 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -3038,14 +3038,13 @@ MachineInstr *RISCVInstrInfo::commuteInstructionImpl(MachineInstr &MI,
 #define CASE_WIDEOP_OPCODE_LMULS(OP)                                           \
   CASE_WIDEOP_OPCODE_COMMON(OP, MF8):                                          \
   case CASE_WIDEOP_OPCODE_LMULS_MF4(OP)
-// clang-format on
 
 #define CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, LMUL)                             \
   case RISCV::PseudoV##OP##_##LMUL##_TIED:                                     \
     NewOpc = RISCV::PseudoV##OP##_##LMUL;                                      \
     break;
 
-#define CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP)                                 \
+#define CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP)                                \
   CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4)                                    \
   CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2)                                    \
   CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1)                                     \
@@ -3056,6 +3055,41 @@ MachineInstr *RISCVInstrInfo::commuteInstructionImpl(MachineInstr &MI,
   CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF8)                                    \
   CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP)
 
+// FP Widening Ops may by SEW aware. Create SEW aware cases for these cases.
+#define CASE_FP_WIDEOP_OPCODE_COMMON(OP, LMUL, SEW)                            \
+  RISCV::PseudoV##OP##_##LMUL##_##SEW##_TIED
+
+#define CASE_FP_WIDEOP_OPCODE_LMULS_MF4(OP)                                    \
+  CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF4, E16):                                  \
+  case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E16):                             \
+  case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E32):                             \
+  case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E16):                              \
+  case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E32):                              \
+  case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E16):                              \
+  case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E32):                              \
+  case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E16):                              \
+  case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E32)                               \
+
+#define CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, LMUL, SEW)                     \
+  case RISCV::PseudoV##OP##_##LMUL##_##SEW##_TIED:                             \
+    NewOpc = RISCV::PseudoV##OP##_##LMUL##_##SEW;                              \
+    break;
+
+#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP)                             \
+  CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4, E16)                            \
+  CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E16)                            \
+  CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E32)                            \
+  CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E16)                             \
+  CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E32)                             \
+  CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E16)                             \
+  CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E32)                             \
+  CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E16)                             \
+  CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E32)                             \
+
+#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS(OP)                                 \
+  CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP)
+// clang-format on
+
 MachineInstr *RISCVInstrInfo::convertToThreeAddress(MachineInstr &MI,
                                                     LiveVariables *LV,
                                                     LiveIntervals *LIS) const {
@@ -3063,8 +3097,8 @@ MachineInstr *RISCVInstrInfo::convertToThreeAddress(MachineInstr &MI,
   switch (MI.getOpcode()) {
   default:
     return nullptr;
-  case CASE_WIDEOP_OPCODE_LMULS_MF4(FWADD_WV):
-  case CASE_WIDEOP_OPCODE_LMULS_MF4(FWSUB_WV): {
+  case CASE_FP_WIDEOP_OPCODE_LMULS_MF4(FWADD_WV):
+  case CASE_FP_WIDEOP_OPCODE_LMULS_MF4(FWSUB_WV): {
     assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags) &&
            MI.getNumExplicitOperands() == 7 &&
            "Expect 7 explicit operands rd, rs2, rs1, rm, vl, sew, policy");
@@ -3077,8 +3111,8 @@ MachineInstr *RISCVInstrInfo::convertToThreeAddress(MachineInstr &MI,
     switch (MI.getOpcode()) {
     default:
       llvm_unreachable("Unexpected opcode");
-    CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4(FWADD_WV)
-    CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4(FWSUB_WV)
+    CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_MF4(FWADD_WV)
+    CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_MF4(FWSUB_WV)
     }
     // clang-format on
 
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index ebe8a29fdfa8..1097eb4fb643 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -2213,11 +2213,13 @@ multiclass VPseudoTiedBinaryRoundingMode<VReg RetClass,
                                          DAGOperand Op2Class,
                                          LMULInfo MInfo,
                                          string Constraint = "",
+                                         int sew = 0,
                                          int TargetConstraintType = 1> {
+    defvar suffix = !if(sew, "_" # MInfo.MX # "_E" # sew, "_" # MInfo.MX);
     let VLMul = MInfo.value in {
-    def "_" # MInfo.MX # "_TIED":
+    def suffix # "_TIED":
       VPseudoTiedBinaryNoMaskRoundingMode<RetClass, Op2Class, Constraint, TargetConstraintType>;
-    def "_" # MInfo.MX # "_MASK_TIED" :
+    def suffix # "_MASK_TIED" :
       VPseudoTiedBinaryMaskRoundingMode<RetClass, Op2Class, Constraint, TargetConstraintType>,
       RISCVMaskedPseudo<MaskIdx=2>;
   }
@@ -2334,9 +2336,9 @@ multiclass VPseudoBinaryW_VV<LMULInfo m> {
                            "@earlyclobber $rd", TargetConstraintType=3>;
 }
 
-multiclass VPseudoBinaryW_VV_RM<LMULInfo m> {
+multiclass VPseudoBinaryW_VV_RM<LMULInfo m, int sew = 0> {
   defm _VV : VPseudoBinaryRoundingMode<m.wvrclass, m.vrclass, m.vrclass, m,
-                                      "@earlyclobber $rd",  UsesVXRM=0,
+                                      "@earlyclobber $rd", sew, UsesVXRM=0,
                                       TargetConstraintType=3>;
 }
 
@@ -2356,10 +2358,10 @@ multiclass VPseudoBinaryW_VF<LMULInfo m, FPR_Info f> {
                                    "@earlyclobber $rd">;
 }
 
-multiclass VPseudoBinaryW_VF_RM<LMULInfo m, FPR_Info f> {
+multiclass VPseudoBinaryW_VF_RM<LMULInfo m, FPR_Info f, int sew = 0> {
   defm "_V" # f.FX : VPseudoBinaryRoundingMode<m.wvrclass, m.vrclass,
                                                f.fprclass, m,
-                                               "@earlyclobber $rd",
+                                               "@earlyclobber $rd", sew,
                                                UsesVXRM=0,
                                                TargetConstraintType=3>;
 }
@@ -2371,11 +2373,13 @@ multiclass VPseudoBinaryW_WV<LMULInfo m> {
                                "@earlyclobber $rd", TargetConstraintType=3>;
 }
 
-multiclass VPseudoBinaryW_WV_RM<LMULInfo m> {
+multiclass VPseudoBinaryW_WV_RM<LMULInfo m, int sew = 0> {
   defm _WV : VPseudoBinaryRoundingMode<m.wvrclass, m.wvrclass, m.vrclass, m,
-                                       "@earlyclobber $rd", UsesVXRM=0, TargetConstraintType=3>;
+                                       "@earlyclobber $rd", sew, UsesVXRM = 0,
+                                       TargetConstraintType = 3>;
   defm _WV : VPseudoTiedBinaryRoundingMode<m.wvrclass, m.vrclass, m,
-                                           "@earlyclobber $rd", TargetConstraintType=3>;
+                                           "@earlyclobber $rd", sew,
+                                           TargetConstraintType = 3>;
 }
 
 multiclass VPseudoBinaryW_WX<LMULInfo m> {
@@ -2387,11 +2391,11 @@ multiclass VPseudoBinaryW_WF<LMULInfo m, FPR_Info f, int TargetConstraintType =
                                    f.fprclass, m, /*Constraint*/ "", TargetConstraintType=TargetConstraintType>;
 }
 
-multiclass VPseudoBinaryW_WF_RM<LMULInfo m, FPR_Info f> {
+multiclass VPseudoBinaryW_WF_RM<LMULInfo m, FPR_Info f, int sew = 0> {
   defm "_W" # f.FX : VPseudoBinaryRoundingMode<m.wvrclass, m.wvrclass,
                                                f.fprclass, m,
                                                Constraint="",
-                                               sew=0,
+                                               sew=sew,
                                                UsesVXRM=0,
                                                TargetConstraintType=3>;
 }
@@ -2979,16 +2983,17 @@ multiclass VPseudoVWMUL_VV_VX {
 
 multiclass VPseudoVWMUL_VV_VF_RM {
   foreach m = MxListFW in {
-    defm "" : VPseudoBinaryW_VV_RM<m>,
+    foreach e = SchedSEWSet<m.MX, isF=1, isWidening=1>.val in
+    defm "" : VPseudoBinaryW_VV_RM<m, sew=e>,
               SchedBinary<"WriteVFWMulV", "ReadVFWMulV", "ReadVFWMulV", m.MX,
-                          forceMergeOpRead=true>;
+                          e, forceMergeOpRead=true>;
   }
 
   foreach f = FPListW in {
     foreach m = f.MxListFW in {
-      defm "" : VPseudoBinaryW_VF_RM<m, f>,
+      defm "" : VPseudoBinaryW_VF_RM<m, f, sew=f.SEW>,
                 SchedBinary<"WriteVFWMulF", "ReadVFWMulV", "ReadVFWMulF", m.MX,
-                          forceMergeOpRead=true>;
+                          f.SEW, forceMergeOpRead=true>;
     }
   }
 }
@@ -3007,31 +3012,33 @@ multiclass VPseudoVWALU_WV_WX {
 
 multiclass VPseudoVFWALU_VV_VF_RM {
   foreach m = MxListFW in {
-    defm "" : VPseudoBinaryW_VV_RM<m>,
-              SchedBinary<"WriteVFWALUV", "ReadVFWALUV", "ReadVFWALUV", m.MX,
-                          forceMergeOpRead=true>;
+    foreach e = SchedSEWSet<m.MX, isF=1, isWidening=1>.val in
+      defm "" : VPseudoBinaryW_VV_RM<m, sew=e>,
+                SchedBinary<"WriteVFWALUV", "ReadVFWALUV", "ReadVFWALUV", m.MX,
+                            e, forceMergeOpRead=true>;
   }
 
   foreach f = FPListW in {
     foreach m = f.MxListFW in {
-      defm "" : VPseudoBinaryW_VF_RM<m, f>,
+      defm "" : VPseudoBinaryW_VF_RM<m, f, sew=f.SEW>,
                 SchedBinary<"WriteVFWALUF", "ReadVFWALUV", "ReadVFWALUF", m.MX,
-                          forceMergeOpRead=true>;
+                          f.SEW, forceMergeOpRead=true>;
     }
   }
 }
 
 multiclass VPseudoVFWALU_WV_WF_RM {
   foreach m = MxListFW in {
-    defm "" : VPseudoBinaryW_WV_RM<m>,
-              SchedBinary<"WriteVFWALUV", "ReadVFWALUV", "ReadVFWALUV", m.MX,
-                          forceMergeOpRead=true>;
+    foreach e = SchedSEWSet<m.MX, isF=1, isWidening=1>.val in
+      defm "" : VPseudoBinaryW_WV_RM<m, sew=e>,
+                SchedBinary<"WriteVFWALUV", "ReadVFWALUV", "ReadVFWALUV", m.MX,
+                            e, forceMergeOpRead=true>;
   }
   foreach f = FPListW in {
     foreach m = f.MxListFW in {
-      defm "" : VPseudoBinaryW_WF_RM<m, f>,
+      defm "" : VPseudoBinaryW_WF_RM<m, f, sew=f.SEW>,
                 SchedBinary<"WriteVFWALUF", "ReadVFWALUV", "ReadVFWALUF", m.MX,
-                          forceMergeOpRead=true>;
+                            f.SEW, forceMergeOpRead=true>;
     }
   }
 }
@@ -5028,13 +5035,16 @@ multiclass VPatBinaryW_VV<string intrinsic, string instruction,
 }
 
 multiclass VPatBinaryW_VV_RM<string intrinsic, string instruction,
-                             list<VTypeInfoToWide> vtilist> {
+                             list<VTypeInfoToWide> vtilist, bit isSEWAware = 0> {
   foreach VtiToWti = vtilist in {
     defvar Vti = VtiToWti.Vti;
     defvar Wti = VtiToWti.Wti;
+    defvar name = !if(isSEWAware,
+                      instruction # "_VV_" # Vti.LMul.MX # "_E" # Vti.SEW,
+                      instruction # "_VV_" # Vti.LMul.MX);
     let Predicates = !listconcat(GetVTypePredicates<Vti>.Predicates,
                                  GetVTypePredicates<Wti>.Predicates) in
-    defm : VPatBinaryRoundingMode<intrinsic, instruction # "_VV_" # Vti.LMul.MX,
+    defm : VPatBinaryRoundingMode<intrinsic, name,
                                   Wti.Vector, Vti.Vector, Vti.Vector, Vti.Mask,
                                   Vti.Log2SEW, Wti.RegClass,
                                   Vti.RegClass, Vti.RegClass>;
@@ -5057,14 +5067,17 @@ multiclass VPatBinaryW_VX<string intrinsic, string instruction,
 }
 
 multiclass VPatBinaryW_VX_RM<string intrinsic, string instruction,
-                          list<VTypeInfoToWide> vtilist> {
+                          list<VTypeInfoToWide> vtilist, bit isSEWAware = 0> {
   foreach VtiToWti = vtilist in {
     defvar Vti = VtiToWti.Vti;
     defvar Wti = VtiToWti.Wti;
     defvar kind = "V"#Vti.ScalarSuffix;
+    defvar name = !if(isSEWAware,
+                      instruction#"_"#kind#"_"#Vti.LMul.MX # "_E" # Vti.SEW,
+                      instruction#"_"#kind#"_"#Vti.LMul.MX);
     let Predicates = !listconcat(GetVTypePredicates<Vti>.Predicates,
                                  GetVTypePredicates<Wti>.Predicates) in
-    defm : VPatBinaryRoundingMode<intrinsic, instruction#"_"#kind#"_"#Vti.LMul.MX,
+    defm : VPatBinaryRoundingMode<intrinsic, name,
                                   Wti.Vector, Vti.Vector, Vti.Scalar, Vti.Mask,
                                   Vti.Log2SEW, Wti.RegClass,
                                   Vti.RegClass, Vti.ScalarRegClass>;
@@ -5101,27 +5114,30 @@ multiclass VPatBinaryW_WV<string intrinsic, string instruction,
 }
 
 multiclass VPatBinaryW_WV_RM<string intrinsic, string instruction,
-                             list<VTypeInfoToWide> vtilist> {
+                             list<VTypeInfoToWide> vtilist, bit isSEWAware = 0> {
   foreach VtiToWti = vtilist in {
     defvar Vti = VtiToWti.Vti;
     defvar Wti = VtiToWti.Wti;
+    defvar name = !if(isSEWAware,
+                      instruction # "_WV_" # Vti.LMul.MX # "_E" # Vti.SEW,
+                      instruction # "_WV_" # Vti.LMul.MX);
     let Predicates = !listconcat(GetVTypePredicates<Vti>.Predicates,
                                  GetVTypePredicates<Wti>.Predicates) in {
-      def : VPatTiedBinaryNoMaskRoundingMode<intrinsic, instruction # "_WV_" # Vti.LMul.MX,
+      def : VPatTiedBinaryNoMaskRoundingMode<intrinsic, name,
                                              Wti.Vector, Vti.Vector,
                                              Vti.Log2SEW, Wti.RegClass, Vti.RegClass>;
-      def : VPatBinaryNoMaskTURoundingMode<intrinsic, instruction # "_WV_" # Vti.LMul.MX,
+      def : VPatBinaryNoMaskTURoundingMode<intrinsic, name,
                                            Wti.Vector, Wti.Vector, Vti.Vector, Vti.Log2SEW,
                                            Wti.RegClass, Wti.RegClass, Vti.RegClass>;
       let AddedComplexity = 1 in {
-      def : VPatTiedBinaryNoMaskTURoundingMode<intrinsic, instruction # "_WV_" # Vti.LMul.MX,
+      def : VPatTiedBinaryNoMaskTURoundingMode<intrinsic, name,
                                                Wti.Vector, Vti.Vector,
                                                Vti.Log2SEW, Wti.RegClass, Vti.RegClass>;
-      def : VPatTiedBinaryMaskRoundingMode<intrinsic, instruction # "_WV_" # Vti.LMul.MX,
+      def : VPatTiedBinaryMaskRoundingMode<intrinsic, name,
                                            Wti.Vector, Vti.Vector, Vti.Mask,
                                            Vti.Log2SEW, Wti.RegClass, Vti.RegClass>;
       }
-      def : VPatBinaryMaskTARoundingMode<intrinsic, instruction # "_WV_" # Vti.LMul.MX,
+      def : VPatBinaryMaskTARoundingMode<intrinsic, name,
                                          Wti.Vector, Wti.Vector, Vti.Vector, Vti.Mask,
                                          Vti.Log2SEW, Wti.RegClass,
                                          Wti.RegClass, Vti.RegClass>;
@@ -5145,14 +5161,17 @@ multiclass VPatBinaryW_WX<string intrinsic, string instruction,
 }
 
 multiclass VPatBinaryW_WX_RM<string intrinsic, string instruction,
-                             list<VTypeInfoToWide> vtilist> {
+                             list<VTypeInfoToWide> vtilist, bit isSEWAware = 0> {
   foreach VtiToWti = vtilist in {
     defvar Vti = VtiToWti.Vti;
     defvar Wti = VtiToWti.Wti;
     defvar kind = "W"#Vti.ScalarSuffix;
+    defvar name = !if(isSEWAware,
+                      instruction#"_"#kind#"_"#Vti.LMul.MX#"_E"#Vti.SEW,
+                      instruction#"_"#kind#"_"#Vti.LMul.MX);
     let Predicates = !listconcat(GetVTypePredicates<Vti>.Predicates,
                                  GetVTypePredicates<Wti>.Predicates) in
-    defm : VPatBinaryRoundingMode<intrinsic, instruction#"_"#kind#"_"#Vti.LMul.MX,
+    defm : VPatBinaryRoundingMode<intrinsic, name,
                                   Wti.Vector, Wti.Vector, Vti.Scalar, Vti.Mask,
                                   Vti.Log2SEW, Wti.RegClass,
                                   Wti.RegClass, Vti.ScalarRegClass>;
@@ -5417,20 +5436,22 @@ multiclass VPatBinaryW_VV_VX<string intrinsic, string instruction,
     : VPatBinaryW_VV<intrinsic, instruction, vtilist>,
       VPatBinaryW_VX<intrinsic, instruction, vtilist>;
 
-multiclass VPatBinaryW_VV_VX_RM<string intrinsic, string instruction,
-                                list<VTypeInfoToWide> vtilist>
-    : VPatBinaryW_VV_RM<intrinsic, instruction, vtilist>,
-      VPatBinaryW_VX_RM<intrinsic, instruction, vtilist>;
+multiclass
+    VPatBinaryW_VV_VX_RM<string intrinsic, string instruction,
+                         list<VTypeInfoToWide> vtilist, bit isSEWAware = 0>
+    : VPatBinaryW_VV_RM<intrinsic, instruction, vtilist, isSEWAware>,
+      VPatBinaryW_VX_RM<intrinsic, instruction, vtilist, isSEWAware>;
 
 multiclass VPatBinaryW_WV_WX<string intrinsic, string instruction,
                              list<VTypeInfoToWide> vtilist>
     : VPatBinaryW_WV<intrinsic, instruction, vtilist>,
       VPatBinaryW_WX<intrinsic, instruction, vtilist>;
 
-multiclass VPatBinaryW_WV_WX_RM<string intrinsic, string instruction,
-                                list<VTypeInfoToWide> vtilist>
-    : VPatBinaryW_WV_RM<intrinsic, instruction, vtilist>,
-      VPatBinaryW_WX_RM<intrinsic, instruction, vtilist>;
+multiclass
+    VPatBinaryW_WV_WX_RM<string intrinsic, string instruction,
+                         list<VTypeInfoToWide> vtilist, bit isSEWAware = 0>
+    : VPatBinaryW_WV_RM<intrinsic, instruction, vtilist, isSEWAware>,
+      VPatBinaryW_WX_RM<intrinsic, instruction, vtilist, isSEWAware>;
 
 multiclass VPatBinaryV_WV_WX_WI<string intrinsic, string instruction,
                                 list<VTypeInfoToWide> vtilist>
@@ -7073,13 +7094,13 @@ defm : VPatBinaryV_VX_RM<"int_riscv_vfrsub", "PseudoVFRSUB", AllFloatVectors,
 // 13.3. Vector Widening Floating-Point Add/Subtract Instructions
 //===----------------------------------------------------------------------===//
 defm : VPatBinaryW_VV_VX_RM<"int_riscv_vfwadd", "PseudoVFWADD",
-                            AllWidenableFloatVectors>;
+                            AllWidenableFloatVectors, isSEWAware=1>;
 defm : VPatBinaryW_VV_VX_RM<"int_riscv_vfwsub", "PseudoVFWSUB",
-                            AllWidenableFloatVectors>;
+                            AllWidenableFloatVectors, isSEWAware=1>;
 defm : VPatBinaryW_WV_WX_RM<"int_riscv_vfwadd_w", "PseudoVFWADD",
-                            AllWidenableFloatVectors>;
+                            AllWidenableFloatVectors, isSEWAware=1>;
 defm : VPatBinaryW_WV_WX_RM<"int_riscv_vfwsub_w", "PseudoVFWSUB",
-                            AllWidenableFloatVectors>;
+                            AllWidenableFloatVectors, isSEWAware=1>;
 
 //===----------------------------------------------------------------------===//
 // 13.4. Vector Single-Width Floating-Point Multiply/Divide Instructions
@@ -7095,7 +7116,7 @@ defm : VPatBinaryV_VX_RM<"int_riscv_vfrdiv", "PseudoVFRDIV",
 // 13.5. Vector Widening Floating-Point Multiply
 //===----------------------------------------------------------------------===//
 defm : VPatBinaryW_VV_VX_RM<"int_riscv_vfwmul", "PseudoVFWMUL",
-                            AllWidenableFloatVectors>;
+                            AllWidenableFloatVectors, isSEWAware=1>;
 
 //===----------------------------------------------------------------------===//
 // 13.6. Vector Single-Width Floating-Point Fused Multiply-Add Instructions
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
index 0d5cd93984af..37acd9667a54 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
@@ -621,7 +621,7 @@ multiclass VPatWidenBinaryFPSDNode_VV_VF_RM<SDNode op, string instruction_name>
                     (wti.Vector (riscv_fpextend_vl_oneuse
                                      (vti.Vector vti.RegClass:$rs1),
                                      (vti.Mask true_mask), (XLenVT srcvalue)))),
-                (!cast<Instruction>(instruction_name#"_VV_"#vti.LMul.MX)
+                (!cast<Instruction>(instruction_name#"_VV_"#vti.LMul.MX#"_E"#vti.SEW)
                   (wti.Vector (IMPLICIT_DEF)), vti.RegClass:$rs2,
                   vti.RegClass:$rs1,
                    // Value to indicate no rounding mode change in
@@ -634,7 +634,7 @@ multiclass VPatWidenBinaryFPSDNode_VV_VF_RM<SDNode op, string instruction_name>
                     (wti.Vector (riscv_fpextend_vl_oneuse
                                      (vti.Vector (SplatFPOp (vti.Scalar vti.ScalarRegClass:$rs1))),
                                      (vti.Mask true_mask), (XLenVT srcvalue)))),
-                (!cast<Instruction>(instruction_name#"_V"#vti.ScalarSuffix#"_"#vti.LMul.MX)
+                (!cast<Instruction>(instruction_name#"_V"#vti.ScalarSuffix#"_"#vti.LMul.MX#"_E"#vti.SEW)
                    (wti.Vector (IMPLICIT_DEF)), vti.RegClass:$rs2,
                    vti.ScalarRegClass:$rs1,
                    // Value to indicate no rounding mode change in
@@ -645,7 +645,7 @@ multiclass VPatWidenBinaryFPSDNode_VV_VF_RM<SDNode op, string instruction_name>
                                      (vti.Vector vti.RegClass:$rs2),
                                      (vti.Mask true_mask), (XLenVT srcvalue))),
                     (wti.Vector (SplatFPOp (fpext_oneuse (vti.Scalar vti.ScalarRegClass:$rs1))))),
-                (!cast<Instruction>(instruction_name#"_V"#vti.ScalarSuffix#"_"#vti.LMul.MX)
+                (!cast<Instruction>(instruction_name#"_V"#vti.ScalarSuffix#"_"#vti.LMul.MX#"_E"#vti.SEW)
                    (wti.Vector (IMPLICIT_DEF)), vti.RegClass:$rs2,
                    vti.ScalarRegClass:$rs1,
                    // Value to indicate no rounding mode change in
@@ -666,7 +666,7 @@ multiclass VPatWidenBinaryFPSDNode_WV_WF_RM<SDNode op, string instruction_name>
                     (wti.Vector (riscv_fpextend_vl_oneuse
                                      (vti.Vector vti.RegClass:$rs1),
                                      (vti.Mask true_mask), (XLenVT srcvalue)))),
-                (!cast<Instruction>(instruction_name#"_WV_"#vti.LMul.MX#"_TIED")
+                (!cast<Instruction>(instruction_name#"_WV_"#vti.LMul.MX#"_E"#vti.SEW#"_TIED")
                    wti.RegClass:$rs2, vti.RegClass:$rs1,
                    // Value to indicate no rounding mode change in
                    // RISCVInsertReadWriteCSR
@@ -677,7 +677,7 @@ multiclass VPatWidenBinaryFPSDNode_WV_WF_RM<SDNode op, string instruction_name>
                     (wti.Vector (riscv_fpextend_vl_oneuse
                                      (vti.Vector (SplatFPOp vti.ScalarRegClass:$rs1)),
                                      (vti.Mask true_mask), (XLenVT srcvalue)))),
-                (!cast<Instruction>(instruction_name#"_W"#vti.ScalarSuffix#"_"#vti.LMul.MX)
+                (!cast<Instruction>(instruction_name#"_W"#vti.ScalarSuffix#"_"#vti.LMul.MX#"_E"#vti.SEW)
                    (wti.Vector (IMPLICIT_DEF)), wti.RegClass:$rs2,
                    vti.ScalarRegClass:$rs1,
                    // Value to indicate no rounding mode change in
@@ -686,7 +686,7 @@ multiclass VPatWidenBinaryFPSDNode_WV_WF_RM<SDNode op, string instruction_name>
                    vti.AVL, vti.Log2SEW, TA_MA)>;
       def : Pat<(op (wti.Vector wti.RegClass:$rs2),
                     (wti.Vector (SplatFPOp (fpext_oneuse (vti.Scalar vti.ScalarRegClass:$rs1))))),
-                (!cast<Instruction>(instruction_name#"_W"#vti.ScalarSuffix#"_"#vti.LMul.MX)
+                (!cast<Instruction>(instruction_name#"_W"#vti.ScalarSuffix#"_"#vti.LMul.MX#"_E"#vti.SEW)
                    (wti.Vector (IMPLICIT_DEF)), wti.RegClass:$rs2,
                    vti.ScalarRegClass:$rs1,
                    // Value to indicate no rounding mode change in
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
index befe8d07e575..d2f575459e32 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
@@ -725,23 +725,27 @@ multiclass VPatTiedBinaryNoMaskVL_V_RM<SDNode vop,
                                        string suffix,
                                        ValueType result_type,
                                        ValueType op2_type,
-                                       int sew,
+                                       int log2sew,
                                        LMULInfo vlmul,
                                        VReg result_reg_class,
-                                       VReg op2_reg_class> {
+                                       VReg op2_reg_class,
+                                       bit isSEWAware = 0> {
+  defvar name = !if(isSEWAware,
+                    instruction_name#"_"#suffix#"_"#vlmul.MX#"_E"#!shl(1, log2sew)#"_TIED",
+                    instruction_name#"_"#suffix#"_"#vlmul.MX#"_TIED");
   def : Pat<(result_type (vop
                          (result_type result_reg_class:$rs1),
                          (op2_type op2_reg_class:$rs2),
                          srcvalue,
                          true_mask,
                          VLOpFrag)),
-        (!cast<Instruction>(instruction_name#"_"#suffix#"_"# vlmul.MX#"_TIED")
+        (!cast<Instruction>(name)
                      result_reg_class:$rs1,
                      op2_reg_class:$rs2,
                      // Value to indicate no rounding mode change in
                      // RISCVInsertReadWriteCSR
                      FRM_DYN,
-                     GPR:$vl, sew, TAIL_AGNOSTIC)>;
+                     GPR:$vl, log2sew, TAIL_AGNOSTIC)>;
   // Tail undisturbed
   def : Pat<(riscv_vmerge_vl true_mask,
              (result_type (vop
@@ -751,13 +755,13 @@ multiclass VPatTiedBinaryNoMaskVL_V_RM<SDNode vop,
                            true_mask,
                            VLOpFrag)),
              result_reg_class:$rs1, result_reg_class:$rs1, VLOpFrag),
-            (!cast<Instruction>(instruction_name#"_"#suffix#"_"# vlmul.MX#"_TIED")
+            (!cast<Instruction>(name)
                      result_reg_class:$rs1,
                      op2_reg_class:$rs2,
                      // Value to indicate no rounding mode change in
                      // RISCVInsertReadWriteCSR
                      FRM_DYN,
-                     GPR:$vl, sew, TU_MU)>;
+                     GPR:$vl, log2sew, TU_MU)>;
 }
 
 class VPatBinaryVL_XI<SDPatternOperator vop,
@@ -1591,7 +1595,8 @@ multiclass VPatBinaryFPWVL_VV_VF<SDNode vop, string instruction_name> {
   }
 }
 
-multiclass VPatBinaryFPWVL_VV_VF_RM<SDNode vop, string instruction_name> {
+multiclass VPatBinaryFPWVL_VV_VF_RM<SDNode vop, string instruction_name,
+                                    bit isSEWAware = 0> {
   foreach fvtiToFWti = AllWidenableFloatVectors in {
     defvar vti = fvtiToFWti.Vti;
     defvar wti = fvtiToFWti.Wti;
@@ -1600,11 +1605,11 @@ multiclass VPatBinaryFPWVL_VV_VF_RM<SDNode vop, string instruction_name> {
       def : VPatBinaryVL_V_RM<vop, instruction_name, "VV",
                                        wti.Vector, vti.Vector, vti.Vector, vti.Mask,
                                        vti.Log2SEW, vti.LMul, wti.RegClass, vti.RegClass,
-                                       vti.RegClass>;
+                                       vti.RegClass, isSEWAware>;
       def : VPatBinaryVL_VF_RM<vop, instruction_name#"_V"#vti.ScalarSuffix,
                                         wti.Vector, vti.Vector, vti.Vector, vti.Mask,
                                         vti.Log2SEW, vti.LMul, wti.RegClass, vti.RegClass,
-                                        vti.ScalarRegClass>;
+                                        vti.ScalarRegClass, isSEWAware>;
     }
   }
 }
@@ -1631,8 +1636,9 @@ multiclass VPatBinaryFPWVL_VV_VF_WV_WF<SDNode vop, SDNode vop_w, string instruct
   }
 }
 
-multiclass VPatBinaryFPWVL_VV_VF_WV_WF_RM<SDNode vop, SDNode vop_w, string instruction_name>
-    : VPatBinaryFPWVL_VV_VF_RM<vop, instruction_name> {
+multiclass VPatBinaryFPWVL_VV_VF_WV_WF_RM<
+    SDNode vop, SDNode vop_w, string instruction_name, bit isSEWAware = 0>
+    : VPatBinaryFPWVL_VV_VF_RM<vop, instruction_name, isSEWAware> {
   foreach fvtiToFWti = AllWidenableFloatVectors in {
     defvar vti = fvtiToFWti.Vti;
     defvar wti = fvtiToFWti.Wti;
@@ -1640,15 +1646,16 @@ multiclass VPatBinaryFPWVL_VV_VF_WV_WF_RM<SDNode vop, SDNode vop_w, string instr
                                  GetVTypePredicates<wti>.Predicates) in {
       defm : VPatTiedBinaryNoMaskVL_V_RM<vop_w, instruction_name, "WV",
                                          wti.Vector, vti.Vector, vti.Log2SEW,
-                                         vti.LMul, wti.RegClass, vti.RegClass>;
+                                         vti.LMul, wti.RegClass, vti.RegClass,
+                                         isSEWAware>;
       def : VPatBinaryVL_V_RM<vop_w, instruction_name, "WV",
                                        wti.Vector, wti.Vector, vti.Vector, vti.Mask,
                                        vti.Log2SEW, vti.LMul, wti.RegClass, wti.RegClass,
-                                       vti.RegClass>;
+                                       vti.RegClass, isSEWAware>;
       def : VPatBinaryVL_VF_RM<vop_w, instruction_name#"_W"#vti.ScalarSuffix,
                                         wti.Vector, wti.Vector, vti.Vector, vti.Mask,
                                         vti.Log2SEW, vti.LMul, wti.RegClass, wti.RegClass,
-                                        vti.ScalarRegClass>;
+                                        vti.ScalarRegClass, isSEWAware>;
     }
   }
 }
@@ -2430,8 +2437,10 @@ defm : VPatBinaryFPVL_VV_VF_RM<any_riscv_fsub_vl, "PseudoVFSUB", isSEWAware=1>;
 defm : VPatBinaryFPVL_R_VF_RM<any_riscv_fsub_vl, "PseudoVFRSUB", isSEWAware=1>;
 
 // 13.3. Vector Widening Floating-Point Add/Subtract Instructions
-defm : VPatBinaryFPWVL_VV_VF_WV_WF_RM<riscv_vfwadd_vl, riscv_vfwadd_w_vl, "PseudoVFWADD">;
-defm : VPatBinaryFPWVL_VV_VF_WV_WF_RM<riscv_vfwsub_vl, riscv_vfwsub_w_vl, "PseudoVFWSUB">;
+defm : VPatBinaryFPWVL_VV_VF_WV_WF_RM<riscv_vfwadd_vl, riscv_vfwadd_w_vl,
+                                      "PseudoVFWADD", isSEWAware=1>;
+defm : VPatBinaryFPWVL_VV_VF_WV_WF_RM<riscv_vfwsub_vl, riscv_vfwsub_w_vl,
+                                      "PseudoVFWSUB", isSEWAware=1>;
 
 // 13.4. Vector Single-Width Floating-Point Multiply/Divide Instructions
 defm : VPatBinaryFPVL_VV_VF_RM<any_riscv_fmul_vl, "PseudoVFMUL">;
@@ -2439,7 +2448,7 @@ defm : VPatBinaryFPVL_VV_VF_RM<any_riscv_fdiv_vl, "PseudoVFDIV", isSEWAware=1>;
 defm : VPatBinaryFPVL_R_VF_RM<any_riscv_fdiv_vl, "PseudoVFRDIV", isSEWAware=1>;
 
 // 13.5. Vector Widening Floating-Point Multiply Instructions
-defm : VPatBinaryFPWVL_VV_VF_RM<riscv_vfwmul_vl, "PseudoVFWMUL">;
+defm : VPatBinaryFPWVL_VV_VF_RM<riscv_vfwmul_vl, "PseudoVFWMUL", isSEWAware=1>;
 
 // 13.6 Vector Single-Width Floating-Point Fused Multiply-Add Instructions.
 defm : VPatFPMulAddVL_VV_VF_RM<any_riscv_vfmadd_vl,  "PseudoVFMADD">;
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
index f14766c98439..2b6fc5e59f80 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
@@ -790,18 +790,26 @@ foreach mx = SchedMxListW in {
     defm "" : LMULWriteResMX<"WriteVFWCvtIToFV", [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
   }
 }
+foreach mx = SchedMxListFW in {
+  foreach sew = SchedSEWSet<mx, isF=1, isWidening=1>.val in {
+    defvar Cycles = SiFive7GetCyclesDefault<mx>.c;
+    defvar IsWorstCase = SiFive7IsWorstCaseMXSEW<mx, sew, SchedMxListFW, isF=1>.c;
+    let Latency = 8, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in {
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFWALUV", [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFWALUF", [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulV", [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulF", [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
+    }
+  }
+}
 foreach mx = SchedMxListFW in {
   defvar Cycles = SiFive7GetCyclesDefault<mx>.c;
   defvar IsWorstCase = SiFive7IsWorstCaseMX<mx, SchedMxListFW>.c;
   let Latency = 8, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in {
-    defm "" : LMULWriteResMX<"WriteVFWALUV",     [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVFWMulV",     [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFWMulAddV",  [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFWCvtFToIV", [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFWCvtFToFV", [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFWMulAddF",  [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVFWMulF",     [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVFWALUF",     [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
   }
 }
 // Narrowing
@@ -1147,14 +1155,14 @@ defm "" : LMULReadAdvanceW<"ReadVNClipX", 0>;
 // 14. Vector Floating-Point Instructions
 defm "" : LMULSEWReadAdvanceF<"ReadVFALUV", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFALUF", 0>;
-defm "" : LMULReadAdvanceFW<"ReadVFWALUV", 0>;
-defm "" : LMULReadAdvanceFW<"ReadVFWALUF", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWALUV", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWALUF", 0>;
 defm "" : LMULReadAdvance<"ReadVFMulV", 0>;
 defm "" : LMULReadAdvance<"ReadVFMulF", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFDivV", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFDivF", 0>;
-defm "" : LMULReadAdvanceFW<"ReadVFWMulV", 0>;
-defm "" : LMULReadAdvanceFW<"ReadVFWMulF", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulV", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulF", 0>;
 defm "" : LMULReadAdvance<"ReadVFMulAddV", 0>;
 defm "" : LMULReadAdvance<"ReadVFMulAddF", 0>;
 defm "" : LMULReadAdvanceFW<"ReadVFWMulAddV", 0>;
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
index 4d81bafdf88b..00e92cde8522 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
@@ -541,16 +541,24 @@ foreach mx = SchedMxListFW in {
     defm "" : LMULWriteResMX<"WriteVFWCvtFToFV", [SiFiveP600VectorArith], mx, IsWorstCase>;
   }
 }
+foreach mx = SchedMxListFW in {
+  foreach sew = SchedSEWSet<mx, isF=1, isWidening=1>.val in {
+    defvar LMulLat = SiFiveP600GetLMulCycles<mx>.c;
+    defvar IsWorstCase = SiFiveP600IsWorstCaseMXSEW<mx, sew, SchedMxListFW, isF=1>.c;
+    let Latency = 6, ReleaseAtCycles = [LMulLat] in {
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFWALUV", [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFWALUF", [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulV", [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulF", [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
+    }
+  }
+}
 foreach mx = SchedMxListFW in {
   defvar LMulLat = SiFiveP600GetLMulCycles<mx>.c;
   defvar IsWorstCase = SiFiveP600IsWorstCaseMX<mx, SchedMxListFW>.c;
   let Latency = 6, ReleaseAtCycles = [LMulLat] in {
-    defm "" : LMULWriteResMX<"WriteVFWALUV",    [SiFiveP600VectorArith], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVFWMulV",    [SiFiveP600VectorArith], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFWMulAddV", [SiFiveP600VectorArith], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFWMulAddF", [SiFiveP600VectorArith], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVFWMulF",    [SiFiveP600VectorArith], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVFWALUF",    [SiFiveP600VectorArith], mx, IsWorstCase>;
   }
 }
 // Narrowing
@@ -935,14 +943,14 @@ defm "" : LMULReadAdvanceW<"ReadVNClipX", 0>;
 // 14. Vector Floating-Point Instructions
 defm "" : LMULSEWReadAdvanceF<"ReadVFALUV", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFALUF", 0>;
-defm "" : LMULReadAdvanceFW<"ReadVFWALUV", 0>;
-defm "" : LMULReadAdvanceFW<"ReadVFWALUF", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWALUV", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWALUF", 0>;
 defm "" : LMULReadAdvance<"ReadVFMulV", 0>;
 defm "" : LMULReadAdvance<"ReadVFMulF", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFDivV", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFDivF", 0>;
-defm "" : LMULReadAdvanceFW<"ReadVFWMulV", 0>;
-defm "" : LMULReadAdvanceFW<"ReadVFWMulF", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulV", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulF", 0>;
 defm "" : LMULReadAdvance<"ReadVFMulAddV", 0>;
 defm "" : LMULReadAdvance<"ReadVFMulAddF", 0>;
 defm "" : LMULReadAdvanceFW<"ReadVFWMulAddV", 0>;
diff --git a/llvm/lib/Target/RISCV/RISCVScheduleV.td b/llvm/lib/Target/RISCV/RISCVScheduleV.td
index 9b9c72e0767a..8b5bd7a0c28a 100644
--- a/llvm/lib/Target/RISCV/RISCVScheduleV.td
+++ b/llvm/lib/Target/RISCV/RISCVScheduleV.td
@@ -253,6 +253,18 @@ multiclass LMULReadAdvanceFW<string name, int val, list<SchedWrite> writes = []>
   : LMULReadAdvanceImpl<name, val, writes>;
 class LMULSchedWriteListFW<list<string> names> : LMULSchedWriteListImpl<names, SchedMxListFW>;
 
+multiclass LMULSEWSchedWritesFW<string name>
+    : LMULSEWSchedWritesImpl<name, SchedMxListFW, isF = 1, isWidening = 1>;
+multiclass LMULSEWSchedReadsFW<string name>
+    : LMULSEWSchedReadsImpl<name, SchedMxListFW, isF = 1, isWidening = 1>;
+multiclass LMULSEWWriteResFW<string name, list<ProcResourceKind> resources>
+    : LMULSEWWriteResImpl<name, resources, SchedMxListFW, isF = 1,
+                          isWidening = 1>;
+multiclass
+    LMULSEWReadAdvanceFW<string name, int val, list<SchedWrite> writes = []>
+    : LMULSEWReadAdvanceImpl<name, val, writes, SchedMxListFW, isF = 1,
+                             isWidening = 1>;
+
 // 3.6 Vector Byte Length vlenb
 def WriteRdVLENB      : SchedWrite;
 
@@ -400,16 +412,16 @@ defm "" : LMULSchedWritesW<"WriteVNClipI">;
 defm "" : LMULSEWSchedWritesF<"WriteVFALUV">;
 defm "" : LMULSEWSchedWritesF<"WriteVFALUF">;
 // 13.3. Vector Widening Floating-Point Add/Subtract Instructions
-defm "" : LMULSchedWritesFW<"WriteVFWALUV">;
-defm "" : LMULSchedWritesFW<"WriteVFWALUF">;
+defm "" : LMULSEWSchedWritesFW<"WriteVFWALUV">;
+defm "" : LMULSEWSchedWritesFW<"WriteVFWALUF">;
 // 13.4. Vector Single-Width Floating-Point Multiply/Divide Instructions
 defm "" : LMULSchedWrites<"WriteVFMulV">;
 defm "" : LMULSchedWrites<"WriteVFMulF">;
 defm "" : LMULSEWSchedWritesF<"WriteVFDivV">;
 defm "" : LMULSEWSchedWritesF<"WriteVFDivF">;
 // 13.5. Vector Widening Floating-Point Multiply
-defm "" : LMULSchedWritesFW<"WriteVFWMulV">;
-defm "" : LMULSchedWritesFW<"WriteVFWMulF">;
+defm "" : LMULSEWSchedWritesFW<"WriteVFWMulV">;
+defm "" : LMULSEWSchedWritesFW<"WriteVFWMulF">;
 // 13.6. Vector Single-Width Floating-Point Fused Multiply-Add Instructions
 defm "" : LMULSchedWrites<"WriteVFMulAddV">;
 defm "" : LMULSchedWrites<"WriteVFMulAddF">;
@@ -625,16 +637,16 @@ defm "" : LMULSchedReadsW<"ReadVNClipX">;
 defm "" : LMULSEWSchedReadsF<"ReadVFALUV">;
 defm "" : LMULSEWSchedReadsF<"ReadVFALUF">;
 // 13.3. Vector Widening Floating-Point Add/Subtract Instructions
-defm "" : LMULSchedReadsFW<"ReadVFWALUV">;
-defm "" : LMULSchedReadsFW<"ReadVFWALUF">;
+defm "" : LMULSEWSchedReadsFW<"ReadVFWALUV">;
+defm "" : LMULSEWSchedReadsFW<"ReadVFWALUF">;
 // 13.4. Vector Single-Width Floating-Point Multiply/Divide Instructions
 defm "" : LMULSchedReads<"ReadVFMulV">;
 defm "" : LMULSchedReads<"ReadVFMulF">;
 defm "" : LMULSEWSchedReadsF<"ReadVFDivV">;
 defm "" : LMULSEWSchedReadsF<"ReadVFDivF">;
 // 13.5. Vector Widening Floating-Point Multiply
-defm "" : LMULSchedReadsFW<"ReadVFWMulV">;
-defm "" : LMULSchedReadsFW<"ReadVFWMulF">;
+defm "" : LMULSEWSchedReadsFW<"ReadVFWMulV">;
+defm "" : LMULSEWSchedReadsFW<"ReadVFWMulF">;
 // 13.6. Vector Single-Width Floating-Point Fused Multiply-Add Instructions
 defm "" : LMULSchedReads<"ReadVFMulAddV">;
 defm "" : LMULSchedReads<"ReadVFMulAddF">;
@@ -870,14 +882,14 @@ defm "" : LMULWriteResW<"WriteVNClipI", []>;
 // 13. Vector Floating-Point Instructions
 defm "" : LMULSEWWriteResF<"WriteVFALUV", []>;
 defm "" : LMULSEWWriteResF<"WriteVFALUF", []>;
-defm "" : LMULWriteResFW<"WriteVFWALUV", []>;
-defm "" : LMULWriteResFW<"WriteVFWALUF", []>;
+defm "" : LMULSEWWriteResFW<"WriteVFWALUV", []>;
+defm "" : LMULSEWWriteResFW<"WriteVFWALUF", []>;
 defm "" : LMULWriteRes<"WriteVFMulV", []>;
 defm "" : LMULWriteRes<"WriteVFMulF", []>;
 defm "" : LMULSEWWriteResF<"WriteVFDivV", []>;
 defm "" : LMULSEWWriteResF<"WriteVFDivF", []>;
-defm "" : LMULWriteResFW<"WriteVFWMulV", []>;
-defm "" : LMULWriteResFW<"WriteVFWMulF", []>;
+defm "" : LMULSEWWriteResFW<"WriteVFWMulV", []>;
+defm "" : LMULSEWWriteResFW<"WriteVFWMulF", []>;
 defm "" : LMULWriteRes<"WriteVFMulAddV", []>;
 defm "" : LMULWriteRes<"WriteVFMulAddF", []>;
 defm "" : LMULWriteResFW<"WriteVFWMulAddV", []>;
@@ -1026,14 +1038,14 @@ defm "" : LMULReadAdvanceW<"ReadVNClipX", 0>;
 // 13. Vector Floating-Point Instructions
 defm "" : LMULSEWReadAdvanceF<"ReadVFALUV", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFALUF", 0>;
-defm "" : LMULReadAdvanceFW<"ReadVFWALUV", 0>;
-defm "" : LMULReadAdvanceFW<"ReadVFWALUF", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWALUV", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWALUF", 0>;
 defm "" : LMULReadAdvance<"ReadVFMulV", 0>;
 defm "" : LMULReadAdvance<"ReadVFMulF", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFDivV", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFDivF", 0>;
-defm "" : LMULReadAdvanceFW<"ReadVFWMulV", 0>;
-defm "" : LMULReadAdvanceFW<"ReadVFWMulF", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulV", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulF", 0>;
 defm "" : LMULReadAdvance<"ReadVFMulAddV", 0>;
 defm "" : LMULReadAdvance<"ReadVFMulAddF", 0>;
 defm "" : LMULReadAdvanceFW<"ReadVFWMulAddV", 0>;
-- 
GitLab


From d309d7e4fe51a0ff0ce6430a8af585e2af2772a3 Mon Sep 17 00:00:00 2001
From: Michael Maitland <michaeltmaitland@gmail.com>
Date: Thu, 4 Apr 2024 08:57:15 -0700
Subject: [PATCH 064/464] [RISCV] Split PseudoVFMUL by SEW

Co-authored-by: Wang Pengcheng <wangpengcheng.pp@bytedance.com>
---
 llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td  | 13 +++++++------
 .../Target/RISCV/RISCVInstrInfoVSDPatterns.td    |  2 +-
 .../Target/RISCV/RISCVInstrInfoVVLPatterns.td    |  2 +-
 llvm/lib/Target/RISCV/RISCVSchedSiFive7.td       |  8 ++++----
 llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td    |  9 +++++----
 llvm/lib/Target/RISCV/RISCVScheduleV.td          | 16 ++++++++--------
 6 files changed, 26 insertions(+), 24 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index 1097eb4fb643..c00c650d2d36 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -2832,16 +2832,17 @@ multiclass VPseudoVDIV_VV_VX {
 
 multiclass VPseudoVFMUL_VV_VF_RM {
   foreach m = MxListF in {
-    defm "" : VPseudoBinaryFV_VV_RM<m>,
-              SchedBinary<"WriteVFMulV", "ReadVFMulV", "ReadVFMulV", m.MX,
-                          forceMergeOpRead=true>;
+    foreach e = SchedSEWSet<m.MX, isF=1>.val in
+      defm "" : VPseudoBinaryFV_VV_RM<m, "", sew=e>,
+                SchedBinary<"WriteVFMulV", "ReadVFMulV", "ReadVFMulV", m.MX, e,
+                            forceMergeOpRead=true>;
   }
 
   foreach f = FPList in {
     foreach m = f.MxList in {
-      defm "" : VPseudoBinaryV_VF_RM<m, f>,
+      defm "" : VPseudoBinaryV_VF_RM<m, f, "", sew=f.SEW>,
                 SchedBinary<"WriteVFMulF", "ReadVFMulV", "ReadVFMulF", m.MX,
-                            forceMergeOpRead=true>;
+                            f.SEW, forceMergeOpRead=true>;
     }
   }
 }
@@ -7106,7 +7107,7 @@ defm : VPatBinaryW_WV_WX_RM<"int_riscv_vfwsub_w", "PseudoVFWSUB",
 // 13.4. Vector Single-Width Floating-Point Multiply/Divide Instructions
 //===----------------------------------------------------------------------===//
 defm : VPatBinaryV_VV_VX_RM<"int_riscv_vfmul", "PseudoVFMUL",
-                            AllFloatVectors>;
+                            AllFloatVectors, isSEWAware=1>;
 defm : VPatBinaryV_VV_VX_RM<"int_riscv_vfdiv", "PseudoVFDIV",
                             AllFloatVectors, isSEWAware=1>;
 defm : VPatBinaryV_VX_RM<"int_riscv_vfrdiv", "PseudoVFRDIV",
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
index 37acd9667a54..26847e1bdd82 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
@@ -1213,7 +1213,7 @@ defm : VPatWidenBinaryFPSDNode_VV_VF_WV_WF_RM<fadd, "PseudoVFWADD">;
 defm : VPatWidenBinaryFPSDNode_VV_VF_WV_WF_RM<fsub, "PseudoVFWSUB">;
 
 // 13.4. Vector Single-Width Floating-Point Multiply/Divide Instructions
-defm : VPatBinaryFPSDNode_VV_VF_RM<any_fmul, "PseudoVFMUL">;
+defm : VPatBinaryFPSDNode_VV_VF_RM<any_fmul, "PseudoVFMUL", isSEWAware=1>;
 defm : VPatBinaryFPSDNode_VV_VF_RM<any_fdiv, "PseudoVFDIV", isSEWAware=1>;
 defm : VPatBinaryFPSDNode_R_VF_RM<any_fdiv, "PseudoVFRDIV", isSEWAware=1>;
 
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
index d2f575459e32..3225260dc6f7 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
@@ -2443,7 +2443,7 @@ defm : VPatBinaryFPWVL_VV_VF_WV_WF_RM<riscv_vfwsub_vl, riscv_vfwsub_w_vl,
                                       "PseudoVFWSUB", isSEWAware=1>;
 
 // 13.4. Vector Single-Width Floating-Point Multiply/Divide Instructions
-defm : VPatBinaryFPVL_VV_VF_RM<any_riscv_fmul_vl, "PseudoVFMUL">;
+defm : VPatBinaryFPVL_VV_VF_RM<any_riscv_fmul_vl, "PseudoVFMUL", isSEWAware=1>;
 defm : VPatBinaryFPVL_VV_VF_RM<any_riscv_fdiv_vl, "PseudoVFDIV", isSEWAware=1>;
 defm : VPatBinaryFPVL_R_VF_RM<any_riscv_fdiv_vl, "PseudoVFRDIV", isSEWAware=1>;
 
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
index 2b6fc5e59f80..af65b706b0e2 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
@@ -739,6 +739,8 @@ foreach mx = SchedMxListF in {
     let Latency = 8, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in {
       defm "" : LMULSEWWriteResMXSEW<"WriteVFALUV",  [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
       defm "" : LMULSEWWriteResMXSEW<"WriteVFALUF",  [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFMulV",  [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFMulF",  [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
     }
   }
 }
@@ -746,8 +748,6 @@ foreach mx = SchedMxList in {
   defvar Cycles = SiFive7GetCyclesDefault<mx>.c;
   defvar IsWorstCase = SiFive7IsWorstCaseMX<mx, SchedMxList>.c;
   let Latency = 8, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in {
-    defm "" : LMULWriteResMX<"WriteVFMulV",      [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVFMulF",      [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFMulAddV",   [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFMulAddF",   [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFRecpV",     [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
@@ -1157,8 +1157,8 @@ defm "" : LMULSEWReadAdvanceF<"ReadVFALUV", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFALUF", 0>;
 defm "" : LMULSEWReadAdvanceFW<"ReadVFWALUV", 0>;
 defm "" : LMULSEWReadAdvanceFW<"ReadVFWALUF", 0>;
-defm "" : LMULReadAdvance<"ReadVFMulV", 0>;
-defm "" : LMULReadAdvance<"ReadVFMulF", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFMulV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFMulF", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFDivV", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFDivF", 0>;
 defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulV", 0>;
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
index 00e92cde8522..5819ce5aa315 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
@@ -493,6 +493,9 @@ foreach mx = SchedMxListF in {
     let Latency = 6, ReleaseAtCycles = [LMulLat] in {
       defm "" : LMULSEWWriteResMXSEW<"WriteVFALUV",  [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
       defm "" : LMULSEWWriteResMXSEW<"WriteVFALUF",  [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFMulV",  [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFMulF",  [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
+
     }
   }
 }
@@ -500,8 +503,6 @@ foreach mx = SchedMxList in {
   defvar LMulLat = SiFiveP600GetLMulCycles<mx>.c;
   defvar IsWorstCase = SiFiveP600IsWorstCaseMX<mx, SchedMxList>.c;
   let Latency = 6, ReleaseAtCycles = [LMulLat] in {
-    defm "" : LMULWriteResMX<"WriteVFMulV",    [SiFiveP600VectorArith], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVFMulF",    [SiFiveP600VectorArith], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFMulAddV", [SiFiveP600VectorArith], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFMulAddF", [SiFiveP600VectorArith], mx, IsWorstCase>;
   }
@@ -945,8 +946,8 @@ defm "" : LMULSEWReadAdvanceF<"ReadVFALUV", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFALUF", 0>;
 defm "" : LMULSEWReadAdvanceFW<"ReadVFWALUV", 0>;
 defm "" : LMULSEWReadAdvanceFW<"ReadVFWALUF", 0>;
-defm "" : LMULReadAdvance<"ReadVFMulV", 0>;
-defm "" : LMULReadAdvance<"ReadVFMulF", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFMulV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFMulF", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFDivV", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFDivF", 0>;
 defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulV", 0>;
diff --git a/llvm/lib/Target/RISCV/RISCVScheduleV.td b/llvm/lib/Target/RISCV/RISCVScheduleV.td
index 8b5bd7a0c28a..9cb3cd1c3804 100644
--- a/llvm/lib/Target/RISCV/RISCVScheduleV.td
+++ b/llvm/lib/Target/RISCV/RISCVScheduleV.td
@@ -415,8 +415,8 @@ defm "" : LMULSEWSchedWritesF<"WriteVFALUF">;
 defm "" : LMULSEWSchedWritesFW<"WriteVFWALUV">;
 defm "" : LMULSEWSchedWritesFW<"WriteVFWALUF">;
 // 13.4. Vector Single-Width Floating-Point Multiply/Divide Instructions
-defm "" : LMULSchedWrites<"WriteVFMulV">;
-defm "" : LMULSchedWrites<"WriteVFMulF">;
+defm "" : LMULSEWSchedWritesF<"WriteVFMulV">;
+defm "" : LMULSEWSchedWritesF<"WriteVFMulF">;
 defm "" : LMULSEWSchedWritesF<"WriteVFDivV">;
 defm "" : LMULSEWSchedWritesF<"WriteVFDivF">;
 // 13.5. Vector Widening Floating-Point Multiply
@@ -640,8 +640,8 @@ defm "" : LMULSEWSchedReadsF<"ReadVFALUF">;
 defm "" : LMULSEWSchedReadsFW<"ReadVFWALUV">;
 defm "" : LMULSEWSchedReadsFW<"ReadVFWALUF">;
 // 13.4. Vector Single-Width Floating-Point Multiply/Divide Instructions
-defm "" : LMULSchedReads<"ReadVFMulV">;
-defm "" : LMULSchedReads<"ReadVFMulF">;
+defm "" : LMULSEWSchedReadsF<"ReadVFMulV">;
+defm "" : LMULSEWSchedReadsF<"ReadVFMulF">;
 defm "" : LMULSEWSchedReadsF<"ReadVFDivV">;
 defm "" : LMULSEWSchedReadsF<"ReadVFDivF">;
 // 13.5. Vector Widening Floating-Point Multiply
@@ -884,8 +884,8 @@ defm "" : LMULSEWWriteResF<"WriteVFALUV", []>;
 defm "" : LMULSEWWriteResF<"WriteVFALUF", []>;
 defm "" : LMULSEWWriteResFW<"WriteVFWALUV", []>;
 defm "" : LMULSEWWriteResFW<"WriteVFWALUF", []>;
-defm "" : LMULWriteRes<"WriteVFMulV", []>;
-defm "" : LMULWriteRes<"WriteVFMulF", []>;
+defm "" : LMULSEWWriteResF<"WriteVFMulV", []>;
+defm "" : LMULSEWWriteResF<"WriteVFMulF", []>;
 defm "" : LMULSEWWriteResF<"WriteVFDivV", []>;
 defm "" : LMULSEWWriteResF<"WriteVFDivF", []>;
 defm "" : LMULSEWWriteResFW<"WriteVFWMulV", []>;
@@ -1040,8 +1040,8 @@ defm "" : LMULSEWReadAdvanceF<"ReadVFALUV", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFALUF", 0>;
 defm "" : LMULSEWReadAdvanceFW<"ReadVFWALUV", 0>;
 defm "" : LMULSEWReadAdvanceFW<"ReadVFWALUF", 0>;
-defm "" : LMULReadAdvance<"ReadVFMulV", 0>;
-defm "" : LMULReadAdvance<"ReadVFMulF", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFMulV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFMulF", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFDivV", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFDivF", 0>;
 defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulV", 0>;
-- 
GitLab


From c6b7944be4dfbb1fb35301c670812726845acaa7 Mon Sep 17 00:00:00 2001
From: Michael Maitland <michaeltmaitland@gmail.com>
Date: Thu, 4 Apr 2024 10:52:54 -0700
Subject: [PATCH 065/464] [RISCV] Split single width floating point fused
 multiple-add pseudo instructions by SEW

Co-authored-by: Wang Pengcheng <wangpengcheng.pp@bytedance.com>
---
 llvm/lib/Target/RISCV/RISCVInstrInfo.cpp      | 254 +++++++++++-------
 .../Target/RISCV/RISCVInstrInfoVPseudos.td    |  93 ++++---
 .../Target/RISCV/RISCVInstrInfoVSDPatterns.td |   2 +-
 .../Target/RISCV/RISCVInstrInfoVVLPatterns.td |   4 +-
 llvm/lib/Target/RISCV/RISCVSchedSiFive7.td    |   8 +-
 llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td |  19 +-
 llvm/lib/Target/RISCV/RISCVScheduleV.td       |  16 +-
 7 files changed, 246 insertions(+), 150 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index 3b76e7158ffd..0eed604e251b 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -2712,31 +2712,54 @@ std::string RISCVInstrInfo::createMIROperandComment(
 }
 
 // clang-format off
-#define CASE_VFMA_OPCODE_COMMON(OP, TYPE, LMUL)                                \
+#define CASE_VMA_OPCODE_COMMON(OP, TYPE, LMUL)                                 \
   RISCV::PseudoV##OP##_##TYPE##_##LMUL
 
-#define CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE)                                    \
-  CASE_VFMA_OPCODE_COMMON(OP, TYPE, M1):                                       \
-  case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M2):                                  \
-  case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M4):                                  \
-  case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M8)
+#define CASE_VMA_OPCODE_LMULS_M1(OP, TYPE)                                     \
+  CASE_VMA_OPCODE_COMMON(OP, TYPE, M1):                                        \
+  case CASE_VMA_OPCODE_COMMON(OP, TYPE, M2):                                   \
+  case CASE_VMA_OPCODE_COMMON(OP, TYPE, M4):                                   \
+  case CASE_VMA_OPCODE_COMMON(OP, TYPE, M8)
 
-#define CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE)                                   \
-  CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF2):                                      \
-  case CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE)
+#define CASE_VMA_OPCODE_LMULS_MF2(OP, TYPE)                                    \
+  CASE_VMA_OPCODE_COMMON(OP, TYPE, MF2):                                       \
+  case CASE_VMA_OPCODE_LMULS_M1(OP, TYPE)
 
-#define CASE_VFMA_OPCODE_LMULS_MF4(OP, TYPE)                                   \
-  CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF4):                                      \
-  case CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE)
+#define CASE_VMA_OPCODE_LMULS_MF4(OP, TYPE)                                    \
+  CASE_VMA_OPCODE_COMMON(OP, TYPE, MF4):                                       \
+  case CASE_VMA_OPCODE_LMULS_MF2(OP, TYPE)
 
-#define CASE_VFMA_OPCODE_LMULS(OP, TYPE)                                       \
-  CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF8):                                      \
-  case CASE_VFMA_OPCODE_LMULS_MF4(OP, TYPE)
+#define CASE_VMA_OPCODE_LMULS(OP, TYPE)                                        \
+  CASE_VMA_OPCODE_COMMON(OP, TYPE, MF8):                                       \
+  case CASE_VMA_OPCODE_LMULS_MF4(OP, TYPE)
+
+// VFMA instructions are SEW specific.
+#define CASE_VFMA_OPCODE_COMMON(OP, TYPE, LMUL, SEW)                           \
+  RISCV::PseudoV##OP##_##TYPE##_##LMUL##_##SEW
+
+#define CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE, SEW)                               \
+  CASE_VFMA_OPCODE_COMMON(OP, TYPE, M1, SEW):                                  \
+  case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M2, SEW):                             \
+  case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M4, SEW):                             \
+  case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M8, SEW)
+
+#define CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE, SEW)                              \
+  CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF2, SEW):                                 \
+  case CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE, SEW)
+
+#define CASE_VFMA_OPCODE_LMULS_MF4(OP, TYPE, SEW)                              \
+  CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF4, SEW):                                 \
+  case CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE, SEW)
+
+#define CASE_VFMA_OPCODE_VV(OP)                                                \
+  CASE_VFMA_OPCODE_LMULS_MF4(OP, VV, E16):                                     \
+  case CASE_VFMA_OPCODE_LMULS_MF2(OP, VV, E32):                                \
+  case CASE_VFMA_OPCODE_LMULS_M1(OP, VV, E64)
 
 #define CASE_VFMA_SPLATS(OP)                                                   \
-  CASE_VFMA_OPCODE_LMULS_MF4(OP, VFPR16):                                      \
-  case CASE_VFMA_OPCODE_LMULS_MF2(OP, VFPR32):                                 \
-  case CASE_VFMA_OPCODE_LMULS_M1(OP, VFPR64)
+  CASE_VFMA_OPCODE_LMULS_MF4(OP, VFPR16, E16):                                 \
+  case CASE_VFMA_OPCODE_LMULS_MF2(OP, VFPR32, E32):                            \
+  case CASE_VFMA_OPCODE_LMULS_M1(OP, VFPR64, E64)
 // clang-format on
 
 bool RISCVInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
@@ -2776,16 +2799,16 @@ bool RISCVInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
   case CASE_VFMA_SPLATS(FNMSUB):
   case CASE_VFMA_SPLATS(FNMACC):
   case CASE_VFMA_SPLATS(FNMSAC):
-  case CASE_VFMA_OPCODE_LMULS_MF4(FMACC, VV):
-  case CASE_VFMA_OPCODE_LMULS_MF4(FMSAC, VV):
-  case CASE_VFMA_OPCODE_LMULS_MF4(FNMACC, VV):
-  case CASE_VFMA_OPCODE_LMULS_MF4(FNMSAC, VV):
-  case CASE_VFMA_OPCODE_LMULS(MADD, VX):
-  case CASE_VFMA_OPCODE_LMULS(NMSUB, VX):
-  case CASE_VFMA_OPCODE_LMULS(MACC, VX):
-  case CASE_VFMA_OPCODE_LMULS(NMSAC, VX):
-  case CASE_VFMA_OPCODE_LMULS(MACC, VV):
-  case CASE_VFMA_OPCODE_LMULS(NMSAC, VV): {
+  case CASE_VFMA_OPCODE_VV(FMACC):
+  case CASE_VFMA_OPCODE_VV(FMSAC):
+  case CASE_VFMA_OPCODE_VV(FNMACC):
+  case CASE_VFMA_OPCODE_VV(FNMSAC):
+  case CASE_VMA_OPCODE_LMULS(MADD, VX):
+  case CASE_VMA_OPCODE_LMULS(NMSUB, VX):
+  case CASE_VMA_OPCODE_LMULS(MACC, VX):
+  case CASE_VMA_OPCODE_LMULS(NMSAC, VX):
+  case CASE_VMA_OPCODE_LMULS(MACC, VV):
+  case CASE_VMA_OPCODE_LMULS(NMSAC, VV): {
     // If the tail policy is undisturbed we can't commute.
     assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags));
     if ((MI.getOperand(MI.getNumExplicitOperands() - 1).getImm() & 1) == 0)
@@ -2800,12 +2823,12 @@ bool RISCVInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
       return false;
     return true;
   }
-  case CASE_VFMA_OPCODE_LMULS_MF4(FMADD, VV):
-  case CASE_VFMA_OPCODE_LMULS_MF4(FMSUB, VV):
-  case CASE_VFMA_OPCODE_LMULS_MF4(FNMADD, VV):
-  case CASE_VFMA_OPCODE_LMULS_MF4(FNMSUB, VV):
-  case CASE_VFMA_OPCODE_LMULS(MADD, VV):
-  case CASE_VFMA_OPCODE_LMULS(NMSUB, VV): {
+  case CASE_VFMA_OPCODE_VV(FMADD):
+  case CASE_VFMA_OPCODE_VV(FMSUB):
+  case CASE_VFMA_OPCODE_VV(FNMADD):
+  case CASE_VFMA_OPCODE_VV(FNMSUB):
+  case CASE_VMA_OPCODE_LMULS(MADD, VV):
+  case CASE_VMA_OPCODE_LMULS(NMSUB, VV): {
     // If the tail policy is undisturbed we can't commute.
     assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags));
     if ((MI.getOperand(MI.getNumExplicitOperands() - 1).getImm() & 1) == 0)
@@ -2874,33 +2897,68 @@ bool RISCVInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
   return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
 }
 
-#define CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL)               \
+// clang-format off
+#define CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL)                \
   case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL:                                \
     Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL;                             \
     break;
 
-#define CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE)                   \
-  CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1)                       \
-  CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2)                       \
-  CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4)                       \
-  CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8)
+#define CASE_VMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE)                    \
+  CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1)                        \
+  CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2)                        \
+  CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4)                        \
+  CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8)
+
+#define CASE_VMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE)                   \
+  CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2)                       \
+  CASE_VMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE)
+
+#define CASE_VMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE)                   \
+  CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4)                       \
+  CASE_VMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE)
+
+#define CASE_VMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE)                       \
+  CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF8)                       \
+  CASE_VMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE)
+
+#define CASE_VMA_CHANGE_OPCODE_SPLATS(OLDOP, NEWOP)                            \
+  CASE_VMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VFPR16)                       \
+  CASE_VMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VFPR32)                       \
+  CASE_VMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VFPR64)
+
+// VFMA depends on SEW.
+#define CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL, SEW)          \
+  case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL##_##SEW:                        \
+    Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL##_##SEW;                     \
+    break;
+
+#define CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE, SEW)              \
+  CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1, SEW)                  \
+  CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2, SEW)                  \
+  CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4, SEW)                  \
+  CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8, SEW)
+
+#define CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE, SEW)             \
+  CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2, SEW)                 \
+  CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE, SEW)
 
-#define CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE)                  \
-  CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2)                      \
-  CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE)
+#define CASE_VFMA_CHANGE_OPCODE_VV(OLDOP, NEWOP)                               \
+  CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, VV, MF4, E16)                   \
+  CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VV, E16)                     \
+  CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VV, E32)
 
-#define CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE)                  \
-  CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4)                      \
-  CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE)
+#define CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE, SEW)             \
+  CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4, SEW)                 \
+  CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE, SEW)
 
-#define CASE_VFMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE)                      \
-  CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF8)                      \
-  CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE)
+#define CASE_VFMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE, SEW)                 \
+  CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF8, SEW)                 \
+  CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE, SEW)
 
 #define CASE_VFMA_CHANGE_OPCODE_SPLATS(OLDOP, NEWOP)                           \
-  CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VFPR16)                      \
-  CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VFPR32)                      \
-  CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VFPR64)
+  CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VFPR16, E16)                 \
+  CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VFPR32, E32)                 \
+  CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VFPR64, E64)
 
 MachineInstr *RISCVInstrInfo::commuteInstructionImpl(MachineInstr &MI,
                                                      bool NewMI,
@@ -2939,16 +2997,16 @@ MachineInstr *RISCVInstrInfo::commuteInstructionImpl(MachineInstr &MI,
   case CASE_VFMA_SPLATS(FNMADD):
   case CASE_VFMA_SPLATS(FNMSAC):
   case CASE_VFMA_SPLATS(FNMSUB):
-  case CASE_VFMA_OPCODE_LMULS_MF4(FMACC, VV):
-  case CASE_VFMA_OPCODE_LMULS_MF4(FMSAC, VV):
-  case CASE_VFMA_OPCODE_LMULS_MF4(FNMACC, VV):
-  case CASE_VFMA_OPCODE_LMULS_MF4(FNMSAC, VV):
-  case CASE_VFMA_OPCODE_LMULS(MADD, VX):
-  case CASE_VFMA_OPCODE_LMULS(NMSUB, VX):
-  case CASE_VFMA_OPCODE_LMULS(MACC, VX):
-  case CASE_VFMA_OPCODE_LMULS(NMSAC, VX):
-  case CASE_VFMA_OPCODE_LMULS(MACC, VV):
-  case CASE_VFMA_OPCODE_LMULS(NMSAC, VV): {
+  case CASE_VFMA_OPCODE_VV(FMACC):
+  case CASE_VFMA_OPCODE_VV(FMSAC):
+  case CASE_VFMA_OPCODE_VV(FNMACC):
+  case CASE_VFMA_OPCODE_VV(FNMSAC):
+  case CASE_VMA_OPCODE_LMULS(MADD, VX):
+  case CASE_VMA_OPCODE_LMULS(NMSUB, VX):
+  case CASE_VMA_OPCODE_LMULS(MACC, VX):
+  case CASE_VMA_OPCODE_LMULS(NMSAC, VX):
+  case CASE_VMA_OPCODE_LMULS(MACC, VV):
+  case CASE_VMA_OPCODE_LMULS(NMSAC, VV): {
     // It only make sense to toggle these between clobbering the
     // addend/subtrahend/minuend one of the multiplicands.
     assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index");
@@ -2965,16 +3023,16 @@ MachineInstr *RISCVInstrInfo::commuteInstructionImpl(MachineInstr &MI,
       CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMADD, FNMACC)
       CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMSAC, FNMSUB)
       CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMSUB, FNMSAC)
-      CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FMACC, FMADD, VV)
-      CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FMSAC, FMSUB, VV)
-      CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FNMACC, FNMADD, VV)
-      CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FNMSAC, FNMSUB, VV)
-      CASE_VFMA_CHANGE_OPCODE_LMULS(MACC, MADD, VX)
-      CASE_VFMA_CHANGE_OPCODE_LMULS(MADD, MACC, VX)
-      CASE_VFMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VX)
-      CASE_VFMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VX)
-      CASE_VFMA_CHANGE_OPCODE_LMULS(MACC, MADD, VV)
-      CASE_VFMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VV)
+      CASE_VFMA_CHANGE_OPCODE_VV(FMACC, FMADD)
+      CASE_VFMA_CHANGE_OPCODE_VV(FMSAC, FMSUB)
+      CASE_VFMA_CHANGE_OPCODE_VV(FNMACC, FNMADD)
+      CASE_VFMA_CHANGE_OPCODE_VV(FNMSAC, FNMSUB)
+      CASE_VMA_CHANGE_OPCODE_LMULS(MACC, MADD, VX)
+      CASE_VMA_CHANGE_OPCODE_LMULS(MADD, MACC, VX)
+      CASE_VMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VX)
+      CASE_VMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VX)
+      CASE_VMA_CHANGE_OPCODE_LMULS(MACC, MADD, VV)
+      CASE_VMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VV)
     }
 
     auto &WorkingMI = cloneIfNew(MI);
@@ -2982,12 +3040,12 @@ MachineInstr *RISCVInstrInfo::commuteInstructionImpl(MachineInstr &MI,
     return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
                                                    OpIdx1, OpIdx2);
   }
-  case CASE_VFMA_OPCODE_LMULS_MF4(FMADD, VV):
-  case CASE_VFMA_OPCODE_LMULS_MF4(FMSUB, VV):
-  case CASE_VFMA_OPCODE_LMULS_MF4(FNMADD, VV):
-  case CASE_VFMA_OPCODE_LMULS_MF4(FNMSUB, VV):
-  case CASE_VFMA_OPCODE_LMULS(MADD, VV):
-  case CASE_VFMA_OPCODE_LMULS(NMSUB, VV): {
+  case CASE_VFMA_OPCODE_VV(FMADD):
+  case CASE_VFMA_OPCODE_VV(FMSUB):
+  case CASE_VFMA_OPCODE_VV(FNMADD):
+  case CASE_VFMA_OPCODE_VV(FNMSUB):
+  case CASE_VMA_OPCODE_LMULS(MADD, VV):
+  case CASE_VMA_OPCODE_LMULS(NMSUB, VV): {
     assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index");
     // If one of the operands, is the addend we need to change opcode.
     // Otherwise we're just swapping 2 of the multiplicands.
@@ -2996,12 +3054,12 @@ MachineInstr *RISCVInstrInfo::commuteInstructionImpl(MachineInstr &MI,
       switch (MI.getOpcode()) {
         default:
           llvm_unreachable("Unexpected opcode");
-        CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FMADD, FMACC, VV)
-        CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FMSUB, FMSAC, VV)
-        CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FNMADD, FNMACC, VV)
-        CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FNMSUB, FNMSAC, VV)
-        CASE_VFMA_CHANGE_OPCODE_LMULS(MADD, MACC, VV)
-        CASE_VFMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VV)
+        CASE_VFMA_CHANGE_OPCODE_VV(FMADD, FMACC)
+        CASE_VFMA_CHANGE_OPCODE_VV(FMSUB, FMSAC)
+        CASE_VFMA_CHANGE_OPCODE_VV(FNMADD, FNMACC)
+        CASE_VFMA_CHANGE_OPCODE_VV(FNMSUB, FNMSAC)
+        CASE_VMA_CHANGE_OPCODE_LMULS(MADD, MACC, VV)
+        CASE_VMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VV)
       }
 
       auto &WorkingMI = cloneIfNew(MI);
@@ -3017,12 +3075,17 @@ MachineInstr *RISCVInstrInfo::commuteInstructionImpl(MachineInstr &MI,
   return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
 }
 
-#undef CASE_VFMA_CHANGE_OPCODE_SPLATS
-#undef CASE_VFMA_CHANGE_OPCODE_LMULS
-#undef CASE_VFMA_CHANGE_OPCODE_COMMON
-#undef CASE_VFMA_SPLATS
-#undef CASE_VFMA_OPCODE_LMULS
+#undef CASE_VMA_OPCODE_COMMON
+#undef CASE_VMA_OPCODE_LMULS_M1
+#undef CASE_VMA_OPCODE_LMULS_MF2
+#undef CASE_VMA_OPCODE_LMULS_MF4
+#undef CASE_VMA_OPCODE_LMULS
 #undef CASE_VFMA_OPCODE_COMMON
+#undef CASE_VFMA_OPCODE_LMULS_M1
+#undef CASE_VFMA_OPCODE_LMULS_MF2
+#undef CASE_VFMA_OPCODE_LMULS_MF4
+#undef CASE_VFMA_OPCODE_VV
+#undef CASE_VFMA_SPLATS
 
 // clang-format off
 #define CASE_WIDEOP_OPCODE_COMMON(OP, LMUL)                                    \
@@ -3190,10 +3253,17 @@ MachineInstr *RISCVInstrInfo::convertToThreeAddress(MachineInstr &MI,
   return MIB;
 }
 
-#undef CASE_WIDEOP_CHANGE_OPCODE_LMULS
-#undef CASE_WIDEOP_CHANGE_OPCODE_COMMON
-#undef CASE_WIDEOP_OPCODE_LMULS
 #undef CASE_WIDEOP_OPCODE_COMMON
+#undef CASE_WIDEOP_OPCODE_LMULS_MF4
+#undef CASE_WIDEOP_OPCODE_LMULS
+#undef CASE_WIDEOP_CHANGE_OPCODE_COMMON
+#undef CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4
+#undef CASE_WIDEOP_CHANGE_OPCODE_LMULS
+#undef CASE_FP_WIDEOP_OPCODE_COMMON
+#undef CASE_FP_WIDEOP_OPCODE_LMULS_MF4
+#undef CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON
+#undef CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_MF4
+#undef CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS
 
 void RISCVInstrInfo::mulImm(MachineFunction &MF, MachineBasicBlock &MBB,
                             MachineBasicBlock::iterator II, const DebugLoc &DL,
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index c00c650d2d36..f1159ee633ea 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -3234,15 +3234,17 @@ multiclass VPseudoTernaryWithPolicyRoundingMode<VReg RetClass,
                                                 DAGOperand Op2Class,
                                                 LMULInfo MInfo,
                                                 string Constraint = "",
+                                                int sew = 0,
                                                 bit Commutable = 0,
                                                 int TargetConstraintType = 1> {
   let VLMul = MInfo.value in {
+    defvar suffix = !if(sew, "_" # MInfo.MX # "_E" # sew, "_" # MInfo.MX);
     let isCommutable = Commutable in
-    def "_" # MInfo.MX :
+    def suffix :
         VPseudoTernaryNoMaskWithPolicyRoundingMode<RetClass, Op1Class,
                                                    Op2Class, Constraint,
                                                    TargetConstraintType>;
-    def "_" # MInfo.MX # "_MASK" :
+    def suffix # "_MASK" :
         VPseudoBinaryMaskPolicyRoundingMode<RetClass, Op1Class,
                                             Op2Class, Constraint,
                                             UsesVXRM_=0,
@@ -3256,9 +3258,9 @@ multiclass VPseudoTernaryV_VV_AAXA<LMULInfo m, string Constraint = ""> {
                                       Constraint, Commutable=1>;
 }
 
-multiclass VPseudoTernaryV_VV_AAXA_RM<LMULInfo m, string Constraint = ""> {
+multiclass VPseudoTernaryV_VV_AAXA_RM<LMULInfo m, string Constraint = "", int sew = 0> {
   defm _VV : VPseudoTernaryWithPolicyRoundingMode<m.vrclass, m.vrclass, m.vrclass, m,
-                                                  Constraint, Commutable=1>;
+                                                  Constraint, sew, Commutable=1>;
 }
 
 multiclass VPseudoTernaryV_VX_AAXA<LMULInfo m, string Constraint = ""> {
@@ -3272,10 +3274,11 @@ multiclass VPseudoTernaryV_VF_AAXA<LMULInfo m, FPR_Info f, string Constraint = "
                                               Commutable=1>;
 }
 
-multiclass VPseudoTernaryV_VF_AAXA_RM<LMULInfo m, FPR_Info f, string Constraint = ""> {
+multiclass VPseudoTernaryV_VF_AAXA_RM<LMULInfo m, FPR_Info f,
+                                      string Constraint = "", int sew = 0> {
   defm "_V" # f.FX : VPseudoTernaryWithPolicyRoundingMode<m.vrclass, f.fprclass,
                                                           m.vrclass, m, Constraint,
-                                                          Commutable=1>;
+                                                          sew, Commutable=1>;
 }
 
 multiclass VPseudoTernaryW_VV<LMULInfo m> {
@@ -3345,16 +3348,17 @@ multiclass VPseudoVMAC_VV_VX_AAXA<string Constraint = ""> {
 
 multiclass VPseudoVMAC_VV_VF_AAXA_RM<string Constraint = ""> {
   foreach m = MxListF in {
-    defm "" : VPseudoTernaryV_VV_AAXA_RM<m, Constraint>,
-              SchedTernary<"WriteVFMulAddV", "ReadVFMulAddV", "ReadVFMulAddV",
-                           "ReadVFMulAddV", m.MX>;
+    foreach e = SchedSEWSet<m.MX, isF=1>.val in
+      defm "" : VPseudoTernaryV_VV_AAXA_RM<m, Constraint, sew=e>,
+                SchedTernary<"WriteVFMulAddV", "ReadVFMulAddV", "ReadVFMulAddV",
+                             "ReadVFMulAddV", m.MX, e>;
   }
 
   foreach f = FPList in {
     foreach m = f.MxList in {
-      defm "" : VPseudoTernaryV_VF_AAXA_RM<m, f, Constraint>,
+      defm "" : VPseudoTernaryV_VF_AAXA_RM<m, f, Constraint, sew=f.SEW>,
                 SchedTernary<"WriteVFMulAddF", "ReadVFMulAddV", "ReadVFMulAddF",
-                             "ReadVFMulAddV", m.MX>;
+                             "ReadVFMulAddV", m.MX, f.SEW>;
     }
   }
 }
@@ -4461,23 +4465,26 @@ class VPatTernaryNoMaskWithPolicyRoundingMode<string intrinsic,
                                   ValueType result_type,
                                   ValueType op1_type,
                                   ValueType op2_type,
-                                  int sew,
+                                  int log2sew,
                                   LMULInfo vlmul,
                                   VReg result_reg_class,
                                   RegisterClass op1_reg_class,
-                                  DAGOperand op2_kind> :
+                                  DAGOperand op2_kind,
+                                  bit isSEWAware = 0> :
   Pat<(result_type (!cast<Intrinsic>(intrinsic)
                     (result_type result_reg_class:$rs3),
                     (op1_type op1_reg_class:$rs1),
                     (op2_type op2_kind:$rs2),
                     (XLenVT timm:$round),
                     VLOpFrag, (XLenVT timm:$policy))),
-                   (!cast<Instruction>(inst#"_"#kind#"_"#vlmul.MX)
+                   (!cast<Instruction>(!if(isSEWAware,
+                          inst#"_"#kind#"_"#vlmul.MX#"_E"#!shl(1, log2sew),
+                          inst#"_"#kind#"_"#vlmul.MX))
                     result_reg_class:$rs3,
                     (op1_type op1_reg_class:$rs1),
                     op2_kind:$rs2,
                     (XLenVT timm:$round),
-                    GPR:$vl, sew, (XLenVT timm:$policy))>;
+                    GPR:$vl, log2sew, (XLenVT timm:$policy))>;
 
 class VPatTernaryMask<string intrinsic,
                       string inst,
@@ -4536,11 +4543,12 @@ class VPatTernaryMaskPolicyRoundingMode<string intrinsic,
                                         ValueType op1_type,
                                         ValueType op2_type,
                                         ValueType mask_type,
-                                        int sew,
+                                        int log2sew,
                                         LMULInfo vlmul,
                                         VReg result_reg_class,
                                         RegisterClass op1_reg_class,
-                                        DAGOperand op2_kind> :
+                                        DAGOperand op2_kind,
+                                        bit isSEWAware = 0> :
   Pat<(result_type (!cast<Intrinsic>(intrinsic#"_mask")
                     (result_type result_reg_class:$rs3),
                     (op1_type op1_reg_class:$rs1),
@@ -4548,13 +4556,15 @@ class VPatTernaryMaskPolicyRoundingMode<string intrinsic,
                     (mask_type V0),
                     (XLenVT timm:$round),
                     VLOpFrag, (XLenVT timm:$policy))),
-                   (!cast<Instruction>(inst#"_"#kind#"_"#vlmul.MX # "_MASK")
+                   (!cast<Instruction>(!if(isSEWAware,
+                          inst#"_"#kind#"_"#vlmul.MX#"_E"#!shl(1, log2sew) # "_MASK",
+                          inst#"_"#kind#"_"#vlmul.MX # "_MASK"))
                     result_reg_class:$rs3,
                     (op1_type op1_reg_class:$rs1),
                     op2_kind:$rs2,
                     (mask_type V0),
                     (XLenVT timm:$round),
-                    GPR:$vl, sew, (XLenVT timm:$policy))>;
+                    GPR:$vl, log2sew, (XLenVT timm:$policy))>;
 
 class VPatTernaryMaskTA<string intrinsic,
                         string inst,
@@ -5564,15 +5574,16 @@ multiclass VPatTernaryWithPolicyRoundingMode<string intrinsic,
                                              LMULInfo vlmul,
                                              VReg result_reg_class,
                                              RegisterClass op1_reg_class,
-                                             DAGOperand op2_kind> {
+                                             DAGOperand op2_kind,
+                                             bit isSEWAware = 0> {
   def : VPatTernaryNoMaskWithPolicyRoundingMode<intrinsic, inst, kind, result_type,
                                                 op1_type, op2_type, sew, vlmul,
                                                 result_reg_class, op1_reg_class,
-                                                op2_kind>;
+                                                op2_kind, isSEWAware>;
   def : VPatTernaryMaskPolicyRoundingMode<intrinsic, inst, kind, result_type, op1_type,
                                                 op2_type, mask_type, sew, vlmul,
                                                 result_reg_class, op1_reg_class,
-                                                op2_kind>;
+                                                op2_kind, isSEWAware>;
 }
 
 multiclass VPatTernaryTA<string intrinsic,
@@ -5626,13 +5637,13 @@ multiclass VPatTernaryV_VV_AAXA<string intrinsic, string instruction,
 }
 
 multiclass VPatTernaryV_VV_AAXA_RM<string intrinsic, string instruction,
-                                list<VTypeInfo> vtilist> {
+                                list<VTypeInfo> vtilist, bit isSEWAware = 0> {
   foreach vti = vtilist in
     let Predicates = GetVTypePredicates<vti>.Predicates in
     defm : VPatTernaryWithPolicyRoundingMode<intrinsic, instruction, "VV",
                                              vti.Vector, vti.Vector, vti.Vector, vti.Mask,
                                              vti.Log2SEW, vti.LMul, vti.RegClass,
-                                             vti.RegClass, vti.RegClass>;
+                                             vti.RegClass, vti.RegClass, isSEWAware>;
 }
 
 multiclass VPatTernaryV_VX<string intrinsic, string instruction,
@@ -5657,14 +5668,14 @@ multiclass VPatTernaryV_VX_AAXA<string intrinsic, string instruction,
 }
 
 multiclass VPatTernaryV_VX_AAXA_RM<string intrinsic, string instruction,
-                           list<VTypeInfo> vtilist> {
+                           list<VTypeInfo> vtilist, bit isSEWAware = 0> {
   foreach vti = vtilist in
     let Predicates = GetVTypePredicates<vti>.Predicates in
     defm : VPatTernaryWithPolicyRoundingMode<intrinsic, instruction,
                                              "V"#vti.ScalarSuffix,
                                              vti.Vector, vti.Scalar, vti.Vector, vti.Mask,
                                              vti.Log2SEW, vti.LMul, vti.RegClass,
-                                             vti.ScalarRegClass, vti.RegClass>;
+                                             vti.ScalarRegClass, vti.RegClass, isSEWAware>;
 }
 
 multiclass VPatTernaryV_VI<string intrinsic, string instruction,
@@ -5742,9 +5753,9 @@ multiclass VPatTernaryV_VV_VX_AAXA<string intrinsic, string instruction,
       VPatTernaryV_VX_AAXA<intrinsic, instruction, vtilist>;
 
 multiclass VPatTernaryV_VV_VX_AAXA_RM<string intrinsic, string instruction,
-                              list<VTypeInfo> vtilist>
-    : VPatTernaryV_VV_AAXA_RM<intrinsic, instruction, vtilist>,
-      VPatTernaryV_VX_AAXA_RM<intrinsic, instruction, vtilist>;
+                              list<VTypeInfo> vtilist, bit isSEWAware = 0>
+    : VPatTernaryV_VV_AAXA_RM<intrinsic, instruction, vtilist, isSEWAware>,
+      VPatTernaryV_VX_AAXA_RM<intrinsic, instruction, vtilist, isSEWAware>;
 
 multiclass VPatTernaryV_VX_VI<string intrinsic, string instruction,
                               list<VTypeInfo> vtilist, Operand Imm_type = simm5>
@@ -7122,14 +7133,22 @@ defm : VPatBinaryW_VV_VX_RM<"int_riscv_vfwmul", "PseudoVFWMUL",
 //===----------------------------------------------------------------------===//
 // 13.6. Vector Single-Width Floating-Point Fused Multiply-Add Instructions
 //===----------------------------------------------------------------------===//
-defm : VPatTernaryV_VV_VX_AAXA_RM<"int_riscv_vfmacc", "PseudoVFMACC", AllFloatVectors>;
-defm : VPatTernaryV_VV_VX_AAXA_RM<"int_riscv_vfnmacc", "PseudoVFNMACC", AllFloatVectors>;
-defm : VPatTernaryV_VV_VX_AAXA_RM<"int_riscv_vfmsac", "PseudoVFMSAC", AllFloatVectors>;
-defm : VPatTernaryV_VV_VX_AAXA_RM<"int_riscv_vfnmsac", "PseudoVFNMSAC", AllFloatVectors>;
-defm : VPatTernaryV_VV_VX_AAXA_RM<"int_riscv_vfmadd", "PseudoVFMADD", AllFloatVectors>;
-defm : VPatTernaryV_VV_VX_AAXA_RM<"int_riscv_vfnmadd", "PseudoVFNMADD", AllFloatVectors>;
-defm : VPatTernaryV_VV_VX_AAXA_RM<"int_riscv_vfmsub", "PseudoVFMSUB", AllFloatVectors>;
-defm : VPatTernaryV_VV_VX_AAXA_RM<"int_riscv_vfnmsub", "PseudoVFNMSUB", AllFloatVectors>;
+defm : VPatTernaryV_VV_VX_AAXA_RM<"int_riscv_vfmacc", "PseudoVFMACC",
+                                  AllFloatVectors, isSEWAware=1>;
+defm : VPatTernaryV_VV_VX_AAXA_RM<"int_riscv_vfnmacc", "PseudoVFNMACC",
+                                  AllFloatVectors, isSEWAware=1>;
+defm : VPatTernaryV_VV_VX_AAXA_RM<"int_riscv_vfmsac", "PseudoVFMSAC",
+                                  AllFloatVectors, isSEWAware=1>;
+defm : VPatTernaryV_VV_VX_AAXA_RM<"int_riscv_vfnmsac", "PseudoVFNMSAC",
+                                  AllFloatVectors, isSEWAware=1>;
+defm : VPatTernaryV_VV_VX_AAXA_RM<"int_riscv_vfmadd", "PseudoVFMADD",
+                                  AllFloatVectors, isSEWAware=1>;
+defm : VPatTernaryV_VV_VX_AAXA_RM<"int_riscv_vfnmadd", "PseudoVFNMADD",
+                                  AllFloatVectors, isSEWAware=1>;
+defm : VPatTernaryV_VV_VX_AAXA_RM<"int_riscv_vfmsub", "PseudoVFMSUB",
+                                  AllFloatVectors, isSEWAware=1>;
+defm : VPatTernaryV_VV_VX_AAXA_RM<"int_riscv_vfnmsub", "PseudoVFNMSUB",
+                                  AllFloatVectors, isSEWAware=1>;
 
 //===----------------------------------------------------------------------===//
 // 13.7. Vector Widening Floating-Point Fused Multiply-Add Instructions
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
index 26847e1bdd82..d9dc373a628d 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
@@ -1224,7 +1224,7 @@ defm : VPatWidenBinaryFPSDNode_VV_VF_RM<fmul, "PseudoVFWMUL">;
 foreach fvti = AllFloatVectors in {
   // NOTE: We choose VFMADD because it has the most commuting freedom. So it
   // works best with how TwoAddressInstructionPass tries commuting.
-  defvar suffix = fvti.LMul.MX;
+  defvar suffix = fvti.LMul.MX # "_E" # fvti.SEW;
   let Predicates = GetVTypePredicates<fvti>.Predicates in {
     def : Pat<(fvti.Vector (any_fma fvti.RegClass:$rs1, fvti.RegClass:$rd,
                                     fvti.RegClass:$rs2)),
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
index 3225260dc6f7..a6f24438daa6 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
@@ -1850,7 +1850,7 @@ multiclass VPatFPMulAddVL_VV_VF<SDPatternOperator vop, string instruction_name>
 
 multiclass VPatFPMulAddVL_VV_VF_RM<SDPatternOperator vop, string instruction_name> {
   foreach vti = AllFloatVectors in {
-  defvar suffix = vti.LMul.MX;
+  defvar suffix = vti.LMul.MX # "_E" # vti.SEW;
   let Predicates = GetVTypePredicates<vti>.Predicates in {
     def : Pat<(vti.Vector (vop vti.RegClass:$rs1, vti.RegClass:$rd,
                                vti.RegClass:$rs2, (vti.Mask V0),
@@ -1916,7 +1916,7 @@ multiclass VPatFPMulAccVL_VV_VF<PatFrag vop, string instruction_name> {
 
 multiclass VPatFPMulAccVL_VV_VF_RM<PatFrag vop, string instruction_name> {
   foreach vti = AllFloatVectors in {
-  defvar suffix = vti.LMul.MX;
+  defvar suffix = vti.LMul.MX # "_E" # vti.SEW;
   let Predicates = GetVTypePredicates<vti>.Predicates in {
     def : Pat<(riscv_vmerge_vl (vti.Mask V0),
                            (vti.Vector (vop vti.RegClass:$rs1, vti.RegClass:$rs2,
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
index af65b706b0e2..16bfb509e085 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
@@ -741,6 +741,8 @@ foreach mx = SchedMxListF in {
       defm "" : LMULSEWWriteResMXSEW<"WriteVFALUF",  [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
       defm "" : LMULSEWWriteResMXSEW<"WriteVFMulV",  [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
       defm "" : LMULSEWWriteResMXSEW<"WriteVFMulF",  [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFMulAddV", [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFMulAddF", [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
     }
   }
 }
@@ -748,8 +750,6 @@ foreach mx = SchedMxList in {
   defvar Cycles = SiFive7GetCyclesDefault<mx>.c;
   defvar IsWorstCase = SiFive7IsWorstCaseMX<mx, SchedMxList>.c;
   let Latency = 8, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in {
-    defm "" : LMULWriteResMX<"WriteVFMulAddV",   [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVFMulAddF",   [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFRecpV",     [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFCvtIToFV",  [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFCvtFToIV",  [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
@@ -1163,8 +1163,8 @@ defm "" : LMULSEWReadAdvanceF<"ReadVFDivV", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFDivF", 0>;
 defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulV", 0>;
 defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulF", 0>;
-defm "" : LMULReadAdvance<"ReadVFMulAddV", 0>;
-defm "" : LMULReadAdvance<"ReadVFMulAddF", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFMulAddV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFMulAddF", 0>;
 defm "" : LMULReadAdvanceFW<"ReadVFWMulAddV", 0>;
 defm "" : LMULReadAdvanceFW<"ReadVFWMulAddF", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFSqrtV", 0>;
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
index 5819ce5aa315..0896f3358d23 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
@@ -499,13 +499,20 @@ foreach mx = SchedMxListF in {
     }
   }
 }
+
+foreach mx = SchedMxListF in {
+  foreach sew = SchedSEWSet<mx, isF=1>.val in {
+    defvar LMulLat = SiFiveP600GetLMulCycles<mx>.c;
+    defvar IsWorstCase = SiFiveP600IsWorstCaseMXSEW<mx, sew, SchedMxListF, isF=1>.c;
+    let Latency = 6, ReleaseAtCycles = [LMulLat] in {
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFMulAddV", [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFMulAddF", [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
+    }
+  }
+}
 foreach mx = SchedMxList in {
   defvar LMulLat = SiFiveP600GetLMulCycles<mx>.c;
   defvar IsWorstCase = SiFiveP600IsWorstCaseMX<mx, SchedMxList>.c;
-  let Latency = 6, ReleaseAtCycles = [LMulLat] in {
-    defm "" : LMULWriteResMX<"WriteVFMulAddV", [SiFiveP600VectorArith], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVFMulAddF", [SiFiveP600VectorArith], mx, IsWorstCase>;
-  }
   let Latency = 3, ReleaseAtCycles = [LMulLat] in {
     defm "" : LMULWriteResMX<"WriteVFCvtIToFV", [SiFiveP600VectorArith], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFCvtFToIV", [SiFiveP600VectorArith], mx, IsWorstCase>;
@@ -952,8 +959,8 @@ defm "" : LMULSEWReadAdvanceF<"ReadVFDivV", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFDivF", 0>;
 defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulV", 0>;
 defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulF", 0>;
-defm "" : LMULReadAdvance<"ReadVFMulAddV", 0>;
-defm "" : LMULReadAdvance<"ReadVFMulAddF", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFMulAddV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFMulAddF", 0>;
 defm "" : LMULReadAdvanceFW<"ReadVFWMulAddV", 0>;
 defm "" : LMULReadAdvanceFW<"ReadVFWMulAddF", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFSqrtV", 0>;
diff --git a/llvm/lib/Target/RISCV/RISCVScheduleV.td b/llvm/lib/Target/RISCV/RISCVScheduleV.td
index 9cb3cd1c3804..40090d1db887 100644
--- a/llvm/lib/Target/RISCV/RISCVScheduleV.td
+++ b/llvm/lib/Target/RISCV/RISCVScheduleV.td
@@ -423,8 +423,8 @@ defm "" : LMULSEWSchedWritesF<"WriteVFDivF">;
 defm "" : LMULSEWSchedWritesFW<"WriteVFWMulV">;
 defm "" : LMULSEWSchedWritesFW<"WriteVFWMulF">;
 // 13.6. Vector Single-Width Floating-Point Fused Multiply-Add Instructions
-defm "" : LMULSchedWrites<"WriteVFMulAddV">;
-defm "" : LMULSchedWrites<"WriteVFMulAddF">;
+defm "" : LMULSEWSchedWritesF<"WriteVFMulAddV">;
+defm "" : LMULSEWSchedWritesF<"WriteVFMulAddF">;
 // 13.7. Vector Widening Floating-Point Fused Multiply-Add Instructions
 defm "" : LMULSchedWritesFW<"WriteVFWMulAddV">;
 defm "" : LMULSchedWritesFW<"WriteVFWMulAddF">;
@@ -648,8 +648,8 @@ defm "" : LMULSEWSchedReadsF<"ReadVFDivF">;
 defm "" : LMULSEWSchedReadsFW<"ReadVFWMulV">;
 defm "" : LMULSEWSchedReadsFW<"ReadVFWMulF">;
 // 13.6. Vector Single-Width Floating-Point Fused Multiply-Add Instructions
-defm "" : LMULSchedReads<"ReadVFMulAddV">;
-defm "" : LMULSchedReads<"ReadVFMulAddF">;
+defm "" : LMULSEWSchedReadsF<"ReadVFMulAddV">;
+defm "" : LMULSEWSchedReadsF<"ReadVFMulAddF">;
 // 13.7. Vector Widening Floating-Point Fused Multiply-Add Instructions
 defm "" : LMULSchedReadsFW<"ReadVFWMulAddV">;
 defm "" : LMULSchedReadsFW<"ReadVFWMulAddF">;
@@ -890,8 +890,8 @@ defm "" : LMULSEWWriteResF<"WriteVFDivV", []>;
 defm "" : LMULSEWWriteResF<"WriteVFDivF", []>;
 defm "" : LMULSEWWriteResFW<"WriteVFWMulV", []>;
 defm "" : LMULSEWWriteResFW<"WriteVFWMulF", []>;
-defm "" : LMULWriteRes<"WriteVFMulAddV", []>;
-defm "" : LMULWriteRes<"WriteVFMulAddF", []>;
+defm "" : LMULSEWWriteResF<"WriteVFMulAddV", []>;
+defm "" : LMULSEWWriteResF<"WriteVFMulAddF", []>;
 defm "" : LMULWriteResFW<"WriteVFWMulAddV", []>;
 defm "" : LMULWriteResFW<"WriteVFWMulAddF", []>;
 defm "" : LMULSEWWriteResF<"WriteVFSqrtV", []>;
@@ -1046,8 +1046,8 @@ defm "" : LMULSEWReadAdvanceF<"ReadVFDivV", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFDivF", 0>;
 defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulV", 0>;
 defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulF", 0>;
-defm "" : LMULReadAdvance<"ReadVFMulAddV", 0>;
-defm "" : LMULReadAdvance<"ReadVFMulAddF", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFMulAddV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFMulAddF", 0>;
 defm "" : LMULReadAdvanceFW<"ReadVFWMulAddV", 0>;
 defm "" : LMULReadAdvanceFW<"ReadVFWMulAddF", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFSqrtV", 0>;
-- 
GitLab


From 43248ffea7b8de3a33b11768e8c21d2434252528 Mon Sep 17 00:00:00 2001
From: Michael Maitland <michaeltmaitland@gmail.com>
Date: Thu, 4 Apr 2024 11:58:29 -0700
Subject: [PATCH 066/464] [RISCV] Split widening floating point fused
 multiple-add pseudo instructions by SEW

Co-authored-by: Wang Pengcheng <wangpengcheng.pp@bytedance.com>
---
 llvm/lib/Target/RISCV/RISCVInstrInfo.cpp      |  4 +-
 .../Target/RISCV/RISCVInstrInfoVPseudos.td    | 68 ++++++++++---------
 .../Target/RISCV/RISCVInstrInfoVSDPatterns.td | 24 ++++---
 .../Target/RISCV/RISCVInstrInfoVVLPatterns.td |  5 +-
 llvm/lib/Target/RISCV/RISCVSchedSiFive7.td    |  8 +--
 llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td | 14 ++--
 llvm/lib/Target/RISCV/RISCVScheduleV.td       | 16 ++---
 .../CodeGen/RISCV/rvv/fixed-vectors-fmf.ll    |  4 +-
 .../RISCV/rvv/pass-fast-math-flags-sdnode.ll  |  4 +-
 9 files changed, 74 insertions(+), 73 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index 0eed604e251b..004a228ccc13 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -2943,9 +2943,9 @@ bool RISCVInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
   CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE, SEW)
 
 #define CASE_VFMA_CHANGE_OPCODE_VV(OLDOP, NEWOP)                               \
-  CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, VV, MF4, E16)                   \
   CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VV, E16)                     \
-  CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VV, E32)
+  CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VV, E32)                     \
+  CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VV, E64)
 
 #define CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE, SEW)             \
   CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4, SEW)                 \
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index f1159ee633ea..77b40999fc6a 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -3287,10 +3287,10 @@ multiclass VPseudoTernaryW_VV<LMULInfo m> {
                                       constraint, /*Commutable*/ 0, TargetConstraintType=3>;
 }
 
-multiclass VPseudoTernaryW_VV_RM<LMULInfo m> {
+multiclass VPseudoTernaryW_VV_RM<LMULInfo m, int sew = 0> {
   defvar constraint = "@earlyclobber $rd";
   defm _VV : VPseudoTernaryWithPolicyRoundingMode<m.wvrclass, m.vrclass, m.vrclass, m,
-                                                  constraint, /* Commutable */ 0,
+                                                  constraint, sew, /* Commutable */ 0,
                                                   TargetConstraintType=3>;
 }
 
@@ -3306,11 +3306,11 @@ multiclass VPseudoTernaryW_VF<LMULInfo m, FPR_Info f, int TargetConstraintType =
                                               m.vrclass, m, constraint, /*Commutable*/ 0, TargetConstraintType>;
 }
 
-multiclass VPseudoTernaryW_VF_RM<LMULInfo m, FPR_Info f> {
+multiclass VPseudoTernaryW_VF_RM<LMULInfo m, FPR_Info f, int sew = 0> {
   defvar constraint = "@earlyclobber $rd";
   defm "_V" # f.FX : VPseudoTernaryWithPolicyRoundingMode<m.wvrclass, f.fprclass,
                                                           m.vrclass, m, constraint,
-                                                          /* Commutable */ 0,
+                                                          sew, /* Commutable */ 0,
                                                           TargetConstraintType=3>;
 }
 
@@ -3396,16 +3396,17 @@ multiclass VPseudoVWMAC_VX {
 
 multiclass VPseudoVWMAC_VV_VF_RM {
   foreach m = MxListFW in {
-    defm "" : VPseudoTernaryW_VV_RM<m>,
-              SchedTernary<"WriteVFWMulAddV", "ReadVFWMulAddV",
-                           "ReadVFWMulAddV", "ReadVFWMulAddV", m.MX>;
+    foreach e = SchedSEWSet<m.MX, isF=1, isWidening=1>.val in
+      defm "" : VPseudoTernaryW_VV_RM<m, sew=e>,
+                SchedTernary<"WriteVFWMulAddV", "ReadVFWMulAddV",
+                             "ReadVFWMulAddV", "ReadVFWMulAddV", m.MX, e>;
   }
 
   foreach f = FPListW in {
     foreach m = f.MxListFW in {
-      defm "" : VPseudoTernaryW_VF_RM<m, f>,
+      defm "" : VPseudoTernaryW_VF_RM<m, f, sew=f.SEW>,
                 SchedTernary<"WriteVFWMulAddF", "ReadVFWMulAddV",
-                             "ReadVFWMulAddF", "ReadVFWMulAddV", m.MX>;
+                             "ReadVFWMulAddF", "ReadVFWMulAddV", m.MX, f.SEW>;
     }
   }
 }
@@ -3413,17 +3414,18 @@ multiclass VPseudoVWMAC_VV_VF_RM {
 multiclass VPseudoVWMAC_VV_VF_BF_RM {
   foreach m = MxListFW in {
     defvar mx = m.MX;
-    defm "" : VPseudoTernaryW_VV_RM<m>,
-              SchedTernary<"WriteVFWMulAddV", "ReadVFWMulAddV",
-                           "ReadVFWMulAddV", "ReadVFWMulAddV", mx>;
+    foreach e = SchedSEWSet<mx, isF=1, isWidening=1>.val in
+      defm "" : VPseudoTernaryW_VV_RM<m, sew=e>,
+                SchedTernary<"WriteVFWMulAddV", "ReadVFWMulAddV",
+                             "ReadVFWMulAddV", "ReadVFWMulAddV", mx, e>;
   }
 
   foreach f = BFPListW in {
     foreach m = f.MxListFW in {
       defvar mx = m.MX;
-      defm "" : VPseudoTernaryW_VF_RM<m, f>,
+      defm "" : VPseudoTernaryW_VF_RM<m, f, sew=f.SEW>,
                 SchedTernary<"WriteVFWMulAddF", "ReadVFWMulAddV",
-                             "ReadVFWMulAddF", "ReadVFWMulAddV", mx>;
+                             "ReadVFWMulAddF", "ReadVFWMulAddV", mx, f.SEW>;
     }
   }
 }
@@ -5703,7 +5705,7 @@ multiclass VPatTernaryW_VV<string intrinsic, string instruction,
 }
 
 multiclass VPatTernaryW_VV_RM<string intrinsic, string instruction,
-                           list<VTypeInfoToWide> vtilist> {
+                           list<VTypeInfoToWide> vtilist, bit isSEWAware = 0> {
   foreach vtiToWti = vtilist in {
     defvar vti = vtiToWti.Vti;
     defvar wti = vtiToWti.Wti;
@@ -5712,7 +5714,8 @@ multiclass VPatTernaryW_VV_RM<string intrinsic, string instruction,
     defm : VPatTernaryWithPolicyRoundingMode<intrinsic, instruction, "VV",
                                              wti.Vector, vti.Vector, vti.Vector,
                                              vti.Mask, vti.Log2SEW, vti.LMul,
-                                             wti.RegClass, vti.RegClass, vti.RegClass>;
+                                             wti.RegClass, vti.RegClass,
+                                             vti.RegClass, isSEWAware>;
   }
 }
 
@@ -5731,19 +5734,18 @@ multiclass VPatTernaryW_VX<string intrinsic, string instruction,
   }
 }
 
-multiclass VPatTernaryW_VX_RM<string intrinsic, string instruction,
-                           list<VTypeInfoToWide> vtilist> {
+multiclass
+    VPatTernaryW_VX_RM<string intrinsic, string instruction,
+                       list<VTypeInfoToWide> vtilist, bit isSEWAware = 0> {
   foreach vtiToWti = vtilist in {
     defvar vti = vtiToWti.Vti;
     defvar wti = vtiToWti.Wti;
     let Predicates = !listconcat(GetVTypePredicates<vti>.Predicates,
-                                 GetVTypePredicates<wti>.Predicates) in
-    defm : VPatTernaryWithPolicyRoundingMode<intrinsic, instruction,
-                                             "V"#vti.ScalarSuffix,
-                                             wti.Vector, vti.Scalar, vti.Vector,
-                                             vti.Mask, vti.Log2SEW, vti.LMul,
-                                             wti.RegClass, vti.ScalarRegClass,
-                                             vti.RegClass>;
+                                 GetVTypePredicates<wti>.Predicates) in defm
+        : VPatTernaryWithPolicyRoundingMode<
+              intrinsic, instruction, "V" #vti.ScalarSuffix, wti.Vector,
+              vti.Scalar, vti.Vector, vti.Mask, vti.Log2SEW, vti.LMul,
+              wti.RegClass, vti.ScalarRegClass, vti.RegClass, isSEWAware>;
   }
 }
 
@@ -5775,9 +5777,9 @@ multiclass VPatTernaryW_VV_VX<string intrinsic, string instruction,
       VPatTernaryW_VX<intrinsic, instruction, vtilist>;
 
 multiclass VPatTernaryW_VV_VX_RM<string intrinsic, string instruction,
-                              list<VTypeInfoToWide> vtilist>
-    : VPatTernaryW_VV_RM<intrinsic, instruction, vtilist>,
-      VPatTernaryW_VX_RM<intrinsic, instruction, vtilist>;
+                              list<VTypeInfoToWide> vtilist, bit isSEWAware = 1>
+    : VPatTernaryW_VV_RM<intrinsic, instruction, vtilist, isSEWAware>,
+      VPatTernaryW_VX_RM<intrinsic, instruction, vtilist, isSEWAware>;
 
 multiclass VPatBinaryM_VV_VX<string intrinsic, string instruction,
                              list<VTypeInfo> vtilist>
@@ -7154,16 +7156,16 @@ defm : VPatTernaryV_VV_VX_AAXA_RM<"int_riscv_vfnmsub", "PseudoVFNMSUB",
 // 13.7. Vector Widening Floating-Point Fused Multiply-Add Instructions
 //===----------------------------------------------------------------------===//
 defm : VPatTernaryW_VV_VX_RM<"int_riscv_vfwmacc", "PseudoVFWMACC",
-                             AllWidenableFloatVectors>;
+                             AllWidenableFloatVectors, isSEWAware=1>;
 defm : VPatTernaryW_VV_VX_RM<"int_riscv_vfwnmacc", "PseudoVFWNMACC",
-                             AllWidenableFloatVectors>;
+                             AllWidenableFloatVectors, isSEWAware=1>;
 defm : VPatTernaryW_VV_VX_RM<"int_riscv_vfwmsac", "PseudoVFWMSAC",
-                             AllWidenableFloatVectors>;
+                             AllWidenableFloatVectors, isSEWAware=1>;
 defm : VPatTernaryW_VV_VX_RM<"int_riscv_vfwnmsac", "PseudoVFWNMSAC",
-                             AllWidenableFloatVectors>;
+                             AllWidenableFloatVectors, isSEWAware=1>;
 let Predicates = [HasStdExtZvfbfwma] in
 defm : VPatTernaryW_VV_VX_RM<"int_riscv_vfwmaccbf16", "PseudoVFWMACCBF16",
-                              AllWidenableBFloatToFloatVectors>;
+                              AllWidenableBFloatToFloatVectors, isSEWAware=1>;
 
 //===----------------------------------------------------------------------===//
 // 13.8. Vector Floating-Point Square-Root Instruction
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
index d9dc373a628d..7c77449b4f6e 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
@@ -706,6 +706,7 @@ multiclass VPatWidenFPMulAccSDNode_VV_VF_RM<string instruction_name> {
   foreach vtiToWti = AllWidenableFloatVectors in {
     defvar vti = vtiToWti.Vti;
     defvar wti = vtiToWti.Wti;
+    defvar suffix = vti.LMul.MX # "_E" # vti.SEW;
     let Predicates = !listconcat(GetVTypePredicates<vti>.Predicates,
                                  GetVTypePredicates<wti>.Predicates) in {
       def : Pat<(fma (wti.Vector (riscv_fpextend_vl_oneuse
@@ -715,7 +716,7 @@ multiclass VPatWidenFPMulAccSDNode_VV_VF_RM<string instruction_name> {
                                       (vti.Vector vti.RegClass:$rs2),
                                       (vti.Mask true_mask), (XLenVT srcvalue))),
                      (wti.Vector wti.RegClass:$rd)),
-                (!cast<Instruction>(instruction_name#"_VV_"#vti.LMul.MX)
+                (!cast<Instruction>(instruction_name#"_VV_"#suffix)
                    wti.RegClass:$rd, vti.RegClass:$rs1, vti.RegClass:$rs2,
                    // Value to indicate no rounding mode change in
                    // RISCVInsertReadWriteCSR
@@ -727,7 +728,7 @@ multiclass VPatWidenFPMulAccSDNode_VV_VF_RM<string instruction_name> {
                                       (vti.Vector vti.RegClass:$rs2),
                                       (vti.Mask true_mask), (XLenVT srcvalue))),
                      (wti.Vector wti.RegClass:$rd)),
-                (!cast<Instruction>(instruction_name#"_V"#vti.ScalarSuffix#"_"#vti.LMul.MX)
+                (!cast<Instruction>(instruction_name#"_V"#vti.ScalarSuffix#"_"#suffix)
                    wti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2,
                    // Value to indicate no rounding mode change in
                    // RISCVInsertReadWriteCSR
@@ -741,6 +742,7 @@ multiclass VPatWidenFPNegMulAccSDNode_VV_VF_RM<string instruction_name> {
   foreach vtiToWti = AllWidenableFloatVectors in {
     defvar vti = vtiToWti.Vti;
     defvar wti = vtiToWti.Wti;
+    defvar suffix = vti.LMul.MX # "_E" # vti.SEW;
     let Predicates = !listconcat(GetVTypePredicates<vti>.Predicates,
                                  GetVTypePredicates<wti>.Predicates) in {
       def : Pat<(fma (fneg (wti.Vector (riscv_fpextend_vl_oneuse
@@ -749,7 +751,7 @@ multiclass VPatWidenFPNegMulAccSDNode_VV_VF_RM<string instruction_name> {
                      (riscv_fpextend_vl_oneuse (vti.Vector vti.RegClass:$rs2),
                                                (vti.Mask true_mask), (XLenVT srcvalue)),
                      (fneg wti.RegClass:$rd)),
-                (!cast<Instruction>(instruction_name#"_VV_"#vti.LMul.MX)
+                (!cast<Instruction>(instruction_name#"_VV_"#suffix)
                    wti.RegClass:$rd, vti.RegClass:$rs1, vti.RegClass:$rs2,
                    // Value to indicate no rounding mode change in
                    // RISCVInsertReadWriteCSR
@@ -760,7 +762,7 @@ multiclass VPatWidenFPNegMulAccSDNode_VV_VF_RM<string instruction_name> {
                                             (vti.Vector vti.RegClass:$rs2),
                                             (vti.Mask true_mask), (XLenVT srcvalue)))),
                      (fneg wti.RegClass:$rd)),
-                (!cast<Instruction>(instruction_name#"_V"#vti.ScalarSuffix#"_"#vti.LMul.MX)
+                (!cast<Instruction>(instruction_name#"_V"#vti.ScalarSuffix#"_"#suffix)
                    wti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2,
                    // Value to indicate no rounding mode change in
                    // RISCVInsertReadWriteCSR
@@ -770,7 +772,7 @@ multiclass VPatWidenFPNegMulAccSDNode_VV_VF_RM<string instruction_name> {
                      (riscv_fpextend_vl_oneuse (vti.Vector vti.RegClass:$rs2),
                                                (vti.Mask true_mask), (XLenVT srcvalue)),
                      (fneg wti.RegClass:$rd)),
-                (!cast<Instruction>(instruction_name#"_V"#vti.ScalarSuffix#"_"#vti.LMul.MX)
+                (!cast<Instruction>(instruction_name#"_V"#vti.ScalarSuffix#"_"#suffix)
                    wti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2,
                    // Value to indicate no rounding mode change in
                    // RISCVInsertReadWriteCSR
@@ -784,6 +786,7 @@ multiclass VPatWidenFPMulSacSDNode_VV_VF_RM<string instruction_name> {
   foreach vtiToWti = AllWidenableFloatVectors in {
     defvar vti = vtiToWti.Vti;
     defvar wti = vtiToWti.Wti;
+    defvar suffix = vti.LMul.MX # "_E" # vti.SEW;
     let Predicates = !listconcat(GetVTypePredicates<vti>.Predicates,
                                  GetVTypePredicates<wti>.Predicates) in {
       def : Pat<(fma (wti.Vector (riscv_fpextend_vl_oneuse
@@ -792,7 +795,7 @@ multiclass VPatWidenFPMulSacSDNode_VV_VF_RM<string instruction_name> {
                      (riscv_fpextend_vl_oneuse (vti.Vector vti.RegClass:$rs2),
                                                (vti.Mask true_mask), (XLenVT srcvalue)),
                      (fneg wti.RegClass:$rd)),
-                (!cast<Instruction>(instruction_name#"_VV_"#vti.LMul.MX)
+                (!cast<Instruction>(instruction_name#"_VV_"#suffix)
                    wti.RegClass:$rd, vti.RegClass:$rs1, vti.RegClass:$rs2,
                    // Value to indicate no rounding mode change in
                    // RISCVInsertReadWriteCSR
@@ -802,7 +805,7 @@ multiclass VPatWidenFPMulSacSDNode_VV_VF_RM<string instruction_name> {
                      (riscv_fpextend_vl_oneuse (vti.Vector vti.RegClass:$rs2),
                                                (vti.Mask true_mask), (XLenVT srcvalue)),
                      (fneg wti.RegClass:$rd)),
-                (!cast<Instruction>(instruction_name#"_V"#vti.ScalarSuffix#"_"#vti.LMul.MX)
+                (!cast<Instruction>(instruction_name#"_V"#vti.ScalarSuffix#"_"#suffix)
                    wti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2,
                    // Value to indicate no rounding mode change in
                    // RISCVInsertReadWriteCSR
@@ -816,6 +819,7 @@ multiclass VPatWidenFPNegMulSacSDNode_VV_VF_RM<string instruction_name> {
   foreach vtiToWti = AllWidenableFloatVectors in {
     defvar vti = vtiToWti.Vti;
     defvar wti = vtiToWti.Wti;
+    defvar suffix = vti.LMul.MX # "_E" # vti.SEW;
     let Predicates = !listconcat(GetVTypePredicates<vti>.Predicates,
                                  GetVTypePredicates<wti>.Predicates) in {
       def : Pat<(fma (fneg (wti.Vector (riscv_fpextend_vl_oneuse
@@ -824,7 +828,7 @@ multiclass VPatWidenFPNegMulSacSDNode_VV_VF_RM<string instruction_name> {
                      (riscv_fpextend_vl_oneuse (vti.Vector vti.RegClass:$rs2),
                                                (vti.Mask true_mask), (XLenVT srcvalue)),
                      wti.RegClass:$rd),
-                (!cast<Instruction>(instruction_name#"_VV_"#vti.LMul.MX)
+                (!cast<Instruction>(instruction_name#"_VV_"#suffix)
                    wti.RegClass:$rd, vti.RegClass:$rs1, vti.RegClass:$rs2,
                    // Value to indicate no rounding mode change in
                    // RISCVInsertReadWriteCSR
@@ -835,7 +839,7 @@ multiclass VPatWidenFPNegMulSacSDNode_VV_VF_RM<string instruction_name> {
                                             (vti.Vector vti.RegClass:$rs2),
                                             (vti.Mask true_mask), (XLenVT srcvalue)))),
                      wti.RegClass:$rd),
-                (!cast<Instruction>(instruction_name#"_V"#vti.ScalarSuffix#"_"#vti.LMul.MX)
+                (!cast<Instruction>(instruction_name#"_V"#vti.ScalarSuffix#"_"#suffix)
                    wti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2,
                    // Value to indicate no rounding mode change in
                    // RISCVInsertReadWriteCSR
@@ -845,7 +849,7 @@ multiclass VPatWidenFPNegMulSacSDNode_VV_VF_RM<string instruction_name> {
                      (riscv_fpextend_vl_oneuse (vti.Vector vti.RegClass:$rs2),
                                                (vti.Mask true_mask), (XLenVT srcvalue)),
                      wti.RegClass:$rd),
-                (!cast<Instruction>(instruction_name#"_V"#vti.ScalarSuffix#"_"#vti.LMul.MX)
+                (!cast<Instruction>(instruction_name#"_V"#vti.ScalarSuffix#"_"#suffix)
                    wti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2,
                    // Value to indicate no rounding mode change in
                    // RISCVInsertReadWriteCSR
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
index a6f24438daa6..322c055306e8 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
@@ -1994,13 +1994,14 @@ multiclass VPatWidenFPMulAccVL_VV_VF_RM<SDNode vop, string instruction_name> {
   foreach vtiToWti = AllWidenableFloatVectors in {
     defvar vti = vtiToWti.Vti;
     defvar wti = vtiToWti.Wti;
+    defvar suffix = vti.LMul.MX # "_E" # vti.SEW;
     let Predicates = !listconcat(GetVTypePredicates<vti>.Predicates,
                                  GetVTypePredicates<wti>.Predicates) in {
       def : Pat<(vop (vti.Vector vti.RegClass:$rs1),
                      (vti.Vector vti.RegClass:$rs2),
                      (wti.Vector wti.RegClass:$rd), (vti.Mask V0),
                      VLOpFrag),
-                (!cast<Instruction>(instruction_name#"_VV_"#vti.LMul.MX #"_MASK")
+                (!cast<Instruction>(instruction_name#"_VV_"#suffix#"_MASK")
                    wti.RegClass:$rd, vti.RegClass:$rs1, vti.RegClass:$rs2,
                    (vti.Mask V0),
                    // Value to indicate no rounding mode change in
@@ -2011,7 +2012,7 @@ multiclass VPatWidenFPMulAccVL_VV_VF_RM<SDNode vop, string instruction_name> {
                      (vti.Vector vti.RegClass:$rs2),
                      (wti.Vector wti.RegClass:$rd), (vti.Mask V0),
                      VLOpFrag),
-                (!cast<Instruction>(instruction_name#"_V"#vti.ScalarSuffix#"_"#vti.LMul.MX #"_MASK")
+                (!cast<Instruction>(instruction_name#"_V"#vti.ScalarSuffix#"_"#suffix#"_MASK")
                    wti.RegClass:$rd, vti.ScalarRegClass:$rs1, vti.RegClass:$rs2,
                    (vti.Mask V0),
                    // Value to indicate no rounding mode change in
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
index 16bfb509e085..1dd4afa4ee48 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
@@ -799,6 +799,8 @@ foreach mx = SchedMxListFW in {
       defm "" : LMULSEWWriteResMXSEW<"WriteVFWALUF", [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
       defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulV", [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
       defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulF", [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulAddV", [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulAddF", [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
     }
   }
 }
@@ -806,10 +808,8 @@ foreach mx = SchedMxListFW in {
   defvar Cycles = SiFive7GetCyclesDefault<mx>.c;
   defvar IsWorstCase = SiFive7IsWorstCaseMX<mx, SchedMxListFW>.c;
   let Latency = 8, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in {
-    defm "" : LMULWriteResMX<"WriteVFWMulAddV",  [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFWCvtFToIV", [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFWCvtFToFV", [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVFWMulAddF",  [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
   }
 }
 // Narrowing
@@ -1165,8 +1165,8 @@ defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulV", 0>;
 defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulF", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFMulAddV", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFMulAddF", 0>;
-defm "" : LMULReadAdvanceFW<"ReadVFWMulAddV", 0>;
-defm "" : LMULReadAdvanceFW<"ReadVFWMulAddF", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulAddV", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulAddF", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFSqrtV", 0>;
 defm "" : LMULReadAdvance<"ReadVFRecpV", 0>;
 defm "" : LMULReadAdvance<"ReadVFMinMaxV", 0>;
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
index 0896f3358d23..f63dcf20e683 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
@@ -558,17 +558,11 @@ foreach mx = SchedMxListFW in {
       defm "" : LMULSEWWriteResMXSEW<"WriteVFWALUF", [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
       defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulV", [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
       defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulF", [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulAddV", [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulAddF", [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
     }
   }
 }
-foreach mx = SchedMxListFW in {
-  defvar LMulLat = SiFiveP600GetLMulCycles<mx>.c;
-  defvar IsWorstCase = SiFiveP600IsWorstCaseMX<mx, SchedMxListFW>.c;
-  let Latency = 6, ReleaseAtCycles = [LMulLat] in {
-    defm "" : LMULWriteResMX<"WriteVFWMulAddV", [SiFiveP600VectorArith], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVFWMulAddF", [SiFiveP600VectorArith], mx, IsWorstCase>;
-  }
-}
 // Narrowing
 foreach mx = SchedMxListW in {
   defvar LMulLat = SiFiveP600GetLMulCycles<mx>.c;
@@ -961,8 +955,8 @@ defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulV", 0>;
 defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulF", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFMulAddV", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFMulAddF", 0>;
-defm "" : LMULReadAdvanceFW<"ReadVFWMulAddV", 0>;
-defm "" : LMULReadAdvanceFW<"ReadVFWMulAddF", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulAddV", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulAddF", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFSqrtV", 0>;
 defm "" : LMULReadAdvance<"ReadVFRecpV", 0>;
 defm "" : LMULReadAdvance<"ReadVFCmpV", 0>;
diff --git a/llvm/lib/Target/RISCV/RISCVScheduleV.td b/llvm/lib/Target/RISCV/RISCVScheduleV.td
index 40090d1db887..b82e86e9fcd5 100644
--- a/llvm/lib/Target/RISCV/RISCVScheduleV.td
+++ b/llvm/lib/Target/RISCV/RISCVScheduleV.td
@@ -426,8 +426,8 @@ defm "" : LMULSEWSchedWritesFW<"WriteVFWMulF">;
 defm "" : LMULSEWSchedWritesF<"WriteVFMulAddV">;
 defm "" : LMULSEWSchedWritesF<"WriteVFMulAddF">;
 // 13.7. Vector Widening Floating-Point Fused Multiply-Add Instructions
-defm "" : LMULSchedWritesFW<"WriteVFWMulAddV">;
-defm "" : LMULSchedWritesFW<"WriteVFWMulAddF">;
+defm "" : LMULSEWSchedWritesFW<"WriteVFWMulAddV">;
+defm "" : LMULSEWSchedWritesFW<"WriteVFWMulAddF">;
 // 13.8. Vector Floating-Point Square-Root Instruction
 defm "" : LMULSEWSchedWritesF<"WriteVFSqrtV">;
 // 13.9. Vector Floating-Point Reciprocal Square-Root Estimate Instruction
@@ -651,8 +651,8 @@ defm "" : LMULSEWSchedReadsFW<"ReadVFWMulF">;
 defm "" : LMULSEWSchedReadsF<"ReadVFMulAddV">;
 defm "" : LMULSEWSchedReadsF<"ReadVFMulAddF">;
 // 13.7. Vector Widening Floating-Point Fused Multiply-Add Instructions
-defm "" : LMULSchedReadsFW<"ReadVFWMulAddV">;
-defm "" : LMULSchedReadsFW<"ReadVFWMulAddF">;
+defm "" : LMULSEWSchedReadsFW<"ReadVFWMulAddV">;
+defm "" : LMULSEWSchedReadsFW<"ReadVFWMulAddF">;
 // 13.8. Vector Floating-Point Square-Root Instruction
 defm "" : LMULSEWSchedReadsF<"ReadVFSqrtV">;
 // 13.9. Vector Floating-Point Reciprocal Square-Root Estimate Instruction
@@ -892,8 +892,8 @@ defm "" : LMULSEWWriteResFW<"WriteVFWMulV", []>;
 defm "" : LMULSEWWriteResFW<"WriteVFWMulF", []>;
 defm "" : LMULSEWWriteResF<"WriteVFMulAddV", []>;
 defm "" : LMULSEWWriteResF<"WriteVFMulAddF", []>;
-defm "" : LMULWriteResFW<"WriteVFWMulAddV", []>;
-defm "" : LMULWriteResFW<"WriteVFWMulAddF", []>;
+defm "" : LMULSEWWriteResFW<"WriteVFWMulAddV", []>;
+defm "" : LMULSEWWriteResFW<"WriteVFWMulAddF", []>;
 defm "" : LMULSEWWriteResF<"WriteVFSqrtV", []>;
 defm "" : LMULWriteRes<"WriteVFRecpV", []>;
 defm "" : LMULWriteRes<"WriteVFMinMaxV", []>;
@@ -1048,8 +1048,8 @@ defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulV", 0>;
 defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulF", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFMulAddV", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFMulAddF", 0>;
-defm "" : LMULReadAdvanceFW<"ReadVFWMulAddV", 0>;
-defm "" : LMULReadAdvanceFW<"ReadVFWMulAddF", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulAddV", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulAddF", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFSqrtV", 0>;
 defm "" : LMULReadAdvance<"ReadVFRecpV", 0>;
 defm "" : LMULReadAdvance<"ReadVFMinMaxV", 0>;
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fmf.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fmf.ll
index 1c1440c3e6d0..a4851e9838fb 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fmf.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fmf.ll
@@ -9,8 +9,8 @@ define <2 x double> @foo(<2 x double> %x, <2 x double> %y) {
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   [[COPY:%[0-9]+]]:vr = COPY $v9
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vr = COPY $v8
-  ; CHECK-NEXT:   [[PseudoVFADD_VV_M1_:%[0-9]+]]:vr = nnan ninf nsz arcp contract afn reassoc nofpexcept PseudoVFADD_VV_M1 $noreg, [[COPY1]], [[COPY]], 7, 2, 6 /* e64 */, 1 /* ta, mu */, implicit $frm
-  ; CHECK-NEXT:   $v8 = COPY [[PseudoVFADD_VV_M1_]]
+  ; CHECK-NEXT:   [[PseudoVFADD_VV_M1_E64_:%[0-9]+]]:vr = nnan ninf nsz arcp contract afn reassoc nofpexcept PseudoVFADD_VV_M1_E64 $noreg, [[COPY1]], [[COPY]], 7, 2, 6 /* e64 */, 1 /* ta, mu */, implicit $frm
+  ; CHECK-NEXT:   $v8 = COPY [[PseudoVFADD_VV_M1_E64_]]
   ; CHECK-NEXT:   PseudoRET implicit $v8
   %1 = fadd fast <2 x double> %x, %y
   ret <2 x double> %1
diff --git a/llvm/test/CodeGen/RISCV/rvv/pass-fast-math-flags-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/pass-fast-math-flags-sdnode.ll
index 1ad8e2d66392..8457f3d2c149 100644
--- a/llvm/test/CodeGen/RISCV/rvv/pass-fast-math-flags-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/pass-fast-math-flags-sdnode.ll
@@ -15,8 +15,8 @@ define <vscale x 1 x double> @foo(<vscale x 1 x double> %x, <vscale x 1 x double
   ; CHECK-NEXT:   [[SLLI:%[0-9]+]]:gpr = SLLI [[COPY]], 32
   ; CHECK-NEXT:   [[SRLI:%[0-9]+]]:gprnox0 = SRLI killed [[SLLI]], 32
   ; CHECK-NEXT:   $v0 = COPY [[COPY1]]
-  ; CHECK-NEXT:   [[PseudoVFMUL_VV_M1_MASK:%[0-9]+]]:vrnov0 = nnan ninf nsz arcp contract afn reassoc nofpexcept PseudoVFMUL_VV_M1_MASK $noreg, [[COPY3]], [[COPY2]], $v0, 7, killed [[SRLI]], 6 /* e64 */, 1 /* ta, mu */, implicit $frm
-  ; CHECK-NEXT:   $v8 = COPY [[PseudoVFMUL_VV_M1_MASK]]
+  ; CHECK-NEXT:   [[PseudoVFMUL_VV_M1_E64_MASK:%[0-9]+]]:vrnov0 = nnan ninf nsz arcp contract afn reassoc nofpexcept PseudoVFMUL_VV_M1_E64_MASK $noreg, [[COPY3]], [[COPY2]], $v0, 7, killed [[SRLI]], 6 /* e64 */, 1 /* ta, mu */, implicit $frm
+  ; CHECK-NEXT:   $v8 = COPY [[PseudoVFMUL_VV_M1_E64_MASK]]
   ; CHECK-NEXT:   PseudoRET implicit $v8
   %1 = call fast <vscale x 1 x double> @llvm.vp.fmul.nxv1f64(<vscale x 1 x double> %x, <vscale x 1 x double> %y, <vscale x 1 x i1> %m, i32 %vl)
   ret <vscale x 1 x double> %1
-- 
GitLab


From b8e1ff322104e5445371328a4fbc9c1d2bf82f01 Mon Sep 17 00:00:00 2001
From: Michael Maitland <michaeltmaitland@gmail.com>
Date: Thu, 4 Apr 2024 12:16:39 -0700
Subject: [PATCH 067/464] [RISCV] Split PseudoVFRSQRT7 and PseudoVFREC7 by SEW

Co-authored-by: Wang Pengcheng <wangpengcheng.pp@bytedance.com>
---
 .../Target/RISCV/RISCVInstrInfoVPseudos.td    | 62 +++++++++++--------
 llvm/lib/Target/RISCV/RISCVSchedSiFive7.td    |  4 +-
 llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td | 11 +++-
 llvm/lib/Target/RISCV/RISCVScheduleV.td       |  8 +--
 4 files changed, 52 insertions(+), 33 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index 77b40999fc6a..a5f56a1c209c 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -2577,14 +2577,16 @@ multiclass VPseudoVSQR_V_RM {
 multiclass VPseudoVRCP_V {
   foreach m = MxListF in {
     defvar mx = m.MX;
-    let VLMul = m.value in {
-      def "_V_" # mx
-          : VPseudoUnaryNoMask<m.vrclass, m.vrclass>,
-            SchedUnary<"WriteVFRecpV", "ReadVFRecpV", mx, forceMergeOpRead=true>;
-      def "_V_" # mx # "_MASK"
-          : VPseudoUnaryMask<m.vrclass, m.vrclass>,
-            RISCVMaskedPseudo<MaskIdx = 2>,
-            SchedUnary<"WriteVFRecpV", "ReadVFRecpV", mx, forceMergeOpRead=true>;
+    foreach e = SchedSEWSet<mx, isF=1>.val in {
+      let VLMul = m.value in {
+        def "_V_" # mx # "_E" # e
+            : VPseudoUnaryNoMask<m.vrclass, m.vrclass>,
+              SchedUnary<"WriteVFRecpV", "ReadVFRecpV", mx, e, forceMergeOpRead=true>;
+        def "_V_" # mx # "_E" # e # "_MASK"
+            : VPseudoUnaryMask<m.vrclass, m.vrclass>,
+              RISCVMaskedPseudo<MaskIdx = 2>,
+              SchedUnary<"WriteVFRecpV", "ReadVFRecpV", mx, e, forceMergeOpRead=true>;
+      }
     }
   }
 }
@@ -2592,14 +2594,16 @@ multiclass VPseudoVRCP_V {
 multiclass VPseudoVRCP_V_RM {
   foreach m = MxListF in {
     defvar mx = m.MX;
-    let VLMul = m.value in {
-      def "_V_" # mx
-          : VPseudoUnaryNoMaskRoundingMode<m.vrclass, m.vrclass>,
-            SchedUnary<"WriteVFRecpV", "ReadVFRecpV", mx, forceMergeOpRead=true>;
-      def "_V_" # mx # "_MASK"
-          : VPseudoUnaryMaskRoundingMode<m.vrclass, m.vrclass>,
-            RISCVMaskedPseudo<MaskIdx = 2>,
-            SchedUnary<"WriteVFRecpV", "ReadVFRecpV", mx, forceMergeOpRead=true>;
+    foreach e = SchedSEWSet<mx, isF=1>.val in {
+      let VLMul = m.value in {
+        def "_V_" # mx # "_E" # e
+            : VPseudoUnaryNoMaskRoundingMode<m.vrclass, m.vrclass>,
+              SchedUnary<"WriteVFRecpV", "ReadVFRecpV", mx, e, forceMergeOpRead=true>;
+        def "_V_" # mx # "_E" # e # "_MASK"
+            : VPseudoUnaryMaskRoundingMode<m.vrclass, m.vrclass>,
+              RISCVMaskedPseudo<MaskIdx = 2>,
+              SchedUnary<"WriteVFRecpV", "ReadVFRecpV", mx, e, forceMergeOpRead=true>;
+      }
     }
   }
 }
@@ -3939,12 +3943,16 @@ class VPatUnaryNoMask<string intrinsic_name,
                       int log2sew,
                       LMULInfo vlmul,
                       VReg result_reg_class,
-                      VReg op2_reg_class> :
+                      VReg op2_reg_class,
+                      bit isSEWAware = 0> :
   Pat<(result_type (!cast<Intrinsic>(intrinsic_name)
                    (result_type result_reg_class:$merge),
                    (op2_type op2_reg_class:$rs2),
                    VLOpFrag)),
-                   (!cast<Instruction>(inst#"_"#kind#"_"#vlmul.MX)
+                   (!cast<Instruction>(
+                     !if(isSEWAware,
+                         inst#"_"#kind#"_"#vlmul.MX#"_E"#!shl(1, log2sew),
+                         inst#"_"#kind#"_"#vlmul.MX))
                    (result_type result_reg_class:$merge),
                    (op2_type op2_reg_class:$rs2),
                    GPR:$vl, log2sew, TU_MU)>;
@@ -3983,13 +3991,17 @@ class VPatUnaryMask<string intrinsic_name,
                     int log2sew,
                     LMULInfo vlmul,
                     VReg result_reg_class,
-                    VReg op2_reg_class> :
+                    VReg op2_reg_class,
+                    bit isSEWAware = 0> :
   Pat<(result_type (!cast<Intrinsic>(intrinsic_name#"_mask")
                    (result_type result_reg_class:$merge),
                    (op2_type op2_reg_class:$rs2),
                    (mask_type V0),
                    VLOpFrag, (XLenVT timm:$policy))),
-                   (!cast<Instruction>(inst#"_"#kind#"_"#vlmul.MX#"_MASK")
+                   (!cast<Instruction>(
+                      !if(isSEWAware,
+                          inst#"_"#kind#"_"#vlmul.MX#"_E"#!shl(1, log2sew)#"_MASK",
+                          inst#"_"#kind#"_"#vlmul.MX#"_MASK"))
                    (result_type result_reg_class:$merge),
                    (op2_type op2_reg_class:$rs2),
                    (mask_type V0), GPR:$vl, log2sew, (XLenVT timm:$policy))>;
@@ -4681,15 +4693,15 @@ multiclass VPatUnaryV_VF<string intrinsic, string instruction, string suffix,
 }
 
 multiclass VPatUnaryV_V<string intrinsic, string instruction,
-                        list<VTypeInfo> vtilist> {
+                        list<VTypeInfo> vtilist, bit isSEWAware = 0> {
   foreach vti = vtilist in {
     let Predicates = GetVTypePredicates<vti>.Predicates in {
       def : VPatUnaryNoMask<intrinsic, instruction, "V",
                             vti.Vector, vti.Vector, vti.Log2SEW,
-                            vti.LMul, vti.RegClass, vti.RegClass>;
+                            vti.LMul, vti.RegClass, vti.RegClass, isSEWAware>;
       def : VPatUnaryMask<intrinsic, instruction, "V",
                           vti.Vector, vti.Vector, vti.Mask, vti.Log2SEW,
-                          vti.LMul, vti.RegClass, vti.RegClass>;
+                          vti.LMul, vti.RegClass, vti.RegClass, isSEWAware>;
     }
   }
 }
@@ -7175,12 +7187,12 @@ defm : VPatUnaryV_V_RM<"int_riscv_vfsqrt", "PseudoVFSQRT", AllFloatVectors, isSE
 //===----------------------------------------------------------------------===//
 // 13.9. Vector Floating-Point Reciprocal Square-Root Estimate Instruction
 //===----------------------------------------------------------------------===//
-defm : VPatUnaryV_V<"int_riscv_vfrsqrt7", "PseudoVFRSQRT7", AllFloatVectors>;
+defm : VPatUnaryV_V<"int_riscv_vfrsqrt7", "PseudoVFRSQRT7", AllFloatVectors, isSEWAware=1>;
 
 //===----------------------------------------------------------------------===//
 // 13.10. Vector Floating-Point Reciprocal Estimate Instruction
 //===----------------------------------------------------------------------===//
-defm : VPatUnaryV_V_RM<"int_riscv_vfrec7", "PseudoVFREC7", AllFloatVectors>;
+defm : VPatUnaryV_V_RM<"int_riscv_vfrec7", "PseudoVFREC7", AllFloatVectors, isSEWAware=1>;
 
 //===----------------------------------------------------------------------===//
 // 13.11. Vector Floating-Point Min/Max Instructions
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
index 1dd4afa4ee48..2a13cb4beac1 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
@@ -743,6 +743,7 @@ foreach mx = SchedMxListF in {
       defm "" : LMULSEWWriteResMXSEW<"WriteVFMulF",  [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
       defm "" : LMULSEWWriteResMXSEW<"WriteVFMulAddV", [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
       defm "" : LMULSEWWriteResMXSEW<"WriteVFMulAddF", [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFRecpV",   [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
     }
   }
 }
@@ -750,7 +751,6 @@ foreach mx = SchedMxList in {
   defvar Cycles = SiFive7GetCyclesDefault<mx>.c;
   defvar IsWorstCase = SiFive7IsWorstCaseMX<mx, SchedMxList>.c;
   let Latency = 8, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in {
-    defm "" : LMULWriteResMX<"WriteVFRecpV",     [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFCvtIToFV",  [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFCvtFToIV",  [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
   }
@@ -1168,7 +1168,7 @@ defm "" : LMULSEWReadAdvanceF<"ReadVFMulAddF", 0>;
 defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulAddV", 0>;
 defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulAddF", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFSqrtV", 0>;
-defm "" : LMULReadAdvance<"ReadVFRecpV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFRecpV", 0>;
 defm "" : LMULReadAdvance<"ReadVFMinMaxV", 0>;
 defm "" : LMULReadAdvance<"ReadVFMinMaxF", 0>;
 defm "" : LMULReadAdvance<"ReadVFSgnjV", 0>;
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
index f63dcf20e683..80090a002714 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
@@ -510,6 +510,14 @@ foreach mx = SchedMxListF in {
     }
   }
 }
+foreach mx = SchedMxListF in {
+  foreach sew = SchedSEWSet<mx, isF=1>.val in {
+    defvar LMulLat = SiFiveP600GetLMulCycles<mx>.c;
+    defvar IsWorstCase = SiFiveP600IsWorstCaseMXSEW<mx, sew, SchedMxListF, isF=1>.c;
+    let Latency = 2, ReleaseAtCycles = [LMulLat] in
+    defm "" : LMULSEWWriteResMXSEW<"WriteVFRecpV", [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
+  }
+}
 foreach mx = SchedMxList in {
   defvar LMulLat = SiFiveP600GetLMulCycles<mx>.c;
   defvar IsWorstCase = SiFiveP600IsWorstCaseMX<mx, SchedMxList>.c;
@@ -520,7 +528,6 @@ foreach mx = SchedMxList in {
   let Latency = 2, ReleaseAtCycles = [LMulLat] in {
     defm "" : LMULWriteResMX<"WriteVFCmpV",  [SiFiveP600VectorArith], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFCmpF",  [SiFiveP600VectorArith], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVFRecpV", [SiFiveP600VectorArith], mx, IsWorstCase>;
   }
   let Latency = 1, ReleaseAtCycles = [LMulLat] in {
     defm "" : LMULWriteResMX<"WriteVFSgnjV",   [SiFiveP600VectorArith], mx, IsWorstCase>;
@@ -958,7 +965,7 @@ defm "" : LMULSEWReadAdvanceF<"ReadVFMulAddF", 0>;
 defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulAddV", 0>;
 defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulAddF", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFSqrtV", 0>;
-defm "" : LMULReadAdvance<"ReadVFRecpV", 0>;
+defm "" : LMULSEWReadAdvance<"ReadVFRecpV", 0>;
 defm "" : LMULReadAdvance<"ReadVFCmpV", 0>;
 defm "" : LMULReadAdvance<"ReadVFCmpF", 0>;
 defm "" : LMULReadAdvance<"ReadVFMinMaxV", 0>;
diff --git a/llvm/lib/Target/RISCV/RISCVScheduleV.td b/llvm/lib/Target/RISCV/RISCVScheduleV.td
index b82e86e9fcd5..6070482ea175 100644
--- a/llvm/lib/Target/RISCV/RISCVScheduleV.td
+++ b/llvm/lib/Target/RISCV/RISCVScheduleV.td
@@ -432,7 +432,7 @@ defm "" : LMULSEWSchedWritesFW<"WriteVFWMulAddF">;
 defm "" : LMULSEWSchedWritesF<"WriteVFSqrtV">;
 // 13.9. Vector Floating-Point Reciprocal Square-Root Estimate Instruction
 // 13.10. Vector Floating-Point Reciprocal Estimate Instruction
-defm "" : LMULSchedWrites<"WriteVFRecpV">;
+defm "" : LMULSEWSchedWritesF<"WriteVFRecpV">;
 // 13.11. Vector Floating-Point MIN/MAX Instructions
 defm "" : LMULSchedWrites<"WriteVFMinMaxV">;
 defm "" : LMULSchedWrites<"WriteVFMinMaxF">;
@@ -657,7 +657,7 @@ defm "" : LMULSEWSchedReadsFW<"ReadVFWMulAddF">;
 defm "" : LMULSEWSchedReadsF<"ReadVFSqrtV">;
 // 13.9. Vector Floating-Point Reciprocal Square-Root Estimate Instruction
 // 13.10. Vector Floating-Point Reciprocal Estimate Instruction
-defm "" : LMULSchedReads<"ReadVFRecpV">;
+defm "" : LMULSEWSchedReadsF<"ReadVFRecpV">;
 // 13.11. Vector Floating-Point MIN/MAX Instructions
 defm "" : LMULSchedReads<"ReadVFMinMaxV">;
 defm "" : LMULSchedReads<"ReadVFMinMaxF">;
@@ -895,7 +895,7 @@ defm "" : LMULSEWWriteResF<"WriteVFMulAddF", []>;
 defm "" : LMULSEWWriteResFW<"WriteVFWMulAddV", []>;
 defm "" : LMULSEWWriteResFW<"WriteVFWMulAddF", []>;
 defm "" : LMULSEWWriteResF<"WriteVFSqrtV", []>;
-defm "" : LMULWriteRes<"WriteVFRecpV", []>;
+defm "" : LMULSEWWriteResF<"WriteVFRecpV", []>;
 defm "" : LMULWriteRes<"WriteVFMinMaxV", []>;
 defm "" : LMULWriteRes<"WriteVFMinMaxF", []>;
 defm "" : LMULWriteRes<"WriteVFSgnjV", []>;
@@ -1051,7 +1051,7 @@ defm "" : LMULSEWReadAdvanceF<"ReadVFMulAddF", 0>;
 defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulAddV", 0>;
 defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulAddF", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFSqrtV", 0>;
-defm "" : LMULReadAdvance<"ReadVFRecpV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFRecpV", 0>;
 defm "" : LMULReadAdvance<"ReadVFMinMaxV", 0>;
 defm "" : LMULReadAdvance<"ReadVFMinMaxF", 0>;
 defm "" : LMULReadAdvance<"ReadVFSgnjV", 0>;
-- 
GitLab


From f4e3226a6c2bd89247192e4aae395a8deb3868b1 Mon Sep 17 00:00:00 2001
From: Xiaoyang Liu <siujoeng.lau@gmail.com>
Date: Fri, 12 Apr 2024 10:09:05 -0400
Subject: [PATCH 068/464] [libc++][ranges] LWG3736: move_iterator missing
 disable_sized_sentinel_for specialization (#85611)

This pull request implements LWG3736: move_iterator missing
disable_sized_sentinel_for specialization.
---
 libcxx/docs/Status/Cxx23Issues.csv            |  2 +-
 libcxx/include/__iterator/move_iterator.h     |  6 ++++
 libcxx/include/iterator                       |  5 +++
 .../sized_sentinel.compile.pass.cpp           | 32 +++++++++++++++++++
 4 files changed, 44 insertions(+), 1 deletion(-)
 create mode 100644 libcxx/test/std/iterators/predef.iterators/move.iterators/sized_sentinel.compile.pass.cpp

diff --git a/libcxx/docs/Status/Cxx23Issues.csv b/libcxx/docs/Status/Cxx23Issues.csv
index fa82e67ac7db..04f6e23375ac 100644
--- a/libcxx/docs/Status/Cxx23Issues.csv
+++ b/libcxx/docs/Status/Cxx23Issues.csv
@@ -201,7 +201,7 @@
 "`3677 <https://wg21.link/LWG3677>`__","Is a cv-qualified ``pair`` specially handled in uses-allocator construction?", "November 2022","|Complete|","18.0",""
 "`3717 <https://wg21.link/LWG3717>`__","``common_view::end`` should improve ``random_access_range`` case", "November 2022","","","|ranges|"
 "`3732 <https://wg21.link/LWG3732>`__","``prepend_range`` and ``append_range`` can't be amortized constant time", "November 2022","|Nothing to do|","","|ranges|"
-"`3736 <https://wg21.link/LWG3736>`__","``move_iterator`` missing ``disable_sized_sentinel_for`` specialization", "November 2022","","","|ranges|"
+"`3736 <https://wg21.link/LWG3736>`__","``move_iterator`` missing ``disable_sized_sentinel_for`` specialization", "November 2022","|Complete|","19.0","|ranges|"
 "`3737 <https://wg21.link/LWG3737>`__","``take_view::sentinel`` should provide ``operator-``", "November 2022","","","|ranges|"
 "`3738 <https://wg21.link/LWG3738>`__","Missing preconditions for ``take_view`` constructor", "November 2022","|Complete|","16.0","|ranges|"
 "`3743 <https://wg21.link/LWG3743>`__","``ranges::to``'s reserve may be ill-formed", "November 2022","","","|ranges|"
diff --git a/libcxx/include/__iterator/move_iterator.h b/libcxx/include/__iterator/move_iterator.h
index eefd5b374845..6382c947060b 100644
--- a/libcxx/include/__iterator/move_iterator.h
+++ b/libcxx/include/__iterator/move_iterator.h
@@ -330,6 +330,12 @@ operator+(typename move_iterator<_Iter>::difference_type __n, const move_iterato
 }
 #endif // _LIBCPP_STD_VER >= 20
 
+#if _LIBCPP_STD_VER >= 20
+template <class _Iter1, class _Iter2>
+  requires(!sized_sentinel_for<_Iter1, _Iter2>)
+inline constexpr bool disable_sized_sentinel_for<move_iterator<_Iter1>, move_iterator<_Iter2>> = true;
+#endif // _LIBCPP_STD_VER >= 20
+
 template <class _Iter>
 inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 move_iterator<_Iter> make_move_iterator(_Iter __i) {
   return move_iterator<_Iter>(std::move(__i));
diff --git a/libcxx/include/iterator b/libcxx/include/iterator
index 5779bf828711..1b9e7eaf0c1e 100644
--- a/libcxx/include/iterator
+++ b/libcxx/include/iterator
@@ -475,6 +475,11 @@ constexpr move_iterator<Iterator> operator+(   // constexpr in C++17
 template <class Iterator>   // constexpr in C++17
 constexpr  move_iterator<Iterator> make_move_iterator(const Iterator& i);
 
+template<class Iterator1, class Iterator2>
+    requires (!sized_sentinel_for<Iterator1, Iterator2>)
+  inline constexpr bool disable_sized_sentinel_for<move_iterator<Iterator1>,    // since C++20
+                                                    move_iterator<Iterator2>> = true;
+
 template<semiregular S>
 class move_sentinel {
 public:
diff --git a/libcxx/test/std/iterators/predef.iterators/move.iterators/sized_sentinel.compile.pass.cpp b/libcxx/test/std/iterators/predef.iterators/move.iterators/sized_sentinel.compile.pass.cpp
new file mode 100644
index 000000000000..cb49086dd680
--- /dev/null
+++ b/libcxx/test/std/iterators/predef.iterators/move.iterators/sized_sentinel.compile.pass.cpp
@@ -0,0 +1,32 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+#include <iterator>
+
+#include "test_iterators.h"
+
+using sized_it = random_access_iterator<int*>;
+static_assert(std::sized_sentinel_for<sized_it, sized_it>);
+static_assert(std::sized_sentinel_for<std::move_iterator<sized_it>, std::move_iterator<sized_it>>);
+
+struct unsized_it {
+  using value_type      = int;
+  using difference_type = std::ptrdiff_t;
+
+  value_type& operator*() const;
+  bool operator==(const unsized_it&) const;
+  difference_type operator-(const unsized_it&) const { return 0; }
+};
+
+template <>
+inline constexpr bool std::disable_sized_sentinel_for<unsized_it, unsized_it> = true;
+
+static_assert(!std::sized_sentinel_for<unsized_it, unsized_it>);
+static_assert(!std::sized_sentinel_for<std::move_iterator<unsized_it>, std::move_iterator<unsized_it>>);
-- 
GitLab


From 9d8ccbc1be78fb4ba0ad1c6ead2dc04b24c66668 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12@gmail.com>
Date: Fri, 12 Apr 2024 16:13:54 +0200
Subject: [PATCH 069/464] [lldb][test] Add tests for evaluating local variables
 whose name clashes with Objective-C types (#87807)

Depends on https://github.com/llvm/llvm-project/pull/87767
---
 .../objc-builtin-types/TestObjCBuiltinTypes.py      | 13 +++++++++++++
 .../API/lang/objcxx/objc-builtin-types/main.cpp     |  8 +++++++-
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/lldb/test/API/lang/objcxx/objc-builtin-types/TestObjCBuiltinTypes.py b/lldb/test/API/lang/objcxx/objc-builtin-types/TestObjCBuiltinTypes.py
index 611c73889990..3cdca31b8969 100644
--- a/lldb/test/API/lang/objcxx/objc-builtin-types/TestObjCBuiltinTypes.py
+++ b/lldb/test/API/lang/objcxx/objc-builtin-types/TestObjCBuiltinTypes.py
@@ -13,6 +13,7 @@ class TestObjCBuiltinTypes(TestBase):
         # Find the line numbers to break inside main().
         self.main_source = "main.cpp"
         self.break_line = line_number(self.main_source, "// Set breakpoint here.")
+        self.bar_break_line = line_number(self.main_source, "return id + Class")
 
     @add_test_categories(["pyapi"])
     def test_with_python_api(self):
@@ -26,6 +27,11 @@ class TestObjCBuiltinTypes(TestBase):
         bpt = target.BreakpointCreateByLocation(self.main_source, self.break_line)
         self.assertTrue(bpt, VALID_BREAKPOINT)
 
+        bar_bpt = target.BreakpointCreateByLocation(
+            self.main_source, self.bar_break_line
+        )
+        self.assertTrue(bar_bpt, VALID_BREAKPOINT)
+
         # Now launch the process, and do not stop at entry point.
         process = target.LaunchSimple(None, None, self.get_process_working_directory())
 
@@ -52,3 +58,10 @@ class TestObjCBuiltinTypes(TestBase):
             patterns=["\(id\) \$.* = nil"],
         )
         self.expect("expr --language C++ -- id my_id = 0; my_id", error=True)
+
+        lldbutil.continue_to_breakpoint(process, bar_bpt)
+
+        self.expect_expr("id", result_value="12", result_type="int")
+        self.expect_expr("Class", result_value="15", result_type="int")
+        self.expect("expr --language Objective-C++ -- id", error=True)
+        self.expect("expr --language Objective-C++ -- Class", error=True)
diff --git a/lldb/test/API/lang/objcxx/objc-builtin-types/main.cpp b/lldb/test/API/lang/objcxx/objc-builtin-types/main.cpp
index 6dd8cbc6e9fe..5b35ec0f0b8c 100644
--- a/lldb/test/API/lang/objcxx/objc-builtin-types/main.cpp
+++ b/lldb/test/API/lang/objcxx/objc-builtin-types/main.cpp
@@ -2,8 +2,14 @@ namespace ns {
   typedef int id;
 };
 
+int bar() {
+  int id = 12;
+  int Class = 15;
+  return id + Class;
+}
+
 int main()
 {
   ns::id foo = 0;
-  return foo; // Set breakpoint here.
+  return foo + bar(); // Set breakpoint here.
 }
-- 
GitLab


From bcf047a4ed9afc49f851156698a6d277ab2c00ea Mon Sep 17 00:00:00 2001
From: Bjorn Pettersson <bjorn.a.pettersson@ericsson.com>
Date: Wed, 3 Apr 2024 19:00:17 +0200
Subject: [PATCH 070/464] [ARM][PowerPC] Add regression tests for narrowing
 load in TargetLowering::SimplifySetCC

These test cases show some miscomplies for big-endian when dealing
with non byte-sized loads. One part of the problem is that LLVM IR
isn't really telling where the padding goes for non byte-sized
loads/stores. So currently TargetLowering::SimplifySetCC can't assume
anything about it. But the implementation also do not consider that
the TypeStoreSize could be larger than the TypeSize, resulting in
the offset calculation being wrong for big-endian.

Pre-commit for https://github.com/llvm/llvm-project/pull/87646
---
 .../CodeGen/ARM/simplifysetcc_narrow_load.ll  | 665 ++++++++++++++++++
 .../PowerPC/simplifysetcc_narrow_load.ll      | 390 ++++++++++
 2 files changed, 1055 insertions(+)
 create mode 100644 llvm/test/CodeGen/ARM/simplifysetcc_narrow_load.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/simplifysetcc_narrow_load.ll

diff --git a/llvm/test/CodeGen/ARM/simplifysetcc_narrow_load.ll b/llvm/test/CodeGen/ARM/simplifysetcc_narrow_load.ll
new file mode 100644
index 000000000000..2cba4b46f9a8
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/simplifysetcc_narrow_load.ll
@@ -0,0 +1,665 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -O1 -mtriple arm -o - %s | FileCheck --check-prefix CHECK-LE %s
+; RUN: llc -O1 -mtriple armv7 -o - %s | FileCheck --check-prefix CHECK-V7-LE %s
+; RUN: llc -O1 -mtriple armeb -o - %s | FileCheck --check-prefix CHECK-BE %s
+; RUN: llc -O1 -mtriple armv7eb -o - %s | FileCheck --check-prefix CHECK-V7-BE %s
+
+; A collection of regression tests to verify the load-narrowing part of
+; TargetLowering::SimplifySetCC (and/or other similar rewrites such as
+; combining AND+LOAD into ZEXTLOAD).
+;
+; Using both arm and armv7 to show that alignment restrictions are
+; considered for the narrowed load (armv7 is a bit more relaxed when it
+; comes to unaligned memory accesses).
+
+;--------------------------------------------------------------------------
+; Test non byte-sized types.
+;
+; As long as LLVM IR isn't defining where the padding goes we can't really
+; optimize these (without adding a target lowering hook that can inform
+; ISel about which bits are padding).
+; --------------------------------------------------------------------------
+
+define i1 @test_129_15_0(ptr %y) {
+; CHECK-LE-LABEL: test_129_15_0:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    ldrh r0, [r0]
+; CHECK-LE-NEXT:    mov r1, #255
+; CHECK-LE-NEXT:    orr r1, r1, #32512
+; CHECK-LE-NEXT:    ands r0, r0, r1
+; CHECK-LE-NEXT:    movne r0, #1
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-V7-LE-LABEL: test_129_15_0:
+; CHECK-V7-LE:       @ %bb.0:
+; CHECK-V7-LE-NEXT:    ldrh r0, [r0]
+; CHECK-V7-LE-NEXT:    bfc r0, #15, #17
+; CHECK-V7-LE-NEXT:    cmp r0, #0
+; CHECK-V7-LE-NEXT:    movwne r0, #1
+; CHECK-V7-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: test_129_15_0:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    ldrh r0, [r0, #14]
+; CHECK-BE-NEXT:    mov r1, #255
+; CHECK-BE-NEXT:    orr r1, r1, #32512
+; CHECK-BE-NEXT:    ands r0, r0, r1
+; CHECK-BE-NEXT:    movne r0, #1
+; CHECK-BE-NEXT:    mov pc, lr
+;
+; CHECK-V7-BE-LABEL: test_129_15_0:
+; CHECK-V7-BE:       @ %bb.0:
+; CHECK-V7-BE-NEXT:    ldrh r0, [r0, #14]
+; CHECK-V7-BE-NEXT:    bfc r0, #15, #17
+; CHECK-V7-BE-NEXT:    cmp r0, #0
+; CHECK-V7-BE-NEXT:    movwne r0, #1
+; CHECK-V7-BE-NEXT:    bx lr
+  %a = load i129, ptr %y
+  %b = and i129 %a, u0x7fff
+  %cmp = icmp ne i129 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @test_126_20_4(ptr %y) {
+; CHECK-LE-LABEL: test_126_20_4:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    ldr r0, [r0]
+; CHECK-LE-NEXT:    mvn r1, #15
+; CHECK-LE-NEXT:    sub r1, r1, #-16777216
+; CHECK-LE-NEXT:    ands r0, r0, r1
+; CHECK-LE-NEXT:    movne r0, #1
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-V7-LE-LABEL: test_126_20_4:
+; CHECK-V7-LE:       @ %bb.0:
+; CHECK-V7-LE-NEXT:    ldr r0, [r0]
+; CHECK-V7-LE-NEXT:    movw r1, #65520
+; CHECK-V7-LE-NEXT:    movt r1, #255
+; CHECK-V7-LE-NEXT:    ands r0, r0, r1
+; CHECK-V7-LE-NEXT:    movwne r0, #1
+; CHECK-V7-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: test_126_20_4:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    ldr r0, [r0, #12]
+; CHECK-BE-NEXT:    mvn r1, #15
+; CHECK-BE-NEXT:    sub r1, r1, #-16777216
+; CHECK-BE-NEXT:    ands r0, r0, r1
+; CHECK-BE-NEXT:    movne r0, #1
+; CHECK-BE-NEXT:    mov pc, lr
+;
+; CHECK-V7-BE-LABEL: test_126_20_4:
+; CHECK-V7-BE:       @ %bb.0:
+; CHECK-V7-BE-NEXT:    ldr r0, [r0, #12]
+; CHECK-V7-BE-NEXT:    movw r1, #65520
+; CHECK-V7-BE-NEXT:    movt r1, #255
+; CHECK-V7-BE-NEXT:    ands r0, r0, r1
+; CHECK-V7-BE-NEXT:    movwne r0, #1
+; CHECK-V7-BE-NEXT:    bx lr
+  %a = load i126, ptr %y
+  %b = and i126 %a, u0xfffff0
+  %cmp = icmp ne i126 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @test_33_8_0(ptr %y) {
+; CHECK-LE-LABEL: test_33_8_0:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    ldrb r0, [r0]
+; CHECK-LE-NEXT:    cmp r0, #0
+; CHECK-LE-NEXT:    movne r0, #1
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-V7-LE-LABEL: test_33_8_0:
+; CHECK-V7-LE:       @ %bb.0:
+; CHECK-V7-LE-NEXT:    ldrb r0, [r0]
+; CHECK-V7-LE-NEXT:    cmp r0, #0
+; CHECK-V7-LE-NEXT:    movwne r0, #1
+; CHECK-V7-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: test_33_8_0:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    ldrb r0, [r0, #3]
+; CHECK-BE-NEXT:    cmp r0, #0
+; CHECK-BE-NEXT:    movne r0, #1
+; CHECK-BE-NEXT:    mov pc, lr
+;
+; CHECK-V7-BE-LABEL: test_33_8_0:
+; CHECK-V7-BE:       @ %bb.0:
+; CHECK-V7-BE-NEXT:    ldrb r0, [r0, #3]
+; CHECK-V7-BE-NEXT:    cmp r0, #0
+; CHECK-V7-BE-NEXT:    movwne r0, #1
+; CHECK-V7-BE-NEXT:    bx lr
+  %a = load i33, ptr %y
+  %b = and i33 %a, u0xff
+  %cmp = icmp ne i33 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @test_33_1_32(ptr %y) {
+; CHECK-LE-LABEL: test_33_1_32:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    ldrb r0, [r0, #4]
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-V7-LE-LABEL: test_33_1_32:
+; CHECK-V7-LE:       @ %bb.0:
+; CHECK-V7-LE-NEXT:    ldrb r0, [r0, #4]
+; CHECK-V7-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: test_33_1_32:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    ldr r0, [r0]
+; CHECK-BE-NEXT:    lsr r0, r0, #24
+; CHECK-BE-NEXT:    mov pc, lr
+;
+; CHECK-V7-BE-LABEL: test_33_1_32:
+; CHECK-V7-BE:       @ %bb.0:
+; CHECK-V7-BE-NEXT:    ldr r0, [r0]
+; CHECK-V7-BE-NEXT:    lsr r0, r0, #24
+; CHECK-V7-BE-NEXT:    bx lr
+  %a = load i33, ptr %y
+  %b = and i33 %a, u0x100000000
+  %cmp = icmp ne i33 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @test_33_1_31(ptr %y) {
+; CHECK-LE-LABEL: test_33_1_31:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    ldrb r0, [r0, #3]
+; CHECK-LE-NEXT:    lsr r0, r0, #7
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-V7-LE-LABEL: test_33_1_31:
+; CHECK-V7-LE:       @ %bb.0:
+; CHECK-V7-LE-NEXT:    ldrb r0, [r0, #3]
+; CHECK-V7-LE-NEXT:    lsr r0, r0, #7
+; CHECK-V7-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: test_33_1_31:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    ldrb r0, [r0]
+; CHECK-BE-NEXT:    lsr r0, r0, #7
+; CHECK-BE-NEXT:    mov pc, lr
+;
+; CHECK-V7-BE-LABEL: test_33_1_31:
+; CHECK-V7-BE:       @ %bb.0:
+; CHECK-V7-BE-NEXT:    ldrb r0, [r0]
+; CHECK-V7-BE-NEXT:    lsr r0, r0, #7
+; CHECK-V7-BE-NEXT:    bx lr
+  %a = load i33, ptr %y
+  %b = and i33 %a, u0x80000000
+  %cmp = icmp ne i33 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @test_33_1_0(ptr %y) {
+; CHECK-LE-LABEL: test_33_1_0:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    ldrb r0, [r0]
+; CHECK-LE-NEXT:    and r0, r0, #1
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-V7-LE-LABEL: test_33_1_0:
+; CHECK-V7-LE:       @ %bb.0:
+; CHECK-V7-LE-NEXT:    ldrb r0, [r0]
+; CHECK-V7-LE-NEXT:    and r0, r0, #1
+; CHECK-V7-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: test_33_1_0:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    ldrb r0, [r0, #3]
+; CHECK-BE-NEXT:    and r0, r0, #1
+; CHECK-BE-NEXT:    mov pc, lr
+;
+; CHECK-V7-BE-LABEL: test_33_1_0:
+; CHECK-V7-BE:       @ %bb.0:
+; CHECK-V7-BE-NEXT:    ldrb r0, [r0, #3]
+; CHECK-V7-BE-NEXT:    and r0, r0, #1
+; CHECK-V7-BE-NEXT:    bx lr
+  %a = load i33, ptr %y
+  %b = and i33 %a, u0x1
+  %cmp = icmp ne i33 %b, 0
+  ret i1 %cmp
+}
+
+;--------------------------------------------------------------------------
+; Test byte-sized types.
+;--------------------------------------------------------------------------
+
+
+define i1 @test_128_20_4(ptr %y) {
+; CHECK-LE-LABEL: test_128_20_4:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    ldr r0, [r0]
+; CHECK-LE-NEXT:    mvn r1, #15
+; CHECK-LE-NEXT:    sub r1, r1, #-16777216
+; CHECK-LE-NEXT:    ands r0, r0, r1
+; CHECK-LE-NEXT:    movne r0, #1
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-V7-LE-LABEL: test_128_20_4:
+; CHECK-V7-LE:       @ %bb.0:
+; CHECK-V7-LE-NEXT:    ldr r0, [r0]
+; CHECK-V7-LE-NEXT:    movw r1, #65520
+; CHECK-V7-LE-NEXT:    movt r1, #255
+; CHECK-V7-LE-NEXT:    ands r0, r0, r1
+; CHECK-V7-LE-NEXT:    movwne r0, #1
+; CHECK-V7-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: test_128_20_4:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    ldr r0, [r0, #12]
+; CHECK-BE-NEXT:    mvn r1, #15
+; CHECK-BE-NEXT:    sub r1, r1, #-16777216
+; CHECK-BE-NEXT:    ands r0, r0, r1
+; CHECK-BE-NEXT:    movne r0, #1
+; CHECK-BE-NEXT:    mov pc, lr
+;
+; CHECK-V7-BE-LABEL: test_128_20_4:
+; CHECK-V7-BE:       @ %bb.0:
+; CHECK-V7-BE-NEXT:    ldr r0, [r0, #12]
+; CHECK-V7-BE-NEXT:    movw r1, #65520
+; CHECK-V7-BE-NEXT:    movt r1, #255
+; CHECK-V7-BE-NEXT:    ands r0, r0, r1
+; CHECK-V7-BE-NEXT:    movwne r0, #1
+; CHECK-V7-BE-NEXT:    bx lr
+  %a = load i128, ptr %y
+  %b = and i128 %a, u0xfffff0
+  %cmp = icmp ne i128 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @test_48_16_0(ptr %y) {
+; CHECK-LE-LABEL: test_48_16_0:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    ldrh r0, [r0]
+; CHECK-LE-NEXT:    cmp r0, #0
+; CHECK-LE-NEXT:    movne r0, #1
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-V7-LE-LABEL: test_48_16_0:
+; CHECK-V7-LE:       @ %bb.0:
+; CHECK-V7-LE-NEXT:    ldrh r0, [r0]
+; CHECK-V7-LE-NEXT:    cmp r0, #0
+; CHECK-V7-LE-NEXT:    movwne r0, #1
+; CHECK-V7-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: test_48_16_0:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    ldrh r0, [r0, #4]
+; CHECK-BE-NEXT:    cmp r0, #0
+; CHECK-BE-NEXT:    movne r0, #1
+; CHECK-BE-NEXT:    mov pc, lr
+;
+; CHECK-V7-BE-LABEL: test_48_16_0:
+; CHECK-V7-BE:       @ %bb.0:
+; CHECK-V7-BE-NEXT:    ldrh r0, [r0, #4]
+; CHECK-V7-BE-NEXT:    cmp r0, #0
+; CHECK-V7-BE-NEXT:    movwne r0, #1
+; CHECK-V7-BE-NEXT:    bx lr
+  %a = load i48, ptr %y
+  %b = and i48 %a, u0xffff
+  %cmp = icmp ne i48 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @test_48_16_8(ptr %y) {
+; CHECK-LE-LABEL: test_48_16_8:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    ldrh r0, [r0, #1]
+; CHECK-LE-NEXT:    cmp r0, #0
+; CHECK-LE-NEXT:    movne r0, #1
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-V7-LE-LABEL: test_48_16_8:
+; CHECK-V7-LE:       @ %bb.0:
+; CHECK-V7-LE-NEXT:    ldrh r0, [r0, #1]
+; CHECK-V7-LE-NEXT:    cmp r0, #0
+; CHECK-V7-LE-NEXT:    movwne r0, #1
+; CHECK-V7-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: test_48_16_8:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    ldrh r0, [r0, #3]
+; CHECK-BE-NEXT:    cmp r0, #0
+; CHECK-BE-NEXT:    movne r0, #1
+; CHECK-BE-NEXT:    mov pc, lr
+;
+; CHECK-V7-BE-LABEL: test_48_16_8:
+; CHECK-V7-BE:       @ %bb.0:
+; CHECK-V7-BE-NEXT:    ldrh r0, [r0, #3]
+; CHECK-V7-BE-NEXT:    cmp r0, #0
+; CHECK-V7-BE-NEXT:    movwne r0, #1
+; CHECK-V7-BE-NEXT:    bx lr
+  %a = load i48, ptr %y
+  %b = and i48 %a, u0xffff00
+  %cmp = icmp ne i48 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @test_48_16_16(ptr %y) {
+; CHECK-LE-LABEL: test_48_16_16:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    ldrh r0, [r0, #2]
+; CHECK-LE-NEXT:    cmp r0, #0
+; CHECK-LE-NEXT:    movne r0, #1
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-V7-LE-LABEL: test_48_16_16:
+; CHECK-V7-LE:       @ %bb.0:
+; CHECK-V7-LE-NEXT:    ldrh r0, [r0, #2]
+; CHECK-V7-LE-NEXT:    cmp r0, #0
+; CHECK-V7-LE-NEXT:    movwne r0, #1
+; CHECK-V7-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: test_48_16_16:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    ldrh r0, [r0, #2]
+; CHECK-BE-NEXT:    cmp r0, #0
+; CHECK-BE-NEXT:    movne r0, #1
+; CHECK-BE-NEXT:    mov pc, lr
+;
+; CHECK-V7-BE-LABEL: test_48_16_16:
+; CHECK-V7-BE:       @ %bb.0:
+; CHECK-V7-BE-NEXT:    ldrh r0, [r0, #2]
+; CHECK-V7-BE-NEXT:    cmp r0, #0
+; CHECK-V7-BE-NEXT:    movwne r0, #1
+; CHECK-V7-BE-NEXT:    bx lr
+  %a = load i48, ptr %y
+  %b = and i48 %a, u0xffff0000
+  %cmp = icmp ne i48 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @test_48_16_32(ptr %y) {
+; CHECK-LE-LABEL: test_48_16_32:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    ldrh r0, [r0, #4]
+; CHECK-LE-NEXT:    cmp r0, #0
+; CHECK-LE-NEXT:    movne r0, #1
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-V7-LE-LABEL: test_48_16_32:
+; CHECK-V7-LE:       @ %bb.0:
+; CHECK-V7-LE-NEXT:    ldrh r0, [r0, #4]
+; CHECK-V7-LE-NEXT:    cmp r0, #0
+; CHECK-V7-LE-NEXT:    movwne r0, #1
+; CHECK-V7-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: test_48_16_32:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    ldrh r0, [r0]
+; CHECK-BE-NEXT:    cmp r0, #0
+; CHECK-BE-NEXT:    movne r0, #1
+; CHECK-BE-NEXT:    mov pc, lr
+;
+; CHECK-V7-BE-LABEL: test_48_16_32:
+; CHECK-V7-BE:       @ %bb.0:
+; CHECK-V7-BE-NEXT:    ldrh r0, [r0]
+; CHECK-V7-BE-NEXT:    cmp r0, #0
+; CHECK-V7-BE-NEXT:    movwne r0, #1
+; CHECK-V7-BE-NEXT:    bx lr
+  %a = load i48, ptr %y
+  %b = and i48 %a, u0xffff00000000
+  %cmp = icmp ne i48 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @test_48_17_0(ptr %y) {
+; CHECK-LE-LABEL: test_48_17_0:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    ldr r0, [r0]
+; CHECK-LE-NEXT:    ldr r1, .LCPI11_0
+; CHECK-LE-NEXT:    ands r0, r0, r1
+; CHECK-LE-NEXT:    movne r0, #1
+; CHECK-LE-NEXT:    mov pc, lr
+; CHECK-LE-NEXT:    .p2align 2
+; CHECK-LE-NEXT:  @ %bb.1:
+; CHECK-LE-NEXT:  .LCPI11_0:
+; CHECK-LE-NEXT:    .long 131071 @ 0x1ffff
+;
+; CHECK-V7-LE-LABEL: test_48_17_0:
+; CHECK-V7-LE:       @ %bb.0:
+; CHECK-V7-LE-NEXT:    ldr r0, [r0]
+; CHECK-V7-LE-NEXT:    bfc r0, #17, #15
+; CHECK-V7-LE-NEXT:    cmp r0, #0
+; CHECK-V7-LE-NEXT:    movwne r0, #1
+; CHECK-V7-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: test_48_17_0:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    ldr r1, [r0]
+; CHECK-BE-NEXT:    ldrh r0, [r0, #4]
+; CHECK-BE-NEXT:    orr r0, r0, r1, lsl #16
+; CHECK-BE-NEXT:    ldr r1, .LCPI11_0
+; CHECK-BE-NEXT:    ands r0, r0, r1
+; CHECK-BE-NEXT:    movne r0, #1
+; CHECK-BE-NEXT:    mov pc, lr
+; CHECK-BE-NEXT:    .p2align 2
+; CHECK-BE-NEXT:  @ %bb.1:
+; CHECK-BE-NEXT:  .LCPI11_0:
+; CHECK-BE-NEXT:    .long 131071 @ 0x1ffff
+;
+; CHECK-V7-BE-LABEL: test_48_17_0:
+; CHECK-V7-BE:       @ %bb.0:
+; CHECK-V7-BE-NEXT:    ldr r1, [r0]
+; CHECK-V7-BE-NEXT:    ldrh r0, [r0, #4]
+; CHECK-V7-BE-NEXT:    orr r0, r0, r1, lsl #16
+; CHECK-V7-BE-NEXT:    bfc r0, #17, #15
+; CHECK-V7-BE-NEXT:    cmp r0, #0
+; CHECK-V7-BE-NEXT:    movwne r0, #1
+; CHECK-V7-BE-NEXT:    bx lr
+  %a = load i48, ptr %y
+  %b = and i48 %a, u0x1ffff
+  %cmp = icmp ne i48 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @test_40_16_0(ptr %y) {
+; CHECK-LE-LABEL: test_40_16_0:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    ldrh r0, [r0]
+; CHECK-LE-NEXT:    cmp r0, #0
+; CHECK-LE-NEXT:    movne r0, #1
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-V7-LE-LABEL: test_40_16_0:
+; CHECK-V7-LE:       @ %bb.0:
+; CHECK-V7-LE-NEXT:    ldrh r0, [r0]
+; CHECK-V7-LE-NEXT:    cmp r0, #0
+; CHECK-V7-LE-NEXT:    movwne r0, #1
+; CHECK-V7-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: test_40_16_0:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    ldrh r0, [r0, #3]
+; CHECK-BE-NEXT:    cmp r0, #0
+; CHECK-BE-NEXT:    movne r0, #1
+; CHECK-BE-NEXT:    mov pc, lr
+;
+; CHECK-V7-BE-LABEL: test_40_16_0:
+; CHECK-V7-BE:       @ %bb.0:
+; CHECK-V7-BE-NEXT:    ldrh r0, [r0, #3]
+; CHECK-V7-BE-NEXT:    cmp r0, #0
+; CHECK-V7-BE-NEXT:    movwne r0, #1
+; CHECK-V7-BE-NEXT:    bx lr
+  %a = load i40, ptr %y
+  %b = and i40 %a, u0xffff
+  %cmp = icmp ne i40 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @test_40_1_32(ptr %y) {
+; CHECK-LE-LABEL: test_40_1_32:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    ldrb r0, [r0, #4]
+; CHECK-LE-NEXT:    and r0, r0, #1
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-V7-LE-LABEL: test_40_1_32:
+; CHECK-V7-LE:       @ %bb.0:
+; CHECK-V7-LE-NEXT:    ldrb r0, [r0, #4]
+; CHECK-V7-LE-NEXT:    and r0, r0, #1
+; CHECK-V7-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: test_40_1_32:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    ldr r0, [r0]
+; CHECK-BE-NEXT:    mov r1, #1
+; CHECK-BE-NEXT:    and r0, r1, r0, lsr #24
+; CHECK-BE-NEXT:    mov pc, lr
+;
+; CHECK-V7-BE-LABEL: test_40_1_32:
+; CHECK-V7-BE:       @ %bb.0:
+; CHECK-V7-BE-NEXT:    ldr r0, [r0]
+; CHECK-V7-BE-NEXT:    ubfx r0, r0, #24, #1
+; CHECK-V7-BE-NEXT:    bx lr
+  %a = load i40, ptr %y
+  %b = and i40 %a, u0x100000000
+  %cmp = icmp ne i40 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @test_24_16_0(ptr %y) {
+; CHECK-LE-LABEL: test_24_16_0:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    ldrh r0, [r0]
+; CHECK-LE-NEXT:    cmp r0, #0
+; CHECK-LE-NEXT:    movne r0, #1
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-V7-LE-LABEL: test_24_16_0:
+; CHECK-V7-LE:       @ %bb.0:
+; CHECK-V7-LE-NEXT:    ldrh r0, [r0]
+; CHECK-V7-LE-NEXT:    cmp r0, #0
+; CHECK-V7-LE-NEXT:    movwne r0, #1
+; CHECK-V7-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: test_24_16_0:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    ldrh r0, [r0, #1]
+; CHECK-BE-NEXT:    cmp r0, #0
+; CHECK-BE-NEXT:    movne r0, #1
+; CHECK-BE-NEXT:    mov pc, lr
+;
+; CHECK-V7-BE-LABEL: test_24_16_0:
+; CHECK-V7-BE:       @ %bb.0:
+; CHECK-V7-BE-NEXT:    ldrh r0, [r0, #1]
+; CHECK-V7-BE-NEXT:    cmp r0, #0
+; CHECK-V7-BE-NEXT:    movwne r0, #1
+; CHECK-V7-BE-NEXT:    bx lr
+  %a = load i24, ptr %y
+  %b = and i24 %a, u0xffff
+  %cmp = icmp ne i24 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @test_24_8_8(ptr %y) {
+; CHECK-LE-LABEL: test_24_8_8:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    ldrb r0, [r0, #1]
+; CHECK-LE-NEXT:    lsls r0, r0, #8
+; CHECK-LE-NEXT:    movne r0, #1
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-V7-LE-LABEL: test_24_8_8:
+; CHECK-V7-LE:       @ %bb.0:
+; CHECK-V7-LE-NEXT:    ldrb r0, [r0, #1]
+; CHECK-V7-LE-NEXT:    lsls r0, r0, #8
+; CHECK-V7-LE-NEXT:    movwne r0, #1
+; CHECK-V7-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: test_24_8_8:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    ldrb r0, [r0, #1]
+; CHECK-BE-NEXT:    lsls r0, r0, #8
+; CHECK-BE-NEXT:    movne r0, #1
+; CHECK-BE-NEXT:    mov pc, lr
+;
+; CHECK-V7-BE-LABEL: test_24_8_8:
+; CHECK-V7-BE:       @ %bb.0:
+; CHECK-V7-BE-NEXT:    ldrb r0, [r0, #1]
+; CHECK-V7-BE-NEXT:    lsls r0, r0, #8
+; CHECK-V7-BE-NEXT:    movwne r0, #1
+; CHECK-V7-BE-NEXT:    bx lr
+  %a = load i24, ptr %y
+  %b = and i24 %a, u0xff00
+  %cmp = icmp ne i24 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @test_24_8_12(ptr %y) {
+; CHECK-LE-LABEL: test_24_8_12:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    ldrb r1, [r0, #2]
+; CHECK-LE-NEXT:    ldrh r0, [r0]
+; CHECK-LE-NEXT:    orr r0, r0, r1, lsl #16
+; CHECK-LE-NEXT:    ands r0, r0, #1044480
+; CHECK-LE-NEXT:    movne r0, #1
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-V7-LE-LABEL: test_24_8_12:
+; CHECK-V7-LE:       @ %bb.0:
+; CHECK-V7-LE-NEXT:    ldrb r1, [r0, #2]
+; CHECK-V7-LE-NEXT:    ldrh r0, [r0]
+; CHECK-V7-LE-NEXT:    orr r0, r0, r1, lsl #16
+; CHECK-V7-LE-NEXT:    ands r0, r0, #1044480
+; CHECK-V7-LE-NEXT:    movwne r0, #1
+; CHECK-V7-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: test_24_8_12:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    ldrh r0, [r0]
+; CHECK-BE-NEXT:    mov r1, #1044480
+; CHECK-BE-NEXT:    ands r0, r1, r0, lsl #8
+; CHECK-BE-NEXT:    movne r0, #1
+; CHECK-BE-NEXT:    mov pc, lr
+;
+; CHECK-V7-BE-LABEL: test_24_8_12:
+; CHECK-V7-BE:       @ %bb.0:
+; CHECK-V7-BE-NEXT:    ldrh r0, [r0]
+; CHECK-V7-BE-NEXT:    mov r1, #1044480
+; CHECK-V7-BE-NEXT:    ands r0, r1, r0, lsl #8
+; CHECK-V7-BE-NEXT:    movwne r0, #1
+; CHECK-V7-BE-NEXT:    bx lr
+  %a = load i24, ptr %y
+  %b = and i24 %a, u0xff000
+  %cmp = icmp ne i24 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @test_24_8_16(ptr %y) {
+; CHECK-LE-LABEL: test_24_8_16:
+; CHECK-LE:       @ %bb.0:
+; CHECK-LE-NEXT:    ldrb r0, [r0, #2]
+; CHECK-LE-NEXT:    lsls r0, r0, #16
+; CHECK-LE-NEXT:    movne r0, #1
+; CHECK-LE-NEXT:    mov pc, lr
+;
+; CHECK-V7-LE-LABEL: test_24_8_16:
+; CHECK-V7-LE:       @ %bb.0:
+; CHECK-V7-LE-NEXT:    ldrb r0, [r0, #2]
+; CHECK-V7-LE-NEXT:    lsls r0, r0, #16
+; CHECK-V7-LE-NEXT:    movwne r0, #1
+; CHECK-V7-LE-NEXT:    bx lr
+;
+; CHECK-BE-LABEL: test_24_8_16:
+; CHECK-BE:       @ %bb.0:
+; CHECK-BE-NEXT:    ldrb r0, [r0]
+; CHECK-BE-NEXT:    lsls r0, r0, #16
+; CHECK-BE-NEXT:    movne r0, #1
+; CHECK-BE-NEXT:    mov pc, lr
+;
+; CHECK-V7-BE-LABEL: test_24_8_16:
+; CHECK-V7-BE:       @ %bb.0:
+; CHECK-V7-BE-NEXT:    ldrb r0, [r0]
+; CHECK-V7-BE-NEXT:    lsls r0, r0, #16
+; CHECK-V7-BE-NEXT:    movwne r0, #1
+; CHECK-V7-BE-NEXT:    bx lr
+  %a = load i24, ptr %y
+  %b = and i24 %a, u0xff0000
+  %cmp = icmp ne i24 %b, 0
+  ret i1 %cmp
+}
diff --git a/llvm/test/CodeGen/PowerPC/simplifysetcc_narrow_load.ll b/llvm/test/CodeGen/PowerPC/simplifysetcc_narrow_load.ll
new file mode 100644
index 000000000000..1a03423fe6ae
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/simplifysetcc_narrow_load.ll
@@ -0,0 +1,390 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -O1 -mtriple ppc32le -o - %s | FileCheck --check-prefix CHECK-LE %s
+; RUN: llc -O1 -mtriple ppc32 -o - %s | FileCheck --check-prefix CHECK-BE %s
+
+; A collection of regression tests to verify the load-narrowing part of
+; TargetLowering::SimplifySetCC (and/or other similar rewrites such as
+; combining AND+LOAD into ZEXTLOAD).
+
+
+;--------------------------------------------------------------------------
+; Test non byte-sized types.
+;
+; As long as LLVM IR isn't defining where the padding goes we can't really
+; optimize these (without adding a target lowering hook that can inform
+; ISel about which bits are padding).
+; --------------------------------------------------------------------------
+
+define i1 @test_129_15_0(ptr %y) {
+; CHECK-LE-LABEL: test_129_15_0:
+; CHECK-LE:       # %bb.0:
+; CHECK-LE-NEXT:    lhz 3, 0(3)
+; CHECK-LE-NEXT:    clrlwi 3, 3, 17
+; CHECK-LE-NEXT:    addic 4, 3, -1
+; CHECK-LE-NEXT:    subfe 3, 4, 3
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_129_15_0:
+; CHECK-BE:       # %bb.0:
+; CHECK-BE-NEXT:    lhz 3, 14(3)
+; CHECK-BE-NEXT:    clrlwi 3, 3, 17
+; CHECK-BE-NEXT:    addic 4, 3, -1
+; CHECK-BE-NEXT:    subfe 3, 4, 3
+; CHECK-BE-NEXT:    blr
+  %a = load i129, ptr %y
+  %b = and i129 %a, u0x7fff
+  %cmp = icmp ne i129 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @test_126_20_4(ptr %y) {
+; CHECK-LE-LABEL: test_126_20_4:
+; CHECK-LE:       # %bb.0:
+; CHECK-LE-NEXT:    lwz 3, 0(3)
+; CHECK-LE-NEXT:    rlwinm 3, 3, 0, 8, 27
+; CHECK-LE-NEXT:    addic 4, 3, -1
+; CHECK-LE-NEXT:    subfe 3, 4, 3
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_126_20_4:
+; CHECK-BE:       # %bb.0:
+; CHECK-BE-NEXT:    lwz 3, 12(3)
+; CHECK-BE-NEXT:    rlwinm 3, 3, 0, 8, 27
+; CHECK-BE-NEXT:    addic 4, 3, -1
+; CHECK-BE-NEXT:    subfe 3, 4, 3
+; CHECK-BE-NEXT:    blr
+  %a = load i126, ptr %y
+  %b = and i126 %a, u0xfffff0
+  %cmp = icmp ne i126 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @test_33_8_0(ptr %y) {
+; CHECK-LE-LABEL: test_33_8_0:
+; CHECK-LE:       # %bb.0:
+; CHECK-LE-NEXT:    lbz 3, 0(3)
+; CHECK-LE-NEXT:    addic 4, 3, -1
+; CHECK-LE-NEXT:    subfe 3, 4, 3
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_33_8_0:
+; CHECK-BE:       # %bb.0:
+; CHECK-BE-NEXT:    lbz 3, 3(3)
+; CHECK-BE-NEXT:    addic 4, 3, -1
+; CHECK-BE-NEXT:    subfe 3, 4, 3
+; CHECK-BE-NEXT:    blr
+  %a = load i33, ptr %y
+  %b = and i33 %a, u0xff
+  %cmp = icmp ne i33 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @test_33_1_32(ptr %y) {
+; CHECK-LE-LABEL: test_33_1_32:
+; CHECK-LE:       # %bb.0:
+; CHECK-LE-NEXT:    lbz 3, 4(3)
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_33_1_32:
+; CHECK-BE:       # %bb.0:
+; CHECK-BE-NEXT:    lwz 3, 0(3)
+; CHECK-BE-NEXT:    srwi 3, 3, 24
+; CHECK-BE-NEXT:    blr
+  %a = load i33, ptr %y
+  %b = and i33 %a, u0x100000000
+  %cmp = icmp ne i33 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @test_33_1_31(ptr %y) {
+; CHECK-LE-LABEL: test_33_1_31:
+; CHECK-LE:       # %bb.0:
+; CHECK-LE-NEXT:    lbz 3, 3(3)
+; CHECK-LE-NEXT:    srwi 3, 3, 7
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_33_1_31:
+; CHECK-BE:       # %bb.0:
+; CHECK-BE-NEXT:    lbz 3, 0(3)
+; CHECK-BE-NEXT:    srwi 3, 3, 7
+; CHECK-BE-NEXT:    blr
+  %a = load i33, ptr %y
+  %b = and i33 %a, u0x80000000
+  %cmp = icmp ne i33 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @test_33_1_0(ptr %y) {
+; CHECK-LE-LABEL: test_33_1_0:
+; CHECK-LE:       # %bb.0:
+; CHECK-LE-NEXT:    lbz 3, 0(3)
+; CHECK-LE-NEXT:    clrlwi 3, 3, 31
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_33_1_0:
+; CHECK-BE:       # %bb.0:
+; CHECK-BE-NEXT:    lbz 3, 3(3)
+; CHECK-BE-NEXT:    clrlwi 3, 3, 31
+; CHECK-BE-NEXT:    blr
+  %a = load i33, ptr %y
+  %b = and i33 %a, u0x1
+  %cmp = icmp ne i33 %b, 0
+  ret i1 %cmp
+}
+
+;--------------------------------------------------------------------------
+; Test byte-sized types.
+;--------------------------------------------------------------------------
+
+
+define i1 @test_128_20_4(ptr %y) {
+; CHECK-LE-LABEL: test_128_20_4:
+; CHECK-LE:       # %bb.0:
+; CHECK-LE-NEXT:    lwz 3, 0(3)
+; CHECK-LE-NEXT:    rlwinm 3, 3, 0, 8, 27
+; CHECK-LE-NEXT:    addic 4, 3, -1
+; CHECK-LE-NEXT:    subfe 3, 4, 3
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_128_20_4:
+; CHECK-BE:       # %bb.0:
+; CHECK-BE-NEXT:    lwz 3, 12(3)
+; CHECK-BE-NEXT:    rlwinm 3, 3, 0, 8, 27
+; CHECK-BE-NEXT:    addic 4, 3, -1
+; CHECK-BE-NEXT:    subfe 3, 4, 3
+; CHECK-BE-NEXT:    blr
+  %a = load i128, ptr %y
+  %b = and i128 %a, u0xfffff0
+  %cmp = icmp ne i128 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @test_48_16_0(ptr %y) {
+; CHECK-LE-LABEL: test_48_16_0:
+; CHECK-LE:       # %bb.0:
+; CHECK-LE-NEXT:    lhz 3, 0(3)
+; CHECK-LE-NEXT:    addic 4, 3, -1
+; CHECK-LE-NEXT:    subfe 3, 4, 3
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_48_16_0:
+; CHECK-BE:       # %bb.0:
+; CHECK-BE-NEXT:    lhz 3, 4(3)
+; CHECK-BE-NEXT:    addic 4, 3, -1
+; CHECK-BE-NEXT:    subfe 3, 4, 3
+; CHECK-BE-NEXT:    blr
+  %a = load i48, ptr %y
+  %b = and i48 %a, u0xffff
+  %cmp = icmp ne i48 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @test_48_16_8(ptr %y) {
+; CHECK-LE-LABEL: test_48_16_8:
+; CHECK-LE:       # %bb.0:
+; CHECK-LE-NEXT:    lhz 3, 1(3)
+; CHECK-LE-NEXT:    addic 4, 3, -1
+; CHECK-LE-NEXT:    subfe 3, 4, 3
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_48_16_8:
+; CHECK-BE:       # %bb.0:
+; CHECK-BE-NEXT:    lhz 3, 3(3)
+; CHECK-BE-NEXT:    addic 4, 3, -1
+; CHECK-BE-NEXT:    subfe 3, 4, 3
+; CHECK-BE-NEXT:    blr
+  %a = load i48, ptr %y
+  %b = and i48 %a, u0xffff00
+  %cmp = icmp ne i48 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @test_48_16_16(ptr %y) {
+; CHECK-LE-LABEL: test_48_16_16:
+; CHECK-LE:       # %bb.0:
+; CHECK-LE-NEXT:    lhz 3, 2(3)
+; CHECK-LE-NEXT:    addic 4, 3, -1
+; CHECK-LE-NEXT:    subfe 3, 4, 3
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_48_16_16:
+; CHECK-BE:       # %bb.0:
+; CHECK-BE-NEXT:    lhz 3, 2(3)
+; CHECK-BE-NEXT:    addic 4, 3, -1
+; CHECK-BE-NEXT:    subfe 3, 4, 3
+; CHECK-BE-NEXT:    blr
+  %a = load i48, ptr %y
+  %b = and i48 %a, u0xffff0000
+  %cmp = icmp ne i48 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @test_48_16_32(ptr %y) {
+; CHECK-LE-LABEL: test_48_16_32:
+; CHECK-LE:       # %bb.0:
+; CHECK-LE-NEXT:    lhz 3, 4(3)
+; CHECK-LE-NEXT:    addic 4, 3, -1
+; CHECK-LE-NEXT:    subfe 3, 4, 3
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_48_16_32:
+; CHECK-BE:       # %bb.0:
+; CHECK-BE-NEXT:    lhz 3, 0(3)
+; CHECK-BE-NEXT:    addic 4, 3, -1
+; CHECK-BE-NEXT:    subfe 3, 4, 3
+; CHECK-BE-NEXT:    blr
+  %a = load i48, ptr %y
+  %b = and i48 %a, u0xffff00000000
+  %cmp = icmp ne i48 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @test_48_17_0(ptr %y) {
+; CHECK-LE-LABEL: test_48_17_0:
+; CHECK-LE:       # %bb.0:
+; CHECK-LE-NEXT:    lwz 3, 0(3)
+; CHECK-LE-NEXT:    clrlwi 3, 3, 15
+; CHECK-LE-NEXT:    addic 4, 3, -1
+; CHECK-LE-NEXT:    subfe 3, 4, 3
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_48_17_0:
+; CHECK-BE:       # %bb.0:
+; CHECK-BE-NEXT:    lhz 4, 4(3)
+; CHECK-BE-NEXT:    lwz 3, 0(3)
+; CHECK-BE-NEXT:    clrlwi 4, 4, 16
+; CHECK-BE-NEXT:    rlwimi 4, 3, 16, 15, 15
+; CHECK-BE-NEXT:    addic 3, 4, -1
+; CHECK-BE-NEXT:    subfe 3, 3, 4
+; CHECK-BE-NEXT:    blr
+  %a = load i48, ptr %y
+  %b = and i48 %a, u0x1ffff
+  %cmp = icmp ne i48 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @test_40_16_0(ptr %y) {
+; CHECK-LE-LABEL: test_40_16_0:
+; CHECK-LE:       # %bb.0:
+; CHECK-LE-NEXT:    lhz 3, 0(3)
+; CHECK-LE-NEXT:    addic 4, 3, -1
+; CHECK-LE-NEXT:    subfe 3, 4, 3
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_40_16_0:
+; CHECK-BE:       # %bb.0:
+; CHECK-BE-NEXT:    lhz 3, 3(3)
+; CHECK-BE-NEXT:    addic 4, 3, -1
+; CHECK-BE-NEXT:    subfe 3, 4, 3
+; CHECK-BE-NEXT:    blr
+  %a = load i40, ptr %y
+  %b = and i40 %a, u0xffff
+  %cmp = icmp ne i40 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @test_40_1_32(ptr %y) {
+; CHECK-LE-LABEL: test_40_1_32:
+; CHECK-LE:       # %bb.0:
+; CHECK-LE-NEXT:    lbz 3, 4(3)
+; CHECK-LE-NEXT:    clrlwi 3, 3, 31
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_40_1_32:
+; CHECK-BE:       # %bb.0:
+; CHECK-BE-NEXT:    lwz 3, 0(3)
+; CHECK-BE-NEXT:    rlwinm 3, 3, 8, 31, 31
+; CHECK-BE-NEXT:    blr
+  %a = load i40, ptr %y
+  %b = and i40 %a, u0x100000000
+  %cmp = icmp ne i40 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @test_24_16_0(ptr %y) {
+; CHECK-LE-LABEL: test_24_16_0:
+; CHECK-LE:       # %bb.0:
+; CHECK-LE-NEXT:    lhz 3, 0(3)
+; CHECK-LE-NEXT:    addic 4, 3, -1
+; CHECK-LE-NEXT:    subfe 3, 4, 3
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_24_16_0:
+; CHECK-BE:       # %bb.0:
+; CHECK-BE-NEXT:    lhz 3, 1(3)
+; CHECK-BE-NEXT:    addic 4, 3, -1
+; CHECK-BE-NEXT:    subfe 3, 4, 3
+; CHECK-BE-NEXT:    blr
+  %a = load i24, ptr %y
+  %b = and i24 %a, u0xffff
+  %cmp = icmp ne i24 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @test_24_8_8(ptr %y) {
+; CHECK-LE-LABEL: test_24_8_8:
+; CHECK-LE:       # %bb.0:
+; CHECK-LE-NEXT:    lbz 3, 1(3)
+; CHECK-LE-NEXT:    slwi 3, 3, 8
+; CHECK-LE-NEXT:    addic 4, 3, -1
+; CHECK-LE-NEXT:    subfe 3, 4, 3
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_24_8_8:
+; CHECK-BE:       # %bb.0:
+; CHECK-BE-NEXT:    lbz 3, 1(3)
+; CHECK-BE-NEXT:    slwi 3, 3, 8
+; CHECK-BE-NEXT:    addic 4, 3, -1
+; CHECK-BE-NEXT:    subfe 3, 4, 3
+; CHECK-BE-NEXT:    blr
+  %a = load i24, ptr %y
+  %b = and i24 %a, u0xff00
+  %cmp = icmp ne i24 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @test_24_8_12(ptr %y) {
+; CHECK-LE-LABEL: test_24_8_12:
+; CHECK-LE:       # %bb.0:
+; CHECK-LE-NEXT:    lhz 4, 0(3)
+; CHECK-LE-NEXT:    lbz 3, 2(3)
+; CHECK-LE-NEXT:    rlwinm 4, 4, 0, 16, 19
+; CHECK-LE-NEXT:    rlwimi 4, 3, 16, 12, 15
+; CHECK-LE-NEXT:    addic 3, 4, -1
+; CHECK-LE-NEXT:    subfe 3, 3, 4
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_24_8_12:
+; CHECK-BE:       # %bb.0:
+; CHECK-BE-NEXT:    lhz 3, 0(3)
+; CHECK-BE-NEXT:    rlwinm 3, 3, 8, 12, 19
+; CHECK-BE-NEXT:    addic 4, 3, -1
+; CHECK-BE-NEXT:    subfe 3, 4, 3
+; CHECK-BE-NEXT:    blr
+  %a = load i24, ptr %y
+  %b = and i24 %a, u0xff000
+  %cmp = icmp ne i24 %b, 0
+  ret i1 %cmp
+}
+
+define i1 @test_24_8_16(ptr %y) {
+; CHECK-LE-LABEL: test_24_8_16:
+; CHECK-LE:       # %bb.0:
+; CHECK-LE-NEXT:    lbz 3, 2(3)
+; CHECK-LE-NEXT:    slwi 3, 3, 16
+; CHECK-LE-NEXT:    addic 4, 3, -1
+; CHECK-LE-NEXT:    subfe 3, 4, 3
+; CHECK-LE-NEXT:    blr
+;
+; CHECK-BE-LABEL: test_24_8_16:
+; CHECK-BE:       # %bb.0:
+; CHECK-BE-NEXT:    lbz 3, 0(3)
+; CHECK-BE-NEXT:    slwi 3, 3, 16
+; CHECK-BE-NEXT:    addic 4, 3, -1
+; CHECK-BE-NEXT:    subfe 3, 4, 3
+; CHECK-BE-NEXT:    blr
+  %a = load i24, ptr %y
+  %b = and i24 %a, u0xff0000
+  %cmp = icmp ne i24 %b, 0
+  ret i1 %cmp
+}
-- 
GitLab


From 33e6b488beda80dddb68016a132ee1f0a3c04aa1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bj=C3=B6rn=20Pettersson?= <bjorn.a.pettersson@ericsson.com>
Date: Fri, 12 Apr 2024 16:18:12 +0200
Subject: [PATCH 071/464] [SelectionDAG] Fix and improve
 TargetLowering::SimplifySetCC (#87646)

The load narrowing part of TargetLowering::SimplifySetCC is updated
according to this:

1) The offset calculation (for big endian) did not work properly for
   non byte-sized types. This is basically solved by an early exit
   if the memory type isn't byte-sized. But the code is also corrected
   to use the store size when calculating the offset.
2) To still allow some optimizations for non-byte-sized types the
   TargetLowering::isPaddedAtMostSignificantBitsWhenStored hook is
   added. By default it assumes that scalar integer types are padded
   starting at the most significant bits, if the type needs padding
   when being stored to memory.
3) Allow optimizing when isPaddedAtMostSignificantBitsWhenStored is
   true, as that hook makes it possible for TargetLowering to know
   how the non byte-sized value is aligned in memory.
4) Update the algorithm to always search for a narrowed load with
   a power-of-2 byte-sized type. In the past the algorithm started
   with the the width of the original load, and then divided it by
   two for each iteration. But for a type such as i48 that would
   just end up trying to narrow the load into a i24 or i12 load,
   and then we would fail sooner or later due to not finding a
   newVT that fulfilled newVT.isRound().
   With this new approach we can narrow the i48 load into either
   an i8, i16 or i32 load. By checking if such a load is allowed
(e.g. alignment wise) for any "multiple of 8 offset", then we can find
   more opportunities for the optimization to trigger. So even for a
   byte-sized type such as i32 we may now end up narrowing the load
   into loading the 16 bits starting at offset 8 (if that is allowed
   by the target). The old algorithm did not even consider that case.
5) Also start using getObjectPtrOffset instead of getMemBasePlusOffset
   when creating the new ptr. This way we get "nsw" on the add.
---
 llvm/include/llvm/CodeGen/TargetLowering.h    |  7 ++
 .../CodeGen/SelectionDAG/TargetLowering.cpp   | 68 +++++++++++--------
 .../CodeGen/ARM/simplifysetcc_narrow_load.ll  | 33 +++++----
 .../PowerPC/simplifysetcc_narrow_load.ll      | 38 ++++-------
 4 files changed, 79 insertions(+), 67 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index a4dc09744618..980d75ad91a0 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -1804,6 +1804,13 @@ public:
   /// where the sext is redundant, and use x directly.
   virtual bool shouldRemoveRedundantExtend(SDValue Op) const { return true; }
 
+  /// Indicates if any padding is guaranteed to go at the most significant bits
+  /// when storing the type to memory and the type size isn't equal to the store
+  /// size.
+  bool isPaddedAtMostSignificantBitsWhenStored(EVT VT) const {
+    return VT.isScalarInteger() && !VT.isByteSized();
+  }
+
   /// When splitting a value of the specified type into parts, does the Lo
   /// or Hi part come first?  This usually follows the endianness, except
   /// for ppcf128, where the Hi part always comes first.
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 409d66adfd67..4939976b8e28 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -4621,7 +4621,10 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
       LoadSDNode *Lod = cast<LoadSDNode>(N0.getOperand(0));
       APInt bestMask;
       unsigned bestWidth = 0, bestOffset = 0;
-      if (Lod->isSimple() && Lod->isUnindexed()) {
+      if (Lod->isSimple() && Lod->isUnindexed() &&
+          (Lod->getMemoryVT().isByteSized() ||
+           isPaddedAtMostSignificantBitsWhenStored(Lod->getMemoryVT()))) {
+        unsigned memWidth = Lod->getMemoryVT().getStoreSizeInBits();
         unsigned origWidth = N0.getValueSizeInBits();
         unsigned maskWidth = origWidth;
         // We can narrow (e.g.) 16-bit extending loads on 32-bit target to
@@ -4629,40 +4632,51 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
         if (Lod->getExtensionType() != ISD::NON_EXTLOAD)
           origWidth = Lod->getMemoryVT().getSizeInBits();
         const APInt &Mask = N0.getConstantOperandAPInt(1);
-        for (unsigned width = origWidth / 2; width>=8; width /= 2) {
+        // Only consider power-of-2 widths (and at least one byte) as candiates
+        // for the narrowed load.
+        for (unsigned width = 8; width < origWidth; width *= 2) {
+          EVT newVT = EVT::getIntegerVT(*DAG.getContext(), width);
+          if (!shouldReduceLoadWidth(Lod, ISD::NON_EXTLOAD, newVT))
+            continue;
           APInt newMask = APInt::getLowBitsSet(maskWidth, width);
-          for (unsigned offset=0; offset<origWidth/width; offset++) {
+          // Avoid accessing any padding here for now (we could use memWidth
+          // instead of origWidth here otherwise).
+          unsigned maxOffset = origWidth - width;
+          for (unsigned offset = 0; offset <= maxOffset; offset += 8) {
             if (Mask.isSubsetOf(newMask)) {
-              if (Layout.isLittleEndian())
-                bestOffset = (uint64_t)offset * (width/8);
-              else
-                bestOffset = (origWidth/width - offset - 1) * (width/8);
-              bestMask = Mask.lshr(offset * (width/8) * 8);
-              bestWidth = width;
-              break;
+              unsigned ptrOffset =
+                  Layout.isLittleEndian() ? offset : memWidth - width - offset;
+              unsigned IsFast = 0;
+              Align NewAlign = commonAlignment(Lod->getAlign(), ptrOffset / 8);
+              if (allowsMemoryAccess(
+                      *DAG.getContext(), Layout, newVT, Lod->getAddressSpace(),
+                      NewAlign, Lod->getMemOperand()->getFlags(), &IsFast) &&
+                  IsFast) {
+                bestOffset = ptrOffset / 8;
+                bestMask = Mask.lshr(offset);
+                bestWidth = width;
+                break;
+              }
             }
-            newMask <<= width;
+            newMask <<= 8;
           }
+          if (bestWidth)
+            break;
         }
       }
       if (bestWidth) {
         EVT newVT = EVT::getIntegerVT(*DAG.getContext(), bestWidth);
-        if (newVT.isRound() &&
-            shouldReduceLoadWidth(Lod, ISD::NON_EXTLOAD, newVT)) {
-          SDValue Ptr = Lod->getBasePtr();
-          if (bestOffset != 0)
-            Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(bestOffset),
-                                           dl);
-          SDValue NewLoad =
-              DAG.getLoad(newVT, dl, Lod->getChain(), Ptr,
-                          Lod->getPointerInfo().getWithOffset(bestOffset),
-                          Lod->getOriginalAlign());
-          return DAG.getSetCC(dl, VT,
-                              DAG.getNode(ISD::AND, dl, newVT, NewLoad,
-                                      DAG.getConstant(bestMask.trunc(bestWidth),
-                                                      dl, newVT)),
-                              DAG.getConstant(0LL, dl, newVT), Cond);
-        }
+        SDValue Ptr = Lod->getBasePtr();
+        if (bestOffset != 0)
+          Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::getFixed(bestOffset));
+        SDValue NewLoad =
+            DAG.getLoad(newVT, dl, Lod->getChain(), Ptr,
+                        Lod->getPointerInfo().getWithOffset(bestOffset),
+                        Lod->getOriginalAlign());
+        SDValue And =
+            DAG.getNode(ISD::AND, dl, newVT, NewLoad,
+                        DAG.getConstant(bestMask.trunc(bestWidth), dl, newVT));
+        return DAG.getSetCC(dl, VT, And, DAG.getConstant(0LL, dl, newVT), Cond);
       }
     }
 
diff --git a/llvm/test/CodeGen/ARM/simplifysetcc_narrow_load.ll b/llvm/test/CodeGen/ARM/simplifysetcc_narrow_load.ll
index 2cba4b46f9a8..838da59f9e41 100644
--- a/llvm/test/CodeGen/ARM/simplifysetcc_narrow_load.ll
+++ b/llvm/test/CodeGen/ARM/simplifysetcc_narrow_load.ll
@@ -40,7 +40,9 @@ define i1 @test_129_15_0(ptr %y) {
 ;
 ; CHECK-BE-LABEL: test_129_15_0:
 ; CHECK-BE:       @ %bb.0:
-; CHECK-BE-NEXT:    ldrh r0, [r0, #14]
+; CHECK-BE-NEXT:    ldr r1, [r0, #12]
+; CHECK-BE-NEXT:    ldrb r0, [r0, #16]
+; CHECK-BE-NEXT:    orr r0, r0, r1, lsl #8
 ; CHECK-BE-NEXT:    mov r1, #255
 ; CHECK-BE-NEXT:    orr r1, r1, #32512
 ; CHECK-BE-NEXT:    ands r0, r0, r1
@@ -49,7 +51,7 @@ define i1 @test_129_15_0(ptr %y) {
 ;
 ; CHECK-V7-BE-LABEL: test_129_15_0:
 ; CHECK-V7-BE:       @ %bb.0:
-; CHECK-V7-BE-NEXT:    ldrh r0, [r0, #14]
+; CHECK-V7-BE-NEXT:    ldrh r0, [r0, #15]
 ; CHECK-V7-BE-NEXT:    bfc r0, #15, #17
 ; CHECK-V7-BE-NEXT:    cmp r0, #0
 ; CHECK-V7-BE-NEXT:    movwne r0, #1
@@ -119,14 +121,14 @@ define i1 @test_33_8_0(ptr %y) {
 ;
 ; CHECK-BE-LABEL: test_33_8_0:
 ; CHECK-BE:       @ %bb.0:
-; CHECK-BE-NEXT:    ldrb r0, [r0, #3]
+; CHECK-BE-NEXT:    ldrb r0, [r0, #4]
 ; CHECK-BE-NEXT:    cmp r0, #0
 ; CHECK-BE-NEXT:    movne r0, #1
 ; CHECK-BE-NEXT:    mov pc, lr
 ;
 ; CHECK-V7-BE-LABEL: test_33_8_0:
 ; CHECK-V7-BE:       @ %bb.0:
-; CHECK-V7-BE-NEXT:    ldrb r0, [r0, #3]
+; CHECK-V7-BE-NEXT:    ldrb r0, [r0, #4]
 ; CHECK-V7-BE-NEXT:    cmp r0, #0
 ; CHECK-V7-BE-NEXT:    movwne r0, #1
 ; CHECK-V7-BE-NEXT:    bx lr
@@ -179,13 +181,13 @@ define i1 @test_33_1_31(ptr %y) {
 ;
 ; CHECK-BE-LABEL: test_33_1_31:
 ; CHECK-BE:       @ %bb.0:
-; CHECK-BE-NEXT:    ldrb r0, [r0]
+; CHECK-BE-NEXT:    ldrb r0, [r0, #1]
 ; CHECK-BE-NEXT:    lsr r0, r0, #7
 ; CHECK-BE-NEXT:    mov pc, lr
 ;
 ; CHECK-V7-BE-LABEL: test_33_1_31:
 ; CHECK-V7-BE:       @ %bb.0:
-; CHECK-V7-BE-NEXT:    ldrb r0, [r0]
+; CHECK-V7-BE-NEXT:    ldrb r0, [r0, #1]
 ; CHECK-V7-BE-NEXT:    lsr r0, r0, #7
 ; CHECK-V7-BE-NEXT:    bx lr
   %a = load i33, ptr %y
@@ -209,13 +211,13 @@ define i1 @test_33_1_0(ptr %y) {
 ;
 ; CHECK-BE-LABEL: test_33_1_0:
 ; CHECK-BE:       @ %bb.0:
-; CHECK-BE-NEXT:    ldrb r0, [r0, #3]
+; CHECK-BE-NEXT:    ldrb r0, [r0, #4]
 ; CHECK-BE-NEXT:    and r0, r0, #1
 ; CHECK-BE-NEXT:    mov pc, lr
 ;
 ; CHECK-V7-BE-LABEL: test_33_1_0:
 ; CHECK-V7-BE:       @ %bb.0:
-; CHECK-V7-BE-NEXT:    ldrb r0, [r0, #3]
+; CHECK-V7-BE-NEXT:    ldrb r0, [r0, #4]
 ; CHECK-V7-BE-NEXT:    and r0, r0, #1
 ; CHECK-V7-BE-NEXT:    bx lr
   %a = load i33, ptr %y
@@ -309,7 +311,7 @@ define i1 @test_48_16_8(ptr %y) {
 ; CHECK-LE-LABEL: test_48_16_8:
 ; CHECK-LE:       @ %bb.0:
 ; CHECK-LE-NEXT:    ldrh r0, [r0, #1]
-; CHECK-LE-NEXT:    cmp r0, #0
+; CHECK-LE-NEXT:    lsls r0, r0, #8
 ; CHECK-LE-NEXT:    movne r0, #1
 ; CHECK-LE-NEXT:    mov pc, lr
 ;
@@ -444,9 +446,7 @@ define i1 @test_48_17_0(ptr %y) {
 ;
 ; CHECK-V7-BE-LABEL: test_48_17_0:
 ; CHECK-V7-BE:       @ %bb.0:
-; CHECK-V7-BE-NEXT:    ldr r1, [r0]
-; CHECK-V7-BE-NEXT:    ldrh r0, [r0, #4]
-; CHECK-V7-BE-NEXT:    orr r0, r0, r1, lsl #16
+; CHECK-V7-BE-NEXT:    ldr r0, [r0, #2]
 ; CHECK-V7-BE-NEXT:    bfc r0, #17, #15
 ; CHECK-V7-BE-NEXT:    cmp r0, #0
 ; CHECK-V7-BE-NEXT:    movwne r0, #1
@@ -506,15 +506,14 @@ define i1 @test_40_1_32(ptr %y) {
 ;
 ; CHECK-BE-LABEL: test_40_1_32:
 ; CHECK-BE:       @ %bb.0:
-; CHECK-BE-NEXT:    ldr r0, [r0]
-; CHECK-BE-NEXT:    mov r1, #1
-; CHECK-BE-NEXT:    and r0, r1, r0, lsr #24
+; CHECK-BE-NEXT:    ldrb r0, [r0]
+; CHECK-BE-NEXT:    and r0, r0, #1
 ; CHECK-BE-NEXT:    mov pc, lr
 ;
 ; CHECK-V7-BE-LABEL: test_40_1_32:
 ; CHECK-V7-BE:       @ %bb.0:
-; CHECK-V7-BE-NEXT:    ldr r0, [r0]
-; CHECK-V7-BE-NEXT:    ubfx r0, r0, #24, #1
+; CHECK-V7-BE-NEXT:    ldrb r0, [r0]
+; CHECK-V7-BE-NEXT:    and r0, r0, #1
 ; CHECK-V7-BE-NEXT:    bx lr
   %a = load i40, ptr %y
   %b = and i40 %a, u0x100000000
diff --git a/llvm/test/CodeGen/PowerPC/simplifysetcc_narrow_load.ll b/llvm/test/CodeGen/PowerPC/simplifysetcc_narrow_load.ll
index 1a03423fe6ae..49b8e2bc2f7b 100644
--- a/llvm/test/CodeGen/PowerPC/simplifysetcc_narrow_load.ll
+++ b/llvm/test/CodeGen/PowerPC/simplifysetcc_narrow_load.ll
@@ -26,7 +26,7 @@ define i1 @test_129_15_0(ptr %y) {
 ;
 ; CHECK-BE-LABEL: test_129_15_0:
 ; CHECK-BE:       # %bb.0:
-; CHECK-BE-NEXT:    lhz 3, 14(3)
+; CHECK-BE-NEXT:    lhz 3, 15(3)
 ; CHECK-BE-NEXT:    clrlwi 3, 3, 17
 ; CHECK-BE-NEXT:    addic 4, 3, -1
 ; CHECK-BE-NEXT:    subfe 3, 4, 3
@@ -69,7 +69,7 @@ define i1 @test_33_8_0(ptr %y) {
 ;
 ; CHECK-BE-LABEL: test_33_8_0:
 ; CHECK-BE:       # %bb.0:
-; CHECK-BE-NEXT:    lbz 3, 3(3)
+; CHECK-BE-NEXT:    lbz 3, 4(3)
 ; CHECK-BE-NEXT:    addic 4, 3, -1
 ; CHECK-BE-NEXT:    subfe 3, 4, 3
 ; CHECK-BE-NEXT:    blr
@@ -105,7 +105,7 @@ define i1 @test_33_1_31(ptr %y) {
 ;
 ; CHECK-BE-LABEL: test_33_1_31:
 ; CHECK-BE:       # %bb.0:
-; CHECK-BE-NEXT:    lbz 3, 0(3)
+; CHECK-BE-NEXT:    lbz 3, 1(3)
 ; CHECK-BE-NEXT:    srwi 3, 3, 7
 ; CHECK-BE-NEXT:    blr
   %a = load i33, ptr %y
@@ -123,7 +123,7 @@ define i1 @test_33_1_0(ptr %y) {
 ;
 ; CHECK-BE-LABEL: test_33_1_0:
 ; CHECK-BE:       # %bb.0:
-; CHECK-BE-NEXT:    lbz 3, 3(3)
+; CHECK-BE-NEXT:    lbz 3, 4(3)
 ; CHECK-BE-NEXT:    clrlwi 3, 3, 31
 ; CHECK-BE-NEXT:    blr
   %a = load i33, ptr %y
@@ -250,12 +250,10 @@ define i1 @test_48_17_0(ptr %y) {
 ;
 ; CHECK-BE-LABEL: test_48_17_0:
 ; CHECK-BE:       # %bb.0:
-; CHECK-BE-NEXT:    lhz 4, 4(3)
-; CHECK-BE-NEXT:    lwz 3, 0(3)
-; CHECK-BE-NEXT:    clrlwi 4, 4, 16
-; CHECK-BE-NEXT:    rlwimi 4, 3, 16, 15, 15
-; CHECK-BE-NEXT:    addic 3, 4, -1
-; CHECK-BE-NEXT:    subfe 3, 3, 4
+; CHECK-BE-NEXT:    lwz 3, 2(3)
+; CHECK-BE-NEXT:    clrlwi 3, 3, 15
+; CHECK-BE-NEXT:    addic 4, 3, -1
+; CHECK-BE-NEXT:    subfe 3, 4, 3
 ; CHECK-BE-NEXT:    blr
   %a = load i48, ptr %y
   %b = and i48 %a, u0x1ffff
@@ -292,8 +290,8 @@ define i1 @test_40_1_32(ptr %y) {
 ;
 ; CHECK-BE-LABEL: test_40_1_32:
 ; CHECK-BE:       # %bb.0:
-; CHECK-BE-NEXT:    lwz 3, 0(3)
-; CHECK-BE-NEXT:    rlwinm 3, 3, 8, 31, 31
+; CHECK-BE-NEXT:    lbz 3, 0(3)
+; CHECK-BE-NEXT:    clrlwi 3, 3, 31
 ; CHECK-BE-NEXT:    blr
   %a = load i40, ptr %y
   %b = and i40 %a, u0x100000000
@@ -325,7 +323,6 @@ define i1 @test_24_8_8(ptr %y) {
 ; CHECK-LE-LABEL: test_24_8_8:
 ; CHECK-LE:       # %bb.0:
 ; CHECK-LE-NEXT:    lbz 3, 1(3)
-; CHECK-LE-NEXT:    slwi 3, 3, 8
 ; CHECK-LE-NEXT:    addic 4, 3, -1
 ; CHECK-LE-NEXT:    subfe 3, 4, 3
 ; CHECK-LE-NEXT:    blr
@@ -333,7 +330,6 @@ define i1 @test_24_8_8(ptr %y) {
 ; CHECK-BE-LABEL: test_24_8_8:
 ; CHECK-BE:       # %bb.0:
 ; CHECK-BE-NEXT:    lbz 3, 1(3)
-; CHECK-BE-NEXT:    slwi 3, 3, 8
 ; CHECK-BE-NEXT:    addic 4, 3, -1
 ; CHECK-BE-NEXT:    subfe 3, 4, 3
 ; CHECK-BE-NEXT:    blr
@@ -346,18 +342,16 @@ define i1 @test_24_8_8(ptr %y) {
 define i1 @test_24_8_12(ptr %y) {
 ; CHECK-LE-LABEL: test_24_8_12:
 ; CHECK-LE:       # %bb.0:
-; CHECK-LE-NEXT:    lhz 4, 0(3)
-; CHECK-LE-NEXT:    lbz 3, 2(3)
-; CHECK-LE-NEXT:    rlwinm 4, 4, 0, 16, 19
-; CHECK-LE-NEXT:    rlwimi 4, 3, 16, 12, 15
-; CHECK-LE-NEXT:    addic 3, 4, -1
-; CHECK-LE-NEXT:    subfe 3, 3, 4
+; CHECK-LE-NEXT:    lhz 3, 1(3)
+; CHECK-LE-NEXT:    rlwinm 3, 3, 0, 20, 27
+; CHECK-LE-NEXT:    addic 4, 3, -1
+; CHECK-LE-NEXT:    subfe 3, 4, 3
 ; CHECK-LE-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test_24_8_12:
 ; CHECK-BE:       # %bb.0:
 ; CHECK-BE-NEXT:    lhz 3, 0(3)
-; CHECK-BE-NEXT:    rlwinm 3, 3, 8, 12, 19
+; CHECK-BE-NEXT:    rlwinm 3, 3, 0, 20, 27
 ; CHECK-BE-NEXT:    addic 4, 3, -1
 ; CHECK-BE-NEXT:    subfe 3, 4, 3
 ; CHECK-BE-NEXT:    blr
@@ -371,7 +365,6 @@ define i1 @test_24_8_16(ptr %y) {
 ; CHECK-LE-LABEL: test_24_8_16:
 ; CHECK-LE:       # %bb.0:
 ; CHECK-LE-NEXT:    lbz 3, 2(3)
-; CHECK-LE-NEXT:    slwi 3, 3, 16
 ; CHECK-LE-NEXT:    addic 4, 3, -1
 ; CHECK-LE-NEXT:    subfe 3, 4, 3
 ; CHECK-LE-NEXT:    blr
@@ -379,7 +372,6 @@ define i1 @test_24_8_16(ptr %y) {
 ; CHECK-BE-LABEL: test_24_8_16:
 ; CHECK-BE:       # %bb.0:
 ; CHECK-BE-NEXT:    lbz 3, 0(3)
-; CHECK-BE-NEXT:    slwi 3, 3, 16
 ; CHECK-BE-NEXT:    addic 4, 3, -1
 ; CHECK-BE-NEXT:    subfe 3, 4, 3
 ; CHECK-BE-NEXT:    blr
-- 
GitLab


From 2c2377d3a9305b86ab110a4f8390b2d16bff9510 Mon Sep 17 00:00:00 2001
From: Adam Fowler <adamfowler71@gmail.com>
Date: Fri, 12 Apr 2024 15:39:22 +0100
Subject: [PATCH 072/464] Add lldb-dap to Swift distributions (#88482)

This includes the lldb-dap executable in the MacOS and Linux
distributions of Swift. Currently there is a commit in the Apple repo to
do this for just MacOS https://github.com/apple/llvm-project/pull/8176.
This PR extends this to both Linux and MacOS and brings the change
upstream.

@JDevlieghere @adrian-prantl
---
 lldb/cmake/caches/Apple-lldb-Linux.cmake | 1 +
 lldb/cmake/caches/Apple-lldb-macOS.cmake | 1 +
 2 files changed, 2 insertions(+)

diff --git a/lldb/cmake/caches/Apple-lldb-Linux.cmake b/lldb/cmake/caches/Apple-lldb-Linux.cmake
index bfa660d8654b..b936929afc42 100644
--- a/lldb/cmake/caches/Apple-lldb-Linux.cmake
+++ b/lldb/cmake/caches/Apple-lldb-Linux.cmake
@@ -5,6 +5,7 @@ set(LLVM_DISTRIBUTION_COMPONENTS
   lldb
   liblldb
   lldb-argdumper
+  lldb-dap
   lldb-server
   lldb-python-scripts
   CACHE STRING "")
diff --git a/lldb/cmake/caches/Apple-lldb-macOS.cmake b/lldb/cmake/caches/Apple-lldb-macOS.cmake
index 2aef41157bab..5155024ac3a0 100644
--- a/lldb/cmake/caches/Apple-lldb-macOS.cmake
+++ b/lldb/cmake/caches/Apple-lldb-macOS.cmake
@@ -22,6 +22,7 @@ set(LLVM_DISTRIBUTION_COMPONENTS
   lldb
   liblldb
   lldb-argdumper
+  lldb-dap
   darwin-debug
   debugserver
   CACHE STRING "")
-- 
GitLab


From 986d0dba85f3239671c5151d9817b73de08664a0 Mon Sep 17 00:00:00 2001
From: yronglin <yronglin777@gmail.com>
Date: Fri, 12 Apr 2024 22:39:40 +0800
Subject: [PATCH 073/464] [Clang] Fix AST dump for {CXXDefaultArgExpr,
 CXXDefaultInitExpr} (#88269)

This PR fix a AST dump issue since
https://github.com/llvm/llvm-project/pull/80001

When Clang dumps `CXXDefaultArgExpr`/`CXXDefaultInitExpr`, there has no
recursively dump the complete `CXXDefaultArgExpr`/`CXXDefaultInitExpr`.

Since this PR, Clang will recursively dump a
`CXXDefaultArgExpr`/`CXXDefaultInitExpr` node, even if the node has no
rewritten init.

*Consider*:
```
struct A {
  int arr[1];
};

struct B {
  const A &a = A{{0}};
};

void test() {
  B b{};
}
```

*Before*:
```
`-FunctionDecl <line:9:1, line:11:1> line:9:6 test 'void ()'
  `-CompoundStmt <col:13, line:11:1>
    `-DeclStmt <line:10:3, col:8>
      `-VarDecl <col:3, col:7> col:5 b 'B' listinit
        `-InitListExpr <col:6, col:7> 'B'
          `-CXXDefaultInitExpr <col:7> 'const A' lvalue has rewritten init
            `-ExprWithCleanups <line:6:16, col:21> 'const A' lvalue
```

*After*:
```
`-FunctionDecl 0x15a9455a8 <line:9:1, line:11:1> line:9:6 test 'void ()'
  `-CompoundStmt 0x15a945850 <col:13, line:11:1>
    `-DeclStmt 0x15a945838 <line:10:3, col:8>
      `-VarDecl 0x15a945708 <col:3, col:7> col:5 b 'B' listinit
        `-InitListExpr 0x15a9457b0 <col:6, col:7> 'B'
          `-CXXDefaultInitExpr 0x15a9457f8 <col:7> 'const A' lvalue has rewritten init
            `-ExprWithCleanups 0x15a945568 <line:6:16, col:21> 'const A' lvalue
              `-MaterializeTemporaryExpr 0x15a945500 <col:16, col:21> 'const A' lvalue extended by Field 0x15a945160 'a' 'const A &'
                `-ImplicitCastExpr 0x15a9454e8 <col:16, col:21> 'const A' <NoOp>
                  `-CXXFunctionalCastExpr 0x15a9454c0 <col:16, col:21> 'A' functional cast to A <NoOp>
                    `-InitListExpr 0x15a9452c0 <col:17, col:21> 'A'
                      `-InitListExpr 0x15a945308 <col:18, col:20> 'int[1]'
                        `-IntegerLiteral 0x15a945210 <col:19> 'int' 0
```

---------

Signed-off-by: yronglin <yronglin777@gmail.com>
---
 clang/include/clang/AST/ASTNodeTraverser.h    |    8 +
 clang/include/clang/AST/JSONNodeDumper.h      |    2 +
 clang/lib/AST/JSONNodeDumper.cpp              |    8 +
 clang/lib/AST/TextNodeDumper.cpp              |   14 +-
 clang/test/AST/ast-dump-default-arg-json.cpp  | 2091 +++++++++++++++++
 clang/test/AST/ast-dump-default-init-json.cpp |  929 ++++++++
 clang/test/AST/ast-dump-default-init.cpp      |   21 +
 .../test/AST/ast-dump-for-range-lifetime.cpp  |   21 +
 clang/unittests/AST/ASTTraverserTest.cpp      |    8 +
 9 files changed, 3090 insertions(+), 12 deletions(-)
 create mode 100644 clang/test/AST/ast-dump-default-arg-json.cpp
 create mode 100644 clang/test/AST/ast-dump-default-init-json.cpp
 create mode 100644 clang/test/AST/ast-dump-default-init.cpp

diff --git a/clang/include/clang/AST/ASTNodeTraverser.h b/clang/include/clang/AST/ASTNodeTraverser.h
index 94e7dd817809..a6e3b05b0059 100644
--- a/clang/include/clang/AST/ASTNodeTraverser.h
+++ b/clang/include/clang/AST/ASTNodeTraverser.h
@@ -932,6 +932,14 @@ public:
       Visit(TArg);
   }
 
+  void VisitCXXDefaultArgExpr(const CXXDefaultArgExpr *Node) {
+    Visit(Node->getExpr());
+  }
+
+  void VisitCXXDefaultInitExpr(const CXXDefaultInitExpr *Node) {
+    Visit(Node->getExpr());
+  }
+
   // Implements Visit methods for Attrs.
 #include "clang/AST/AttrNodeTraverse.inc"
 };
diff --git a/clang/include/clang/AST/JSONNodeDumper.h b/clang/include/clang/AST/JSONNodeDumper.h
index 7a60f362650c..55bd583e304e 100644
--- a/clang/include/clang/AST/JSONNodeDumper.h
+++ b/clang/include/clang/AST/JSONNodeDumper.h
@@ -310,6 +310,8 @@ public:
   void VisitMaterializeTemporaryExpr(const MaterializeTemporaryExpr *MTE);
   void VisitCXXDependentScopeMemberExpr(const CXXDependentScopeMemberExpr *ME);
   void VisitRequiresExpr(const RequiresExpr *RE);
+  void VisitCXXDefaultArgExpr(const CXXDefaultArgExpr *Node);
+  void VisitCXXDefaultInitExpr(const CXXDefaultInitExpr *Node);
 
   void VisitObjCEncodeExpr(const ObjCEncodeExpr *OEE);
   void VisitObjCMessageExpr(const ObjCMessageExpr *OME);
diff --git a/clang/lib/AST/JSONNodeDumper.cpp b/clang/lib/AST/JSONNodeDumper.cpp
index fb3494393f75..78115200f4f0 100644
--- a/clang/lib/AST/JSONNodeDumper.cpp
+++ b/clang/lib/AST/JSONNodeDumper.cpp
@@ -1576,6 +1576,14 @@ void JSONNodeDumper::VisitMaterializeTemporaryExpr(
   attributeOnlyIfTrue("boundToLValueRef", MTE->isBoundToLvalueReference());
 }
 
+void JSONNodeDumper::VisitCXXDefaultArgExpr(const CXXDefaultArgExpr *Node) {
+  attributeOnlyIfTrue("hasRewrittenInit", Node->hasRewrittenInit());
+}
+
+void JSONNodeDumper::VisitCXXDefaultInitExpr(const CXXDefaultInitExpr *Node) {
+  attributeOnlyIfTrue("hasRewrittenInit", Node->hasRewrittenInit());
+}
+
 void JSONNodeDumper::VisitCXXDependentScopeMemberExpr(
     const CXXDependentScopeMemberExpr *DSME) {
   JOS.attribute("isArrow", DSME->isArrow());
diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp
index 085a7f51ce99..f4b2df7730ae 100644
--- a/clang/lib/AST/TextNodeDumper.cpp
+++ b/clang/lib/AST/TextNodeDumper.cpp
@@ -1450,23 +1450,13 @@ void TextNodeDumper::VisitExpressionTraitExpr(const ExpressionTraitExpr *Node) {
 }
 
 void TextNodeDumper::VisitCXXDefaultArgExpr(const CXXDefaultArgExpr *Node) {
-  if (Node->hasRewrittenInit()) {
+  if (Node->hasRewrittenInit())
     OS << " has rewritten init";
-    AddChild([=] {
-      ColorScope Color(OS, ShowColors, StmtColor);
-      Visit(Node->getExpr());
-    });
-  }
 }
 
 void TextNodeDumper::VisitCXXDefaultInitExpr(const CXXDefaultInitExpr *Node) {
-  if (Node->hasRewrittenInit()) {
+  if (Node->hasRewrittenInit())
     OS << " has rewritten init";
-    AddChild([=] {
-      ColorScope Color(OS, ShowColors, StmtColor);
-      Visit(Node->getExpr());
-    });
-  }
 }
 
 void TextNodeDumper::VisitMaterializeTemporaryExpr(
diff --git a/clang/test/AST/ast-dump-default-arg-json.cpp b/clang/test/AST/ast-dump-default-arg-json.cpp
new file mode 100644
index 000000000000..b6a138934caf
--- /dev/null
+++ b/clang/test/AST/ast-dump-default-arg-json.cpp
@@ -0,0 +1,2091 @@
+// RUN: %clang_cc1 -std=c++23 -triple x86_64-linux-gnu -fsyntax-only -ast-dump=json %s | FileCheck %s
+
+struct S {
+  int arr[1];
+  const int *begin() const { return arr; }
+  const int *end() const { return &arr[1]; }
+  S() {}
+  ~S() {}
+};
+
+S func(const int &, const S &s = S());
+
+void test() {
+  for (auto v : func(1)) {}
+}
+
+// NOTE: CHECK lines have been autogenerated by gen_ast_dump_json_test.py
+
+
+// CHECK-NOT: {{^}}Dumping
+// CHECK:  "kind": "TranslationUnitDecl",
+// CHECK-NEXT:  "loc": {},
+// CHECK-NEXT:  "range": {
+// CHECK-NEXT:   "begin": {},
+// CHECK-NEXT:   "end": {}
+// CHECK-NEXT:  },
+// CHECK-NEXT:  "inner": [
+// CHECK-NEXT:   {
+// CHECK-NEXT:    "id": "0x{{.*}}",
+// CHECK-NEXT:    "kind": "TypedefDecl",
+// CHECK-NEXT:    "loc": {},
+// CHECK-NEXT:    "range": {
+// CHECK-NEXT:     "begin": {},
+// CHECK-NEXT:     "end": {}
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "isImplicit": true,
+// CHECK-NEXT:    "name": "__int128_t",
+// CHECK-NEXT:    "type": {
+// CHECK-NEXT:     "qualType": "__int128"
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "inner": [
+// CHECK-NEXT:     {
+// CHECK-NEXT:      "id": "0x{{.*}}",
+// CHECK-NEXT:      "kind": "BuiltinType",
+// CHECK-NEXT:      "type": {
+// CHECK-NEXT:       "qualType": "__int128"
+// CHECK-NEXT:      }
+// CHECK-NEXT:     }
+// CHECK-NEXT:    ]
+// CHECK-NEXT:   },
+// CHECK-NEXT:   {
+// CHECK-NEXT:    "id": "0x{{.*}}",
+// CHECK-NEXT:    "kind": "TypedefDecl",
+// CHECK-NEXT:    "loc": {},
+// CHECK-NEXT:    "range": {
+// CHECK-NEXT:     "begin": {},
+// CHECK-NEXT:     "end": {}
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "isImplicit": true,
+// CHECK-NEXT:    "name": "__uint128_t",
+// CHECK-NEXT:    "type": {
+// CHECK-NEXT:     "qualType": "unsigned __int128"
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "inner": [
+// CHECK-NEXT:     {
+// CHECK-NEXT:      "id": "0x{{.*}}",
+// CHECK-NEXT:      "kind": "BuiltinType",
+// CHECK-NEXT:      "type": {
+// CHECK-NEXT:       "qualType": "unsigned __int128"
+// CHECK-NEXT:      }
+// CHECK-NEXT:     }
+// CHECK-NEXT:    ]
+// CHECK-NEXT:   },
+// CHECK-NEXT:   {
+// CHECK-NEXT:    "id": "0x{{.*}}",
+// CHECK-NEXT:    "kind": "TypedefDecl",
+// CHECK-NEXT:    "loc": {},
+// CHECK-NEXT:    "range": {
+// CHECK-NEXT:     "begin": {},
+// CHECK-NEXT:     "end": {}
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "isImplicit": true,
+// CHECK-NEXT:    "name": "__NSConstantString",
+// CHECK-NEXT:    "type": {
+// CHECK-NEXT:     "qualType": "__NSConstantString_tag"
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "inner": [
+// CHECK-NEXT:     {
+// CHECK-NEXT:      "id": "0x{{.*}}",
+// CHECK-NEXT:      "kind": "RecordType",
+// CHECK-NEXT:      "type": {
+// CHECK-NEXT:       "qualType": "__NSConstantString_tag"
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "decl": {
+// CHECK-NEXT:       "id": "0x{{.*}}",
+// CHECK-NEXT:       "kind": "CXXRecordDecl",
+// CHECK-NEXT:       "name": "__NSConstantString_tag"
+// CHECK-NEXT:      }
+// CHECK-NEXT:     }
+// CHECK-NEXT:    ]
+// CHECK-NEXT:   },
+// CHECK-NEXT:   {
+// CHECK-NEXT:    "id": "0x{{.*}}",
+// CHECK-NEXT:    "kind": "TypedefDecl",
+// CHECK-NEXT:    "loc": {},
+// CHECK-NEXT:    "range": {
+// CHECK-NEXT:     "begin": {},
+// CHECK-NEXT:     "end": {}
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "isImplicit": true,
+// CHECK-NEXT:    "name": "__builtin_ms_va_list",
+// CHECK-NEXT:    "type": {
+// CHECK-NEXT:     "qualType": "char *"
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "inner": [
+// CHECK-NEXT:     {
+// CHECK-NEXT:      "id": "0x{{.*}}",
+// CHECK-NEXT:      "kind": "PointerType",
+// CHECK-NEXT:      "type": {
+// CHECK-NEXT:       "qualType": "char *"
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "inner": [
+// CHECK-NEXT:       {
+// CHECK-NEXT:        "id": "0x{{.*}}",
+// CHECK-NEXT:        "kind": "BuiltinType",
+// CHECK-NEXT:        "type": {
+// CHECK-NEXT:         "qualType": "char"
+// CHECK-NEXT:        }
+// CHECK-NEXT:       }
+// CHECK-NEXT:      ]
+// CHECK-NEXT:     }
+// CHECK-NEXT:    ]
+// CHECK-NEXT:   },
+// CHECK-NEXT:   {
+// CHECK-NEXT:    "id": "0x{{.*}}",
+// CHECK-NEXT:    "kind": "TypedefDecl",
+// CHECK-NEXT:    "loc": {},
+// CHECK-NEXT:    "range": {
+// CHECK-NEXT:     "begin": {},
+// CHECK-NEXT:     "end": {}
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "isImplicit": true,
+// CHECK-NEXT:    "name": "__builtin_va_list",
+// CHECK-NEXT:    "type": {
+// CHECK-NEXT:     "qualType": "__va_list_tag[1]"
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "inner": [
+// CHECK-NEXT:     {
+// CHECK-NEXT:      "id": "0x{{.*}}",
+// CHECK-NEXT:      "kind": "ConstantArrayType",
+// CHECK-NEXT:      "type": {
+// CHECK-NEXT:       "qualType": "__va_list_tag[1]"
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "size": 1,
+// CHECK-NEXT:      "inner": [
+// CHECK-NEXT:       {
+// CHECK-NEXT:        "id": "0x{{.*}}",
+// CHECK-NEXT:        "kind": "RecordType",
+// CHECK-NEXT:        "type": {
+// CHECK-NEXT:         "qualType": "__va_list_tag"
+// CHECK-NEXT:        },
+// CHECK-NEXT:        "decl": {
+// CHECK-NEXT:         "id": "0x{{.*}}",
+// CHECK-NEXT:         "kind": "CXXRecordDecl",
+// CHECK-NEXT:         "name": "__va_list_tag"
+// CHECK-NEXT:        }
+// CHECK-NEXT:       }
+// CHECK-NEXT:      ]
+// CHECK-NEXT:     }
+// CHECK-NEXT:    ]
+// CHECK-NEXT:   },
+// CHECK-NEXT:   {
+// CHECK-NEXT:    "id": "0x{{.*}}",
+// CHECK-NEXT:    "kind": "CXXRecordDecl",
+// CHECK-NEXT:    "loc": {
+// CHECK-NEXT:     "offset": 110,
+// CHECK-NEXT:     "file": "{{.*}}",
+// CHECK-NEXT:     "line": 3,
+// CHECK-NEXT:     "col": 8,
+// CHECK-NEXT:     "tokLen": 1
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "range": {
+// CHECK-NEXT:     "begin": {
+// CHECK-NEXT:      "offset": 103,
+// CHECK-NEXT:      "col": 1,
+// CHECK-NEXT:      "tokLen": 6
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "end": {
+// CHECK-NEXT:      "offset": 235,
+// CHECK-NEXT:      "line": 9,
+// CHECK-NEXT:      "col": 1,
+// CHECK-NEXT:      "tokLen": 1
+// CHECK-NEXT:     }
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "isReferenced": true,
+// CHECK-NEXT:    "name": "S",
+// CHECK-NEXT:    "tagUsed": "struct",
+// CHECK-NEXT:    "completeDefinition": true,
+// CHECK-NEXT:    "definitionData": {
+// CHECK-NEXT:     "canConstDefaultInit": true,
+// CHECK-NEXT:     "copyAssign": {
+// CHECK-NEXT:      "hasConstParam": true,
+// CHECK-NEXT:      "implicitHasConstParam": true,
+// CHECK-NEXT:      "needsImplicit": true,
+// CHECK-NEXT:      "simple": true,
+// CHECK-NEXT:      "trivial": true
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "copyCtor": {
+// CHECK-NEXT:      "hasConstParam": true,
+// CHECK-NEXT:      "implicitHasConstParam": true,
+// CHECK-NEXT:      "simple": true,
+// CHECK-NEXT:      "trivial": true
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "defaultCtor": {
+// CHECK-NEXT:      "defaultedIsConstexpr": true,
+// CHECK-NEXT:      "exists": true,
+// CHECK-NEXT:      "nonTrivial": true,
+// CHECK-NEXT:      "userProvided": true
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "dtor": {
+// CHECK-NEXT:      "nonTrivial": true,
+// CHECK-NEXT:      "userDeclared": true
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "hasUserDeclaredConstructor": true,
+// CHECK-NEXT:     "isStandardLayout": true,
+// CHECK-NEXT:     "moveAssign": {},
+// CHECK-NEXT:     "moveCtor": {}
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "inner": [
+// CHECK-NEXT:     {
+// CHECK-NEXT:      "id": "0x{{.*}}",
+// CHECK-NEXT:      "kind": "CXXRecordDecl",
+// CHECK-NEXT:      "loc": {
+// CHECK-NEXT:       "offset": 110,
+// CHECK-NEXT:       "line": 3,
+// CHECK-NEXT:       "col": 8,
+// CHECK-NEXT:       "tokLen": 1
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "range": {
+// CHECK-NEXT:       "begin": {
+// CHECK-NEXT:        "offset": 103,
+// CHECK-NEXT:        "col": 1,
+// CHECK-NEXT:        "tokLen": 6
+// CHECK-NEXT:       },
+// CHECK-NEXT:       "end": {
+// CHECK-NEXT:        "offset": 110,
+// CHECK-NEXT:        "col": 8,
+// CHECK-NEXT:        "tokLen": 1
+// CHECK-NEXT:       }
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "isImplicit": true,
+// CHECK-NEXT:      "isReferenced": true,
+// CHECK-NEXT:      "name": "S",
+// CHECK-NEXT:      "tagUsed": "struct"
+// CHECK-NEXT:     },
+// CHECK-NEXT:     {
+// CHECK-NEXT:      "id": "0x{{.*}}",
+// CHECK-NEXT:      "kind": "FieldDecl",
+// CHECK-NEXT:      "loc": {
+// CHECK-NEXT:       "offset": 120,
+// CHECK-NEXT:       "line": 4,
+// CHECK-NEXT:       "col": 7,
+// CHECK-NEXT:       "tokLen": 3
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "range": {
+// CHECK-NEXT:       "begin": {
+// CHECK-NEXT:        "offset": 116,
+// CHECK-NEXT:        "col": 3,
+// CHECK-NEXT:        "tokLen": 3
+// CHECK-NEXT:       },
+// CHECK-NEXT:       "end": {
+// CHECK-NEXT:        "offset": 125,
+// CHECK-NEXT:        "col": 12,
+// CHECK-NEXT:        "tokLen": 1
+// CHECK-NEXT:       }
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "isReferenced": true,
+// CHECK-NEXT:      "name": "arr",
+// CHECK-NEXT:      "type": {
+// CHECK-NEXT:       "qualType": "int[1]"
+// CHECK-NEXT:      }
+// CHECK-NEXT:     },
+// CHECK-NEXT:     {
+// CHECK-NEXT:      "id": "0x{{.*}}",
+// CHECK-NEXT:      "kind": "CXXMethodDecl",
+// CHECK-NEXT:      "loc": {
+// CHECK-NEXT:       "offset": 141,
+// CHECK-NEXT:       "line": 5,
+// CHECK-NEXT:       "col": 14,
+// CHECK-NEXT:       "tokLen": 5
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "range": {
+// CHECK-NEXT:       "begin": {
+// CHECK-NEXT:        "offset": 130,
+// CHECK-NEXT:        "col": 3,
+// CHECK-NEXT:        "tokLen": 5
+// CHECK-NEXT:       },
+// CHECK-NEXT:       "end": {
+// CHECK-NEXT:        "offset": 169,
+// CHECK-NEXT:        "col": 42,
+// CHECK-NEXT:        "tokLen": 1
+// CHECK-NEXT:       }
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "isUsed": true,
+// CHECK-NEXT:      "name": "begin",
+// CHECK-NEXT:      "mangledName": "_ZNK1S5beginEv",
+// CHECK-NEXT:      "type": {
+// CHECK-NEXT:       "qualType": "const int *() const"
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "inner": [
+// CHECK-NEXT:       {
+// CHECK-NEXT:        "id": "0x{{.*}}",
+// CHECK-NEXT:        "kind": "CompoundStmt",
+// CHECK-NEXT:        "range": {
+// CHECK-NEXT:         "begin": {
+// CHECK-NEXT:          "offset": 155,
+// CHECK-NEXT:          "col": 28,
+// CHECK-NEXT:          "tokLen": 1
+// CHECK-NEXT:         },
+// CHECK-NEXT:         "end": {
+// CHECK-NEXT:          "offset": 169,
+// CHECK-NEXT:          "col": 42,
+// CHECK-NEXT:          "tokLen": 1
+// CHECK-NEXT:         }
+// CHECK-NEXT:        },
+// CHECK-NEXT:        "inner": [
+// CHECK-NEXT:         {
+// CHECK-NEXT:          "id": "0x{{.*}}",
+// CHECK-NEXT:          "kind": "ReturnStmt",
+// CHECK-NEXT:          "range": {
+// CHECK-NEXT:           "begin": {
+// CHECK-NEXT:            "offset": 157,
+// CHECK-NEXT:            "col": 30,
+// CHECK-NEXT:            "tokLen": 6
+// CHECK-NEXT:           },
+// CHECK-NEXT:           "end": {
+// CHECK-NEXT:            "offset": 164,
+// CHECK-NEXT:            "col": 37,
+// CHECK-NEXT:            "tokLen": 3
+// CHECK-NEXT:           }
+// CHECK-NEXT:          },
+// CHECK-NEXT:          "inner": [
+// CHECK-NEXT:           {
+// CHECK-NEXT:            "id": "0x{{.*}}",
+// CHECK-NEXT:            "kind": "ImplicitCastExpr",
+// CHECK-NEXT:            "range": {
+// CHECK-NEXT:             "begin": {
+// CHECK-NEXT:              "offset": 164,
+// CHECK-NEXT:              "col": 37,
+// CHECK-NEXT:              "tokLen": 3
+// CHECK-NEXT:             },
+// CHECK-NEXT:             "end": {
+// CHECK-NEXT:              "offset": 164,
+// CHECK-NEXT:              "col": 37,
+// CHECK-NEXT:              "tokLen": 3
+// CHECK-NEXT:             }
+// CHECK-NEXT:            },
+// CHECK-NEXT:            "type": {
+// CHECK-NEXT:             "qualType": "const int *"
+// CHECK-NEXT:            },
+// CHECK-NEXT:            "valueCategory": "prvalue",
+// CHECK-NEXT:            "castKind": "ArrayToPointerDecay",
+// CHECK-NEXT:            "inner": [
+// CHECK-NEXT:             {
+// CHECK-NEXT:              "id": "0x{{.*}}",
+// CHECK-NEXT:              "kind": "MemberExpr",
+// CHECK-NEXT:              "range": {
+// CHECK-NEXT:               "begin": {
+// CHECK-NEXT:                "offset": 164,
+// CHECK-NEXT:                "col": 37,
+// CHECK-NEXT:                "tokLen": 3
+// CHECK-NEXT:               },
+// CHECK-NEXT:               "end": {
+// CHECK-NEXT:                "offset": 164,
+// CHECK-NEXT:                "col": 37,
+// CHECK-NEXT:                "tokLen": 3
+// CHECK-NEXT:               }
+// CHECK-NEXT:              },
+// CHECK-NEXT:              "type": {
+// CHECK-NEXT:               "qualType": "const int[1]"
+// CHECK-NEXT:              },
+// CHECK-NEXT:              "valueCategory": "lvalue",
+// CHECK-NEXT:              "name": "arr",
+// CHECK-NEXT:              "isArrow": true,
+// CHECK-NEXT:              "referencedMemberDecl": "0x{{.*}}",
+// CHECK-NEXT:              "inner": [
+// CHECK-NEXT:               {
+// CHECK-NEXT:                "id": "0x{{.*}}",
+// CHECK-NEXT:                "kind": "CXXThisExpr",
+// CHECK-NEXT:                "range": {
+// CHECK-NEXT:                 "begin": {
+// CHECK-NEXT:                  "offset": 164,
+// CHECK-NEXT:                  "col": 37,
+// CHECK-NEXT:                  "tokLen": 3
+// CHECK-NEXT:                 },
+// CHECK-NEXT:                 "end": {
+// CHECK-NEXT:                  "offset": 164,
+// CHECK-NEXT:                  "col": 37,
+// CHECK-NEXT:                  "tokLen": 3
+// CHECK-NEXT:                 }
+// CHECK-NEXT:                },
+// CHECK-NEXT:                "type": {
+// CHECK-NEXT:                 "qualType": "const S *"
+// CHECK-NEXT:                },
+// CHECK-NEXT:                "valueCategory": "prvalue",
+// CHECK-NEXT:                "implicit": true
+// CHECK-NEXT:               }
+// CHECK-NEXT:              ]
+// CHECK-NEXT:             }
+// CHECK-NEXT:            ]
+// CHECK-NEXT:           }
+// CHECK-NEXT:          ]
+// CHECK-NEXT:         }
+// CHECK-NEXT:        ]
+// CHECK-NEXT:       }
+// CHECK-NEXT:      ]
+// CHECK-NEXT:     },
+// CHECK-NEXT:     {
+// CHECK-NEXT:      "id": "0x{{.*}}",
+// CHECK-NEXT:      "kind": "CXXMethodDecl",
+// CHECK-NEXT:      "loc": {
+// CHECK-NEXT:       "offset": 184,
+// CHECK-NEXT:       "line": 6,
+// CHECK-NEXT:       "col": 14,
+// CHECK-NEXT:       "tokLen": 3
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "range": {
+// CHECK-NEXT:       "begin": {
+// CHECK-NEXT:        "offset": 173,
+// CHECK-NEXT:        "col": 3,
+// CHECK-NEXT:        "tokLen": 5
+// CHECK-NEXT:       },
+// CHECK-NEXT:       "end": {
+// CHECK-NEXT:        "offset": 214,
+// CHECK-NEXT:        "col": 44,
+// CHECK-NEXT:        "tokLen": 1
+// CHECK-NEXT:       }
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "isUsed": true,
+// CHECK-NEXT:      "name": "end",
+// CHECK-NEXT:      "mangledName": "_ZNK1S3endEv",
+// CHECK-NEXT:      "type": {
+// CHECK-NEXT:       "qualType": "const int *() const"
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "inner": [
+// CHECK-NEXT:       {
+// CHECK-NEXT:        "id": "0x{{.*}}",
+// CHECK-NEXT:        "kind": "CompoundStmt",
+// CHECK-NEXT:        "range": {
+// CHECK-NEXT:         "begin": {
+// CHECK-NEXT:          "offset": 196,
+// CHECK-NEXT:          "col": 26,
+// CHECK-NEXT:          "tokLen": 1
+// CHECK-NEXT:         },
+// CHECK-NEXT:         "end": {
+// CHECK-NEXT:          "offset": 214,
+// CHECK-NEXT:          "col": 44,
+// CHECK-NEXT:          "tokLen": 1
+// CHECK-NEXT:         }
+// CHECK-NEXT:        },
+// CHECK-NEXT:        "inner": [
+// CHECK-NEXT:         {
+// CHECK-NEXT:          "id": "0x{{.*}}",
+// CHECK-NEXT:          "kind": "ReturnStmt",
+// CHECK-NEXT:          "range": {
+// CHECK-NEXT:           "begin": {
+// CHECK-NEXT:            "offset": 198,
+// CHECK-NEXT:            "col": 28,
+// CHECK-NEXT:            "tokLen": 6
+// CHECK-NEXT:           },
+// CHECK-NEXT:           "end": {
+// CHECK-NEXT:            "offset": 211,
+// CHECK-NEXT:            "col": 41,
+// CHECK-NEXT:            "tokLen": 1
+// CHECK-NEXT:           }
+// CHECK-NEXT:          },
+// CHECK-NEXT:          "inner": [
+// CHECK-NEXT:           {
+// CHECK-NEXT:            "id": "0x{{.*}}",
+// CHECK-NEXT:            "kind": "UnaryOperator",
+// CHECK-NEXT:            "range": {
+// CHECK-NEXT:             "begin": {
+// CHECK-NEXT:              "offset": 205,
+// CHECK-NEXT:              "col": 35,
+// CHECK-NEXT:              "tokLen": 1
+// CHECK-NEXT:             },
+// CHECK-NEXT:             "end": {
+// CHECK-NEXT:              "offset": 211,
+// CHECK-NEXT:              "col": 41,
+// CHECK-NEXT:              "tokLen": 1
+// CHECK-NEXT:             }
+// CHECK-NEXT:            },
+// CHECK-NEXT:            "type": {
+// CHECK-NEXT:             "qualType": "const int *"
+// CHECK-NEXT:            },
+// CHECK-NEXT:            "valueCategory": "prvalue",
+// CHECK-NEXT:            "isPostfix": false,
+// CHECK-NEXT:            "opcode": "&",
+// CHECK-NEXT:            "canOverflow": false,
+// CHECK-NEXT:            "inner": [
+// CHECK-NEXT:             {
+// CHECK-NEXT:              "id": "0x{{.*}}",
+// CHECK-NEXT:              "kind": "ArraySubscriptExpr",
+// CHECK-NEXT:              "range": {
+// CHECK-NEXT:               "begin": {
+// CHECK-NEXT:                "offset": 206,
+// CHECK-NEXT:                "col": 36,
+// CHECK-NEXT:                "tokLen": 3
+// CHECK-NEXT:               },
+// CHECK-NEXT:               "end": {
+// CHECK-NEXT:                "offset": 211,
+// CHECK-NEXT:                "col": 41,
+// CHECK-NEXT:                "tokLen": 1
+// CHECK-NEXT:               }
+// CHECK-NEXT:              },
+// CHECK-NEXT:              "type": {
+// CHECK-NEXT:               "qualType": "const int"
+// CHECK-NEXT:              },
+// CHECK-NEXT:              "valueCategory": "lvalue",
+// CHECK-NEXT:              "inner": [
+// CHECK-NEXT:               {
+// CHECK-NEXT:                "id": "0x{{.*}}",
+// CHECK-NEXT:                "kind": "ImplicitCastExpr",
+// CHECK-NEXT:                "range": {
+// CHECK-NEXT:                 "begin": {
+// CHECK-NEXT:                  "offset": 206,
+// CHECK-NEXT:                  "col": 36,
+// CHECK-NEXT:                  "tokLen": 3
+// CHECK-NEXT:                 },
+// CHECK-NEXT:                 "end": {
+// CHECK-NEXT:                  "offset": 206,
+// CHECK-NEXT:                  "col": 36,
+// CHECK-NEXT:                  "tokLen": 3
+// CHECK-NEXT:                 }
+// CHECK-NEXT:                },
+// CHECK-NEXT:                "type": {
+// CHECK-NEXT:                 "qualType": "const int *"
+// CHECK-NEXT:                },
+// CHECK-NEXT:                "valueCategory": "prvalue",
+// CHECK-NEXT:                "castKind": "ArrayToPointerDecay",
+// CHECK-NEXT:                "inner": [
+// CHECK-NEXT:                 {
+// CHECK-NEXT:                  "id": "0x{{.*}}",
+// CHECK-NEXT:                  "kind": "MemberExpr",
+// CHECK-NEXT:                  "range": {
+// CHECK-NEXT:                   "begin": {
+// CHECK-NEXT:                    "offset": 206,
+// CHECK-NEXT:                    "col": 36,
+// CHECK-NEXT:                    "tokLen": 3
+// CHECK-NEXT:                   },
+// CHECK-NEXT:                   "end": {
+// CHECK-NEXT:                    "offset": 206,
+// CHECK-NEXT:                    "col": 36,
+// CHECK-NEXT:                    "tokLen": 3
+// CHECK-NEXT:                   }
+// CHECK-NEXT:                  },
+// CHECK-NEXT:                  "type": {
+// CHECK-NEXT:                   "qualType": "const int[1]"
+// CHECK-NEXT:                  },
+// CHECK-NEXT:                  "valueCategory": "lvalue",
+// CHECK-NEXT:                  "name": "arr",
+// CHECK-NEXT:                  "isArrow": true,
+// CHECK-NEXT:                  "referencedMemberDecl": "0x{{.*}}",
+// CHECK-NEXT:                  "inner": [
+// CHECK-NEXT:                   {
+// CHECK-NEXT:                    "id": "0x{{.*}}",
+// CHECK-NEXT:                    "kind": "CXXThisExpr",
+// CHECK-NEXT:                    "range": {
+// CHECK-NEXT:                     "begin": {
+// CHECK-NEXT:                      "offset": 206,
+// CHECK-NEXT:                      "col": 36,
+// CHECK-NEXT:                      "tokLen": 3
+// CHECK-NEXT:                     },
+// CHECK-NEXT:                     "end": {
+// CHECK-NEXT:                      "offset": 206,
+// CHECK-NEXT:                      "col": 36,
+// CHECK-NEXT:                      "tokLen": 3
+// CHECK-NEXT:                     }
+// CHECK-NEXT:                    },
+// CHECK-NEXT:                    "type": {
+// CHECK-NEXT:                     "qualType": "const S *"
+// CHECK-NEXT:                    },
+// CHECK-NEXT:                    "valueCategory": "prvalue",
+// CHECK-NEXT:                    "implicit": true
+// CHECK-NEXT:                   }
+// CHECK-NEXT:                  ]
+// CHECK-NEXT:                 }
+// CHECK-NEXT:                ]
+// CHECK-NEXT:               },
+// CHECK-NEXT:               {
+// CHECK-NEXT:                "id": "0x{{.*}}",
+// CHECK-NEXT:                "kind": "IntegerLiteral",
+// CHECK-NEXT:                "range": {
+// CHECK-NEXT:                 "begin": {
+// CHECK-NEXT:                  "offset": 210,
+// CHECK-NEXT:                  "col": 40,
+// CHECK-NEXT:                  "tokLen": 1
+// CHECK-NEXT:                 },
+// CHECK-NEXT:                 "end": {
+// CHECK-NEXT:                  "offset": 210,
+// CHECK-NEXT:                  "col": 40,
+// CHECK-NEXT:                  "tokLen": 1
+// CHECK-NEXT:                 }
+// CHECK-NEXT:                },
+// CHECK-NEXT:                "type": {
+// CHECK-NEXT:                 "qualType": "int"
+// CHECK-NEXT:                },
+// CHECK-NEXT:                "valueCategory": "prvalue",
+// CHECK-NEXT:                "value": "1"
+// CHECK-NEXT:               }
+// CHECK-NEXT:              ]
+// CHECK-NEXT:             }
+// CHECK-NEXT:            ]
+// CHECK-NEXT:           }
+// CHECK-NEXT:          ]
+// CHECK-NEXT:         }
+// CHECK-NEXT:        ]
+// CHECK-NEXT:       }
+// CHECK-NEXT:      ]
+// CHECK-NEXT:     },
+// CHECK-NEXT:     {
+// CHECK-NEXT:      "id": "0x{{.*}}",
+// CHECK-NEXT:      "kind": "CXXConstructorDecl",
+// CHECK-NEXT:      "loc": {
+// CHECK-NEXT:       "offset": 218,
+// CHECK-NEXT:       "line": 7,
+// CHECK-NEXT:       "col": 3,
+// CHECK-NEXT:       "tokLen": 1
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "range": {
+// CHECK-NEXT:       "begin": {
+// CHECK-NEXT:        "offset": 218,
+// CHECK-NEXT:        "col": 3,
+// CHECK-NEXT:        "tokLen": 1
+// CHECK-NEXT:       },
+// CHECK-NEXT:       "end": {
+// CHECK-NEXT:        "offset": 223,
+// CHECK-NEXT:        "col": 8,
+// CHECK-NEXT:        "tokLen": 1
+// CHECK-NEXT:       }
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "isUsed": true,
+// CHECK-NEXT:      "name": "S",
+// CHECK-NEXT:      "mangledName": "_ZN1SC1Ev",
+// CHECK-NEXT:      "type": {
+// CHECK-NEXT:       "qualType": "void ()"
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "inner": [
+// CHECK-NEXT:       {
+// CHECK-NEXT:        "id": "0x{{.*}}",
+// CHECK-NEXT:        "kind": "CompoundStmt",
+// CHECK-NEXT:        "range": {
+// CHECK-NEXT:         "begin": {
+// CHECK-NEXT:          "offset": 222,
+// CHECK-NEXT:          "col": 7,
+// CHECK-NEXT:          "tokLen": 1
+// CHECK-NEXT:         },
+// CHECK-NEXT:         "end": {
+// CHECK-NEXT:          "offset": 223,
+// CHECK-NEXT:          "col": 8,
+// CHECK-NEXT:          "tokLen": 1
+// CHECK-NEXT:         }
+// CHECK-NEXT:        }
+// CHECK-NEXT:       }
+// CHECK-NEXT:      ]
+// CHECK-NEXT:     },
+// CHECK-NEXT:     {
+// CHECK-NEXT:      "id": "0x{{.*}}",
+// CHECK-NEXT:      "kind": "CXXDestructorDecl",
+// CHECK-NEXT:      "loc": {
+// CHECK-NEXT:       "offset": 227,
+// CHECK-NEXT:       "line": 8,
+// CHECK-NEXT:       "col": 3,
+// CHECK-NEXT:       "tokLen": 1
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "range": {
+// CHECK-NEXT:       "begin": {
+// CHECK-NEXT:        "offset": 227,
+// CHECK-NEXT:        "col": 3,
+// CHECK-NEXT:        "tokLen": 1
+// CHECK-NEXT:       },
+// CHECK-NEXT:       "end": {
+// CHECK-NEXT:        "offset": 233,
+// CHECK-NEXT:        "col": 9,
+// CHECK-NEXT:        "tokLen": 1
+// CHECK-NEXT:       }
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "isUsed": true,
+// CHECK-NEXT:      "name": "~S",
+// CHECK-NEXT:      "mangledName": "_ZN1SD1Ev",
+// CHECK-NEXT:      "type": {
+// CHECK-NEXT:       "qualType": "void () noexcept"
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "inner": [
+// CHECK-NEXT:       {
+// CHECK-NEXT:        "id": "0x{{.*}}",
+// CHECK-NEXT:        "kind": "CompoundStmt",
+// CHECK-NEXT:        "range": {
+// CHECK-NEXT:         "begin": {
+// CHECK-NEXT:          "offset": 232,
+// CHECK-NEXT:          "col": 8,
+// CHECK-NEXT:          "tokLen": 1
+// CHECK-NEXT:         },
+// CHECK-NEXT:         "end": {
+// CHECK-NEXT:          "offset": 233,
+// CHECK-NEXT:          "col": 9,
+// CHECK-NEXT:          "tokLen": 1
+// CHECK-NEXT:         }
+// CHECK-NEXT:        }
+// CHECK-NEXT:       }
+// CHECK-NEXT:      ]
+// CHECK-NEXT:     },
+// CHECK-NEXT:     {
+// CHECK-NEXT:      "id": "0x{{.*}}",
+// CHECK-NEXT:      "kind": "CXXConstructorDecl",
+// CHECK-NEXT:      "loc": {
+// CHECK-NEXT:       "offset": 110,
+// CHECK-NEXT:       "line": 3,
+// CHECK-NEXT:       "col": 8,
+// CHECK-NEXT:       "tokLen": 1
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "range": {
+// CHECK-NEXT:       "begin": {
+// CHECK-NEXT:        "offset": 110,
+// CHECK-NEXT:        "col": 8,
+// CHECK-NEXT:        "tokLen": 1
+// CHECK-NEXT:       },
+// CHECK-NEXT:       "end": {
+// CHECK-NEXT:        "offset": 110,
+// CHECK-NEXT:        "col": 8,
+// CHECK-NEXT:        "tokLen": 1
+// CHECK-NEXT:       }
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "isImplicit": true,
+// CHECK-NEXT:      "name": "S",
+// CHECK-NEXT:      "mangledName": "_ZN1SC1ERKS_",
+// CHECK-NEXT:      "type": {
+// CHECK-NEXT:       "qualType": "void (const S &)"
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "inline": true,
+// CHECK-NEXT:      "constexpr": true,
+// CHECK-NEXT:      "explicitlyDefaulted": "default",
+// CHECK-NEXT:      "inner": [
+// CHECK-NEXT:       {
+// CHECK-NEXT:        "id": "0x{{.*}}",
+// CHECK-NEXT:        "kind": "ParmVarDecl",
+// CHECK-NEXT:        "loc": {
+// CHECK-NEXT:         "offset": 110,
+// CHECK-NEXT:         "col": 8,
+// CHECK-NEXT:         "tokLen": 1
+// CHECK-NEXT:        },
+// CHECK-NEXT:        "range": {
+// CHECK-NEXT:         "begin": {
+// CHECK-NEXT:          "offset": 110,
+// CHECK-NEXT:          "col": 8,
+// CHECK-NEXT:          "tokLen": 1
+// CHECK-NEXT:         },
+// CHECK-NEXT:         "end": {
+// CHECK-NEXT:          "offset": 110,
+// CHECK-NEXT:          "col": 8,
+// CHECK-NEXT:          "tokLen": 1
+// CHECK-NEXT:         }
+// CHECK-NEXT:        },
+// CHECK-NEXT:        "type": {
+// CHECK-NEXT:         "qualType": "const S &"
+// CHECK-NEXT:        }
+// CHECK-NEXT:       }
+// CHECK-NEXT:      ]
+// CHECK-NEXT:     }
+// CHECK-NEXT:    ]
+// CHECK-NEXT:   },
+// CHECK-NEXT:   {
+// CHECK-NEXT:    "id": "0x{{.*}}",
+// CHECK-NEXT:    "kind": "FunctionDecl",
+// CHECK-NEXT:    "loc": {
+// CHECK-NEXT:     "offset": 241,
+// CHECK-NEXT:     "line": 11,
+// CHECK-NEXT:     "col": 3,
+// CHECK-NEXT:     "tokLen": 4
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "range": {
+// CHECK-NEXT:     "begin": {
+// CHECK-NEXT:      "offset": 239,
+// CHECK-NEXT:      "col": 1,
+// CHECK-NEXT:      "tokLen": 1
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "end": {
+// CHECK-NEXT:      "offset": 275,
+// CHECK-NEXT:      "col": 37,
+// CHECK-NEXT:      "tokLen": 1
+// CHECK-NEXT:     }
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "isUsed": true,
+// CHECK-NEXT:    "name": "func",
+// CHECK-NEXT:    "mangledName": "_Z4funcRKiRK1S",
+// CHECK-NEXT:    "type": {
+// CHECK-NEXT:     "qualType": "S (const int &, const S &)"
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "inner": [
+// CHECK-NEXT:     {
+// CHECK-NEXT:      "id": "0x{{.*}}",
+// CHECK-NEXT:      "kind": "ParmVarDecl",
+// CHECK-NEXT:      "loc": {
+// CHECK-NEXT:       "offset": 257,
+// CHECK-NEXT:       "col": 19,
+// CHECK-NEXT:       "tokLen": 1
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "range": {
+// CHECK-NEXT:       "begin": {
+// CHECK-NEXT:        "offset": 246,
+// CHECK-NEXT:        "col": 8,
+// CHECK-NEXT:        "tokLen": 5
+// CHECK-NEXT:       },
+// CHECK-NEXT:       "end": {
+// CHECK-NEXT:        "offset": 256,
+// CHECK-NEXT:        "col": 18,
+// CHECK-NEXT:        "tokLen": 1
+// CHECK-NEXT:       }
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "type": {
+// CHECK-NEXT:       "qualType": "const int &"
+// CHECK-NEXT:      }
+// CHECK-NEXT:     },
+// CHECK-NEXT:     {
+// CHECK-NEXT:      "id": "0x{{.*}}",
+// CHECK-NEXT:      "kind": "ParmVarDecl",
+// CHECK-NEXT:      "loc": {
+// CHECK-NEXT:       "offset": 268,
+// CHECK-NEXT:       "col": 30,
+// CHECK-NEXT:       "tokLen": 1
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "range": {
+// CHECK-NEXT:       "begin": {
+// CHECK-NEXT:        "offset": 259,
+// CHECK-NEXT:        "col": 21,
+// CHECK-NEXT:        "tokLen": 5
+// CHECK-NEXT:       },
+// CHECK-NEXT:       "end": {
+// CHECK-NEXT:        "offset": 274,
+// CHECK-NEXT:        "col": 36,
+// CHECK-NEXT:        "tokLen": 1
+// CHECK-NEXT:       }
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "name": "s",
+// CHECK-NEXT:      "type": {
+// CHECK-NEXT:       "qualType": "const S &"
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "init": "c",
+// CHECK-NEXT:      "inner": [
+// CHECK-NEXT:       {
+// CHECK-NEXT:        "id": "0x{{.*}}",
+// CHECK-NEXT:        "kind": "ExprWithCleanups",
+// CHECK-NEXT:        "range": {
+// CHECK-NEXT:         "begin": {
+// CHECK-NEXT:          "offset": 272,
+// CHECK-NEXT:          "col": 34,
+// CHECK-NEXT:          "tokLen": 1
+// CHECK-NEXT:         },
+// CHECK-NEXT:         "end": {
+// CHECK-NEXT:          "offset": 274,
+// CHECK-NEXT:          "col": 36,
+// CHECK-NEXT:          "tokLen": 1
+// CHECK-NEXT:         }
+// CHECK-NEXT:        },
+// CHECK-NEXT:        "type": {
+// CHECK-NEXT:         "qualType": "const S"
+// CHECK-NEXT:        },
+// CHECK-NEXT:        "valueCategory": "lvalue",
+// CHECK-NEXT:        "cleanupsHaveSideEffects": true,
+// CHECK-NEXT:        "inner": [
+// CHECK-NEXT:         {
+// CHECK-NEXT:          "id": "0x{{.*}}",
+// CHECK-NEXT:          "kind": "MaterializeTemporaryExpr",
+// CHECK-NEXT:          "range": {
+// CHECK-NEXT:           "begin": {
+// CHECK-NEXT:            "offset": 272,
+// CHECK-NEXT:            "col": 34,
+// CHECK-NEXT:            "tokLen": 1
+// CHECK-NEXT:           },
+// CHECK-NEXT:           "end": {
+// CHECK-NEXT:            "offset": 274,
+// CHECK-NEXT:            "col": 36,
+// CHECK-NEXT:            "tokLen": 1
+// CHECK-NEXT:           }
+// CHECK-NEXT:          },
+// CHECK-NEXT:          "type": {
+// CHECK-NEXT:           "qualType": "const S"
+// CHECK-NEXT:          },
+// CHECK-NEXT:          "valueCategory": "lvalue",
+// CHECK-NEXT:          "storageDuration": "full expression",
+// CHECK-NEXT:          "boundToLValueRef": true,
+// CHECK-NEXT:          "inner": [
+// CHECK-NEXT:           {
+// CHECK-NEXT:            "id": "0x{{.*}}",
+// CHECK-NEXT:            "kind": "ImplicitCastExpr",
+// CHECK-NEXT:            "range": {
+// CHECK-NEXT:             "begin": {
+// CHECK-NEXT:              "offset": 272,
+// CHECK-NEXT:              "col": 34,
+// CHECK-NEXT:              "tokLen": 1
+// CHECK-NEXT:             },
+// CHECK-NEXT:             "end": {
+// CHECK-NEXT:              "offset": 274,
+// CHECK-NEXT:              "col": 36,
+// CHECK-NEXT:              "tokLen": 1
+// CHECK-NEXT:             }
+// CHECK-NEXT:            },
+// CHECK-NEXT:            "type": {
+// CHECK-NEXT:             "qualType": "const S"
+// CHECK-NEXT:            },
+// CHECK-NEXT:            "valueCategory": "prvalue",
+// CHECK-NEXT:            "castKind": "NoOp",
+// CHECK-NEXT:            "inner": [
+// CHECK-NEXT:             {
+// CHECK-NEXT:              "id": "0x{{.*}}",
+// CHECK-NEXT:              "kind": "CXXBindTemporaryExpr",
+// CHECK-NEXT:              "range": {
+// CHECK-NEXT:               "begin": {
+// CHECK-NEXT:                "offset": 272,
+// CHECK-NEXT:                "col": 34,
+// CHECK-NEXT:                "tokLen": 1
+// CHECK-NEXT:               },
+// CHECK-NEXT:               "end": {
+// CHECK-NEXT:                "offset": 274,
+// CHECK-NEXT:                "col": 36,
+// CHECK-NEXT:                "tokLen": 1
+// CHECK-NEXT:               }
+// CHECK-NEXT:              },
+// CHECK-NEXT:              "type": {
+// CHECK-NEXT:               "qualType": "S"
+// CHECK-NEXT:              },
+// CHECK-NEXT:              "valueCategory": "prvalue",
+// CHECK-NEXT:              "temp": "0x{{.*}}",
+// CHECK-NEXT:              "dtor": {
+// CHECK-NEXT:               "id": "0x{{.*}}",
+// CHECK-NEXT:               "kind": "CXXDestructorDecl",
+// CHECK-NEXT:               "name": "~S",
+// CHECK-NEXT:               "type": {
+// CHECK-NEXT:                "qualType": "void () noexcept"
+// CHECK-NEXT:               }
+// CHECK-NEXT:              },
+// CHECK-NEXT:              "inner": [
+// CHECK-NEXT:               {
+// CHECK-NEXT:                "id": "0x{{.*}}",
+// CHECK-NEXT:                "kind": "CXXTemporaryObjectExpr",
+// CHECK-NEXT:                "range": {
+// CHECK-NEXT:                 "begin": {
+// CHECK-NEXT:                  "offset": 272,
+// CHECK-NEXT:                  "col": 34,
+// CHECK-NEXT:                  "tokLen": 1
+// CHECK-NEXT:                 },
+// CHECK-NEXT:                 "end": {
+// CHECK-NEXT:                  "offset": 274,
+// CHECK-NEXT:                  "col": 36,
+// CHECK-NEXT:                  "tokLen": 1
+// CHECK-NEXT:                 }
+// CHECK-NEXT:                },
+// CHECK-NEXT:                "type": {
+// CHECK-NEXT:                 "qualType": "S"
+// CHECK-NEXT:                },
+// CHECK-NEXT:                "valueCategory": "prvalue",
+// CHECK-NEXT:                "ctorType": {
+// CHECK-NEXT:                 "qualType": "void ()"
+// CHECK-NEXT:                },
+// CHECK-NEXT:                "hadMultipleCandidates": true,
+// CHECK-NEXT:                "constructionKind": "complete"
+// CHECK-NEXT:               }
+// CHECK-NEXT:              ]
+// CHECK-NEXT:             }
+// CHECK-NEXT:            ]
+// CHECK-NEXT:           }
+// CHECK-NEXT:          ]
+// CHECK-NEXT:         }
+// CHECK-NEXT:        ]
+// CHECK-NEXT:       }
+// CHECK-NEXT:      ]
+// CHECK-NEXT:     }
+// CHECK-NEXT:    ]
+// CHECK-NEXT:   },
+// CHECK-NEXT:   {
+// CHECK-NEXT:    "id": "0x{{.*}}",
+// CHECK-NEXT:    "kind": "FunctionDecl",
+// CHECK-NEXT:    "loc": {
+// CHECK-NEXT:     "offset": 284,
+// CHECK-NEXT:     "line": 13,
+// CHECK-NEXT:     "col": 6,
+// CHECK-NEXT:     "tokLen": 4
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "range": {
+// CHECK-NEXT:     "begin": {
+// CHECK-NEXT:      "offset": 279,
+// CHECK-NEXT:      "col": 1,
+// CHECK-NEXT:      "tokLen": 4
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "end": {
+// CHECK-NEXT:      "offset": 321,
+// CHECK-NEXT:      "line": 15,
+// CHECK-NEXT:      "col": 1,
+// CHECK-NEXT:      "tokLen": 1
+// CHECK-NEXT:     }
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "name": "test",
+// CHECK-NEXT:    "mangledName": "_Z4testv",
+// CHECK-NEXT:    "type": {
+// CHECK-NEXT:     "qualType": "void ()"
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "inner": [
+// CHECK-NEXT:     {
+// CHECK-NEXT:      "id": "0x{{.*}}",
+// CHECK-NEXT:      "kind": "CompoundStmt",
+// CHECK-NEXT:      "range": {
+// CHECK-NEXT:       "begin": {
+// CHECK-NEXT:        "offset": 291,
+// CHECK-NEXT:        "line": 13,
+// CHECK-NEXT:        "col": 13,
+// CHECK-NEXT:        "tokLen": 1
+// CHECK-NEXT:       },
+// CHECK-NEXT:       "end": {
+// CHECK-NEXT:        "offset": 321,
+// CHECK-NEXT:        "line": 15,
+// CHECK-NEXT:        "col": 1,
+// CHECK-NEXT:        "tokLen": 1
+// CHECK-NEXT:       }
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "inner": [
+// CHECK-NEXT:       {
+// CHECK-NEXT:        "id": "0x{{.*}}",
+// CHECK-NEXT:        "kind": "CXXForRangeStmt",
+// CHECK-NEXT:        "range": {
+// CHECK-NEXT:         "begin": {
+// CHECK-NEXT:          "offset": 295,
+// CHECK-NEXT:          "line": 14,
+// CHECK-NEXT:          "col": 3,
+// CHECK-NEXT:          "tokLen": 3
+// CHECK-NEXT:         },
+// CHECK-NEXT:         "end": {
+// CHECK-NEXT:          "offset": 319,
+// CHECK-NEXT:          "col": 27,
+// CHECK-NEXT:          "tokLen": 1
+// CHECK-NEXT:         }
+// CHECK-NEXT:        },
+// CHECK-NEXT:        "inner": [
+// CHECK-NEXT:         {},
+// CHECK-NEXT:         {
+// CHECK-NEXT:          "id": "0x{{.*}}",
+// CHECK-NEXT:          "kind": "DeclStmt",
+// CHECK-NEXT:          "range": {
+// CHECK-NEXT:           "begin": {
+// CHECK-NEXT:            "offset": 309,
+// CHECK-NEXT:            "col": 17,
+// CHECK-NEXT:            "tokLen": 4
+// CHECK-NEXT:           },
+// CHECK-NEXT:           "end": {
+// CHECK-NEXT:            "offset": 309,
+// CHECK-NEXT:            "col": 17,
+// CHECK-NEXT:            "tokLen": 4
+// CHECK-NEXT:           }
+// CHECK-NEXT:          },
+// CHECK-NEXT:          "inner": [
+// CHECK-NEXT:           {
+// CHECK-NEXT:            "id": "0x{{.*}}",
+// CHECK-NEXT:            "kind": "VarDecl",
+// CHECK-NEXT:            "loc": {
+// CHECK-NEXT:             "offset": 309,
+// CHECK-NEXT:             "col": 17,
+// CHECK-NEXT:             "tokLen": 4
+// CHECK-NEXT:            },
+// CHECK-NEXT:            "range": {
+// CHECK-NEXT:             "begin": {
+// CHECK-NEXT:              "offset": 309,
+// CHECK-NEXT:              "col": 17,
+// CHECK-NEXT:              "tokLen": 4
+// CHECK-NEXT:             },
+// CHECK-NEXT:             "end": {
+// CHECK-NEXT:              "offset": 315,
+// CHECK-NEXT:              "col": 23,
+// CHECK-NEXT:              "tokLen": 1
+// CHECK-NEXT:             }
+// CHECK-NEXT:            },
+// CHECK-NEXT:            "isImplicit": true,
+// CHECK-NEXT:            "isUsed": true,
+// CHECK-NEXT:            "name": "__range1",
+// CHECK-NEXT:            "type": {
+// CHECK-NEXT:             "qualType": "S &&"
+// CHECK-NEXT:            },
+// CHECK-NEXT:            "init": "c",
+// CHECK-NEXT:            "inner": [
+// CHECK-NEXT:             {
+// CHECK-NEXT:              "id": "0x{{.*}}",
+// CHECK-NEXT:              "kind": "ExprWithCleanups",
+// CHECK-NEXT:              "range": {
+// CHECK-NEXT:               "begin": {
+// CHECK-NEXT:                "offset": 309,
+// CHECK-NEXT:                "col": 17,
+// CHECK-NEXT:                "tokLen": 4
+// CHECK-NEXT:               },
+// CHECK-NEXT:               "end": {
+// CHECK-NEXT:                "offset": 315,
+// CHECK-NEXT:                "col": 23,
+// CHECK-NEXT:                "tokLen": 1
+// CHECK-NEXT:               }
+// CHECK-NEXT:              },
+// CHECK-NEXT:              "type": {
+// CHECK-NEXT:               "qualType": "S"
+// CHECK-NEXT:              },
+// CHECK-NEXT:              "valueCategory": "xvalue",
+// CHECK-NEXT:              "cleanupsHaveSideEffects": true,
+// CHECK-NEXT:              "inner": [
+// CHECK-NEXT:               {
+// CHECK-NEXT:                "id": "0x{{.*}}",
+// CHECK-NEXT:                "kind": "MaterializeTemporaryExpr",
+// CHECK-NEXT:                "range": {
+// CHECK-NEXT:                 "begin": {
+// CHECK-NEXT:                  "offset": 309,
+// CHECK-NEXT:                  "col": 17,
+// CHECK-NEXT:                  "tokLen": 4
+// CHECK-NEXT:                 },
+// CHECK-NEXT:                 "end": {
+// CHECK-NEXT:                  "offset": 315,
+// CHECK-NEXT:                  "col": 23,
+// CHECK-NEXT:                  "tokLen": 1
+// CHECK-NEXT:                 }
+// CHECK-NEXT:                },
+// CHECK-NEXT:                "type": {
+// CHECK-NEXT:                 "qualType": "S"
+// CHECK-NEXT:                },
+// CHECK-NEXT:                "valueCategory": "xvalue",
+// CHECK-NEXT:                "extendingDecl": {
+// CHECK-NEXT:                 "id": "0x{{.*}}",
+// CHECK-NEXT:                 "kind": "VarDecl",
+// CHECK-NEXT:                 "name": "__range1",
+// CHECK-NEXT:                 "type": {
+// CHECK-NEXT:                  "qualType": "S &&"
+// CHECK-NEXT:                 }
+// CHECK-NEXT:                },
+// CHECK-NEXT:                "storageDuration": "automatic",
+// CHECK-NEXT:                "inner": [
+// CHECK-NEXT:                 {
+// CHECK-NEXT:                  "id": "0x{{.*}}",
+// CHECK-NEXT:                  "kind": "CXXBindTemporaryExpr",
+// CHECK-NEXT:                  "range": {
+// CHECK-NEXT:                   "begin": {
+// CHECK-NEXT:                    "offset": 309,
+// CHECK-NEXT:                    "col": 17,
+// CHECK-NEXT:                    "tokLen": 4
+// CHECK-NEXT:                   },
+// CHECK-NEXT:                   "end": {
+// CHECK-NEXT:                    "offset": 315,
+// CHECK-NEXT:                    "col": 23,
+// CHECK-NEXT:                    "tokLen": 1
+// CHECK-NEXT:                   }
+// CHECK-NEXT:                  },
+// CHECK-NEXT:                  "type": {
+// CHECK-NEXT:                   "qualType": "S"
+// CHECK-NEXT:                  },
+// CHECK-NEXT:                  "valueCategory": "prvalue",
+// CHECK-NEXT:                  "temp": "0x{{.*}}",
+// CHECK-NEXT:                  "dtor": {
+// CHECK-NEXT:                   "id": "0x{{.*}}",
+// CHECK-NEXT:                   "kind": "CXXDestructorDecl",
+// CHECK-NEXT:                   "name": "~S",
+// CHECK-NEXT:                   "type": {
+// CHECK-NEXT:                    "qualType": "void () noexcept"
+// CHECK-NEXT:                   }
+// CHECK-NEXT:                  },
+// CHECK-NEXT:                  "inner": [
+// CHECK-NEXT:                   {
+// CHECK-NEXT:                    "id": "0x{{.*}}",
+// CHECK-NEXT:                    "kind": "CallExpr",
+// CHECK-NEXT:                    "range": {
+// CHECK-NEXT:                     "begin": {
+// CHECK-NEXT:                      "offset": 309,
+// CHECK-NEXT:                      "col": 17,
+// CHECK-NEXT:                      "tokLen": 4
+// CHECK-NEXT:                     },
+// CHECK-NEXT:                     "end": {
+// CHECK-NEXT:                      "offset": 315,
+// CHECK-NEXT:                      "col": 23,
+// CHECK-NEXT:                      "tokLen": 1
+// CHECK-NEXT:                     }
+// CHECK-NEXT:                    },
+// CHECK-NEXT:                    "type": {
+// CHECK-NEXT:                     "qualType": "S"
+// CHECK-NEXT:                    },
+// CHECK-NEXT:                    "valueCategory": "prvalue",
+// CHECK-NEXT:                    "inner": [
+// CHECK-NEXT:                     {
+// CHECK-NEXT:                      "id": "0x{{.*}}",
+// CHECK-NEXT:                      "kind": "ImplicitCastExpr",
+// CHECK-NEXT:                      "range": {
+// CHECK-NEXT:                       "begin": {
+// CHECK-NEXT:                        "offset": 309,
+// CHECK-NEXT:                        "col": 17,
+// CHECK-NEXT:                        "tokLen": 4
+// CHECK-NEXT:                       },
+// CHECK-NEXT:                       "end": {
+// CHECK-NEXT:                        "offset": 309,
+// CHECK-NEXT:                        "col": 17,
+// CHECK-NEXT:                        "tokLen": 4
+// CHECK-NEXT:                       }
+// CHECK-NEXT:                      },
+// CHECK-NEXT:                      "type": {
+// CHECK-NEXT:                       "qualType": "S (*)(const int &, const S &)"
+// CHECK-NEXT:                      },
+// CHECK-NEXT:                      "valueCategory": "prvalue",
+// CHECK-NEXT:                      "castKind": "FunctionToPointerDecay",
+// CHECK-NEXT:                      "inner": [
+// CHECK-NEXT:                       {
+// CHECK-NEXT:                        "id": "0x{{.*}}",
+// CHECK-NEXT:                        "kind": "DeclRefExpr",
+// CHECK-NEXT:                        "range": {
+// CHECK-NEXT:                         "begin": {
+// CHECK-NEXT:                          "offset": 309,
+// CHECK-NEXT:                          "col": 17,
+// CHECK-NEXT:                          "tokLen": 4
+// CHECK-NEXT:                         },
+// CHECK-NEXT:                         "end": {
+// CHECK-NEXT:                          "offset": 309,
+// CHECK-NEXT:                          "col": 17,
+// CHECK-NEXT:                          "tokLen": 4
+// CHECK-NEXT:                         }
+// CHECK-NEXT:                        },
+// CHECK-NEXT:                        "type": {
+// CHECK-NEXT:                         "qualType": "S (const int &, const S &)"
+// CHECK-NEXT:                        },
+// CHECK-NEXT:                        "valueCategory": "lvalue",
+// CHECK-NEXT:                        "referencedDecl": {
+// CHECK-NEXT:                         "id": "0x{{.*}}",
+// CHECK-NEXT:                         "kind": "FunctionDecl",
+// CHECK-NEXT:                         "name": "func",
+// CHECK-NEXT:                         "type": {
+// CHECK-NEXT:                          "qualType": "S (const int &, const S &)"
+// CHECK-NEXT:                         }
+// CHECK-NEXT:                        }
+// CHECK-NEXT:                       }
+// CHECK-NEXT:                      ]
+// CHECK-NEXT:                     },
+// CHECK-NEXT:                     {
+// CHECK-NEXT:                      "id": "0x{{.*}}",
+// CHECK-NEXT:                      "kind": "MaterializeTemporaryExpr",
+// CHECK-NEXT:                      "range": {
+// CHECK-NEXT:                       "begin": {
+// CHECK-NEXT:                        "offset": 314,
+// CHECK-NEXT:                        "col": 22,
+// CHECK-NEXT:                        "tokLen": 1
+// CHECK-NEXT:                       },
+// CHECK-NEXT:                       "end": {
+// CHECK-NEXT:                        "offset": 314,
+// CHECK-NEXT:                        "col": 22,
+// CHECK-NEXT:                        "tokLen": 1
+// CHECK-NEXT:                       }
+// CHECK-NEXT:                      },
+// CHECK-NEXT:                      "type": {
+// CHECK-NEXT:                       "qualType": "const int"
+// CHECK-NEXT:                      },
+// CHECK-NEXT:                      "valueCategory": "lvalue",
+// CHECK-NEXT:                      "extendingDecl": {
+// CHECK-NEXT:                       "id": "0x{{.*}}",
+// CHECK-NEXT:                       "kind": "VarDecl",
+// CHECK-NEXT:                       "name": "__range1",
+// CHECK-NEXT:                       "type": {
+// CHECK-NEXT:                        "qualType": "S &&"
+// CHECK-NEXT:                       }
+// CHECK-NEXT:                      },
+// CHECK-NEXT:                      "storageDuration": "automatic",
+// CHECK-NEXT:                      "boundToLValueRef": true,
+// CHECK-NEXT:                      "inner": [
+// CHECK-NEXT:                       {
+// CHECK-NEXT:                        "id": "0x{{.*}}",
+// CHECK-NEXT:                        "kind": "ImplicitCastExpr",
+// CHECK-NEXT:                        "range": {
+// CHECK-NEXT:                         "begin": {
+// CHECK-NEXT:                          "offset": 314,
+// CHECK-NEXT:                          "col": 22,
+// CHECK-NEXT:                          "tokLen": 1
+// CHECK-NEXT:                         },
+// CHECK-NEXT:                         "end": {
+// CHECK-NEXT:                          "offset": 314,
+// CHECK-NEXT:                          "col": 22,
+// CHECK-NEXT:                          "tokLen": 1
+// CHECK-NEXT:                         }
+// CHECK-NEXT:                        },
+// CHECK-NEXT:                        "type": {
+// CHECK-NEXT:                         "qualType": "const int"
+// CHECK-NEXT:                        },
+// CHECK-NEXT:                        "valueCategory": "prvalue",
+// CHECK-NEXT:                        "castKind": "NoOp",
+// CHECK-NEXT:                        "inner": [
+// CHECK-NEXT:                         {
+// CHECK-NEXT:                          "id": "0x{{.*}}",
+// CHECK-NEXT:                          "kind": "IntegerLiteral",
+// CHECK-NEXT:                          "range": {
+// CHECK-NEXT:                           "begin": {
+// CHECK-NEXT:                            "offset": 314,
+// CHECK-NEXT:                            "col": 22,
+// CHECK-NEXT:                            "tokLen": 1
+// CHECK-NEXT:                           },
+// CHECK-NEXT:                           "end": {
+// CHECK-NEXT:                            "offset": 314,
+// CHECK-NEXT:                            "col": 22,
+// CHECK-NEXT:                            "tokLen": 1
+// CHECK-NEXT:                           }
+// CHECK-NEXT:                          },
+// CHECK-NEXT:                          "type": {
+// CHECK-NEXT:                           "qualType": "int"
+// CHECK-NEXT:                          },
+// CHECK-NEXT:                          "valueCategory": "prvalue",
+// CHECK-NEXT:                          "value": "1"
+// CHECK-NEXT:                         }
+// CHECK-NEXT:                        ]
+// CHECK-NEXT:                       }
+// CHECK-NEXT:                      ]
+// CHECK-NEXT:                     },
+// CHECK-NEXT:                     {
+// CHECK-NEXT:                      "id": "0x{{.*}}",
+// CHECK-NEXT:                      "kind": "CXXDefaultArgExpr",
+// CHECK-NEXT:                      "range": {
+// CHECK-NEXT:                       "begin": {},
+// CHECK-NEXT:                       "end": {}
+// CHECK-NEXT:                      },
+// CHECK-NEXT:                      "type": {
+// CHECK-NEXT:                       "qualType": "const S"
+// CHECK-NEXT:                      },
+// CHECK-NEXT:                      "valueCategory": "lvalue",
+// CHECK-NEXT:                      "hasRewrittenInit": true,
+// CHECK-NEXT:                      "inner": [
+// CHECK-NEXT:                       {
+// CHECK-NEXT:                        "id": "0x{{.*}}",
+// CHECK-NEXT:                        "kind": "MaterializeTemporaryExpr",
+// CHECK-NEXT:                        "range": {
+// CHECK-NEXT:                         "begin": {
+// CHECK-NEXT:                          "offset": 272,
+// CHECK-NEXT:                          "line": 11,
+// CHECK-NEXT:                          "col": 34,
+// CHECK-NEXT:                          "tokLen": 1
+// CHECK-NEXT:                         },
+// CHECK-NEXT:                         "end": {
+// CHECK-NEXT:                          "offset": 274,
+// CHECK-NEXT:                          "col": 36,
+// CHECK-NEXT:                          "tokLen": 1
+// CHECK-NEXT:                         }
+// CHECK-NEXT:                        },
+// CHECK-NEXT:                        "type": {
+// CHECK-NEXT:                         "qualType": "const S"
+// CHECK-NEXT:                        },
+// CHECK-NEXT:                        "valueCategory": "lvalue",
+// CHECK-NEXT:                        "extendingDecl": {
+// CHECK-NEXT:                         "id": "0x{{.*}}",
+// CHECK-NEXT:                         "kind": "VarDecl",
+// CHECK-NEXT:                         "name": "__range1",
+// CHECK-NEXT:                         "type": {
+// CHECK-NEXT:                          "qualType": "S &&"
+// CHECK-NEXT:                         }
+// CHECK-NEXT:                        },
+// CHECK-NEXT:                        "storageDuration": "automatic",
+// CHECK-NEXT:                        "boundToLValueRef": true,
+// CHECK-NEXT:                        "inner": [
+// CHECK-NEXT:                         {
+// CHECK-NEXT:                          "id": "0x{{.*}}",
+// CHECK-NEXT:                          "kind": "ImplicitCastExpr",
+// CHECK-NEXT:                          "range": {
+// CHECK-NEXT:                           "begin": {
+// CHECK-NEXT:                            "offset": 272,
+// CHECK-NEXT:                            "col": 34,
+// CHECK-NEXT:                            "tokLen": 1
+// CHECK-NEXT:                           },
+// CHECK-NEXT:                           "end": {
+// CHECK-NEXT:                            "offset": 274,
+// CHECK-NEXT:                            "col": 36,
+// CHECK-NEXT:                            "tokLen": 1
+// CHECK-NEXT:                           }
+// CHECK-NEXT:                          },
+// CHECK-NEXT:                          "type": {
+// CHECK-NEXT:                           "qualType": "const S"
+// CHECK-NEXT:                          },
+// CHECK-NEXT:                          "valueCategory": "prvalue",
+// CHECK-NEXT:                          "castKind": "NoOp",
+// CHECK-NEXT:                          "inner": [
+// CHECK-NEXT:                           {
+// CHECK-NEXT:                            "id": "0x{{.*}}",
+// CHECK-NEXT:                            "kind": "CXXBindTemporaryExpr",
+// CHECK-NEXT:                            "range": {
+// CHECK-NEXT:                             "begin": {
+// CHECK-NEXT:                              "offset": 272,
+// CHECK-NEXT:                              "col": 34,
+// CHECK-NEXT:                              "tokLen": 1
+// CHECK-NEXT:                             },
+// CHECK-NEXT:                             "end": {
+// CHECK-NEXT:                              "offset": 274,
+// CHECK-NEXT:                              "col": 36,
+// CHECK-NEXT:                              "tokLen": 1
+// CHECK-NEXT:                             }
+// CHECK-NEXT:                            },
+// CHECK-NEXT:                            "type": {
+// CHECK-NEXT:                             "qualType": "S"
+// CHECK-NEXT:                            },
+// CHECK-NEXT:                            "valueCategory": "prvalue",
+// CHECK-NEXT:                            "temp": "0x{{.*}}",
+// CHECK-NEXT:                            "dtor": {
+// CHECK-NEXT:                             "id": "0x{{.*}}",
+// CHECK-NEXT:                             "kind": "CXXDestructorDecl",
+// CHECK-NEXT:                             "name": "~S",
+// CHECK-NEXT:                             "type": {
+// CHECK-NEXT:                              "qualType": "void () noexcept"
+// CHECK-NEXT:                             }
+// CHECK-NEXT:                            },
+// CHECK-NEXT:                            "inner": [
+// CHECK-NEXT:                             {
+// CHECK-NEXT:                              "id": "0x{{.*}}",
+// CHECK-NEXT:                              "kind": "CXXTemporaryObjectExpr",
+// CHECK-NEXT:                              "range": {
+// CHECK-NEXT:                               "begin": {
+// CHECK-NEXT:                                "offset": 272,
+// CHECK-NEXT:                                "col": 34,
+// CHECK-NEXT:                                "tokLen": 1
+// CHECK-NEXT:                               },
+// CHECK-NEXT:                               "end": {
+// CHECK-NEXT:                                "offset": 274,
+// CHECK-NEXT:                                "col": 36,
+// CHECK-NEXT:                                "tokLen": 1
+// CHECK-NEXT:                               }
+// CHECK-NEXT:                              },
+// CHECK-NEXT:                              "type": {
+// CHECK-NEXT:                               "qualType": "S"
+// CHECK-NEXT:                              },
+// CHECK-NEXT:                              "valueCategory": "prvalue",
+// CHECK-NEXT:                              "ctorType": {
+// CHECK-NEXT:                               "qualType": "void ()"
+// CHECK-NEXT:                              },
+// CHECK-NEXT:                              "hadMultipleCandidates": true,
+// CHECK-NEXT:                              "constructionKind": "complete"
+// CHECK-NEXT:                             }
+// CHECK-NEXT:                            ]
+// CHECK-NEXT:                           }
+// CHECK-NEXT:                          ]
+// CHECK-NEXT:                         }
+// CHECK-NEXT:                        ]
+// CHECK-NEXT:                       }
+// CHECK-NEXT:                      ]
+// CHECK-NEXT:                     }
+// CHECK-NEXT:                    ]
+// CHECK-NEXT:                   }
+// CHECK-NEXT:                  ]
+// CHECK-NEXT:                 }
+// CHECK-NEXT:                ]
+// CHECK-NEXT:               }
+// CHECK-NEXT:              ]
+// CHECK-NEXT:             }
+// CHECK-NEXT:            ]
+// CHECK-NEXT:           }
+// CHECK-NEXT:          ]
+// CHECK-NEXT:         },
+// CHECK-NEXT:         {
+// CHECK-NEXT:          "id": "0x{{.*}}",
+// CHECK-NEXT:          "kind": "DeclStmt",
+// CHECK-NEXT:          "range": {
+// CHECK-NEXT:           "begin": {
+// CHECK-NEXT:            "offset": 307,
+// CHECK-NEXT:            "line": 14,
+// CHECK-NEXT:            "col": 15,
+// CHECK-NEXT:            "tokLen": 1
+// CHECK-NEXT:           },
+// CHECK-NEXT:           "end": {
+// CHECK-NEXT:            "offset": 307,
+// CHECK-NEXT:            "col": 15,
+// CHECK-NEXT:            "tokLen": 1
+// CHECK-NEXT:           }
+// CHECK-NEXT:          },
+// CHECK-NEXT:          "inner": [
+// CHECK-NEXT:           {
+// CHECK-NEXT:            "id": "0x{{.*}}",
+// CHECK-NEXT:            "kind": "VarDecl",
+// CHECK-NEXT:            "loc": {
+// CHECK-NEXT:             "offset": 307,
+// CHECK-NEXT:             "col": 15,
+// CHECK-NEXT:             "tokLen": 1
+// CHECK-NEXT:            },
+// CHECK-NEXT:            "range": {
+// CHECK-NEXT:             "begin": {
+// CHECK-NEXT:              "offset": 307,
+// CHECK-NEXT:              "col": 15,
+// CHECK-NEXT:              "tokLen": 1
+// CHECK-NEXT:             },
+// CHECK-NEXT:             "end": {
+// CHECK-NEXT:              "offset": 307,
+// CHECK-NEXT:              "col": 15,
+// CHECK-NEXT:              "tokLen": 1
+// CHECK-NEXT:             }
+// CHECK-NEXT:            },
+// CHECK-NEXT:            "isImplicit": true,
+// CHECK-NEXT:            "isUsed": true,
+// CHECK-NEXT:            "name": "__begin1",
+// CHECK-NEXT:            "type": {
+// CHECK-NEXT:             "qualType": "const int *"
+// CHECK-NEXT:            },
+// CHECK-NEXT:            "init": "c",
+// CHECK-NEXT:            "inner": [
+// CHECK-NEXT:             {
+// CHECK-NEXT:              "id": "0x{{.*}}",
+// CHECK-NEXT:              "kind": "CXXMemberCallExpr",
+// CHECK-NEXT:              "range": {
+// CHECK-NEXT:               "begin": {
+// CHECK-NEXT:                "offset": 307,
+// CHECK-NEXT:                "col": 15,
+// CHECK-NEXT:                "tokLen": 1
+// CHECK-NEXT:               },
+// CHECK-NEXT:               "end": {
+// CHECK-NEXT:                "offset": 307,
+// CHECK-NEXT:                "col": 15,
+// CHECK-NEXT:                "tokLen": 1
+// CHECK-NEXT:               }
+// CHECK-NEXT:              },
+// CHECK-NEXT:              "type": {
+// CHECK-NEXT:               "qualType": "const int *"
+// CHECK-NEXT:              },
+// CHECK-NEXT:              "valueCategory": "prvalue",
+// CHECK-NEXT:              "inner": [
+// CHECK-NEXT:               {
+// CHECK-NEXT:                "id": "0x{{.*}}",
+// CHECK-NEXT:                "kind": "MemberExpr",
+// CHECK-NEXT:                "range": {
+// CHECK-NEXT:                 "begin": {
+// CHECK-NEXT:                  "offset": 307,
+// CHECK-NEXT:                  "col": 15,
+// CHECK-NEXT:                  "tokLen": 1
+// CHECK-NEXT:                 },
+// CHECK-NEXT:                 "end": {
+// CHECK-NEXT:                  "offset": 307,
+// CHECK-NEXT:                  "col": 15,
+// CHECK-NEXT:                  "tokLen": 1
+// CHECK-NEXT:                 }
+// CHECK-NEXT:                },
+// CHECK-NEXT:                "type": {
+// CHECK-NEXT:                 "qualType": "<bound member function type>"
+// CHECK-NEXT:                },
+// CHECK-NEXT:                "valueCategory": "prvalue",
+// CHECK-NEXT:                "name": "begin",
+// CHECK-NEXT:                "isArrow": false,
+// CHECK-NEXT:                "referencedMemberDecl": "0x{{.*}}",
+// CHECK-NEXT:                "inner": [
+// CHECK-NEXT:                 {
+// CHECK-NEXT:                  "id": "0x{{.*}}",
+// CHECK-NEXT:                  "kind": "ImplicitCastExpr",
+// CHECK-NEXT:                  "range": {
+// CHECK-NEXT:                   "begin": {
+// CHECK-NEXT:                    "offset": 307,
+// CHECK-NEXT:                    "col": 15,
+// CHECK-NEXT:                    "tokLen": 1
+// CHECK-NEXT:                   },
+// CHECK-NEXT:                   "end": {
+// CHECK-NEXT:                    "offset": 307,
+// CHECK-NEXT:                    "col": 15,
+// CHECK-NEXT:                    "tokLen": 1
+// CHECK-NEXT:                   }
+// CHECK-NEXT:                  },
+// CHECK-NEXT:                  "type": {
+// CHECK-NEXT:                   "qualType": "const S"
+// CHECK-NEXT:                  },
+// CHECK-NEXT:                  "valueCategory": "lvalue",
+// CHECK-NEXT:                  "castKind": "NoOp",
+// CHECK-NEXT:                  "inner": [
+// CHECK-NEXT:                   {
+// CHECK-NEXT:                    "id": "0x{{.*}}",
+// CHECK-NEXT:                    "kind": "DeclRefExpr",
+// CHECK-NEXT:                    "range": {
+// CHECK-NEXT:                     "begin": {
+// CHECK-NEXT:                      "offset": 307,
+// CHECK-NEXT:                      "col": 15,
+// CHECK-NEXT:                      "tokLen": 1
+// CHECK-NEXT:                     },
+// CHECK-NEXT:                     "end": {
+// CHECK-NEXT:                      "offset": 307,
+// CHECK-NEXT:                      "col": 15,
+// CHECK-NEXT:                      "tokLen": 1
+// CHECK-NEXT:                     }
+// CHECK-NEXT:                    },
+// CHECK-NEXT:                    "type": {
+// CHECK-NEXT:                     "qualType": "S"
+// CHECK-NEXT:                    },
+// CHECK-NEXT:                    "valueCategory": "lvalue",
+// CHECK-NEXT:                    "referencedDecl": {
+// CHECK-NEXT:                     "id": "0x{{.*}}",
+// CHECK-NEXT:                     "kind": "VarDecl",
+// CHECK-NEXT:                     "name": "__range1",
+// CHECK-NEXT:                     "type": {
+// CHECK-NEXT:                      "qualType": "S &&"
+// CHECK-NEXT:                     }
+// CHECK-NEXT:                    }
+// CHECK-NEXT:                   }
+// CHECK-NEXT:                  ]
+// CHECK-NEXT:                 }
+// CHECK-NEXT:                ]
+// CHECK-NEXT:               }
+// CHECK-NEXT:              ]
+// CHECK-NEXT:             }
+// CHECK-NEXT:            ]
+// CHECK-NEXT:           }
+// CHECK-NEXT:          ]
+// CHECK-NEXT:         },
+// CHECK-NEXT:         {
+// CHECK-NEXT:          "id": "0x{{.*}}",
+// CHECK-NEXT:          "kind": "DeclStmt",
+// CHECK-NEXT:          "range": {
+// CHECK-NEXT:           "begin": {
+// CHECK-NEXT:            "offset": 307,
+// CHECK-NEXT:            "col": 15,
+// CHECK-NEXT:            "tokLen": 1
+// CHECK-NEXT:           },
+// CHECK-NEXT:           "end": {
+// CHECK-NEXT:            "offset": 307,
+// CHECK-NEXT:            "col": 15,
+// CHECK-NEXT:            "tokLen": 1
+// CHECK-NEXT:           }
+// CHECK-NEXT:          },
+// CHECK-NEXT:          "inner": [
+// CHECK-NEXT:           {
+// CHECK-NEXT:            "id": "0x{{.*}}",
+// CHECK-NEXT:            "kind": "VarDecl",
+// CHECK-NEXT:            "loc": {
+// CHECK-NEXT:             "offset": 307,
+// CHECK-NEXT:             "col": 15,
+// CHECK-NEXT:             "tokLen": 1
+// CHECK-NEXT:            },
+// CHECK-NEXT:            "range": {
+// CHECK-NEXT:             "begin": {
+// CHECK-NEXT:              "offset": 307,
+// CHECK-NEXT:              "col": 15,
+// CHECK-NEXT:              "tokLen": 1
+// CHECK-NEXT:             },
+// CHECK-NEXT:             "end": {
+// CHECK-NEXT:              "offset": 307,
+// CHECK-NEXT:              "col": 15,
+// CHECK-NEXT:              "tokLen": 1
+// CHECK-NEXT:             }
+// CHECK-NEXT:            },
+// CHECK-NEXT:            "isImplicit": true,
+// CHECK-NEXT:            "isUsed": true,
+// CHECK-NEXT:            "name": "__end1",
+// CHECK-NEXT:            "type": {
+// CHECK-NEXT:             "qualType": "const int *"
+// CHECK-NEXT:            },
+// CHECK-NEXT:            "init": "c",
+// CHECK-NEXT:            "inner": [
+// CHECK-NEXT:             {
+// CHECK-NEXT:              "id": "0x{{.*}}",
+// CHECK-NEXT:              "kind": "CXXMemberCallExpr",
+// CHECK-NEXT:              "range": {
+// CHECK-NEXT:               "begin": {
+// CHECK-NEXT:                "offset": 307,
+// CHECK-NEXT:                "col": 15,
+// CHECK-NEXT:                "tokLen": 1
+// CHECK-NEXT:               },
+// CHECK-NEXT:               "end": {
+// CHECK-NEXT:                "offset": 307,
+// CHECK-NEXT:                "col": 15,
+// CHECK-NEXT:                "tokLen": 1
+// CHECK-NEXT:               }
+// CHECK-NEXT:              },
+// CHECK-NEXT:              "type": {
+// CHECK-NEXT:               "qualType": "const int *"
+// CHECK-NEXT:              },
+// CHECK-NEXT:              "valueCategory": "prvalue",
+// CHECK-NEXT:              "inner": [
+// CHECK-NEXT:               {
+// CHECK-NEXT:                "id": "0x{{.*}}",
+// CHECK-NEXT:                "kind": "MemberExpr",
+// CHECK-NEXT:                "range": {
+// CHECK-NEXT:                 "begin": {
+// CHECK-NEXT:                  "offset": 307,
+// CHECK-NEXT:                  "col": 15,
+// CHECK-NEXT:                  "tokLen": 1
+// CHECK-NEXT:                 },
+// CHECK-NEXT:                 "end": {
+// CHECK-NEXT:                  "offset": 307,
+// CHECK-NEXT:                  "col": 15,
+// CHECK-NEXT:                  "tokLen": 1
+// CHECK-NEXT:                 }
+// CHECK-NEXT:                },
+// CHECK-NEXT:                "type": {
+// CHECK-NEXT:                 "qualType": "<bound member function type>"
+// CHECK-NEXT:                },
+// CHECK-NEXT:                "valueCategory": "prvalue",
+// CHECK-NEXT:                "name": "end",
+// CHECK-NEXT:                "isArrow": false,
+// CHECK-NEXT:                "referencedMemberDecl": "0x{{.*}}",
+// CHECK-NEXT:                "inner": [
+// CHECK-NEXT:                 {
+// CHECK-NEXT:                  "id": "0x{{.*}}",
+// CHECK-NEXT:                  "kind": "ImplicitCastExpr",
+// CHECK-NEXT:                  "range": {
+// CHECK-NEXT:                   "begin": {
+// CHECK-NEXT:                    "offset": 307,
+// CHECK-NEXT:                    "col": 15,
+// CHECK-NEXT:                    "tokLen": 1
+// CHECK-NEXT:                   },
+// CHECK-NEXT:                   "end": {
+// CHECK-NEXT:                    "offset": 307,
+// CHECK-NEXT:                    "col": 15,
+// CHECK-NEXT:                    "tokLen": 1
+// CHECK-NEXT:                   }
+// CHECK-NEXT:                  },
+// CHECK-NEXT:                  "type": {
+// CHECK-NEXT:                   "qualType": "const S"
+// CHECK-NEXT:                  },
+// CHECK-NEXT:                  "valueCategory": "lvalue",
+// CHECK-NEXT:                  "castKind": "NoOp",
+// CHECK-NEXT:                  "inner": [
+// CHECK-NEXT:                   {
+// CHECK-NEXT:                    "id": "0x{{.*}}",
+// CHECK-NEXT:                    "kind": "DeclRefExpr",
+// CHECK-NEXT:                    "range": {
+// CHECK-NEXT:                     "begin": {
+// CHECK-NEXT:                      "offset": 307,
+// CHECK-NEXT:                      "col": 15,
+// CHECK-NEXT:                      "tokLen": 1
+// CHECK-NEXT:                     },
+// CHECK-NEXT:                     "end": {
+// CHECK-NEXT:                      "offset": 307,
+// CHECK-NEXT:                      "col": 15,
+// CHECK-NEXT:                      "tokLen": 1
+// CHECK-NEXT:                     }
+// CHECK-NEXT:                    },
+// CHECK-NEXT:                    "type": {
+// CHECK-NEXT:                     "qualType": "S"
+// CHECK-NEXT:                    },
+// CHECK-NEXT:                    "valueCategory": "lvalue",
+// CHECK-NEXT:                    "referencedDecl": {
+// CHECK-NEXT:                     "id": "0x{{.*}}",
+// CHECK-NEXT:                     "kind": "VarDecl",
+// CHECK-NEXT:                     "name": "__range1",
+// CHECK-NEXT:                     "type": {
+// CHECK-NEXT:                      "qualType": "S &&"
+// CHECK-NEXT:                     }
+// CHECK-NEXT:                    }
+// CHECK-NEXT:                   }
+// CHECK-NEXT:                  ]
+// CHECK-NEXT:                 }
+// CHECK-NEXT:                ]
+// CHECK-NEXT:               }
+// CHECK-NEXT:              ]
+// CHECK-NEXT:             }
+// CHECK-NEXT:            ]
+// CHECK-NEXT:           }
+// CHECK-NEXT:          ]
+// CHECK-NEXT:         },
+// CHECK-NEXT:         {
+// CHECK-NEXT:          "id": "0x{{.*}}",
+// CHECK-NEXT:          "kind": "BinaryOperator",
+// CHECK-NEXT:          "range": {
+// CHECK-NEXT:           "begin": {
+// CHECK-NEXT:            "offset": 307,
+// CHECK-NEXT:            "col": 15,
+// CHECK-NEXT:            "tokLen": 1
+// CHECK-NEXT:           },
+// CHECK-NEXT:           "end": {
+// CHECK-NEXT:            "offset": 307,
+// CHECK-NEXT:            "col": 15,
+// CHECK-NEXT:            "tokLen": 1
+// CHECK-NEXT:           }
+// CHECK-NEXT:          },
+// CHECK-NEXT:          "type": {
+// CHECK-NEXT:           "qualType": "bool"
+// CHECK-NEXT:          },
+// CHECK-NEXT:          "valueCategory": "prvalue",
+// CHECK-NEXT:          "opcode": "!=",
+// CHECK-NEXT:          "inner": [
+// CHECK-NEXT:           {
+// CHECK-NEXT:            "id": "0x{{.*}}",
+// CHECK-NEXT:            "kind": "ImplicitCastExpr",
+// CHECK-NEXT:            "range": {
+// CHECK-NEXT:             "begin": {
+// CHECK-NEXT:              "offset": 307,
+// CHECK-NEXT:              "col": 15,
+// CHECK-NEXT:              "tokLen": 1
+// CHECK-NEXT:             },
+// CHECK-NEXT:             "end": {
+// CHECK-NEXT:              "offset": 307,
+// CHECK-NEXT:              "col": 15,
+// CHECK-NEXT:              "tokLen": 1
+// CHECK-NEXT:             }
+// CHECK-NEXT:            },
+// CHECK-NEXT:            "type": {
+// CHECK-NEXT:             "qualType": "const int *"
+// CHECK-NEXT:            },
+// CHECK-NEXT:            "valueCategory": "prvalue",
+// CHECK-NEXT:            "castKind": "LValueToRValue",
+// CHECK-NEXT:            "inner": [
+// CHECK-NEXT:             {
+// CHECK-NEXT:              "id": "0x{{.*}}",
+// CHECK-NEXT:              "kind": "DeclRefExpr",
+// CHECK-NEXT:              "range": {
+// CHECK-NEXT:               "begin": {
+// CHECK-NEXT:                "offset": 307,
+// CHECK-NEXT:                "col": 15,
+// CHECK-NEXT:                "tokLen": 1
+// CHECK-NEXT:               },
+// CHECK-NEXT:               "end": {
+// CHECK-NEXT:                "offset": 307,
+// CHECK-NEXT:                "col": 15,
+// CHECK-NEXT:                "tokLen": 1
+// CHECK-NEXT:               }
+// CHECK-NEXT:              },
+// CHECK-NEXT:              "type": {
+// CHECK-NEXT:               "qualType": "const int *"
+// CHECK-NEXT:              },
+// CHECK-NEXT:              "valueCategory": "lvalue",
+// CHECK-NEXT:              "referencedDecl": {
+// CHECK-NEXT:               "id": "0x{{.*}}",
+// CHECK-NEXT:               "kind": "VarDecl",
+// CHECK-NEXT:               "name": "__begin1",
+// CHECK-NEXT:               "type": {
+// CHECK-NEXT:                "qualType": "const int *"
+// CHECK-NEXT:               }
+// CHECK-NEXT:              }
+// CHECK-NEXT:             }
+// CHECK-NEXT:            ]
+// CHECK-NEXT:           },
+// CHECK-NEXT:           {
+// CHECK-NEXT:            "id": "0x{{.*}}",
+// CHECK-NEXT:            "kind": "ImplicitCastExpr",
+// CHECK-NEXT:            "range": {
+// CHECK-NEXT:             "begin": {
+// CHECK-NEXT:              "offset": 307,
+// CHECK-NEXT:              "col": 15,
+// CHECK-NEXT:              "tokLen": 1
+// CHECK-NEXT:             },
+// CHECK-NEXT:             "end": {
+// CHECK-NEXT:              "offset": 307,
+// CHECK-NEXT:              "col": 15,
+// CHECK-NEXT:              "tokLen": 1
+// CHECK-NEXT:             }
+// CHECK-NEXT:            },
+// CHECK-NEXT:            "type": {
+// CHECK-NEXT:             "qualType": "const int *"
+// CHECK-NEXT:            },
+// CHECK-NEXT:            "valueCategory": "prvalue",
+// CHECK-NEXT:            "castKind": "LValueToRValue",
+// CHECK-NEXT:            "inner": [
+// CHECK-NEXT:             {
+// CHECK-NEXT:              "id": "0x{{.*}}",
+// CHECK-NEXT:              "kind": "DeclRefExpr",
+// CHECK-NEXT:              "range": {
+// CHECK-NEXT:               "begin": {
+// CHECK-NEXT:                "offset": 307,
+// CHECK-NEXT:                "col": 15,
+// CHECK-NEXT:                "tokLen": 1
+// CHECK-NEXT:               },
+// CHECK-NEXT:               "end": {
+// CHECK-NEXT:                "offset": 307,
+// CHECK-NEXT:                "col": 15,
+// CHECK-NEXT:                "tokLen": 1
+// CHECK-NEXT:               }
+// CHECK-NEXT:              },
+// CHECK-NEXT:              "type": {
+// CHECK-NEXT:               "qualType": "const int *"
+// CHECK-NEXT:              },
+// CHECK-NEXT:              "valueCategory": "lvalue",
+// CHECK-NEXT:              "referencedDecl": {
+// CHECK-NEXT:               "id": "0x{{.*}}",
+// CHECK-NEXT:               "kind": "VarDecl",
+// CHECK-NEXT:               "name": "__end1",
+// CHECK-NEXT:               "type": {
+// CHECK-NEXT:                "qualType": "const int *"
+// CHECK-NEXT:               }
+// CHECK-NEXT:              }
+// CHECK-NEXT:             }
+// CHECK-NEXT:            ]
+// CHECK-NEXT:           }
+// CHECK-NEXT:          ]
+// CHECK-NEXT:         },
+// CHECK-NEXT:         {
+// CHECK-NEXT:          "id": "0x{{.*}}",
+// CHECK-NEXT:          "kind": "UnaryOperator",
+// CHECK-NEXT:          "range": {
+// CHECK-NEXT:           "begin": {
+// CHECK-NEXT:            "offset": 307,
+// CHECK-NEXT:            "col": 15,
+// CHECK-NEXT:            "tokLen": 1
+// CHECK-NEXT:           },
+// CHECK-NEXT:           "end": {
+// CHECK-NEXT:            "offset": 307,
+// CHECK-NEXT:            "col": 15,
+// CHECK-NEXT:            "tokLen": 1
+// CHECK-NEXT:           }
+// CHECK-NEXT:          },
+// CHECK-NEXT:          "type": {
+// CHECK-NEXT:           "qualType": "const int *"
+// CHECK-NEXT:          },
+// CHECK-NEXT:          "valueCategory": "lvalue",
+// CHECK-NEXT:          "isPostfix": false,
+// CHECK-NEXT:          "opcode": "++",
+// CHECK-NEXT:          "inner": [
+// CHECK-NEXT:           {
+// CHECK-NEXT:            "id": "0x{{.*}}",
+// CHECK-NEXT:            "kind": "DeclRefExpr",
+// CHECK-NEXT:            "range": {
+// CHECK-NEXT:             "begin": {
+// CHECK-NEXT:              "offset": 307,
+// CHECK-NEXT:              "col": 15,
+// CHECK-NEXT:              "tokLen": 1
+// CHECK-NEXT:             },
+// CHECK-NEXT:             "end": {
+// CHECK-NEXT:              "offset": 307,
+// CHECK-NEXT:              "col": 15,
+// CHECK-NEXT:              "tokLen": 1
+// CHECK-NEXT:             }
+// CHECK-NEXT:            },
+// CHECK-NEXT:            "type": {
+// CHECK-NEXT:             "qualType": "const int *"
+// CHECK-NEXT:            },
+// CHECK-NEXT:            "valueCategory": "lvalue",
+// CHECK-NEXT:            "referencedDecl": {
+// CHECK-NEXT:             "id": "0x{{.*}}",
+// CHECK-NEXT:             "kind": "VarDecl",
+// CHECK-NEXT:             "name": "__begin1",
+// CHECK-NEXT:             "type": {
+// CHECK-NEXT:              "qualType": "const int *"
+// CHECK-NEXT:             }
+// CHECK-NEXT:            }
+// CHECK-NEXT:           }
+// CHECK-NEXT:          ]
+// CHECK-NEXT:         },
+// CHECK-NEXT:         {
+// CHECK-NEXT:          "id": "0x{{.*}}",
+// CHECK-NEXT:          "kind": "DeclStmt",
+// CHECK-NEXT:          "range": {
+// CHECK-NEXT:           "begin": {
+// CHECK-NEXT:            "offset": 300,
+// CHECK-NEXT:            "col": 8,
+// CHECK-NEXT:            "tokLen": 4
+// CHECK-NEXT:           },
+// CHECK-NEXT:           "end": {
+// CHECK-NEXT:            "offset": 316,
+// CHECK-NEXT:            "col": 24,
+// CHECK-NEXT:            "tokLen": 1
+// CHECK-NEXT:           }
+// CHECK-NEXT:          },
+// CHECK-NEXT:          "inner": [
+// CHECK-NEXT:           {
+// CHECK-NEXT:            "id": "0x{{.*}}",
+// CHECK-NEXT:            "kind": "VarDecl",
+// CHECK-NEXT:            "loc": {
+// CHECK-NEXT:             "offset": 305,
+// CHECK-NEXT:             "col": 13,
+// CHECK-NEXT:             "tokLen": 1
+// CHECK-NEXT:            },
+// CHECK-NEXT:            "range": {
+// CHECK-NEXT:             "begin": {
+// CHECK-NEXT:              "offset": 300,
+// CHECK-NEXT:              "col": 8,
+// CHECK-NEXT:              "tokLen": 4
+// CHECK-NEXT:             },
+// CHECK-NEXT:             "end": {
+// CHECK-NEXT:              "offset": 307,
+// CHECK-NEXT:              "col": 15,
+// CHECK-NEXT:              "tokLen": 1
+// CHECK-NEXT:             }
+// CHECK-NEXT:            },
+// CHECK-NEXT:            "name": "v",
+// CHECK-NEXT:            "type": {
+// CHECK-NEXT:             "qualType": "int"
+// CHECK-NEXT:            },
+// CHECK-NEXT:            "init": "c",
+// CHECK-NEXT:            "inner": [
+// CHECK-NEXT:             {
+// CHECK-NEXT:              "id": "0x{{.*}}",
+// CHECK-NEXT:              "kind": "ImplicitCastExpr",
+// CHECK-NEXT:              "range": {
+// CHECK-NEXT:               "begin": {
+// CHECK-NEXT:                "offset": 307,
+// CHECK-NEXT:                "col": 15,
+// CHECK-NEXT:                "tokLen": 1
+// CHECK-NEXT:               },
+// CHECK-NEXT:               "end": {
+// CHECK-NEXT:                "offset": 307,
+// CHECK-NEXT:                "col": 15,
+// CHECK-NEXT:                "tokLen": 1
+// CHECK-NEXT:               }
+// CHECK-NEXT:              },
+// CHECK-NEXT:              "type": {
+// CHECK-NEXT:               "qualType": "int"
+// CHECK-NEXT:              },
+// CHECK-NEXT:              "valueCategory": "prvalue",
+// CHECK-NEXT:              "castKind": "LValueToRValue",
+// CHECK-NEXT:              "inner": [
+// CHECK-NEXT:               {
+// CHECK-NEXT:                "id": "0x{{.*}}",
+// CHECK-NEXT:                "kind": "UnaryOperator",
+// CHECK-NEXT:                "range": {
+// CHECK-NEXT:                 "begin": {
+// CHECK-NEXT:                  "offset": 307,
+// CHECK-NEXT:                  "col": 15,
+// CHECK-NEXT:                  "tokLen": 1
+// CHECK-NEXT:                 },
+// CHECK-NEXT:                 "end": {
+// CHECK-NEXT:                  "offset": 307,
+// CHECK-NEXT:                  "col": 15,
+// CHECK-NEXT:                  "tokLen": 1
+// CHECK-NEXT:                 }
+// CHECK-NEXT:                },
+// CHECK-NEXT:                "type": {
+// CHECK-NEXT:                 "qualType": "const int"
+// CHECK-NEXT:                },
+// CHECK-NEXT:                "valueCategory": "lvalue",
+// CHECK-NEXT:                "isPostfix": false,
+// CHECK-NEXT:                "opcode": "*",
+// CHECK-NEXT:                "canOverflow": false,
+// CHECK-NEXT:                "inner": [
+// CHECK-NEXT:                 {
+// CHECK-NEXT:                  "id": "0x{{.*}}",
+// CHECK-NEXT:                  "kind": "ImplicitCastExpr",
+// CHECK-NEXT:                  "range": {
+// CHECK-NEXT:                   "begin": {
+// CHECK-NEXT:                    "offset": 307,
+// CHECK-NEXT:                    "col": 15,
+// CHECK-NEXT:                    "tokLen": 1
+// CHECK-NEXT:                   },
+// CHECK-NEXT:                   "end": {
+// CHECK-NEXT:                    "offset": 307,
+// CHECK-NEXT:                    "col": 15,
+// CHECK-NEXT:                    "tokLen": 1
+// CHECK-NEXT:                   }
+// CHECK-NEXT:                  },
+// CHECK-NEXT:                  "type": {
+// CHECK-NEXT:                   "qualType": "const int *"
+// CHECK-NEXT:                  },
+// CHECK-NEXT:                  "valueCategory": "prvalue",
+// CHECK-NEXT:                  "castKind": "LValueToRValue",
+// CHECK-NEXT:                  "inner": [
+// CHECK-NEXT:                   {
+// CHECK-NEXT:                    "id": "0x{{.*}}",
+// CHECK-NEXT:                    "kind": "DeclRefExpr",
+// CHECK-NEXT:                    "range": {
+// CHECK-NEXT:                     "begin": {
+// CHECK-NEXT:                      "offset": 307,
+// CHECK-NEXT:                      "col": 15,
+// CHECK-NEXT:                      "tokLen": 1
+// CHECK-NEXT:                     },
+// CHECK-NEXT:                     "end": {
+// CHECK-NEXT:                      "offset": 307,
+// CHECK-NEXT:                      "col": 15,
+// CHECK-NEXT:                      "tokLen": 1
+// CHECK-NEXT:                     }
+// CHECK-NEXT:                    },
+// CHECK-NEXT:                    "type": {
+// CHECK-NEXT:                     "qualType": "const int *"
+// CHECK-NEXT:                    },
+// CHECK-NEXT:                    "valueCategory": "lvalue",
+// CHECK-NEXT:                    "referencedDecl": {
+// CHECK-NEXT:                     "id": "0x{{.*}}",
+// CHECK-NEXT:                     "kind": "VarDecl",
+// CHECK-NEXT:                     "name": "__begin1",
+// CHECK-NEXT:                     "type": {
+// CHECK-NEXT:                      "qualType": "const int *"
+// CHECK-NEXT:                     }
+// CHECK-NEXT:                    }
+// CHECK-NEXT:                   }
+// CHECK-NEXT:                  ]
+// CHECK-NEXT:                 }
+// CHECK-NEXT:                ]
+// CHECK-NEXT:               }
+// CHECK-NEXT:              ]
+// CHECK-NEXT:             }
+// CHECK-NEXT:            ]
+// CHECK-NEXT:           }
+// CHECK-NEXT:          ]
+// CHECK-NEXT:         },
+// CHECK-NEXT:         {
+// CHECK-NEXT:          "id": "0x{{.*}}",
+// CHECK-NEXT:          "kind": "CompoundStmt",
+// CHECK-NEXT:          "range": {
+// CHECK-NEXT:           "begin": {
+// CHECK-NEXT:            "offset": 318,
+// CHECK-NEXT:            "col": 26,
+// CHECK-NEXT:            "tokLen": 1
+// CHECK-NEXT:           },
+// CHECK-NEXT:           "end": {
+// CHECK-NEXT:            "offset": 319,
+// CHECK-NEXT:            "col": 27,
+// CHECK-NEXT:            "tokLen": 1
+// CHECK-NEXT:           }
+// CHECK-NEXT:          }
+// CHECK-NEXT:         }
+// CHECK-NEXT:        ]
+// CHECK-NEXT:       }
+// CHECK-NEXT:      ]
+// CHECK-NEXT:     }
+// CHECK-NEXT:    ]
+// CHECK-NEXT:   }
+// CHECK-NEXT:  ]
+// CHECK-NEXT: }
diff --git a/clang/test/AST/ast-dump-default-init-json.cpp b/clang/test/AST/ast-dump-default-init-json.cpp
new file mode 100644
index 000000000000..1058b4e3ea4d
--- /dev/null
+++ b/clang/test/AST/ast-dump-default-init-json.cpp
@@ -0,0 +1,929 @@
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -fsyntax-only -ast-dump=json %s | FileCheck %s
+
+struct A {
+  int arr[1];
+};
+
+struct B {
+  const A &a = A{{0}};
+};
+
+void test() {
+  B b{};
+}
+
+// NOTE: CHECK lines have been autogenerated by gen_ast_dump_json_test.py
+
+
+// CHECK-NOT: {{^}}Dumping
+// CHECK:  "kind": "TranslationUnitDecl",
+// CHECK-NEXT:  "loc": {},
+// CHECK-NEXT:  "range": {
+// CHECK-NEXT:   "begin": {},
+// CHECK-NEXT:   "end": {}
+// CHECK-NEXT:  },
+// CHECK-NEXT:  "inner": [
+// CHECK-NEXT:   {
+// CHECK-NEXT:    "id": "0x{{.*}}",
+// CHECK-NEXT:    "kind": "TypedefDecl",
+// CHECK-NEXT:    "loc": {},
+// CHECK-NEXT:    "range": {
+// CHECK-NEXT:     "begin": {},
+// CHECK-NEXT:     "end": {}
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "isImplicit": true,
+// CHECK-NEXT:    "name": "__int128_t",
+// CHECK-NEXT:    "type": {
+// CHECK-NEXT:     "qualType": "__int128"
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "inner": [
+// CHECK-NEXT:     {
+// CHECK-NEXT:      "id": "0x{{.*}}",
+// CHECK-NEXT:      "kind": "BuiltinType",
+// CHECK-NEXT:      "type": {
+// CHECK-NEXT:       "qualType": "__int128"
+// CHECK-NEXT:      }
+// CHECK-NEXT:     }
+// CHECK-NEXT:    ]
+// CHECK-NEXT:   },
+// CHECK-NEXT:   {
+// CHECK-NEXT:    "id": "0x{{.*}}",
+// CHECK-NEXT:    "kind": "TypedefDecl",
+// CHECK-NEXT:    "loc": {},
+// CHECK-NEXT:    "range": {
+// CHECK-NEXT:     "begin": {},
+// CHECK-NEXT:     "end": {}
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "isImplicit": true,
+// CHECK-NEXT:    "name": "__uint128_t",
+// CHECK-NEXT:    "type": {
+// CHECK-NEXT:     "qualType": "unsigned __int128"
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "inner": [
+// CHECK-NEXT:     {
+// CHECK-NEXT:      "id": "0x{{.*}}",
+// CHECK-NEXT:      "kind": "BuiltinType",
+// CHECK-NEXT:      "type": {
+// CHECK-NEXT:       "qualType": "unsigned __int128"
+// CHECK-NEXT:      }
+// CHECK-NEXT:     }
+// CHECK-NEXT:    ]
+// CHECK-NEXT:   },
+// CHECK-NEXT:   {
+// CHECK-NEXT:    "id": "0x{{.*}}",
+// CHECK-NEXT:    "kind": "TypedefDecl",
+// CHECK-NEXT:    "loc": {},
+// CHECK-NEXT:    "range": {
+// CHECK-NEXT:     "begin": {},
+// CHECK-NEXT:     "end": {}
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "isImplicit": true,
+// CHECK-NEXT:    "name": "__NSConstantString",
+// CHECK-NEXT:    "type": {
+// CHECK-NEXT:     "qualType": "__NSConstantString_tag"
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "inner": [
+// CHECK-NEXT:     {
+// CHECK-NEXT:      "id": "0x{{.*}}",
+// CHECK-NEXT:      "kind": "RecordType",
+// CHECK-NEXT:      "type": {
+// CHECK-NEXT:       "qualType": "__NSConstantString_tag"
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "decl": {
+// CHECK-NEXT:       "id": "0x{{.*}}",
+// CHECK-NEXT:       "kind": "CXXRecordDecl",
+// CHECK-NEXT:       "name": "__NSConstantString_tag"
+// CHECK-NEXT:      }
+// CHECK-NEXT:     }
+// CHECK-NEXT:    ]
+// CHECK-NEXT:   },
+// CHECK-NEXT:   {
+// CHECK-NEXT:    "id": "0x{{.*}}",
+// CHECK-NEXT:    "kind": "TypedefDecl",
+// CHECK-NEXT:    "loc": {},
+// CHECK-NEXT:    "range": {
+// CHECK-NEXT:     "begin": {},
+// CHECK-NEXT:     "end": {}
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "isImplicit": true,
+// CHECK-NEXT:    "name": "__builtin_ms_va_list",
+// CHECK-NEXT:    "type": {
+// CHECK-NEXT:     "qualType": "char *"
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "inner": [
+// CHECK-NEXT:     {
+// CHECK-NEXT:      "id": "0x{{.*}}",
+// CHECK-NEXT:      "kind": "PointerType",
+// CHECK-NEXT:      "type": {
+// CHECK-NEXT:       "qualType": "char *"
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "inner": [
+// CHECK-NEXT:       {
+// CHECK-NEXT:        "id": "0x{{.*}}",
+// CHECK-NEXT:        "kind": "BuiltinType",
+// CHECK-NEXT:        "type": {
+// CHECK-NEXT:         "qualType": "char"
+// CHECK-NEXT:        }
+// CHECK-NEXT:       }
+// CHECK-NEXT:      ]
+// CHECK-NEXT:     }
+// CHECK-NEXT:    ]
+// CHECK-NEXT:   },
+// CHECK-NEXT:   {
+// CHECK-NEXT:    "id": "0x{{.*}}",
+// CHECK-NEXT:    "kind": "TypedefDecl",
+// CHECK-NEXT:    "loc": {},
+// CHECK-NEXT:    "range": {
+// CHECK-NEXT:     "begin": {},
+// CHECK-NEXT:     "end": {}
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "isImplicit": true,
+// CHECK-NEXT:    "name": "__builtin_va_list",
+// CHECK-NEXT:    "type": {
+// CHECK-NEXT:     "qualType": "__va_list_tag[1]"
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "inner": [
+// CHECK-NEXT:     {
+// CHECK-NEXT:      "id": "0x{{.*}}",
+// CHECK-NEXT:      "kind": "ConstantArrayType",
+// CHECK-NEXT:      "type": {
+// CHECK-NEXT:       "qualType": "__va_list_tag[1]"
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "size": 1,
+// CHECK-NEXT:      "inner": [
+// CHECK-NEXT:       {
+// CHECK-NEXT:        "id": "0x{{.*}}",
+// CHECK-NEXT:        "kind": "RecordType",
+// CHECK-NEXT:        "type": {
+// CHECK-NEXT:         "qualType": "__va_list_tag"
+// CHECK-NEXT:        },
+// CHECK-NEXT:        "decl": {
+// CHECK-NEXT:         "id": "0x{{.*}}",
+// CHECK-NEXT:         "kind": "CXXRecordDecl",
+// CHECK-NEXT:         "name": "__va_list_tag"
+// CHECK-NEXT:        }
+// CHECK-NEXT:       }
+// CHECK-NEXT:      ]
+// CHECK-NEXT:     }
+// CHECK-NEXT:    ]
+// CHECK-NEXT:   },
+// CHECK-NEXT:   {
+// CHECK-NEXT:    "id": "0x{{.*}}",
+// CHECK-NEXT:    "kind": "CXXRecordDecl",
+// CHECK-NEXT:    "loc": {
+// CHECK-NEXT:     "offset": 99,
+// CHECK-NEXT:     "file": "{{.*}}",
+// CHECK-NEXT:     "line": 3,
+// CHECK-NEXT:     "col": 8,
+// CHECK-NEXT:     "tokLen": 1
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "range": {
+// CHECK-NEXT:     "begin": {
+// CHECK-NEXT:      "offset": 92,
+// CHECK-NEXT:      "col": 1,
+// CHECK-NEXT:      "tokLen": 6
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "end": {
+// CHECK-NEXT:      "offset": 117,
+// CHECK-NEXT:      "line": 5,
+// CHECK-NEXT:      "col": 1,
+// CHECK-NEXT:      "tokLen": 1
+// CHECK-NEXT:     }
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "isReferenced": true,
+// CHECK-NEXT:    "name": "A",
+// CHECK-NEXT:    "tagUsed": "struct",
+// CHECK-NEXT:    "completeDefinition": true,
+// CHECK-NEXT:    "definitionData": {
+// CHECK-NEXT:     "canPassInRegisters": true,
+// CHECK-NEXT:     "copyAssign": {
+// CHECK-NEXT:      "hasConstParam": true,
+// CHECK-NEXT:      "implicitHasConstParam": true,
+// CHECK-NEXT:      "needsImplicit": true,
+// CHECK-NEXT:      "simple": true,
+// CHECK-NEXT:      "trivial": true
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "copyCtor": {
+// CHECK-NEXT:      "hasConstParam": true,
+// CHECK-NEXT:      "implicitHasConstParam": true,
+// CHECK-NEXT:      "needsImplicit": true,
+// CHECK-NEXT:      "simple": true,
+// CHECK-NEXT:      "trivial": true
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "defaultCtor": {
+// CHECK-NEXT:      "exists": true,
+// CHECK-NEXT:      "needsImplicit": true,
+// CHECK-NEXT:      "trivial": true
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "dtor": {
+// CHECK-NEXT:      "irrelevant": true,
+// CHECK-NEXT:      "simple": true,
+// CHECK-NEXT:      "trivial": true
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "isAggregate": true,
+// CHECK-NEXT:     "isLiteral": true,
+// CHECK-NEXT:     "isPOD": true,
+// CHECK-NEXT:     "isStandardLayout": true,
+// CHECK-NEXT:     "isTrivial": true,
+// CHECK-NEXT:     "isTriviallyCopyable": true,
+// CHECK-NEXT:     "moveAssign": {
+// CHECK-NEXT:      "exists": true,
+// CHECK-NEXT:      "needsImplicit": true,
+// CHECK-NEXT:      "simple": true,
+// CHECK-NEXT:      "trivial": true
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "moveCtor": {
+// CHECK-NEXT:      "exists": true,
+// CHECK-NEXT:      "needsImplicit": true,
+// CHECK-NEXT:      "simple": true,
+// CHECK-NEXT:      "trivial": true
+// CHECK-NEXT:     }
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "inner": [
+// CHECK-NEXT:     {
+// CHECK-NEXT:      "id": "0x{{.*}}",
+// CHECK-NEXT:      "kind": "CXXRecordDecl",
+// CHECK-NEXT:      "loc": {
+// CHECK-NEXT:       "offset": 99,
+// CHECK-NEXT:       "line": 3,
+// CHECK-NEXT:       "col": 8,
+// CHECK-NEXT:       "tokLen": 1
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "range": {
+// CHECK-NEXT:       "begin": {
+// CHECK-NEXT:        "offset": 92,
+// CHECK-NEXT:        "col": 1,
+// CHECK-NEXT:        "tokLen": 6
+// CHECK-NEXT:       },
+// CHECK-NEXT:       "end": {
+// CHECK-NEXT:        "offset": 99,
+// CHECK-NEXT:        "col": 8,
+// CHECK-NEXT:        "tokLen": 1
+// CHECK-NEXT:       }
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "isImplicit": true,
+// CHECK-NEXT:      "name": "A",
+// CHECK-NEXT:      "tagUsed": "struct"
+// CHECK-NEXT:     },
+// CHECK-NEXT:     {
+// CHECK-NEXT:      "id": "0x{{.*}}",
+// CHECK-NEXT:      "kind": "FieldDecl",
+// CHECK-NEXT:      "loc": {
+// CHECK-NEXT:       "offset": 109,
+// CHECK-NEXT:       "line": 4,
+// CHECK-NEXT:       "col": 7,
+// CHECK-NEXT:       "tokLen": 3
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "range": {
+// CHECK-NEXT:       "begin": {
+// CHECK-NEXT:        "offset": 105,
+// CHECK-NEXT:        "col": 3,
+// CHECK-NEXT:        "tokLen": 3
+// CHECK-NEXT:       },
+// CHECK-NEXT:       "end": {
+// CHECK-NEXT:        "offset": 114,
+// CHECK-NEXT:        "col": 12,
+// CHECK-NEXT:        "tokLen": 1
+// CHECK-NEXT:       }
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "name": "arr",
+// CHECK-NEXT:      "type": {
+// CHECK-NEXT:       "qualType": "int[1]"
+// CHECK-NEXT:      }
+// CHECK-NEXT:     },
+// CHECK-NEXT:     {
+// CHECK-NEXT:      "id": "0x{{.*}}",
+// CHECK-NEXT:      "kind": "CXXDestructorDecl",
+// CHECK-NEXT:      "loc": {
+// CHECK-NEXT:       "offset": 99,
+// CHECK-NEXT:       "line": 3,
+// CHECK-NEXT:       "col": 8,
+// CHECK-NEXT:       "tokLen": 1
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "range": {
+// CHECK-NEXT:       "begin": {
+// CHECK-NEXT:        "offset": 99,
+// CHECK-NEXT:        "col": 8,
+// CHECK-NEXT:        "tokLen": 1
+// CHECK-NEXT:       },
+// CHECK-NEXT:       "end": {
+// CHECK-NEXT:        "offset": 99,
+// CHECK-NEXT:        "col": 8,
+// CHECK-NEXT:        "tokLen": 1
+// CHECK-NEXT:       }
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "isImplicit": true,
+// CHECK-NEXT:      "isReferenced": true,
+// CHECK-NEXT:      "name": "~A",
+// CHECK-NEXT:      "mangledName": "_ZN1AD1Ev",
+// CHECK-NEXT:      "type": {
+// CHECK-NEXT:       "qualType": "void () noexcept"
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "inline": true,
+// CHECK-NEXT:      "explicitlyDefaulted": "default"
+// CHECK-NEXT:     }
+// CHECK-NEXT:    ]
+// CHECK-NEXT:   },
+// CHECK-NEXT:   {
+// CHECK-NEXT:    "id": "0x{{.*}}",
+// CHECK-NEXT:    "kind": "CXXRecordDecl",
+// CHECK-NEXT:    "loc": {
+// CHECK-NEXT:     "offset": 128,
+// CHECK-NEXT:     "line": 7,
+// CHECK-NEXT:     "col": 8,
+// CHECK-NEXT:     "tokLen": 1
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "range": {
+// CHECK-NEXT:     "begin": {
+// CHECK-NEXT:      "offset": 121,
+// CHECK-NEXT:      "col": 1,
+// CHECK-NEXT:      "tokLen": 6
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "end": {
+// CHECK-NEXT:      "offset": 155,
+// CHECK-NEXT:      "line": 9,
+// CHECK-NEXT:      "col": 1,
+// CHECK-NEXT:      "tokLen": 1
+// CHECK-NEXT:     }
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "isReferenced": true,
+// CHECK-NEXT:    "name": "B",
+// CHECK-NEXT:    "tagUsed": "struct",
+// CHECK-NEXT:    "completeDefinition": true,
+// CHECK-NEXT:    "definitionData": {
+// CHECK-NEXT:     "canConstDefaultInit": true,
+// CHECK-NEXT:     "canPassInRegisters": true,
+// CHECK-NEXT:     "copyAssign": {
+// CHECK-NEXT:      "hasConstParam": true,
+// CHECK-NEXT:      "implicitHasConstParam": true,
+// CHECK-NEXT:      "needsImplicit": true,
+// CHECK-NEXT:      "trivial": true
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "copyCtor": {
+// CHECK-NEXT:      "hasConstParam": true,
+// CHECK-NEXT:      "implicitHasConstParam": true,
+// CHECK-NEXT:      "needsImplicit": true,
+// CHECK-NEXT:      "simple": true,
+// CHECK-NEXT:      "trivial": true
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "defaultCtor": {
+// CHECK-NEXT:      "defaultedIsConstexpr": true,
+// CHECK-NEXT:      "exists": true,
+// CHECK-NEXT:      "isConstexpr": true,
+// CHECK-NEXT:      "needsImplicit": true,
+// CHECK-NEXT:      "nonTrivial": true
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "dtor": {
+// CHECK-NEXT:      "irrelevant": true,
+// CHECK-NEXT:      "needsImplicit": true,
+// CHECK-NEXT:      "simple": true,
+// CHECK-NEXT:      "trivial": true
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "hasConstexprNonCopyMoveConstructor": true,
+// CHECK-NEXT:     "isAggregate": true,
+// CHECK-NEXT:     "isLiteral": true,
+// CHECK-NEXT:     "isTriviallyCopyable": true,
+// CHECK-NEXT:     "moveAssign": {
+// CHECK-NEXT:      "exists": true,
+// CHECK-NEXT:      "needsImplicit": true,
+// CHECK-NEXT:      "trivial": true
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "moveCtor": {
+// CHECK-NEXT:      "exists": true,
+// CHECK-NEXT:      "needsImplicit": true,
+// CHECK-NEXT:      "simple": true,
+// CHECK-NEXT:      "trivial": true
+// CHECK-NEXT:     }
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "inner": [
+// CHECK-NEXT:     {
+// CHECK-NEXT:      "id": "0x{{.*}}",
+// CHECK-NEXT:      "kind": "CXXRecordDecl",
+// CHECK-NEXT:      "loc": {
+// CHECK-NEXT:       "offset": 128,
+// CHECK-NEXT:       "line": 7,
+// CHECK-NEXT:       "col": 8,
+// CHECK-NEXT:       "tokLen": 1
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "range": {
+// CHECK-NEXT:       "begin": {
+// CHECK-NEXT:        "offset": 121,
+// CHECK-NEXT:        "col": 1,
+// CHECK-NEXT:        "tokLen": 6
+// CHECK-NEXT:       },
+// CHECK-NEXT:       "end": {
+// CHECK-NEXT:        "offset": 128,
+// CHECK-NEXT:        "col": 8,
+// CHECK-NEXT:        "tokLen": 1
+// CHECK-NEXT:       }
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "isImplicit": true,
+// CHECK-NEXT:      "name": "B",
+// CHECK-NEXT:      "tagUsed": "struct"
+// CHECK-NEXT:     },
+// CHECK-NEXT:     {
+// CHECK-NEXT:      "id": "0x{{.*}}",
+// CHECK-NEXT:      "kind": "FieldDecl",
+// CHECK-NEXT:      "loc": {
+// CHECK-NEXT:       "offset": 143,
+// CHECK-NEXT:       "line": 8,
+// CHECK-NEXT:       "col": 12,
+// CHECK-NEXT:       "tokLen": 1
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "range": {
+// CHECK-NEXT:       "begin": {
+// CHECK-NEXT:        "offset": 134,
+// CHECK-NEXT:        "col": 3,
+// CHECK-NEXT:        "tokLen": 5
+// CHECK-NEXT:       },
+// CHECK-NEXT:       "end": {
+// CHECK-NEXT:        "offset": 152,
+// CHECK-NEXT:        "col": 21,
+// CHECK-NEXT:        "tokLen": 1
+// CHECK-NEXT:       }
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "name": "a",
+// CHECK-NEXT:      "type": {
+// CHECK-NEXT:       "qualType": "const A &"
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "hasInClassInitializer": true,
+// CHECK-NEXT:      "inner": [
+// CHECK-NEXT:       {
+// CHECK-NEXT:        "id": "0x{{.*}}",
+// CHECK-NEXT:        "kind": "ExprWithCleanups",
+// CHECK-NEXT:        "range": {
+// CHECK-NEXT:         "begin": {
+// CHECK-NEXT:          "offset": 147,
+// CHECK-NEXT:          "col": 16,
+// CHECK-NEXT:          "tokLen": 1
+// CHECK-NEXT:         },
+// CHECK-NEXT:         "end": {
+// CHECK-NEXT:          "offset": 152,
+// CHECK-NEXT:          "col": 21,
+// CHECK-NEXT:          "tokLen": 1
+// CHECK-NEXT:         }
+// CHECK-NEXT:        },
+// CHECK-NEXT:        "type": {
+// CHECK-NEXT:         "qualType": "const A"
+// CHECK-NEXT:        },
+// CHECK-NEXT:        "valueCategory": "lvalue",
+// CHECK-NEXT:        "inner": [
+// CHECK-NEXT:         {
+// CHECK-NEXT:          "id": "0x{{.*}}",
+// CHECK-NEXT:          "kind": "MaterializeTemporaryExpr",
+// CHECK-NEXT:          "range": {
+// CHECK-NEXT:           "begin": {
+// CHECK-NEXT:            "offset": 147,
+// CHECK-NEXT:            "col": 16,
+// CHECK-NEXT:            "tokLen": 1
+// CHECK-NEXT:           },
+// CHECK-NEXT:           "end": {
+// CHECK-NEXT:            "offset": 152,
+// CHECK-NEXT:            "col": 21,
+// CHECK-NEXT:            "tokLen": 1
+// CHECK-NEXT:           }
+// CHECK-NEXT:          },
+// CHECK-NEXT:          "type": {
+// CHECK-NEXT:           "qualType": "const A"
+// CHECK-NEXT:          },
+// CHECK-NEXT:          "valueCategory": "lvalue",
+// CHECK-NEXT:          "extendingDecl": {
+// CHECK-NEXT:           "id": "0x{{.*}}",
+// CHECK-NEXT:           "kind": "FieldDecl",
+// CHECK-NEXT:           "name": "a",
+// CHECK-NEXT:           "type": {
+// CHECK-NEXT:            "qualType": "const A &"
+// CHECK-NEXT:           }
+// CHECK-NEXT:          },
+// CHECK-NEXT:          "storageDuration": "automatic",
+// CHECK-NEXT:          "boundToLValueRef": true,
+// CHECK-NEXT:          "inner": [
+// CHECK-NEXT:           {
+// CHECK-NEXT:            "id": "0x{{.*}}",
+// CHECK-NEXT:            "kind": "ImplicitCastExpr",
+// CHECK-NEXT:            "range": {
+// CHECK-NEXT:             "begin": {
+// CHECK-NEXT:              "offset": 147,
+// CHECK-NEXT:              "col": 16,
+// CHECK-NEXT:              "tokLen": 1
+// CHECK-NEXT:             },
+// CHECK-NEXT:             "end": {
+// CHECK-NEXT:              "offset": 152,
+// CHECK-NEXT:              "col": 21,
+// CHECK-NEXT:              "tokLen": 1
+// CHECK-NEXT:             }
+// CHECK-NEXT:            },
+// CHECK-NEXT:            "type": {
+// CHECK-NEXT:             "qualType": "const A"
+// CHECK-NEXT:            },
+// CHECK-NEXT:            "valueCategory": "prvalue",
+// CHECK-NEXT:            "castKind": "NoOp",
+// CHECK-NEXT:            "inner": [
+// CHECK-NEXT:             {
+// CHECK-NEXT:              "id": "0x{{.*}}",
+// CHECK-NEXT:              "kind": "CXXFunctionalCastExpr",
+// CHECK-NEXT:              "range": {
+// CHECK-NEXT:               "begin": {
+// CHECK-NEXT:                "offset": 147,
+// CHECK-NEXT:                "col": 16,
+// CHECK-NEXT:                "tokLen": 1
+// CHECK-NEXT:               },
+// CHECK-NEXT:               "end": {
+// CHECK-NEXT:                "offset": 152,
+// CHECK-NEXT:                "col": 21,
+// CHECK-NEXT:                "tokLen": 1
+// CHECK-NEXT:               }
+// CHECK-NEXT:              },
+// CHECK-NEXT:              "type": {
+// CHECK-NEXT:               "qualType": "A"
+// CHECK-NEXT:              },
+// CHECK-NEXT:              "valueCategory": "prvalue",
+// CHECK-NEXT:              "castKind": "NoOp",
+// CHECK-NEXT:              "inner": [
+// CHECK-NEXT:               {
+// CHECK-NEXT:                "id": "0x{{.*}}",
+// CHECK-NEXT:                "kind": "InitListExpr",
+// CHECK-NEXT:                "range": {
+// CHECK-NEXT:                 "begin": {
+// CHECK-NEXT:                  "offset": 148,
+// CHECK-NEXT:                  "col": 17,
+// CHECK-NEXT:                  "tokLen": 1
+// CHECK-NEXT:                 },
+// CHECK-NEXT:                 "end": {
+// CHECK-NEXT:                  "offset": 152,
+// CHECK-NEXT:                  "col": 21,
+// CHECK-NEXT:                  "tokLen": 1
+// CHECK-NEXT:                 }
+// CHECK-NEXT:                },
+// CHECK-NEXT:                "type": {
+// CHECK-NEXT:                 "qualType": "A"
+// CHECK-NEXT:                },
+// CHECK-NEXT:                "valueCategory": "prvalue",
+// CHECK-NEXT:                "inner": [
+// CHECK-NEXT:                 {
+// CHECK-NEXT:                  "id": "0x{{.*}}",
+// CHECK-NEXT:                  "kind": "InitListExpr",
+// CHECK-NEXT:                  "range": {
+// CHECK-NEXT:                   "begin": {
+// CHECK-NEXT:                    "offset": 149,
+// CHECK-NEXT:                    "col": 18,
+// CHECK-NEXT:                    "tokLen": 1
+// CHECK-NEXT:                   },
+// CHECK-NEXT:                   "end": {
+// CHECK-NEXT:                    "offset": 151,
+// CHECK-NEXT:                    "col": 20,
+// CHECK-NEXT:                    "tokLen": 1
+// CHECK-NEXT:                   }
+// CHECK-NEXT:                  },
+// CHECK-NEXT:                  "type": {
+// CHECK-NEXT:                   "qualType": "int[1]"
+// CHECK-NEXT:                  },
+// CHECK-NEXT:                  "valueCategory": "prvalue",
+// CHECK-NEXT:                  "inner": [
+// CHECK-NEXT:                   {
+// CHECK-NEXT:                    "id": "0x{{.*}}",
+// CHECK-NEXT:                    "kind": "IntegerLiteral",
+// CHECK-NEXT:                    "range": {
+// CHECK-NEXT:                     "begin": {
+// CHECK-NEXT:                      "offset": 150,
+// CHECK-NEXT:                      "col": 19,
+// CHECK-NEXT:                      "tokLen": 1
+// CHECK-NEXT:                     },
+// CHECK-NEXT:                     "end": {
+// CHECK-NEXT:                      "offset": 150,
+// CHECK-NEXT:                      "col": 19,
+// CHECK-NEXT:                      "tokLen": 1
+// CHECK-NEXT:                     }
+// CHECK-NEXT:                    },
+// CHECK-NEXT:                    "type": {
+// CHECK-NEXT:                     "qualType": "int"
+// CHECK-NEXT:                    },
+// CHECK-NEXT:                    "valueCategory": "prvalue",
+// CHECK-NEXT:                    "value": "0"
+// CHECK-NEXT:                   }
+// CHECK-NEXT:                  ]
+// CHECK-NEXT:                 }
+// CHECK-NEXT:                ]
+// CHECK-NEXT:               }
+// CHECK-NEXT:              ]
+// CHECK-NEXT:             }
+// CHECK-NEXT:            ]
+// CHECK-NEXT:           }
+// CHECK-NEXT:          ]
+// CHECK-NEXT:         }
+// CHECK-NEXT:        ]
+// CHECK-NEXT:       }
+// CHECK-NEXT:      ]
+// CHECK-NEXT:     }
+// CHECK-NEXT:    ]
+// CHECK-NEXT:   },
+// CHECK-NEXT:   {
+// CHECK-NEXT:    "id": "0x{{.*}}",
+// CHECK-NEXT:    "kind": "FunctionDecl",
+// CHECK-NEXT:    "loc": {
+// CHECK-NEXT:     "offset": 164,
+// CHECK-NEXT:     "line": 11,
+// CHECK-NEXT:     "col": 6,
+// CHECK-NEXT:     "tokLen": 4
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "range": {
+// CHECK-NEXT:     "begin": {
+// CHECK-NEXT:      "offset": 159,
+// CHECK-NEXT:      "col": 1,
+// CHECK-NEXT:      "tokLen": 4
+// CHECK-NEXT:     },
+// CHECK-NEXT:     "end": {
+// CHECK-NEXT:      "offset": 182,
+// CHECK-NEXT:      "line": 13,
+// CHECK-NEXT:      "col": 1,
+// CHECK-NEXT:      "tokLen": 1
+// CHECK-NEXT:     }
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "name": "test",
+// CHECK-NEXT:    "mangledName": "_Z4testv",
+// CHECK-NEXT:    "type": {
+// CHECK-NEXT:     "qualType": "void ()"
+// CHECK-NEXT:    },
+// CHECK-NEXT:    "inner": [
+// CHECK-NEXT:     {
+// CHECK-NEXT:      "id": "0x{{.*}}",
+// CHECK-NEXT:      "kind": "CompoundStmt",
+// CHECK-NEXT:      "range": {
+// CHECK-NEXT:       "begin": {
+// CHECK-NEXT:        "offset": 171,
+// CHECK-NEXT:        "line": 11,
+// CHECK-NEXT:        "col": 13,
+// CHECK-NEXT:        "tokLen": 1
+// CHECK-NEXT:       },
+// CHECK-NEXT:       "end": {
+// CHECK-NEXT:        "offset": 182,
+// CHECK-NEXT:        "line": 13,
+// CHECK-NEXT:        "col": 1,
+// CHECK-NEXT:        "tokLen": 1
+// CHECK-NEXT:       }
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "inner": [
+// CHECK-NEXT:       {
+// CHECK-NEXT:        "id": "0x{{.*}}",
+// CHECK-NEXT:        "kind": "DeclStmt",
+// CHECK-NEXT:        "range": {
+// CHECK-NEXT:         "begin": {
+// CHECK-NEXT:          "offset": 175,
+// CHECK-NEXT:          "line": 12,
+// CHECK-NEXT:          "col": 3,
+// CHECK-NEXT:          "tokLen": 1
+// CHECK-NEXT:         },
+// CHECK-NEXT:         "end": {
+// CHECK-NEXT:          "offset": 180,
+// CHECK-NEXT:          "col": 8,
+// CHECK-NEXT:          "tokLen": 1
+// CHECK-NEXT:         }
+// CHECK-NEXT:        },
+// CHECK-NEXT:        "inner": [
+// CHECK-NEXT:         {
+// CHECK-NEXT:          "id": "0x{{.*}}",
+// CHECK-NEXT:          "kind": "VarDecl",
+// CHECK-NEXT:          "loc": {
+// CHECK-NEXT:           "offset": 177,
+// CHECK-NEXT:           "col": 5,
+// CHECK-NEXT:           "tokLen": 1
+// CHECK-NEXT:          },
+// CHECK-NEXT:          "range": {
+// CHECK-NEXT:           "begin": {
+// CHECK-NEXT:            "offset": 175,
+// CHECK-NEXT:            "col": 3,
+// CHECK-NEXT:            "tokLen": 1
+// CHECK-NEXT:           },
+// CHECK-NEXT:           "end": {
+// CHECK-NEXT:            "offset": 179,
+// CHECK-NEXT:            "col": 7,
+// CHECK-NEXT:            "tokLen": 1
+// CHECK-NEXT:           }
+// CHECK-NEXT:          },
+// CHECK-NEXT:          "name": "b",
+// CHECK-NEXT:          "type": {
+// CHECK-NEXT:           "qualType": "B"
+// CHECK-NEXT:          },
+// CHECK-NEXT:          "init": "list",
+// CHECK-NEXT:          "inner": [
+// CHECK-NEXT:           {
+// CHECK-NEXT:            "id": "0x{{.*}}",
+// CHECK-NEXT:            "kind": "InitListExpr",
+// CHECK-NEXT:            "range": {
+// CHECK-NEXT:             "begin": {
+// CHECK-NEXT:              "offset": 178,
+// CHECK-NEXT:              "col": 6,
+// CHECK-NEXT:              "tokLen": 1
+// CHECK-NEXT:             },
+// CHECK-NEXT:             "end": {
+// CHECK-NEXT:              "offset": 179,
+// CHECK-NEXT:              "col": 7,
+// CHECK-NEXT:              "tokLen": 1
+// CHECK-NEXT:             }
+// CHECK-NEXT:            },
+// CHECK-NEXT:            "type": {
+// CHECK-NEXT:             "qualType": "B"
+// CHECK-NEXT:            },
+// CHECK-NEXT:            "valueCategory": "prvalue",
+// CHECK-NEXT:            "inner": [
+// CHECK-NEXT:             {
+// CHECK-NEXT:              "id": "0x{{.*}}",
+// CHECK-NEXT:              "kind": "CXXDefaultInitExpr",
+// CHECK-NEXT:              "range": {
+// CHECK-NEXT:               "begin": {
+// CHECK-NEXT:                "offset": 179,
+// CHECK-NEXT:                "col": 7,
+// CHECK-NEXT:                "tokLen": 1
+// CHECK-NEXT:               },
+// CHECK-NEXT:               "end": {
+// CHECK-NEXT:                "offset": 179,
+// CHECK-NEXT:                "col": 7,
+// CHECK-NEXT:                "tokLen": 1
+// CHECK-NEXT:               }
+// CHECK-NEXT:              },
+// CHECK-NEXT:              "type": {
+// CHECK-NEXT:               "qualType": "const A"
+// CHECK-NEXT:              },
+// CHECK-NEXT:              "valueCategory": "lvalue",
+// CHECK-NEXT:              "hasRewrittenInit": true,
+// CHECK-NEXT:              "inner": [
+// CHECK-NEXT:               {
+// CHECK-NEXT:                "id": "0x{{.*}}",
+// CHECK-NEXT:                "kind": "ExprWithCleanups",
+// CHECK-NEXT:                "range": {
+// CHECK-NEXT:                 "begin": {
+// CHECK-NEXT:                  "offset": 147,
+// CHECK-NEXT:                  "line": 8,
+// CHECK-NEXT:                  "col": 16,
+// CHECK-NEXT:                  "tokLen": 1
+// CHECK-NEXT:                 },
+// CHECK-NEXT:                 "end": {
+// CHECK-NEXT:                  "offset": 152,
+// CHECK-NEXT:                  "col": 21,
+// CHECK-NEXT:                  "tokLen": 1
+// CHECK-NEXT:                 }
+// CHECK-NEXT:                },
+// CHECK-NEXT:                "type": {
+// CHECK-NEXT:                 "qualType": "const A"
+// CHECK-NEXT:                },
+// CHECK-NEXT:                "valueCategory": "lvalue",
+// CHECK-NEXT:                "inner": [
+// CHECK-NEXT:                 {
+// CHECK-NEXT:                  "id": "0x{{.*}}",
+// CHECK-NEXT:                  "kind": "MaterializeTemporaryExpr",
+// CHECK-NEXT:                  "range": {
+// CHECK-NEXT:                   "begin": {
+// CHECK-NEXT:                    "offset": 147,
+// CHECK-NEXT:                    "col": 16,
+// CHECK-NEXT:                    "tokLen": 1
+// CHECK-NEXT:                   },
+// CHECK-NEXT:                   "end": {
+// CHECK-NEXT:                    "offset": 152,
+// CHECK-NEXT:                    "col": 21,
+// CHECK-NEXT:                    "tokLen": 1
+// CHECK-NEXT:                   }
+// CHECK-NEXT:                  },
+// CHECK-NEXT:                  "type": {
+// CHECK-NEXT:                   "qualType": "const A"
+// CHECK-NEXT:                  },
+// CHECK-NEXT:                  "valueCategory": "lvalue",
+// CHECK-NEXT:                  "extendingDecl": {
+// CHECK-NEXT:                   "id": "0x{{.*}}",
+// CHECK-NEXT:                   "kind": "FieldDecl",
+// CHECK-NEXT:                   "name": "a",
+// CHECK-NEXT:                   "type": {
+// CHECK-NEXT:                    "qualType": "const A &"
+// CHECK-NEXT:                   }
+// CHECK-NEXT:                  },
+// CHECK-NEXT:                  "storageDuration": "automatic",
+// CHECK-NEXT:                  "boundToLValueRef": true,
+// CHECK-NEXT:                  "inner": [
+// CHECK-NEXT:                   {
+// CHECK-NEXT:                    "id": "0x{{.*}}",
+// CHECK-NEXT:                    "kind": "ImplicitCastExpr",
+// CHECK-NEXT:                    "range": {
+// CHECK-NEXT:                     "begin": {
+// CHECK-NEXT:                      "offset": 147,
+// CHECK-NEXT:                      "col": 16,
+// CHECK-NEXT:                      "tokLen": 1
+// CHECK-NEXT:                     },
+// CHECK-NEXT:                     "end": {
+// CHECK-NEXT:                      "offset": 152,
+// CHECK-NEXT:                      "col": 21,
+// CHECK-NEXT:                      "tokLen": 1
+// CHECK-NEXT:                     }
+// CHECK-NEXT:                    },
+// CHECK-NEXT:                    "type": {
+// CHECK-NEXT:                     "qualType": "const A"
+// CHECK-NEXT:                    },
+// CHECK-NEXT:                    "valueCategory": "prvalue",
+// CHECK-NEXT:                    "castKind": "NoOp",
+// CHECK-NEXT:                    "inner": [
+// CHECK-NEXT:                     {
+// CHECK-NEXT:                      "id": "0x{{.*}}",
+// CHECK-NEXT:                      "kind": "CXXFunctionalCastExpr",
+// CHECK-NEXT:                      "range": {
+// CHECK-NEXT:                       "begin": {
+// CHECK-NEXT:                        "offset": 147,
+// CHECK-NEXT:                        "col": 16,
+// CHECK-NEXT:                        "tokLen": 1
+// CHECK-NEXT:                       },
+// CHECK-NEXT:                       "end": {
+// CHECK-NEXT:                        "offset": 152,
+// CHECK-NEXT:                        "col": 21,
+// CHECK-NEXT:                        "tokLen": 1
+// CHECK-NEXT:                       }
+// CHECK-NEXT:                      },
+// CHECK-NEXT:                      "type": {
+// CHECK-NEXT:                       "qualType": "A"
+// CHECK-NEXT:                      },
+// CHECK-NEXT:                      "valueCategory": "prvalue",
+// CHECK-NEXT:                      "castKind": "NoOp",
+// CHECK-NEXT:                      "inner": [
+// CHECK-NEXT:                       {
+// CHECK-NEXT:                        "id": "0x{{.*}}",
+// CHECK-NEXT:                        "kind": "InitListExpr",
+// CHECK-NEXT:                        "range": {
+// CHECK-NEXT:                         "begin": {
+// CHECK-NEXT:                          "offset": 148,
+// CHECK-NEXT:                          "col": 17,
+// CHECK-NEXT:                          "tokLen": 1
+// CHECK-NEXT:                         },
+// CHECK-NEXT:                         "end": {
+// CHECK-NEXT:                          "offset": 152,
+// CHECK-NEXT:                          "col": 21,
+// CHECK-NEXT:                          "tokLen": 1
+// CHECK-NEXT:                         }
+// CHECK-NEXT:                        },
+// CHECK-NEXT:                        "type": {
+// CHECK-NEXT:                         "qualType": "A"
+// CHECK-NEXT:                        },
+// CHECK-NEXT:                        "valueCategory": "prvalue",
+// CHECK-NEXT:                        "inner": [
+// CHECK-NEXT:                         {
+// CHECK-NEXT:                          "id": "0x{{.*}}",
+// CHECK-NEXT:                          "kind": "InitListExpr",
+// CHECK-NEXT:                          "range": {
+// CHECK-NEXT:                           "begin": {
+// CHECK-NEXT:                            "offset": 149,
+// CHECK-NEXT:                            "col": 18,
+// CHECK-NEXT:                            "tokLen": 1
+// CHECK-NEXT:                           },
+// CHECK-NEXT:                           "end": {
+// CHECK-NEXT:                            "offset": 151,
+// CHECK-NEXT:                            "col": 20,
+// CHECK-NEXT:                            "tokLen": 1
+// CHECK-NEXT:                           }
+// CHECK-NEXT:                          },
+// CHECK-NEXT:                          "type": {
+// CHECK-NEXT:                           "qualType": "int[1]"
+// CHECK-NEXT:                          },
+// CHECK-NEXT:                          "valueCategory": "prvalue",
+// CHECK-NEXT:                          "inner": [
+// CHECK-NEXT:                           {
+// CHECK-NEXT:                            "id": "0x{{.*}}",
+// CHECK-NEXT:                            "kind": "IntegerLiteral",
+// CHECK-NEXT:                            "range": {
+// CHECK-NEXT:                             "begin": {
+// CHECK-NEXT:                              "offset": 150,
+// CHECK-NEXT:                              "col": 19,
+// CHECK-NEXT:                              "tokLen": 1
+// CHECK-NEXT:                             },
+// CHECK-NEXT:                             "end": {
+// CHECK-NEXT:                              "offset": 150,
+// CHECK-NEXT:                              "col": 19,
+// CHECK-NEXT:                              "tokLen": 1
+// CHECK-NEXT:                             }
+// CHECK-NEXT:                            },
+// CHECK-NEXT:                            "type": {
+// CHECK-NEXT:                             "qualType": "int"
+// CHECK-NEXT:                            },
+// CHECK-NEXT:                            "valueCategory": "prvalue",
+// CHECK-NEXT:                            "value": "0"
+// CHECK-NEXT:                           }
+// CHECK-NEXT:                          ]
+// CHECK-NEXT:                         }
+// CHECK-NEXT:                        ]
+// CHECK-NEXT:                       }
+// CHECK-NEXT:                      ]
+// CHECK-NEXT:                     }
+// CHECK-NEXT:                    ]
+// CHECK-NEXT:                   }
+// CHECK-NEXT:                  ]
+// CHECK-NEXT:                 }
+// CHECK-NEXT:                ]
+// CHECK-NEXT:               }
+// CHECK-NEXT:              ]
+// CHECK-NEXT:             }
+// CHECK-NEXT:            ]
+// CHECK-NEXT:           }
+// CHECK-NEXT:          ]
+// CHECK-NEXT:         }
+// CHECK-NEXT:        ]
+// CHECK-NEXT:       }
+// CHECK-NEXT:      ]
+// CHECK-NEXT:     }
+// CHECK-NEXT:    ]
+// CHECK-NEXT:   }
+// CHECK-NEXT:  ]
+// CHECK-NEXT: }
diff --git a/clang/test/AST/ast-dump-default-init.cpp b/clang/test/AST/ast-dump-default-init.cpp
new file mode 100644
index 000000000000..9fe945ee6e93
--- /dev/null
+++ b/clang/test/AST/ast-dump-default-init.cpp
@@ -0,0 +1,21 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-unknown -fsyntax-only -ast-dump %s | FileCheck %s
+
+struct A {
+  int arr[1];
+};
+
+struct B {
+  const A &a = A{{0}};
+};
+
+void test() {
+  B b{};
+}
+// CHECK: -CXXDefaultInitExpr 0x{{[^ ]*}} <{{.*}}> 'const A' lvalue has rewritten init
+// CHECK-NEXT:  `-ExprWithCleanups 0x{{[^ ]*}} <{{.*}}> 'const A' lvalue
+// CHECK-NEXT:    `-MaterializeTemporaryExpr 0x{{[^ ]*}} <{{.*}}> 'const A' lvalue extended by Field 0x{{[^ ]*}} 'a' 'const A &'
+// CHECK-NEXT:      `-ImplicitCastExpr 0x{{[^ ]*}} <{{.*}}> 'const A' <NoOp>
+// CHECK-NEXT:        `-CXXFunctionalCastExpr 0x{{[^ ]*}} <{{.*}}> 'A' functional cast to A <NoOp>
+// CHECK-NEXT:          `-InitListExpr 0x{{[^ ]*}} <{{.*}}> 'A'
+// CHECK-NEXT:            `-InitListExpr 0x{{[^ ]*}} <{{.*}}> 'int[1]'
+// CHECK-NEXT:              `-IntegerLiteral 0x{{[^ ]*}} <{{.*}}> 'int' 0
diff --git a/clang/test/AST/ast-dump-for-range-lifetime.cpp b/clang/test/AST/ast-dump-for-range-lifetime.cpp
index 88b838268be2..0e92b6990ed5 100644
--- a/clang/test/AST/ast-dump-for-range-lifetime.cpp
+++ b/clang/test/AST/ast-dump-for-range-lifetime.cpp
@@ -132,6 +132,9 @@ void test4() {
   // CHECK-NEXT:  |       | `-DeclRefExpr {{.*}} 'int (&(const A &))[3]' lvalue Function {{.*}} 'default_arg_fn' 'int (&(const A &))[3]'
   // CHECK-NEXT:  |       `-CXXDefaultArgExpr {{.*}} <<invalid sloc>> 'const A':'const P2718R0::A' lvalue has rewritten init
   // CHECK-NEXT:  |         `-MaterializeTemporaryExpr {{.*}} 'const A':'const P2718R0::A' lvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
+  // CHECK-NEXT:  |           `-ImplicitCastExpr {{.*}} 'const A':'const P2718R0::A' <NoOp>
+  // CHECK-NEXT:  |             `-CXXBindTemporaryExpr {{.*}} 'A':'P2718R0::A' (CXXTemporary {{.*}})
+  // CHECK-NEXT:  |               `-CXXTemporaryObjectExpr {{.*}} 'A':'P2718R0::A' 'void ()'
   for (auto e : default_arg_fn()) 
     bar(e);
 }
@@ -179,10 +182,19 @@ void test5() {
   // CHECK-NEXT:  |               |       |       |     `-CXXTemporaryObjectExpr {{.*}} 'A':'P2718R0::A' 'void ()'
   // CHECK-NEXT:  |               |       |       `-CXXDefaultArgExpr {{.*}} <<invalid sloc>> 'const DefaultA':'const P2718R0::DefaultA' lvalue has rewritten init
   // CHECK-NEXT:  |               |       |         `-MaterializeTemporaryExpr {{.*}} 'const DefaultA':'const P2718R0::DefaultA' lvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
+  // CHECK-NEXT:  |               |       |           `-ImplicitCastExpr {{.*}} 'const DefaultA':'const P2718R0::DefaultA' <NoOp>
+  // CHECK-NEXT:  |               |       |             `-CXXBindTemporaryExpr {{.*}} 'DefaultA':'P2718R0::DefaultA' (CXXTemporary {{.*}})
+  // CHECK-NEXT:  |               |       |               `-CXXTemporaryObjectExpr {{.*}} 'DefaultA':'P2718R0::DefaultA' 'void ()'
   // CHECK-NEXT:  |               |       `-CXXDefaultArgExpr {{.*}} <<invalid sloc>> 'const DefaultA':'const P2718R0::DefaultA' lvalue has rewritten init
   // CHECK-NEXT:  |               |         `-MaterializeTemporaryExpr {{.*}} 'const DefaultA':'const P2718R0::DefaultA' lvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
+  // CHECK-NEXT:  |               |           `-ImplicitCastExpr {{.*}} 'const DefaultA':'const P2718R0::DefaultA' <NoOp>
+  // CHECK-NEXT:  |               |             `-CXXBindTemporaryExpr {{.*}} 'DefaultA':'P2718R0::DefaultA' (CXXTemporary {{.*}})
+  // CHECK-NEXT:  |               |               `-CXXTemporaryObjectExpr {{.*}} 'DefaultA':'P2718R0::DefaultA' 'void ()'
   // CHECK-NEXT:  |               `-CXXDefaultArgExpr {{.*}} <<invalid sloc>> 'const DefaultA':'const P2718R0::DefaultA' lvalue has rewritten init
   // CHECK-NEXT:  |                 `-MaterializeTemporaryExpr {{.*}} 'const DefaultA':'const P2718R0::DefaultA' lvalue extended by Var {{.*}} '__range1' 'int (&)[3]'
+  // CHECK-NEXT:  |                   `-ImplicitCastExpr {{.*}} 'const DefaultA':'const P2718R0::DefaultA' <NoOp>
+  // CHECK-NEXT:  |                     `-CXXBindTemporaryExpr {{.*}} 'DefaultA':'P2718R0::DefaultA' (CXXTemporary {{.*}})
+  // CHECK-NEXT:  |                       `-CXXTemporaryObjectExpr {{.*}} 'DefaultA':'P2718R0::DefaultA' 'void ()'
   for (auto e : default_arg_fn(foo(foo(foo(A())))))
     bar(e);
 }
@@ -219,10 +231,19 @@ void test6() {
   // CHECK-NEXT:  |           |       |       |     `-CXXTemporaryObjectExpr {{.*}} 'C':'P2718R0::C' 'void ()'
   // CHECK-NEXT:  |           |       |       `-CXXDefaultArgExpr {{.*}} <<invalid sloc>> 'const DefaultA':'const P2718R0::DefaultA' lvalue has rewritten init
   // CHECK-NEXT:  |           |       |         `-MaterializeTemporaryExpr {{.*}} 'const DefaultA':'const P2718R0::DefaultA' lvalue extended by Var {{.*}} '__range1' 'C &&'
+  // CHECK-NEXT:  |           |       |           `-ImplicitCastExpr {{.*}} 'const DefaultA':'const P2718R0::DefaultA' <NoOp>
+  // CHECK-NEXT:  |           |       |             `-CXXBindTemporaryExpr {{.*}} 'DefaultA':'P2718R0::DefaultA' (CXXTemporary {{.*}})
+  // CHECK-NEXT:  |           |       |               `-CXXTemporaryObjectExpr {{.*}} 'DefaultA':'P2718R0::DefaultA' 'void ()'
   // CHECK-NEXT:  |           |       `-CXXDefaultArgExpr {{.*}} <<invalid sloc>> 'const DefaultA':'const P2718R0::DefaultA' lvalue has rewritten init
   // CHECK-NEXT:  |           |         `-MaterializeTemporaryExpr {{.*}} 'const DefaultA':'const P2718R0::DefaultA' lvalue extended by Var {{.*}} '__range1' 'C &&'
+  // CHECK-NEXT:  |           |           `-ImplicitCastExpr {{.*}} 'const DefaultA':'const P2718R0::DefaultA' <NoOp>
+  // CHECK-NEXT:  |           |             `-CXXBindTemporaryExpr {{.*}} 'DefaultA':'P2718R0::DefaultA' (CXXTemporary {{.*}})
+  // CHECK-NEXT:  |           |               `-CXXTemporaryObjectExpr {{.*}} 'DefaultA':'P2718R0::DefaultA' 'void ()'
   // CHECK-NEXT:  |           `-CXXDefaultArgExpr {{.*}} <<invalid sloc>> 'const DefaultA':'const P2718R0::DefaultA' lvalue has rewritten init
   // CHECK-NEXT:  |             `-MaterializeTemporaryExpr {{.*}} 'const DefaultA':'const P2718R0::DefaultA' lvalue extended by Var {{.*}} '__range1' 'C &&'
+  // CHECK-NEXT:  |               `-ImplicitCastExpr {{.*}} 'const DefaultA':'const P2718R0::DefaultA' <NoOp>
+  // CHECK-NEXT:  |                 `-CXXBindTemporaryExpr {{.*}} 'DefaultA':'P2718R0::DefaultA' (CXXTemporary {{.*}})
+  // CHECK-NEXT:  |                   `-CXXTemporaryObjectExpr {{.*}} 'DefaultA':'P2718R0::DefaultA' 'void ()'
   for (auto e : C(0, C(0, C(0, C()))))
     bar(e);
 }
diff --git a/clang/unittests/AST/ASTTraverserTest.cpp b/clang/unittests/AST/ASTTraverserTest.cpp
index 362f37655a0a..8b6e3e90c0ea 100644
--- a/clang/unittests/AST/ASTTraverserTest.cpp
+++ b/clang/unittests/AST/ASTTraverserTest.cpp
@@ -368,6 +368,8 @@ FunctionDecl 'stringConstruct'
   |             |-ImplicitCastExpr
   |             | `-StringLiteral
   |             `-CXXDefaultArgExpr
+  |               `-UnaryOperator
+  |                 `-IntegerLiteral
   `-ExprWithCleanups
     `-CXXOperatorCallExpr
       |-ImplicitCastExpr
@@ -378,6 +380,8 @@ FunctionDecl 'stringConstruct'
           |-ImplicitCastExpr
           | `-StringLiteral
           `-CXXDefaultArgExpr
+            `-UnaryOperator
+              `-IntegerLiteral
 )cpp");
 
     EXPECT_EQ(dumpASTString(TK_IgnoreUnlessSpelledInSource,
@@ -415,6 +419,8 @@ FunctionDecl 'overloadCall'
   |             |-ImplicitCastExpr
   |             | `-StringLiteral
   |             `-CXXDefaultArgExpr
+  |               `-UnaryOperator
+  |                 `-IntegerLiteral
   `-CXXMemberCallExpr
     `-MemberExpr
       `-ParenExpr
@@ -1219,6 +1225,7 @@ CXXRecordDecl 'Record'
 | | `-IntegerLiteral
 | |-CXXCtorInitializer 'm_i2'
 | | `-CXXDefaultInitExpr
+| |   `-IntegerLiteral
 | |-CXXCtorInitializer 'm_s'
 | | `-CXXConstructExpr
 | `-CompoundStmt
@@ -1485,6 +1492,7 @@ CallExpr
 | `-DeclRefExpr 'hasDefaultArg'
 |-IntegerLiteral
 `-CXXDefaultArgExpr
+  `-IntegerLiteral
 )cpp");
     EXPECT_EQ(dumpASTString(TK_IgnoreUnlessSpelledInSource,
                             BN[0].getNodeAs<CallExpr>("funcCall")),
-- 
GitLab


From a27ab3f54377a0addeef1ec9ba7c826fd885a304 Mon Sep 17 00:00:00 2001
From: Dominik Adamski <dominik.adamski@amd.com>
Date: Fri, 12 Apr 2024 16:41:34 +0200
Subject: [PATCH 074/464] [Flang][AMDGPU] Add rocm-path flag (#88190)

ROCm installation path is used for finding and automatically linking
required bitcode libraries for OpenMP AMDGPU offload.

Reported issue: https://github.com/llvm/llvm-project/issues/82553
---
 clang/include/clang/Driver/Options.td                       | 3 ++-
 .../Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/asanrtl.bc | 0
 .../Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/hip.bc     | 0
 .../Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/ockl.bc    | 0
 .../bitcode-no-abi-ver/oclc_correctly_rounded_sqrt_off.bc   | 0
 .../bitcode-no-abi-ver/oclc_correctly_rounded_sqrt_on.bc    | 0
 .../rocm/amdgcn/bitcode-no-abi-ver/oclc_daz_opt_off.bc      | 0
 .../rocm/amdgcn/bitcode-no-abi-ver/oclc_daz_opt_on.bc       | 0
 .../rocm/amdgcn/bitcode-no-abi-ver/oclc_finite_only_off.bc  | 0
 .../rocm/amdgcn/bitcode-no-abi-ver/oclc_finite_only_on.bc   | 0
 .../rocm/amdgcn/bitcode-no-abi-ver/oclc_isa_version_1010.bc | 0
 .../rocm/amdgcn/bitcode-no-abi-ver/oclc_isa_version_1011.bc | 0
 .../rocm/amdgcn/bitcode-no-abi-ver/oclc_isa_version_1012.bc | 0
 .../rocm/amdgcn/bitcode-no-abi-ver/oclc_isa_version_803.bc  | 0
 .../rocm/amdgcn/bitcode-no-abi-ver/oclc_isa_version_900.bc  | 0
 .../rocm/amdgcn/bitcode-no-abi-ver/oclc_isa_version_908.bc  | 0
 .../rocm/amdgcn/bitcode-no-abi-ver/oclc_unsafe_math_off.bc  | 0
 .../rocm/amdgcn/bitcode-no-abi-ver/oclc_unsafe_math_on.bc   | 0
 .../amdgcn/bitcode-no-abi-ver/oclc_wavefrontsize64_off.bc   | 0
 .../amdgcn/bitcode-no-abi-ver/oclc_wavefrontsize64_on.bc    | 0
 .../Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/ocml.bc    | 0
 .../Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/opencl.bc  | 0
 flang/test/Driver/Inputs/rocm/amdgcn/bitcode/asanrtl.bc     | 0
 flang/test/Driver/Inputs/rocm/amdgcn/bitcode/hip.bc         | 0
 flang/test/Driver/Inputs/rocm/amdgcn/bitcode/ockl.bc        | 0
 .../Inputs/rocm/amdgcn/bitcode/oclc_abi_version_400.bc      | 0
 .../Inputs/rocm/amdgcn/bitcode/oclc_abi_version_500.bc      | 0
 .../Inputs/rocm/amdgcn/bitcode/oclc_abi_version_600.bc      | 0
 .../rocm/amdgcn/bitcode/oclc_correctly_rounded_sqrt_off.bc  | 0
 .../rocm/amdgcn/bitcode/oclc_correctly_rounded_sqrt_on.bc   | 0
 .../Driver/Inputs/rocm/amdgcn/bitcode/oclc_daz_opt_off.bc   | 0
 .../Driver/Inputs/rocm/amdgcn/bitcode/oclc_daz_opt_on.bc    | 0
 .../Inputs/rocm/amdgcn/bitcode/oclc_finite_only_off.bc      | 0
 .../Inputs/rocm/amdgcn/bitcode/oclc_finite_only_on.bc       | 0
 .../Inputs/rocm/amdgcn/bitcode/oclc_isa_version_1010.bc     | 0
 .../Inputs/rocm/amdgcn/bitcode/oclc_isa_version_1011.bc     | 0
 .../Inputs/rocm/amdgcn/bitcode/oclc_isa_version_1012.bc     | 0
 .../Inputs/rocm/amdgcn/bitcode/oclc_isa_version_803.bc      | 0
 .../Inputs/rocm/amdgcn/bitcode/oclc_isa_version_900.bc      | 0
 .../Inputs/rocm/amdgcn/bitcode/oclc_isa_version_908.bc      | 0
 .../Inputs/rocm/amdgcn/bitcode/oclc_unsafe_math_off.bc      | 0
 .../Inputs/rocm/amdgcn/bitcode/oclc_unsafe_math_on.bc       | 0
 .../Inputs/rocm/amdgcn/bitcode/oclc_wavefrontsize64_off.bc  | 0
 .../Inputs/rocm/amdgcn/bitcode/oclc_wavefrontsize64_on.bc   | 0
 flang/test/Driver/Inputs/rocm/amdgcn/bitcode/ocml.bc        | 0
 flang/test/Driver/Inputs/rocm/amdgcn/bitcode/opencl.bc      | 0
 flang/test/Driver/Inputs/rocm/bin/.hipVersion               | 6 ++++++
 flang/test/Driver/Inputs/rocm/include/hip/hip_runtime.h     | 0
 flang/test/Driver/Inputs/rocm/share/hip/hipVersion          | 6 ++++++
 flang/test/Driver/driver-help-hidden.f90                    | 1 +
 flang/test/Driver/driver-help.f90                           | 1 +
 flang/test/Driver/omp-driver-offload.f90                    | 6 ++++++
 52 files changed, 22 insertions(+), 1 deletion(-)
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/asanrtl.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/hip.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/ockl.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_correctly_rounded_sqrt_off.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_correctly_rounded_sqrt_on.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_daz_opt_off.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_daz_opt_on.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_finite_only_off.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_finite_only_on.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_isa_version_1010.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_isa_version_1011.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_isa_version_1012.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_isa_version_803.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_isa_version_900.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_isa_version_908.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_unsafe_math_off.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_unsafe_math_on.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_wavefrontsize64_off.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_wavefrontsize64_on.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/ocml.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/opencl.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode/asanrtl.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode/hip.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode/ockl.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_abi_version_400.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_abi_version_500.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_abi_version_600.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_correctly_rounded_sqrt_off.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_correctly_rounded_sqrt_on.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_daz_opt_off.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_daz_opt_on.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_finite_only_off.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_finite_only_on.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_isa_version_1010.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_isa_version_1011.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_isa_version_1012.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_isa_version_803.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_isa_version_900.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_isa_version_908.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_unsafe_math_off.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_unsafe_math_on.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_wavefrontsize64_off.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_wavefrontsize64_on.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode/ocml.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/amdgcn/bitcode/opencl.bc
 create mode 100644 flang/test/Driver/Inputs/rocm/bin/.hipVersion
 create mode 100644 flang/test/Driver/Inputs/rocm/include/hip/hip_runtime.h
 create mode 100644 flang/test/Driver/Inputs/rocm/share/hip/hipVersion

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 400f72f2250a..9a0b5d3304ca 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -1341,7 +1341,8 @@ def hip_link : Flag<["--"], "hip-link">, Group<opencl_Group>,
   HelpText<"Link clang-offload-bundler bundles for HIP">;
 def no_hip_rt: Flag<["-"], "no-hip-rt">, Group<hip_Group>,
   HelpText<"Do not link against HIP runtime libraries">;
-def rocm_path_EQ : Joined<["--"], "rocm-path=">, Group<hip_Group>,
+def rocm_path_EQ : Joined<["--"], "rocm-path=">,
+  Visibility<[FlangOption]>, Group<hip_Group>,
   HelpText<"ROCm installation path, used for finding and automatically linking required bitcode libraries.">;
 def hip_path_EQ : Joined<["--"], "hip-path=">, Group<hip_Group>,
   HelpText<"HIP runtime installation path, used for finding HIP version and adding HIP include path.">;
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/asanrtl.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/asanrtl.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/hip.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/hip.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/ockl.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/ockl.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_correctly_rounded_sqrt_off.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_correctly_rounded_sqrt_off.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_correctly_rounded_sqrt_on.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_correctly_rounded_sqrt_on.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_daz_opt_off.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_daz_opt_off.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_daz_opt_on.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_daz_opt_on.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_finite_only_off.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_finite_only_off.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_finite_only_on.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_finite_only_on.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_isa_version_1010.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_isa_version_1010.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_isa_version_1011.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_isa_version_1011.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_isa_version_1012.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_isa_version_1012.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_isa_version_803.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_isa_version_803.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_isa_version_900.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_isa_version_900.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_isa_version_908.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_isa_version_908.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_unsafe_math_off.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_unsafe_math_off.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_unsafe_math_on.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_unsafe_math_on.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_wavefrontsize64_off.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_wavefrontsize64_off.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_wavefrontsize64_on.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/oclc_wavefrontsize64_on.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/ocml.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/ocml.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/opencl.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode-no-abi-ver/opencl.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/asanrtl.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/asanrtl.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/hip.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/hip.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/ockl.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/ockl.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_abi_version_400.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_abi_version_400.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_abi_version_500.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_abi_version_500.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_abi_version_600.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_abi_version_600.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_correctly_rounded_sqrt_off.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_correctly_rounded_sqrt_off.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_correctly_rounded_sqrt_on.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_correctly_rounded_sqrt_on.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_daz_opt_off.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_daz_opt_off.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_daz_opt_on.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_daz_opt_on.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_finite_only_off.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_finite_only_off.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_finite_only_on.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_finite_only_on.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_isa_version_1010.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_isa_version_1010.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_isa_version_1011.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_isa_version_1011.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_isa_version_1012.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_isa_version_1012.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_isa_version_803.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_isa_version_803.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_isa_version_900.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_isa_version_900.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_isa_version_908.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_isa_version_908.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_unsafe_math_off.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_unsafe_math_off.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_unsafe_math_on.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_unsafe_math_on.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_wavefrontsize64_off.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_wavefrontsize64_off.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_wavefrontsize64_on.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/oclc_wavefrontsize64_on.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/ocml.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/ocml.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/opencl.bc b/flang/test/Driver/Inputs/rocm/amdgcn/bitcode/opencl.bc
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/bin/.hipVersion b/flang/test/Driver/Inputs/rocm/bin/.hipVersion
new file mode 100644
index 000000000000..677293c09139
--- /dev/null
+++ b/flang/test/Driver/Inputs/rocm/bin/.hipVersion
@@ -0,0 +1,6 @@
+# Auto-generated by cmake
+# NOTE: The trailing whitespace is added on purpose to verify that these
+# whitespaces are trimmed before paring.
+HIP_VERSION_MAJOR=3 
+HIP_VERSION_MINOR=6
+HIP_VERSION_PATCH=20214-a2917cd
diff --git a/flang/test/Driver/Inputs/rocm/include/hip/hip_runtime.h b/flang/test/Driver/Inputs/rocm/include/hip/hip_runtime.h
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/flang/test/Driver/Inputs/rocm/share/hip/hipVersion b/flang/test/Driver/Inputs/rocm/share/hip/hipVersion
new file mode 100644
index 000000000000..677293c09139
--- /dev/null
+++ b/flang/test/Driver/Inputs/rocm/share/hip/hipVersion
@@ -0,0 +1,6 @@
+# Auto-generated by cmake
+# NOTE: The trailing whitespace is added on purpose to verify that these
+# whitespaces are trimmed before paring.
+HIP_VERSION_MAJOR=3 
+HIP_VERSION_MINOR=6
+HIP_VERSION_PATCH=20214-a2917cd
diff --git a/flang/test/Driver/driver-help-hidden.f90 b/flang/test/Driver/driver-help-hidden.f90
index eb00949c2a95..de2fe3048f99 100644
--- a/flang/test/Driver/driver-help-hidden.f90
+++ b/flang/test/Driver/driver-help-hidden.f90
@@ -148,6 +148,7 @@
 ! CHECK-NEXT: -pthread                Support POSIX threads in generated code
 ! CHECK-NEXT: -P                      Disable linemarker output in -E mode
 ! CHECK-NEXT: -resource-dir <value>   The directory which holds the compiler resource files
+! CHECK-NEXT: --rocm-path=<value> ROCm installation path, used for finding and automatically linking required bitcode libraries.
 ! CHECK-NEXT: -Rpass-analysis=<value> Report transformation analysis from optimization passes whose name matches the given POSIX regular expression
 ! CHECK-NEXT: -Rpass-missed=<value>   Report missed transformations by optimization passes whose name matches the given POSIX regular expression
 ! CHECK-NEXT: -Rpass=<value>          Report transformations performed by optimization passes whose name matches the given POSIX regular expression
diff --git a/flang/test/Driver/driver-help.f90 b/flang/test/Driver/driver-help.f90
index 05f50ea348c2..b258eb59c186 100644
--- a/flang/test/Driver/driver-help.f90
+++ b/flang/test/Driver/driver-help.f90
@@ -135,6 +135,7 @@
 ! HELP-NEXT: -print-target-triple    Print the normalized target triple
 ! HELP-NEXT: -pthread                Support POSIX threads in generated code
 ! HELP-NEXT: -P                      Disable linemarker output in -E mode
+! HELP-NEXT:  --rocm-path=<value> ROCm installation path, used for finding and automatically linking required bitcode libraries.
 ! HELP-NEXT: -Rpass-analysis=<value> Report transformation analysis from optimization passes whose name matches the given POSIX regular expression
 ! HELP-NEXT: -Rpass-missed=<value>   Report missed transformations by optimization passes whose name matches the given POSIX regular expression
 ! HELP-NEXT: -Rpass=<value>          Report transformations performed by optimization passes whose name matches the given POSIX regular expression
diff --git a/flang/test/Driver/omp-driver-offload.f90 b/flang/test/Driver/omp-driver-offload.f90
index 7e9a73627cd7..8f48ca75114c 100644
--- a/flang/test/Driver/omp-driver-offload.f90
+++ b/flang/test/Driver/omp-driver-offload.f90
@@ -201,3 +201,9 @@
 ! RUN:      -nogpulibc %s 2>&1 \
 ! RUN:   | FileCheck --check-prefix=NO-LIBC-GPU-AMDGPU %s
 ! NO-LIBC-GPU-AMDGPU-NOT: "-lcgpu-amdgpu"
+
+! RUN:   %flang -### -v --target=x86_64-unknown-linux-gnu -fopenmp  \
+! RUN:      --offload-arch=gfx900 \
+! RUN:      --rocm-path=%S/Inputs/rocm %s 2>&1 \
+! RUN:   | FileCheck --check-prefix=ROCM-PATH %s
+! ROCM-PATH: Found HIP installation: {{.*Inputs.*rocm}}, version 3.6.20214-a2917cd
-- 
GitLab


From 5752e3196bc52fdac70e3650abc703570ff6209b Mon Sep 17 00:00:00 2001
From: Vincent Belliard <81770341+v-bulle@users.noreply.github.com>
Date: Fri, 12 Apr 2024 07:48:41 -0700
Subject: [PATCH 075/464] [lldb] fix dead lock in TypeCategoryMap.cpp (#87540)

FormatManager::GetCategoryForLanguage and
FormatManager::GetCategory(can_create = true) can be called concurrently
and they both take the TypeCategory::m_map_mutex and the
FormatManager::m_language_categories_mutex but in reverse order.

On one thread, GetCategoryForLanguage takes m_language_categories_mutex
and then ends calling TypeCategoryMap::Get which takes m_map_mutex

On another thread GetCategory calls TypeCategoryMap::Add which takes
m_map_mutex and then calls FormatManager::Changed() which takes
m_language_categories_mutex

If both threads are running concurrently, we have a dead lock.

The patch releases the m_map_mutex before calling Changed which avoids
the dead lock.

---------

Co-authored-by: Vincent Belliard <v-bulle@github.com>
---
 .../source/DataFormatters/TypeCategoryMap.cpp | 40 ++++++++++++++-----
 1 file changed, 29 insertions(+), 11 deletions(-)

diff --git a/lldb/source/DataFormatters/TypeCategoryMap.cpp b/lldb/source/DataFormatters/TypeCategoryMap.cpp
index fd76bab95826..ce2cf369b5be 100644
--- a/lldb/source/DataFormatters/TypeCategoryMap.cpp
+++ b/lldb/source/DataFormatters/TypeCategoryMap.cpp
@@ -25,19 +25,31 @@ TypeCategoryMap::TypeCategoryMap(IFormatChangeListener *lst)
 }
 
 void TypeCategoryMap::Add(KeyType name, const TypeCategoryImplSP &entry) {
-  std::lock_guard<std::recursive_mutex> guard(m_map_mutex);
-  m_map[name] = entry;
+  {
+    std::lock_guard<std::recursive_mutex> guard(m_map_mutex);
+    m_map[name] = entry;
+  }
+  // Release the mutex to avoid a potential deadlock between
+  // TypeCategoryMap::m_map_mutex and
+  // FormatManager::m_language_categories_mutex which can be acquired in
+  // reverse order when calling FormatManager::Changed.
   if (listener)
     listener->Changed();
 }
 
 bool TypeCategoryMap::Delete(KeyType name) {
-  std::lock_guard<std::recursive_mutex> guard(m_map_mutex);
-  MapIterator iter = m_map.find(name);
-  if (iter == m_map.end())
-    return false;
-  m_map.erase(name);
-  Disable(name);
+  {
+    std::lock_guard<std::recursive_mutex> guard(m_map_mutex);
+    MapIterator iter = m_map.find(name);
+    if (iter == m_map.end())
+      return false;
+    m_map.erase(name);
+    Disable(name);
+  }
+  // Release the mutex to avoid a potential deadlock between
+  // TypeCategoryMap::m_map_mutex and
+  // FormatManager::m_language_categories_mutex which can be acquired in
+  // reverse order when calling FormatManager::Changed.
   if (listener)
     listener->Changed();
   return true;
@@ -123,9 +135,15 @@ void TypeCategoryMap::DisableAllCategories() {
 }
 
 void TypeCategoryMap::Clear() {
-  std::lock_guard<std::recursive_mutex> guard(m_map_mutex);
-  m_map.clear();
-  m_active_categories.clear();
+  {
+    std::lock_guard<std::recursive_mutex> guard(m_map_mutex);
+    m_map.clear();
+    m_active_categories.clear();
+  }
+  // Release the mutex to avoid a potential deadlock between
+  // TypeCategoryMap::m_map_mutex and
+  // FormatManager::m_language_categories_mutex which can be acquired in
+  // reverse order when calling FormatManager::Changed.
   if (listener)
     listener->Changed();
 }
-- 
GitLab


From 4dd20b0728223ff20f4817edc2f89901385cb990 Mon Sep 17 00:00:00 2001
From: Nathan Sidwell <nathan@acm.org>
Date: Fri, 12 Apr 2024 11:35:23 -0400
Subject: [PATCH 076/464] [BOLT][NFC] Refactor relocation loop (#88424)

Use the std `if () continue;` idiom before falling into the
processing.
---
 bolt/lib/Rewrite/RewriteInstance.cpp | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index cbbe7d00d35d..f22bede002da 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -2306,9 +2306,13 @@ void RewriteInstance::processRelocations() {
     return;
 
   for (const SectionRef &Section : InputFile->sections()) {
-    if (cantFail(Section.getRelocatedSection()) != InputFile->section_end() &&
-        !BinarySection(*BC, Section).isAllocatable())
-      readRelocations(Section);
+    section_iterator SecIter = cantFail(Section.getRelocatedSection());
+    if (SecIter == InputFile->section_end())
+      continue;
+    if (BinarySection(*BC, Section).isAllocatable())
+      continue;
+
+    readRelocations(Section);
   }
 
   if (NumFailedRelocations)
-- 
GitLab


From 040efafa9fef101924d3cf67db20f6429ce1c871 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Fri, 12 Apr 2024 08:45:03 -0700
Subject: [PATCH 077/464] [RISCV] Support uimm32 immediates in
 RISCVInstrInfo::movImm for RV32. (#88464)

This allows us to support larger stack offsets for FrameLowering.

Fixes #88365.
---
 llvm/lib/Target/RISCV/RISCVInstrInfo.cpp | 11 ++++++++--
 llvm/test/CodeGen/RISCV/pr88365.ll       | 26 ++++++++++++++++++++++++
 2 files changed, 35 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/CodeGen/RISCV/pr88365.ll

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index 004a228ccc13..508f607fab20 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -767,8 +767,15 @@ void RISCVInstrInfo::movImm(MachineBasicBlock &MBB,
                             bool DstIsDead) const {
   Register SrcReg = RISCV::X0;
 
-  if (!STI.is64Bit() && !isInt<32>(Val))
-    report_fatal_error("Should only materialize 32-bit constants for RV32");
+  // For RV32, allow a sign or unsigned 32 bit value.
+  if (!STI.is64Bit() && !isInt<32>(Val)) {
+    // If have a uimm32 it will still fit in a register so we can allow it.
+    if (!isUInt<32>(Val))
+      report_fatal_error("Should only materialize 32-bit constants for RV32");
+
+    // Sign extend for generateInstSeq.
+    Val = SignExtend64<32>(Val);
+  }
 
   RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val, STI);
   assert(!Seq.empty());
diff --git a/llvm/test/CodeGen/RISCV/pr88365.ll b/llvm/test/CodeGen/RISCV/pr88365.ll
new file mode 100644
index 000000000000..73010fdf4044
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/pr88365.ll
@@ -0,0 +1,26 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc < %s -mtriple=riscv32 | FileCheck %s
+
+define void @foo() {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -2032
+; CHECK-NEXT:    .cfi_def_cfa_offset 2032
+; CHECK-NEXT:    sw ra, 2028(sp) # 4-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset ra, -4
+; CHECK-NEXT:    li a0, -2048
+; CHECK-NEXT:    sub sp, sp, a0
+; CHECK-NEXT:    .cfi_def_cfa_offset -16
+; CHECK-NEXT:    addi a0, sp, 4
+; CHECK-NEXT:    call use
+; CHECK-NEXT:    li a0, -2048
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    lw ra, 2028(sp) # 4-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 2032
+; CHECK-NEXT:    ret
+  %1 = alloca [1073741818 x i32], align 4
+  call void @use(ptr %1)
+  ret void
+}
+
+declare void @use(ptr)
-- 
GitLab


From 72dfee114bd38dc4b424d392bc14cd6b7dfb79e5 Mon Sep 17 00:00:00 2001
From: Francis Visoiu Mistrih <890283+francisvm@users.noreply.github.com>
Date: Fri, 12 Apr 2024 08:45:47 -0700
Subject: [PATCH 078/464] [RISCV] Remove mayLoad = 1 from store-conditional
 (#88470)

lr.[wd]: mayLoad = 1, mayStore = 0
sc.[wd]: mayLoad = 0, mayStore = 1

all other AMOs: mayLoad = 1, mayStore = 1
---
 llvm/lib/Target/RISCV/RISCVInstrInfoA.td | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoA.td b/llvm/lib/Target/RISCV/RISCVInstrInfoA.td
index 36842ceb49bf..814e0ddf111e 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoA.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoA.td
@@ -30,6 +30,19 @@ multiclass LR_r_aq_rl<bits<3> funct3, string opcodestr> {
   def _AQ_RL : LR_r<1, 1, funct3, opcodestr # ".aqrl">;
 }
 
+let hasSideEffects = 0, mayLoad = 0, mayStore = 1 in
+class SC_r<bit aq, bit rl, bits<3> funct3, string opcodestr>
+    : RVInstRAtomic<0b00011, aq, rl, funct3, OPC_AMO,
+                    (outs GPR:$rd), (ins GPRMemZeroOffset:$rs1, GPR:$rs2),
+                    opcodestr, "$rd, $rs2, $rs1">;
+
+multiclass SC_r_aq_rl<bits<3> funct3, string opcodestr> {
+  def ""     : SC_r<0, 0, funct3, opcodestr>;
+  def _AQ    : SC_r<1, 0, funct3, opcodestr # ".aq">;
+  def _RL    : SC_r<0, 1, funct3, opcodestr # ".rl">;
+  def _AQ_RL : SC_r<1, 1, funct3, opcodestr # ".aqrl">;
+}
+
 let hasSideEffects = 0, mayLoad = 1, mayStore = 1 in
 class AMO_rr<bits<5> funct5, bit aq, bit rl, bits<3> funct3, string opcodestr>
     : RVInstRAtomic<funct5, aq, rl, funct3, OPC_AMO,
@@ -49,7 +62,7 @@ multiclass AMO_rr_aq_rl<bits<5> funct5, bits<3> funct3, string opcodestr> {
 
 let Predicates = [HasStdExtAOrZalrsc], IsSignExtendingOpW = 1 in {
 defm LR_W       : LR_r_aq_rl<0b010, "lr.w">, Sched<[WriteAtomicLDW, ReadAtomicLDW]>;
-defm SC_W       : AMO_rr_aq_rl<0b00011, 0b010, "sc.w">,
+defm SC_W       : SC_r_aq_rl<0b010, "sc.w">,
                   Sched<[WriteAtomicSTW, ReadAtomicSTW, ReadAtomicSTW]>;
 } // Predicates = [HasStdExtAOrZalrsc], IsSignExtendingOpW = 1
 
@@ -76,7 +89,7 @@ defm AMOMAXU_W  : AMO_rr_aq_rl<0b11100, 0b010, "amomaxu.w">,
 
 let Predicates = [HasStdExtAOrZalrsc, IsRV64] in {
 defm LR_D       : LR_r_aq_rl<0b011, "lr.d">, Sched<[WriteAtomicLDD, ReadAtomicLDD]>;
-defm SC_D       : AMO_rr_aq_rl<0b00011, 0b011, "sc.d">,
+defm SC_D       : SC_r_aq_rl<0b011, "sc.d">,
                   Sched<[WriteAtomicSTD, ReadAtomicSTD, ReadAtomicSTD]>;
 } // Predicates = [HasStdExtAOrZalrsc, IsRV64]
 
-- 
GitLab


From e0a628715a8464e220c8ba9e9aaaf2561139198a Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333@gmail.com>
Date: Fri, 12 Apr 2024 23:47:20 +0800
Subject: [PATCH 079/464] [ValueTracking] Convert `isKnownNonZero` to use
 SimplifyQuery (#85863)

This patch converts `isKnownNonZero` to use SimplifyQuery. Then we can
use the context information from `DomCondCache`.

Fixes https://github.com/llvm/llvm-project/issues/85823.
Alive2: https://alive2.llvm.org/ce/z/QUvHVj
---
 clang/lib/CodeGen/CGCall.cpp                  |   3 +-
 llvm/include/llvm/Analysis/ValueTracking.h    |   6 +-
 llvm/lib/Analysis/BasicAliasAnalysis.cpp      |   3 +-
 llvm/lib/Analysis/InstructionSimplify.cpp     |  29 ++--
 llvm/lib/Analysis/LazyValueInfo.cpp           |   5 +-
 llvm/lib/Analysis/Loads.cpp                   |   6 +-
 llvm/lib/Analysis/ScalarEvolution.cpp         |   2 +-
 llvm/lib/Analysis/ValueTracking.cpp           |  17 +--
 llvm/lib/CodeGen/CodeGenPrepare.cpp           |   2 +-
 .../Transforms/IPO/AttributorAttributes.cpp   |   5 +-
 llvm/lib/Transforms/IPO/FunctionAttrs.cpp     |   2 +-
 .../InstCombine/InstCombineAddSub.cpp         |   2 +-
 .../InstCombine/InstCombineAndOrXor.cpp       |   8 +-
 .../InstCombine/InstCombineCalls.cpp          |  13 +-
 .../InstCombine/InstCombineCompares.cpp       |  21 ++-
 .../Transforms/InstCombine/InstCombinePHI.cpp |   3 +-
 .../InstCombine/InstructionCombining.cpp      |   2 +-
 .../Instrumentation/MemorySanitizer.cpp       |   4 +-
 .../Utils/PromoteMemoryToRegister.cpp         |   2 +-
 .../lib/Transforms/Utils/SimplifyLibCalls.cpp |  20 +--
 .../Transforms/Vectorize/VectorCombine.cpp    |   3 +-
 llvm/test/Transforms/InstCombine/icmp-dom.ll  | 131 ++++++++++++++++++
 llvm/unittests/Analysis/ValueTrackingTest.cpp |  15 +-
 23 files changed, 213 insertions(+), 91 deletions(-)

diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index 3f5463a9a70e..7a0bc6fa77b8 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -4124,7 +4124,8 @@ static bool isProvablyNull(llvm::Value *addr) {
 }
 
 static bool isProvablyNonNull(Address Addr, CodeGenFunction &CGF) {
-  return llvm::isKnownNonZero(Addr.getBasePointer(), CGF.CGM.getDataLayout());
+  return llvm::isKnownNonZero(Addr.getBasePointer(), /*Depth=*/0,
+                              CGF.CGM.getDataLayout());
 }
 
 /// Emit the actual writing-back of a writeback.
diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h
index 3970efba18cc..9db0894162af 100644
--- a/llvm/include/llvm/Analysis/ValueTracking.h
+++ b/llvm/include/llvm/Analysis/ValueTracking.h
@@ -124,11 +124,7 @@ bool isOnlyUsedInZeroEqualityComparison(const Instruction *CxtI);
 /// specified, perform context-sensitive analysis and return true if the
 /// pointer couldn't possibly be null at the specified instruction.
 /// Supports values with integer or pointer type and vectors of integers.
-bool isKnownNonZero(const Value *V, const DataLayout &DL, unsigned Depth = 0,
-                    AssumptionCache *AC = nullptr,
-                    const Instruction *CxtI = nullptr,
-                    const DominatorTree *DT = nullptr,
-                    bool UseInstrInfo = true);
+bool isKnownNonZero(const Value *V, unsigned Depth, const SimplifyQuery &Q);
 
 /// Return true if the two given values are negation.
 /// Currently can recoginze Value pair:
diff --git a/llvm/lib/Analysis/BasicAliasAnalysis.cpp b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
index 8dfc4b2a1cb1..b082dfe8fbd2 100644
--- a/llvm/lib/Analysis/BasicAliasAnalysis.cpp
+++ b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
@@ -1283,7 +1283,8 @@ AliasResult BasicAAResult::aliasGEP(
     // VarIndex = Scale*V.
     const VariableGEPIndex &Var = DecompGEP1.VarIndices[0];
     if (Var.Val.TruncBits == 0 &&
-        isKnownNonZero(Var.Val.V, DL, 0, &AC, Var.CxtI, DT)) {
+        isKnownNonZero(Var.Val.V, /*Depth=*/0,
+                       SimplifyQuery(DL, DT, &AC, Var.CxtI))) {
       // Check if abs(V*Scale) >= abs(Scale) holds in the presence of
       // potentially wrapping math.
       auto MultiplyByScaleNoWrap = [](const VariableGEPIndex &Var) {
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index 1aaa4d658803..e83657df20f2 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -1586,12 +1586,10 @@ static Value *simplifyUnsignedRangeCheck(ICmpInst *ZeroICmp,
     if (match(UnsignedICmp,
               m_c_ICmp(UnsignedPred, m_Specific(Y), m_Specific(A)))) {
       if (UnsignedPred == ICmpInst::ICMP_UGE && IsAnd &&
-          EqPred == ICmpInst::ICMP_NE &&
-          isKnownNonZero(B, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT))
+          EqPred == ICmpInst::ICMP_NE && isKnownNonZero(B, /*Depth=*/0, Q))
         return UnsignedICmp;
       if (UnsignedPred == ICmpInst::ICMP_ULT && !IsAnd &&
-          EqPred == ICmpInst::ICMP_EQ &&
-          isKnownNonZero(B, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT))
+          EqPred == ICmpInst::ICMP_EQ && isKnownNonZero(B, /*Depth=*/0, Q))
         return UnsignedICmp;
     }
   }
@@ -1609,13 +1607,13 @@ static Value *simplifyUnsignedRangeCheck(ICmpInst *ZeroICmp,
   // X > Y && Y == 0  -->  Y == 0  iff X != 0
   // X > Y || Y == 0  -->  X > Y   iff X != 0
   if (UnsignedPred == ICmpInst::ICMP_UGT && EqPred == ICmpInst::ICMP_EQ &&
-      isKnownNonZero(X, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT))
+      isKnownNonZero(X, /*Depth=*/0, Q))
     return IsAnd ? ZeroICmp : UnsignedICmp;
 
   // X <= Y && Y != 0  -->  X <= Y  iff X != 0
   // X <= Y || Y != 0  -->  Y != 0  iff X != 0
   if (UnsignedPred == ICmpInst::ICMP_ULE && EqPred == ICmpInst::ICMP_NE &&
-      isKnownNonZero(X, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT))
+      isKnownNonZero(X, /*Depth=*/0, Q))
     return IsAnd ? UnsignedICmp : ZeroICmp;
 
   // The transforms below here are expected to be handled more generally with
@@ -2821,11 +2819,10 @@ static Constant *computePointerICmp(CmpInst::Predicate Pred, Value *LHS,
     // the other operand can not be based on the alloc - if it were, then
     // the cmp itself would be a capture.
     Value *MI = nullptr;
-    if (isAllocLikeFn(LHS, TLI) &&
-        llvm::isKnownNonZero(RHS, DL, 0, nullptr, CxtI, DT))
+    if (isAllocLikeFn(LHS, TLI) && llvm::isKnownNonZero(RHS, /*Depth=*/0, Q))
       MI = LHS;
     else if (isAllocLikeFn(RHS, TLI) &&
-             llvm::isKnownNonZero(LHS, DL, 0, nullptr, CxtI, DT))
+             llvm::isKnownNonZero(LHS, /*Depth=*/0, Q))
       MI = RHS;
     if (MI) {
       // FIXME: This is incorrect, see PR54002. While we can assume that the
@@ -2981,12 +2978,12 @@ static Value *simplifyICmpWithZero(CmpInst::Predicate Pred, Value *LHS,
     return getTrue(ITy);
   case ICmpInst::ICMP_EQ:
   case ICmpInst::ICMP_ULE:
-    if (isKnownNonZero(LHS, Q.DL, 0, Q.AC, Q.CxtI, Q.DT, Q.IIQ.UseInstrInfo))
+    if (isKnownNonZero(LHS, /*Depth=*/0, Q))
       return getFalse(ITy);
     break;
   case ICmpInst::ICMP_NE:
   case ICmpInst::ICMP_UGT:
-    if (isKnownNonZero(LHS, Q.DL, 0, Q.AC, Q.CxtI, Q.DT, Q.IIQ.UseInstrInfo))
+    if (isKnownNonZero(LHS, /*Depth=*/0, Q))
       return getTrue(ITy);
     break;
   case ICmpInst::ICMP_SLT: {
@@ -3001,8 +2998,7 @@ static Value *simplifyICmpWithZero(CmpInst::Predicate Pred, Value *LHS,
     KnownBits LHSKnown = computeKnownBits(LHS, /* Depth */ 0, Q);
     if (LHSKnown.isNegative())
       return getTrue(ITy);
-    if (LHSKnown.isNonNegative() &&
-        isKnownNonZero(LHS, Q.DL, 0, Q.AC, Q.CxtI, Q.DT))
+    if (LHSKnown.isNonNegative() && isKnownNonZero(LHS, /*Depth=*/0, Q))
       return getFalse(ITy);
     break;
   }
@@ -3018,8 +3014,7 @@ static Value *simplifyICmpWithZero(CmpInst::Predicate Pred, Value *LHS,
     KnownBits LHSKnown = computeKnownBits(LHS, /* Depth */ 0, Q);
     if (LHSKnown.isNegative())
       return getFalse(ITy);
-    if (LHSKnown.isNonNegative() &&
-        isKnownNonZero(LHS, Q.DL, 0, Q.AC, Q.CxtI, Q.DT))
+    if (LHSKnown.isNonNegative() && isKnownNonZero(LHS, /*Depth=*/0, Q))
       return getTrue(ITy);
     break;
   }
@@ -3172,7 +3167,7 @@ static Value *simplifyICmpWithBinOpOnLHS(CmpInst::Predicate Pred,
   const APInt *C;
   if ((match(LBO, m_LShr(m_Specific(RHS), m_APInt(C))) && *C != 0) ||
       (match(LBO, m_UDiv(m_Specific(RHS), m_APInt(C))) && *C != 1)) {
-    if (isKnownNonZero(RHS, Q.DL, 0, Q.AC, Q.CxtI, Q.DT)) {
+    if (isKnownNonZero(RHS, /*Depth=*/0, Q)) {
       switch (Pred) {
       default:
         break;
@@ -3405,7 +3400,7 @@ static Value *simplifyICmpWithBinOp(CmpInst::Predicate Pred, Value *LHS,
       bool NUW = Q.IIQ.hasNoUnsignedWrap(LBO) && Q.IIQ.hasNoUnsignedWrap(RBO);
       bool NSW = Q.IIQ.hasNoSignedWrap(LBO) && Q.IIQ.hasNoSignedWrap(RBO);
       if (!NUW || (ICmpInst::isSigned(Pred) && !NSW) ||
-          !isKnownNonZero(LBO->getOperand(0), Q.DL))
+          !isKnownNonZero(LBO->getOperand(0), /*Depth=*/0, Q))
         break;
       if (Value *V = simplifyICmpInst(Pred, LBO->getOperand(1),
                                       RBO->getOperand(1), Q, MaxRecurse - 1))
diff --git a/llvm/lib/Analysis/LazyValueInfo.cpp b/llvm/lib/Analysis/LazyValueInfo.cpp
index 6cded828c25f..3223b0564e6c 100644
--- a/llvm/lib/Analysis/LazyValueInfo.cpp
+++ b/llvm/lib/Analysis/LazyValueInfo.cpp
@@ -645,7 +645,7 @@ LazyValueInfoImpl::solveBlockValueImpl(Value *Val, BasicBlock *BB) {
   // instruction is placed, even if it could legally be hoisted much higher.
   // That is unfortunate.
   PointerType *PT = dyn_cast<PointerType>(BBI->getType());
-  if (PT && isKnownNonZero(BBI, DL))
+  if (PT && isKnownNonZero(BBI, /*Depth=*/0, DL))
     return ValueLatticeElement::getNot(ConstantPointerNull::get(PT));
 
   if (BBI->getType()->isIntegerTy()) {
@@ -1863,7 +1863,8 @@ LazyValueInfo::getPredicateAt(unsigned Pred, Value *V, Constant *C,
   Module *M = CxtI->getModule();
   const DataLayout &DL = M->getDataLayout();
   if (V->getType()->isPointerTy() && C->isNullValue() &&
-      isKnownNonZero(V->stripPointerCastsSameRepresentation(), DL)) {
+      isKnownNonZero(V->stripPointerCastsSameRepresentation(), /*Depth=*/0,
+                     DL)) {
     if (Pred == ICmpInst::ICMP_EQ)
       return LazyValueInfo::False;
     else if (Pred == ICmpInst::ICMP_NE)
diff --git a/llvm/lib/Analysis/Loads.cpp b/llvm/lib/Analysis/Loads.cpp
index 5916d2ab48ec..b5403408cf2a 100644
--- a/llvm/lib/Analysis/Loads.cpp
+++ b/llvm/lib/Analysis/Loads.cpp
@@ -99,7 +99,8 @@ static bool isDereferenceableAndAlignedPointer(
                                                           CheckForFreed));
   if (KnownDerefBytes.getBoolValue() && KnownDerefBytes.uge(Size) &&
       !CheckForFreed)
-    if (!CheckForNonNull || isKnownNonZero(V, DL, 0, AC, CtxI, DT)) {
+    if (!CheckForNonNull ||
+        isKnownNonZero(V, /*Depth=*/0, SimplifyQuery(DL, DT, AC, CtxI))) {
       // As we recursed through GEPs to get here, we've incrementally checked
       // that each step advanced by a multiple of the alignment. If our base is
       // properly aligned, then the original offset accessed must also be.
@@ -133,7 +134,8 @@ static bool isDereferenceableAndAlignedPointer(
     if (getObjectSize(V, ObjSize, DL, TLI, Opts)) {
       APInt KnownDerefBytes(Size.getBitWidth(), ObjSize);
       if (KnownDerefBytes.getBoolValue() && KnownDerefBytes.uge(Size) &&
-          isKnownNonZero(V, DL, 0, AC, CtxI, DT) && !V->canBeFreed()) {
+          isKnownNonZero(V, /*Depth=*/0, SimplifyQuery(DL, DT, AC, CtxI)) &&
+          !V->canBeFreed()) {
         // As we recursed through GEPs to get here, we've incrementally
         // checked that each step advanced by a multiple of the alignment. If
         // our base is properly aligned, then the original offset accessed
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 95440dda3b4c..1c98b0295e52 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -6900,7 +6900,7 @@ const ConstantRange &ScalarEvolution::getRangeRef(
         uint64_t Rem = MaxVal.urem(Align);
         MaxVal -= APInt(BitWidth, Rem);
         APInt MinVal = APInt::getZero(BitWidth);
-        if (llvm::isKnownNonZero(V, DL))
+        if (llvm::isKnownNonZero(V, /*Depth=*/0, DL))
           MinVal = Align;
         ConservativeResult = ConservativeResult.intersectWith(
             ConstantRange::getNonEmpty(MinVal, MaxVal + 1), RangeType);
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index ca257cc6aefd..b83e7e6769c2 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -274,16 +274,6 @@ bool llvm::isKnownToBeAPowerOfTwo(const Value *V, const DataLayout &DL,
 static bool isKnownNonZero(const Value *V, const APInt &DemandedElts,
                            unsigned Depth, const SimplifyQuery &Q);
 
-static bool isKnownNonZero(const Value *V, unsigned Depth,
-                           const SimplifyQuery &Q);
-
-bool llvm::isKnownNonZero(const Value *V, const DataLayout &DL, unsigned Depth,
-                          AssumptionCache *AC, const Instruction *CxtI,
-                          const DominatorTree *DT, bool UseInstrInfo) {
-  return ::isKnownNonZero(
-      V, Depth, SimplifyQuery(DL, DT, AC, safeCxtI(V, CxtI), UseInstrInfo));
-}
-
 bool llvm::isKnownNonNegative(const Value *V, const SimplifyQuery &SQ,
                               unsigned Depth) {
   return computeKnownBits(V, Depth, SQ).isNonNegative();
@@ -298,7 +288,7 @@ bool llvm::isKnownPositive(const Value *V, const SimplifyQuery &SQ,
   // this updated.
   KnownBits Known = computeKnownBits(V, Depth, SQ);
   return Known.isNonNegative() &&
-         (Known.isNonZero() || ::isKnownNonZero(V, Depth, SQ));
+         (Known.isNonZero() || isKnownNonZero(V, Depth, SQ));
 }
 
 bool llvm::isKnownNegative(const Value *V, const SimplifyQuery &SQ,
@@ -3093,11 +3083,12 @@ bool isKnownNonZero(const Value *V, const APInt &DemandedElts, unsigned Depth,
   return false;
 }
 
-bool isKnownNonZero(const Value *V, unsigned Depth, const SimplifyQuery &Q) {
+bool llvm::isKnownNonZero(const Value *V, unsigned Depth,
+                          const SimplifyQuery &Q) {
   auto *FVTy = dyn_cast<FixedVectorType>(V->getType());
   APInt DemandedElts =
       FVTy ? APInt::getAllOnes(FVTy->getNumElements()) : APInt(1, 1);
-  return isKnownNonZero(V, DemandedElts, Depth, Q);
+  return ::isKnownNonZero(V, DemandedElts, Depth, Q);
 }
 
 /// If the pair of operators are the same invertible function, return the
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index e657872c3828..22dbb3198a9f 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -2314,7 +2314,7 @@ static bool despeculateCountZeros(IntrinsicInst *CountZeros,
 
   // Bail if the value is never zero.
   Use &Op = CountZeros->getOperandUse(0);
-  if (isKnownNonZero(Op, *DL))
+  if (isKnownNonZero(Op, /*Depth=*/0, *DL))
     return false;
 
   // The intrinsic will be sunk behind a compare against zero and branch.
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index ff680e998e71..f27d8d64a104 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -2452,8 +2452,9 @@ bool AANonNull::isImpliedByIR(Attributor &A, const IRPosition &IRP,
   }
 
   if (llvm::any_of(Worklist, [&](AA::ValueAndContext VAC) {
-        return !isKnownNonZero(VAC.getValue(), A.getDataLayout(), 0, AC,
-                               VAC.getCtxI(), DT);
+        return !isKnownNonZero(
+            VAC.getValue(), /*Depth=*/0,
+            SimplifyQuery(A.getDataLayout(), DT, AC, VAC.getCtxI()));
       }))
     return false;
 
diff --git a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
index 7ebf265e17ba..14612b251d1a 100644
--- a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
+++ b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
@@ -1175,7 +1175,7 @@ static bool isReturnNonNull(Function *F, const SCCNodeSet &SCCNodes,
     Value *RetVal = FlowsToReturn[i];
 
     // If this value is locally known to be non-null, we're good
-    if (isKnownNonZero(RetVal, DL))
+    if (isKnownNonZero(RetVal, /*Depth=*/0, DL))
       continue;
 
     // Otherwise, we need to look upwards since we can't make any local
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
index fa1e2280991f..07c50d866544 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
@@ -988,7 +988,7 @@ Instruction *InstCombinerImpl::foldAddWithConstant(BinaryOperator &Add) {
   if (C->isOne()) {
     if (match(Op0, m_ZExt(m_Add(m_Value(X), m_AllOnes())))) {
       const SimplifyQuery Q = SQ.getWithInstruction(&Add);
-      if (llvm::isKnownNonZero(X, DL, 0, Q.AC, Q.CxtI, Q.DT))
+      if (llvm::isKnownNonZero(X, /*Depth=*/0, Q))
         return new ZExtInst(X, Ty);
     }
   }
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index c0cf1a7db726..2c0c4ee46e80 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -1031,10 +1031,6 @@ static Value *foldUnsignedUnderflowCheck(ICmpInst *ZeroICmp,
       !ICmpInst::isEquality(EqPred))
     return nullptr;
 
-  auto IsKnownNonZero = [&](Value *V) {
-    return isKnownNonZero(V, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT);
-  };
-
   ICmpInst::Predicate UnsignedPred;
 
   Value *A, *B;
@@ -1043,9 +1039,9 @@ static Value *foldUnsignedUnderflowCheck(ICmpInst *ZeroICmp,
       match(ZeroCmpOp, m_c_Add(m_Specific(A), m_Value(B))) &&
       (ZeroICmp->hasOneUse() || UnsignedICmp->hasOneUse())) {
     auto GetKnownNonZeroAndOther = [&](Value *&NonZero, Value *&Other) {
-      if (!IsKnownNonZero(NonZero))
+      if (!isKnownNonZero(NonZero, /*Depth=*/0, Q))
         std::swap(NonZero, Other);
-      return IsKnownNonZero(NonZero);
+      return isKnownNonZero(NonZero, /*Depth=*/0, Q);
     };
 
     // Given  ZeroCmpOp = (A + B)
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index b1017f4d6bc1..b35ab8751a21 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -601,8 +601,8 @@ static Instruction *foldCttzCtlz(IntrinsicInst &II, InstCombinerImpl &IC) {
   // then change the 'ZeroIsPoison' parameter to 'true'
   // because we know the zero behavior can't affect the result.
   if (!Known.One.isZero() ||
-      isKnownNonZero(Op0, IC.getDataLayout(), 0, &IC.getAssumptionCache(), &II,
-                     &IC.getDominatorTree())) {
+      isKnownNonZero(Op0, /*Depth=*/0,
+                     IC.getSimplifyQuery().getWithInstruction(&II))) {
     if (!match(II.getArgOperand(1), m_One()))
       return IC.replaceOperand(II, 1, IC.Builder.getTrue());
   }
@@ -2061,7 +2061,8 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
     // See if we can deduce non-null.
     if (!CI.hasRetAttr(Attribute::NonNull) &&
         (Known.isNonZero() ||
-         isKnownNonZero(II, DL, /*Depth*/ 0, &AC, II, &DT))) {
+         isKnownNonZero(II, /*Depth=*/0,
+                        getSimplifyQuery().getWithInstruction(II)))) {
       CI.addRetAttr(Attribute::NonNull);
       Changed = true;
     }
@@ -3648,7 +3649,8 @@ Instruction *InstCombinerImpl::visitCallBase(CallBase &Call) {
   for (Value *V : Call.args()) {
     if (V->getType()->isPointerTy() &&
         !Call.paramHasAttr(ArgNo, Attribute::NonNull) &&
-        isKnownNonZero(V, DL, 0, &AC, &Call, &DT))
+        isKnownNonZero(V, /*Depth=*/0,
+                       getSimplifyQuery().getWithInstruction(&Call)))
       ArgNos.push_back(ArgNo);
     ArgNo++;
   }
@@ -3828,7 +3830,8 @@ Instruction *InstCombinerImpl::visitCallBase(CallBase &Call) {
 
         // isKnownNonNull -> nonnull attribute
         if (!GCR.hasRetAttr(Attribute::NonNull) &&
-            isKnownNonZero(DerivedPtr, DL, 0, &AC, &Call, &DT)) {
+            isKnownNonZero(DerivedPtr, /*Depth=*/0,
+                           getSimplifyQuery().getWithInstruction(&Call))) {
           GCR.addRetAttr(Attribute::NonNull);
           // We discovered new fact, re-check users.
           Worklist.pushUsersToWorkList(GCR);
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index 7292bb62702a..90550cdbdf89 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -1273,12 +1273,12 @@ Instruction *InstCombinerImpl::foldICmpWithZero(ICmpInst &Cmp) {
 
       // if X non-zero and NoOverflow(X * Y)
       //    (icmp eq/ne Y)
-      if (!XKnown.One.isZero() || isKnownNonZero(X, DL, 0, Q.AC, Q.CxtI, Q.DT))
+      if (!XKnown.One.isZero() || isKnownNonZero(X, /*Depth=*/0, Q))
         return new ICmpInst(Pred, Y, Cmp.getOperand(1));
 
       // if Y non-zero and NoOverflow(X * Y)
       //    (icmp eq/ne X)
-      if (!YKnown.One.isZero() || isKnownNonZero(Y, DL, 0, Q.AC, Q.CxtI, Q.DT))
+      if (!YKnown.One.isZero() || isKnownNonZero(Y, /*Depth=*/0, Q))
         return new ICmpInst(Pred, X, Cmp.getOperand(1));
     }
     // Note, we are skipping cases:
@@ -3087,7 +3087,7 @@ Instruction *InstCombinerImpl::foldICmpAddConstant(ICmpInst &Cmp,
   // (X + -1) <u C --> X <=u C (if X is never null)
   if (Pred == CmpInst::ICMP_ULT && C2->isAllOnes()) {
     const SimplifyQuery Q = SQ.getWithInstruction(&Cmp);
-    if (llvm::isKnownNonZero(X, DL, 0, Q.AC, Q.CxtI, Q.DT))
+    if (llvm::isKnownNonZero(X, /*Depth=*/0, Q))
       return new ICmpInst(ICmpInst::ICMP_ULE, X, ConstantInt::get(Ty, C));
   }
 
@@ -4275,8 +4275,7 @@ static Value *foldICmpWithLowBitMaskedVal(ICmpInst::Predicate Pred, Value *Op0,
 
       // Look for: x & ~Mask pred ~Mask
       if (isMaskOrZero(X, /*Not=*/true, Q)) {
-        return !ICmpInst::isSigned(Pred) ||
-               isKnownNonZero(X, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT);
+        return !ICmpInst::isSigned(Pred) || isKnownNonZero(X, /*Depth=*/0, Q);
       }
       return false;
     }
@@ -4780,8 +4779,7 @@ static Instruction *foldICmpXorXX(ICmpInst &I, const SimplifyQuery &Q,
   // icmp (X ^ Y_NonZero) s>= X --> icmp (X ^ Y_NonZero) s> X
   // icmp (X ^ Y_NonZero) s<= X --> icmp (X ^ Y_NonZero) s< X
   CmpInst::Predicate PredOut = CmpInst::getStrictPredicate(Pred);
-  if (PredOut != Pred &&
-      isKnownNonZero(A, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT))
+  if (PredOut != Pred && isKnownNonZero(A, /*Depth=*/0, Q))
     return new ICmpInst(PredOut, Op0, Op1);
 
   return nullptr;
@@ -5064,11 +5062,11 @@ Instruction *InstCombinerImpl::foldICmpBinOp(ICmpInst &I,
     return new ICmpInst(Pred, C, D);
   // (A - B) u>=/u< A --> B u>/u<= A  iff B != 0
   if (A == Op1 && (Pred == ICmpInst::ICMP_UGE || Pred == ICmpInst::ICMP_ULT) &&
-      isKnownNonZero(B, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT))
+      isKnownNonZero(B, /*Depth=*/0, Q))
     return new ICmpInst(CmpInst::getFlippedStrictnessPredicate(Pred), B, A);
   // C u<=/u> (C - D) --> C u</u>= D  iff B != 0
   if (C == Op0 && (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_UGT) &&
-      isKnownNonZero(D, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT))
+      isKnownNonZero(D, /*Depth=*/0, Q))
     return new ICmpInst(CmpInst::getFlippedStrictnessPredicate(Pred), C, D);
 
   // icmp (A-B), (C-B) -> icmp A, C for equalities or if there is no overflow.
@@ -5110,14 +5108,13 @@ Instruction *InstCombinerImpl::foldICmpBinOp(ICmpInst &I,
         //    X * Z eq/ne Y * Z -> X eq/ne Y
         if (ZKnown.countMaxTrailingZeros() == 0)
           return new ICmpInst(Pred, X, Y);
-        NonZero = !ZKnown.One.isZero() ||
-                  isKnownNonZero(Z, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT);
+        NonZero = !ZKnown.One.isZero() || isKnownNonZero(Z, /*Depth=*/0, Q);
         // if Z != 0 and nsw(X * Z) and nsw(Y * Z)
         //    X * Z eq/ne Y * Z -> X eq/ne Y
         if (NonZero && BO0 && BO1 && Op0HasNSW && Op1HasNSW)
           return new ICmpInst(Pred, X, Y);
       } else
-        NonZero = isKnownNonZero(Z, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT);
+        NonZero = isKnownNonZero(Z, /*Depth=*/0, Q);
 
       // If Z != 0 and nuw(X * Z) and nuw(Y * Z)
       //    X * Z u{lt/le/gt/ge}/eq/ne Y * Z -> X u{lt/le/gt/ge}/eq/ne Y
diff --git a/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp b/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp
index 46bca4b722a0..9838e2aa9f3a 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp
@@ -1537,7 +1537,8 @@ Instruction *InstCombinerImpl::visitPHINode(PHINode &PN) {
       for (unsigned I = 0, E = PN.getNumIncomingValues(); I != E; ++I) {
         Instruction *CtxI = PN.getIncomingBlock(I)->getTerminator();
         Value *VA = PN.getIncomingValue(I);
-        if (isKnownNonZero(VA, DL, 0, &AC, CtxI, &DT)) {
+        if (isKnownNonZero(VA, 0,
+                           getSimplifyQuery().getWithInstruction(CtxI))) {
           if (!NonZeroConst)
             NonZeroConst = getAnyNonZeroConstInt(PN);
           if (NonZeroConst != VA) {
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 7c40fb4fc860..4cfafa7ac800 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -1431,7 +1431,7 @@ Instruction *InstCombinerImpl::foldFBinOpOfIntCastsFromSign(
     if (OpsKnown[OpNo].hasKnownBits() &&
         OpsKnown[OpNo].getKnownBits(SQ).isNonZero())
       return true;
-    return isKnownNonZero(IntOps[OpNo], SQ.DL);
+    return isKnownNonZero(IntOps[OpNo], /*Depth=*/0, SQ);
   };
 
   auto IsNonNeg = [&](unsigned OpNo) -> bool {
diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index ee3531bbd68d..a72b0ee9a08e 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -1281,7 +1281,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
         // ignored.
         return;
       }
-      if (llvm::isKnownNonZero(ConvertedShadow, DL)) {
+      if (llvm::isKnownNonZero(ConvertedShadow, /*Depth=*/0, DL)) {
         // Copy origin as the value is definitely uninitialized.
         paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
                     OriginAlignment);
@@ -1427,7 +1427,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
           // Skip, value is initialized or const shadow is ignored.
           continue;
         }
-        if (llvm::isKnownNonZero(ConvertedShadow, DL)) {
+        if (llvm::isKnownNonZero(ConvertedShadow, /*Depth=*/0, DL)) {
           // Report as the value is definitely uninitialized.
           insertWarningFn(IRB, ShadowData.Origin);
           if (!MS.Recover)
diff --git a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
index adcf161b313b..f376b5f7d68d 100644
--- a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
+++ b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
@@ -459,7 +459,7 @@ static void convertMetadataToAssumes(LoadInst *LI, Value *Val,
   // we can only do this if the value is known non-poison.
   if (AC && LI->getMetadata(LLVMContext::MD_nonnull) &&
       LI->getMetadata(LLVMContext::MD_noundef) &&
-      !isKnownNonZero(Val, DL, 0, AC, LI, DT))
+      !isKnownNonZero(Val, /*Depth=*/0, SimplifyQuery(DL, DT, AC, LI)))
     addAssumeNonNull(AC, LI);
 }
 
diff --git a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
index 2e68a9c01898..7e9e91606fe2 100644
--- a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
@@ -305,7 +305,7 @@ static void annotateNonNullAndDereferenceable(CallInst *CI, ArrayRef<unsigned> A
   if (ConstantInt *LenC = dyn_cast<ConstantInt>(Size)) {
     annotateNonNullNoUndefBasedOnAccess(CI, ArgNos);
     annotateDereferenceableBytes(CI, ArgNos, LenC->getZExtValue());
-  } else if (isKnownNonZero(Size, DL)) {
+  } else if (isKnownNonZero(Size, /*Depth=*/0, DL)) {
     annotateNonNullNoUndefBasedOnAccess(CI, ArgNos);
     const APInt *X, *Y;
     uint64_t DerefMin = 1;
@@ -394,7 +394,7 @@ Value *LibCallSimplifier::optimizeStrNCat(CallInst *CI, IRBuilderBase &B) {
   Value *Size = CI->getArgOperand(2);
   uint64_t Len;
   annotateNonNullNoUndefBasedOnAccess(CI, 0);
-  if (isKnownNonZero(Size, DL))
+  if (isKnownNonZero(Size, /*Depth=*/0, DL))
     annotateNonNullNoUndefBasedOnAccess(CI, 1);
 
   // We don't do anything if length is not constant.
@@ -613,7 +613,7 @@ Value *LibCallSimplifier::optimizeStrNCmp(CallInst *CI, IRBuilderBase &B) {
   if (Str1P == Str2P) // strncmp(x,x,n)  -> 0
     return ConstantInt::get(CI->getType(), 0);
 
-  if (isKnownNonZero(Size, DL))
+  if (isKnownNonZero(Size, /*Depth=*/0, DL))
     annotateNonNullNoUndefBasedOnAccess(CI, {0, 1});
   // Get the length argument if it is constant.
   uint64_t Length;
@@ -749,7 +749,7 @@ Value *LibCallSimplifier::optimizeStpCpy(CallInst *CI, IRBuilderBase &B) {
 
 Value *LibCallSimplifier::optimizeStrLCpy(CallInst *CI, IRBuilderBase &B) {
   Value *Size = CI->getArgOperand(2);
-  if (isKnownNonZero(Size, DL))
+  if (isKnownNonZero(Size, /*Depth=*/0, DL))
     // Like snprintf, the function stores into the destination only when
     // the size argument is nonzero.
     annotateNonNullNoUndefBasedOnAccess(CI, 0);
@@ -833,7 +833,7 @@ Value *LibCallSimplifier::optimizeStringNCpy(CallInst *CI, bool RetEnd,
   Value *Src = CI->getArgOperand(1);
   Value *Size = CI->getArgOperand(2);
 
-  if (isKnownNonZero(Size, DL)) {
+  if (isKnownNonZero(Size, /*Depth=*/0, DL)) {
     // Both st{p,r}ncpy(D, S, N) access the source and destination arrays
     // only when N is nonzero.
     annotateNonNullNoUndefBasedOnAccess(CI, 0);
@@ -926,7 +926,7 @@ Value *LibCallSimplifier::optimizeStringLength(CallInst *CI, IRBuilderBase &B,
   Type *CharTy = B.getIntNTy(CharSize);
 
   if (isOnlyUsedInZeroEqualityComparison(CI) &&
-      (!Bound || isKnownNonZero(Bound, DL))) {
+      (!Bound || isKnownNonZero(Bound, /*Depth=*/0, DL))) {
     // Fold strlen:
     //   strlen(x) != 0 --> *x != 0
     //   strlen(x) == 0 --> *x == 0
@@ -1047,7 +1047,7 @@ Value *LibCallSimplifier::optimizeStrNLen(CallInst *CI, IRBuilderBase &B) {
   if (Value *V = optimizeStringLength(CI, B, 8, Bound))
     return V;
 
-  if (isKnownNonZero(Bound, DL))
+  if (isKnownNonZero(Bound, /*Depth=*/0, DL))
     annotateNonNullNoUndefBasedOnAccess(CI, 0);
   return nullptr;
 }
@@ -1291,7 +1291,7 @@ Value *LibCallSimplifier::optimizeMemChr(CallInst *CI, IRBuilderBase &B) {
   Value *SrcStr = CI->getArgOperand(0);
   Value *Size = CI->getArgOperand(2);
 
-  if (isKnownNonZero(Size, DL)) {
+  if (isKnownNonZero(Size, /*Depth=*/0, DL)) {
     annotateNonNullNoUndefBasedOnAccess(CI, 0);
     if (isOnlyUsedInEqualityComparison(CI, SrcStr))
       return memChrToCharCompare(CI, Size, B, DL);
@@ -2976,7 +2976,7 @@ Value *LibCallSimplifier::optimizeStrToInt(CallInst *CI, IRBuilderBase &B,
     // It would be readonly too, except that it still may write to errno.
     CI->addParamAttr(0, Attribute::NoCapture);
     EndPtr = nullptr;
-  } else if (!isKnownNonZero(EndPtr, DL))
+  } else if (!isKnownNonZero(EndPtr, /*Depth=*/0, DL))
     return nullptr;
 
   StringRef Str;
@@ -3402,7 +3402,7 @@ Value *LibCallSimplifier::optimizeSnPrintF(CallInst *CI, IRBuilderBase &B) {
     return V;
   }
 
-  if (isKnownNonZero(CI->getOperand(1), DL))
+  if (isKnownNonZero(CI->getOperand(1), /*Depth=*/0, DL))
     annotateNonNullNoUndefBasedOnAccess(CI, 0);
   return nullptr;
 }
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 653e0ff7a3ec..e0e2f50c89ad 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -885,7 +885,8 @@ bool VectorCombine::scalarizeVPIntrinsic(Instruction &I) {
   else
     SafeToSpeculate = isSafeToSpeculativelyExecuteWithOpcode(
         *FunctionalOpcode, &VPI, nullptr, &AC, &DT);
-  if (!SafeToSpeculate && !isKnownNonZero(EVL, *DL, 0, &AC, &VPI, &DT))
+  if (!SafeToSpeculate &&
+      !isKnownNonZero(EVL, /*Depth=*/0, SimplifyQuery(*DL, &DT, &AC, &VPI)))
     return false;
 
   Value *ScalarVal =
diff --git a/llvm/test/Transforms/InstCombine/icmp-dom.ll b/llvm/test/Transforms/InstCombine/icmp-dom.ll
index f4b9022d1434..83cedd5ea9cb 100644
--- a/llvm/test/Transforms/InstCombine/icmp-dom.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-dom.ll
@@ -403,3 +403,134 @@ truelabel:
 falselabel:
   ret i8 0
 }
+
+define i1 @and_mask1_eq(i32 %conv) {
+; CHECK-LABEL: @and_mask1_eq(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[CONV:%.*]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    ret i1 false
+; CHECK:       else:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %and = and i32 %conv, 1
+  %cmp = icmp eq i32 %and, 0
+  br i1 %cmp, label %then, label %else
+
+then:
+  ret i1 0
+
+else:
+  %and1 = and i32 %conv, 3
+  %cmp1 = icmp eq i32 %and1, 0
+  ret i1 %cmp1
+}
+
+define i1 @and_mask1_ne(i32 %conv) {
+; CHECK-LABEL: @and_mask1_ne(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[CONV:%.*]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    ret i1 false
+; CHECK:       else:
+; CHECK-NEXT:    ret i1 true
+;
+entry:
+  %and = and i32 %conv, 1
+  %cmp = icmp eq i32 %and, 0
+  br i1 %cmp, label %then, label %else
+
+then:
+  ret i1 0
+
+else:
+  %and1 = and i32 %conv, 3
+  %cmp1 = icmp ne i32 %and1, 0
+  ret i1 %cmp1
+}
+
+define i1 @and_mask2(i32 %conv) {
+; CHECK-LABEL: @and_mask2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[CONV:%.*]], 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    ret i1 false
+; CHECK:       else:
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[CONV]], 3
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[AND1]], 0
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+entry:
+  %and = and i32 %conv, 4
+  %cmp = icmp eq i32 %and, 0
+  br i1 %cmp, label %then, label %else
+
+then:
+  ret i1 0
+
+else:
+  %and1 = and i32 %conv, 3
+  %cmp1 = icmp eq i32 %and1, 0
+  ret i1 %cmp1
+}
+
+; TODO: %cmp1 can be folded into false.
+
+define i1 @and_mask3(i32 %conv) {
+; CHECK-LABEL: @and_mask3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[CONV:%.*]], 3
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    ret i1 false
+; CHECK:       else:
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[CONV]], 7
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[AND1]], 0
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
+entry:
+  %and = and i32 %conv, 3
+  %cmp = icmp eq i32 %and, 0
+  br i1 %cmp, label %then, label %else
+
+then:
+  ret i1 0
+
+else:
+  %and1 = and i32 %conv, 7
+  %cmp1 = icmp eq i32 %and1, 0
+  ret i1 %cmp1
+}
+
+define i1 @and_mask4(i32 %conv) {
+; CHECK-LABEL: @and_mask4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[CONV:%.*]], 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[THEN:%.*]], label [[ELSE:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    ret i1 false
+; CHECK:       else:
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %and = and i32 %conv, 4
+  %cmp = icmp eq i32 %and, 0
+  br i1 %cmp, label %then, label %else
+
+then:
+  ret i1 0
+
+else:
+  %and1 = and i32 %conv, 7
+  %cmp1 = icmp eq i32 %and1, 0
+  ret i1 %cmp1
+}
diff --git a/llvm/unittests/Analysis/ValueTrackingTest.cpp b/llvm/unittests/Analysis/ValueTrackingTest.cpp
index b4d2270d7070..6b1044876d36 100644
--- a/llvm/unittests/Analysis/ValueTrackingTest.cpp
+++ b/llvm/unittests/Analysis/ValueTrackingTest.cpp
@@ -2055,7 +2055,8 @@ TEST_F(ValueTrackingTest, isNonZeroRecurrence) {
   )");
   const DataLayout &DL = M->getDataLayout();
   AssumptionCache AC(*F);
-  EXPECT_TRUE(isKnownNonZero(A, DL, 0, &AC, CxtI));
+  EXPECT_TRUE(isKnownNonZero(A, /*Depth=*/0,
+                             SimplifyQuery(DL, /*DT=*/nullptr, &AC, CxtI)));
 }
 
 TEST_F(ValueTrackingTest, KnownNonZeroFromDomCond) {
@@ -2078,8 +2079,10 @@ TEST_F(ValueTrackingTest, KnownNonZeroFromDomCond) {
   AssumptionCache AC(*F);
   DominatorTree DT(*F);
   const DataLayout &DL = M->getDataLayout();
-  EXPECT_EQ(isKnownNonZero(A, DL, 0, &AC, CxtI, &DT), true);
-  EXPECT_EQ(isKnownNonZero(A, DL, 0, &AC, CxtI2, &DT), false);
+  const SimplifyQuery SQ(DL, &DT, &AC);
+  EXPECT_EQ(isKnownNonZero(A, /*Depth=*/0, SQ.getWithInstruction(CxtI)), true);
+  EXPECT_EQ(isKnownNonZero(A, /*Depth=*/0, SQ.getWithInstruction(CxtI2)),
+            false);
 }
 
 TEST_F(ValueTrackingTest, KnownNonZeroFromDomCond2) {
@@ -2102,8 +2105,10 @@ TEST_F(ValueTrackingTest, KnownNonZeroFromDomCond2) {
   AssumptionCache AC(*F);
   DominatorTree DT(*F);
   const DataLayout &DL = M->getDataLayout();
-  EXPECT_EQ(isKnownNonZero(A, DL, 0, &AC, CxtI, &DT), true);
-  EXPECT_EQ(isKnownNonZero(A, DL, 0, &AC, CxtI2, &DT), false);
+  const SimplifyQuery SQ(DL, &DT, &AC);
+  EXPECT_EQ(isKnownNonZero(A, /*Depth=*/0, SQ.getWithInstruction(CxtI)), true);
+  EXPECT_EQ(isKnownNonZero(A, /*Depth=*/0, SQ.getWithInstruction(CxtI2)),
+            false);
 }
 
 TEST_F(ValueTrackingTest, IsImpliedConditionAnd) {
-- 
GitLab


From 6f1e23b47d428d792866993ed26f4173d479d43d Mon Sep 17 00:00:00 2001
From: Kunwar Grover <groverkss@gmail.com>
Date: Fri, 12 Apr 2024 17:56:46 +0200
Subject: [PATCH 080/464] [MLIR][Bufferization] Choose default memory space in
 tensor copy insertion (#88500)

Tensor copy insertion currently uses memory_space = 0 when creating a
tensor copy using alloc_tensor. This memory space should instead be the
default memory space provided in bufferization options.
---
 .../Dialect/Bufferization/IR/BufferizableOpInterface.cpp   | 7 ++++---
 .../Bufferization/Transforms/tensor-copy-insertion.mlir    | 6 +++---
 mlir/test/Dialect/SparseTensor/sparse_sddmm.mlir           | 2 +-
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
index 55c9299c58ef..c2b2b99fc008 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
@@ -193,10 +193,11 @@ FailureOr<Value> bufferization::allocateTensorForShapedValue(
   FailureOr<BaseMemRefType> copyBufferType = getBufferType(tensor, options);
   if (failed(copyBufferType))
     return failure();
-  Attribute memorySpace = copyBufferType->getMemorySpace();
+  std::optional<Attribute> memorySpace = copyBufferType->getMemorySpace();
   if (!memorySpace)
-    memorySpace = b.getI64IntegerAttr(0);
-  allocTensorOp.setMemorySpaceAttr(memorySpace);
+    memorySpace = options.defaultMemorySpaceFn(tensorType);
+  if (memorySpace.has_value())
+    allocTensorOp.setMemorySpaceAttr(memorySpace.value());
   return allocTensorOp.getResult();
 }
 
diff --git a/mlir/test/Dialect/Bufferization/Transforms/tensor-copy-insertion.mlir b/mlir/test/Dialect/Bufferization/Transforms/tensor-copy-insertion.mlir
index 72cf08df5978..55dcca193cbe 100644
--- a/mlir/test/Dialect/Bufferization/Transforms/tensor-copy-insertion.mlir
+++ b/mlir/test/Dialect/Bufferization/Transforms/tensor-copy-insertion.mlir
@@ -34,7 +34,7 @@ func.func @do_not_copy_undefined_tensor(%f: f32, %idx: index)
 {
   // The second alloc_tensor should not have a copy operand.
   // CHECK: bufferization.alloc_tensor() : tensor<5xf32>
-  // CHECK: bufferization.alloc_tensor() {memory_space = 0 : i64} : tensor<5xf32>
+  // CHECK: bufferization.alloc_tensor() : tensor<5xf32>
   %0 = bufferization.alloc_tensor() : tensor<5xf32>
   %1 = tensor.insert %f into %0[%idx] : tensor<5xf32>
   return %0, %1 : tensor<5xf32>, tensor<5xf32>
@@ -46,7 +46,7 @@ func.func @do_not_copy_undefined_tensor(%f: f32, %idx: index)
 func.func @do_not_copy_when_overwritten(%t: tensor<5xf32>, %f: f32)
   -> (tensor<5xf32>, tensor<5xf32>)
 {
-  // CHECK: %[[alloc:.*]] = bufferization.alloc_tensor() {memory_space = 0 : i64} : tensor<5xf32>
+  // CHECK: %[[alloc:.*]] = bufferization.alloc_tensor() : tensor<5xf32>
   // CHECK: linalg.generic {{.*}} outs(%[[alloc]] : tensor<5xf32>)
   %r = linalg.generic {
     indexing_maps = [affine_map<(d0) -> (d0)>],
@@ -65,7 +65,7 @@ func.func @do_not_copy_when_result_not_read(%t: tensor<5xf32>, %f: f32)
   -> (tensor<3xf32>)
 {
   %0 = tensor.extract_slice %t[0][3][1] : tensor<5xf32> to tensor<3xf32>
-  // CHECK: %[[alloc:.*]] = bufferization.alloc_tensor() {memory_space = 0 : i64} : tensor<3xf32>
+  // CHECK: %[[alloc:.*]] = bufferization.alloc_tensor() : tensor<3xf32>
   // CHECK: linalg.generic {{.*}} outs(%[[alloc]] : tensor<3xf32>)
   %r = linalg.generic {
     indexing_maps = [affine_map<(d0) -> (d0)>],
diff --git a/mlir/test/Dialect/SparseTensor/sparse_sddmm.mlir b/mlir/test/Dialect/SparseTensor/sparse_sddmm.mlir
index 6fc2db481c6f..fcd69bea426d 100644
--- a/mlir/test/Dialect/SparseTensor/sparse_sddmm.mlir
+++ b/mlir/test/Dialect/SparseTensor/sparse_sddmm.mlir
@@ -63,7 +63,7 @@ func.func @fold_yield_direct_zero() -> tensor<32xf64> {
 // CHECK-DAG:       %[[VAL_5:.*]] = arith.constant 1 : index
 // CHECK-DAG:       %[[VAL_6:.*]] = arith.constant dense<0.000000e+00> : tensor<8x8xf64>
 // CHECK-DAG:       %[[VAL_7:.*]] = bufferization.alloc_tensor() copy(%[[VAL_6]]) : tensor<8x8xf64>
-// CHECK-DAG:       %[[VAL_8:.*]] = bufferization.alloc_tensor() copy(%[[VAL_6]]) {memory_space = 0 : i64} : tensor<8x8xf64>
+// CHECK-DAG:       %[[VAL_8:.*]] = bufferization.alloc_tensor() copy(%[[VAL_6]]) : tensor<8x8xf64>
 // CHECK-DAG:       %[[VAL_9:.*]] = bufferization.to_memref %[[VAL_1]] : memref<8x8xf64>
 // CHECK-DAG:       %[[VAL_10:.*]] = bufferization.to_memref %[[VAL_2]] : memref<8x8xf64>
 // CHECK-DAG:       %[[VAL_11:.*]] = sparse_tensor.positions %[[VAL_0]] {level = 0 : index} : tensor<8x8xf64, #sparse{{[0-9]*}}> to memref<?xindex>
-- 
GitLab


From b614e5b0340f783ad355899248c52cb22a04b014 Mon Sep 17 00:00:00 2001
From: Dominik Steenken <dost@de.ibm.com>
Date: Fri, 12 Apr 2024 18:03:57 +0200
Subject: [PATCH 081/464] [SystemZ] Add missing (dis-)assembly tests. (#88498)

### Assembly Tests
This adds assembly tests for
```
lcdfr, lndfr, lpdfr
```

### Disassembly Tests
This also adds a number of disassembly tests for the following
menmonics:
```
cgit, cgrt, cit, clfit, clgit, clgrt, clgt, clrt, clt, crt, lcdfr, lndfr, lpdfr
```
For `cutfu` / `cu12`, it also adds a check for bad register choice.
---
 llvm/test/MC/Disassembler/SystemZ/insns.txt   | 96 +++++++++++++++++++
 .../MC/Disassembler/SystemZ/invalid-regs.txt  |  7 ++
 llvm/test/MC/SystemZ/insn-good.s              | 30 ++++++
 3 files changed, 133 insertions(+)

diff --git a/llvm/test/MC/Disassembler/SystemZ/insns.txt b/llvm/test/MC/Disassembler/SystemZ/insns.txt
index f2f942ed5d36..23714dfc3a8e 100644
--- a/llvm/test/MC/Disassembler/SystemZ/insns.txt
+++ b/llvm/test/MC/Disassembler/SystemZ/insns.txt
@@ -2956,6 +2956,12 @@
 # CHECK: cgib %r0, 0, 15, 0
 0xec 0x0f 0x00 0x00 0x00 0xfc
 
+# CHECK: cgit %r0, 0, 0
+0xec 0x00 0x00 0x00 0x00 0x70
+
+# CHECK: cgit %r0, 0, 15
+0xec 0x00 0x00 0x00 0xf0 0x70
+
 # CHECK: cgith %r0, 0
 0xec 0x00 0x00 0x00 0x20 0x70
 
@@ -3052,6 +3058,12 @@
 # CHECK: cgrb %r0, %r0, 15, 0
 0xec 0x00 0x00 0x00 0xf0 0xe4
 
+# CHECK: cgrt %r0, %r1, 0
+0xb9 0x60 0x00 0x01
+
+# CHECK: cgrt %r0, %r1, 15
+0xb9 0x60 0xf0 0x01
+
 # CHECK: cgrth %r0, %r1
 0xb9 0x60 0x20 0x01
 
@@ -3430,6 +3442,12 @@
 # CHECK: cih %r15, 0
 0xcc 0xfd 0x00 0x00 0x00 0x00
 
+# CHECK: cit %r0, 0, 0
+0xec 0x00 0x00 0x00 0x00 0x72
+
+# CHECK: cit %r0, 0, 15
+0xec 0x00 0x00 0x00 0xf0 0x72
+
 # CHECK: cith %r0, 0
 0xec 0x00 0x00 0x00 0x20 0x72
 
@@ -3862,6 +3880,12 @@
 # CHECK: clfi %r15, 0
 0xc2 0xff 0x00 0x00 0x00 0x00
 
+# CHECK: clfit %r0, 0, 0
+0xec 0x00 0x00 0x00 0x00 0x73
+
+# CHECK: clfit %r0, 0, 15
+0xec 0x00 0x00 0x00 0xf0 0x73
+
 # CHECK: clfith %r0, 0
 0xec 0x00 0x00 0x00 0x20 0x73
 
@@ -3910,6 +3934,12 @@
 # CHECK: clg %r15, 0
 0xe3 0xf0 0x00 0x00 0x00 0x21
 
+# CHECK: clgit %r0, 0, 0
+0xec 0x00 0x00 0x00 0x00 0x71
+
+# CHECK: clgit %r0, 0, 15
+0xec 0x00 0x00 0x00 0xf0 0x71
+
 # CHECK: clgith %r0, 0
 0xec 0x00 0x00 0x00 0x20 0x71
 
@@ -4447,6 +4477,12 @@
 # CHECK: clrb %r0, %r0, 15, 0
 0xec 0x00 0x00 0x00 0xf0 0xf7
 
+# CHECK: clgrt %r0, %r1, 0
+0xb9 0x61 0x00 0x01
+
+# CHECK: clgrt %r0, %r1, 15
+0xb9 0x61 0xf0 0x01
+
 # CHECK: clgrth %r0, %r1
 0xb9 0x61 0x20 0x01
 
@@ -4465,6 +4501,12 @@
 # CHECK: clgrtle %r0, %r1
 0xb9 0x61 0xc0 0x01
 
+# CHECK: clrt %r0, %r1, 0
+0xb9 0x73 0x00 0x01
+
+# CHECK: clrt %r0, %r1, 15
+0xb9 0x73 0xf0 0x01
+
 # CHECK: clrth %r0, %r1
 0xb9 0x73 0x20 0x01
 
@@ -4483,6 +4525,12 @@
 # CHECK: clrtle %r0, %r1
 0xb9 0x73 0xc0 0x01
 
+# CHECK: clt %r0, 0, -524288
+0xeb 0x00 0x00 0x00 0x80 0x23
+
+# CHECK: clt %r0, 15, 32768
+0xeb 0x0f 0x00 0x00 0x08 0x23
+
 # CHECK: clth %r0, -524288
 0xeb 0x02 0x00 0x00 0x80 0x23
 
@@ -4525,6 +4573,12 @@
 # CHECK: cltle %r1, 0
 0xeb 0x1c 0x00 0x00 0x00 0x23
 
+# CHECK: clgt %r0, 0, -524288
+0xeb 0x00 0x00 0x00 0x80 0x2b
+
+# CHECK: clgt %r0, 15, 32768
+0xeb 0x0f 0x00 0x00 0x08 0x2b
+
 # CHECK: clgth %r0, -524288
 0xeb 0x02 0x00 0x00 0x80 0x2b
 
@@ -4792,6 +4846,12 @@
 # CHECK: crdte %r4, %r5, %r6, 7
 0xb9 0x8f 0x57 0x46
 
+# CHECK: crt %r0, %r1, 0
+0xb9 0x72 0x00 0x01
+
+# CHECK: crt %r0, %r1, 15
+0xb9 0x72 0xf0 0x01
+
 # CHECK: crth %r0, %r1
 0xb9 0x72 0x20 0x01
 
@@ -7972,6 +8032,18 @@
 # CHECK: lcdbr %f15, %f9
 0xb3 0x13 0x00 0xf9
 
+# CHECK: lcdfr %f0, %f9
+0xb3 0x73 0x00 0x09
+
+# CHECK: lcdfr %f0, %f15
+0xb3 0x73 0x00 0x0f
+
+# CHECK: lcdfr %f15, %f0
+0xb3 0x73 0x00 0xf0
+
+# CHECK: lcdfr %f15, %f9
+0xb3 0x73 0x00 0xf9
+
 # CHECK: lcdr %f0, %f9
 0x23 0x09
 
@@ -9570,6 +9642,18 @@
 # CHECK: lndbr %f15, %f9
 0xb3 0x11 0x00 0xf9
 
+# CHECK: lndfr %f0, %f9
+0xb3 0x71 0x00 0x09
+
+# CHECK: lndfr %f0, %f15
+0xb3 0x71 0x00 0x0f
+
+# CHECK: lndfr %f15, %f0
+0xb3 0x71 0x00 0xf0
+
+# CHECK: lndfr %f15, %f9
+0xb3 0x71 0x00 0xf9
+
 # CHECK: lndr %f0, %f9
 0x21 0x09
 
@@ -9909,6 +9993,18 @@
 # CHECK: lpdbr %f15, %f9
 0xb3 0x10 0x00 0xf9
 
+# CHECK: lpdfr %f0, %f9
+0xb3 0x70 0x00 0x09
+
+# CHECK: lpdfr %f0, %f15
+0xb3 0x70 0x00 0x0f
+
+# CHECK: lpdfr %f15, %f0
+0xb3 0x70 0x00 0xf0
+
+# CHECK: lpdfr %f15, %f9
+0xb3 0x70 0x00 0xf9
+
 # CHECK: lpdg %r0, 0, 0
 0xc8 0x05 0x00 0x00 0x00 0x00
 
diff --git a/llvm/test/MC/Disassembler/SystemZ/invalid-regs.txt b/llvm/test/MC/Disassembler/SystemZ/invalid-regs.txt
index 12440677ba34..0ce960d5ddcc 100644
--- a/llvm/test/MC/Disassembler/SystemZ/invalid-regs.txt
+++ b/llvm/test/MC/Disassembler/SystemZ/invalid-regs.txt
@@ -20,3 +20,10 @@
 # CHECK-NEXT: 0xb9 0x97 0x00 0x18
 # CHECK-NEXT: ^
 0xb9 0x97 0x00 0x18
+
+# This would be "cutfu %r1, %r2", but %r1 is invalid.
+#
+# CHECK-NEXT: warning: invalid instruction encoding
+# CHECK-NEXT: 0xb2 0xa7 0x00 0x12
+# CHECK-NEXT: ^
+0xb2 0xa7 0x00 0x12
diff --git a/llvm/test/MC/SystemZ/insn-good.s b/llvm/test/MC/SystemZ/insn-good.s
index 91120f7f97aa..f5dd672f9dd8 100644
--- a/llvm/test/MC/SystemZ/insn-good.s
+++ b/llvm/test/MC/SystemZ/insn-good.s
@@ -9038,6 +9038,36 @@
 	lcdbr	%f15,%f0
 	lcdbr	%f15,%f9
 
+#CHECK: lcdfr	%f0, %f9                # encoding: [0xb3,0x73,0x00,0x09]
+#CHECK: lcdfr	%f0, %f15               # encoding: [0xb3,0x73,0x00,0x0f]
+#CHECK: lcdfr	%f15, %f0               # encoding: [0xb3,0x73,0x00,0xf0]
+#CHECK: lcdfr	%f15, %f9               # encoding: [0xb3,0x73,0x00,0xf9]
+
+	lcdfr	%f0,%f9
+	lcdfr	%f0,%f15
+	lcdfr	%f15,%f0
+	lcdfr	%f15,%f9
+
+#CHECK: lndfr	%f0, %f9                # encoding: [0xb3,0x71,0x00,0x09]
+#CHECK: lndfr	%f0, %f15               # encoding: [0xb3,0x71,0x00,0x0f]
+#CHECK: lndfr	%f15, %f0               # encoding: [0xb3,0x71,0x00,0xf0]
+#CHECK: lndfr	%f15, %f9               # encoding: [0xb3,0x71,0x00,0xf9]
+
+	lndfr	%f0,%f9
+	lndfr	%f0,%f15
+	lndfr	%f15,%f0
+	lndfr	%f15,%f9
+
+#CHECK: lpdfr	%f0, %f9                # encoding: [0xb3,0x70,0x00,0x09]
+#CHECK: lpdfr	%f0, %f15               # encoding: [0xb3,0x70,0x00,0x0f]
+#CHECK: lpdfr	%f15, %f0               # encoding: [0xb3,0x70,0x00,0xf0]
+#CHECK: lpdfr	%f15, %f9               # encoding: [0xb3,0x70,0x00,0xf9]
+
+	lpdfr	%f0,%f9
+	lpdfr	%f0,%f15
+	lpdfr	%f15,%f0
+	lpdfr	%f15,%f9
+
 #CHECK: lcdr	%f0, %f9                # encoding: [0x23,0x09]
 #CHECK: lcdr	%f0, %f15               # encoding: [0x23,0x0f]
 #CHECK: lcdr	%f15, %f0               # encoding: [0x23,0xf0]
-- 
GitLab


From b794dc23255505dd7735f995b8ff1192305a072e Mon Sep 17 00:00:00 2001
From: Dominik Steenken <dost@de.ibm.com>
Date: Fri, 12 Apr 2024 18:05:30 +0200
Subject: [PATCH 082/464] [SystemZ] Add custom handling of legal vectors with
 reduce-add. (#88495)

This commit skips the expansion of the `vector.reduce.add` intrinsic on
vector-enabled SystemZ targets in order to introduce custom handling of
`vector.reduce.add` for legal vector types using the VSUM instructions.
This is limited to full vectors with scalar types up to `i32` due to
performance concerns.

It also adds testing for the generation of such custom handling, and
adapts the related cost computation, as well as the testing for that.

The expected result is a performance boost in certain benchmarks that
make heavy use of `vector.reduce.add` with other benchmarks remaining
constant.

For instance, the assembly for `vector.reduce.add<4 x i32>` changes from
```hlasm
        vmrlg   %v0, %v24, %v24
        vaf     %v0, %v24, %v0
        vrepf   %v1, %v0, 1
        vaf     %v0, %v0, %v1
        vlgvf   %r2, %v0, 0
```
to
```hlasm
        vgbm    %v0, 0
        vsumqf  %v0, %v24, %v0
        vlgvf   %r2, %v0, 3
```
---
 .../Target/SystemZ/SystemZISelLowering.cpp    |  43 +++
 llvm/lib/Target/SystemZ/SystemZISelLowering.h |   1 +
 .../SystemZ/SystemZTargetTransformInfo.cpp    |  31 +-
 .../SystemZ/SystemZTargetTransformInfo.h      |   2 +
 .../Analysis/CostModel/SystemZ/reduce-add.ll  |  24 +-
 .../test/CodeGen/SystemZ/vec-reduce-add-01.ll | 289 ++++++++++++++++++
 6 files changed, 373 insertions(+), 17 deletions(-)
 create mode 100644 llvm/test/CodeGen/SystemZ/vec-reduce-add-01.ll

diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index efffd669d268..3b3057f8fe96 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -16,6 +16,7 @@
 #include "SystemZMachineFunctionInfo.h"
 #include "SystemZTargetMachine.h"
 #include "llvm/CodeGen/CallingConvLower.h"
+#include "llvm/CodeGen/ISDOpcodes.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
@@ -23,6 +24,7 @@
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/IntrinsicsS390.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/KnownBits.h"
 #include <cctype>
 #include <optional>
@@ -451,6 +453,10 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::SRL, VT, Custom);
       setOperationAction(ISD::ROTL, VT, Custom);
 
+      // Add ISD::VECREDUCE_ADD as custom in order to implement
+      // it with VZERO+VSUM
+      setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
+
       // Map SETCCs onto one of VCE, VCH or VCHL, swapping the operands
       // and inverting the result as necessary.
       setOperationAction(ISD::SETCC, VT, Custom);
@@ -6167,6 +6173,8 @@ SDValue SystemZTargetLowering::LowerOperation(SDValue Op,
     return lowerOR(Op, DAG);
   case ISD::CTPOP:
     return lowerCTPOP(Op, DAG);
+  case ISD::VECREDUCE_ADD:
+    return lowerVECREDUCE_ADD(Op, DAG);
   case ISD::ATOMIC_FENCE:
     return lowerATOMIC_FENCE(Op, DAG);
   case ISD::ATOMIC_SWAP:
@@ -9600,3 +9608,38 @@ SDValue SystemZTargetLowering::lowerGET_ROUNDING(SDValue Op,
 
   return DAG.getMergeValues({RetVal, Chain}, dl);
 }
+
+SDValue SystemZTargetLowering::lowerVECREDUCE_ADD(SDValue Op,
+                                                  SelectionDAG &DAG) const {
+  EVT VT = Op.getValueType();
+  Op = Op.getOperand(0);
+  EVT OpVT = Op.getValueType();
+
+  assert(OpVT.isVector() && "Operand type for VECREDUCE_ADD is not a vector.");
+
+  SDLoc DL(Op);
+
+  // load a 0 vector for the third operand of VSUM.
+  SDValue Zero = DAG.getSplatBuildVector(OpVT, DL, DAG.getConstant(0, DL, VT));
+
+  // execute VSUM.
+  switch (OpVT.getScalarSizeInBits()) {
+  case 8:
+  case 16:
+    Op = DAG.getNode(SystemZISD::VSUM, DL, MVT::v4i32, Op, Zero);
+    LLVM_FALLTHROUGH;
+  case 32:
+  case 64:
+    Op = DAG.getNode(SystemZISD::VSUM, DL, MVT::i128, Op,
+                     DAG.getBitcast(Op.getValueType(), Zero));
+    break;
+  case 128:
+    break; // VSUM over v1i128 should not happen and would be a noop
+  default:
+    llvm_unreachable("Unexpected scalar size.");
+  }
+  // Cast to original vector type, retrieve last element.
+  return DAG.getNode(
+      ISD::EXTRACT_VECTOR_ELT, DL, VT, DAG.getBitcast(OpVT, Op),
+      DAG.getConstant(OpVT.getVectorNumElements() - 1, DL, MVT::i32));
+}
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
index 7140287a886c..2290a7d62e89 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
@@ -696,6 +696,7 @@ private:
   SDValue lowerBITCAST(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerOR(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerCTPOP(SDValue Op, SelectionDAG &DAG) const;
+  SDValue lowerVECREDUCE_ADD(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerATOMIC_LDST_I128(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerATOMIC_LOAD_OP(SDValue Op, SelectionDAG &DAG,
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
index 17e534f405c0..4c9e78c05dbc 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
@@ -19,6 +19,7 @@
 #include "llvm/CodeGen/CostTable.h"
 #include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Intrinsics.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/MathExtras.h"
 
@@ -1293,18 +1294,14 @@ getVectorIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy,
   if (ID == Intrinsic::vector_reduce_add) {
     // Retrieve number and size of elements for the vector op.
     auto *VTy = cast<FixedVectorType>(ParamTys.front());
-    unsigned NumElements = VTy->getNumElements();
     unsigned ScalarSize = VTy->getScalarSizeInBits();
     // For scalar sizes >128 bits, we fall back to the generic cost estimate.
     if (ScalarSize > SystemZ::VectorBits)
       return -1;
-    // A single vector register can hold this many elements.
-    unsigned MaxElemsPerVector = SystemZ::VectorBits / ScalarSize;
     // This many vector regs are needed to represent the input elements (V).
     unsigned VectorRegsNeeded = getNumVectorRegs(VTy);
     // This many instructions are needed for the final sum of vector elems (S).
-    unsigned LastVectorHandling =
-        2 * Log2_32_Ceil(std::min(NumElements, MaxElemsPerVector));
+    unsigned LastVectorHandling = (ScalarSize < 32) ? 3 : 2;
     // We use vector adds to create a sum vector, which takes
     // V/2 + V/4 + ... = V - 1 operations.
     // Then, we need S operations to sum up the elements of that sum vector,
@@ -1324,3 +1321,27 @@ SystemZTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
     return Cost;
   return BaseT::getIntrinsicInstrCost(ICA, CostKind);
 }
+
+bool SystemZTTIImpl::shouldExpandReduction(const IntrinsicInst *II) const {
+  // Always expand on Subtargets without vector instructions
+  if (!ST->hasVector())
+    return true;
+
+  // Always expand for operands that do not fill one vector reg
+  auto *Type = cast<FixedVectorType>(II->getOperand(0)->getType());
+  unsigned NumElts = Type->getNumElements();
+  unsigned ScalarSize = Type->getScalarSizeInBits();
+  unsigned MaxElts = SystemZ::VectorBits / ScalarSize;
+  if (NumElts < MaxElts)
+    return true;
+
+  // Otherwise
+  switch (II->getIntrinsicID()) {
+  // Do not expand vector.reduce.add
+  case Intrinsic::vector_reduce_add:
+    // Except for i64, since the performance benefit is dubious there
+    return ScalarSize >= 64;
+  default:
+    return true;
+  }
+}
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
index 1d824d353d8f..696d887c1d5d 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
@@ -127,6 +127,8 @@ public:
 
   InstructionCost getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
                                         TTI::TargetCostKind CostKind);
+
+  bool shouldExpandReduction(const IntrinsicInst *II) const;
   /// @}
 };
 
diff --git a/llvm/test/Analysis/CostModel/SystemZ/reduce-add.ll b/llvm/test/Analysis/CostModel/SystemZ/reduce-add.ll
index 061e5ece44a4..90b5b746c914 100644
--- a/llvm/test/Analysis/CostModel/SystemZ/reduce-add.ll
+++ b/llvm/test/Analysis/CostModel/SystemZ/reduce-add.ll
@@ -7,19 +7,19 @@ define void @reduce(ptr %src, ptr %dst) {
 ; CHECK:  Cost Model: Found an estimated cost of 5 for instruction: %R8_64 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %V8_64)
 ; CHECK:  Cost Model: Found an estimated cost of 9 for instruction: %R16_64 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %V16_64)
 ; CHECK:  Cost Model: Found an estimated cost of 2 for instruction: %R2_32 = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %V2_32)
-; CHECK:  Cost Model: Found an estimated cost of 4 for instruction: %R4_32 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %V4_32)
-; CHECK:  Cost Model: Found an estimated cost of 5 for instruction: %R8_32 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %V8_32)
-; CHECK:  Cost Model: Found an estimated cost of 7 for instruction: %R16_32 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %V16_32)
-; CHECK:  Cost Model: Found an estimated cost of 2 for instruction: %R2_16 = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> %V2_16)
-; CHECK:  Cost Model: Found an estimated cost of 4 for instruction: %R4_16 = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> %V4_16)
-; CHECK:  Cost Model: Found an estimated cost of 6 for instruction: %R8_16 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %V8_16)
-; CHECK:  Cost Model: Found an estimated cost of 7 for instruction: %R16_16 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %V16_16)
-; CHECK:  Cost Model: Found an estimated cost of 2 for instruction: %R2_8 = call i8 @llvm.vector.reduce.add.v2i8(<2 x i8> %V2_8)
-; CHECK:  Cost Model: Found an estimated cost of 4 for instruction: %R4_8 = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> %V4_8)
-; CHECK:  Cost Model: Found an estimated cost of 6 for instruction: %R8_8 = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> %V8_8)
-; CHECK:  Cost Model: Found an estimated cost of 8 for instruction: %R16_8 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %V16_8)
+; CHECK:  Cost Model: Found an estimated cost of 2 for instruction: %R4_32 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %V4_32)
+; CHECK:  Cost Model: Found an estimated cost of 3 for instruction: %R8_32 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %V8_32)
+; CHECK:  Cost Model: Found an estimated cost of 5 for instruction: %R16_32 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %V16_32)
+; CHECK:  Cost Model: Found an estimated cost of 3 for instruction: %R2_16 = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> %V2_16)
+; CHECK:  Cost Model: Found an estimated cost of 3 for instruction: %R4_16 = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> %V4_16)
+; CHECK:  Cost Model: Found an estimated cost of 3 for instruction: %R8_16 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %V8_16)
+; CHECK:  Cost Model: Found an estimated cost of 4 for instruction: %R16_16 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %V16_16)
+; CHECK:  Cost Model: Found an estimated cost of 3 for instruction: %R2_8 = call i8 @llvm.vector.reduce.add.v2i8(<2 x i8> %V2_8)
+; CHECK:  Cost Model: Found an estimated cost of 3 for instruction: %R4_8 = call i8 @llvm.vector.reduce.add.v4i8(<4 x i8> %V4_8)
+; CHECK:  Cost Model: Found an estimated cost of 3 for instruction: %R8_8 = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> %V8_8)
+; CHECK:  Cost Model: Found an estimated cost of 3 for instruction: %R16_8 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %V16_8)
 ;
-; CHECK:  Cost Model: Found an estimated cost of 15 for instruction: %R128_8 = call i8 @llvm.vector.reduce.add.v128i8(<128 x i8> %V128_8)
+; CHECK:  Cost Model: Found an estimated cost of 10 for instruction: %R128_8 = call i8 @llvm.vector.reduce.add.v128i8(<128 x i8> %V128_8)
 ; CHECK:  Cost Model: Found an estimated cost of 20 for instruction: %R4_256 = call i256 @llvm.vector.reduce.add.v4i256(<4 x i256> %V4_256)
 
   ; REDUCEADD64
diff --git a/llvm/test/CodeGen/SystemZ/vec-reduce-add-01.ll b/llvm/test/CodeGen/SystemZ/vec-reduce-add-01.ll
new file mode 100644
index 000000000000..56b151d7f941
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/vec-reduce-add-01.ll
@@ -0,0 +1,289 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; Test vector add reduction instrinsic
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z16 | FileCheck %s
+
+; 1 vector length
+declare i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %a)
+declare i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %a)
+declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %a)
+declare i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a)
+declare i128 @llvm.vector.reduce.add.v1i128(<1 x i128> %a)
+; 2 vector lengths
+declare i8 @llvm.vector.reduce.add.v32i8(<32 x i8> %a)
+declare i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %a)
+declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %a)
+declare i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %a)
+declare i128 @llvm.vector.reduce.add.v2i128(<2 x i128> %a)
+; ; TODO
+; ; 4 vector lengths
+declare i8 @llvm.vector.reduce.add.v64i8(<64 x i8> %a)
+declare i16 @llvm.vector.reduce.add.v32i16(<32 x i16> %a)
+declare i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %a)
+declare i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %a)
+declare i128 @llvm.vector.reduce.add.v4i128(<4 x i128> %a)
+; ; Subvector lengths
+declare i8 @llvm.vector.reduce.add.v8i8(<8 x i8> %a)
+declare i16 @llvm.vector.reduce.add.v4i16(<4 x i16> %a)
+declare i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %a)
+declare i64 @llvm.vector.reduce.add.v1i64(<1 x i64> %a)
+
+define i8 @f1_1(<16 x i8> %a) {
+; CHECK-LABEL: f1_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vgbm %v0, 0
+; CHECK-NEXT:    vsumb %v1, %v24, %v0
+; CHECK-NEXT:    vsumqf %v0, %v1, %v0
+; CHECK-NEXT:    vlgvf %r2, %v0, 3
+; CHECK-NEXT:    # kill: def $r2l killed $r2l killed $r2d
+; CHECK-NEXT:    br %r14
+  %redadd = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %a)
+  ret i8 %redadd
+}
+
+define i16 @f1_2(<8 x i16> %a) {
+; CHECK-LABEL: f1_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vgbm %v0, 0
+; CHECK-NEXT:    vsumh %v1, %v24, %v0
+; CHECK-NEXT:    vsumqf %v0, %v1, %v0
+; CHECK-NEXT:    vlgvf %r2, %v0, 3
+; CHECK-NEXT:    # kill: def $r2l killed $r2l killed $r2d
+; CHECK-NEXT:    br %r14
+  %redadd = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %a)
+  ret i16 %redadd
+}
+
+define i32 @f1_3(<4 x i32> %a) {
+; CHECK-LABEL: f1_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vgbm %v0, 0
+; CHECK-NEXT:    vsumqf %v0, %v24, %v0
+; CHECK-NEXT:    vlgvf %r2, %v0, 3
+; CHECK-NEXT:    # kill: def $r2l killed $r2l killed $r2d
+; CHECK-NEXT:    br %r14
+
+  %redadd = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %a)
+  ret i32 %redadd
+}
+
+define i64 @f1_4(<2 x i64> %a) {
+; CHECK-LABEL: f1_4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vrepg %v0, %v24, 1
+; CHECK-NEXT:    vag %v0, %v24, %v0
+; CHECK-NEXT:    vlgvg %r2, %v0, 0
+; CHECK-NEXT:    br %r14
+
+  %redadd = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %a)
+  ret i64 %redadd
+}
+
+define i128 @f1_5(<1 x i128> %a) {
+; CHECK-LABEL: f1_5:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vst %v24, 0(%r2), 3
+; CHECK-NEXT:    br %r14
+  %redadd = call i128 @llvm.vector.reduce.add.v1i128(<1 x i128> %a)
+  ret i128 %redadd
+}
+
+define i8 @f2_1(<32 x i8> %a) {
+; CHECK-LABEL: f2_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vab %v0, %v24, %v26
+; CHECK-NEXT:    vgbm %v1, 0
+; CHECK-NEXT:    vsumb %v0, %v0, %v1
+; CHECK-NEXT:    vsumqf %v0, %v0, %v1
+; CHECK-NEXT:    vlgvf %r2, %v0, 3
+; CHECK-NEXT:    # kill: def $r2l killed $r2l killed $r2d
+; CHECK-NEXT:    br %r14
+  %redadd = call i8 @llvm.vector.reduce.add.v32i8(<32 x i8> %a)
+  ret i8 %redadd
+}
+
+define i16 @f2_2(<16 x i16> %a) {
+; CHECK-LABEL: f2_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vah %v0, %v24, %v26
+; CHECK-NEXT:    vgbm %v1, 0
+; CHECK-NEXT:    vsumh %v0, %v0, %v1
+; CHECK-NEXT:    vsumqf %v0, %v0, %v1
+; CHECK-NEXT:    vlgvf %r2, %v0, 3
+; CHECK-NEXT:    # kill: def $r2l killed $r2l killed $r2d
+; CHECK-NEXT:    br %r14
+  %redadd = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %a)
+  ret i16 %redadd
+}
+
+define i32 @f2_3(<8 x i32> %a) {
+; CHECK-LABEL: f2_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vaf %v0, %v24, %v26
+; CHECK-NEXT:    vgbm %v1, 0
+; CHECK-NEXT:    vsumqf %v0, %v0, %v1
+; CHECK-NEXT:    vlgvf %r2, %v0, 3
+; CHECK-NEXT:    # kill: def $r2l killed $r2l killed $r2d
+; CHECK-NEXT:    br %r14
+
+  %redadd = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %a)
+  ret i32 %redadd
+}
+
+define i64 @f2_4(<4 x i64> %a) {
+; CHECK-LABEL: f2_4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vag %v0, %v24, %v26
+; CHECK-NEXT:    vrepg %v1, %v0, 1
+; CHECK-NEXT:    vag %v0, %v0, %v1
+; CHECK-NEXT:    vlgvg %r2, %v0, 0
+; CHECK-NEXT:    br %r14
+
+  %redadd = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %a)
+  ret i64 %redadd
+}
+
+define i128 @f2_5(<2 x i128> %a) {
+; CHECK-LABEL: f2_5:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl %v0, 16(%r3), 3
+; CHECK-NEXT:    vl %v1, 0(%r3), 3
+; CHECK-NEXT:    vaq %v0, %v1, %v0
+; CHECK-NEXT:    vst %v0, 0(%r2), 3
+; CHECK-NEXT:    br %r14
+  %redadd = call i128 @llvm.vector.reduce.add.v2i128(<2 x i128> %a)
+  ret i128 %redadd
+}
+
+define i8 @f3_1(<64 x i8> %a) {
+; CHECK-LABEL: f3_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vab %v0, %v26, %v30
+; CHECK-NEXT:    vab %v1, %v24, %v28
+; CHECK-NEXT:    vab %v0, %v1, %v0
+; CHECK-NEXT:    vgbm %v1, 0
+; CHECK-NEXT:    vsumb %v0, %v0, %v1
+; CHECK-NEXT:    vsumqf %v0, %v0, %v1
+; CHECK-NEXT:    vlgvf %r2, %v0, 3
+; CHECK-NEXT:    # kill: def $r2l killed $r2l killed $r2d
+; CHECK-NEXT:    br %r14
+  %redadd = call i8 @llvm.vector.reduce.add.v64i8(<64 x i8> %a)
+  ret i8 %redadd
+}
+
+define i16 @f3_2(<32 x i16> %a) {
+; CHECK-LABEL: f3_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vah %v0, %v26, %v30
+; CHECK-NEXT:    vah %v1, %v24, %v28
+; CHECK-NEXT:    vah %v0, %v1, %v0
+; CHECK-NEXT:    vgbm %v1, 0
+; CHECK-NEXT:    vsumh %v0, %v0, %v1
+; CHECK-NEXT:    vsumqf %v0, %v0, %v1
+; CHECK-NEXT:    vlgvf %r2, %v0, 3
+; CHECK-NEXT:    # kill: def $r2l killed $r2l killed $r2d
+; CHECK-NEXT:    br %r14
+  %redadd = call i16 @llvm.vector.reduce.add.v32i16(<32 x i16> %a)
+  ret i16 %redadd
+}
+
+define i32 @f3_3(<16 x i32> %a) {
+; CHECK-LABEL: f3_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vaf %v0, %v26, %v30
+; CHECK-NEXT:    vaf %v1, %v24, %v28
+; CHECK-NEXT:    vaf %v0, %v1, %v0
+; CHECK-NEXT:    vgbm %v1, 0
+; CHECK-NEXT:    vsumqf %v0, %v0, %v1
+; CHECK-NEXT:    vlgvf %r2, %v0, 3
+; CHECK-NEXT:    # kill: def $r2l killed $r2l killed $r2d
+; CHECK-NEXT:    br %r14
+
+  %redadd = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %a)
+  ret i32 %redadd
+}
+
+define i64 @f3_4(<8 x i64> %a) {
+; CHECK-LABEL: f3_4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vag %v0, %v26, %v30
+; CHECK-NEXT:    vag %v1, %v24, %v28
+; CHECK-NEXT:    vag %v0, %v1, %v0
+; CHECK-NEXT:    vrepg %v1, %v0, 1
+; CHECK-NEXT:    vag %v0, %v0, %v1
+; CHECK-NEXT:    vlgvg %r2, %v0, 0
+; CHECK-NEXT:    br %r14
+
+  %redadd = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %a)
+  ret i64 %redadd
+}
+
+define i128 @f3_5(<4 x i128> %a) {
+; CHECK-LABEL: f3_5:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl %v0, 32(%r3), 3
+; CHECK-NEXT:    vl %v1, 0(%r3), 3
+; CHECK-NEXT:    vl %v2, 48(%r3), 3
+; CHECK-NEXT:    vl %v3, 16(%r3), 3
+; CHECK-NEXT:    vaq %v2, %v3, %v2
+; CHECK-NEXT:    vaq %v0, %v1, %v0
+; CHECK-NEXT:    vaq %v0, %v0, %v2
+; CHECK-NEXT:    vst %v0, 0(%r2), 3
+; CHECK-NEXT:    br %r14
+  %redadd = call i128 @llvm.vector.reduce.add.v4i128(<4 x i128> %a)
+  ret i128 %redadd
+}
+
+
+define i8 @f4_1(<8 x i8> %a) {
+; CHECK-LABEL: f4_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpkg %v0, %v24, %v24
+; CHECK-NEXT:    vab %v0, %v24, %v0
+; CHECK-NEXT:    vpkf %v1, %v0, %v0
+; CHECK-NEXT:    vab %v0, %v0, %v1
+; CHECK-NEXT:    vrepb %v1, %v0, 1
+; CHECK-NEXT:    vab %v0, %v0, %v1
+; CHECK-NEXT:    vlgvb %r2, %v0, 0
+; CHECK-NEXT:    # kill: def $r2l killed $r2l killed $r2d
+; CHECK-NEXT:    br %r14
+  %redadd = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> %a)
+  ret i8 %redadd
+}
+
+define i16 @f4_2(<4 x i16> %a) {
+; CHECK-LABEL: f4_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpkg %v0, %v24, %v24
+; CHECK-NEXT:    vah %v0, %v24, %v0
+; CHECK-NEXT:    vreph %v1, %v0, 1
+; CHECK-NEXT:    vah %v0, %v0, %v1
+; CHECK-NEXT:    vlgvh %r2, %v0, 0
+; CHECK-NEXT:    # kill: def $r2l killed $r2l killed $r2d
+; CHECK-NEXT:    br %r14
+  %redadd = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> %a)
+  ret i16 %redadd
+}
+
+define i32 @f4_3(<2 x i32> %a) {
+; CHECK-LABEL: f4_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vrepf %v0, %v24, 1
+; CHECK-NEXT:    vaf %v0, %v24, %v0
+; CHECK-NEXT:    vlgvf %r2, %v0, 0
+; CHECK-NEXT:    # kill: def $r2l killed $r2l killed $r2d
+; CHECK-NEXT:    br %r14
+
+  %redadd = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %a)
+  ret i32 %redadd
+}
+
+define i64 @f4_4(<1 x i64> %a) {
+; CHECK-LABEL: f4_4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vlgvg %r2, %v24, 0
+; CHECK-NEXT:    br %r14
+
+  %redadd = call i64 @llvm.vector.reduce.add.v1i64(<1 x i64> %a)
+  ret i64 %redadd
+}
-- 
GitLab


From 8d468c132eed7ffe34d601b224220efd51655eb3 Mon Sep 17 00:00:00 2001
From: Evan Wilde <etceterawilde@gmail.com>
Date: Fri, 12 Apr 2024 09:05:59 -0700
Subject: [PATCH 083/464] NFC: Make clang resource headers an interface library
 (#88317)

Making the clang resource headers into an interface library instead of a
custom target means that we can attach the header search paths to the
library. Targets that "link" against this library will automatically
have the appropriate paths added to their header search paths to find
them. Then downstream projects that embed a copy of clang can query the
generated `ClangTargets.cmake` file for the header location instead of
attempting to compute them.
---
 clang/lib/Headers/CMakeLists.txt | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/clang/lib/Headers/CMakeLists.txt b/clang/lib/Headers/CMakeLists.txt
index 97104ccd8db5..e6ae4e19e81d 100644
--- a/clang/lib/Headers/CMakeLists.txt
+++ b/clang/lib/Headers/CMakeLists.txt
@@ -437,14 +437,14 @@ foreach( f ${generated_files} )
 endforeach( f )
 
 function(add_header_target target_name file_list)
-  add_custom_target(${target_name} DEPENDS ${file_list})
+  add_library(${target_name} INTERFACE ${file_list})
   set_target_properties(${target_name} PROPERTIES
     FOLDER "Misc"
     RUNTIME_OUTPUT_DIRECTORY "${output_dir}")
 endfunction()
 
 # The catch-all clang-resource-headers target
-add_custom_target("clang-resource-headers" ALL DEPENDS ${out_files})
+add_library(clang-resource-headers INTERFACE ${out_files})
 set_target_properties("clang-resource-headers" PROPERTIES
   FOLDER "Misc"
   RUNTIME_OUTPUT_DIRECTORY "${output_dir}")
@@ -501,6 +501,10 @@ add_header_target("windows-resource-headers" ${windows_only_files})
 add_header_target("utility-resource-headers" ${utility_files})
 
 get_clang_resource_dir(header_install_dir SUBDIR include)
+target_include_directories(clang-resource-headers INTERFACE
+  $<BUILD_INTERFACE:${output_dir}>
+  $<INSTALL_INTERFACE:${header_install_dir}>)
+set_property(GLOBAL APPEND PROPERTY CLANG_EXPORTS clang-resource-headers)
 
 #############################################################
 # Install rules for the catch-all clang-resource-headers target
-- 
GitLab


From 05d8b5e62d7a1cf2f94582346d6fd2d39667a26b Mon Sep 17 00:00:00 2001
From: JOSTAR <52376093+shenjunjiekoda@users.noreply.github.com>
Date: Sat, 13 Apr 2024 00:15:33 +0800
Subject: [PATCH 084/464] [analyzer] Support `PointerType` in
 `getCXXRecordDecl` for `ContainerModeling` (#87787)

---
 .../lib/StaticAnalyzer/Checkers/ContainerModeling.cpp  |  4 ++++
 clang/test/Analysis/invalidated-iterator.cpp           | 10 +++++++++-
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/clang/lib/StaticAnalyzer/Checkers/ContainerModeling.cpp b/clang/lib/StaticAnalyzer/Checkers/ContainerModeling.cpp
index 65a2ec4076fd..009c0d3fb936 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ContainerModeling.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ContainerModeling.cpp
@@ -770,6 +770,10 @@ const CXXRecordDecl *getCXXRecordDecl(ProgramStateRef State,
     Type = RefT->getPointeeType();
   }
 
+  if (const auto *PtrT = Type->getAs<PointerType>()) {
+    Type = PtrT->getPointeeType();
+  }
+
   return Type->getUnqualifiedDesugaredType()->getAsCXXRecordDecl();
 }
 
diff --git a/clang/test/Analysis/invalidated-iterator.cpp b/clang/test/Analysis/invalidated-iterator.cpp
index 778a8e01d993..c940dbf7276d 100644
--- a/clang/test/Analysis/invalidated-iterator.cpp
+++ b/clang/test/Analysis/invalidated-iterator.cpp
@@ -130,6 +130,14 @@ struct cont_with_ptr_iterator {
   T* erase(T*);
 };
 
+void invalidated_access_via_end_iterator_after_push_back() {
+  cont_with_ptr_iterator<int> C;
+  C.push_back(1);
+  auto i = C.end();
+  C.push_back(2);
+  auto j = i[-1]; // expected-warning{{Invalidated iterator accessed}}
+}
+
 void invalidated_dereference_end_ptr_iterator(cont_with_ptr_iterator<int> &C) {
   auto i = C.begin();
   C.erase(i);
@@ -196,4 +204,4 @@ void invalidated_subscript_end_ptr_iterator(cont_with_ptr_iterator<int> &C) {
   auto i = C.begin();
   C.erase(i);
   (void) i[1]; // expected-warning{{Invalidated iterator accessed}}
-}
+}
\ No newline at end of file
-- 
GitLab


From 37575f5262d0e37a1e0b10c59fb8f59182cfe951 Mon Sep 17 00:00:00 2001
From: Jordan Rupprecht <rupprecht@google.com>
Date: Fri, 12 Apr 2024 16:27:53 +0000
Subject: [PATCH 085/464] [NFC][ValueTracking] Fix Wunused-variable

For e0a628715a8464e220c8ba9e9aaaf2561139198a
---
 llvm/lib/Analysis/InstructionSimplify.cpp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index e83657df20f2..4e6e66692267 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -2713,8 +2713,6 @@ static Constant *computePointerICmp(CmpInst::Predicate Pred, Value *LHS,
   assert(LHS->getType() == RHS->getType() && "Must have same types");
   const DataLayout &DL = Q.DL;
   const TargetLibraryInfo *TLI = Q.TLI;
-  const DominatorTree *DT = Q.DT;
-  const Instruction *CxtI = Q.CxtI;
 
   // We can only fold certain predicates on pointer comparisons.
   switch (Pred) {
-- 
GitLab


From 54a6798e0a3630e705ed32dbbd63414a16331085 Mon Sep 17 00:00:00 2001
From: Edwin Vane <revane@google.com>
Date: Fri, 12 Apr 2024 12:49:25 -0400
Subject: [PATCH 086/464] [clang-tidy] Simplify RenamerClangTidyCheck API
 (#88268)

Some functions allow a null SourceManager, no SourceManager, or a
SourceManager in an inconsistent argument position. Since SourceManager
is generally not null and it doesn't make sense to apply renaming
without one, these inconsistencies are now gone.
---
 .../utils/RenamerClangTidyCheck.cpp           | 46 +++++++++++--------
 .../clang-tidy/utils/RenamerClangTidyCheck.h  | 11 +++--
 2 files changed, 32 insertions(+), 25 deletions(-)

diff --git a/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.cpp b/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.cpp
index ad8048e2a92b..962a243ce94d 100644
--- a/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.cpp
+++ b/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.cpp
@@ -169,14 +169,14 @@ public:
       return;
     if (SM.isWrittenInCommandLineFile(MacroNameTok.getLocation()))
       return;
-    Check->checkMacro(SM, MacroNameTok, Info);
+    Check->checkMacro(MacroNameTok, Info, SM);
   }
 
   /// MacroExpands calls expandMacro for macros in the main file
   void MacroExpands(const Token &MacroNameTok, const MacroDefinition &MD,
                     SourceRange /*Range*/,
                     const MacroArgs * /*Args*/) override {
-    Check->expandMacro(MacroNameTok, MD.getMacroInfo());
+    Check->expandMacro(MacroNameTok, MD.getMacroInfo(), SM);
   }
 
 private:
@@ -187,7 +187,7 @@ private:
 class RenamerClangTidyVisitor
     : public RecursiveASTVisitor<RenamerClangTidyVisitor> {
 public:
-  RenamerClangTidyVisitor(RenamerClangTidyCheck *Check, const SourceManager *SM,
+  RenamerClangTidyVisitor(RenamerClangTidyCheck *Check, const SourceManager &SM,
                           bool AggressiveDependentMemberLookup)
       : Check(Check), SM(SM),
         AggressiveDependentMemberLookup(AggressiveDependentMemberLookup) {}
@@ -258,7 +258,7 @@ public:
     // Fix overridden methods
     if (const auto *Method = dyn_cast<CXXMethodDecl>(Decl)) {
       if (const CXXMethodDecl *Overridden = getOverrideMethod(Method)) {
-        Check->addUsage(Overridden, Method->getLocation());
+        Check->addUsage(Overridden, Method->getLocation(), SM);
         return true; // Don't try to add the actual decl as a Failure.
       }
     }
@@ -268,7 +268,7 @@ public:
     if (isa<ClassTemplateSpecializationDecl>(Decl))
       return true;
 
-    Check->checkNamedDecl(Decl, *SM);
+    Check->checkNamedDecl(Decl, SM);
     return true;
   }
 
@@ -385,7 +385,7 @@ public:
 
 private:
   RenamerClangTidyCheck *Check;
-  const SourceManager *SM;
+  const SourceManager &SM;
   const bool AggressiveDependentMemberLookup;
 };
 
@@ -415,7 +415,7 @@ void RenamerClangTidyCheck::registerPPCallbacks(
 
 void RenamerClangTidyCheck::addUsage(
     const RenamerClangTidyCheck::NamingCheckId &Decl, SourceRange Range,
-    const SourceManager *SourceMgr) {
+    const SourceManager &SourceMgr) {
   // Do nothing if the provided range is invalid.
   if (Range.isInvalid())
     return;
@@ -425,8 +425,7 @@ void RenamerClangTidyCheck::addUsage(
   // spelling location to different source locations, and we only want to fix
   // the token once, before it is expanded by the macro.
   SourceLocation FixLocation = Range.getBegin();
-  if (SourceMgr)
-    FixLocation = SourceMgr->getSpellingLoc(FixLocation);
+  FixLocation = SourceMgr.getSpellingLoc(FixLocation);
   if (FixLocation.isInvalid())
     return;
 
@@ -440,15 +439,15 @@ void RenamerClangTidyCheck::addUsage(
   if (!Failure.shouldFix())
     return;
 
-  if (SourceMgr && SourceMgr->isWrittenInScratchSpace(FixLocation))
+  if (SourceMgr.isWrittenInScratchSpace(FixLocation))
     Failure.FixStatus = RenamerClangTidyCheck::ShouldFixStatus::InsideMacro;
 
-  if (!utils::rangeCanBeFixed(Range, SourceMgr))
+  if (!utils::rangeCanBeFixed(Range, &SourceMgr))
     Failure.FixStatus = RenamerClangTidyCheck::ShouldFixStatus::InsideMacro;
 }
 
 void RenamerClangTidyCheck::addUsage(const NamedDecl *Decl, SourceRange Range,
-                                     const SourceManager *SourceMgr) {
+                                     const SourceManager &SourceMgr) {
   // Don't keep track for non-identifier names.
   auto *II = Decl->getIdentifier();
   if (!II)
@@ -489,18 +488,24 @@ void RenamerClangTidyCheck::checkNamedDecl(const NamedDecl *Decl,
   }
 
   Failure.Info = std::move(Info);
-  addUsage(Decl, Range, &SourceMgr);
+  addUsage(Decl, Range, SourceMgr);
 }
 
 void RenamerClangTidyCheck::check(const MatchFinder::MatchResult &Result) {
-  RenamerClangTidyVisitor Visitor(this, Result.SourceManager,
+  if (!Result.SourceManager) {
+    // In principle SourceManager is not null but going only by the definition
+    // of MatchResult it must be handled. Cannot rename anything without a
+    // SourceManager.
+    return;
+  }
+  RenamerClangTidyVisitor Visitor(this, *Result.SourceManager,
                                   AggressiveDependentMemberLookup);
   Visitor.TraverseAST(*Result.Context);
 }
 
-void RenamerClangTidyCheck::checkMacro(const SourceManager &SourceMgr,
-                                       const Token &MacroNameTok,
-                                       const MacroInfo *MI) {
+void RenamerClangTidyCheck::checkMacro(const Token &MacroNameTok,
+                                       const MacroInfo *MI,
+                                       const SourceManager &SourceMgr) {
   std::optional<FailureInfo> MaybeFailure =
       getMacroFailureInfo(MacroNameTok, SourceMgr);
   if (!MaybeFailure)
@@ -515,11 +520,12 @@ void RenamerClangTidyCheck::checkMacro(const SourceManager &SourceMgr,
     Failure.FixStatus = ShouldFixStatus::FixInvalidIdentifier;
 
   Failure.Info = std::move(Info);
-  addUsage(ID, Range);
+  addUsage(ID, Range, SourceMgr);
 }
 
 void RenamerClangTidyCheck::expandMacro(const Token &MacroNameTok,
-                                        const MacroInfo *MI) {
+                                        const MacroInfo *MI,
+                                        const SourceManager &SourceMgr) {
   StringRef Name = MacroNameTok.getIdentifierInfo()->getName();
   NamingCheckId ID(MI->getDefinitionLoc(), Name);
 
@@ -528,7 +534,7 @@ void RenamerClangTidyCheck::expandMacro(const Token &MacroNameTok,
     return;
 
   SourceRange Range(MacroNameTok.getLocation(), MacroNameTok.getEndLoc());
-  addUsage(ID, Range);
+  addUsage(ID, Range, SourceMgr);
 }
 
 static std::string
diff --git a/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.h b/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.h
index 38228fb59bf6..be5b6f0c7f76 100644
--- a/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.h
+++ b/clang-tools-extra/clang-tidy/utils/RenamerClangTidyCheck.h
@@ -108,18 +108,19 @@ public:
       llvm::DenseMap<NamingCheckId, NamingCheckFailure>;
 
   /// Check Macros for style violations.
-  void checkMacro(const SourceManager &SourceMgr, const Token &MacroNameTok,
-                  const MacroInfo *MI);
+  void checkMacro(const Token &MacroNameTok, const MacroInfo *MI,
+                  const SourceManager &SourceMgr);
 
   /// Add a usage of a macro if it already has a violation.
-  void expandMacro(const Token &MacroNameTok, const MacroInfo *MI);
+  void expandMacro(const Token &MacroNameTok, const MacroInfo *MI,
+                   const SourceManager &SourceMgr);
 
   void addUsage(const RenamerClangTidyCheck::NamingCheckId &Decl,
-                SourceRange Range, const SourceManager *SourceMgr = nullptr);
+                SourceRange Range, const SourceManager &SourceMgr);
 
   /// Convenience method when the usage to be added is a NamedDecl.
   void addUsage(const NamedDecl *Decl, SourceRange Range,
-                const SourceManager *SourceMgr = nullptr);
+                const SourceManager &SourceMgr);
 
   void checkNamedDecl(const NamedDecl *Decl, const SourceManager &SourceMgr);
 
-- 
GitLab


From 3652b2a877c3691b524a19bc38b338f1ba1dde37 Mon Sep 17 00:00:00 2001
From: Mike Rice <michael.p.rice@intel.com>
Date: Fri, 12 Apr 2024 10:01:01 -0700
Subject: [PATCH 087/464] [clang][CodeGen][OpenMP] Fix casting of atomic update
 of ptr types (#88215)

In 4d5e834c5b7f0ccccd90a6d543e182df602f6bc8, casts were removed for
pointers but one case was missed. Add missing check.
---
 clang/lib/CodeGen/CGAtomic.cpp              |  6 +++++-
 clang/test/OpenMP/atomic_update_codegen.cpp | 11 +++++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/clang/lib/CodeGen/CGAtomic.cpp b/clang/lib/CodeGen/CGAtomic.cpp
index d35ce0409d72..07452b18a85e 100644
--- a/clang/lib/CodeGen/CGAtomic.cpp
+++ b/clang/lib/CodeGen/CGAtomic.cpp
@@ -1806,7 +1806,11 @@ void AtomicInfo::EmitAtomicUpdateOp(
                                              /*NumReservedValues=*/2);
   PHI->addIncoming(OldVal, CurBB);
   Address NewAtomicAddr = CreateTempAlloca();
-  Address NewAtomicIntAddr = castToAtomicIntPointer(NewAtomicAddr);
+  Address NewAtomicIntAddr =
+      shouldCastToInt(NewAtomicAddr.getElementType(), /*CmpXchg=*/true)
+          ? castToAtomicIntPointer(NewAtomicAddr)
+          : NewAtomicAddr;
+
   if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
       requiresMemSetZero(getAtomicAddress().getElementType())) {
     CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
diff --git a/clang/test/OpenMP/atomic_update_codegen.cpp b/clang/test/OpenMP/atomic_update_codegen.cpp
index ce0765118922..fe745590a9f9 100644
--- a/clang/test/OpenMP/atomic_update_codegen.cpp
+++ b/clang/test/OpenMP/atomic_update_codegen.cpp
@@ -27,6 +27,7 @@ long double ldv, ldx;
 _Complex int civ, cix;
 _Complex float cfv, cfx;
 _Complex double cdv, cdx;
+char *cpx;
 
 typedef int int4 __attribute__((__vector_size__(16)));
 int4 int4x;
@@ -851,6 +852,16 @@ int main(void) {
 // CHECK: call{{.*}} @__kmpc_flush(
 #pragma omp atomic seq_cst
   rix = dv / rix;
+
+// CHECK: [[LD_CPX:%.+]] = load atomic ptr, ptr @cpx monotonic
+// CHECK: br label %[[CONT:.+]]
+// CHECK: [[CONT]]
+// CHECK: [[PHI:%.+]] = phi ptr
+// CHECK: [[RES:%.+]] = cmpxchg ptr @cpx,
+// CHECK: br i1 %{{.+}}, label %[[EXIT:.+]], label %[[CONT]]
+  #pragma omp atomic update
+  cpx += 1;
+
   return 0;
 }
 
-- 
GitLab


From d61edecbfd099143e0e2617505fec921524938f8 Mon Sep 17 00:00:00 2001
From: Owen Pan <owenpiano@gmail.com>
Date: Fri, 12 Apr 2024 10:12:24 -0700
Subject: [PATCH 088/464] [clang-format] Fix a regression in
 ContinuationIndenter (#88414)

Commit d06b92391513 caused a regression that breaks after a block
comment adjacent to a function paramter that follows.

Fixes #86573.
---
 clang/lib/Format/ContinuationIndenter.cpp     | 8 +++++++-
 clang/unittests/Format/FormatTestComments.cpp | 4 ++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/clang/lib/Format/ContinuationIndenter.cpp b/clang/lib/Format/ContinuationIndenter.cpp
index 700bce35c868..ad0e2c3c620c 100644
--- a/clang/lib/Format/ContinuationIndenter.cpp
+++ b/clang/lib/Format/ContinuationIndenter.cpp
@@ -684,7 +684,13 @@ void ContinuationIndenter::addTokenOnCurrentLine(LineState &State, bool DryRun,
         // arguments to function calls. We do this by ensuring that either all
         // arguments (including any lambdas) go on the same line as the function
         // call, or we break before the first argument.
-        auto PrevNonComment = Current.getPreviousNonComment();
+        const auto *Prev = Current.Previous;
+        if (!Prev)
+          return false;
+        // For example, `/*Newline=*/false`.
+        if (Prev->is(TT_BlockComment) && Current.SpacesRequiredBefore == 0)
+          return false;
+        const auto *PrevNonComment = Current.getPreviousNonComment();
         if (!PrevNonComment || PrevNonComment->isNot(tok::l_paren))
           return false;
         if (Current.isOneOf(tok::comment, tok::l_paren, TT_LambdaLSquare))
diff --git a/clang/unittests/Format/FormatTestComments.cpp b/clang/unittests/Format/FormatTestComments.cpp
index d705cf34d8af..d2baace6a7d8 100644
--- a/clang/unittests/Format/FormatTestComments.cpp
+++ b/clang/unittests/Format/FormatTestComments.cpp
@@ -376,6 +376,10 @@ TEST_F(FormatTestComments, RemovesTrailingWhitespaceOfComments) {
 TEST_F(FormatTestComments, UnderstandsBlockComments) {
   verifyFormat("f(/*noSpaceAfterParameterNamingComment=*/true);");
   verifyFormat("void f() { g(/*aaa=*/x, /*bbb=*/!y, /*c=*/::c); }");
+  verifyFormat("fooooooooooooooooooooooooooooo(\n"
+               "    /*qq_=*/move(q), [this, b](bar<void(uint32_t)> b) {},\n"
+               "    c);",
+               getLLVMStyleWithColumns(60));
   EXPECT_EQ("f(aaaaaaaaaaaaaaaaaaaaaaaaa, /* Trailing comment for aa... */\n"
             "  bbbbbbbbbbbbbbbbbbbbbbbbb);",
             format("f(aaaaaaaaaaaaaaaaaaaaaaaaa ,   \\\n"
-- 
GitLab


From dcd097c475163b9bd41ff009a9157e86c6f2f171 Mon Sep 17 00:00:00 2001
From: Stanislav Mekhanoshin <rampitec@users.noreply.github.com>
Date: Fri, 12 Apr 2024 10:15:59 -0700
Subject: [PATCH 089/464] Add IIT_V6 to support 6-element vectors in
 intrinsics. (#88196)

Needed for the future patch.
---
 llvm/include/llvm/IR/Intrinsics.td | 1 +
 llvm/lib/IR/Function.cpp           | 4 ++++
 2 files changed, 5 insertions(+)

diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index f0723a633f0f..bdd8465883fc 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -319,6 +319,7 @@ def IIT_FUNCREF : IIT_VT<funcref, 55>;
 def IIT_I2 : IIT_Int<2, 57>;
 def IIT_I4 : IIT_Int<4, 58>;
 def IIT_AARCH64_SVCOUNT : IIT_VT<aarch64svcount, 59>;
+def IIT_V6 : IIT_Vec<6, 60>;
 }
 
 defvar IIT_all_FixedTypes = !filter(iit, IIT_all,
diff --git a/llvm/lib/IR/Function.cpp b/llvm/lib/IR/Function.cpp
index b5fda9bb3d12..96953ac49c19 100644
--- a/llvm/lib/IR/Function.cpp
+++ b/llvm/lib/IR/Function.cpp
@@ -1172,6 +1172,10 @@ static void DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos,
     OutputTable.push_back(IITDescriptor::getVector(4, IsScalableVector));
     DecodeIITType(NextElt, Infos, Info, OutputTable);
     return;
+  case IIT_V6:
+    OutputTable.push_back(IITDescriptor::getVector(6, IsScalableVector));
+    DecodeIITType(NextElt, Infos, Info, OutputTable);
+    return;
   case IIT_V8:
     OutputTable.push_back(IITDescriptor::getVector(8, IsScalableVector));
     DecodeIITType(NextElt, Infos, Info, OutputTable);
-- 
GitLab


From 6a85cf8fc0437d4885fc829948befe32c1e5a21d Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Fri, 12 Apr 2024 10:22:21 -0700
Subject: [PATCH 090/464] [SelectionDAG] Verify SPLAT_VECTOR nodes when they
 are created. (#88305)

This applies the same rules we have for the scalar operands of a
BUILD_VECTOR where the scalar type must match the element type or for
integer vectors we allow the scalar type to be larger than the element
type. Hexagon uses i32 for an FP zero vector so we allow that as an
exception.
---
 llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index ddb12a101e02..412e1de9fc41 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -6032,6 +6032,17 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
     if (N1.getValueType().getScalarType() == MVT::i1)
       return getNode(ISD::VECREDUCE_AND, DL, VT, N1);
     break;
+  case ISD::SPLAT_VECTOR:
+    assert(VT.isVector() && "Wrong return type!");
+    // FIXME: Hexagon uses i32 scalar for a floating point zero vector so allow
+    // that for now.
+    assert((VT.getVectorElementType() == N1.getValueType() ||
+            (VT.isFloatingPoint() && N1.getValueType() == MVT::i32) ||
+            (VT.getVectorElementType().isInteger() &&
+             N1.getValueType().isInteger() &&
+             VT.getVectorElementType().bitsLE(N1.getValueType()))) &&
+           "Wrong operand type!");
+    break;
   }
 
   SDNode *N;
-- 
GitLab


From 7cafe04e0d74c1e9f7e3871a0bcdf4ccb1c89f0c Mon Sep 17 00:00:00 2001
From: Xiaoyang Liu <siujoeng.lau@gmail.com>
Date: Fri, 12 Apr 2024 13:25:22 -0400
Subject: [PATCH 091/464] [libc++] P3029R1: Better `mdspan`'s CTAD (#87873)

## Abstract

This pull request implements [P3029R1](https://wg21.link/P3029R1). The
paper discusses the current behavior of `mdspan`'s most common
pointer-indices CTAD, where the `Extents` template parameter is deduced
as `dextents` (dynamic extents), even when passing compile-time constant
values. The author believes this behavior is suboptimal, as it doesn't
take advantage of the compile-time information. The proposed change
suggests deducing static extents if `integral_constant`-like constants
are passed, resulting in more intuitive syntax and less error-prone
code.

## Reference

- [P3029R1](https://wg21.link/P3029R1)
- [Draft C++ Standard: [span.syn]](https://eel.is/c++draft/span.syn)
- [Draft C++ Standard: [mdspan.syn]](https://eel.is/c++draft/mdspan.syn)
---
 libcxx/docs/ReleaseNotes/19.rst               |  1 +
 libcxx/docs/Status/Cxx2cPapers.csv            |  2 +-
 libcxx/include/__mdspan/mdspan.h              |  7 +++
 libcxx/include/mdspan                         |  6 ++-
 libcxx/include/span                           | 44 ++++++++++++++++++-
 .../views/mdspan/mdspan/deduction.pass.cpp    | 18 +++++++-
 .../views.span/span.cons/deduct.pass.cpp      | 11 +++++
 7 files changed, 84 insertions(+), 5 deletions(-)

diff --git a/libcxx/docs/ReleaseNotes/19.rst b/libcxx/docs/ReleaseNotes/19.rst
index 81c05b9112bd..7bc0148c9ff0 100644
--- a/libcxx/docs/ReleaseNotes/19.rst
+++ b/libcxx/docs/ReleaseNotes/19.rst
@@ -47,6 +47,7 @@ Implemented Papers
 - P3142R0 - Printing Blank Lines with ``println`` (as DR against C++23)
 - P2302R4 - ``std::ranges::contains``
 - P1659R3 - ``std::ranges::starts_with`` and ``std::ranges::ends_with``
+- P3029R1 - Better ``mdspan``'s CTAD
 
 Improvements and New Features
 -----------------------------
diff --git a/libcxx/docs/Status/Cxx2cPapers.csv b/libcxx/docs/Status/Cxx2cPapers.csv
index fa11da62bc08..8516070c72b8 100644
--- a/libcxx/docs/Status/Cxx2cPapers.csv
+++ b/libcxx/docs/Status/Cxx2cPapers.csv
@@ -61,5 +61,5 @@
 "`P1068R11 <https://wg21.link/P1068R11>`__","LWG","Vector API for random number generation","Tokyo March 2024","","",""
 "`P2944R3 <https://wg21.link/P2944R3>`__","LWG","Comparisons for ``reference_wrapper``","Tokyo March 2024","","",""
 "`P2642R6 <https://wg21.link/P2642R6>`__","LWG","Padded ``mdspan`` layouts","Tokyo March 2024","","",""
-"`P3029R1 <https://wg21.link/P3029R1>`__","LWG","Better ``mdspan``'s CTAD","Tokyo March 2024","","",""
+"`P3029R1 <https://wg21.link/P3029R1>`__","LWG","Better ``mdspan``'s CTAD","Tokyo March 2024","|Complete|","19.0",""
 "","","","","","",""
diff --git a/libcxx/include/__mdspan/mdspan.h b/libcxx/include/__mdspan/mdspan.h
index d9a0108d6d35..09665af316e7 100644
--- a/libcxx/include/__mdspan/mdspan.h
+++ b/libcxx/include/__mdspan/mdspan.h
@@ -266,10 +266,17 @@ private:
   friend class mdspan;
 };
 
+#  if _LIBCPP_STD_VER >= 26
+template <class _ElementType, class... _OtherIndexTypes>
+  requires((is_convertible_v<_OtherIndexTypes, size_t> && ...) && (sizeof...(_OtherIndexTypes) > 0))
+explicit mdspan(_ElementType*,
+                _OtherIndexTypes...) -> mdspan<_ElementType, extents<size_t, __maybe_static_ext<_OtherIndexTypes>...>>;
+#  else
 template <class _ElementType, class... _OtherIndexTypes>
   requires((is_convertible_v<_OtherIndexTypes, size_t> && ...) && (sizeof...(_OtherIndexTypes) > 0))
 explicit mdspan(_ElementType*, _OtherIndexTypes...)
     -> mdspan<_ElementType, dextents<size_t, sizeof...(_OtherIndexTypes)>>;
+#  endif
 
 template <class _Pointer>
   requires(is_pointer_v<remove_reference_t<_Pointer>>)
diff --git a/libcxx/include/mdspan b/libcxx/include/mdspan
index c13d9eef001a..8d443f4acd1d 100644
--- a/libcxx/include/mdspan
+++ b/libcxx/include/mdspan
@@ -370,7 +370,11 @@ namespace std {
   template<class ElementType, class... Integrals>
     requires((is_convertible_v<Integrals, size_t> && ...) && sizeof...(Integrals) > 0)
     explicit mdspan(ElementType*, Integrals...)
-      -> mdspan<ElementType, dextents<size_t, sizeof...(Integrals)>>;
+      -> mdspan<ElementType, dextents<size_t, sizeof...(Integrals)>>;            // until C++26
+  template<class ElementType, class... Integrals>
+    requires((is_convertible_v<Integrals, size_t> && ...) && sizeof...(Integrals) > 0)
+    explicit mdspan(ElementType*, Integrals...)
+      -> mdspan<ElementType, extents<size_t, maybe-static-ext<Integrals>...>>;  // since C++26
 
   template<class ElementType, class OtherIndexType, size_t N>
     mdspan(ElementType*, span<OtherIndexType, N>)
diff --git a/libcxx/include/span b/libcxx/include/span
index c0fe25ddb4be..0307edcb55c3 100644
--- a/libcxx/include/span
+++ b/libcxx/include/span
@@ -18,6 +18,20 @@ namespace std {
 // constants
 inline constexpr size_t dynamic_extent = numeric_limits<size_t>::max();
 
+template<class T>
+  concept integral-constant-like =                           // exposition only, since C++26
+    is_integral_v<decltype(T::value)> &&
+    !is_same_v<bool, remove_const_t<decltype(T::value)>> &&
+    convertible_to<T, decltype(T::value)> &&
+    equality_comparable_with<T, decltype(T::value)> &&
+    bool_constant<T() == T::value>::value &&
+    bool_constant<static_cast<decltype(T::value)>(T()) == T::value>::value;
+
+template<class T>
+  constexpr size_t maybe-static-ext = dynamic_extent;        // exposition only, since C++26
+template<integral-constant-like T>
+  constexpr size_t maybe-static-ext<T> = {T::value};
+
 // [views.span], class template span
 template <class ElementType, size_t Extent = dynamic_extent>
     class span;
@@ -110,7 +124,9 @@ private:
 };
 
 template<class It, class EndOrSize>
-    span(It, EndOrSize) -> span<remove_reference_t<iter_reference_t<_It>>>;
+    span(It, EndOrSize) -> span<remove_reference_t<iter_reference_t<_It>>>;                             // until C++26
+template<class It, class EndOrSize>
+    span(It, EndOrSize) -> span<remove_reference_t<iter_reference_t<It>>, maybe-static-ext<EndOrSize>>; // since C++26
 
 template<class T, size_t N>
     span(T (&)[N]) -> span<T, N>;
@@ -129,6 +145,8 @@ template<class R>
 */
 
 #include <__assert>
+#include <__concepts/convertible_to.h>
+#include <__concepts/equality_comparable.h>
 #include <__config>
 #include <__fwd/array.h>
 #include <__fwd/span.h>
@@ -143,15 +161,19 @@ template<class R>
 #include <__ranges/enable_borrowed_range.h>
 #include <__ranges/enable_view.h>
 #include <__ranges/size.h>
+#include <__type_traits/integral_constant.h>
 #include <__type_traits/is_array.h>
 #include <__type_traits/is_const.h>
 #include <__type_traits/is_convertible.h>
+#include <__type_traits/is_integral.h>
+#include <__type_traits/is_same.h>
+#include <__type_traits/remove_const.h>
 #include <__type_traits/remove_cv.h>
 #include <__type_traits/remove_cvref.h>
 #include <__type_traits/remove_reference.h>
 #include <__type_traits/type_identity.h>
 #include <__utility/forward.h>
-#include <cstddef>      // for byte
+#include <cstddef> // for byte
 #include <initializer_list>
 #include <stdexcept>
 #include <version>
@@ -563,8 +585,26 @@ _LIBCPP_HIDE_FROM_ABI auto as_writable_bytes(span<_Tp, _Extent> __s) noexcept {
   return __s.__as_writable_bytes();
 }
 
+#  if _LIBCPP_STD_VER >= 26
+template <class _Tp>
+concept __integral_constant_like =
+    is_integral_v<decltype(_Tp::value)> && !is_same_v<bool, remove_const_t<decltype(_Tp::value)>> &&
+    convertible_to<_Tp, decltype(_Tp::value)> && equality_comparable_with<_Tp, decltype(_Tp::value)> &&
+    bool_constant<_Tp() == _Tp::value>::value &&
+    bool_constant<static_cast<decltype(_Tp::value)>(_Tp()) == _Tp::value>::value;
+
+template <class _Tp>
+inline constexpr size_t __maybe_static_ext = dynamic_extent;
+
+template <__integral_constant_like _Tp>
+inline constexpr size_t __maybe_static_ext<_Tp> = {_Tp::value};
+
+template <contiguous_iterator _It, class _EndOrSize>
+span(_It, _EndOrSize) -> span<remove_reference_t<iter_reference_t<_It>>, __maybe_static_ext<_EndOrSize>>;
+#  else
 template <contiguous_iterator _It, class _EndOrSize>
 span(_It, _EndOrSize) -> span<remove_reference_t<iter_reference_t<_It>>>;
+#  endif
 
 template <class _Tp, size_t _Sz>
 span(_Tp (&)[_Sz]) -> span<_Tp, _Sz>;
diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/deduction.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/deduction.pass.cpp
index 557829dcbad9..876a3e84d695 100644
--- a/libcxx/test/std/containers/views/mdspan/mdspan/deduction.pass.cpp
+++ b/libcxx/test/std/containers/views/mdspan/mdspan/deduction.pass.cpp
@@ -22,7 +22,11 @@
 //  template<class ElementType, class... Integrals>
 //    requires((is_convertible_v<Integrals, size_t> && ...) && sizeof...(Integrals) > 0)
 //    explicit mdspan(ElementType*, Integrals...)
-//      -> mdspan<ElementType, dextents<size_t, sizeof...(Integrals)>>;
+//      -> mdspan<ElementType, dextents<size_t, sizeof...(Integrals)>>;            // until C++26
+//  template<class ElementType, class... Integrals>
+//    requires((is_convertible_v<Integrals, size_t> && ...) && sizeof...(Integrals) > 0)
+//    explicit mdspan(ElementType*, Integrals...)
+//      -> mdspan<ElementType, extents<size_t, maybe-static-ext<Integrals>...>>;  // since C++26
 //
 //  template<class ElementType, class OtherIndexType, size_t N>
 //    mdspan(ElementType*, span<OtherIndexType, N>)
@@ -102,6 +106,18 @@ constexpr bool test_no_layout_deduction_guides(const H& handle, const A&) {
   // deduction from pointer and integral like
   ASSERT_SAME_TYPE(decltype(std::mdspan(handle, 5, SizeTIntType(6))), std::mdspan<T, std::dextents<size_t, 2>>);
 
+#if _LIBCPP_STD_VER >= 26
+  // P3029R1: deduction from `integral_constant`
+  ASSERT_SAME_TYPE(
+      decltype(std::mdspan(handle, std::integral_constant<size_t, 5>{})), std::mdspan<T, std::extents<size_t, 5>>);
+  ASSERT_SAME_TYPE(decltype(std::mdspan(handle, std::integral_constant<size_t, 5>{}, std::dynamic_extent)),
+                   std::mdspan<T, std::extents<size_t, 5, std::dynamic_extent>>);
+  ASSERT_SAME_TYPE(
+      decltype(std::mdspan(
+          handle, std::integral_constant<size_t, 5>{}, std::dynamic_extent, std::integral_constant<size_t, 7>{})),
+      std::mdspan<T, std::extents<size_t, 5, std::dynamic_extent, 7>>);
+#endif
+
   std::array<char, 3> exts;
   // deduction from pointer and array
   ASSERT_SAME_TYPE(decltype(std::mdspan(handle, exts)), std::mdspan<T, std::dextents<size_t, 3>>);
diff --git a/libcxx/test/std/containers/views/views.span/span.cons/deduct.pass.cpp b/libcxx/test/std/containers/views/views.span/span.cons/deduct.pass.cpp
index 398862dccdb6..92bb7c9caea8 100644
--- a/libcxx/test/std/containers/views/views.span/span.cons/deduct.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.cons/deduct.pass.cpp
@@ -31,6 +31,7 @@
 #include <iterator>
 #include <memory>
 #include <string>
+#include <type_traits>
 
 #include "test_macros.h"
 
@@ -48,6 +49,16 @@ void test_iterator_sentinel() {
   assert(s.size() == std::size(arr));
   assert(s.data() == std::data(arr));
   }
+
+#if _LIBCPP_STD_VER >= 26
+  // P3029R1: deduction from `integral_constant`
+  {
+    std::span s{std::begin(arr), std::integral_constant<size_t, 3>{}};
+    ASSERT_SAME_TYPE(decltype(s), std::span<int, 3>);
+    assert(s.size() == std::size(arr));
+    assert(s.data() == std::data(arr));
+  }
+#endif
 }
 
 void test_c_array() {
-- 
GitLab


From fe59cb256289d9f2f6ebe75571389f3cc90bbdec Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda@apple.com>
Date: Fri, 12 Apr 2024 09:43:10 -0700
Subject: [PATCH 092/464] [clang][deps] NFCI: Extract `FilenameForLookup` into
 new function

This is an NFC change split from https://github.com/llvm/llvm-project/pull/68645.
---
 .../DependencyScanningFilesystem.h            |  4 ++
 .../DependencyScanningFilesystem.cpp          | 41 ++++++++++++-------
 2 files changed, 30 insertions(+), 15 deletions(-)

diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
index 9a522a3e2fe2..4cd0f958fcff 100644
--- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
+++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
@@ -402,6 +402,10 @@ private:
   llvm::ErrorOr<std::string> WorkingDirForCacheLookup;
 
   void updateWorkingDirForCacheLookup();
+
+  llvm::ErrorOr<StringRef>
+  tryGetFilenameForLookup(StringRef OriginalFilename,
+                          llvm::SmallVectorImpl<char> &PathBuf) const;
 };
 
 } // end namespace dependencies
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
index 9b7812a1adb9..c66780d50fa1 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
@@ -233,24 +233,15 @@ DependencyScanningWorkerFilesystem::computeAndStoreResult(
 llvm::ErrorOr<EntryRef>
 DependencyScanningWorkerFilesystem::getOrCreateFileSystemEntry(
     StringRef OriginalFilename) {
-  StringRef FilenameForLookup;
   SmallString<256> PathBuf;
-  if (llvm::sys::path::is_absolute_gnu(OriginalFilename)) {
-    FilenameForLookup = OriginalFilename;
-  } else if (!WorkingDirForCacheLookup) {
-    return WorkingDirForCacheLookup.getError();
-  } else {
-    StringRef RelFilename = OriginalFilename;
-    RelFilename.consume_front("./");
-    PathBuf = *WorkingDirForCacheLookup;
-    llvm::sys::path::append(PathBuf, RelFilename);
-    FilenameForLookup = PathBuf.str();
-  }
-  assert(llvm::sys::path::is_absolute_gnu(FilenameForLookup));
+  auto FilenameForLookup = tryGetFilenameForLookup(OriginalFilename, PathBuf);
+  if (!FilenameForLookup)
+    return FilenameForLookup.getError();
+
   if (const auto *Entry =
-          findEntryByFilenameWithWriteThrough(FilenameForLookup))
+          findEntryByFilenameWithWriteThrough(*FilenameForLookup))
     return EntryRef(OriginalFilename, *Entry).unwrapError();
-  auto MaybeEntry = computeAndStoreResult(OriginalFilename, FilenameForLookup);
+  auto MaybeEntry = computeAndStoreResult(OriginalFilename, *FilenameForLookup);
   if (!MaybeEntry)
     return MaybeEntry.getError();
   return EntryRef(OriginalFilename, *MaybeEntry).unwrapError();
@@ -351,4 +342,24 @@ void DependencyScanningWorkerFilesystem::updateWorkingDirForCacheLookup() {
          llvm::sys::path::is_absolute_gnu(*WorkingDirForCacheLookup));
 }
 
+llvm::ErrorOr<StringRef>
+DependencyScanningWorkerFilesystem::tryGetFilenameForLookup(
+    StringRef OriginalFilename, llvm::SmallVectorImpl<char> &PathBuf) const {
+  StringRef FilenameForLookup;
+  if (llvm::sys::path::is_absolute_gnu(OriginalFilename)) {
+    FilenameForLookup = OriginalFilename;
+  } else if (!WorkingDirForCacheLookup) {
+    return WorkingDirForCacheLookup.getError();
+  } else {
+    StringRef RelFilename = OriginalFilename;
+    RelFilename.consume_front("./");
+    PathBuf.assign(WorkingDirForCacheLookup->begin(),
+                   WorkingDirForCacheLookup->end());
+    llvm::sys::path::append(PathBuf, RelFilename);
+    FilenameForLookup = StringRef{PathBuf.begin(), PathBuf.size()};
+  }
+  assert(llvm::sys::path::is_absolute_gnu(FilenameForLookup));
+  return FilenameForLookup;
+}
+
 const char DependencyScanningWorkerFilesystem::ID = 0;
-- 
GitLab


From edd7fed9da48c0e708cce9bd4d305ae43d8bd77c Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda@apple.com>
Date: Fri, 12 Apr 2024 10:09:04 -0700
Subject: [PATCH 093/464] [llvm][vfs] NFCI: Remove `const` from
 `VFS::getRealPath()`

This is an NFC change split from https://github.com/llvm/llvm-project/pull/68645.
---
 llvm/include/llvm/Support/VirtualFileSystem.h | 10 ++++-----
 llvm/lib/Support/FileCollector.cpp            |  2 +-
 llvm/lib/Support/VirtualFileSystem.cpp        | 21 ++++++++-----------
 .../Support/VirtualFileSystemTest.cpp         |  2 +-
 4 files changed, 16 insertions(+), 19 deletions(-)

diff --git a/llvm/include/llvm/Support/VirtualFileSystem.h b/llvm/include/llvm/Support/VirtualFileSystem.h
index 770ca8764426..4b1ca0c3d262 100644
--- a/llvm/include/llvm/Support/VirtualFileSystem.h
+++ b/llvm/include/llvm/Support/VirtualFileSystem.h
@@ -298,7 +298,7 @@ public:
   /// symlinks. For real file system, this uses `llvm::sys::fs::real_path`.
   /// This returns errc::operation_not_permitted if not implemented by subclass.
   virtual std::error_code getRealPath(const Twine &Path,
-                                      SmallVectorImpl<char> &Output) const;
+                                      SmallVectorImpl<char> &Output);
 
   /// Check whether a file exists. Provided for convenience.
   bool exists(const Twine &Path);
@@ -393,7 +393,7 @@ public:
   std::error_code setCurrentWorkingDirectory(const Twine &Path) override;
   std::error_code isLocal(const Twine &Path, bool &Result) override;
   std::error_code getRealPath(const Twine &Path,
-                              SmallVectorImpl<char> &Output) const override;
+                              SmallVectorImpl<char> &Output) override;
 
   using iterator = FileSystemList::reverse_iterator;
   using const_iterator = FileSystemList::const_reverse_iterator;
@@ -453,7 +453,7 @@ public:
     return FS->setCurrentWorkingDirectory(Path);
   }
   std::error_code getRealPath(const Twine &Path,
-                              SmallVectorImpl<char> &Output) const override {
+                              SmallVectorImpl<char> &Output) override {
     return FS->getRealPath(Path, Output);
   }
   std::error_code isLocal(const Twine &Path, bool &Result) override {
@@ -622,7 +622,7 @@ public:
   /// This doesn't resolve symlinks as they are not supported in in-memory file
   /// system.
   std::error_code getRealPath(const Twine &Path,
-                              SmallVectorImpl<char> &Output) const override;
+                              SmallVectorImpl<char> &Output) override;
   std::error_code isLocal(const Twine &Path, bool &Result) override;
   std::error_code setCurrentWorkingDirectory(const Twine &Path) override;
 
@@ -1047,7 +1047,7 @@ public:
   ErrorOr<std::unique_ptr<File>> openFileForRead(const Twine &Path) override;
 
   std::error_code getRealPath(const Twine &Path,
-                              SmallVectorImpl<char> &Output) const override;
+                              SmallVectorImpl<char> &Output) override;
 
   llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override;
 
diff --git a/llvm/lib/Support/FileCollector.cpp b/llvm/lib/Support/FileCollector.cpp
index 92bcdf00b8a8..be0b06b0aff8 100644
--- a/llvm/lib/Support/FileCollector.cpp
+++ b/llvm/lib/Support/FileCollector.cpp
@@ -281,7 +281,7 @@ public:
   }
 
   std::error_code getRealPath(const Twine &Path,
-                              SmallVectorImpl<char> &Output) const override {
+                              SmallVectorImpl<char> &Output) override {
     auto EC = FS->getRealPath(Path, Output);
     if (!EC) {
       Collector->addFile(Path);
diff --git a/llvm/lib/Support/VirtualFileSystem.cpp b/llvm/lib/Support/VirtualFileSystem.cpp
index 057f8eae0552..32b480028e71 100644
--- a/llvm/lib/Support/VirtualFileSystem.cpp
+++ b/llvm/lib/Support/VirtualFileSystem.cpp
@@ -138,7 +138,7 @@ std::error_code FileSystem::makeAbsolute(SmallVectorImpl<char> &Path) const {
 }
 
 std::error_code FileSystem::getRealPath(const Twine &Path,
-                                        SmallVectorImpl<char> &Output) const {
+                                        SmallVectorImpl<char> &Output) {
   return errc::operation_not_permitted;
 }
 
@@ -275,7 +275,7 @@ public:
   std::error_code setCurrentWorkingDirectory(const Twine &Path) override;
   std::error_code isLocal(const Twine &Path, bool &Result) override;
   std::error_code getRealPath(const Twine &Path,
-                              SmallVectorImpl<char> &Output) const override;
+                              SmallVectorImpl<char> &Output) override;
 
 protected:
   void printImpl(raw_ostream &OS, PrintType Type,
@@ -357,9 +357,8 @@ std::error_code RealFileSystem::isLocal(const Twine &Path, bool &Result) {
   return llvm::sys::fs::is_local(adjustPath(Path, Storage), Result);
 }
 
-std::error_code
-RealFileSystem::getRealPath(const Twine &Path,
-                            SmallVectorImpl<char> &Output) const {
+std::error_code RealFileSystem::getRealPath(const Twine &Path,
+                                            SmallVectorImpl<char> &Output) {
   SmallString<256> Storage;
   return llvm::sys::fs::real_path(adjustPath(Path, Storage), Output);
 }
@@ -471,9 +470,8 @@ std::error_code OverlayFileSystem::isLocal(const Twine &Path, bool &Result) {
   return errc::no_such_file_or_directory;
 }
 
-std::error_code
-OverlayFileSystem::getRealPath(const Twine &Path,
-                               SmallVectorImpl<char> &Output) const {
+std::error_code OverlayFileSystem::getRealPath(const Twine &Path,
+                                               SmallVectorImpl<char> &Output) {
   for (const auto &FS : FSList)
     if (FS->exists(Path))
       return FS->getRealPath(Path, Output);
@@ -1157,9 +1155,8 @@ std::error_code InMemoryFileSystem::setCurrentWorkingDirectory(const Twine &P) {
   return {};
 }
 
-std::error_code
-InMemoryFileSystem::getRealPath(const Twine &Path,
-                                SmallVectorImpl<char> &Output) const {
+std::error_code InMemoryFileSystem::getRealPath(const Twine &Path,
+                                                SmallVectorImpl<char> &Output) {
   auto CWD = getCurrentWorkingDirectory();
   if (!CWD || CWD->empty())
     return errc::operation_not_permitted;
@@ -2535,7 +2532,7 @@ RedirectingFileSystem::openFileForRead(const Twine &OriginalPath) {
 
 std::error_code
 RedirectingFileSystem::getRealPath(const Twine &OriginalPath,
-                                   SmallVectorImpl<char> &Output) const {
+                                   SmallVectorImpl<char> &Output) {
   SmallString<256> Path;
   OriginalPath.toVector(Path);
 
diff --git a/llvm/unittests/Support/VirtualFileSystemTest.cpp b/llvm/unittests/Support/VirtualFileSystemTest.cpp
index 695b09343257..49a2e19e4f74 100644
--- a/llvm/unittests/Support/VirtualFileSystemTest.cpp
+++ b/llvm/unittests/Support/VirtualFileSystemTest.cpp
@@ -82,7 +82,7 @@ public:
   }
   // Map any symlink to "/symlink".
   std::error_code getRealPath(const Twine &Path,
-                              SmallVectorImpl<char> &Output) const override {
+                              SmallVectorImpl<char> &Output) override {
     auto I = findEntry(Path);
     if (I == FilesAndDirs.end())
       return make_error_code(llvm::errc::no_such_file_or_directory);
-- 
GitLab


From c11976f525f3b9b9dc6080f0b23d2ef1ec5fe8fd Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda@apple.com>
Date: Fri, 12 Apr 2024 10:26:58 -0700
Subject: [PATCH 094/464] [clang][deps] Create separate unittest directory

This is an NFC change split from https://github.com/llvm/llvm-project/pull/68645.
---
 clang/unittests/Tooling/CMakeLists.txt                          | 2 +-
 .../Tooling/{ => DependencyScanning}/DependencyScannerTest.cpp  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename clang/unittests/Tooling/{ => DependencyScanning}/DependencyScannerTest.cpp (99%)

diff --git a/clang/unittests/Tooling/CMakeLists.txt b/clang/unittests/Tooling/CMakeLists.txt
index 5a10a6b28539..2ff493ef5fc3 100644
--- a/clang/unittests/Tooling/CMakeLists.txt
+++ b/clang/unittests/Tooling/CMakeLists.txt
@@ -13,7 +13,6 @@ add_clang_unittest(ToolingTests
   CastExprTest.cpp
   CommentHandlerTest.cpp
   CompilationDatabaseTest.cpp
-  DependencyScannerTest.cpp
   DiagnosticsYamlTest.cpp
   ExecutionTest.cpp
   FixItTest.cpp
@@ -24,6 +23,7 @@ add_clang_unittest(ToolingTests
   LookupTest.cpp
   QualTypeNamesTest.cpp
   RangeSelectorTest.cpp
+  DependencyScanning/DependencyScannerTest.cpp
   RecursiveASTVisitorTests/Attr.cpp
   RecursiveASTVisitorTests/BitfieldInitializer.cpp
   RecursiveASTVisitorTests/CallbacksLeaf.cpp
diff --git a/clang/unittests/Tooling/DependencyScannerTest.cpp b/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp
similarity index 99%
rename from clang/unittests/Tooling/DependencyScannerTest.cpp
rename to clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp
index 8735fcad2004..ec0e143be4a2 100644
--- a/clang/unittests/Tooling/DependencyScannerTest.cpp
+++ b/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp
@@ -1,4 +1,4 @@
-//===- unittest/Tooling/ToolingTest.cpp - Tooling unit tests --------------===//
+//===- DependencyScannerTest.cpp ------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
-- 
GitLab


From a11a4324bb27c01e7a005e1a7f49fb8284098e8c Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda@apple.com>
Date: Fri, 12 Apr 2024 10:34:42 -0700
Subject: [PATCH 095/464] [clang][deps] Cache `VFS::getRealPath()` (#68645)

This PR starts caching calls to
`DependencyScanningWorkerFilesystem::getRealPath()` that we use whenever
we canonicalize module map path. In the case of the real VFS, this
functions performs an expensive syscall that we'd like to do as rarely
as possible.

This PR keeps the real path out of `CachedFileSystemEntry`, since that's
**immutable**; populating the real path on creation of this data
structure (every stat/open) would be expensive.
---
 .../DependencyScanningFilesystem.h            |  71 +++++++-
 .../DependencyScanningFilesystem.cpp          | 109 +++++++++++--
 clang/unittests/Tooling/CMakeLists.txt        |   1 +
 .../DependencyScanningFilesystemTest.cpp      | 152 ++++++++++++++++++
 4 files changed, 314 insertions(+), 19 deletions(-)
 create mode 100644 clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp

diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
index 4cd0f958fcff..8b6f149c7cb2 100644
--- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
+++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
@@ -142,6 +142,8 @@ private:
   CachedFileContents *Contents;
 };
 
+using CachedRealPath = llvm::ErrorOr<std::string>;
+
 /// This class is a shared cache, that caches the 'stat' and 'open' calls to the
 /// underlying real file system, and the scanned preprocessor directives of
 /// files.
@@ -154,9 +156,11 @@ public:
     /// The mutex that needs to be locked before mutation of any member.
     mutable std::mutex CacheLock;
 
-    /// Map from filenames to cached entries.
-    llvm::StringMap<const CachedFileSystemEntry *, llvm::BumpPtrAllocator>
-        EntriesByFilename;
+    /// Map from filenames to cached entries and real paths.
+    llvm::StringMap<
+        std::pair<const CachedFileSystemEntry *, const CachedRealPath *>,
+        llvm::BumpPtrAllocator>
+        CacheByFilename;
 
     /// Map from unique IDs to cached entries.
     llvm::DenseMap<llvm::sys::fs::UniqueID, const CachedFileSystemEntry *>
@@ -168,6 +172,9 @@ public:
     /// The backing storage for cached contents.
     llvm::SpecificBumpPtrAllocator<CachedFileContents> ContentsStorage;
 
+    /// The backing storage for cached real paths.
+    llvm::SpecificBumpPtrAllocator<CachedRealPath> RealPathStorage;
+
     /// Returns entry associated with the filename or nullptr if none is found.
     const CachedFileSystemEntry *findEntryByFilename(StringRef Filename) const;
 
@@ -194,6 +201,17 @@ public:
     const CachedFileSystemEntry &
     getOrInsertEntryForFilename(StringRef Filename,
                                 const CachedFileSystemEntry &Entry);
+
+    /// Returns the real path associated with the filename or nullptr if none is
+    /// found.
+    const CachedRealPath *findRealPathByFilename(StringRef Filename) const;
+
+    /// Returns the real path associated with the filename if there is some.
+    /// Otherwise, constructs new one with the given one, associates it with the
+    /// filename and returns the result.
+    const CachedRealPath &
+    getOrEmplaceRealPathForFilename(StringRef Filename,
+                                    llvm::ErrorOr<StringRef> RealPath);
   };
 
   DependencyScanningFilesystemSharedCache();
@@ -210,14 +228,17 @@ private:
 /// This class is a local cache, that caches the 'stat' and 'open' calls to the
 /// underlying real file system.
 class DependencyScanningFilesystemLocalCache {
-  llvm::StringMap<const CachedFileSystemEntry *, llvm::BumpPtrAllocator> Cache;
+  llvm::StringMap<
+      std::pair<const CachedFileSystemEntry *, const CachedRealPath *>,
+      llvm::BumpPtrAllocator>
+      Cache;
 
 public:
   /// Returns entry associated with the filename or nullptr if none is found.
   const CachedFileSystemEntry *findEntryByFilename(StringRef Filename) const {
     assert(llvm::sys::path::is_absolute_gnu(Filename));
     auto It = Cache.find(Filename);
-    return It == Cache.end() ? nullptr : It->getValue();
+    return It == Cache.end() ? nullptr : It->getValue().first;
   }
 
   /// Associates the given entry with the filename and returns the given entry
@@ -226,9 +247,40 @@ public:
   insertEntryForFilename(StringRef Filename,
                          const CachedFileSystemEntry &Entry) {
     assert(llvm::sys::path::is_absolute_gnu(Filename));
-    const auto *InsertedEntry = Cache.insert({Filename, &Entry}).first->second;
-    assert(InsertedEntry == &Entry && "entry already present");
-    return *InsertedEntry;
+    auto [It, Inserted] = Cache.insert({Filename, {&Entry, nullptr}});
+    auto &[CachedEntry, CachedRealPath] = It->getValue();
+    if (!Inserted) {
+      // The file is already present in the local cache. If we got here, it only
+      // contains the real path. Let's make sure the entry is populated too.
+      assert((!CachedEntry && CachedRealPath) && "entry already present");
+      CachedEntry = &Entry;
+    }
+    return *CachedEntry;
+  }
+
+  /// Returns real path associated with the filename or nullptr if none is
+  /// found.
+  const CachedRealPath *findRealPathByFilename(StringRef Filename) const {
+    assert(llvm::sys::path::is_absolute_gnu(Filename));
+    auto It = Cache.find(Filename);
+    return It == Cache.end() ? nullptr : It->getValue().second;
+  }
+
+  /// Associates the given real path with the filename and returns the given
+  /// entry pointer (for convenience).
+  const CachedRealPath &
+  insertRealPathForFilename(StringRef Filename,
+                            const CachedRealPath &RealPath) {
+    assert(llvm::sys::path::is_absolute_gnu(Filename));
+    auto [It, Inserted] = Cache.insert({Filename, {nullptr, &RealPath}});
+    auto &[CachedEntry, CachedRealPath] = It->getValue();
+    if (!Inserted) {
+      // The file is already present in the local cache. If we got here, it only
+      // contains the entry. Let's make sure the real path is populated too.
+      assert((!CachedRealPath && CachedEntry) && "real path already present");
+      CachedRealPath = &RealPath;
+    }
+    return *CachedRealPath;
   }
 };
 
@@ -296,6 +348,9 @@ public:
   llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
   openFileForRead(const Twine &Path) override;
 
+  std::error_code getRealPath(const Twine &Path,
+                              SmallVectorImpl<char> &Output) override;
+
   std::error_code setCurrentWorkingDirectory(const Twine &Path) override;
 
   /// Returns entry for the given filename.
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
index c66780d50fa1..84185c931b55 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
@@ -113,8 +113,8 @@ DependencyScanningFilesystemSharedCache::CacheShard::findEntryByFilename(
     StringRef Filename) const {
   assert(llvm::sys::path::is_absolute_gnu(Filename));
   std::lock_guard<std::mutex> LockGuard(CacheLock);
-  auto It = EntriesByFilename.find(Filename);
-  return It == EntriesByFilename.end() ? nullptr : It->getValue();
+  auto It = CacheByFilename.find(Filename);
+  return It == CacheByFilename.end() ? nullptr : It->getValue().first;
 }
 
 const CachedFileSystemEntry *
@@ -130,11 +130,16 @@ DependencyScanningFilesystemSharedCache::CacheShard::
     getOrEmplaceEntryForFilename(StringRef Filename,
                                  llvm::ErrorOr<llvm::vfs::Status> Stat) {
   std::lock_guard<std::mutex> LockGuard(CacheLock);
-  auto Insertion = EntriesByFilename.insert({Filename, nullptr});
-  if (Insertion.second)
-    Insertion.first->second =
+  auto [It, Inserted] = CacheByFilename.insert({Filename, {nullptr, nullptr}});
+  auto &[CachedEntry, CachedRealPath] = It->getValue();
+  if (!CachedEntry) {
+    // The entry is not present in the shared cache. Either the cache doesn't
+    // know about the file at all, or it only knows about its real path.
+    assert((Inserted || CachedRealPath) && "existing file with empty pair");
+    CachedEntry =
         new (EntryStorage.Allocate()) CachedFileSystemEntry(std::move(Stat));
-  return *Insertion.first->second;
+  }
+  return *CachedEntry;
 }
 
 const CachedFileSystemEntry &
@@ -142,16 +147,17 @@ DependencyScanningFilesystemSharedCache::CacheShard::getOrEmplaceEntryForUID(
     llvm::sys::fs::UniqueID UID, llvm::vfs::Status Stat,
     std::unique_ptr<llvm::MemoryBuffer> Contents) {
   std::lock_guard<std::mutex> LockGuard(CacheLock);
-  auto Insertion = EntriesByUID.insert({UID, nullptr});
-  if (Insertion.second) {
+  auto [It, Inserted] = EntriesByUID.insert({UID, nullptr});
+  auto &CachedEntry = It->getSecond();
+  if (Inserted) {
     CachedFileContents *StoredContents = nullptr;
     if (Contents)
       StoredContents = new (ContentsStorage.Allocate())
           CachedFileContents(std::move(Contents));
-    Insertion.first->second = new (EntryStorage.Allocate())
+    CachedEntry = new (EntryStorage.Allocate())
         CachedFileSystemEntry(std::move(Stat), StoredContents);
   }
-  return *Insertion.first->second;
+  return *CachedEntry;
 }
 
 const CachedFileSystemEntry &
@@ -159,7 +165,40 @@ DependencyScanningFilesystemSharedCache::CacheShard::
     getOrInsertEntryForFilename(StringRef Filename,
                                 const CachedFileSystemEntry &Entry) {
   std::lock_guard<std::mutex> LockGuard(CacheLock);
-  return *EntriesByFilename.insert({Filename, &Entry}).first->getValue();
+  auto [It, Inserted] = CacheByFilename.insert({Filename, {&Entry, nullptr}});
+  auto &[CachedEntry, CachedRealPath] = It->getValue();
+  if (!Inserted || !CachedEntry)
+    CachedEntry = &Entry;
+  return *CachedEntry;
+}
+
+const CachedRealPath *
+DependencyScanningFilesystemSharedCache::CacheShard::findRealPathByFilename(
+    StringRef Filename) const {
+  assert(llvm::sys::path::is_absolute_gnu(Filename));
+  std::lock_guard<std::mutex> LockGuard(CacheLock);
+  auto It = CacheByFilename.find(Filename);
+  return It == CacheByFilename.end() ? nullptr : It->getValue().second;
+}
+
+const CachedRealPath &DependencyScanningFilesystemSharedCache::CacheShard::
+    getOrEmplaceRealPathForFilename(StringRef Filename,
+                                    llvm::ErrorOr<llvm::StringRef> RealPath) {
+  std::lock_guard<std::mutex> LockGuard(CacheLock);
+
+  const CachedRealPath *&StoredRealPath = CacheByFilename[Filename].second;
+  if (!StoredRealPath) {
+    auto OwnedRealPath = [&]() -> CachedRealPath {
+      if (!RealPath)
+        return RealPath.getError();
+      return RealPath->str();
+    }();
+
+    StoredRealPath = new (RealPathStorage.Allocate())
+        CachedRealPath(std::move(OwnedRealPath));
+  }
+
+  return *StoredRealPath;
 }
 
 static bool shouldCacheStatFailures(StringRef Filename) {
@@ -321,6 +360,54 @@ DependencyScanningWorkerFilesystem::openFileForRead(const Twine &Path) {
   return DepScanFile::create(Result.get());
 }
 
+std::error_code
+DependencyScanningWorkerFilesystem::getRealPath(const Twine &Path,
+                                                SmallVectorImpl<char> &Output) {
+  SmallString<256> OwnedFilename;
+  StringRef OriginalFilename = Path.toStringRef(OwnedFilename);
+
+  SmallString<256> PathBuf;
+  auto FilenameForLookup = tryGetFilenameForLookup(OriginalFilename, PathBuf);
+  if (!FilenameForLookup)
+    return FilenameForLookup.getError();
+
+  auto HandleCachedRealPath =
+      [&Output](const CachedRealPath &RealPath) -> std::error_code {
+    if (!RealPath)
+      return RealPath.getError();
+    Output.assign(RealPath->begin(), RealPath->end());
+    return {};
+  };
+
+  // If we already have the result in local cache, no work required.
+  if (const auto *RealPath =
+          LocalCache.findRealPathByFilename(*FilenameForLookup))
+    return HandleCachedRealPath(*RealPath);
+
+  // If we have the result in the shared cache, cache it locally.
+  auto &Shard = SharedCache.getShardForFilename(*FilenameForLookup);
+  if (const auto *ShardRealPath =
+          Shard.findRealPathByFilename(*FilenameForLookup)) {
+    const auto &RealPath = LocalCache.insertRealPathForFilename(
+        *FilenameForLookup, *ShardRealPath);
+    return HandleCachedRealPath(RealPath);
+  }
+
+  // If we don't know the real path, compute it...
+  std::error_code EC = getUnderlyingFS().getRealPath(OriginalFilename, Output);
+  llvm::ErrorOr<llvm::StringRef> ComputedRealPath = EC;
+  if (!EC)
+    ComputedRealPath = StringRef{Output.data(), Output.size()};
+
+  // ...and try to write it into the shared cache. In case some other thread won
+  // this race and already wrote its own result there, just adopt it. Write
+  // whatever is in the shared cache into the local one.
+  const auto &RealPath = Shard.getOrEmplaceRealPathForFilename(
+      *FilenameForLookup, ComputedRealPath);
+  return HandleCachedRealPath(
+      LocalCache.insertRealPathForFilename(*FilenameForLookup, RealPath));
+}
+
 std::error_code DependencyScanningWorkerFilesystem::setCurrentWorkingDirectory(
     const Twine &Path) {
   std::error_code EC = ProxyFileSystem::setCurrentWorkingDirectory(Path);
diff --git a/clang/unittests/Tooling/CMakeLists.txt b/clang/unittests/Tooling/CMakeLists.txt
index 2ff493ef5fc3..0eb612f8d949 100644
--- a/clang/unittests/Tooling/CMakeLists.txt
+++ b/clang/unittests/Tooling/CMakeLists.txt
@@ -24,6 +24,7 @@ add_clang_unittest(ToolingTests
   QualTypeNamesTest.cpp
   RangeSelectorTest.cpp
   DependencyScanning/DependencyScannerTest.cpp
+  DependencyScanning/DependencyScanningFilesystemTest.cpp
   RecursiveASTVisitorTests/Attr.cpp
   RecursiveASTVisitorTests/BitfieldInitializer.cpp
   RecursiveASTVisitorTests/CallbacksLeaf.cpp
diff --git a/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp b/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
new file mode 100644
index 000000000000..50cad72d223e
--- /dev/null
+++ b/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
@@ -0,0 +1,152 @@
+//===- DependencyScanningFilesystemTest.cpp -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/VirtualFileSystem.h"
+#include "gtest/gtest.h"
+
+using namespace clang::tooling::dependencies;
+
+namespace {
+struct InstrumentingFilesystem
+    : llvm::RTTIExtends<InstrumentingFilesystem, llvm::vfs::ProxyFileSystem> {
+  unsigned NumStatusCalls = 0;
+  unsigned NumGetRealPathCalls = 0;
+
+  using llvm::RTTIExtends<InstrumentingFilesystem,
+                          llvm::vfs::ProxyFileSystem>::RTTIExtends;
+
+  llvm::ErrorOr<llvm::vfs::Status> status(const llvm::Twine &Path) override {
+    ++NumStatusCalls;
+    return ProxyFileSystem::status(Path);
+  }
+
+  std::error_code getRealPath(const llvm::Twine &Path,
+                              llvm::SmallVectorImpl<char> &Output) override {
+    ++NumGetRealPathCalls;
+    return ProxyFileSystem::getRealPath(Path, Output);
+  }
+};
+} // namespace
+
+TEST(DependencyScanningWorkerFilesystem, CacheStatusFailures) {
+  auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
+
+  auto InstrumentingFS =
+      llvm::makeIntrusiveRefCnt<InstrumentingFilesystem>(InMemoryFS);
+
+  DependencyScanningFilesystemSharedCache SharedCache;
+  DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS);
+  DependencyScanningWorkerFilesystem DepFS2(SharedCache, InstrumentingFS);
+
+  DepFS.status("/foo.c");
+  EXPECT_EQ(InstrumentingFS->NumStatusCalls, 1u);
+
+  DepFS.status("/foo.c");
+  EXPECT_EQ(InstrumentingFS->NumStatusCalls, 1u); // Cached, no increase.
+
+  DepFS.status("/bar.c");
+  EXPECT_EQ(InstrumentingFS->NumStatusCalls, 2u);
+
+  DepFS2.status("/foo.c");
+  EXPECT_EQ(InstrumentingFS->NumStatusCalls, 2u); // Shared cache.
+}
+
+TEST(DependencyScanningFilesystem, CacheGetRealPath) {
+  auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
+  InMemoryFS->setCurrentWorkingDirectory("/");
+  InMemoryFS->addFile("/foo", 0, llvm::MemoryBuffer::getMemBuffer(""));
+  InMemoryFS->addFile("/bar", 0, llvm::MemoryBuffer::getMemBuffer(""));
+
+  auto InstrumentingFS =
+      llvm::makeIntrusiveRefCnt<InstrumentingFilesystem>(InMemoryFS);
+
+  DependencyScanningFilesystemSharedCache SharedCache;
+  DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS);
+  DependencyScanningWorkerFilesystem DepFS2(SharedCache, InstrumentingFS);
+
+  {
+    llvm::SmallString<128> Result;
+    DepFS.getRealPath("/foo", Result);
+    EXPECT_EQ(InstrumentingFS->NumGetRealPathCalls, 1u);
+  }
+
+  {
+    llvm::SmallString<128> Result;
+    DepFS.getRealPath("/foo", Result);
+    EXPECT_EQ(InstrumentingFS->NumGetRealPathCalls, 1u); // Cached, no increase.
+  }
+
+  {
+    llvm::SmallString<128> Result;
+    DepFS.getRealPath("/bar", Result);
+    EXPECT_EQ(InstrumentingFS->NumGetRealPathCalls, 2u);
+  }
+
+  {
+    llvm::SmallString<128> Result;
+    DepFS2.getRealPath("/foo", Result);
+    EXPECT_EQ(InstrumentingFS->NumGetRealPathCalls, 2u); // Shared cache.
+  }
+}
+
+TEST(DependencyScanningFilesystem, RealPathAndStatusInvariants) {
+  auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
+  InMemoryFS->setCurrentWorkingDirectory("/");
+  InMemoryFS->addFile("/foo.c", 0, llvm::MemoryBuffer::getMemBuffer(""));
+  InMemoryFS->addFile("/bar.c", 0, llvm::MemoryBuffer::getMemBuffer(""));
+
+  auto InstrumentingFS =
+      llvm::makeIntrusiveRefCnt<InstrumentingFilesystem>(InMemoryFS);
+
+  DependencyScanningFilesystemSharedCache SharedCache;
+  DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS);
+
+  // Success.
+  {
+    DepFS.status("/foo.c");
+
+    llvm::SmallString<128> Result;
+    DepFS.getRealPath("/foo.c", Result);
+  }
+  {
+    llvm::SmallString<128> Result;
+    DepFS.getRealPath("/bar.c", Result);
+
+    DepFS.status("/bar.c");
+  }
+
+  // Failure.
+  {
+    DepFS.status("/foo.m");
+
+    llvm::SmallString<128> Result;
+    DepFS.getRealPath("/foo.m", Result);
+  }
+  {
+    llvm::SmallString<128> Result;
+    DepFS.getRealPath("/bar.m", Result);
+
+    DepFS.status("/bar.m");
+  }
+
+  // Failure without caching.
+  {
+    DepFS.status("/foo");
+
+    llvm::SmallString<128> Result;
+    DepFS.getRealPath("/foo", Result);
+  }
+  {
+    llvm::SmallString<128> Result;
+    DepFS.getRealPath("/bar", Result);
+
+    DepFS.status("/bar");
+  }
+}
-- 
GitLab


From 2620ccc3cdebcaa53b96b5f4d79ffe3951534e07 Mon Sep 17 00:00:00 2001
From: Kalesh Singh <kaleshsingh96@gmail.com>
Date: Fri, 12 Apr 2024 10:43:59 -0700
Subject: [PATCH 096/464] ANDROID: x86_64: Set default max-page-size to 16kB
 (#87413)

Android now supports both 4kB and 16kB page sizes. The vast majority of
android apps are developed on x86_64 machines. In order to provide
emulators that support larger page sizes, Android emulates the page-size
in x86_64 to support testing apps for large page size support.

For this reason, update Android x86_64 ELFs default max-page-size to
16384 to support both 4kB and 16kB page-size devices.

Increase max-page-size raises concerns of increased disk space and extra
VMA slab memory.

In Android, RO partitions use sparse images, so that the holes on ELFs
don't allocate blocks on disk; and PackageManager ensures to punch holes
in ELF-paddings on the /data partition when apps are installed.

Extra VMA slab memory is addressed by the bionic loader, which extends
segment VMAs to cover the gaps between consecutive segment mappings, to
avoid the extra VMAs needed for the gap PROT_NONE mappings (---p). This
optimization is done in the crt_pad_segment note [1] is present in the
ELF.

[1]
https://cs.android.com/android/platform/superproject/main/+/189e480390ef13199d59e1fb54078e8b78ea6f79:bionic/libc/arch-common/bionic/crt_pad_segment.S

---------

Signed-off-by: Kalesh Singh <kaleshsingh@google.com>
Co-authored-by: Kalesh Singh <kaleshsingh@google.com>
---
 clang/lib/Driver/ToolChains/Linux.cpp |  3 ++-
 clang/test/Driver/android-link.cpp    | 38 +++++++++++++++++----------
 2 files changed, 26 insertions(+), 15 deletions(-)

diff --git a/clang/lib/Driver/ToolChains/Linux.cpp b/clang/lib/Driver/ToolChains/Linux.cpp
index 6c2f23e57bce..fb65881061ef 100644
--- a/clang/lib/Driver/ToolChains/Linux.cpp
+++ b/clang/lib/Driver/ToolChains/Linux.cpp
@@ -244,8 +244,9 @@ Linux::Linux(const Driver &D, const llvm::Triple &Triple, const ArgList &Args)
       // Android ARM uses max-page-size=4096 to reduce VMA usage.
       ExtraOpts.push_back("-z");
       ExtraOpts.push_back("max-page-size=4096");
-    } else if (Triple.isAArch64()) {
+    } else if (Triple.isAArch64() || Triple.getArch() == llvm::Triple::x86_64) {
       // Android AArch64 uses max-page-size=16384 to support 4k/16k page sizes.
+      // Android emulates a 16k page size for app testing on x86_64 machines.
       ExtraOpts.push_back("-z");
       ExtraOpts.push_back("max-page-size=16384");
     }
diff --git a/clang/test/Driver/android-link.cpp b/clang/test/Driver/android-link.cpp
index f9bdd00507d7..ab7dae540558 100644
--- a/clang/test/Driver/android-link.cpp
+++ b/clang/test/Driver/android-link.cpp
@@ -1,26 +1,36 @@
-// Check that we add relevant linker flags for Android ARM/AArch64.
+// Check that we add relevant linker flags for Android ARM/AArch64/i386/x86_64.
 
-// RUN: %clang -### -target arm-linux-androideabi %s 2>&1 | \
-// RUN:   FileCheck --check-prefix=MAX-PAGE-SIZE %s
+// RUN: %clang -### --target=arm-linux-androideabi %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=MAX-PAGE-SIZE-4KB %s
 
-// RUN: %clang -target aarch64-none-linux-android \
+// RUN: %clang --target=aarch64-none-linux-android \
 // RUN:   -### -v %s 2> %t
 // RUN: FileCheck -check-prefix=GENERIC-ARM < %t %s
-//
-// RUN: %clang -target aarch64-none-linux-android \
+
+// RUN: %clang --target=aarch64-none-linux-android \
 // RUN:   -mcpu=cortex-a53 -### -v %s 2> %t
 // RUN: FileCheck -check-prefix=CORTEX-A53 < %t %s
-//
-// RUN: %clang -target aarch64-none-linux-android \
+
+// RUN: %clang --target=aarch64-none-linux-android \
 // RUN:   -mcpu=cortex-a57 -### -v %s 2> %t
 // RUN: FileCheck -check-prefix=CORTEX-A57 < %t %s
-//
-// RUN: %clang -target aarch64-none-linux-android \
+
+// RUN: %clang --target=aarch64-none-linux-android \
 // RUN:   -### -v %s 2> %t
-// RUN: FileCheck -check-prefix=MAX-PAGE-SIZE-AARCH64 < %t %s
-//
+// RUN: FileCheck -check-prefix=MAX-PAGE-SIZE-16KB < %t %s
+
+// RUN: %clang -### --target=i386-none-linux-android %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=NO-MAX-PAGE-SIZE-16KB %s
+
+// RUN: %clang -### --target=x86_64-none-linux-gnu %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=NO-MAX-PAGE-SIZE-16KB %s
+
+// RUN: %clang -### --target=x86_64-none-linux-android %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=MAX-PAGE-SIZE-16KB %s
+
 // GENERIC-ARM: --fix-cortex-a53-843419
 // CORTEX-A53: --fix-cortex-a53-843419
 // CORTEX-A57-NOT: --fix-cortex-a53-843419
-// MAX-PAGE-SIZE: "-z" "max-page-size=4096"
-// MAX-PAGE-SIZE-AARCH64: "-z" "max-page-size=16384"
+// MAX-PAGE-SIZE-4KB: "-z" "max-page-size=4096"
+// MAX-PAGE-SIZE-16KB: "-z" "max-page-size=16384"
+// NO-MAX-PAGE-SIZE-16KB-NOT: "-z" "max-page-size=16384"
-- 
GitLab


From 446d38c65f72fd5d242a64182b05683577f683d3 Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda@apple.com>
Date: Fri, 12 Apr 2024 10:48:54 -0700
Subject: [PATCH 097/464] [lldb] Fix test build failure

Caused by commit edd7fed9da48c0e708cce9bd4d305ae43d8bd77c
---
 lldb/unittests/Host/FileSystemTest.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lldb/unittests/Host/FileSystemTest.cpp b/lldb/unittests/Host/FileSystemTest.cpp
index 4ed9beff4d30..3b5ee7c8bc22 100644
--- a/lldb/unittests/Host/FileSystemTest.cpp
+++ b/lldb/unittests/Host/FileSystemTest.cpp
@@ -74,7 +74,7 @@ public:
   }
   // Map any symlink to "/symlink".
   std::error_code getRealPath(const Twine &Path,
-                              SmallVectorImpl<char> &Output) const override {
+                              SmallVectorImpl<char> &Output) override {
     auto I = FilesAndDirs.find(Path.str());
     if (I == FilesAndDirs.end())
       return make_error_code(llvm::errc::no_such_file_or_directory);
-- 
GitLab


From 9022d9c102a8c47616fbeb6a45b015db6bd2c17a Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Fri, 12 Apr 2024 10:49:39 -0700
Subject: [PATCH 098/464] [xray] Apply default visibility to
 __xray_CustomEvent/__xray_TypedEvent

`__xray_customevent` and `__xray_typedevent` are built-in functions in
Clang. With -fxray-instrument, they are lowered to `__xray_CustomEvent`
(with 2 arguments) or `__xray_TypedEvent` (with 3 arguments).

xray patching is supported for shared objects, but they may contain
`__xray_customevent` and `__xray_typedevent` references that need to be
satisfied by default visibility definitions exported by the executable.

lld since df54f627fad789ccb11c72a9fddf116decbeba0e, like GNU ld, catches
the scenario at link time.
---
 compiler-rt/lib/xray/xray_trampoline_AArch64.S | 4 ++--
 compiler-rt/lib/xray/xray_trampoline_x86_64.S  | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/compiler-rt/lib/xray/xray_trampoline_AArch64.S b/compiler-rt/lib/xray/xray_trampoline_AArch64.S
index 6f10dda3602b..536a79e0d150 100644
--- a/compiler-rt/lib/xray/xray_trampoline_AArch64.S
+++ b/compiler-rt/lib/xray/xray_trampoline_AArch64.S
@@ -138,8 +138,9 @@ ASM_SYMBOL(__xray_ArgLoggerEntry):
   ret
 ASM_SIZE(__xray_ArgLoggerEntry)
 
+// __xray_*Event have default visibility so that they can be referenced by user
+// DSOs that do not link against the runtime.
 .global ASM_SYMBOL(__xray_CustomEvent)
-ASM_HIDDEN(__xray_CustomEvent)
 ASM_TYPE_FUNCTION(__xray_CustomEvent)
 ASM_SYMBOL(__xray_CustomEvent):
   SAVE_REGISTERS
@@ -153,7 +154,6 @@ ASM_SYMBOL(__xray_CustomEvent):
 ASM_SIZE(__xray_CustomEvent)
 
 .global ASM_SYMBOL(__xray_TypedEvent)
-ASM_HIDDEN(__xray_TypedEvent)
 ASM_TYPE_FUNCTION(__xray_TypedEvent)
 ASM_SYMBOL(__xray_TypedEvent):
   SAVE_REGISTERS
diff --git a/compiler-rt/lib/xray/xray_trampoline_x86_64.S b/compiler-rt/lib/xray/xray_trampoline_x86_64.S
index 0f00bcc41508..ff3ac91071a6 100644
--- a/compiler-rt/lib/xray/xray_trampoline_x86_64.S
+++ b/compiler-rt/lib/xray/xray_trampoline_x86_64.S
@@ -257,8 +257,9 @@ LOCAL_LABEL(arg1entryFail):
 
 //===----------------------------------------------------------------------===//
 
+// __xray_*Event have default visibility so that they can be referenced by user
+// DSOs that do not link against the runtime.
 	.global ASM_SYMBOL(__xray_CustomEvent)
-	ASM_HIDDEN(__xray_CustomEvent)
 	.align 16, 0x90
 	ASM_TYPE_FUNCTION(__xray_CustomEvent)
 # LLVM-MCA-BEGIN __xray_CustomEvent
@@ -284,7 +285,6 @@ LOCAL_LABEL(customEventCleanup):
 //===----------------------------------------------------------------------===//
 
 	.global ASM_SYMBOL(__xray_TypedEvent)
-	ASM_HIDDEN(__xray_TypedEvent)
 	.align 16, 0x90
 	ASM_TYPE_FUNCTION(__xray_TypedEvent)
 # LLVM-MCA-BEGIN __xray_TypedEvent
-- 
GitLab


From 4b0beb4f5ec42aea58461df7994e2fa40f335bb6 Mon Sep 17 00:00:00 2001
From: jimingham <jingham@apple.com>
Date: Fri, 12 Apr 2024 10:51:49 -0700
Subject: [PATCH 099/464] =?UTF-8?q?Reapply=20"Fix=20error=20in=20unrecogni?=
 =?UTF-8?q?zed=20register=20name=20handling=20for=20"SBFram=E2=80=A6e.regi?=
 =?UTF-8?q?ster""=20(#88468)"=20(#88535)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The only change is a fix for the "register" iterator test to not rely on
particular register names.

I mistook where the artificial "pc" register is generated. It isn't
added to the register list or the register sets (except on arm where
that's the name of the actual register), so I can't use it in this test.
I instead just assert that the "register" generator produces the same
list as flattening the register sets from "registers".

This reverts commit 9f14914753599f3879e4c273191959e2f1b3632c.
---
 lldb/bindings/interface/SBFrameExtensions.i  | 12 +++++++++++-
 lldb/test/API/python_api/frame/TestFrames.py | 16 ++++++++++++++--
 2 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/lldb/bindings/interface/SBFrameExtensions.i b/lldb/bindings/interface/SBFrameExtensions.i
index 43b22ed7a6b3..e0472280666a 100644
--- a/lldb/bindings/interface/SBFrameExtensions.i
+++ b/lldb/bindings/interface/SBFrameExtensions.i
@@ -44,6 +44,16 @@ STRING_EXTENSION_OUTSIDE(SBFrame)
                 def __init__(self, regs):
                     self.regs = regs
 
+                def __iter__(self):
+                    return self.get_registers()
+
+                def get_registers(self):
+                    for i in range(0,len(self.regs)):
+                        rs = self.regs[i]
+                        for j in range (0,rs.num_children):
+                            reg = rs.GetChildAtIndex(j)
+                            yield reg
+                          
                 def __getitem__(self, key):
                     if type(key) is str:
                         for i in range(0,len(self.regs)):
@@ -52,7 +62,7 @@ STRING_EXTENSION_OUTSIDE(SBFrame)
                                 reg = rs.GetChildAtIndex(j)
                                 if reg.name == key: return reg
                     else:
-                        return lldb.SBValue()
+                        return SBValue()
 
             return registers_access(self.registers)
 
diff --git a/lldb/test/API/python_api/frame/TestFrames.py b/lldb/test/API/python_api/frame/TestFrames.py
index a82b129bc809..0ec12206024a 100644
--- a/lldb/test/API/python_api/frame/TestFrames.py
+++ b/lldb/test/API/python_api/frame/TestFrames.py
@@ -73,10 +73,12 @@ class FrameAPITestCase(TestBase):
                 gpr_reg_set = lldbutil.get_GPRs(frame)
                 pc_value = gpr_reg_set.GetChildMemberWithName("pc")
                 self.assertTrue(pc_value, "We should have a valid PC.")
-                pc_value_int = int(pc_value.GetValue(), 0)
+                               
+    
                 # Make sure on arm targets we dont mismatch PC value on the basis of thumb bit.
                 # Frame PC will not have thumb bit set in case of a thumb
                 # instruction as PC.
+                pc_value_int = int(pc_value.GetValue(), 0)
                 if self.getArchitecture() in ["arm", "armv7", "armv7k"]:
                     pc_value_int &= ~1
                 self.assertEqual(
@@ -91,7 +93,17 @@ class FrameAPITestCase(TestBase):
                     frame.GetSP(),
                     "SP gotten as a value should equal frame's GetSP",
                 )
-
+                # Test that the "register" property's flat list matches the list from
+                # the "registers" property that returns register sets:
+                register_regs = set()
+                flattened_regs = set()
+                for reg_set in frame.registers:
+                    for reg in reg_set:
+                        flattened_regs.add(reg.name)
+                for reg in frame.register:
+                    register_regs.add(reg.name)
+                self.assertEqual(register_regs, flattened_regs, "register matches registers")
+                        
             print("---", file=session)
             process.Continue()
 
-- 
GitLab


From 83dc41992dc602070f5429e2717352f60aad931c Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Fri, 12 Apr 2024 11:14:20 -0700
Subject: [PATCH 100/464] [memprof] Clean up writer traits (NFC) (#88549)

RecordWriter does not live past the end of writeMemProfRecords, so it
can be safely on stack.

The constructor of FrameWriter does not take any parameter, so we can
let OnDiskChainedHashTableGenerator::Emit (with a single parameter)
default-construct an instance of the writer trait inside Emit.
---
 llvm/lib/ProfileData/InstrProfWriter.cpp | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp
index 7c56cde3e6ce..ede042dd6b9c 100644
--- a/llvm/lib/ProfileData/InstrProfWriter.cpp
+++ b/llvm/lib/ProfileData/InstrProfWriter.cpp
@@ -428,14 +428,13 @@ static uint64_t writeMemProfRecords(
     llvm::MapVector<GlobalValue::GUID, memprof::IndexedMemProfRecord>
         &MemProfRecordData,
     memprof::MemProfSchema *Schema) {
-  auto RecordWriter =
-      std::make_unique<memprof::RecordWriterTrait>(memprof::Version1);
-  RecordWriter->Schema = Schema;
+  memprof::RecordWriterTrait RecordWriter(memprof::Version1);
+  RecordWriter.Schema = Schema;
   OnDiskChainedHashTableGenerator<memprof::RecordWriterTrait>
       RecordTableGenerator;
   for (auto &[GUID, Record] : MemProfRecordData) {
     // Insert the key (func hash) and value (memprof record).
-    RecordTableGenerator.insert(GUID, Record, *RecordWriter.get());
+    RecordTableGenerator.insert(GUID, Record, RecordWriter);
   }
   // Release the memory of this MapVector as it is no longer needed.
   MemProfRecordData.clear();
@@ -443,14 +442,13 @@ static uint64_t writeMemProfRecords(
   // The call to Emit invokes RecordWriterTrait::EmitData which destructs
   // the memprof record copies owned by the RecordTableGenerator. This works
   // because the RecordTableGenerator is not used after this point.
-  return RecordTableGenerator.Emit(OS.OS, *RecordWriter);
+  return RecordTableGenerator.Emit(OS.OS, RecordWriter);
 }
 
 // Serialize MemProfFrameData.  Return FrameTableOffset.
 static uint64_t writeMemProfFrames(
     ProfOStream &OS,
     llvm::MapVector<memprof::FrameId, memprof::Frame> &MemProfFrameData) {
-  auto FrameWriter = std::make_unique<memprof::FrameWriterTrait>();
   OnDiskChainedHashTableGenerator<memprof::FrameWriterTrait>
       FrameTableGenerator;
   for (auto &[FrameId, Frame] : MemProfFrameData) {
@@ -460,7 +458,7 @@ static uint64_t writeMemProfFrames(
   // Release the memory of this MapVector as it is no longer needed.
   MemProfFrameData.clear();
 
-  return FrameTableGenerator.Emit(OS.OS, *FrameWriter);
+  return FrameTableGenerator.Emit(OS.OS);
 }
 
 static Error writeMemProfV0(
-- 
GitLab


From 0318ce8552896df7b180ee5481463eada65c755e Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron@aaronballman.com>
Date: Fri, 12 Apr 2024 14:15:22 -0400
Subject: [PATCH 101/464] [C99] Claim conformance for _Complex support (#88161)

There's so much overlap between the cited papers so this condenses the
status page into a single entry rather than trying to test conformance
against multiple papers doing conflicting things.
---
 clang/test/C/C99/n809.c   | 122 ++++++++++++++++++++++++++++++++++++++
 clang/test/C/C99/n809_2.c |  64 ++++++++++++++++++++
 clang/test/C/C99/n809_3.c |  12 ++++
 clang/www/c_status.html   |  37 +++++-------
 4 files changed, 213 insertions(+), 22 deletions(-)
 create mode 100644 clang/test/C/C99/n809.c
 create mode 100644 clang/test/C/C99/n809_2.c
 create mode 100644 clang/test/C/C99/n809_3.c

diff --git a/clang/test/C/C99/n809.c b/clang/test/C/C99/n809.c
new file mode 100644
index 000000000000..7297443a777c
--- /dev/null
+++ b/clang/test/C/C99/n809.c
@@ -0,0 +1,122 @@
+// RUN: %clang_cc1 -verify -std=c99 %s
+
+/* WG14 N620, N638, N657, N694, N809: Partial
+ * Complex and imaginary support in <complex.h>
+ *
+ * NB: Clang supports _Complex but not _Imaginary. In C99, _Complex support is
+ * required outside of freestanding, but _Imaginary support is fully optional.
+ * In C11, both are made fully optional.
+ *
+ * NB: _Complex support requires an underlying support library such as
+ * compiler-rt to provide functions like __divsc3. Compiler-rt is not supported
+ * on Windows.
+ *
+ * Because the functionality is so intertwined between the various papers,
+ * we're testing all of the functionality in one file.
+ */
+
+// Demonstrate that we support spelling complex floating-point objects.
+float _Complex f1;
+_Complex float f2;
+
+double _Complex d1;
+_Complex double d2;
+
+long double _Complex ld1;
+_Complex long double ld2;
+
+// Show that we don't support spelling imaginary types.
+float _Imaginary fi1; // expected-error {{imaginary types are not supported}}
+_Imaginary float fi2; // expected-error {{imaginary types are not supported}}
+
+double _Imaginary di1; // expected-error {{imaginary types are not supported}}
+_Imaginary double di2; // expected-error {{imaginary types are not supported}}
+
+long double _Imaginary ldi1; // expected-error {{imaginary types are not supported}}
+_Imaginary long double ldi2; // expected-error {{imaginary types are not supported}}
+
+// Each complex type has the same representation and alignment as an array
+// containing two elements of the corresponding real type. Note, it is not
+// mandatory that the alignment of a structure containing an array of two
+// elements has the same alignment as an array of two elements outside of a
+// structure, but this is a property Clang supports.
+_Static_assert(sizeof(float _Complex) == sizeof(struct { float mem[2]; }), "");
+_Static_assert(_Alignof(float _Complex) == _Alignof(struct { float mem[2]; }), "");
+
+_Static_assert(sizeof(double _Complex) == sizeof(struct { double mem[2]; }), "");
+_Static_assert(_Alignof(double _Complex) == _Alignof(struct { double mem[2]; }), "");
+
+_Static_assert(sizeof(long double _Complex) == sizeof(struct { long double mem[2]; }), "");
+_Static_assert(_Alignof(long double _Complex) == _Alignof(struct { long double mem[2]; }), "");
+
+// The first element corresponds to the real part and the second element
+// corresponds to the imaginary part.
+_Static_assert(__real((float _Complex){ 1.0f, 2.0f }) == 1.0f, "");
+_Static_assert(__imag((float _Complex){ 1.0f, 2.0f }) == 2.0f, "");
+
+_Static_assert(__real((double _Complex){ 1.0, 2.0 }) == 1.0, "");
+_Static_assert(__imag((double _Complex){ 1.0, 2.0 }) == 2.0, "");
+
+_Static_assert(__real((long double _Complex){ 1.0L, 2.0L }) == 1.0L, "");
+_Static_assert(__imag((long double _Complex){ 1.0L, 2.0L }) == 2.0L, "");
+
+// When a real value is converted to a complex value, the real part follows the
+// usual conversion rules and the imaginary part should be zero.
+_Static_assert(__real((float _Complex)1.0f) == 1.0f, "");
+_Static_assert(__imag((float _Complex)1.0f) == 0.0f, "");
+
+_Static_assert(__real((double _Complex)1.0f) == 1.0, "");
+_Static_assert(__imag((double _Complex)1.0f) == 0.0, "");
+
+_Static_assert(__real((long double _Complex)1.0f) == 1.0L, "");
+_Static_assert(__imag((long double _Complex)1.0f) == 0.0L, "");
+
+// When a complex value is converted to a real value, the real part follows the
+// usual conversion rules and the imaginary part is discarded.
+_Static_assert((float)(float _Complex){ 1.0f, 2.0f } == 1.0f, "");
+_Static_assert((double)(float _Complex){ 1.0f, 2.0f } == 1.0, "");
+_Static_assert((long double)(float _Complex){ 1.0f, 2.0f } == 1.0L, "");
+
+// Complex values are only equal if both the real and imaginary parts are equal.
+_Static_assert((float _Complex){ 1.0f, 2.0f } == (float _Complex){ 1.0f, 2.0f }, "");
+_Static_assert((double _Complex){ 1.0, 2.0 } == (double _Complex){ 1.0, 2.0 }, "");
+_Static_assert((long double _Complex){ 1.0L, 2.0L } == (long double _Complex){ 1.0L, 2.0L }, "");
+
+_Static_assert((float _Complex){ 1.0f, 2.0f } != (float _Complex){ 2.0f, 0.0f }, "");
+_Static_assert((double _Complex){ 1.0, 2.0 } != (double _Complex){ 2.0, 0.0 }, "");
+_Static_assert((long double _Complex){ 1.0L, 2.0L } != (long double _Complex){ 2.0L, 0.0L }, "");
+
+// You cannot use relational operator on complex values.
+int i1 = (float _Complex){ 1.0f, 2.0f } < 10;        // expected-error {{invalid operands to binary expression}}
+int i2 = (double _Complex){ 1.0f, 2.0f } > 10;       // expected-error {{invalid operands to binary expression}}
+int i3 = (long double _Complex){ 1.0f, 2.0f } <= 10; // expected-error {{invalid operands to binary expression}}
+int i4 = (float _Complex){ 1.0f, 2.0f } >= 10;       // expected-error {{invalid operands to binary expression}}
+
+// As a type specifier, _Complex cannot appear alone; however, we support it as
+// an extension by assuming _Complex double.
+_Complex c = 1.0f; // expected-warning {{plain '_Complex' requires a type specifier; assuming '_Complex double'}}
+// Because we don't support imaginary types, we don't extend the extension to
+// that type specifier.
+// FIXME: the warning diagnostic here is incorrect and should not be emitted.
+_Imaginary i = 1.0f; // expected-warning {{plain '_Complex' requires a type specifier; assuming '_Complex double'}} \
+                        expected-error {{imaginary types are not supported}}
+
+void func(void) {
+#pragma clang diagnostic push
+#pragma clang diagnostic warning "-Wpedantic"
+  // Increment and decrement operators have a constraint that their operand be
+  // a real type; Clang supports this as an extension on complex types as well.
+  _Complex float cf = 0.0f;
+
+  cf++; // expected-warning {{'++' on an object of complex type is a Clang extension}}
+  ++cf; // expected-warning {{'++' on an object of complex type is a Clang extension}}
+
+  cf--; // expected-warning {{'--' on an object of complex type is a Clang extension}}
+  --cf; // expected-warning {{'--' on an object of complex type is a Clang extension}}
+
+  // However, unary + and - are fine, as is += 1.
+  (void)-cf;
+  (void)+cf;
+  cf += 1;
+#pragma clang diagnostic pop
+}
diff --git a/clang/test/C/C99/n809_2.c b/clang/test/C/C99/n809_2.c
new file mode 100644
index 000000000000..3bf163126521
--- /dev/null
+++ b/clang/test/C/C99/n809_2.c
@@ -0,0 +1,64 @@
+// RUN: %clang_cc1 -ast-dump -std=c99 %s | FileCheck %s
+
+void variadic(int i, ...);
+
+void func(void) {
+  // CHECK: FunctionDecl {{.*}} func 'void (void)'
+
+  // Show that we correctly convert between two complex domains.
+  _Complex float cf = 1.0f;
+  _Complex double cd;
+
+  cd = cf;
+  // CHECK: BinaryOperator {{.*}} '_Complex double' '='
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'cd'
+  // CHECK-NEXT: ImplicitCastExpr {{.*}} '_Complex double' <FloatingComplexCast>
+  // CHECK-NEXT: ImplicitCastExpr {{.*}} '_Complex float' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'cf'
+
+  cf = cd;
+  // CHECK: BinaryOperator {{.*}} '_Complex float' '='
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'cf'
+  // CHECK-NEXT: ImplicitCastExpr {{.*}} '_Complex float' <FloatingComplexCast>
+  // CHECK-NEXT: ImplicitCastExpr {{.*}} '_Complex double' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'cd'
+
+  // Show that we correctly convert to the common type of a complex and real.
+  // This should convert the _Complex float to a _Complex double ("without
+  // change of domain" c.f. C99 6.3.1.8p1).
+  (void)(cf + 1.0);
+  // CHECK: BinaryOperator {{.*}} '_Complex double' '+'
+  // CHECK-NEXT: ImplicitCastExpr {{.*}} '_Complex double' <FloatingComplexCast>
+  // CHECK-NEXT: ImplicitCastExpr {{.*}} '_Complex float' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'cf'
+  // CHECK-NEXT: FloatingLiteral {{.*}} 'double' 1.0
+
+  // This should convert the float constant to double, then produce a
+  // _Complex double.
+  (void)(cd + 1.0f);
+  // CHECK: BinaryOperator {{.*}} '_Complex double' '+'
+  // CHECK-NEXT: ImplicitCastExpr {{.*}} '_Complex double' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'cd'
+  // CHECK-NEXT: ImplicitCastExpr {{.*}} 'double' <FloatingCast>
+  // CHECK-NEXT: FloatingLiteral {{.*}} 'float' 1.0
+
+  // This should convert the int constant to float, then produce a
+  // _Complex float.
+  (void)(cf + 1);
+  // CHECK: BinaryOperator {{.*}} '_Complex float' '+'
+  // CHECK-NEXT: ImplicitCastExpr {{.*}} '_Complex float' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'cf'
+  // CHECK-NEXT: ImplicitCastExpr {{.*}} 'float' <IntegralToFloating>
+  // CHECK-NEXT: IntegerLiteral {{.*}} 'int' 1
+
+  // Show that we do not promote a _Complex float to _Complex double as part of
+  // the default argument promotions when passing to a variadic function.
+  variadic(1, cf);
+  // CHECK: CallExpr
+  // CHECK-NEXT: ImplicitCastExpr {{.*}} <FunctionToPointerDecay>
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'variadic'
+  // CHECK-NEXT: IntegerLiteral {{.*}} 'int' 1
+  // CHECK-NEXT: ImplicitCastExpr {{.*}} '_Complex float' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr {{.*}} 'cf'
+}
+
diff --git a/clang/test/C/C99/n809_3.c b/clang/test/C/C99/n809_3.c
new file mode 100644
index 000000000000..f1283f5fe163
--- /dev/null
+++ b/clang/test/C/C99/n809_3.c
@@ -0,0 +1,12 @@
+// RUN: %clang_cc1 -emit-llvm -std=c99 %s -o - | FileCheck %s
+
+// Demonstrate that statics are properly zero initialized.
+static _Complex float f_global;
+void func(void) {
+  static _Complex double d_local;
+  d_local = f_global;
+}
+
+// CHECK-DAG: @func.d_local = internal global { double, double } zeroinitializer
+// CHECK-DAG: @f_global = internal global { float, float } zeroinitializer
+
diff --git a/clang/www/c_status.html b/clang/www/c_status.html
index 411f55447be7..9893170ae847 100644
--- a/clang/www/c_status.html
+++ b/clang/www/c_status.html
@@ -132,29 +132,22 @@ conformance.</p>
       <td>Unknown</td>
       <td class="full" align="center">Yes</td>
     </tr>
-    <tr id="complex">
-      <td rowspan="6">complex and imaginary support in &lt;complex.h&gt;</td>
+    <tr>
+      <td>complex and imaginary support in &lt;complex.h&gt;</td>
+      <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n693.ps">N693</a></td>
+      <td class="partial" align="center">
+        <details><summary>Partial</summary>
+          Clang supports <code>_Complex</code> type specifiers but does not
+          support <code>_Imaginary</code> type specifiers. Support for
+          <code>_Imaginary</code> is optional in C99 and Clang does not claim
+          conformance to Annex G.<br />
+          <br />
+          <code>_Complex</code> support requires an underlying support library
+          such as compiler-rt to provide functions like <code>__divsc3</code>,
+          but compiler-rt is not supported on Windows.
+        </details>
+      </td>
     </tr>
-      <tr>
-        <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n620.ps">N620</a></td>
-        <td class="unknown" align="center">Unknown</td>
-      </tr>
-      <tr>
-        <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n638.ps">N638</a></td>
-        <td class="unknown" align="center">Unknown</td>
-      </tr>
-      <tr>
-        <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n657.ps">N657</a></td>
-        <td class="unknown" align="center">Unknown</td>
-      </tr>
-      <tr>
-        <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n694.ps">N694</a></td>
-        <td class="unknown" align="center">Unknown</td>
-      </tr>
-      <tr>
-        <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n809.ps">N809</a></td>
-        <td class="unknown" align="center">Unknown</td>
-      </tr>
     <tr>
       <td>type-generic math macros in &lt;tgmath.h&gt;</td>
       <td><a href="https://www.open-std.org/jtc1/sc22/wg14/www/docs/n693.ps">N693</a></td>
-- 
GitLab


From b074f25329501487e312b59e463a2d5f743090f8 Mon Sep 17 00:00:00 2001
From: elizabethandrews <elizabeth.andrews@intel.com>
Date: Fri, 12 Apr 2024 14:41:49 -0400
Subject: [PATCH 102/464] [NFC][Clang] Fix static analyzer concern (#88179)

Fix static analyzer concerns about dereferencing
null values.
---
 clang/lib/AST/Interp/InterpState.h | 6 +++++-
 clang/lib/Sema/SemaAPINotes.cpp    | 2 ++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/clang/lib/AST/Interp/InterpState.h b/clang/lib/AST/Interp/InterpState.h
index 8f84bf6ed2ea..c17cfad11b1e 100644
--- a/clang/lib/AST/Interp/InterpState.h
+++ b/clang/lib/AST/Interp/InterpState.h
@@ -89,7 +89,11 @@ public:
 
   /// Delegates source mapping to the mapper.
   SourceInfo getSource(const Function *F, CodePtr PC) const override {
-    return M ? M->getSource(F, PC) : F->getSource(PC);
+    if (M)
+      return M->getSource(F, PC);
+
+    assert(F && "Function cannot be null");
+    return F->getSource(PC);
   }
 
   Context &getContext() const { return Ctx; }
diff --git a/clang/lib/Sema/SemaAPINotes.cpp b/clang/lib/Sema/SemaAPINotes.cpp
index a3128306c664..4c445f28bba8 100644
--- a/clang/lib/Sema/SemaAPINotes.cpp
+++ b/clang/lib/Sema/SemaAPINotes.cpp
@@ -463,6 +463,8 @@ static void ProcessAPINotes(Sema &S, FunctionOrMethod AnyFunc,
     D = MD;
   }
 
+  assert((FD || MD) && "Expecting Function or ObjCMethod");
+
   // Nullability of return type.
   if (Info.NullabilityAudited)
     applyNullability(S, D, Info.getReturnTypeInfo(), Metadata);
-- 
GitLab


From c777c011a709dffd4fa5e79cad7947b7c3405d02 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=BCtt?= <schuett@gmail.com>
Date: Fri, 12 Apr 2024 21:10:19 +0200
Subject: [PATCH 103/464] [GlobalIsel] Import vscale (#88240)

https://github.com/llvm/llvm-project/pull/84542
---
 llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp  |  4 ++
 .../AArch64/GlobalISel/irtranslator-vscale.ll | 52 +++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-vscale.ll

diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 312e564f5d80..9b4575f7f34d 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -2550,6 +2550,10 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
     MIRBuilder.buildInstr(TargetOpcode::G_RESET_FPMODE, {}, {});
     return true;
   }
+  case Intrinsic::vscale: {
+    MIRBuilder.buildVScale(getOrCreateVReg(CI), 1);
+    return true;
+  }
   case Intrinsic::prefetch: {
     Value *Addr = CI.getOperand(0);
     unsigned RW = cast<ConstantInt>(CI.getOperand(1))->getZExtValue();
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-vscale.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-vscale.ll
new file mode 100644
index 000000000000..d06e2c64c245
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-vscale.ll
@@ -0,0 +1,52 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -O0 -mtriple=aarch64-linux-gnu -global-isel -stop-after=irtranslator %s -o - | FileCheck %s
+
+define i64 @call_vscale_i64() {
+  ; CHECK-LABEL: name: call_vscale_i64
+  ; CHECK: bb.1.entry:
+  ; CHECK-NEXT:   [[VSCALE:%[0-9]+]]:_(s64) = G_VSCALE i64 1
+  ; CHECK-NEXT:   $x0 = COPY [[VSCALE]](s64)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $x0
+entry:
+  %vscale = call i64 @llvm.vscale.64()
+  ret i64 %vscale
+}
+
+define i64 @call_vscale_i32() {
+  ; CHECK-LABEL: name: call_vscale_i32
+  ; CHECK: bb.1.entry:
+  ; CHECK-NEXT:   [[VSCALE:%[0-9]+]]:_(s32) = G_VSCALE i32 1
+  ; CHECK-NEXT:   [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[VSCALE]](s32)
+  ; CHECK-NEXT:   $x0 = COPY [[ZEXT]](s64)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $x0
+entry:
+  %vscale = call i32 @llvm.vscale.32()
+  %zext = zext i32 %vscale to i64
+  ret i64 %zext
+}
+
+define i64 @call_vscale_i16() {
+  ; CHECK-LABEL: name: call_vscale_i16
+  ; CHECK: bb.1.entry:
+  ; CHECK-NEXT:   [[VSCALE:%[0-9]+]]:_(s16) = G_VSCALE i16 1
+  ; CHECK-NEXT:   [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[VSCALE]](s16)
+  ; CHECK-NEXT:   $x0 = COPY [[ZEXT]](s64)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $x0
+entry:
+  %vscale = call i16 @llvm.vscale.16()
+  %zext = zext i16 %vscale to i64
+  ret i64 %zext
+}
+
+define i64 @call_vscale_i8() {
+  ; CHECK-LABEL: name: call_vscale_i8
+  ; CHECK: bb.1.entry:
+  ; CHECK-NEXT:   [[VSCALE:%[0-9]+]]:_(s8) = G_VSCALE i8 1
+  ; CHECK-NEXT:   [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[VSCALE]](s8)
+  ; CHECK-NEXT:   $x0 = COPY [[ZEXT]](s64)
+  ; CHECK-NEXT:   RET_ReallyLR implicit $x0
+entry:
+  %vscale = call i8 @llvm.vscale.8()
+  %zext = zext i8 %vscale to i64
+  ret i64 %zext
+}
-- 
GitLab


From a9111d4a26514e169ca57c11e6533ca7c5408832 Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda@apple.com>
Date: Fri, 12 Apr 2024 12:33:19 -0700
Subject: [PATCH 104/464] [clang] Remove unused variable in test

---
 .../DependencyScanning/DependencyScanningFilesystemTest.cpp  | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp b/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
index 50cad72d223e..697b7d70ff03 100644
--- a/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
+++ b/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
@@ -102,11 +102,8 @@ TEST(DependencyScanningFilesystem, RealPathAndStatusInvariants) {
   InMemoryFS->addFile("/foo.c", 0, llvm::MemoryBuffer::getMemBuffer(""));
   InMemoryFS->addFile("/bar.c", 0, llvm::MemoryBuffer::getMemBuffer(""));
 
-  auto InstrumentingFS =
-      llvm::makeIntrusiveRefCnt<InstrumentingFilesystem>(InMemoryFS);
-
   DependencyScanningFilesystemSharedCache SharedCache;
-  DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS);
+  DependencyScanningWorkerFilesystem DepFS(SharedCache, InMemoryFS);
 
   // Success.
   {
-- 
GitLab


From 6dbd4bb3b4c00d62bf1552e54e4370c6816ea060 Mon Sep 17 00:00:00 2001
From: Jing Wang <2019426+jingw@users.noreply.github.com>
Date: Fri, 12 Apr 2024 12:41:06 -0700
Subject: [PATCH 105/464] Add txtpb to the list of supported TextProto
 extensions (#88355)

According to
https://protobuf.dev/reference/protobuf/textformat-spec/#text-format-files,
txtpb is the canonical extension

Co-authored-by: Jing Wang <99jingw@gmail.com>
---
 clang/docs/ClangFormat.rst               | 2 +-
 clang/lib/Format/Format.cpp              | 6 +++++-
 clang/test/Format/lit.local.cfg          | 1 +
 clang/tools/clang-format/ClangFormat.cpp | 2 +-
 4 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/clang/docs/ClangFormat.rst b/clang/docs/ClangFormat.rst
index 80dc38a075c8..dbd9c91ae508 100644
--- a/clang/docs/ClangFormat.rst
+++ b/clang/docs/ClangFormat.rst
@@ -54,7 +54,7 @@ to format C/C++/Java/JavaScript/JSON/Objective-C/Protobuf/C# code.
                                        Objective-C: .m .mm
                                        Proto: .proto .protodevel
                                        TableGen: .td
-                                       TextProto: .textpb .pb.txt .textproto .asciipb
+                                       TextProto: .txtpb .textpb .pb.txt .textproto .asciipb
                                        Verilog: .sv .svh .v .vh
     --cursor=<uint>                - The position of the cursor when invoking
                                      clang-format from an editor integration
diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp
index 89e6c19b0af4..ccb2c9190e2e 100644
--- a/clang/lib/Format/Format.cpp
+++ b/clang/lib/Format/Format.cpp
@@ -3891,7 +3891,11 @@ static FormatStyle::LanguageKind getLanguageByFileName(StringRef FileName) {
       FileName.ends_with_insensitive(".protodevel")) {
     return FormatStyle::LK_Proto;
   }
-  if (FileName.ends_with_insensitive(".textpb") ||
+  // txtpb is the canonical extension, and textproto is the legacy canonical
+  // extension
+  // https://protobuf.dev/reference/protobuf/textformat-spec/#text-format-files
+  if (FileName.ends_with_insensitive(".txtpb") ||
+      FileName.ends_with_insensitive(".textpb") ||
       FileName.ends_with_insensitive(".pb.txt") ||
       FileName.ends_with_insensitive(".textproto") ||
       FileName.ends_with_insensitive(".asciipb")) {
diff --git a/clang/test/Format/lit.local.cfg b/clang/test/Format/lit.local.cfg
index 3851397a3f7c..3eb0f54ceaa6 100644
--- a/clang/test/Format/lit.local.cfg
+++ b/clang/test/Format/lit.local.cfg
@@ -12,6 +12,7 @@ config.suffixes = [
     ".proto",
     ".protodevel",
     ".pb.txt",
+    ".txtpb",
     ".textproto",
     ".textpb",
     ".asciipb",
diff --git a/clang/tools/clang-format/ClangFormat.cpp b/clang/tools/clang-format/ClangFormat.cpp
index ed401135ad84..feb733fe3c9e 100644
--- a/clang/tools/clang-format/ClangFormat.cpp
+++ b/clang/tools/clang-format/ClangFormat.cpp
@@ -92,7 +92,7 @@ static cl::opt<std::string> AssumeFileName(
              "  Objective-C: .m .mm\n"
              "  Proto: .proto .protodevel\n"
              "  TableGen: .td\n"
-             "  TextProto: .textpb .pb.txt .textproto .asciipb\n"
+             "  TextProto: .txtpb .textpb .pb.txt .textproto .asciipb\n"
              "  Verilog: .sv .svh .v .vh"),
     cl::init("<stdin>"), cl::cat(ClangFormatCategory));
 
-- 
GitLab


From 4078afc6d23e25df6baedad61b224ef86a94d42f Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Fri, 12 Apr 2024 14:52:24 -0500
Subject: [PATCH 106/464] [LLVM][OpenMP] Add "nowait" clause as valid for
 "workshare" (#88426)

Add the "nowait" clause to the list of allowed clauses for the "workshare"
directive. This will make it consistent with other directives (which are
shared between C/C++ and Fortran).

The parser will still reject "nowait" on "!$omp workshare", so this has no
effect on accepting/rejecting Fortran source code.
---
 flang/lib/Semantics/check-omp-structure.h         | 1 +
 flang/test/Semantics/OpenMP/clause-validity01.f90 | 3 +++
 llvm/include/llvm/Frontend/OpenMP/OMP.td          | 3 +++
 3 files changed, 7 insertions(+)

diff --git a/flang/lib/Semantics/check-omp-structure.h b/flang/lib/Semantics/check-omp-structure.h
index 33243d926cf1..8287653458e1 100644
--- a/flang/lib/Semantics/check-omp-structure.h
+++ b/flang/lib/Semantics/check-omp-structure.h
@@ -42,6 +42,7 @@ static const OmpDirectiveSet noWaitClauseNotAllowedSet{
     Directive::OMPD_do_simd,
     Directive::OMPD_sections,
     Directive::OMPD_single,
+    Directive::OMPD_workshare,
 };
 } // namespace omp
 } // namespace llvm
diff --git a/flang/test/Semantics/OpenMP/clause-validity01.f90 b/flang/test/Semantics/OpenMP/clause-validity01.f90
index 977a52986461..74f154bb0ad6 100644
--- a/flang/test/Semantics/OpenMP/clause-validity01.f90
+++ b/flang/test/Semantics/OpenMP/clause-validity01.f90
@@ -342,6 +342,9 @@ use omp_lib
   a = 1.0
   !ERROR: COPYPRIVATE clause is not allowed on the END WORKSHARE directive
   !$omp end workshare nowait copyprivate(a)
+  !$omp workshare nowait
+  !ERROR: NOWAIT clause is not allowed on the WORKSHARE directive, use it on OMP END WORKSHARE directive
+  !$omp end workshare
   !$omp end parallel
 
 ! 2.8.1 simd-clause -> safelen-clause |
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index d9a931438b42..e91169e8da1a 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -2115,6 +2115,9 @@ def OMP_scope : Directive<"scope"> {
   let association = AS_Block;
 }
 def OMP_Workshare : Directive<"workshare"> {
+  let allowedOnceClauses = [
+    VersionedClause<OMPC_NoWait>
+  ];
   let association = AS_Block;
 }
 def OMP_ParallelWorkshare : Directive<"parallel workshare"> {
-- 
GitLab


From 41ff91e143a72dcd5755c9ad2160cf3c333b50c6 Mon Sep 17 00:00:00 2001
From: OverMighty <its.overmighty@gmail.com>
Date: Fri, 12 Apr 2024 21:01:36 +0100
Subject: [PATCH 107/464] Reapply "[libc][math][c23] Add remaining linux/*
 entrypoints for {,u}fromfp{,x}* (#86692)" (#88567)

This reverts commit 8a071678a9091d536eae29912ca7be6238105956.

The test failure on 32-bit Arm should have been fixed by #86892.

cc @nickdesaulniers @lntue
---
 libc/config/linux/aarch64/entrypoints.txt | 16 ++++++++++++++++
 libc/config/linux/arm/entrypoints.txt     | 12 ++++++++++++
 libc/config/linux/riscv/entrypoints.txt   | 16 ++++++++++++++++
 3 files changed, 44 insertions(+)

diff --git a/libc/config/linux/aarch64/entrypoints.txt b/libc/config/linux/aarch64/entrypoints.txt
index f5f5c437685a..2952baacdd67 100644
--- a/libc/config/linux/aarch64/entrypoints.txt
+++ b/libc/config/linux/aarch64/entrypoints.txt
@@ -398,6 +398,12 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.frexp
     libc.src.math.frexpf
     libc.src.math.frexpl
+    libc.src.math.fromfp
+    libc.src.math.fromfpf
+    libc.src.math.fromfpl
+    libc.src.math.fromfpx
+    libc.src.math.fromfpxf
+    libc.src.math.fromfpxl
     libc.src.math.hypot
     libc.src.math.hypotf
     libc.src.math.ilogb
@@ -480,6 +486,12 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.trunc
     libc.src.math.truncf
     libc.src.math.truncl
+    libc.src.math.ufromfp
+    libc.src.math.ufromfpf
+    libc.src.math.ufromfpl
+    libc.src.math.ufromfpx
+    libc.src.math.ufromfpxf
+    libc.src.math.ufromfpxl
 )
 
 if(LIBC_TYPES_HAS_FLOAT128)
@@ -502,6 +514,8 @@ if(LIBC_TYPES_HAS_FLOAT128)
     libc.src.math.fminimum_mag_numf128
     libc.src.math.fmodf128
     libc.src.math.frexpf128
+    libc.src.math.fromfpf128
+    libc.src.math.fromfpxf128
     libc.src.math.ilogbf128
     libc.src.math.ldexpf128
     libc.src.math.llogbf128
@@ -519,6 +533,8 @@ if(LIBC_TYPES_HAS_FLOAT128)
     libc.src.math.roundf128
     libc.src.math.sqrtf128
     libc.src.math.truncf128
+    libc.src.math.ufromfpf128
+    libc.src.math.ufromfpxf128
   )
 endif()
 
diff --git a/libc/config/linux/arm/entrypoints.txt b/libc/config/linux/arm/entrypoints.txt
index fca50735d320..35fd588a9a6c 100644
--- a/libc/config/linux/arm/entrypoints.txt
+++ b/libc/config/linux/arm/entrypoints.txt
@@ -265,6 +265,12 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.frexp
     libc.src.math.frexpf
     libc.src.math.frexpl
+    libc.src.math.fromfp
+    libc.src.math.fromfpf
+    libc.src.math.fromfpl
+    libc.src.math.fromfpx
+    libc.src.math.fromfpxf
+    libc.src.math.fromfpxl
     libc.src.math.hypot
     libc.src.math.hypotf
     libc.src.math.ilogb
@@ -347,6 +353,12 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.trunc
     libc.src.math.truncf
     libc.src.math.truncl
+    libc.src.math.ufromfp
+    libc.src.math.ufromfpf
+    libc.src.math.ufromfpl
+    libc.src.math.ufromfpx
+    libc.src.math.ufromfpxf
+    libc.src.math.ufromfpxl
 )
 
 set(TARGET_LLVMLIBC_ENTRYPOINTS
diff --git a/libc/config/linux/riscv/entrypoints.txt b/libc/config/linux/riscv/entrypoints.txt
index 71289789158f..47c03a61c45a 100644
--- a/libc/config/linux/riscv/entrypoints.txt
+++ b/libc/config/linux/riscv/entrypoints.txt
@@ -406,6 +406,12 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.frexp
     libc.src.math.frexpf
     libc.src.math.frexpl
+    libc.src.math.fromfp
+    libc.src.math.fromfpf
+    libc.src.math.fromfpl
+    libc.src.math.fromfpx
+    libc.src.math.fromfpxf
+    libc.src.math.fromfpxl
     libc.src.math.hypot
     libc.src.math.hypotf
     libc.src.math.ilogb
@@ -488,6 +494,12 @@ set(TARGET_LIBM_ENTRYPOINTS
     libc.src.math.trunc
     libc.src.math.truncf
     libc.src.math.truncl
+    libc.src.math.ufromfp
+    libc.src.math.ufromfpf
+    libc.src.math.ufromfpl
+    libc.src.math.ufromfpx
+    libc.src.math.ufromfpxf
+    libc.src.math.ufromfpxl
 )
 
 if(LIBC_TYPES_HAS_FLOAT128)
@@ -510,6 +522,8 @@ if(LIBC_TYPES_HAS_FLOAT128)
     libc.src.math.fminimum_mag_numf128
     libc.src.math.fmodf128
     libc.src.math.frexpf128
+    libc.src.math.fromfpf128
+    libc.src.math.fromfpxf128
     libc.src.math.ilogbf128
     libc.src.math.ldexpf128
     libc.src.math.llogbf128
@@ -527,6 +541,8 @@ if(LIBC_TYPES_HAS_FLOAT128)
     libc.src.math.roundf128
     libc.src.math.sqrtf128
     libc.src.math.truncf128
+    libc.src.math.ufromfpf128
+    libc.src.math.ufromfpxf128
   )
 endif()
 
-- 
GitLab


From 5300a6731e98fbcf7bca68374e934de737166698 Mon Sep 17 00:00:00 2001
From: Jonathan Peyton <jonathan.l.peyton@intel.com>
Date: Fri, 12 Apr 2024 15:13:59 -0500
Subject: [PATCH 108/464] [OpenMP] Fix re-locking hang found in issue 86684
 (#88539)

This was initially reported here (including stacktraces):
https://stackoverflow.com/questions/78183545/does-compiling-imagick-with-openmp-enabled-in-freebsd-13-2-cause-sched-yield

If `__kmp_register_library_startup()` detects that another instance of
the library is present, `__kmp_is_address_mapped()` is eventually
called. which uses `kmpc_alloc()` to allocate memory. This function
calls `__kmp_entry_thread()` to access the thread-local memory pool,
which is a bad idea during initialization. This macro internally calls
`__kmp_get_global_thread_id_reg()` which sets the bootstrap lock at the
beginning (before calling `__kmp_register_library_startup()`).

The fix is to use `KMP_INTERNAL_MALLOC()`/`KMP_INTERNAL_FREE()` instead
of `kmpc_malloc()`/`kmpc_free()`. `KMP_INTERNAL_MALLOC` and
`KMP_INTERNAL_FREE` do not use any bootstrap locks. They just translate
to `malloc()`/`free()` and are meant to be used during library
initialization before other library-specific allocators have been
initialized.

Fixes: #86684
---
 openmp/runtime/src/z_Linux_util.cpp | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/openmp/runtime/src/z_Linux_util.cpp b/openmp/runtime/src/z_Linux_util.cpp
index be47be0dcec2..11ce083f4801 100644
--- a/openmp/runtime/src/z_Linux_util.cpp
+++ b/openmp/runtime/src/z_Linux_util.cpp
@@ -2141,10 +2141,10 @@ int __kmp_is_address_mapped(void *addr) {
   // We pass from number of vm entry's semantic
   // to size of whole entry map list.
   lstsz = lstsz * 4 / 3;
-  buf = reinterpret_cast<char *>(kmpc_malloc(lstsz));
+  buf = reinterpret_cast<char *>(KMP_INTERNAL_MALLOC(lstsz));
   rc = sysctl(mib, 4, buf, &lstsz, NULL, 0);
   if (rc < 0) {
-    kmpc_free(buf);
+    KMP_INTERNAL_FREE(buf);
     return 0;
   }
 
@@ -2168,7 +2168,7 @@ int __kmp_is_address_mapped(void *addr) {
     }
     lw += cursz;
   }
-  kmpc_free(buf);
+  KMP_INTERNAL_FREE(buf);
 #elif KMP_OS_DRAGONFLY
   char err[_POSIX2_LINE_MAX];
   kinfo_proc *proc;
@@ -2234,12 +2234,12 @@ int __kmp_is_address_mapped(void *addr) {
     return 0;
   }
 
-  buf = kmpc_malloc(sz);
+  buf = KMP_INTERNAL_MALLOC(sz);
 
   while (sz > 0 && (rd = pread(file, buf, sz, 0)) == sz) {
     void *newbuf;
     sz <<= 1;
-    newbuf = kmpc_realloc(buf, sz);
+    newbuf = KMP_INTERNAL_REALLOC(buf, sz);
     buf = newbuf;
   }
 
@@ -2255,7 +2255,7 @@ int __kmp_is_address_mapped(void *addr) {
     }
   }
 
-  kmpc_free(map);
+  KMP_INTERNAL_FREE(map);
   close(file);
   KMP_INTERNAL_FREE(name);
 #elif KMP_OS_DARWIN
-- 
GitLab


From dfafe3822b1e59f938c256c25a92a7b555f78469 Mon Sep 17 00:00:00 2001
From: Alexander Richardson <alexrichardson@google.com>
Date: Fri, 12 Apr 2024 13:20:30 -0700
Subject: [PATCH 109/464] Reland "[compiler-rt] Allow running tests without
 installing first"

Currently, the testsuite uses the default runtimes path to find the
runtimes libraries which may or may not match the just-built runtimes.
This change uses the `-resource-dir` flag for clang whenever
`COMPILER_RT_TEST_STANDALONE_BUILD_LIBS` is set to ensure that we are
actually testing the currently built libraries rather than the ones
bundled with `${COMPILER_RT_TEST_COMPILER}`.

The existing logic works fine when clang and compiler-rt share the same
build directory ``-DLLVM_ENABLE_PROJECTS=clang;compiler-rt`, but when
building compiler-rt separately we need to tell the compiler used for
the tests where it can find the just-built libraries.

This reduces the fixes check-all failures to one in my configuration:
```
cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -G Ninja
-DCMAKE_C_COMPILER=$HOME/output/upstream-llvm/bin/clang
-DCMAKE_CXX_COMPILER=$HOME/output/upstream-llvm/bin/clang++
-DCOMPILER_RT_INCLUDE_TESTS=ON
-DLLVM_EXTERNAL_LIT=$HOME/build/upstream-llvm-project-build/bin/llvm-lit
-DLLVM_CMAKE_DIR=$HOME/output/upstream-llvm
-DCOMPILER_RT_DEBUG=OFF
-S $HOME/src/upstream-llvm-project/compiler-rt
-B $HOME/src/upstream-llvm-project/compiler-rt/cmake-build-all-sanitizers
```

This relands the previous PR with fixes for Windows.
Depends on https://github.com/llvm/llvm-project/pull/88074 to be merged
first for GCC buildbots.

Pull Request: https://github.com/llvm/llvm-project/pull/88075
---
 compiler-rt/CMakeLists.txt                    | 23 +++++
 compiler-rt/cmake/Modules/AddCompilerRT.cmake |  1 +
 compiler-rt/test/CMakeLists.txt               |  8 --
 compiler-rt/test/fuzzer/lit.cfg.py            |  1 +
 compiler-rt/test/lit.common.cfg.py            | 89 ++++++++++++++-----
 compiler-rt/test/lit.common.configured.in     |  1 +
 compiler-rt/test/safestack/lit.cfg.py         | 10 ++-
 7 files changed, 101 insertions(+), 32 deletions(-)

diff --git a/compiler-rt/CMakeLists.txt b/compiler-rt/CMakeLists.txt
index f4e92f14db85..8649507ce1c7 100644
--- a/compiler-rt/CMakeLists.txt
+++ b/compiler-rt/CMakeLists.txt
@@ -584,6 +584,29 @@ string(APPEND COMPILER_RT_TEST_COMPILER_CFLAGS " ${stdlib_flag}")
 string(REPLACE " " ";" COMPILER_RT_UNITTEST_CFLAGS "${COMPILER_RT_TEST_COMPILER_CFLAGS}")
 set(COMPILER_RT_UNITTEST_LINK_FLAGS ${COMPILER_RT_UNITTEST_CFLAGS})
 
+option(COMPILER_RT_TEST_STANDALONE_BUILD_LIBS
+  "When set to ON and testing in a standalone build, test the runtime \
+  libraries built by this standalone build rather than the runtime libraries \
+  shipped with the compiler (used for testing). When set to OFF and testing \
+  in a standalone build, test the runtime libraries shipped with the compiler \
+  (used for testing). This option has no effect if the compiler and this \
+  build are configured to use the same runtime library path."
+  ON)
+if (COMPILER_RT_TEST_STANDALONE_BUILD_LIBS)
+  # Ensure that the unit tests can find the sanitizer headers prior to installation.
+  list(APPEND COMPILER_RT_UNITTEST_CFLAGS "-I${CMAKE_CURRENT_LIST_DIR}/include")
+  # Ensure that unit tests link against the just-built runtime libraries instead
+  # of the ones bundled with the compiler by overriding the resource directory.
+  #
+  if ("${COMPILER_RT_TEST_COMPILER_ID}" MATCHES "Clang")
+    list(APPEND COMPILER_RT_UNITTEST_LINK_FLAGS "-resource-dir=${COMPILER_RT_OUTPUT_DIR}")
+  endif()
+  get_compiler_rt_output_dir(${COMPILER_RT_DEFAULT_TARGET_ARCH} rtlib_dir)
+  if (NOT WIN32)
+    list(APPEND COMPILER_RT_UNITTEST_LINK_FLAGS "-Wl,-rpath,${rtlib_dir}")
+  endif()
+endif()
+
 if(COMPILER_RT_USE_LLVM_UNWINDER)
   # We're linking directly against the libunwind that we're building so don't
   # try to link in the toolchain's default libunwind which may be missing.
diff --git a/compiler-rt/cmake/Modules/AddCompilerRT.cmake b/compiler-rt/cmake/Modules/AddCompilerRT.cmake
index dc36f16c661c..6e0d9dbff65a 100644
--- a/compiler-rt/cmake/Modules/AddCompilerRT.cmake
+++ b/compiler-rt/cmake/Modules/AddCompilerRT.cmake
@@ -771,6 +771,7 @@ function(configure_compiler_rt_lit_site_cfg input output)
   get_compiler_rt_output_dir(${COMPILER_RT_DEFAULT_TARGET_ARCH} output_dir)
 
   string(REPLACE ${CMAKE_CFG_INTDIR} ${LLVM_BUILD_MODE} COMPILER_RT_RESOLVED_TEST_COMPILER ${COMPILER_RT_TEST_COMPILER})
+  string(REPLACE ${CMAKE_CFG_INTDIR} ${LLVM_BUILD_MODE} COMPILER_RT_RESOLVED_OUTPUT_DIR ${COMPILER_RT_OUTPUT_DIR})
   string(REPLACE ${CMAKE_CFG_INTDIR} ${LLVM_BUILD_MODE} COMPILER_RT_RESOLVED_LIBRARY_OUTPUT_DIR ${output_dir})
 
   configure_lit_site_cfg(${input} ${output})
diff --git a/compiler-rt/test/CMakeLists.txt b/compiler-rt/test/CMakeLists.txt
index c186be1e44fd..edc007aaf477 100644
--- a/compiler-rt/test/CMakeLists.txt
+++ b/compiler-rt/test/CMakeLists.txt
@@ -1,14 +1,6 @@
 # Needed for lit support in standalone builds.
 include(AddLLVM)
 
-option(COMPILER_RT_TEST_STANDALONE_BUILD_LIBS
-  "When set to ON and testing in a standalone build, test the runtime \
-  libraries built by this standalone build rather than the runtime libraries \
-  shipped with the compiler (used for testing). When set to OFF and testing \
-  in a standalone build, test the runtime libraries shipped with the compiler \
-  (used for testing). This option has no effect if the compiler and this \
-  build are configured to use the same runtime library path."
-  ON)
 pythonize_bool(COMPILER_RT_TEST_STANDALONE_BUILD_LIBS)
 
 pythonize_bool(LLVM_ENABLE_EXPENSIVE_CHECKS)
diff --git a/compiler-rt/test/fuzzer/lit.cfg.py b/compiler-rt/test/fuzzer/lit.cfg.py
index 9084254b3b15..0c0550ac729d 100644
--- a/compiler-rt/test/fuzzer/lit.cfg.py
+++ b/compiler-rt/test/fuzzer/lit.cfg.py
@@ -106,6 +106,7 @@ def generate_compiler_cmd(is_cpp=True, fuzzer_enabled=True, msan_enabled=False):
     return " ".join(
         [
             compiler_cmd,
+            config.target_cflags,
             std_cmd,
             "-O2 -gline-tables-only",
             sanitizers_cmd,
diff --git a/compiler-rt/test/lit.common.cfg.py b/compiler-rt/test/lit.common.cfg.py
index 0ac20a9831d9..28f126a11b16 100644
--- a/compiler-rt/test/lit.common.cfg.py
+++ b/compiler-rt/test/lit.common.cfg.py
@@ -14,6 +14,27 @@ import lit.formats
 import lit.util
 
 
+def get_path_from_clang(args, allow_failure):
+    clang_cmd = [
+        config.clang.strip(),
+        f"--target={config.target_triple}",
+        *args,
+    ]
+    path = None
+    try:
+        result = subprocess.run(
+            clang_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True
+        )
+        path = result.stdout.decode().strip()
+    except subprocess.CalledProcessError as e:
+        msg = f"Failed to run {clang_cmd}\nrc:{e.returncode}\nstdout:{e.stdout}\ne.stderr{e.stderr}"
+        if allow_failure:
+            lit_config.warning(msg)
+        else:
+            lit_config.fatal(msg)
+    return path, clang_cmd
+
+
 def find_compiler_libdir():
     """
     Returns the path to library resource directory used
@@ -26,26 +47,6 @@ def find_compiler_libdir():
         # TODO: Support other compilers.
         return None
 
-    def get_path_from_clang(args, allow_failure):
-        clang_cmd = [
-            config.clang.strip(),
-            f"--target={config.target_triple}",
-        ]
-        clang_cmd.extend(args)
-        path = None
-        try:
-            result = subprocess.run(
-                clang_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True
-            )
-            path = result.stdout.decode().strip()
-        except subprocess.CalledProcessError as e:
-            msg = f"Failed to run {clang_cmd}\nrc:{e.returncode}\nstdout:{e.stdout}\ne.stderr{e.stderr}"
-            if allow_failure:
-                lit_config.warning(msg)
-            else:
-                lit_config.fatal(msg)
-        return path, clang_cmd
-
     # Try using `-print-runtime-dir`. This is only supported by very new versions of Clang.
     # so allow failure here.
     runtime_dir, clang_cmd = get_path_from_clang(
@@ -130,6 +131,7 @@ if execute_external:
     config.available_features.add("shell")
 
 target_is_msvc = bool(re.match(r".*-windows-msvc$", config.target_triple))
+target_is_windows = bool(re.match(r".*-windows.*$", config.target_triple))
 
 compiler_id = getattr(config, "compiler_id", None)
 if compiler_id == "Clang":
@@ -168,10 +170,53 @@ if config.enable_per_target_runtime_dir:
             r"/i386(?=-[^/]+$)", "/x86_64", config.compiler_rt_libdir
         )
 
+
+# Check if the test compiler resource dir matches the local build directory
+# (which happens with -DLLVM_ENABLE_PROJECTS=clang;compiler-rt) or if we are
+# using an installed clang to test compiler-rt standalone. In the latter case
+# we may need to override the resource dir to match the path of the just-built
+# compiler-rt libraries.
+test_cc_resource_dir, _ = get_path_from_clang(
+    shlex.split(config.target_cflags) + ["-print-resource-dir"], allow_failure=True
+)
+# Normalize the path for comparison
+if test_cc_resource_dir is not None:
+    test_cc_resource_dir = os.path.realpath(test_cc_resource_dir)
+if lit_config.debug:
+    lit_config.note(f"Resource dir for {config.clang} is {test_cc_resource_dir}")
+local_build_resource_dir = os.path.realpath(config.compiler_rt_output_dir)
+if test_cc_resource_dir != local_build_resource_dir and config.test_standalone_build_libs:
+    if config.compiler_id == "Clang":
+        if lit_config.debug:
+            lit_config.note(
+                f"Overriding test compiler resource dir to use "
+                f'libraries in "{config.compiler_rt_libdir}"'
+            )
+        # Ensure that we use the just-built static libraries when linking by
+        # overriding the Clang resource directory. Additionally, we want to use
+        # the builtin headers shipped with clang (e.g. stdint.h), so we
+        # explicitly add this as an include path (since the headers are not
+        # going to be in the current compiler-rt build directory).
+        # We also tell the linker to add an RPATH entry for the local library
+        # directory so that the just-built shared libraries are used.
+        config.target_cflags += f" -nobuiltininc"
+        config.target_cflags += f" -I{config.compiler_rt_src_root}/include"
+        config.target_cflags += f" -idirafter {test_cc_resource_dir}/include"
+        config.target_cflags += f" -resource-dir={config.compiler_rt_output_dir}"
+        if not target_is_windows:
+            # Avoid passing -rpath on Windows where it is not supported.
+            config.target_cflags += f" -Wl,-rpath,{config.compiler_rt_libdir}"
+    else:
+        lit_config.warning(
+            f"Cannot override compiler-rt library directory with non-Clang "
+            f"compiler: {config.compiler_id}"
+        )
+
+
 # Ask the compiler for the path to libraries it is going to use. If this
 # doesn't match config.compiler_rt_libdir then it means we might be testing the
 # compiler's own runtime libraries rather than the ones we just built.
-# Warn about about this and handle appropriately.
+# Warn about this and handle appropriately.
 compiler_libdir = find_compiler_libdir()
 if compiler_libdir:
     compiler_rt_libdir_real = os.path.realpath(config.compiler_rt_libdir)
@@ -182,7 +227,7 @@ if compiler_libdir:
             f'compiler-rt libdir:  "{compiler_rt_libdir_real}"'
         )
         if config.test_standalone_build_libs:
-            # Use just built runtime libraries, i.e. the the libraries this built just built.
+            # Use just built runtime libraries, i.e. the libraries this build just built.
             if not config.test_suite_supports_overriding_runtime_lib_path:
                 # Test suite doesn't support this configuration.
                 # TODO(dliew): This should be an error but it seems several bots are
diff --git a/compiler-rt/test/lit.common.configured.in b/compiler-rt/test/lit.common.configured.in
index fff5dc6cc750..8889b816b149 100644
--- a/compiler-rt/test/lit.common.configured.in
+++ b/compiler-rt/test/lit.common.configured.in
@@ -27,6 +27,7 @@ set_default("compiler_id", "@COMPILER_RT_TEST_COMPILER_ID@")
 set_default("python_executable", "@Python3_EXECUTABLE@")
 set_default("compiler_rt_debug", @COMPILER_RT_DEBUG_PYBOOL@)
 set_default("compiler_rt_intercept_libdispatch", @COMPILER_RT_INTERCEPT_LIBDISPATCH_PYBOOL@)
+set_default("compiler_rt_output_dir", "@COMPILER_RT_RESOLVED_OUTPUT_DIR@")
 set_default("compiler_rt_libdir", "@COMPILER_RT_RESOLVED_LIBRARY_OUTPUT_DIR@")
 set_default("emulator", "@COMPILER_RT_EMULATOR@")
 set_default("asan_shadow_scale", "@COMPILER_RT_ASAN_SHADOW_SCALE@")
diff --git a/compiler-rt/test/safestack/lit.cfg.py b/compiler-rt/test/safestack/lit.cfg.py
index adf27a0d7e5e..aadb8bf0d5c7 100644
--- a/compiler-rt/test/safestack/lit.cfg.py
+++ b/compiler-rt/test/safestack/lit.cfg.py
@@ -13,10 +13,16 @@ config.suffixes = [".c", ".cpp", ".m", ".mm", ".ll", ".test"]
 
 # Add clang substitutions.
 config.substitutions.append(
-    ("%clang_nosafestack ", config.clang + " -O0 -fno-sanitize=safe-stack ")
+    (
+        "%clang_nosafestack ",
+        config.clang + config.target_cflags + " -O0 -fno-sanitize=safe-stack ",
+    )
 )
 config.substitutions.append(
-    ("%clang_safestack ", config.clang + " -O0 -fsanitize=safe-stack ")
+    (
+        "%clang_safestack ",
+        config.clang + config.target_cflags + " -O0 -fsanitize=safe-stack ",
+    )
 )
 
 if config.lto_supported:
-- 
GitLab


From 588987405a979f724ab0aa7ad788cc5e87af3649 Mon Sep 17 00:00:00 2001
From: Artem Chikin <achikin@apple.com>
Date: Fri, 12 Apr 2024 13:34:47 -0700
Subject: [PATCH 110/464] [llvm][vfs] Make vfs::FileSystem::exists() virtual
 NFC (#88575)

Allow a `vfs::FileSystem` to provide a more efficient implementation of
`exists()` if they are able to. The existing `FileSystem`
implementations continue to default to using `status()` except that
overlay, proxy, and redirecting filesystems are taught to forward calls
to `exists()` correctly to their wrapped/external filesystem.

Co-authored-by: Ben Langmuir <blangmuir@apple.com>
---
 llvm/include/llvm/Support/VirtualFileSystem.h |   8 +-
 llvm/lib/Support/VirtualFileSystem.cpp        |  56 ++++++
 .../Support/VirtualFileSystemTest.cpp         | 165 ++++++++++++++++++
 3 files changed, 227 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/Support/VirtualFileSystem.h b/llvm/include/llvm/Support/VirtualFileSystem.h
index 4b1ca0c3d262..0113d6b7da25 100644
--- a/llvm/include/llvm/Support/VirtualFileSystem.h
+++ b/llvm/include/llvm/Support/VirtualFileSystem.h
@@ -300,8 +300,9 @@ public:
   virtual std::error_code getRealPath(const Twine &Path,
                                       SmallVectorImpl<char> &Output);
 
-  /// Check whether a file exists. Provided for convenience.
-  bool exists(const Twine &Path);
+  /// Check whether \p Path exists. By default this uses \c status(), but
+  /// filesystems may provide a more efficient implementation if available.
+  virtual bool exists(const Twine &Path);
 
   /// Is the file mounted on a local filesystem?
   virtual std::error_code isLocal(const Twine &Path, bool &Result);
@@ -386,6 +387,7 @@ public:
   void pushOverlay(IntrusiveRefCntPtr<FileSystem> FS);
 
   llvm::ErrorOr<Status> status(const Twine &Path) override;
+  bool exists(const Twine &Path) override;
   llvm::ErrorOr<std::unique_ptr<File>>
   openFileForRead(const Twine &Path) override;
   directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override;
@@ -439,6 +441,7 @@ public:
   llvm::ErrorOr<Status> status(const Twine &Path) override {
     return FS->status(Path);
   }
+  bool exists(const Twine &Path) override { return FS->exists(Path); }
   llvm::ErrorOr<std::unique_ptr<File>>
   openFileForRead(const Twine &Path) override {
     return FS->openFileForRead(Path);
@@ -1044,6 +1047,7 @@ public:
          bool UseExternalNames, FileSystem &ExternalFS);
 
   ErrorOr<Status> status(const Twine &Path) override;
+  bool exists(const Twine &Path) override;
   ErrorOr<std::unique_ptr<File>> openFileForRead(const Twine &Path) override;
 
   std::error_code getRealPath(const Twine &Path,
diff --git a/llvm/lib/Support/VirtualFileSystem.cpp b/llvm/lib/Support/VirtualFileSystem.cpp
index 32b480028e71..921af30bfcde 100644
--- a/llvm/lib/Support/VirtualFileSystem.cpp
+++ b/llvm/lib/Support/VirtualFileSystem.cpp
@@ -438,6 +438,15 @@ ErrorOr<Status> OverlayFileSystem::status(const Twine &Path) {
   return make_error_code(llvm::errc::no_such_file_or_directory);
 }
 
+bool OverlayFileSystem::exists(const Twine &Path) {
+  // FIXME: handle symlinks that cross file systems
+  for (iterator I = overlays_begin(), E = overlays_end(); I != E; ++I) {
+    if ((*I)->exists(Path))
+      return true;
+  }
+  return false;
+}
+
 ErrorOr<std::unique_ptr<File>>
 OverlayFileSystem::openFileForRead(const llvm::Twine &Path) {
   // FIXME: handle symlinks that cross file systems
@@ -2428,6 +2437,53 @@ ErrorOr<Status> RedirectingFileSystem::status(const Twine &OriginalPath) {
   return S;
 }
 
+bool RedirectingFileSystem::exists(const Twine &OriginalPath) {
+  SmallString<256> Path;
+  OriginalPath.toVector(Path);
+
+  if (makeAbsolute(Path))
+    return false;
+
+  if (Redirection == RedirectKind::Fallback) {
+    // Attempt to find the original file first, only falling back to the
+    // mapped file if that fails.
+    if (ExternalFS->exists(Path))
+      return true;
+  }
+
+  ErrorOr<RedirectingFileSystem::LookupResult> Result = lookupPath(Path);
+  if (!Result) {
+    // Was not able to map file, fallthrough to using the original path if
+    // that was the specified redirection type.
+    if (Redirection == RedirectKind::Fallthrough &&
+        isFileNotFound(Result.getError()))
+      return ExternalFS->exists(Path);
+    return false;
+  }
+
+  std::optional<StringRef> ExtRedirect = Result->getExternalRedirect();
+  if (!ExtRedirect) {
+    assert(isa<RedirectingFileSystem::DirectoryEntry>(Result->E));
+    return true;
+  }
+
+  SmallString<256> RemappedPath((*ExtRedirect).str());
+  if (makeAbsolute(RemappedPath))
+    return false;
+
+  if (ExternalFS->exists(RemappedPath))
+    return true;
+
+  if (Redirection == RedirectKind::Fallthrough) {
+    // Mapped the file but it wasn't found in the underlying filesystem,
+    // fallthrough to using the original path if that was the specified
+    // redirection type.
+    return ExternalFS->exists(Path);
+  }
+
+  return false;
+}
+
 namespace {
 
 /// Provide a file wrapper with an overriden status.
diff --git a/llvm/unittests/Support/VirtualFileSystemTest.cpp b/llvm/unittests/Support/VirtualFileSystemTest.cpp
index 49a2e19e4f74..e9b4ac3d92e1 100644
--- a/llvm/unittests/Support/VirtualFileSystemTest.cpp
+++ b/llvm/unittests/Support/VirtualFileSystemTest.cpp
@@ -201,6 +201,21 @@ class ErrorDummyFileSystem : public DummyFileSystem {
   }
 };
 
+/// A version of \c DummyFileSystem that aborts on \c status() to test that
+/// \c exists() is being used.
+class NoStatusDummyFileSystem : public DummyFileSystem {
+public:
+  ErrorOr<vfs::Status> status(const Twine &Path) override {
+    llvm::report_fatal_error(
+        "unexpected call to NoStatusDummyFileSystem::status");
+  }
+
+  bool exists(const Twine &Path) override {
+    auto Status = DummyFileSystem::status(Path);
+    return Status && Status->exists();
+  }
+};
+
 /// Replace back-slashes by front-slashes.
 std::string getPosixPath(const Twine &S) {
   SmallString<128> Result;
@@ -968,6 +983,30 @@ TEST(OverlayFileSystemTest, PrintOutput) {
             Output);
 }
 
+TEST(OverlayFileSystemTest, Exists) {
+  IntrusiveRefCntPtr<DummyFileSystem> Lower(new NoStatusDummyFileSystem());
+  IntrusiveRefCntPtr<DummyFileSystem> Upper(new NoStatusDummyFileSystem());
+  IntrusiveRefCntPtr<vfs::OverlayFileSystem> O(
+      new vfs::OverlayFileSystem(Lower));
+  O->pushOverlay(Upper);
+
+  Lower->addDirectory("/both");
+  Upper->addDirectory("/both");
+  Lower->addRegularFile("/both/lower_file");
+  Upper->addRegularFile("/both/upper_file");
+  Lower->addDirectory("/lower");
+  Upper->addDirectory("/upper");
+
+  EXPECT_TRUE(O->exists("/both"));
+  EXPECT_TRUE(O->exists("/both"));
+  EXPECT_TRUE(O->exists("/both/lower_file"));
+  EXPECT_TRUE(O->exists("/both/upper_file"));
+  EXPECT_TRUE(O->exists("/lower"));
+  EXPECT_TRUE(O->exists("/upper"));
+  EXPECT_FALSE(O->exists("/both/nope"));
+  EXPECT_FALSE(O->exists("/nope"));
+}
+
 TEST(ProxyFileSystemTest, Basic) {
   IntrusiveRefCntPtr<vfs::InMemoryFileSystem> Base(
       new vfs::InMemoryFileSystem());
@@ -3364,6 +3403,11 @@ TEST(RedirectingFileSystemTest, ExternalPaths) {
       SeenPaths.push_back(Dir.str());
       return ProxyFileSystem::dir_begin(Dir, EC);
     }
+
+    bool exists(const Twine &Path) override {
+      SeenPaths.push_back(Path.str());
+      return ProxyFileSystem::exists(Path);
+    }
   };
 
   std::error_code EC;
@@ -3395,3 +3439,124 @@ TEST(RedirectingFileSystemTest, ExternalPaths) {
 
   EXPECT_EQ(CheckFS->SeenPaths, Expected);
 }
+
+TEST(RedirectingFileSystemTest, Exists) {
+  IntrusiveRefCntPtr<DummyFileSystem> Dummy(new NoStatusDummyFileSystem());
+  auto YAML =
+    MemoryBuffer::getMemBuffer("{\n"
+                               "  'version': 0,\n"
+                               "  'roots': [\n"
+                               "    {\n"
+                               "      'type': 'directory-remap',\n"
+                               "      'name': '/dremap',\n"
+                               "      'external-contents': '/a',\n"
+                               "    },"
+                               "    {\n"
+                               "      'type': 'directory-remap',\n"
+                               "      'name': '/dmissing',\n"
+                               "      'external-contents': '/dmissing',\n"
+                               "    },"
+                               "    {\n"
+                               "      'type': 'directory',\n"
+                               "      'name': '/both',\n"
+                               "      'contents': [\n"
+                               "        {\n"
+                               "          'type': 'file',\n"
+                               "          'name': 'vfile',\n"
+                               "          'external-contents': '/c'\n"
+                               "        }\n"
+                               "      ]\n"
+                               "    },\n"
+                               "    {\n"
+                               "      'type': 'directory',\n"
+                               "      'name': '/vdir',\n"
+                               "      'contents': ["
+                               "        {\n"
+                               "          'type': 'directory-remap',\n"
+                               "          'name': 'dremap',\n"
+                               "          'external-contents': '/b'\n"
+                               "        },\n"
+                               "        {\n"
+                               "          'type': 'file',\n"
+                               "          'name': 'missing',\n"
+                               "          'external-contents': '/missing'\n"
+                               "        },\n"
+                               "        {\n"
+                               "          'type': 'file',\n"
+                               "          'name': 'vfile',\n"
+                               "          'external-contents': '/c'\n"
+                               "        }]\n"
+                               "    }]\n"
+                               "}");
+
+  Dummy->addDirectory("/a");
+  Dummy->addRegularFile("/a/foo");
+  Dummy->addDirectory("/b");
+  Dummy->addRegularFile("/c");
+  Dummy->addRegularFile("/both/foo");
+
+  auto Redirecting = vfs::RedirectingFileSystem::create(
+							std::move(YAML), nullptr, "", nullptr, Dummy);
+
+  EXPECT_TRUE(Redirecting->exists("/dremap"));
+  EXPECT_FALSE(Redirecting->exists("/dmissing"));
+  EXPECT_FALSE(Redirecting->exists("/unknown"));
+  EXPECT_TRUE(Redirecting->exists("/both"));
+  EXPECT_TRUE(Redirecting->exists("/both/foo"));
+  EXPECT_TRUE(Redirecting->exists("/both/vfile"));
+  EXPECT_TRUE(Redirecting->exists("/vdir"));
+  EXPECT_TRUE(Redirecting->exists("/vdir/dremap"));
+  EXPECT_FALSE(Redirecting->exists("/vdir/missing"));
+  EXPECT_TRUE(Redirecting->exists("/vdir/vfile"));
+  EXPECT_FALSE(Redirecting->exists("/vdir/unknown"));
+}
+
+TEST(RedirectingFileSystemTest, ExistsFallback) {
+  IntrusiveRefCntPtr<DummyFileSystem> Dummy(new NoStatusDummyFileSystem());
+  auto YAML =
+    MemoryBuffer::getMemBuffer("{\n"
+                               "  'version': 0,\n"
+                               "  'redirecting-with': 'fallback',\n"
+                               "  'roots': [\n"
+                               "    {\n"
+                               "      'type': 'file',\n"
+                               "      'name': '/fallback',\n"
+                               "      'external-contents': '/missing',\n"
+                               "    },"
+                               "  ]\n"
+                               "}");
+
+  Dummy->addRegularFile("/fallback");
+
+  auto Redirecting = vfs::RedirectingFileSystem::create(
+							std::move(YAML), nullptr, "", nullptr, Dummy);
+
+  EXPECT_TRUE(Redirecting->exists("/fallback"));
+  EXPECT_FALSE(Redirecting->exists("/missing"));
+}
+
+TEST(RedirectingFileSystemTest, ExistsRedirectOnly) {
+  IntrusiveRefCntPtr<DummyFileSystem> Dummy(new NoStatusDummyFileSystem());
+  auto YAML =
+    MemoryBuffer::getMemBuffer("{\n"
+                               "  'version': 0,\n"
+                               "  'redirecting-with': 'redirect-only',\n"
+                               "  'roots': [\n"
+                               "    {\n"
+                               "      'type': 'file',\n"
+                               "      'name': '/vfile',\n"
+                               "      'external-contents': '/a',\n"
+                               "    },"
+                               "  ]\n"
+                               "}");
+
+  Dummy->addRegularFile("/a");
+  Dummy->addRegularFile("/b");
+
+  auto Redirecting = vfs::RedirectingFileSystem::create(
+							std::move(YAML), nullptr, "", nullptr, Dummy);
+
+  EXPECT_FALSE(Redirecting->exists("/a"));
+  EXPECT_FALSE(Redirecting->exists("/b"));
+  EXPECT_TRUE(Redirecting->exists("/vfile"));
+}
-- 
GitLab


From 62624a4bfed4ea468dcb677b799aee3518a24bf7 Mon Sep 17 00:00:00 2001
From: "Harrison,Hao" <57025411+TSWorld1314@users.noreply.github.com>
Date: Sat, 13 Apr 2024 04:43:58 +0800
Subject: [PATCH 111/464] [Support] Fix the issue where the character being
 saved in Unicode causes a warning to be treated as an error in Visual Studio
 2022. (#88513)

Fix the issue where the character being saved in Unicode causes a
warning to be treated as an error in Visual Studio 2022.

![image](https://github.com/llvm/llvm-project/assets/57025411/07353525-6520-4b74-b4f5-5b3f5afc47e1)
---
 llvm/lib/Support/Parallel.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Support/Parallel.cpp b/llvm/lib/Support/Parallel.cpp
index 9b14b05b5211..af35947192c0 100644
--- a/llvm/lib/Support/Parallel.cpp
+++ b/llvm/lib/Support/Parallel.cpp
@@ -57,7 +57,7 @@ public:
     Threads.resize(1);
     std::lock_guard<std::mutex> Lock(Mutex);
     // Use operator[] before creating the thread to avoid data race in .size()
-    // in “safe libc++” mode.
+    // in 'safe libc++' mode.
     auto &Thread0 = Threads[0];
     Thread0 = std::thread([this, S] {
       for (unsigned I = 1; I < ThreadCount; ++I) {
-- 
GitLab


From 8a4b7de91dc334c828674aa2cad927c6ffb9cf37 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Fri, 12 Apr 2024 12:45:47 -0700
Subject: [PATCH 112/464] [SLP][NFC]Make TTIRef capture argument instead of
 outer declaration.

---
 llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index df891371fdf7..7b5b0efca8d3 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -4895,14 +4895,10 @@ void BoUpSLP::reorderTopToBottom() {
   // Maps a TreeEntry to the reorder indices of external users.
   DenseMap<const TreeEntry *, SmallVector<OrdersType, 1>>
       ExternalUserReorderMap;
-  // FIXME: Workaround for syntax error reported by MSVC buildbots.
-  TargetTransformInfo &TTIRef = *TTI;
   // Find all reorderable nodes with the given VF.
   // Currently the are vectorized stores,loads,extracts + some gathering of
   // extracts.
-  for_each(VectorizableTree, [this, &TTIRef, &VFToOrderedEntries,
-                              &GathersToOrders, &ExternalUserReorderMap,
-                              &AltShufflesToOrders, &PhisToOrders](
+  for_each(VectorizableTree, [&, &TTIRef = *TTI](
                                  const std::unique_ptr<TreeEntry> &TE) {
     // Look for external users that will probably be vectorized.
     SmallVector<OrdersType, 1> ExternalUserReorderIndices =
@@ -9288,11 +9284,9 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
       (void)E;
       return TTI->getInstructionCost(VI, CostKind);
     };
-    // FIXME: Workaround for syntax error reported by MSVC buildbots.
-    TargetTransformInfo &TTIRef = *TTI;
     // Need to clear CommonCost since the final shuffle cost is included into
     // vector cost.
-    auto GetVectorCost = [&](InstructionCost) {
+    auto GetVectorCost = [&, &TTIRef = *TTI](InstructionCost) {
       // VecCost is equal to sum of the cost of creating 2 vectors
       // and the cost of creating shuffle.
       InstructionCost VecCost = 0;
-- 
GitLab


From 0f82469314f34a086669dfcd190a9f89260fbee5 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Fri, 12 Apr 2024 22:07:29 +0100
Subject: [PATCH 113/464] [Passes] Run SimpleLoopUnswitch after introducing
 invariant branches. (#81271)

IndVars may be able to replace a loop dependent condition with a loop
invariant one, but loop-unswitch runs before IndVars, so the invariant
check remains in the loop.

For an example, consider a read-only loop with a bounds check:
https://godbolt.org/z/8cdj4qhbG

This patch uses a approach similar to the way extra cleanup passes are
run on demand after vectorization (added in acea6e9cfa4c4a0e8678c7).

It introduces a new ShouldRunExtraSimpleLoopUnswitch analysis marker,
which IndVars can use to indicate that extra unswitching is beneficial.

ExtraSimpleLoopUnswitchPassManager uses this analysis to determine
whether to run its passes on a loop.

Compile-time impact (geomean) ranges from +0.0% to 0.02%
https://llvm-compile-time-tracker.com/compare.php?from=138c0beb109ffe47f75a0fe8c4dc2cdabe8a6532&to=19e6e99eeb280d426907ea73a21b139ba7225627&stat=instructions%3Au

Compile-time impact (geomean) of unconditionally running
SimpleLoopUnswitch ranges from +0.05% - +0.16%

https://llvm-compile-time-tracker.com/compare.php?from=138c0beb109ffe47f75a0fe8c4dc2cdabe8a6532&to=2930dfd5accdce2e6f8d5146ae4d626add2065a2&stat=instructions:u

Unconditionally running SimpleLoopUnswitch seems to indicate that there
are multiple other scenarios where we fail to run unswitching when
opportunities remain.


Fixes https://github.com/llvm/llvm-project/issues/85551.

PR: https://github.com/llvm/llvm-project/pull/81271
---
 .../Transforms/Scalar/SimpleLoopUnswitch.h    |  35 ++++++
 .../llvm/Transforms/Utils/SimplifyIndVar.h    |  17 ++-
 llvm/lib/Passes/PassBuilderPipelines.cpp      |   7 ++
 llvm/lib/Passes/PassRegistry.def              |   3 +
 llvm/lib/Transforms/Scalar/IndVarSimplify.cpp |  17 ++-
 .../Transforms/Scalar/SimpleLoopUnswitch.cpp  |   1 +
 llvm/lib/Transforms/Utils/SimplifyIndVar.cpp  |  20 +++-
 .../AArch64/hoist-runtime-checks.ll           | 106 ++++++++++++++----
 8 files changed, 169 insertions(+), 37 deletions(-)

diff --git a/llvm/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h b/llvm/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h
index b97ee23fc0e6..a40b0a2a9ab7 100644
--- a/llvm/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h
+++ b/llvm/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h
@@ -12,6 +12,7 @@
 #include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/Analysis/LoopAnalysisManager.h"
 #include "llvm/IR/PassManager.h"
+#include "llvm/Transforms/Scalar/LoopPassManager.h"
 
 namespace llvm {
 
@@ -20,6 +21,40 @@ class Loop;
 class StringRef;
 class raw_ostream;
 
+struct ShouldRunExtraSimpleLoopUnswitch
+    : public AnalysisInfoMixin<ShouldRunExtraSimpleLoopUnswitch> {
+  static AnalysisKey Key;
+  struct Result {
+    bool invalidate(Loop &L, const PreservedAnalyses &PA,
+                    LoopAnalysisManager::Invalidator &) {
+      // Check whether the analysis has been explicitly invalidated. Otherwise,
+      // it remains preserved.
+      auto PAC = PA.getChecker<ShouldRunExtraSimpleLoopUnswitch>();
+      return !PAC.preservedWhenStateless();
+    }
+  };
+
+  Result run(Loop &L, LoopAnalysisManager &AM,
+             LoopStandardAnalysisResults &AR) {
+    return Result();
+  }
+
+  static bool isRequired() { return true; }
+};
+
+struct ExtraSimpleLoopUnswitchPassManager : public LoopPassManager {
+  PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM,
+                        LoopStandardAnalysisResults &AR, LPMUpdater &U) {
+    auto PA = PreservedAnalyses::all();
+    if (AM.getCachedResult<ShouldRunExtraSimpleLoopUnswitch>(L))
+      PA.intersect(LoopPassManager::run(L, AM, AR, U));
+    PA.abandon<ShouldRunExtraSimpleLoopUnswitch>();
+    return PA;
+  }
+
+  static bool isRequired() { return true; }
+};
+
 /// This pass transforms loops that contain branches or switches on loop-
 /// invariant conditions to have multiple loops. For example, it turns the left
 /// into the right code:
diff --git a/llvm/include/llvm/Transforms/Utils/SimplifyIndVar.h b/llvm/include/llvm/Transforms/Utils/SimplifyIndVar.h
index ff60811b6168..bd1718dd8cad 100644
--- a/llvm/include/llvm/Transforms/Utils/SimplifyIndVar.h
+++ b/llvm/include/llvm/Transforms/Utils/SimplifyIndVar.h
@@ -15,6 +15,8 @@
 #ifndef LLVM_TRANSFORMS_UTILS_SIMPLIFYINDVAR_H
 #define LLVM_TRANSFORMS_UTILS_SIMPLIFYINDVAR_H
 
+#include <utility>
+
 namespace llvm {
 
 class Type;
@@ -46,11 +48,16 @@ public:
 };
 
 /// simplifyUsersOfIV - Simplify instructions that use this induction variable
-/// by using ScalarEvolution to analyze the IV's recurrence.
-bool simplifyUsersOfIV(PHINode *CurrIV, ScalarEvolution *SE, DominatorTree *DT,
-                       LoopInfo *LI, const TargetTransformInfo *TTI,
-                       SmallVectorImpl<WeakTrackingVH> &Dead,
-                       SCEVExpander &Rewriter, IVVisitor *V = nullptr);
+/// by using ScalarEvolution to analyze the IV's recurrence. Returns a pair
+/// where the first entry indicates that the function makes changes and the
+/// second entry indicates that it introduced new opportunities for loop
+/// unswitching.
+std::pair<bool, bool> simplifyUsersOfIV(PHINode *CurrIV, ScalarEvolution *SE,
+                                        DominatorTree *DT, LoopInfo *LI,
+                                        const TargetTransformInfo *TTI,
+                                        SmallVectorImpl<WeakTrackingVH> &Dead,
+                                        SCEVExpander &Rewriter,
+                                        IVVisitor *V = nullptr);
 
 /// SimplifyLoopIVs - Simplify users of induction variables within this
 /// loop. This does not actually change or add IVs.
diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp
index cb892e30c4a0..3bb2ce0ae346 100644
--- a/llvm/lib/Passes/PassBuilderPipelines.cpp
+++ b/llvm/lib/Passes/PassBuilderPipelines.cpp
@@ -651,6 +651,13 @@ PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
   LPM2.addPass(LoopIdiomRecognizePass());
   LPM2.addPass(IndVarSimplifyPass());
 
+  {
+    ExtraSimpleLoopUnswitchPassManager ExtraPasses;
+    ExtraPasses.addPass(SimpleLoopUnswitchPass(/* NonTrivial */ Level ==
+                                               OptimizationLevel::O3));
+    LPM2.addPass(std::move(ExtraPasses));
+  }
+
   invokeLateLoopOptimizationsEPCallbacks(LPM2, Level);
 
   LPM2.addPass(LoopDeletionPass());
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index d15f58d7adfa..2fbc7f7d88ba 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -587,6 +587,9 @@ LOOP_ANALYSIS("ddg", DDGAnalysis())
 LOOP_ANALYSIS("iv-users", IVUsersAnalysis())
 LOOP_ANALYSIS("no-op-loop", NoOpLoopAnalysis())
 LOOP_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis(PIC))
+LOOP_ANALYSIS("should-run-extra-simple-loop-unswitch",
+                  ShouldRunExtraSimpleLoopUnswitch())
+
 #undef LOOP_ANALYSIS
 
 #ifndef LOOP_PASS
diff --git a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
index 38104afa7f78..ba392e187b8b 100644
--- a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
+++ b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
@@ -70,6 +70,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Transforms/Utils/LoopUtils.h"
@@ -137,6 +138,8 @@ class IndVarSimplify {
   SmallVector<WeakTrackingVH, 16> DeadInsts;
   bool WidenIndVars;
 
+  bool RunUnswitching = false;
+
   bool handleFloatingPointIV(Loop *L, PHINode *PH);
   bool rewriteNonIntegerIVs(Loop *L);
 
@@ -170,6 +173,8 @@ public:
   }
 
   bool run(Loop *L);
+
+  bool runUnswitching() const { return RunUnswitching; }
 };
 
 } // end anonymous namespace
@@ -615,9 +620,11 @@ bool IndVarSimplify::simplifyAndExtend(Loop *L,
       // Information about sign/zero extensions of CurrIV.
       IndVarSimplifyVisitor Visitor(CurrIV, SE, TTI, DT);
 
-      Changed |= simplifyUsersOfIV(CurrIV, SE, DT, LI, TTI, DeadInsts, Rewriter,
-                                   &Visitor);
+      const auto &[C, U] = simplifyUsersOfIV(CurrIV, SE, DT, LI, TTI, DeadInsts,
+                                             Rewriter, &Visitor);
 
+      Changed |= C;
+      RunUnswitching |= U;
       if (Visitor.WI.WidestNativeType) {
         WideIVs.push_back(Visitor.WI);
       }
@@ -1874,6 +1881,7 @@ bool IndVarSimplify::predicateLoopExits(Loop *L, SCEVExpander &Rewriter) {
     if (OldCond->use_empty())
       DeadInsts.emplace_back(OldCond);
     Changed = true;
+    RunUnswitching = true;
   }
 
   return Changed;
@@ -2059,6 +2067,11 @@ PreservedAnalyses IndVarSimplifyPass::run(Loop &L, LoopAnalysisManager &AM,
 
   auto PA = getLoopPassPreservedAnalyses();
   PA.preserveSet<CFGAnalyses>();
+  if (IVS.runUnswitching()) {
+    AM.getResult<ShouldRunExtraSimpleLoopUnswitch>(L, AR);
+    PA.preserve<ShouldRunExtraSimpleLoopUnswitch>();
+  }
+
   if (AR.MSSA)
     PA.preserve<MemorySSAAnalysis>();
   return PA;
diff --git a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
index 64b850a6a5c2..d763b1ee0aa1 100644
--- a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
+++ b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
@@ -133,6 +133,7 @@ static cl::opt<unsigned> InjectInvariantConditionHotnesThreshold(
                          "not-taken 1/<this option> times or less."),
     cl::init(16));
 
+AnalysisKey ShouldRunExtraSimpleLoopUnswitch::Key;
 namespace {
 struct CompareDesc {
   BranchInst *Term;
diff --git a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
index f6d72265305d..440fe0790d79 100644
--- a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
@@ -60,6 +60,7 @@ namespace {
     SmallVectorImpl<WeakTrackingVH> &DeadInsts;
 
     bool Changed = false;
+    bool RunUnswitching = false;
 
   public:
     SimplifyIndvar(Loop *Loop, ScalarEvolution *SE, DominatorTree *DT,
@@ -72,6 +73,7 @@ namespace {
     }
 
     bool hasChanged() const { return Changed; }
+    bool runUnswitching() const { return RunUnswitching; }
 
     /// Iteratively perform simplification on a worklist of users of the
     /// specified induction variable. This is the top-level driver that applies
@@ -233,6 +235,7 @@ bool SimplifyIndvar::makeIVComparisonInvariant(ICmpInst *ICmp,
   ICmp->setPredicate(InvariantPredicate);
   ICmp->setOperand(0, NewLHS);
   ICmp->setOperand(1, NewRHS);
+  RunUnswitching = true;
   return true;
 }
 
@@ -993,14 +996,18 @@ void IVVisitor::anchor() { }
 
 /// Simplify instructions that use this induction variable
 /// by using ScalarEvolution to analyze the IV's recurrence.
-bool simplifyUsersOfIV(PHINode *CurrIV, ScalarEvolution *SE, DominatorTree *DT,
-                       LoopInfo *LI, const TargetTransformInfo *TTI,
-                       SmallVectorImpl<WeakTrackingVH> &Dead,
-                       SCEVExpander &Rewriter, IVVisitor *V) {
+///  Returns a pair where the first entry indicates that the function makes
+///  changes and the second entry indicates that it introduced new opportunities
+///  for loop unswitching.
+std::pair<bool, bool> simplifyUsersOfIV(PHINode *CurrIV, ScalarEvolution *SE,
+                                        DominatorTree *DT, LoopInfo *LI,
+                                        const TargetTransformInfo *TTI,
+                                        SmallVectorImpl<WeakTrackingVH> &Dead,
+                                        SCEVExpander &Rewriter, IVVisitor *V) {
   SimplifyIndvar SIV(LI->getLoopFor(CurrIV->getParent()), SE, DT, LI, TTI,
                      Rewriter, Dead);
   SIV.simplifyUsers(CurrIV, V);
-  return SIV.hasChanged();
+  return {SIV.hasChanged(), SIV.runUnswitching()};
 }
 
 /// Simplify users of induction variables within this
@@ -1014,8 +1021,9 @@ bool simplifyLoopIVs(Loop *L, ScalarEvolution *SE, DominatorTree *DT,
 #endif
   bool Changed = false;
   for (BasicBlock::iterator I = L->getHeader()->begin(); isa<PHINode>(I); ++I) {
-    Changed |=
+    const auto &[C, _] =
         simplifyUsersOfIV(cast<PHINode>(I), SE, DT, LI, TTI, Dead, Rewriter);
+    Changed |= C;
   }
   return Changed;
 }
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll
index a140e17a0dd1..55dd28b70170 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/hoist-runtime-checks.ll
@@ -14,24 +14,50 @@ define i32 @read_only_loop_with_runtime_check(ptr noundef %array, i32 noundef %c
 ; CHECK-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
 ; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[N]], -1
 ; CHECK-NEXT:    [[DOTNOT_NOT:%.*]] = icmp ult i32 [[TMP1]], [[COUNT]]
+; CHECK-NEXT:    br i1 [[DOTNOT_NOT]], label [[FOR_BODY_PREHEADER10:%.*]], label [[IF_THEN:%.*]]
+; CHECK:       for.body.preheader10:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], 8
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[FOR_BODY_PREHEADER13:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[TMP0]], 4294967288
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI11:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[ARRAY]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 16
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD12:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP4]] = add <4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-NEXT:    [[TMP5]] = add <4 x i32> [[WIDE_LOAD12]], [[VEC_PHI11]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[TMP0]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY_PREHEADER13]]
+; CHECK:       for.body.preheader13:
+; CHECK-NEXT:    [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER10]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[SUM_07_PH:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER10]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.cond.cleanup:
-; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD:%.*]], [[IF_END:%.*]] ]
+; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP7]], [[MIDDLE_BLOCK]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    ret i32 [[SUM_0_LCSSA]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[IF_END]] ]
-; CHECK-NEXT:    [[SUM_07:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[ADD]], [[IF_END]] ]
-; CHECK-NEXT:    br i1 [[DOTNOT_NOT]], label [[IF_END]], label [[IF_THEN:%.*]]
-; CHECK:       if.then:
-; CHECK-NEXT:    tail call void @llvm.trap()
-; CHECK-NEXT:    unreachable
-; CHECK:       if.end:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ [[INDVARS_IV_PH]], [[FOR_BODY_PREHEADER13]] ]
+; CHECK-NEXT:    [[SUM_07:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[SUM_07_PH]], [[FOR_BODY_PREHEADER13]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[ARRAY]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT:    [[ADD]] = add nsw i32 [[TMP2]], [[SUM_07]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[ADD]] = add nsw i32 [[TMP8]], [[SUM_07]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[TMP0]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       if.then:
+; CHECK-NEXT:    tail call void @llvm.trap()
+; CHECK-NEXT:    unreachable
 ;
 entry:
   %array.addr = alloca ptr, align 8
@@ -103,24 +129,50 @@ define dso_local noundef i32 @sum_prefix_with_sum(ptr %s.coerce0, i64 %s.coerce1
 ; CHECK:       for.body.preheader:
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[N]], -1
 ; CHECK-NEXT:    [[DOTNOT_NOT:%.*]] = icmp ult i64 [[TMP0]], [[S_COERCE1]]
+; CHECK-NEXT:    br i1 [[DOTNOT_NOT]], label [[ENTRY:%.*]], label [[COND_FALSE_I:%.*]], !prof [[PROF4:![0-9]+]]
+; CHECK:       for.body.preheader8:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[FOR_BODY_PREHEADER11:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i64 [[N]], -8
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI9:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[S_COERCE0]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 16
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD10:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP3]] = add <4 x i32> [[WIDE_LOAD]], [[VEC_PHI]]
+; CHECK-NEXT:    [[TMP4]] = add <4 x i32> [[WIDE_LOAD10]], [[VEC_PHI9]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[SPAN_CHECKED_ACCESS_EXIT:%.*]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP4]], [[TMP3]]
+; CHECK-NEXT:    [[ADD:%.*]] = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY_PREHEADER11]]
+; CHECK:       for.body.preheader11:
+; CHECK-NEXT:    [[I_07_PH:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[N_VEC]], [[SPAN_CHECKED_ACCESS_EXIT]] ]
+; CHECK-NEXT:    [[RET_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[ADD]], [[SPAN_CHECKED_ACCESS_EXIT]] ]
+; CHECK-NEXT:    br label [[FOR_BODY1:%.*]]
 ; CHECK:       for.cond.cleanup:
-; CHECK-NEXT:    [[RET_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD:%.*]], [[SPAN_CHECKED_ACCESS_EXIT:%.*]] ]
-; CHECK-NEXT:    ret i32 [[RET_0_LCSSA]]
+; CHECK-NEXT:    [[RET_0_LCSSA1:%.*]] = phi i32 [ 0, [[ENTRY1:%.*]] ], [ [[ADD]], [[SPAN_CHECKED_ACCESS_EXIT]] ], [ [[ADD1:%.*]], [[FOR_BODY1]] ]
+; CHECK-NEXT:    ret i32 [[RET_0_LCSSA1]]
 ; CHECK:       for.body:
-; CHECK-NEXT:    [[I_07:%.*]] = phi i64 [ [[INC:%.*]], [[SPAN_CHECKED_ACCESS_EXIT]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT:    [[RET_06:%.*]] = phi i32 [ [[ADD]], [[SPAN_CHECKED_ACCESS_EXIT]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT:    br i1 [[DOTNOT_NOT]], label [[SPAN_CHECKED_ACCESS_EXIT]], label [[COND_FALSE_I:%.*]], !prof [[PROF0:![0-9]+]]
-; CHECK:       cond.false.i:
-; CHECK-NEXT:    tail call void @llvm.trap()
-; CHECK-NEXT:    unreachable
-; CHECK:       span_checked_access.exit:
+; CHECK-NEXT:    [[I_07:%.*]] = phi i64 [ [[INC:%.*]], [[FOR_BODY1]] ], [ [[I_07_PH]], [[FOR_BODY_PREHEADER11]] ]
+; CHECK-NEXT:    [[RET_06:%.*]] = phi i32 [ [[ADD1]], [[FOR_BODY1]] ], [ [[RET_0_LCSSA]], [[FOR_BODY_PREHEADER11]] ]
 ; CHECK-NEXT:    [[ARRAYIDX_I:%.*]] = getelementptr inbounds i32, ptr [[S_COERCE0]], i64 [[I_07]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX_I]], align 4
-; CHECK-NEXT:    [[ADD]] = add nsw i32 [[TMP7]], [[RET_06]]
+; CHECK-NEXT:    [[ADD1]] = add nsw i32 [[TMP7]], [[RET_06]]
 ; CHECK-NEXT:    [[INC]] = add nuw i64 [[I_07]], 1
 ; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]]
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]]
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY1]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       cond.false.i:
+; CHECK-NEXT:    tail call void @llvm.trap()
+; CHECK-NEXT:    unreachable
 ;
 entry:
   %s = alloca %"class.std::__1::span", align 8
@@ -176,7 +228,7 @@ define hidden noundef nonnull align 4 dereferenceable(4) ptr @span_checked_acces
 ; CHECK-NEXT:    [[__SIZE__I:%.*]] = getelementptr inbounds i8, ptr [[THIS]], i64 8
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[__SIZE__I]], align 8
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i64 [[TMP0]], [[__IDX]]
-; CHECK-NEXT:    br i1 [[CMP]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]], !prof [[PROF0]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[COND_END:%.*]], label [[COND_FALSE:%.*]], !prof [[PROF4]]
 ; CHECK:       cond.false:
 ; CHECK-NEXT:    tail call void @llvm.trap()
 ; CHECK-NEXT:    unreachable
@@ -237,5 +289,11 @@ declare void @llvm.trap()
 
 declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
 ;.
-; CHECK: [[PROF0]] = !{!"branch_weights", i32 2000, i32 1}
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK: [[PROF4]] = !{!"branch_weights", i32 2000, i32 1}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]}
+; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META2]], [[META1]]}
 ;.
-- 
GitLab


From c6cd4608c8c4a5356ae41ef05e67df3dc5d9500b Mon Sep 17 00:00:00 2001
From: Matthias Braun <matze@braunis.de>
Date: Fri, 12 Apr 2024 14:10:07 -0700
Subject: [PATCH 114/464] IRVerifier: Allow GlobalValue as
 llvm.threadlocal.address operand (#88321)

Loosen `llvm.threadlocal.address` verifier checks to allow any
`GlobalValue` with `isThreadLocal()` set to true.
---
 llvm/docs/LangRef.rst                  |  3 ++-
 llvm/lib/IR/Verifier.cpp               |  8 +++---
 llvm/test/Verifier/threadlocal-pass.ll | 13 ++++++++++
 llvm/test/Verifier/threadlocal.ll      | 34 ++++++++++++++++++++++++++
 4 files changed, 53 insertions(+), 5 deletions(-)
 create mode 100644 llvm/test/Verifier/threadlocal-pass.ll
 create mode 100644 llvm/test/Verifier/threadlocal.ll

diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index f6ada292b93b..9592929d79fe 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -28130,7 +28130,8 @@ Syntax:
 Arguments:
 """"""""""
 
-The first argument is a thread local :ref:`global variable <globalvars>`.
+The `llvm.threadlocal.address` intrinsic requires a global value argument (a
+:ref:`global variable <globalvars>` or alias) that is thread local.
 
 Semantics:
 """"""""""
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 4092f0cb12ff..25cb99f0f9bd 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -6226,10 +6226,10 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
   }
   case Intrinsic::threadlocal_address: {
     const Value &Arg0 = *Call.getArgOperand(0);
-    Check(isa<GlobalVariable>(Arg0),
-          "llvm.threadlocal.address first argument must be a GlobalVariable");
-    Check(cast<GlobalVariable>(Arg0).isThreadLocal(),
-          "llvm.threadlocal.address operand isThreadLocal() must no be false");
+    Check(isa<GlobalValue>(Arg0),
+          "llvm.threadlocal.address first argument must be a GlobalValue");
+    Check(cast<GlobalValue>(Arg0).isThreadLocal(),
+          "llvm.threadlocal.address operand isThreadLocal() must be true");
     break;
   }
   };
diff --git a/llvm/test/Verifier/threadlocal-pass.ll b/llvm/test/Verifier/threadlocal-pass.ll
new file mode 100644
index 000000000000..0cee4af82439
--- /dev/null
+++ b/llvm/test/Verifier/threadlocal-pass.ll
@@ -0,0 +1,13 @@
+; RUN: opt -passes=verify -S < %s | FileCheck %s
+
+@var = thread_local global i32 0
+@alias = thread_local alias i32, ptr @var
+
+; CHECK-LABEL: @should_pass
+define void @should_pass() {
+  %p0 = call ptr @llvm.threadlocal.address(ptr @var)
+  store i32 42, ptr %p0, align 4
+  %p1 = call ptr @llvm.threadlocal.address(ptr @alias)
+  store i32 13, ptr %p1, align 4
+  ret void
+}
diff --git a/llvm/test/Verifier/threadlocal.ll b/llvm/test/Verifier/threadlocal.ll
new file mode 100644
index 000000000000..8af23c361cc4
--- /dev/null
+++ b/llvm/test/Verifier/threadlocal.ll
@@ -0,0 +1,34 @@
+; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s
+
+@var = global i32 0
+@tlsvar = thread_local addrspace(1) global i32 0
+
+define void @fail0(ptr %arg) {
+; CHECK: llvm.threadlocal.address first argument must be a GlobalValue
+  %p0 = call ptr @llvm.threadlocal.address(ptr %arg)
+  store i32 42, ptr %p0, align 4
+  ret void
+}
+
+define void @fail1() {
+; CHECK: llvm.threadlocal.address first argument must be a GlobalValue
+  %p0 = call ptr @llvm.threadlocal.address.p0(ptr addrspacecast (ptr addrspace(1) @tlsvar to ptr addrspace(0)))
+  store i32 42, ptr %p0, align 4
+  ret void
+}
+
+
+
+define void @fail2() {
+; CHECK: llvm.threadlocal.address operand isThreadLocal() must be true
+  %p0 = call ptr @llvm.threadlocal.address(ptr @var)
+  store i32 42, ptr %p0, align 4
+  ret void
+}
+
+define void @fail3() {
+; CHECK: llvm.threadlocal.address operand isThreadLocal() must be true
+  %p0 = call ptr @llvm.threadlocal.address(ptr @fail2)
+  store i32 42, ptr %p0, align 4
+  ret void
+}
-- 
GitLab


From daa88364df15683bfa0e58d62cc96d33cd49d34b Mon Sep 17 00:00:00 2001
From: Erich Keane <ekeane@nvidia.com>
Date: Fri, 12 Apr 2024 14:13:31 -0700
Subject: [PATCH 115/464] [OpenACC] Implement 'if' clause for Compute
 Constructs (#88411)

Like with the 'default' clause, this is being applied to only Compute
Constructs for now. The 'if' clause takes a condition expression which
is used as a runtime value.

This is not a particularly complex semantic implementation, as there
isn't much to this clause, other than its interactions with 'self',
  which will be managed in the patch to implement that.
---
 clang/include/clang/AST/ASTNodeTraverser.h    |   3 +-
 clang/include/clang/AST/OpenACCClause.h       |  81 +++++++++++-
 clang/include/clang/Basic/OpenACCClauses.def  |  21 +++
 clang/include/clang/Parse/Parser.h            |   5 +
 clang/include/clang/Sema/SemaOpenACC.h        |  28 +++-
 clang/lib/AST/OpenACCClause.cpp               |  39 ++++++
 clang/lib/AST/StmtProfile.cpp                 |  19 ++-
 clang/lib/AST/TextNodeDumper.cpp              |   5 +
 clang/lib/Parse/ParseOpenACC.cpp              |  32 +++--
 clang/lib/Sema/SemaOpenACC.cpp                |  73 +++++++++--
 clang/lib/Sema/TreeTransform.h                |  14 ++
 clang/lib/Serialization/ASTReader.cpp         |   7 +-
 clang/lib/Serialization/ASTWriter.cpp         |   7 +-
 clang/test/ParserOpenACC/parse-clauses.c      |   2 -
 .../compute-construct-clause-ast.cpp          | 120 +++++++++++++++++-
 .../SemaOpenACC/compute-construct-if-clause.c |  62 +++++++++
 .../compute-construct-if-clause.cpp           |  33 +++++
 clang/tools/libclang/CIndex.cpp               |  43 +++++++
 18 files changed, 557 insertions(+), 37 deletions(-)
 create mode 100644 clang/include/clang/Basic/OpenACCClauses.def
 create mode 100644 clang/test/SemaOpenACC/compute-construct-if-clause.c
 create mode 100644 clang/test/SemaOpenACC/compute-construct-if-clause.cpp

diff --git a/clang/include/clang/AST/ASTNodeTraverser.h b/clang/include/clang/AST/ASTNodeTraverser.h
index a6e3b05b0059..f5c47d8a7c21 100644
--- a/clang/include/clang/AST/ASTNodeTraverser.h
+++ b/clang/include/clang/AST/ASTNodeTraverser.h
@@ -243,7 +243,8 @@ public:
   void Visit(const OpenACCClause *C) {
     getNodeDelegate().AddChild([=] {
       getNodeDelegate().Visit(C);
-      // TODO OpenACC: Switch on clauses that have children, and add them.
+      for (const auto *S : C->children())
+        Visit(S);
     });
   }
 
diff --git a/clang/include/clang/AST/OpenACCClause.h b/clang/include/clang/AST/OpenACCClause.h
index 27e4e1a12c98..712ad46c0f23 100644
--- a/clang/include/clang/AST/OpenACCClause.h
+++ b/clang/include/clang/AST/OpenACCClause.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_CLANG_AST_OPENACCCLAUSE_H
 #define LLVM_CLANG_AST_OPENACCCLAUSE_H
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/StmtIterator.h"
 #include "clang/Basic/OpenACCKinds.h"
 
 namespace clang {
@@ -34,6 +35,17 @@ public:
 
   static bool classof(const OpenACCClause *) { return true; }
 
+  using child_iterator = StmtIterator;
+  using const_child_iterator = ConstStmtIterator;
+  using child_range = llvm::iterator_range<child_iterator>;
+  using const_child_range = llvm::iterator_range<const_child_iterator>;
+
+  child_range children();
+  const_child_range children() const {
+    auto Children = const_cast<OpenACCClause *>(this)->children();
+    return const_child_range(Children.begin(), Children.end());
+  }
+
   virtual ~OpenACCClause() = default;
 };
 
@@ -49,6 +61,13 @@ protected:
 
 public:
   SourceLocation getLParenLoc() const { return LParenLoc; }
+
+  child_range children() {
+    return child_range(child_iterator(), child_iterator());
+  }
+  const_child_range children() const {
+    return const_child_range(const_child_iterator(), const_child_iterator());
+  }
 };
 
 /// A 'default' clause, has the optional 'none' or 'present' argument.
@@ -81,6 +100,51 @@ public:
                                       SourceLocation EndLoc);
 };
 
+/// Represents one of the handful of classes that has an optional/required
+/// 'condition' expression as an argument.
+class OpenACCClauseWithCondition : public OpenACCClauseWithParams {
+  Expr *ConditionExpr = nullptr;
+
+protected:
+  OpenACCClauseWithCondition(OpenACCClauseKind K, SourceLocation BeginLoc,
+                             SourceLocation LParenLoc, Expr *ConditionExpr,
+                             SourceLocation EndLoc)
+      : OpenACCClauseWithParams(K, BeginLoc, LParenLoc, EndLoc),
+        ConditionExpr(ConditionExpr) {}
+
+public:
+  bool hasConditionExpr() const { return ConditionExpr; }
+  const Expr *getConditionExpr() const { return ConditionExpr; }
+  Expr *getConditionExpr() { return ConditionExpr; }
+
+  child_range children() {
+    if (ConditionExpr)
+      return child_range(reinterpret_cast<Stmt **>(&ConditionExpr),
+                         reinterpret_cast<Stmt **>(&ConditionExpr + 1));
+    return child_range(child_iterator(), child_iterator());
+  }
+
+  const_child_range children() const {
+    if (ConditionExpr)
+      return const_child_range(
+          reinterpret_cast<Stmt *const *>(&ConditionExpr),
+          reinterpret_cast<Stmt *const *>(&ConditionExpr + 1));
+    return const_child_range(const_child_iterator(), const_child_iterator());
+  }
+};
+
+/// An 'if' clause, which has a required condition expression.
+class OpenACCIfClause : public OpenACCClauseWithCondition {
+protected:
+  OpenACCIfClause(SourceLocation BeginLoc, SourceLocation LParenLoc,
+                  Expr *ConditionExpr, SourceLocation EndLoc);
+
+public:
+  static OpenACCIfClause *Create(const ASTContext &C, SourceLocation BeginLoc,
+                                 SourceLocation LParenLoc, Expr *ConditionExpr,
+                                 SourceLocation EndLoc);
+};
+
 template <class Impl> class OpenACCClauseVisitor {
   Impl &getDerived() { return static_cast<Impl &>(*this); }
 
@@ -98,6 +162,9 @@ public:
     case OpenACCClauseKind::Default:
       VisitOpenACCDefaultClause(*cast<OpenACCDefaultClause>(C));
       return;
+    case OpenACCClauseKind::If:
+      VisitOpenACCIfClause(*cast<OpenACCIfClause>(C));
+      return;
     case OpenACCClauseKind::Finalize:
     case OpenACCClauseKind::IfPresent:
     case OpenACCClauseKind::Seq:
@@ -106,7 +173,6 @@ public:
     case OpenACCClauseKind::Worker:
     case OpenACCClauseKind::Vector:
     case OpenACCClauseKind::NoHost:
-    case OpenACCClauseKind::If:
     case OpenACCClauseKind::Self:
     case OpenACCClauseKind::Copy:
     case OpenACCClauseKind::UseDevice:
@@ -145,9 +211,13 @@ public:
     llvm_unreachable("Invalid Clause kind");
   }
 
-  void VisitOpenACCDefaultClause(const OpenACCDefaultClause &Clause) {
-    return getDerived().VisitOpenACCDefaultClause(Clause);
+#define VISIT_CLAUSE(CLAUSE_NAME)                                              \
+  void VisitOpenACC##CLAUSE_NAME##Clause(                                      \
+      const OpenACC##CLAUSE_NAME##Clause &Clause) {                            \
+    return getDerived().VisitOpenACC##CLAUSE_NAME##Clause(Clause);             \
   }
+
+#include "clang/Basic/OpenACCClauses.def"
 };
 
 class OpenACCClausePrinter final
@@ -165,7 +235,10 @@ public:
   }
   OpenACCClausePrinter(raw_ostream &OS) : OS(OS) {}
 
-  void VisitOpenACCDefaultClause(const OpenACCDefaultClause &Clause);
+#define VISIT_CLAUSE(CLAUSE_NAME)                                              \
+  void VisitOpenACC##CLAUSE_NAME##Clause(                                      \
+      const OpenACC##CLAUSE_NAME##Clause &Clause);
+#include "clang/Basic/OpenACCClauses.def"
 };
 
 } // namespace clang
diff --git a/clang/include/clang/Basic/OpenACCClauses.def b/clang/include/clang/Basic/OpenACCClauses.def
new file mode 100644
index 000000000000..7fd2720e02ce
--- /dev/null
+++ b/clang/include/clang/Basic/OpenACCClauses.def
@@ -0,0 +1,21 @@
+//===-- OpenACCClauses.def - List of implemented OpenACC Clauses -- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines a list of currently implemented OpenACC Clauses (and
+// eventually, the entire list) in a way that makes generating 'visitor' and
+// other lists easier.
+//
+// The primary macro is a single-argument version taking the name of the Clause
+// as used in Clang source (so `Default` instead of `default`).
+//
+// VISIT_CLAUSE(CLAUSE_NAME)
+
+VISIT_CLAUSE(Default)
+VISIT_CLAUSE(If)
+
+#undef VISIT_CLAUSE
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index 3a055c10ffb3..db90d38c7dac 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -3611,6 +3611,9 @@ private:
   OpenACCClauseParseResult OpenACCCannotContinue();
   OpenACCClauseParseResult OpenACCSuccess(OpenACCClause *Clause);
 
+  using OpenACCConditionExprParseResult =
+      std::pair<ExprResult, OpenACCParseCanContinue>;
+
   /// Parses the OpenACC directive (the entire pragma) including the clause
   /// list, but does not produce the main AST node.
   OpenACCDirectiveParseInfo ParseOpenACCDirective();
@@ -3657,6 +3660,8 @@ private:
   bool ParseOpenACCGangArgList();
   /// Parses a 'gang-arg', used for the 'gang' clause.
   bool ParseOpenACCGangArg();
+  /// Parses a 'condition' expr, ensuring it results in a
+  ExprResult ParseOpenACCConditionExpr();
 
 private:
   //===--------------------------------------------------------------------===//
diff --git a/clang/include/clang/Sema/SemaOpenACC.h b/clang/include/clang/Sema/SemaOpenACC.h
index 27aaee164a28..c1fe0f5b9c0f 100644
--- a/clang/include/clang/Sema/SemaOpenACC.h
+++ b/clang/include/clang/Sema/SemaOpenACC.h
@@ -40,7 +40,11 @@ public:
       OpenACCDefaultClauseKind DefaultClauseKind;
     };
 
-    std::variant<DefaultDetails> Details;
+    struct ConditionDetails {
+      Expr *ConditionExpr;
+    };
+
+    std::variant<DefaultDetails, ConditionDetails> Details;
 
   public:
     OpenACCParsedClause(OpenACCDirectiveKind DirKind,
@@ -63,6 +67,16 @@ public:
       return std::get<DefaultDetails>(Details).DefaultClauseKind;
     }
 
+    const Expr *getConditionExpr() const {
+      return const_cast<OpenACCParsedClause *>(this)->getConditionExpr();
+    }
+
+    Expr *getConditionExpr() {
+      assert(ClauseKind == OpenACCClauseKind::If &&
+             "Parsed clause kind does not have a condition expr");
+      return std::get<ConditionDetails>(Details).ConditionExpr;
+    }
+
     void setLParenLoc(SourceLocation EndLoc) { LParenLoc = EndLoc; }
     void setEndLoc(SourceLocation EndLoc) { ClauseRange.setEnd(EndLoc); }
 
@@ -71,6 +85,18 @@ public:
              "Parsed clause is not a default clause");
       Details = DefaultDetails{DefKind};
     }
+
+    void setConditionDetails(Expr *ConditionExpr) {
+      assert(ClauseKind == OpenACCClauseKind::If &&
+             "Parsed clause kind does not have a condition expr");
+      // In C++ we can count on this being a 'bool', but in C this gets left as
+      // some sort of scalar that codegen will have to take care of converting.
+      assert((!ConditionExpr || ConditionExpr->isInstantiationDependent() ||
+              ConditionExpr->getType()->isScalarType()) &&
+             "Condition expression type not scalar/dependent");
+
+      Details = ConditionDetails{ConditionExpr};
+    }
   };
 
   SemaOpenACC(Sema &S);
diff --git a/clang/lib/AST/OpenACCClause.cpp b/clang/lib/AST/OpenACCClause.cpp
index c83128b60e3a..2bc8d14fa129 100644
--- a/clang/lib/AST/OpenACCClause.cpp
+++ b/clang/lib/AST/OpenACCClause.cpp
@@ -13,6 +13,7 @@
 
 #include "clang/AST/OpenACCClause.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/Expr.h"
 
 using namespace clang;
 
@@ -27,6 +28,40 @@ OpenACCDefaultClause *OpenACCDefaultClause::Create(const ASTContext &C,
   return new (Mem) OpenACCDefaultClause(K, BeginLoc, LParenLoc, EndLoc);
 }
 
+OpenACCIfClause *OpenACCIfClause::Create(const ASTContext &C,
+                                         SourceLocation BeginLoc,
+                                         SourceLocation LParenLoc,
+                                         Expr *ConditionExpr,
+                                         SourceLocation EndLoc) {
+  void *Mem = C.Allocate(sizeof(OpenACCIfClause), alignof(OpenACCIfClause));
+  return new (Mem) OpenACCIfClause(BeginLoc, LParenLoc, ConditionExpr, EndLoc);
+}
+
+OpenACCIfClause::OpenACCIfClause(SourceLocation BeginLoc,
+                                 SourceLocation LParenLoc, Expr *ConditionExpr,
+                                 SourceLocation EndLoc)
+    : OpenACCClauseWithCondition(OpenACCClauseKind::If, BeginLoc, LParenLoc,
+                                 ConditionExpr, EndLoc) {
+  assert(ConditionExpr && "if clause requires condition expr");
+  assert((ConditionExpr->isInstantiationDependent() ||
+          ConditionExpr->getType()->isScalarType()) &&
+         "Condition expression type not scalar/dependent");
+}
+
+OpenACCClause::child_range OpenACCClause::children() {
+  switch (getClauseKind()) {
+  default:
+    assert(false && "Clause children function not implemented");
+    break;
+#define VISIT_CLAUSE(CLAUSE_NAME)                                              \
+  case OpenACCClauseKind::CLAUSE_NAME:                                         \
+    return cast<OpenACC##CLAUSE_NAME##Clause>(this)->children();
+
+#include "clang/Basic/OpenACCClauses.def"
+  }
+  return child_range(child_iterator(), child_iterator());
+}
+
 //===----------------------------------------------------------------------===//
 //  OpenACC clauses printing methods
 //===----------------------------------------------------------------------===//
@@ -34,3 +69,7 @@ void OpenACCClausePrinter::VisitOpenACCDefaultClause(
     const OpenACCDefaultClause &C) {
   OS << "default(" << C.getDefaultClauseKind() << ")";
 }
+
+void OpenACCClausePrinter::VisitOpenACCIfClause(const OpenACCIfClause &C) {
+  OS << "if(" << C.getConditionExpr() << ")";
+}
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index 01e1d1cc8289..44b635bfb843 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -2445,9 +2445,10 @@ void StmtProfiler::VisitTemplateArgument(const TemplateArgument &Arg) {
 namespace {
 class OpenACCClauseProfiler
     : public OpenACCClauseVisitor<OpenACCClauseProfiler> {
+  StmtProfiler &Profiler;
 
 public:
-  OpenACCClauseProfiler() = default;
+  OpenACCClauseProfiler(StmtProfiler &P) : Profiler(P) {}
 
   void VisitOpenACCClauseList(ArrayRef<const OpenACCClause *> Clauses) {
     for (const OpenACCClause *Clause : Clauses) {
@@ -2456,12 +2457,24 @@ public:
       Visit(Clause);
     }
   }
-  void VisitOpenACCDefaultClause(const OpenACCDefaultClause &Clause);
+
+#define VISIT_CLAUSE(CLAUSE_NAME)                                              \
+  void VisitOpenACC##CLAUSE_NAME##Clause(                                      \
+      const OpenACC##CLAUSE_NAME##Clause &Clause);
+
+#include "clang/Basic/OpenACCClauses.def"
 };
 
 /// Nothing to do here, there are no sub-statements.
 void OpenACCClauseProfiler::VisitOpenACCDefaultClause(
     const OpenACCDefaultClause &Clause) {}
+
+void OpenACCClauseProfiler::VisitOpenACCIfClause(
+    const OpenACCIfClause &Clause) {
+  assert(Clause.hasConditionExpr() &&
+         "if clause requires a valid condition expr");
+  Profiler.VisitStmt(Clause.getConditionExpr());
+}
 } // namespace
 
 void StmtProfiler::VisitOpenACCComputeConstruct(
@@ -2469,7 +2482,7 @@ void StmtProfiler::VisitOpenACCComputeConstruct(
   // VisitStmt handles children, so the AssociatedStmt is handled.
   VisitStmt(S);
 
-  OpenACCClauseProfiler P;
+  OpenACCClauseProfiler P{*this};
   P.VisitOpenACCClauseList(S->clauses());
 }
 
diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp
index f4b2df7730ae..0973e63183bd 100644
--- a/clang/lib/AST/TextNodeDumper.cpp
+++ b/clang/lib/AST/TextNodeDumper.cpp
@@ -397,6 +397,11 @@ void TextNodeDumper::Visit(const OpenACCClause *C) {
     case OpenACCClauseKind::Default:
       OS << '(' << cast<OpenACCDefaultClause>(C)->getDefaultClauseKind() << ')';
       break;
+    case OpenACCClauseKind::If:
+      // The condition expression will be printed as a part of the 'children',
+      // but print 'clause' here so it is clear what is happening from the dump.
+      OS << " clause";
+      break;
     default:
       // Nothing to do here.
       break;
diff --git a/clang/lib/Parse/ParseOpenACC.cpp b/clang/lib/Parse/ParseOpenACC.cpp
index b487a1968d1e..91f2b8afcf0c 100644
--- a/clang/lib/Parse/ParseOpenACC.cpp
+++ b/clang/lib/Parse/ParseOpenACC.cpp
@@ -535,14 +535,6 @@ bool ClauseHasRequiredParens(OpenACCDirectiveKind DirKind,
   return getClauseParensKind(DirKind, Kind) == ClauseParensKind::Required;
 }
 
-ExprResult ParseOpenACCConditionalExpr(Parser &P) {
-  // FIXME: It isn't clear if the spec saying 'condition' means the same as
-  // it does in an if/while/etc (See ParseCXXCondition), however as it was
-  // written with Fortran/C in mind, we're going to assume it just means an
-  // 'expression evaluating to boolean'.
-  return P.getActions().CorrectDelayedTyposInExpr(P.ParseExpression());
-}
-
 // Skip until we see the end of pragma token, but don't consume it. This is us
 // just giving up on the rest of the pragma so we can continue executing. We
 // have to do this because 'SkipUntil' considers paren balancing, which isn't
@@ -595,6 +587,23 @@ Parser::OpenACCClauseParseResult Parser::OpenACCSuccess(OpenACCClause *Clause) {
   return {Clause, OpenACCParseCanContinue::Can};
 }
 
+ExprResult Parser::ParseOpenACCConditionExpr() {
+  // FIXME: It isn't clear if the spec saying 'condition' means the same as
+  // it does in an if/while/etc (See ParseCXXCondition), however as it was
+  // written with Fortran/C in mind, we're going to assume it just means an
+  // 'expression evaluating to boolean'.
+  ExprResult ER = getActions().CorrectDelayedTyposInExpr(ParseExpression());
+
+  if (!ER.isUsable())
+    return ER;
+
+  Sema::ConditionResult R =
+      getActions().ActOnCondition(getCurScope(), ER.get()->getExprLoc(),
+                                  ER.get(), Sema::ConditionKind::Boolean);
+
+  return R.isInvalid() ? ExprError() : R.get().second;
+}
+
 // OpenACC 3.3, section 1.7:
 // To simplify the specification and convey appropriate constraint information,
 // a pqr-list is a comma-separated list of pdr items. The one exception is a
@@ -842,12 +851,15 @@ Parser::OpenACCClauseParseResult Parser::ParseOpenACCClauseParams(
       break;
     }
     case OpenACCClauseKind::If: {
-      ExprResult CondExpr = ParseOpenACCConditionalExpr(*this);
+      ExprResult CondExpr = ParseOpenACCConditionExpr();
+      ParsedClause.setConditionDetails(CondExpr.isUsable() ? CondExpr.get()
+                                                           : nullptr);
 
       if (CondExpr.isInvalid()) {
         Parens.skipToEnd();
         return OpenACCCanContinue();
       }
+
       break;
     }
     case OpenACCClauseKind::CopyIn:
@@ -964,7 +976,7 @@ Parser::OpenACCClauseParseResult Parser::ParseOpenACCClauseParams(
       switch (ClauseKind) {
       case OpenACCClauseKind::Self: {
         assert(DirKind != OpenACCDirectiveKind::Update);
-        ExprResult CondExpr = ParseOpenACCConditionalExpr(*this);
+        ExprResult CondExpr = ParseOpenACCConditionExpr();
 
         if (CondExpr.isInvalid()) {
           Parens.skipToEnd();
diff --git a/clang/lib/Sema/SemaOpenACC.cpp b/clang/lib/Sema/SemaOpenACC.cpp
index a6f4453e525d..1249136c8765 100644
--- a/clang/lib/Sema/SemaOpenACC.cpp
+++ b/clang/lib/Sema/SemaOpenACC.cpp
@@ -55,12 +55,49 @@ bool doesClauseApplyToDirective(OpenACCDirectiveKind DirectiveKind,
     default:
       return false;
     }
+  case OpenACCClauseKind::If:
+    switch (DirectiveKind) {
+    case OpenACCDirectiveKind::Parallel:
+    case OpenACCDirectiveKind::Serial:
+    case OpenACCDirectiveKind::Kernels:
+    case OpenACCDirectiveKind::Data:
+    case OpenACCDirectiveKind::EnterData:
+    case OpenACCDirectiveKind::ExitData:
+    case OpenACCDirectiveKind::HostData:
+    case OpenACCDirectiveKind::Init:
+    case OpenACCDirectiveKind::Shutdown:
+    case OpenACCDirectiveKind::Set:
+    case OpenACCDirectiveKind::Update:
+    case OpenACCDirectiveKind::Wait:
+    case OpenACCDirectiveKind::ParallelLoop:
+    case OpenACCDirectiveKind::SerialLoop:
+    case OpenACCDirectiveKind::KernelsLoop:
+      return true;
+    default:
+      return false;
+    }
   default:
     // Do nothing so we can go to the 'unimplemented' diagnostic instead.
     return true;
   }
   llvm_unreachable("Invalid clause kind");
 }
+
+bool checkAlreadyHasClauseOfKind(
+    SemaOpenACC &S, ArrayRef<const OpenACCClause *> ExistingClauses,
+    SemaOpenACC::OpenACCParsedClause &Clause) {
+  const auto *Itr = llvm::find_if(ExistingClauses, [&](const OpenACCClause *C) {
+    return C->getClauseKind() == Clause.getClauseKind();
+  });
+  if (Itr != ExistingClauses.end()) {
+    S.Diag(Clause.getBeginLoc(), diag::err_acc_duplicate_clause_disallowed)
+        << Clause.getDirectiveKind() << Clause.getClauseKind();
+    S.Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here);
+    return true;
+  }
+  return false;
+}
+
 } // namespace
 
 SemaOpenACC::SemaOpenACC(Sema &S) : SemaBase(S) {}
@@ -97,22 +134,38 @@ SemaOpenACC::ActOnClause(ArrayRef<const OpenACCClause *> ExistingClauses,
     // At most one 'default' clause may appear, and it must have a value of
     // either 'none' or 'present'.
     // Second half of the sentence is diagnosed during parsing.
-    auto Itr = llvm::find_if(ExistingClauses, [](const OpenACCClause *C) {
-      return C->getClauseKind() == OpenACCClauseKind::Default;
-    });
-
-    if (Itr != ExistingClauses.end()) {
-      Diag(Clause.getBeginLoc(),
-                   diag::err_acc_duplicate_clause_disallowed)
-          << Clause.getDirectiveKind() << Clause.getClauseKind();
-      Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here);
+    if (checkAlreadyHasClauseOfKind(*this, ExistingClauses, Clause))
       return nullptr;
-    }
 
     return OpenACCDefaultClause::Create(
         getASTContext(), Clause.getDefaultClauseKind(), Clause.getBeginLoc(),
         Clause.getLParenLoc(), Clause.getEndLoc());
   }
+
+  case OpenACCClauseKind::If: {
+    // Restrictions only properly implemented on 'compute' constructs, and
+    // 'compute' constructs are the only construct that can do anything with
+    // this yet, so skip/treat as unimplemented in this case.
+    if (Clause.getDirectiveKind() != OpenACCDirectiveKind::Parallel &&
+        Clause.getDirectiveKind() != OpenACCDirectiveKind::Serial &&
+        Clause.getDirectiveKind() != OpenACCDirectiveKind::Kernels)
+      break;
+
+    // There is no prose in the standard that says duplicates aren't allowed,
+    // but this diagnostic is present in other compilers, as well as makes
+    // sense.
+    if (checkAlreadyHasClauseOfKind(*this, ExistingClauses, Clause))
+      return nullptr;
+
+    // The parser has ensured that we have a proper condition expr, so there
+    // isn't really much to do here.
+
+    // TODO OpenACC: When we implement 'self', this clauses causes us to
+    // 'ignore' the self clause, so we should implement a warning here.
+    return OpenACCIfClause::Create(
+        getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(),
+        Clause.getConditionExpr(), Clause.getEndLoc());
+  }
   default:
     break;
   }
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 284c9173e68e..ee6bd228b459 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -11103,6 +11103,20 @@ OpenACCClause *TreeTransform<Derived>::TransformOpenACCClause(
     ParsedClause.setDefaultDetails(
         cast<OpenACCDefaultClause>(OldClause)->getDefaultClauseKind());
     break;
+  case OpenACCClauseKind::If: {
+    Expr *Cond = const_cast<Expr *>(
+        cast<OpenACCIfClause>(OldClause)->getConditionExpr());
+    assert(Cond && "If constructed with invalid Condition");
+    Sema::ConditionResult Res =
+        TransformCondition(Cond->getExprLoc(), /*Var=*/nullptr, Cond,
+                           Sema::ConditionKind::Boolean);
+
+    if (Res.isInvalid() || !Res.get().second)
+      return nullptr;
+
+    ParsedClause.setConditionDetails(Res.get().second);
+    break;
+  }
   default:
     assert(false && "Unhandled OpenACC clause in TreeTransform");
     return nullptr;
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index ce96ce2bdbcc..a47c7a393d63 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -11787,6 +11787,12 @@ OpenACCClause *ASTRecordReader::readOpenACCClause() {
     return OpenACCDefaultClause::Create(getContext(), DCK, BeginLoc, LParenLoc,
                                         EndLoc);
   }
+  case OpenACCClauseKind::If: {
+    SourceLocation LParenLoc = readSourceLocation();
+    Expr *CondExpr = readSubExpr();
+    return OpenACCIfClause::Create(getContext(), BeginLoc, LParenLoc, CondExpr,
+                                   EndLoc);
+  }
   case OpenACCClauseKind::Finalize:
   case OpenACCClauseKind::IfPresent:
   case OpenACCClauseKind::Seq:
@@ -11795,7 +11801,6 @@ OpenACCClause *ASTRecordReader::readOpenACCClause() {
   case OpenACCClauseKind::Worker:
   case OpenACCClauseKind::Vector:
   case OpenACCClauseKind::NoHost:
-  case OpenACCClauseKind::If:
   case OpenACCClauseKind::Self:
   case OpenACCClauseKind::Copy:
   case OpenACCClauseKind::UseDevice:
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index d9ba10ab6087..d45695784354 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -7517,6 +7517,12 @@ void ASTRecordWriter::writeOpenACCClause(const OpenACCClause *C) {
     writeEnum(DC->getDefaultClauseKind());
     return;
   }
+  case OpenACCClauseKind::If: {
+    const auto *IC = cast<OpenACCIfClause>(C);
+    writeSourceLocation(IC->getLParenLoc());
+    AddStmt(const_cast<Expr *>(IC->getConditionExpr()));
+    return;
+  }
   case OpenACCClauseKind::Finalize:
   case OpenACCClauseKind::IfPresent:
   case OpenACCClauseKind::Seq:
@@ -7525,7 +7531,6 @@ void ASTRecordWriter::writeOpenACCClause(const OpenACCClause *C) {
   case OpenACCClauseKind::Worker:
   case OpenACCClauseKind::Vector:
   case OpenACCClauseKind::NoHost:
-  case OpenACCClauseKind::If:
   case OpenACCClauseKind::Self:
   case OpenACCClauseKind::Copy:
   case OpenACCClauseKind::UseDevice:
diff --git a/clang/test/ParserOpenACC/parse-clauses.c b/clang/test/ParserOpenACC/parse-clauses.c
index b363a0cb1362..2369df58308a 100644
--- a/clang/test/ParserOpenACC/parse-clauses.c
+++ b/clang/test/ParserOpenACC/parse-clauses.c
@@ -283,11 +283,9 @@ void IfClause() {
 
   int i, j;
 
-  // expected-warning@+1{{OpenACC clause 'if' not yet implemented, clause ignored}}
 #pragma acc serial if(i > j)
   for(;;){}
 
-  // expected-warning@+2{{OpenACC clause 'if' not yet implemented, clause ignored}}
   // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}}
 #pragma acc serial if(1+5>3), seq
   for(;;){}
diff --git a/clang/test/SemaOpenACC/compute-construct-clause-ast.cpp b/clang/test/SemaOpenACC/compute-construct-clause-ast.cpp
index bd8010344502..018f0b68c781 100644
--- a/clang/test/SemaOpenACC/compute-construct-clause-ast.cpp
+++ b/clang/test/SemaOpenACC/compute-construct-clause-ast.cpp
@@ -6,8 +6,10 @@
 
 #ifndef PCH_HELPER
 #define PCH_HELPER
-void NormalFunc() {
+void NormalFunc(int i, float f) {
   // CHECK: FunctionDecl{{.*}}NormalFunc
+  // CHECK-NEXT: ParmVarDecl
+  // CHECK-NEXT: ParmVarDecl
   // CHECK-NEXT: CompoundStmt
 #pragma acc parallel default(none)
   while(true);
@@ -24,6 +26,20 @@ void NormalFunc() {
   // CHECK-NEXT: WhileStmt
   // CHECK-NEXT: CXXBoolLiteralExpr
   // CHECK-NEXT: NullStmt
+
+#pragma acc kernels if( i < f)
+  while(true);
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}kernels
+  // CHECK-NEXT: if clause
+  // CHECK-NEXT: BinaryOperator{{.*}} 'bool' '<'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'float' <IntegralToFloating>
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'int' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'int' lvalue ParmVar{{.*}} 'i' 'int'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'float' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'float' lvalue ParmVar{{.*}} 'f' 'float'
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: NullStmt
 }
 
 template<typename T>
@@ -51,24 +67,120 @@ void TemplFunc() {
   // CHECK-NEXT: CXXBoolLiteralExpr
   // CHECK-NEXT: NullStmt
 
+#pragma acc parallel if(T::SomeFloat < typename T::IntTy{})
+  while(true);
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: if clause
+  // CHECK-NEXT: BinaryOperator{{.*}} '<dependent type>' '<'
+  // CHECK-NEXT: DependentScopeDeclRefExpr{{.*}} '<dependent type>' lvalue
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'T'
+  // CHECK-NEXT: CXXUnresolvedConstructExpr{{.*}} 'typename T::IntTy' 'typename T::IntTy'
+  // CHECK-NEXT: InitListExpr{{.*}} 'void'
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: NullStmt
+
+#pragma acc serial if(typename T::IntTy{})
+  while(true);
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}serial
+  // CHECK-NEXT: if clause
+  // CHECK-NEXT: CXXUnresolvedConstructExpr{{.*}} 'typename T::IntTy' 'typename T::IntTy'
+  // CHECK-NEXT: InitListExpr{{.*}} 'void'
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: NullStmt
+
+#pragma acc kernels if(T::SomeFloat)
+  while(true);
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}kernels
+  // CHECK-NEXT: if clause
+  // CHECK-NEXT: DependentScopeDeclRefExpr{{.*}} '<dependent type>' lvalue
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'T'
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: NullStmt
+
+#pragma acc parallel if(T::BC)
+  while(true);
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: if clause
+  // CHECK-NEXT: DependentScopeDeclRefExpr{{.*}} '<dependent type>' lvalue
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'T'
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: NullStmt
+
   // Match the instantiation:
   // CHECK: FunctionDecl{{.*}}TemplFunc{{.*}}implicit_instantiation
-  // CHECK-NEXT: TemplateArgument type 'int'
-  // CHECK-NEXT: BuiltinType
+  // CHECK-NEXT: TemplateArgument type 'InstTy'
+  // CHECK-NEXT: RecordType{{.*}} 'InstTy'
+  // CHECK-NEXT: CXXRecord{{.*}} 'InstTy'
   // CHECK-NEXT: CompoundStmt
+
   // CHECK-NEXT: OpenACCComputeConstruct{{.*}}kernels
   // CHECK-NEXT: default(none)
   // CHECK-NEXT: WhileStmt
   // CHECK-NEXT: CXXBoolLiteralExpr
   // CHECK-NEXT: NullStmt
+
   // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
   // CHECK-NEXT: default(present)
   // CHECK-NEXT: WhileStmt
   // CHECK-NEXT: CXXBoolLiteralExpr
   // CHECK-NEXT: NullStmt
+
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: if clause
+  // CHECK-NEXT: BinaryOperator{{.*}} 'bool' '<'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'float' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'const float' lvalue Var{{.*}} 'SomeFloat' 'const float'
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'InstTy'
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'float' <IntegralToFloating>
+  // CHECK-NEXT: CXXFunctionalCastExpr{{.*}}'typename InstTy::IntTy':'int' functional cast to typename struct InstTy::IntTy <NoOp>
+  // CHECK-NEXT: InitListExpr {{.*}}'typename InstTy::IntTy':'int'
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: NullStmt
+
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}serial
+  // CHECK-NEXT: if clause
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'bool' <IntegralToBoolean>
+  // CHECK-NEXT: CXXFunctionalCastExpr{{.*}}'typename InstTy::IntTy':'int' functional cast to typename struct InstTy::IntTy <NoOp>
+  // CHECK-NEXT: InitListExpr {{.*}}'typename InstTy::IntTy':'int'
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: NullStmt
+
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}kernels
+  // CHECK-NEXT: if clause
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'bool' <FloatingToBoolean>
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'float' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'const float' lvalue Var{{.*}} 'SomeFloat' 'const float'
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'InstTy'
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: NullStmt
+
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: if clause
+  // CHECK-NEXT: ImplicitCastExpr{{.*}} 'bool' <UserDefinedConversion>
+  // CHECK-NEXT: CXXMemberCallExpr{{.*}} 'bool'
+  // CHECK-NEXT: MemberExpr{{.*}} .operator bool
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'const BoolConversion' lvalue Var{{.*}} 'BC' 'const BoolConversion'
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'InstTy'
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: NullStmt
 }
 
+struct BoolConversion{ operator bool() const;};
+struct InstTy {
+  using IntTy = int;
+  static constexpr float SomeFloat = 5.0;
+  static constexpr BoolConversion BC;
+};
+
 void Instantiate() {
-  TemplFunc<int>();
+  TemplFunc<InstTy>();
 }
 #endif
diff --git a/clang/test/SemaOpenACC/compute-construct-if-clause.c b/clang/test/SemaOpenACC/compute-construct-if-clause.c
new file mode 100644
index 000000000000..767b8414b3a6
--- /dev/null
+++ b/clang/test/SemaOpenACC/compute-construct-if-clause.c
@@ -0,0 +1,62 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+void BoolExpr(int *I, float *F) {
+
+  typedef struct {} SomeStruct;
+  int Array[5];
+
+  struct C{};
+  // expected-error@+1{{expected expression}}
+#pragma acc parallel if (struct C f())
+  while(0);
+
+  // expected-error@+1{{unexpected type name 'SomeStruct': expected expression}}
+#pragma acc serial if (SomeStruct)
+  while(0);
+
+  // expected-error@+1{{unexpected type name 'SomeStruct': expected expression}}
+#pragma acc serial if (SomeStruct())
+  while(0);
+
+  SomeStruct S;
+  // expected-error@+1{{statement requires expression of scalar type ('SomeStruct' invalid)}}
+#pragma acc serial if (S)
+  while(0);
+
+  // expected-warning@+1{{address of array 'Array' will always evaluate to 'true'}}
+#pragma acc kernels if (Array)
+  while(0);
+
+  // expected-warning@+4{{incompatible pointer types assigning to 'int *' from 'float *'}}
+  // expected-warning@+3{{using the result of an assignment as a condition without parentheses}}
+  // expected-note@+2{{place parentheses around the assignment to silence this warning}}
+  // expected-note@+1{{use '==' to turn this assignment into an equality comparison}}
+#pragma acc kernels if (I = F)
+  while(0);
+
+#pragma acc parallel if (I)
+  while(0);
+
+#pragma acc serial if (F)
+  while(0);
+
+#pragma acc kernels if (*I < *F)
+  while(0);
+
+  // expected-warning@+2{{OpenACC construct 'data' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause 'if' not yet implemented}}
+#pragma acc data if (*I < *F)
+  while(0);
+  // expected-warning@+2{{OpenACC construct 'parallel loop' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause 'if' not yet implemented}}
+#pragma acc parallel loop if (*I < *F)
+  while(0);
+  // expected-warning@+2{{OpenACC construct 'serial loop' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause 'if' not yet implemented}}
+#pragma acc serial loop if (*I < *F)
+  while(0);
+  // expected-warning@+2{{OpenACC construct 'kernels loop' not yet implemented}}
+  // expected-warning@+1{{OpenACC clause 'if' not yet implemented}}
+#pragma acc kernels loop if (*I < *F)
+  while(0);
+}
diff --git a/clang/test/SemaOpenACC/compute-construct-if-clause.cpp b/clang/test/SemaOpenACC/compute-construct-if-clause.cpp
new file mode 100644
index 000000000000..2a9bb8a3d5c4
--- /dev/null
+++ b/clang/test/SemaOpenACC/compute-construct-if-clause.cpp
@@ -0,0 +1,33 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+struct NoBoolConversion{};
+struct BoolConversion{
+  operator bool();
+};
+
+template <typename T, typename U>
+void BoolExpr() {
+
+  // expected-error@+1{{value of type 'NoBoolConversion' is not contextually convertible to 'bool'}}
+#pragma acc parallel if (NoBoolConversion{})
+  while(0);
+
+  // expected-error@+2{{no member named 'NotValid' in 'NoBoolConversion'}}
+  // expected-note@#INST{{in instantiation of function template specialization}}
+#pragma acc parallel if (T::NotValid)
+  while(0);
+
+#pragma acc parallel if (BoolConversion{})
+  while(0);
+
+  // expected-error@+1{{value of type 'NoBoolConversion' is not contextually convertible to 'bool'}}
+#pragma acc parallel if (T{})
+  while(0);
+
+#pragma acc parallel if (U{})
+  while(0);
+}
+
+void Instantiate() {
+  BoolExpr<NoBoolConversion, BoolConversion>(); // #INST
+}
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index 00ddc4f24500..936e8872513a 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -28,6 +28,7 @@
 #include "clang/AST/Expr.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/Mangle.h"
+#include "clang/AST/OpenACCClause.h"
 #include "clang/AST/OpenMPClause.h"
 #include "clang/AST/OperationKinds.h"
 #include "clang/AST/StmtVisitor.h"
@@ -2115,6 +2116,7 @@ public:
 };
 class EnqueueVisitor : public ConstStmtVisitor<EnqueueVisitor, void>,
                        public ConstAttrVisitor<EnqueueVisitor, void> {
+  friend class OpenACCClauseEnqueue;
   friend class OMPClauseEnqueue;
   VisitorWorkList &WL;
   CXCursor Parent;
@@ -2171,6 +2173,7 @@ public:
   void VisitConceptSpecializationExpr(const ConceptSpecializationExpr *E);
   void VisitRequiresExpr(const RequiresExpr *E);
   void VisitCXXParenListInitExpr(const CXXParenListInitExpr *E);
+  void VisitOpenACCComputeConstruct(const OpenACCComputeConstruct *D);
   void VisitOMPExecutableDirective(const OMPExecutableDirective *D);
   void VisitOMPLoopBasedDirective(const OMPLoopBasedDirective *D);
   void VisitOMPLoopDirective(const OMPLoopDirective *D);
@@ -2269,6 +2272,7 @@ private:
   void AddDecl(const Decl *D, bool isFirst = true);
   void AddTypeLoc(TypeSourceInfo *TI);
   void EnqueueChildren(const Stmt *S);
+  void EnqueueChildren(const OpenACCClause *S);
   void EnqueueChildren(const OMPClause *S);
   void EnqueueChildren(const AnnotateAttr *A);
 };
@@ -2771,6 +2775,38 @@ void EnqueueVisitor::EnqueueChildren(const OMPClause *S) {
   std::reverse(I, E);
 }
 
+namespace {
+class OpenACCClauseEnqueue : public OpenACCClauseVisitor<OpenACCClauseEnqueue> {
+  EnqueueVisitor &Visitor;
+
+public:
+  OpenACCClauseEnqueue(EnqueueVisitor &V) : Visitor(V) {}
+
+#define VISIT_CLAUSE(CLAUSE_NAME)                                              \
+  void VisitOpenACC##CLAUSE_NAME##Clause(const OpenACC##CLAUSE_NAME##Clause &C);
+#include "clang/Basic/OpenACCClauses.def"
+};
+
+void OpenACCClauseEnqueue::VisitOpenACCDefaultClause(
+    const OpenACCDefaultClause &C) {}
+void OpenACCClauseEnqueue::VisitOpenACCIfClause(const OpenACCIfClause &C) {
+  Visitor.AddStmt(C.getConditionExpr());
+}
+} // namespace
+
+void EnqueueVisitor::EnqueueChildren(const OpenACCClause *C) {
+  unsigned size = WL.size();
+  OpenACCClauseEnqueue Visitor(*this);
+  Visitor.Visit(C);
+
+  if (size == WL.size())
+    return;
+  // Now reverse the entries we just added.  This will match the DFS
+  // ordering performed by the worklist.
+  VisitorWorkList::iterator I = WL.begin() + size, E = WL.end();
+  std::reverse(I, E);
+}
+
 void EnqueueVisitor::EnqueueChildren(const AnnotateAttr *A) {
   unsigned size = WL.size();
   for (const Expr *Arg : A->args()) {
@@ -3386,6 +3422,13 @@ void EnqueueVisitor::VisitOMPTargetTeamsDistributeSimdDirective(
   VisitOMPLoopDirective(D);
 }
 
+void EnqueueVisitor::VisitOpenACCComputeConstruct(
+    const OpenACCComputeConstruct *C) {
+  EnqueueChildren(C);
+  for (auto *Clause : C->clauses())
+    EnqueueChildren(Clause);
+}
+
 void EnqueueVisitor::VisitAnnotateAttr(const AnnotateAttr *A) {
   EnqueueChildren(A);
 }
-- 
GitLab


From 7b6b023121fe7d612dc571de0ff3dcaaf8477765 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Fri, 12 Apr 2024 14:28:05 -0700
Subject: [PATCH 116/464] [flang][cuda] Fix crash in semantic (#88577)

Fix for #88451

Do not perform semantic check about data transfer on assignment
statement in device context.
---
 flang/lib/Semantics/check-cuda.cpp |  8 +++++++-
 flang/test/Semantics/cuf11.cuf     | 12 ++++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/flang/lib/Semantics/check-cuda.cpp b/flang/lib/Semantics/check-cuda.cpp
index e0a796972441..2cb15437a235 100644
--- a/flang/lib/Semantics/check-cuda.cpp
+++ b/flang/lib/Semantics/check-cuda.cpp
@@ -480,10 +480,16 @@ void CUDAChecker::Enter(const parser::CUFKernelDoConstruct &x) {
 }
 
 void CUDAChecker::Enter(const parser::AssignmentStmt &x) {
+  auto lhsLoc{std::get<parser::Variable>(x.t).GetSource()};
+  const auto &scope{context_.FindScope(lhsLoc)};
+  const Scope &progUnit{GetProgramUnitContaining(scope)};
+  if (IsCUDADeviceContext(&progUnit)) {
+    return; // Data transfer with assignment is only perform on host.
+  }
+
   const evaluate::Assignment *assign{semantics::GetAssignment(x)};
   int nbLhs{evaluate::GetNbOfCUDASymbols(assign->lhs)};
   int nbRhs{evaluate::GetNbOfCUDASymbols(assign->rhs)};
-  auto lhsLoc{std::get<parser::Variable>(x.t).GetSource()};
 
   // device to host transfer with more than one device object on the rhs is not
   // legal.
diff --git a/flang/test/Semantics/cuf11.cuf b/flang/test/Semantics/cuf11.cuf
index 96108e2b2455..b915c7246ed9 100644
--- a/flang/test/Semantics/cuf11.cuf
+++ b/flang/test/Semantics/cuf11.cuf
@@ -1,5 +1,17 @@
 ! RUN: %python %S/test_errors.py %s %flang_fc1
 
+module mod1
+contains
+  attributes(global) subroutine sub1(adev)
+    real :: adev(10)
+    integer :: tid
+    tid = threadIdx%x
+! Use to crash the compiler. Make sure we have the proper semantic error.
+!ERROR: Actual argument for 'i=' has bad type 'REAL(4)'
+    adev(tid + 1) = scale(real(tid), 2.0) 
+  end subroutine sub1
+end module
+
 subroutine sub1()
   real, device :: adev(10), bdev(10)
   real :: ahost(10)
-- 
GitLab


From 900be9013fdc3bab9fce906f8a71e59ecd8873b4 Mon Sep 17 00:00:00 2001
From: Victor Toni <ViToni@users.noreply.github.com>
Date: Fri, 12 Apr 2024 23:33:35 +0200
Subject: [PATCH 117/464] Fix typos (#88565)

---
 clang/lib/StaticAnalyzer/Core/ExprEngineCXX.cpp         | 2 +-
 libc/src/__support/str_to_float.h                       | 8 ++++----
 llvm/lib/Analysis/LoopCacheAnalysis.cpp                 | 2 +-
 llvm/lib/Support/APInt.cpp                              | 2 +-
 llvm/lib/Transforms/Scalar/LoopInterchange.cpp          | 2 +-
 llvm/test/Transforms/JumpThreading/free_instructions.ll | 2 +-
 llvm/utils/lint/common_lint.py                          | 2 +-
 7 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngineCXX.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngineCXX.cpp
index 504fd7f05e0f..c50db1e0e2f8 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngineCXX.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngineCXX.cpp
@@ -229,7 +229,7 @@ SVal ExprEngine::computeObjectUnderConstruction(
         // We are on the top frame of the analysis. We do not know where is the
         // object returned to. Conjure a symbolic region for the return value.
         // TODO: We probably need a new MemRegion kind to represent the storage
-        // of that SymbolicRegion, so that we cound produce a fancy symbol
+        // of that SymbolicRegion, so that we could produce a fancy symbol
         // instead of an anonymous conjured symbol.
         // TODO: Do we need to track the region to avoid having it dead
         // too early? It does die too early, at least in C++17, but because
diff --git a/libc/src/__support/str_to_float.h b/libc/src/__support/str_to_float.h
index cd0c07629f87..fa466ca8492f 100644
--- a/libc/src/__support/str_to_float.h
+++ b/libc/src/__support/str_to_float.h
@@ -695,10 +695,10 @@ LIBC_INLINE FloatConvertReturn<T> decimal_exp_to_float(
     // If the mantissa is truncated, then the result may be off by the LSB, so
     // check if rounding the mantissa up changes the result. If not, then it's
     // safe, else use the fallback.
-    auto secound_output = eisel_lemire<T>({mantissa + 1, exp10}, round);
-    if (secound_output.has_value()) {
-      if (opt_output->mantissa == secound_output->mantissa &&
-          opt_output->exponent == secound_output->exponent) {
+    auto second_output = eisel_lemire<T>({mantissa + 1, exp10}, round);
+    if (second_output.has_value()) {
+      if (opt_output->mantissa == second_output->mantissa &&
+          opt_output->exponent == second_output->exponent) {
         return {opt_output.value(), 0};
       }
     }
diff --git a/llvm/lib/Analysis/LoopCacheAnalysis.cpp b/llvm/lib/Analysis/LoopCacheAnalysis.cpp
index c3a56639b5c8..284d8d16d264 100644
--- a/llvm/lib/Analysis/LoopCacheAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopCacheAnalysis.cpp
@@ -315,7 +315,7 @@ CacheCostTy IndexedReference::computeRefCost(const Loop &L,
     RefCost = TripCount;
 
     int Index = getSubscriptIndex(L);
-    assert(Index >= 0 && "Cound not locate a valid Index");
+    assert(Index >= 0 && "Could not locate a valid Index");
 
     for (unsigned I = Index + 1; I < getNumSubscripts() - 1; ++I) {
       const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(getSubscript(I));
diff --git a/llvm/lib/Support/APInt.cpp b/llvm/lib/Support/APInt.cpp
index 18feca4c0553..24e136fcb9c7 100644
--- a/llvm/lib/Support/APInt.cpp
+++ b/llvm/lib/Support/APInt.cpp
@@ -2663,7 +2663,7 @@ int APInt::tcDivide(WordType *lhs, const WordType *rhs,
   return false;
 }
 
-/// Shift a bignum left Cound bits in-place. Shifted in bits are zero. There are
+/// Shift a bignum left Count bits in-place. Shifted in bits are zero. There are
 /// no restrictions on Count.
 void APInt::tcShiftLeft(WordType *Dst, unsigned Words, unsigned Count) {
   // Don't bother performing a no-op shift.
diff --git a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
index 277f530ee25f..400973fd9fc9 100644
--- a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
@@ -976,7 +976,7 @@ bool LoopInterchangeLegality::canInterchangeLoops(unsigned InnerLoopId,
       }
 
   if (!findInductions(InnerLoop, InnerLoopInductions)) {
-    LLVM_DEBUG(dbgs() << "Cound not find inner loop induction variables.\n");
+    LLVM_DEBUG(dbgs() << "Could not find inner loop induction variables.\n");
     return false;
   }
 
diff --git a/llvm/test/Transforms/JumpThreading/free_instructions.ll b/llvm/test/Transforms/JumpThreading/free_instructions.ll
index 3485bf8c98da..48bedc72c2d3 100644
--- a/llvm/test/Transforms/JumpThreading/free_instructions.ll
+++ b/llvm/test/Transforms/JumpThreading/free_instructions.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -passes=jump-threading -jump-threading-threshold=3 < %s | FileCheck %s
 
-; TODO: In this test case, only the stores in %join should cound towards
+; TODO: In this test case, only the stores in %join should count towards
 ; the jump threading threshold, as everything else are free instructions.
 define i32 @free_instructions(i1 %c, ptr %p) {
 ; CHECK-LABEL: @free_instructions(
diff --git a/llvm/utils/lint/common_lint.py b/llvm/utils/lint/common_lint.py
index 1bf1695659d8..07b8fc5899e5 100644
--- a/llvm/utils/lint/common_lint.py
+++ b/llvm/utils/lint/common_lint.py
@@ -97,7 +97,7 @@ def RunLintOverAllFiles(linter, filenames):
     for filename in filenames:
         file = open(filename, "r")
         if not file:
-            print("Cound not open %s" % filename)
+            print("Could not open %s" % filename)
             continue
         lines = file.readlines()
         lint.extend(linter.RunOnFile(filename, lines))
-- 
GitLab


From 2996cb7970fb536d7db1088d571dc8d2f9ad35fa Mon Sep 17 00:00:00 2001
From: Michael Flanders <flanders.michaelk@gmail.com>
Date: Fri, 12 Apr 2024 14:35:10 -0700
Subject: [PATCH 118/464] [libc][docs] use docgen to generate stdbit rst docs
 (#88540)

Closes #88066.

Compared to before, the function names in the stdbit table are sorted by
function name, not order-of-appearance in the standard. Since macros
aren't printed by docgen.py and are still a TODO in the code, they are
also not printed in the new stdbit.h docs.

Adds some checks to docgen.py for conditions that tripped me up.

Add code to docgen.py to add the include of the `|check|` rewriter,
since all other generated files need it.
---
 libc/docs/stdbit.rst          | 350 +++++++++++++++++++++-------------
 libc/utils/docgen/docgen.py   |   8 +
 libc/utils/docgen/stdbit.json | 234 +++++++++++++++++++++++
 3 files changed, 461 insertions(+), 131 deletions(-)
 create mode 100644 libc/utils/docgen/stdbit.json

diff --git a/libc/docs/stdbit.rst b/libc/docs/stdbit.rst
index d42f79382462..0a12b2b6d7b5 100644
--- a/libc/docs/stdbit.rst
+++ b/libc/docs/stdbit.rst
@@ -1,135 +1,223 @@
-=========================
-Bitwise Utility Functions
-=========================
-
 .. include:: check.rst
 
----------------
-Source Location
----------------
-
--   The main source for bitwise utility functions is located at:
-    ``libc/src/stdbit``.
-
--   The source for internal helpers used to implement these is located at:
-    ``libc/src/__support/CPP/bit.h``.
-
--   The tests are located at:
-    ``libc/test/src/stdbit/``, ``libc/test/include/stdbit_test.cpp``, and
-    ``src/__support/CPP/bit_test.cpp``.
-
----------------------
-Implementation Status
----------------------
-
-Functions
-=========
-
-..
-  Do not order these, they are as they appear in the standard.
-
-============================ =========
-Function Name                Available
-============================ =========
-stdc_leading_zeros_uc        |check|
-stdc_leading_zeros_us        |check|
-stdc_leading_zeros_ui        |check|
-stdc_leading_zeros_ul        |check|
-stdc_leading_zeros_ull       |check|
-stdc_leading_ones_uc         |check|
-stdc_leading_ones_us         |check|
-stdc_leading_ones_ui         |check|
-stdc_leading_ones_ul         |check|
-stdc_leading_ones_ull        |check|
-stdc_trailing_zeros_uc       |check|
-stdc_trailing_zeros_us       |check|
-stdc_trailing_zeros_ui       |check|
-stdc_trailing_zeros_ul       |check|
-stdc_trailing_zeros_ull      |check|
-stdc_trailing_ones_uc        |check|
-stdc_trailing_ones_us        |check|
-stdc_trailing_ones_ui        |check|
-stdc_trailing_ones_ul        |check|
-stdc_trailing_ones_ull       |check|
-stdc_first_leading_zero_uc   |check|
-stdc_first_leading_zero_us   |check|
-stdc_first_leading_zero_ui   |check|
-stdc_first_leading_zero_ul   |check|
-stdc_first_leading_zero_ull  |check|
-stdc_first_leading_one_uc    |check|
-stdc_first_leading_one_us    |check|
-stdc_first_leading_one_ui    |check|
-stdc_first_leading_one_ul    |check|
-stdc_first_leading_one_ull   |check|
-stdc_first_trailing_zero_uc  |check|
-stdc_first_trailing_zero_us  |check|
-stdc_first_trailing_zero_ui  |check|
-stdc_first_trailing_zero_ul  |check|
-stdc_first_trailing_zero_ull |check|
-stdc_first_trailing_one_uc   |check|
-stdc_first_trailing_one_us   |check|
-stdc_first_trailing_one_ui   |check|
-stdc_first_trailing_one_ul   |check|
-stdc_first_trailing_one_ull  |check|
-stdc_count_zeros_uc          |check|
-stdc_count_zeros_us          |check|
-stdc_count_zeros_ui          |check|
-stdc_count_zeros_ul          |check|
-stdc_count_zeros_ull         |check|
-stdc_count_ones_uc           |check|
-stdc_count_ones_us           |check|
-stdc_count_ones_ui           |check|
-stdc_count_ones_ul           |check|
-stdc_count_ones_ull          |check|
-stdc_has_single_bit_uc       |check|
-stdc_has_single_bit_us       |check|
-stdc_has_single_bit_ui       |check|
-stdc_has_single_bit_ul       |check|
-stdc_has_single_bit_ull      |check|
-stdc_bit_width_uc            |check|
-stdc_bit_width_us            |check|
-stdc_bit_width_ui            |check|
-stdc_bit_width_ul            |check|
-stdc_bit_width_ull           |check|
-stdc_bit_floor_uc            |check|
-stdc_bit_floor_us            |check|
-stdc_bit_floor_ui            |check|
-stdc_bit_floor_ul            |check|
-stdc_bit_floor_ull           |check|
-stdc_bit_ceil_uc             |check|
-stdc_bit_ceil_us             |check|
-stdc_bit_ceil_ui             |check|
-stdc_bit_ceil_ul             |check|
-stdc_bit_ceil_ull            |check|
-============================ =========
-
-
-Macros
-======
+stdbit.h Functions
+==================
 
-=========================  =========
-Macro Name                 Available
-=========================  =========
-__STDC_VERSION_STDBIT_H__  |check|
-__STDC_ENDIAN_LITTLE__     |check|
-__STDC_ENDIAN_BIG__        |check|
-__STDC_ENDIAN_NATIVE__     |check|
-stdc_leading_zeros         |check|
-stdc_leading_ones          |check|
-stdc_trailing_zeros        |check|
-stdc_trailing_ones         |check|
-stdc_first_leading_zero    |check|
-stdc_first_leading_one     |check|
-stdc_first_trailing_zero   |check|
-stdc_first_trailing_one    |check|
-stdc_count_zeros           |check|
-stdc_count_ones            |check|
-stdc_has_single_bit        |check|
-stdc_bit_width             |check|
-stdc_bit_floor             |check|
-stdc_bit_ceil              |check|
-=========================  =========
+.. list-table::
+  :widths: auto
+  :align: center
+  :header-rows: 1
 
-Standards
-=========
-stdbit.h was specified as part of C23 in section 7.18 "Bit and byte utilities."
+  * - Function
+    - Implemented
+    - Standard
+  * - stdc_bit_ceil_uc
+    - |check|
+    - 7.18.16
+  * - stdc_bit_ceil_ui
+    - |check|
+    - 7.18.16
+  * - stdc_bit_ceil_ul
+    - |check|
+    - 7.18.16
+  * - stdc_bit_ceil_ull
+    - |check|
+    - 7.18.16
+  * - stdc_bit_ceil_us
+    - |check|
+    - 7.18.16
+  * - stdc_bit_floor_uc
+    - |check|
+    - 7.18.15
+  * - stdc_bit_floor_ui
+    - |check|
+    - 7.18.15
+  * - stdc_bit_floor_ul
+    - |check|
+    - 7.18.15
+  * - stdc_bit_floor_ull
+    - |check|
+    - 7.18.15
+  * - stdc_bit_floor_us
+    - |check|
+    - 7.18.15
+  * - stdc_bit_width_uc
+    - |check|
+    - 7.18.14
+  * - stdc_bit_width_ui
+    - |check|
+    - 7.18.14
+  * - stdc_bit_width_ul
+    - |check|
+    - 7.18.14
+  * - stdc_bit_width_ull
+    - |check|
+    - 7.18.14
+  * - stdc_bit_width_us
+    - |check|
+    - 7.18.14
+  * - stdc_count_ones_uc
+    - |check|
+    - 7.18.12
+  * - stdc_count_ones_ui
+    - |check|
+    - 7.18.12
+  * - stdc_count_ones_ul
+    - |check|
+    - 7.18.12
+  * - stdc_count_ones_ull
+    - |check|
+    - 7.18.12
+  * - stdc_count_ones_us
+    - |check|
+    - 7.18.12
+  * - stdc_count_zeros_uc
+    - |check|
+    - 7.18.11
+  * - stdc_count_zeros_ui
+    - |check|
+    - 7.18.11
+  * - stdc_count_zeros_ul
+    - |check|
+    - 7.18.11
+  * - stdc_count_zeros_ull
+    - |check|
+    - 7.18.11
+  * - stdc_count_zeros_us
+    - |check|
+    - 7.18.11
+  * - stdc_first_leading_one_uc
+    - |check|
+    - 7.18.8
+  * - stdc_first_leading_one_ui
+    - |check|
+    - 7.18.8
+  * - stdc_first_leading_one_ul
+    - |check|
+    - 7.18.8
+  * - stdc_first_leading_one_ull
+    - |check|
+    - 7.18.8
+  * - stdc_first_leading_one_us
+    - |check|
+    - 7.18.8
+  * - stdc_first_leading_zero_uc
+    - |check|
+    - 7.18.7
+  * - stdc_first_leading_zero_ui
+    - |check|
+    - 7.18.7
+  * - stdc_first_leading_zero_ul
+    - |check|
+    - 7.18.7
+  * - stdc_first_leading_zero_ull
+    - |check|
+    - 7.18.7
+  * - stdc_first_leading_zero_us
+    - |check|
+    - 7.18.7
+  * - stdc_first_trailing_one_uc
+    - |check|
+    - 7.18.10
+  * - stdc_first_trailing_one_ui
+    - |check|
+    - 7.18.10
+  * - stdc_first_trailing_one_ul
+    - |check|
+    - 7.18.10
+  * - stdc_first_trailing_one_ull
+    - |check|
+    - 7.18.10
+  * - stdc_first_trailing_one_us
+    - |check|
+    - 7.18.10
+  * - stdc_first_trailing_zero_uc
+    - |check|
+    - 7.18.9
+  * - stdc_first_trailing_zero_ui
+    - |check|
+    - 7.18.9
+  * - stdc_first_trailing_zero_ul
+    - |check|
+    - 7.18.9
+  * - stdc_first_trailing_zero_ull
+    - |check|
+    - 7.18.9
+  * - stdc_first_trailing_zero_us
+    - |check|
+    - 7.18.9
+  * - stdc_has_single_bit_uc
+    - |check|
+    - 7.18.13
+  * - stdc_has_single_bit_ui
+    - |check|
+    - 7.18.13
+  * - stdc_has_single_bit_ul
+    - |check|
+    - 7.18.13
+  * - stdc_has_single_bit_ull
+    - |check|
+    - 7.18.13
+  * - stdc_has_single_bit_us
+    - |check|
+    - 7.18.13
+  * - stdc_leading_ones_uc
+    - |check|
+    - 7.18.4
+  * - stdc_leading_ones_ui
+    - |check|
+    - 7.18.4
+  * - stdc_leading_ones_ul
+    - |check|
+    - 7.18.4
+  * - stdc_leading_ones_ull
+    - |check|
+    - 7.18.4
+  * - stdc_leading_ones_us
+    - |check|
+    - 7.18.4
+  * - stdc_leading_zeros_uc
+    - |check|
+    - 7.18.3
+  * - stdc_leading_zeros_ui
+    - |check|
+    - 7.18.3
+  * - stdc_leading_zeros_ul
+    - |check|
+    - 7.18.3
+  * - stdc_leading_zeros_ull
+    - |check|
+    - 7.18.3
+  * - stdc_leading_zeros_us
+    - |check|
+    - 7.18.3
+  * - stdc_trailing_ones_uc
+    - |check|
+    - 7.18.6
+  * - stdc_trailing_ones_ui
+    - |check|
+    - 7.18.6
+  * - stdc_trailing_ones_ul
+    - |check|
+    - 7.18.6
+  * - stdc_trailing_ones_ull
+    - |check|
+    - 7.18.6
+  * - stdc_trailing_ones_us
+    - |check|
+    - 7.18.6
+  * - stdc_trailing_zeros_uc
+    - |check|
+    - 7.18.5
+  * - stdc_trailing_zeros_ui
+    - |check|
+    - 7.18.5
+  * - stdc_trailing_zeros_ul
+    - |check|
+    - 7.18.5
+  * - stdc_trailing_zeros_ull
+    - |check|
+    - 7.18.5
+  * - stdc_trailing_zeros_us
+    - |check|
+    - 7.18.5
diff --git a/libc/utils/docgen/docgen.py b/libc/utils/docgen/docgen.py
index 36eb409421b4..23d45305fe51 100755
--- a/libc/utils/docgen/docgen.py
+++ b/libc/utils/docgen/docgen.py
@@ -28,6 +28,13 @@ def is_implemented(hname: str, fname: str) -> bool:
         "src",
         hname.rstrip(".h")
     )
+
+    if not path.exists():
+        raise FileNotFoundError(f"implementation dir does not exist: {path}")
+
+    if not path.is_dir():
+        raise NotADirectoryError(f"implementation dir is not a dir: {path}")
+
     # Recursively search for the target source file in the subdirectories under
     # libc/src/{hname}.
     for _ in path.glob("**/" + fname + ".cpp"):
@@ -53,6 +60,7 @@ def print_functions(header: str, functions: Dict):
 
 
 def print_header(header: str, api: Dict):
+    print(".. include:: check.rst\n")
     fns = f"{header} Functions"
     print(fns)
     print("=" * (len(fns)))
diff --git a/libc/utils/docgen/stdbit.json b/libc/utils/docgen/stdbit.json
new file mode 100644
index 000000000000..9dda0cb0f538
--- /dev/null
+++ b/libc/utils/docgen/stdbit.json
@@ -0,0 +1,234 @@
+{
+  "macros": [
+    "__STDC_VERSION_STDBIT_H__",
+    "__STDC_ENDIAN_LITTLE__",
+    "__STDC_ENDIAN_BIG__",
+    "__STDC_ENDIAN_NATIVE__",
+    "stdc_leading_zeros",
+    "stdc_leading_ones",
+    "stdc_trailing_zeros",
+    "stdc_trailing_ones",
+    "stdc_first_leading_zero",
+    "stdc_first_leading_one",
+    "stdc_first_trailing_zero",
+    "stdc_first_trailing_one",
+    "stdc_count_zeros",
+    "stdc_count_ones",
+    "stdc_has_single_bit",
+    "stdc_bit_width",
+    "stdc_bit_floor",
+    "stdc_bit_ceil"
+  ],
+  "functions": {
+    "stdc_leading_zeros_uc": {
+      "defined": "7.18.3"
+    },
+    "stdc_leading_zeros_us": {
+      "defined": "7.18.3"
+    },
+    "stdc_leading_zeros_ui": {
+      "defined": "7.18.3"
+    },
+    "stdc_leading_zeros_ul": {
+      "defined": "7.18.3"
+    },
+    "stdc_leading_zeros_ull": {
+      "defined": "7.18.3"
+    },
+    "stdc_leading_ones_uc": {
+      "defined": "7.18.4"
+    },
+    "stdc_leading_ones_us": {
+      "defined": "7.18.4"
+    },
+    "stdc_leading_ones_ui": {
+      "defined": "7.18.4"
+    },
+    "stdc_leading_ones_ul": {
+      "defined": "7.18.4"
+    },
+    "stdc_leading_ones_ull": {
+      "defined": "7.18.4"
+    },
+    "stdc_trailing_zeros_uc": {
+      "defined": "7.18.5"
+    },
+    "stdc_trailing_zeros_us": {
+      "defined": "7.18.5"
+    },
+    "stdc_trailing_zeros_ui": {
+      "defined": "7.18.5"
+    },
+    "stdc_trailing_zeros_ul": {
+      "defined": "7.18.5"
+    },
+    "stdc_trailing_zeros_ull": {
+      "defined": "7.18.5"
+    },
+    "stdc_trailing_ones_uc": {
+      "defined": "7.18.6"
+    },
+    "stdc_trailing_ones_us": {
+      "defined": "7.18.6"
+    },
+    "stdc_trailing_ones_ui": {
+      "defined": "7.18.6"
+    },
+    "stdc_trailing_ones_ul": {
+      "defined": "7.18.6"
+    },
+    "stdc_trailing_ones_ull": {
+      "defined": "7.18.6"
+    },
+    "stdc_first_leading_zero_uc": {
+      "defined": "7.18.7"
+    },
+    "stdc_first_leading_zero_us": {
+      "defined": "7.18.7"
+    },
+    "stdc_first_leading_zero_ui": {
+      "defined": "7.18.7"
+    },
+    "stdc_first_leading_zero_ul": {
+      "defined": "7.18.7"
+    },
+    "stdc_first_leading_zero_ull": {
+      "defined": "7.18.7"
+    },
+    "stdc_first_leading_one_uc": {
+      "defined": "7.18.8"
+    },
+    "stdc_first_leading_one_us": {
+      "defined": "7.18.8"
+    },
+    "stdc_first_leading_one_ui": {
+      "defined": "7.18.8"
+    },
+    "stdc_first_leading_one_ul": {
+      "defined": "7.18.8"
+    },
+    "stdc_first_leading_one_ull": {
+      "defined": "7.18.8"
+    },
+    "stdc_first_trailing_zero_uc": {
+      "defined": "7.18.9"
+    },
+    "stdc_first_trailing_zero_us": {
+      "defined": "7.18.9"
+    },
+    "stdc_first_trailing_zero_ui": {
+      "defined": "7.18.9"
+    },
+    "stdc_first_trailing_zero_ul": {
+      "defined": "7.18.9"
+    },
+    "stdc_first_trailing_zero_ull": {
+      "defined": "7.18.9"
+    },
+    "stdc_first_trailing_one_uc": {
+      "defined": "7.18.10"
+    },
+    "stdc_first_trailing_one_us": {
+      "defined": "7.18.10"
+    },
+    "stdc_first_trailing_one_ui": {
+      "defined": "7.18.10"
+    },
+    "stdc_first_trailing_one_ul": {
+      "defined": "7.18.10"
+    },
+    "stdc_first_trailing_one_ull": {
+      "defined": "7.18.10"
+    },
+    "stdc_count_zeros_uc": {
+      "defined": "7.18.11"
+    },
+    "stdc_count_zeros_us": {
+      "defined": "7.18.11"
+    },
+    "stdc_count_zeros_ui": {
+      "defined": "7.18.11"
+    },
+    "stdc_count_zeros_ul": {
+      "defined": "7.18.11"
+    },
+    "stdc_count_zeros_ull": {
+      "defined": "7.18.11"
+    },
+    "stdc_count_ones_uc": {
+      "defined": "7.18.12"
+    },
+    "stdc_count_ones_us": {
+      "defined": "7.18.12"
+    },
+    "stdc_count_ones_ui": {
+      "defined": "7.18.12"
+    },
+    "stdc_count_ones_ul": {
+      "defined": "7.18.12"
+    },
+    "stdc_count_ones_ull": {
+      "defined": "7.18.12"
+    },
+    "stdc_has_single_bit_uc": {
+      "defined": "7.18.13"
+    },
+    "stdc_has_single_bit_us": {
+      "defined": "7.18.13"
+    },
+    "stdc_has_single_bit_ui": {
+      "defined": "7.18.13"
+    },
+    "stdc_has_single_bit_ul": {
+      "defined": "7.18.13"
+    },
+    "stdc_has_single_bit_ull": {
+      "defined": "7.18.13"
+    },
+    "stdc_bit_width_uc": {
+      "defined": "7.18.14"
+    },
+    "stdc_bit_width_us": {
+      "defined": "7.18.14"
+    },
+    "stdc_bit_width_ui": {
+      "defined": "7.18.14"
+    },
+    "stdc_bit_width_ul": {
+      "defined": "7.18.14"
+    },
+    "stdc_bit_width_ull": {
+      "defined": "7.18.14"
+    },
+    "stdc_bit_floor_uc": {
+      "defined": "7.18.15"
+    },
+    "stdc_bit_floor_us": {
+      "defined": "7.18.15"
+    },
+    "stdc_bit_floor_ui": {
+      "defined": "7.18.15"
+    },
+    "stdc_bit_floor_ul": {
+      "defined": "7.18.15"
+    },
+    "stdc_bit_floor_ull": {
+      "defined": "7.18.15"
+    },
+    "stdc_bit_ceil_uc": {
+      "defined": "7.18.16"
+    },
+    "stdc_bit_ceil_us": {
+      "defined": "7.18.16"
+    },
+    "stdc_bit_ceil_ui": {
+      "defined": "7.18.16"
+    },
+    "stdc_bit_ceil_ul": {
+      "defined": "7.18.16"
+    },
+    "stdc_bit_ceil_ull": {
+      "defined": "7.18.16"
+    }
+  }
+}
-- 
GitLab


From eaa3947106e50d6d837e6681fb55cf92c2bb0f9a Mon Sep 17 00:00:00 2001
From: erichkeane <ekeane@nvidia.com>
Date: Fri, 12 Apr 2024 14:32:37 -0700
Subject: [PATCH 119/464] [NFC] Remove unused type alias in OpenACC

Leftover from a previous commit, this ends up not being used, so remove
it.
---
 clang/include/clang/Parse/Parser.h | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index db90d38c7dac..c719218731c3 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -3611,9 +3611,6 @@ private:
   OpenACCClauseParseResult OpenACCCannotContinue();
   OpenACCClauseParseResult OpenACCSuccess(OpenACCClause *Clause);
 
-  using OpenACCConditionExprParseResult =
-      std::pair<ExprResult, OpenACCParseCanContinue>;
-
   /// Parses the OpenACC directive (the entire pragma) including the clause
   /// list, but does not produce the main AST node.
   OpenACCDirectiveParseInfo ParseOpenACCDirective();
-- 
GitLab


From a472e6473e6921f22427d41c3d29e9a26eb4ab32 Mon Sep 17 00:00:00 2001
From: erichkeane <ekeane@nvidia.com>
Date: Fri, 12 Apr 2024 14:41:20 -0700
Subject: [PATCH 120/464] [NFC][OpenACC] Rename visitor functions to omit
 redundant OpenACC

While working on a followup patch, it became clear that this extra bit
of 'OpenACC' before each clause name was redundant with the visitors, so
remove it to make this a little less verbose.
---
 clang/include/clang/AST/OpenACCClause.h | 11 +++++------
 clang/lib/AST/OpenACCClause.cpp         |  4 ++--
 clang/lib/AST/StmtProfile.cpp           |  6 +++---
 clang/tools/libclang/CIndex.cpp         |  5 ++---
 4 files changed, 12 insertions(+), 14 deletions(-)

diff --git a/clang/include/clang/AST/OpenACCClause.h b/clang/include/clang/AST/OpenACCClause.h
index 712ad46c0f23..401b8e904a1b 100644
--- a/clang/include/clang/AST/OpenACCClause.h
+++ b/clang/include/clang/AST/OpenACCClause.h
@@ -160,10 +160,10 @@ public:
 
     switch (C->getClauseKind()) {
     case OpenACCClauseKind::Default:
-      VisitOpenACCDefaultClause(*cast<OpenACCDefaultClause>(C));
+      VisitDefaultClause(*cast<OpenACCDefaultClause>(C));
       return;
     case OpenACCClauseKind::If:
-      VisitOpenACCIfClause(*cast<OpenACCIfClause>(C));
+      VisitIfClause(*cast<OpenACCIfClause>(C));
       return;
     case OpenACCClauseKind::Finalize:
     case OpenACCClauseKind::IfPresent:
@@ -212,9 +212,9 @@ public:
   }
 
 #define VISIT_CLAUSE(CLAUSE_NAME)                                              \
-  void VisitOpenACC##CLAUSE_NAME##Clause(                                      \
+  void Visit##CLAUSE_NAME##Clause(                                             \
       const OpenACC##CLAUSE_NAME##Clause &Clause) {                            \
-    return getDerived().VisitOpenACC##CLAUSE_NAME##Clause(Clause);             \
+    return getDerived().Visit##CLAUSE_NAME##Clause(Clause);                    \
   }
 
 #include "clang/Basic/OpenACCClauses.def"
@@ -236,8 +236,7 @@ public:
   OpenACCClausePrinter(raw_ostream &OS) : OS(OS) {}
 
 #define VISIT_CLAUSE(CLAUSE_NAME)                                              \
-  void VisitOpenACC##CLAUSE_NAME##Clause(                                      \
-      const OpenACC##CLAUSE_NAME##Clause &Clause);
+  void Visit##CLAUSE_NAME##Clause(const OpenACC##CLAUSE_NAME##Clause &Clause);
 #include "clang/Basic/OpenACCClauses.def"
 };
 
diff --git a/clang/lib/AST/OpenACCClause.cpp b/clang/lib/AST/OpenACCClause.cpp
index 2bc8d14fa129..446e47003204 100644
--- a/clang/lib/AST/OpenACCClause.cpp
+++ b/clang/lib/AST/OpenACCClause.cpp
@@ -65,11 +65,11 @@ OpenACCClause::child_range OpenACCClause::children() {
 //===----------------------------------------------------------------------===//
 //  OpenACC clauses printing methods
 //===----------------------------------------------------------------------===//
-void OpenACCClausePrinter::VisitOpenACCDefaultClause(
+void OpenACCClausePrinter::VisitDefaultClause(
     const OpenACCDefaultClause &C) {
   OS << "default(" << C.getDefaultClauseKind() << ")";
 }
 
-void OpenACCClausePrinter::VisitOpenACCIfClause(const OpenACCIfClause &C) {
+void OpenACCClausePrinter::VisitIfClause(const OpenACCIfClause &C) {
   OS << "if(" << C.getConditionExpr() << ")";
 }
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index 44b635bfb843..68ba7475f4ca 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -2459,17 +2459,17 @@ public:
   }
 
 #define VISIT_CLAUSE(CLAUSE_NAME)                                              \
-  void VisitOpenACC##CLAUSE_NAME##Clause(                                      \
+  void Visit##CLAUSE_NAME##Clause(                                      \
       const OpenACC##CLAUSE_NAME##Clause &Clause);
 
 #include "clang/Basic/OpenACCClauses.def"
 };
 
 /// Nothing to do here, there are no sub-statements.
-void OpenACCClauseProfiler::VisitOpenACCDefaultClause(
+void OpenACCClauseProfiler::VisitDefaultClause(
     const OpenACCDefaultClause &Clause) {}
 
-void OpenACCClauseProfiler::VisitOpenACCIfClause(
+void OpenACCClauseProfiler::VisitIfClause(
     const OpenACCIfClause &Clause) {
   assert(Clause.hasConditionExpr() &&
          "if clause requires a valid condition expr");
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index 936e8872513a..b999750c0f5e 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2787,9 +2787,8 @@ public:
 #include "clang/Basic/OpenACCClauses.def"
 };
 
-void OpenACCClauseEnqueue::VisitOpenACCDefaultClause(
-    const OpenACCDefaultClause &C) {}
-void OpenACCClauseEnqueue::VisitOpenACCIfClause(const OpenACCIfClause &C) {
+void OpenACCClauseEnqueue::VisitDefaultClause(const OpenACCDefaultClause &C) {}
+void OpenACCClauseEnqueue::VisitIfClause(const OpenACCIfClause &C) {
   Visitor.AddStmt(C.getConditionExpr());
 }
 } // namespace
-- 
GitLab


From db8e182952573fa9198424a6ed1b1bb1676eae62 Mon Sep 17 00:00:00 2001
From: erichkeane <ekeane@nvidia.com>
Date: Fri, 12 Apr 2024 14:44:20 -0700
Subject: [PATCH 121/464] [NFC] Clang-format previous OpenACC Patch

---
 clang/lib/AST/OpenACCClause.cpp | 3 +--
 clang/lib/AST/StmtProfile.cpp   | 6 ++----
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/clang/lib/AST/OpenACCClause.cpp b/clang/lib/AST/OpenACCClause.cpp
index 446e47003204..dcb512cb5141 100644
--- a/clang/lib/AST/OpenACCClause.cpp
+++ b/clang/lib/AST/OpenACCClause.cpp
@@ -65,8 +65,7 @@ OpenACCClause::child_range OpenACCClause::children() {
 //===----------------------------------------------------------------------===//
 //  OpenACC clauses printing methods
 //===----------------------------------------------------------------------===//
-void OpenACCClausePrinter::VisitDefaultClause(
-    const OpenACCDefaultClause &C) {
+void OpenACCClausePrinter::VisitDefaultClause(const OpenACCDefaultClause &C) {
   OS << "default(" << C.getDefaultClauseKind() << ")";
 }
 
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index 68ba7475f4ca..d2aac1e64038 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -2459,8 +2459,7 @@ public:
   }
 
 #define VISIT_CLAUSE(CLAUSE_NAME)                                              \
-  void Visit##CLAUSE_NAME##Clause(                                      \
-      const OpenACC##CLAUSE_NAME##Clause &Clause);
+  void Visit##CLAUSE_NAME##Clause(const OpenACC##CLAUSE_NAME##Clause &Clause);
 
 #include "clang/Basic/OpenACCClauses.def"
 };
@@ -2469,8 +2468,7 @@ public:
 void OpenACCClauseProfiler::VisitDefaultClause(
     const OpenACCDefaultClause &Clause) {}
 
-void OpenACCClauseProfiler::VisitIfClause(
-    const OpenACCIfClause &Clause) {
+void OpenACCClauseProfiler::VisitIfClause(const OpenACCIfClause &Clause) {
   assert(Clause.hasConditionExpr() &&
          "if clause requires a valid condition expr");
   Profiler.VisitStmt(Clause.getConditionExpr());
-- 
GitLab


From 779ba60417b467a6d2d25101b11711c009694315 Mon Sep 17 00:00:00 2001
From: Artem Chikin <achikin@apple.com>
Date: Fri, 12 Apr 2024 14:46:56 -0700
Subject: [PATCH 122/464] [clang][deps] Overload `Filesystem::exists` in
 `DependencyScanningFilesystem` to have it use cached `status` (#88152)

As-is, calls to `exists()` fallback on the implementation in
`ProxyFileSystem::exists` which explicitly calls out to the underlying
`FS`, which for the `DependencyScanningFilesystem` (overlay) is the real
underlying filesystem.

Instead, directly overloading `exists` allows us to have it rely on the
cached `status` behavior used elsewhere by the
`DependencyScanningFilesystem`.
---
 .../DependencyScanningFilesystem.h            |  4 +++
 .../DependencyScanningFilesystem.cpp          | 11 ++++++++
 .../DependencyScanningFilesystemTest.cpp      | 27 +++++++++++++++++++
 3 files changed, 42 insertions(+)

diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
index 8b6f149c7cb2..f7b4510d7f7b 100644
--- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
+++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
@@ -365,6 +365,10 @@ public:
   /// false if not (i.e. this entry is not a file or its scan fails).
   bool ensureDirectiveTokensArePopulated(EntryRef Entry);
 
+  /// Check whether \p Path exists. By default checks cached result of \c
+  /// status(), and falls back on FS if unable to do so.
+  bool exists(const Twine &Path) override;
+
 private:
   /// For a filename that's not yet associated with any entry in the caches,
   /// uses the underlying filesystem to either look up the entry based in the
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
index 84185c931b55..0cab17a34244 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
@@ -300,6 +300,17 @@ DependencyScanningWorkerFilesystem::status(const Twine &Path) {
   return Result->getStatus();
 }
 
+bool DependencyScanningWorkerFilesystem::exists(const Twine &Path) {
+  // While some VFS overlay filesystems may implement more-efficient
+  // mechanisms for `exists` queries, `DependencyScanningWorkerFilesystem`
+  // typically wraps `RealFileSystem` which does not specialize `exists`,
+  // so it is not likely to benefit from such optimizations. Instead,
+  // it is more-valuable to have this query go through the
+  // cached-`status` code-path of the `DependencyScanningWorkerFilesystem`.
+  llvm::ErrorOr<llvm::vfs::Status> Status = status(Path);
+  return Status && Status->exists();
+}
+
 namespace {
 
 /// The VFS that is used by clang consumes the \c CachedFileSystemEntry using
diff --git a/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp b/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
index 697b7d70ff03..87bb67cfd932 100644
--- a/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
+++ b/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
@@ -18,6 +18,7 @@ struct InstrumentingFilesystem
     : llvm::RTTIExtends<InstrumentingFilesystem, llvm::vfs::ProxyFileSystem> {
   unsigned NumStatusCalls = 0;
   unsigned NumGetRealPathCalls = 0;
+  unsigned NumExistsCalls = 0;
 
   using llvm::RTTIExtends<InstrumentingFilesystem,
                           llvm::vfs::ProxyFileSystem>::RTTIExtends;
@@ -32,6 +33,11 @@ struct InstrumentingFilesystem
     ++NumGetRealPathCalls;
     return ProxyFileSystem::getRealPath(Path, Output);
   }
+
+  bool exists(const llvm::Twine &Path) override {
+    ++NumExistsCalls;
+    return ProxyFileSystem::exists(Path);
+  }
 };
 } // namespace
 
@@ -147,3 +153,24 @@ TEST(DependencyScanningFilesystem, RealPathAndStatusInvariants) {
     DepFS.status("/bar");
   }
 }
+
+TEST(DependencyScanningFilesystem, CacheStatOnExists) {
+  auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
+  auto InstrumentingFS =
+      llvm::makeIntrusiveRefCnt<InstrumentingFilesystem>(InMemoryFS);
+  InMemoryFS->setCurrentWorkingDirectory("/");
+  InMemoryFS->addFile("/foo", 0, llvm::MemoryBuffer::getMemBuffer(""));
+  InMemoryFS->addFile("/bar", 0, llvm::MemoryBuffer::getMemBuffer(""));
+  DependencyScanningFilesystemSharedCache SharedCache;
+  DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS);
+
+  DepFS.status("/foo");
+  DepFS.status("/foo");
+  DepFS.status("/bar");
+  EXPECT_EQ(InstrumentingFS->NumStatusCalls, 2u);
+
+  DepFS.exists("/foo");
+  DepFS.exists("/bar");
+  EXPECT_EQ(InstrumentingFS->NumStatusCalls, 2u);
+  EXPECT_EQ(InstrumentingFS->NumExistsCalls, 0u);
+}
-- 
GitLab


From 03c393db1821ef4c362c4c228492c6fb03f8876f Mon Sep 17 00:00:00 2001
From: erichkeane <ekeane@nvidia.com>
Date: Fri, 12 Apr 2024 14:48:39 -0700
Subject: [PATCH 123/464] Fix build failure from a472e6473e69

Looks like I forgot to do build CIndex.cpp when validating myself!
---
 clang/tools/libclang/CIndex.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index b999750c0f5e..f304786ff9df 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2783,7 +2783,7 @@ public:
   OpenACCClauseEnqueue(EnqueueVisitor &V) : Visitor(V) {}
 
 #define VISIT_CLAUSE(CLAUSE_NAME)                                              \
-  void VisitOpenACC##CLAUSE_NAME##Clause(const OpenACC##CLAUSE_NAME##Clause &C);
+  void Visit##CLAUSE_NAME##Clause(const OpenACC##CLAUSE_NAME##Clause &C);
 #include "clang/Basic/OpenACCClauses.def"
 };
 
-- 
GitLab


From 0d2bb7f017f13ceae793fab7d83d3e67e8d8d8f8 Mon Sep 17 00:00:00 2001
From: Tom Stellard <tstellar@redhat.com>
Date: Fri, 12 Apr 2024 15:10:04 -0700
Subject: [PATCH 124/464] [CMake][Release] Disable PGO (#88465)

Running the test-release.sh script with PGO enabled causes build errors
like:

ld.lld: error: Function Import: link error: linking module flags
'ProfileSummary': IDs have conflicting values

I believe this a build system bug due to the PGO profile data being
generated unconditionally. If you run `ninja check-all` and then `ninja
install` like we do in test-release.sh, then the profile data is
regenerated during `ninja install` and some of the clang tools which are
not test dependencies get build during the ninja install step with
different profile data. When these tools link against the LLVM
libraries, like libSupport, we end up with these errors.
---
 clang/cmake/caches/Release.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/cmake/caches/Release.cmake b/clang/cmake/caches/Release.cmake
index 1ca9138b9807..bd1f688d61a7 100644
--- a/clang/cmake/caches/Release.cmake
+++ b/clang/cmake/caches/Release.cmake
@@ -4,7 +4,7 @@
 
 # General Options
 set(LLVM_RELEASE_ENABLE_LTO THIN CACHE STRING "")
-set(LLVM_RELEASE_ENABLE_PGO ON CACHE BOOL "")
+set(LLVM_RELEASE_ENABLE_PGO OFF CACHE BOOL "")
 
 set(CMAKE_BUILD_TYPE RELEASE CACHE STRING "")
 
-- 
GitLab


From 9bd10853e505b61f5fe2e3b3592c76787a06aa31 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sat, 13 Apr 2024 00:24:12 +0200
Subject: [PATCH 125/464] AMDGPU: Undo atomicrmw add/sub/xor 0 -> atomicrmw or
 canonicalization (#87533)

InstCombine transforms add of 0 to or of 0. For system atomics, this is
problematic because while PCIe supports add, it does not support the
other operations. Undo this for system scope atomics.
---
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     |  27 +-
 .../AMDGPU/global_atomics_i32_system.ll       | 723 ++++++++++--------
 ...expand-atomicrmw-integer-ops-0-to-add-0.ll | 130 ++++
 3 files changed, 572 insertions(+), 308 deletions(-)
 create mode 100644 llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomicrmw-integer-ops-0-to-add-0.ll

diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 14948ef9ea8d..6fde73119f81 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -16027,6 +16027,19 @@ SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
       SSID == RMW->getContext().getOrInsertSyncScopeID("one-as");
 
   switch (RMW->getOperation()) {
+  case AtomicRMWInst::Sub:
+  case AtomicRMWInst::Or:
+  case AtomicRMWInst::Xor: {
+    // Atomic sub/or/xor do not work over PCI express, but atomic add
+    // does. InstCombine transforms these with 0 to or, so undo that.
+    if (HasSystemScope && AMDGPU::isFlatGlobalAddrSpace(AS)) {
+      if (Constant *ConstVal = dyn_cast<Constant>(RMW->getValOperand());
+          ConstVal && ConstVal->isNullValue())
+        return AtomicExpansionKind::Expand;
+    }
+
+    break;
+  }
   case AtomicRMWInst::FAdd: {
     Type *Ty = RMW->getType();
 
@@ -16312,14 +16325,24 @@ bool SITargetLowering::checkForPhysRegDependency(
 }
 
 void SITargetLowering::emitExpandAtomicRMW(AtomicRMWInst *AI) const {
+  AtomicRMWInst::BinOp Op = AI->getOperation();
+
+  if (Op == AtomicRMWInst::Sub || Op == AtomicRMWInst::Or ||
+      Op == AtomicRMWInst::Xor) {
+    // atomicrmw or %ptr, 0 -> atomicrmw add %ptr, 0
+    assert(cast<Constant>(AI->getValOperand())->isNullValue() &&
+           "this cannot be replaced with add");
+    AI->setOperation(AtomicRMWInst::Add);
+    return;
+  }
+
   assert(Subtarget->hasAtomicFaddInsts() &&
          "target should have atomic fadd instructions");
   assert(AI->getType()->isFloatTy() &&
          AI->getPointerAddressSpace() == AMDGPUAS::FLAT_ADDRESS &&
          "generic atomicrmw expansion only supports FP32 operand in flat "
          "address space");
-  assert(AI->getOperation() == AtomicRMWInst::FAdd &&
-         "only fadd is supported for now");
+  assert(Op == AtomicRMWInst::FAdd && "only fadd is supported for now");
 
   // Given: atomicrmw fadd ptr %addr, float %val ordering
   //
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll
index 99d02ffaa523..4598d23e088b 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll
@@ -1403,6 +1403,43 @@ define amdgpu_gfx i32 @global_atomic_sub_i32_ret_offset_scalar(ptr addrspace(1)
   ret i32 %result
 }
 
+define i32 @global_atomic_sub_0_i32_ret(ptr addrspace(1) %ptr) {
+; SI-LABEL: global_atomic_sub_0_i32_ret:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_mov_b32 s4, s6
+; SI-NEXT:    s_mov_b32 s5, s6
+; SI-NEXT:    buffer_atomic_add v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_wbinvl1
+; SI-NEXT:    v_mov_b32_e32 v0, v2
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: global_atomic_sub_0_i32_ret:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, 0
+; VI-NEXT:    flat_atomic_add v0, v[0:1], v2 glc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: global_atomic_sub_0_i32_ret:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    global_atomic_add v0, v[0:1], v2, off glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = atomicrmw sub ptr addrspace(1) %ptr, i32 0 seq_cst
+  ret i32 %result
+}
+
 ; ---------------------------------------------------------------------
 ; atomicrmw and
 ; ---------------------------------------------------------------------
@@ -1767,7 +1804,7 @@ define void @global_atomic_nand_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; SI-NEXT:    s_mov_b32 s5, s6
 ; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
 ; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB40_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB41_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v3, v4, v2
@@ -1782,7 +1819,7 @@ define void @global_atomic_nand_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
 ; SI-NEXT:    v_mov_b32_e32 v4, v5
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB40_1
+; SI-NEXT:    s_cbranch_execnz .LBB41_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -1793,7 +1830,7 @@ define void @global_atomic_nand_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    flat_load_dword v4, v[0:1]
 ; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB40_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB41_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_and_b32_e32 v3, v4, v2
@@ -1805,7 +1842,7 @@ define void @global_atomic_nand_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; VI-NEXT:    v_mov_b32_e32 v4, v3
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB40_1
+; VI-NEXT:    s_cbranch_execnz .LBB41_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -1815,7 +1852,7 @@ define void @global_atomic_nand_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    global_load_dword v4, v[0:1], off
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB40_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB41_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_and_b32_e32 v3, v4, v2
@@ -1827,7 +1864,7 @@ define void @global_atomic_nand_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v4, v3
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB40_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB41_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -1845,7 +1882,7 @@ define void @global_atomic_nand_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; SI-NEXT:    s_mov_b32 s5, s6
 ; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:16
 ; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB41_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB42_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v3, v4, v2
@@ -1860,7 +1897,7 @@ define void @global_atomic_nand_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
 ; SI-NEXT:    v_mov_b32_e32 v4, v5
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB41_1
+; SI-NEXT:    s_cbranch_execnz .LBB42_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -1873,7 +1910,7 @@ define void @global_atomic_nand_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dword v4, v[0:1]
 ; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB41_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB42_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_and_b32_e32 v3, v4, v2
@@ -1885,7 +1922,7 @@ define void @global_atomic_nand_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; VI-NEXT:    v_mov_b32_e32 v4, v3
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB41_1
+; VI-NEXT:    s_cbranch_execnz .LBB42_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -1895,7 +1932,7 @@ define void @global_atomic_nand_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:16
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB41_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB42_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_and_b32_e32 v3, v4, v2
@@ -1907,7 +1944,7 @@ define void @global_atomic_nand_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v4, v3
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB41_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB42_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -1926,7 +1963,7 @@ define i32 @global_atomic_nand_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; SI-NEXT:    s_mov_b32 s5, s6
 ; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
 ; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB42_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB43_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v5, v3
@@ -1941,7 +1978,7 @@ define i32 @global_atomic_nand_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
 ; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB42_1
+; SI-NEXT:    s_cbranch_execnz .LBB43_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    v_mov_b32_e32 v0, v3
@@ -1953,7 +1990,7 @@ define i32 @global_atomic_nand_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    flat_load_dword v3, v[0:1]
 ; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB42_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB43_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v4, v3
@@ -1965,7 +2002,7 @@ define i32 @global_atomic_nand_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
 ; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB42_1
+; VI-NEXT:    s_cbranch_execnz .LBB43_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    v_mov_b32_e32 v0, v3
@@ -1976,7 +2013,7 @@ define i32 @global_atomic_nand_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    global_load_dword v3, v[0:1], off
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB42_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB43_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v4, v3
@@ -1988,7 +2025,7 @@ define i32 @global_atomic_nand_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
 ; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB42_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB43_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, v3
@@ -2007,7 +2044,7 @@ define i32 @global_atomic_nand_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; SI-NEXT:    s_mov_b32 s5, s6
 ; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:16
 ; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB43_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB44_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v5, v3
@@ -2022,7 +2059,7 @@ define i32 @global_atomic_nand_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
 ; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB43_1
+; SI-NEXT:    s_cbranch_execnz .LBB44_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    v_mov_b32_e32 v0, v3
@@ -2036,7 +2073,7 @@ define i32 @global_atomic_nand_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dword v0, v[3:4]
 ; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB43_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB44_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, v0
@@ -2048,7 +2085,7 @@ define i32 @global_atomic_nand_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
 ; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB43_1
+; VI-NEXT:    s_cbranch_execnz .LBB44_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -2058,7 +2095,7 @@ define i32 @global_atomic_nand_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    global_load_dword v3, v[0:1], off offset:16
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB43_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB44_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v4, v3
@@ -2070,7 +2107,7 @@ define i32 @global_atomic_nand_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
 ; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB43_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB44_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, v3
@@ -2095,7 +2132,7 @@ define amdgpu_gfx void @global_atomic_nand_i32_noret_scalar(ptr addrspace(1) inr
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0
 ; SI-NEXT:    s_mov_b64 s[36:37], 0
-; SI-NEXT:  .LBB44_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB45_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v0, s34, v1
@@ -2110,7 +2147,7 @@ define amdgpu_gfx void @global_atomic_nand_i32_noret_scalar(ptr addrspace(1) inr
 ; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
 ; SI-NEXT:    v_mov_b32_e32 v1, v2
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT:    s_cbranch_execnz .LBB44_1
+; SI-NEXT:    s_cbranch_execnz .LBB45_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; SI-NEXT:    v_readlane_b32 s7, v4, 1
@@ -2128,7 +2165,7 @@ define amdgpu_gfx void @global_atomic_nand_i32_noret_scalar(ptr addrspace(1) inr
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
 ; VI-NEXT:    flat_load_dword v3, v[0:1]
 ; VI-NEXT:    s_mov_b64 s[34:35], 0
-; VI-NEXT:  .LBB44_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB45_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_and_b32_e32 v2, s6, v3
@@ -2140,7 +2177,7 @@ define amdgpu_gfx void @global_atomic_nand_i32_noret_scalar(ptr addrspace(1) inr
 ; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; VI-NEXT:    v_mov_b32_e32 v3, v2
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT:    s_cbranch_execnz .LBB44_1
+; VI-NEXT:    s_cbranch_execnz .LBB45_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -2151,7 +2188,7 @@ define amdgpu_gfx void @global_atomic_nand_i32_noret_scalar(ptr addrspace(1) inr
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    global_load_dword v1, v2, s[4:5]
 ; GFX9-NEXT:    s_mov_b64 s[34:35], 0
-; GFX9-NEXT:  .LBB44_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB45_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_and_b32_e32 v0, s6, v1
@@ -2163,7 +2200,7 @@ define amdgpu_gfx void @global_atomic_nand_i32_noret_scalar(ptr addrspace(1) inr
 ; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; GFX9-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT:    s_cbranch_execnz .LBB44_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB45_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -2186,7 +2223,7 @@ define amdgpu_gfx void @global_atomic_nand_i32_noret_offset_scalar(ptr addrspace
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0 offset:16
 ; SI-NEXT:    s_mov_b64 s[36:37], 0
-; SI-NEXT:  .LBB45_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB46_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_and_b32_e32 v0, s34, v1
@@ -2201,7 +2238,7 @@ define amdgpu_gfx void @global_atomic_nand_i32_noret_offset_scalar(ptr addrspace
 ; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
 ; SI-NEXT:    v_mov_b32_e32 v1, v2
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT:    s_cbranch_execnz .LBB45_1
+; SI-NEXT:    s_cbranch_execnz .LBB46_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; SI-NEXT:    v_readlane_b32 s7, v4, 1
@@ -2221,7 +2258,7 @@ define amdgpu_gfx void @global_atomic_nand_i32_noret_offset_scalar(ptr addrspace
 ; VI-NEXT:    v_mov_b32_e32 v1, s35
 ; VI-NEXT:    flat_load_dword v3, v[0:1]
 ; VI-NEXT:    s_mov_b64 s[34:35], 0
-; VI-NEXT:  .LBB45_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB46_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_and_b32_e32 v2, s6, v3
@@ -2233,7 +2270,7 @@ define amdgpu_gfx void @global_atomic_nand_i32_noret_offset_scalar(ptr addrspace
 ; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; VI-NEXT:    v_mov_b32_e32 v3, v2
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT:    s_cbranch_execnz .LBB45_1
+; VI-NEXT:    s_cbranch_execnz .LBB46_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -2244,7 +2281,7 @@ define amdgpu_gfx void @global_atomic_nand_i32_noret_offset_scalar(ptr addrspace
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    global_load_dword v1, v2, s[4:5] offset:16
 ; GFX9-NEXT:    s_mov_b64 s[34:35], 0
-; GFX9-NEXT:  .LBB45_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB46_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_and_b32_e32 v0, s6, v1
@@ -2256,7 +2293,7 @@ define amdgpu_gfx void @global_atomic_nand_i32_noret_offset_scalar(ptr addrspace
 ; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; GFX9-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT:    s_cbranch_execnz .LBB45_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB46_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -2280,7 +2317,7 @@ define amdgpu_gfx i32 @global_atomic_nand_i32_ret_scalar(ptr addrspace(1) inreg
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0
 ; SI-NEXT:    s_mov_b64 s[36:37], 0
-; SI-NEXT:  .LBB46_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB47_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v2, v0
@@ -2295,7 +2332,7 @@ define amdgpu_gfx i32 @global_atomic_nand_i32_ret_scalar(ptr addrspace(1) inreg
 ; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
 ; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT:    s_cbranch_execnz .LBB46_1
+; SI-NEXT:    s_cbranch_execnz .LBB47_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; SI-NEXT:    v_readlane_b32 s7, v3, 1
@@ -2315,7 +2352,7 @@ define amdgpu_gfx i32 @global_atomic_nand_i32_ret_scalar(ptr addrspace(1) inreg
 ; VI-NEXT:    v_mov_b32_e32 v1, s4
 ; VI-NEXT:    s_mov_b64 s[34:35], 0
 ; VI-NEXT:    v_mov_b32_e32 v2, s5
-; VI-NEXT:  .LBB46_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB47_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v4, v0
@@ -2327,7 +2364,7 @@ define amdgpu_gfx i32 @global_atomic_nand_i32_ret_scalar(ptr addrspace(1) inreg
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
 ; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT:    s_cbranch_execnz .LBB46_1
+; VI-NEXT:    s_cbranch_execnz .LBB47_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -2338,7 +2375,7 @@ define amdgpu_gfx i32 @global_atomic_nand_i32_ret_scalar(ptr addrspace(1) inreg
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    global_load_dword v0, v1, s[4:5]
 ; GFX9-NEXT:    s_mov_b64 s[34:35], 0
-; GFX9-NEXT:  .LBB46_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB47_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v3, v0
@@ -2350,7 +2387,7 @@ define amdgpu_gfx i32 @global_atomic_nand_i32_ret_scalar(ptr addrspace(1) inreg
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
 ; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT:    s_cbranch_execnz .LBB46_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB47_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -2373,7 +2410,7 @@ define amdgpu_gfx i32 @global_atomic_nand_i32_ret_offset_scalar(ptr addrspace(1)
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0 offset:16
 ; SI-NEXT:    s_mov_b64 s[36:37], 0
-; SI-NEXT:  .LBB47_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB48_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v2, v0
@@ -2388,7 +2425,7 @@ define amdgpu_gfx i32 @global_atomic_nand_i32_ret_offset_scalar(ptr addrspace(1)
 ; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
 ; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT:    s_cbranch_execnz .LBB47_1
+; SI-NEXT:    s_cbranch_execnz .LBB48_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; SI-NEXT:    v_readlane_b32 s7, v3, 1
@@ -2408,7 +2445,7 @@ define amdgpu_gfx i32 @global_atomic_nand_i32_ret_offset_scalar(ptr addrspace(1)
 ; VI-NEXT:    v_mov_b32_e32 v2, s35
 ; VI-NEXT:    flat_load_dword v0, v[1:2]
 ; VI-NEXT:    s_mov_b64 s[34:35], 0
-; VI-NEXT:  .LBB47_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB48_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v4, v0
@@ -2420,7 +2457,7 @@ define amdgpu_gfx i32 @global_atomic_nand_i32_ret_offset_scalar(ptr addrspace(1)
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
 ; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT:    s_cbranch_execnz .LBB47_1
+; VI-NEXT:    s_cbranch_execnz .LBB48_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -2431,7 +2468,7 @@ define amdgpu_gfx i32 @global_atomic_nand_i32_ret_offset_scalar(ptr addrspace(1)
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    global_load_dword v0, v1, s[4:5] offset:16
 ; GFX9-NEXT:    s_mov_b64 s[34:35], 0
-; GFX9-NEXT:  .LBB47_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB48_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v3, v0
@@ -2443,7 +2480,7 @@ define amdgpu_gfx i32 @global_atomic_nand_i32_ret_offset_scalar(ptr addrspace(1)
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
 ; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT:    s_cbranch_execnz .LBB47_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB48_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -2802,6 +2839,43 @@ define amdgpu_gfx i32 @global_atomic_or_i32_ret_offset_scalar(ptr addrspace(1) i
   ret i32 %result
 }
 
+define i32 @global_atomic_or_0_i32_ret(ptr addrspace(1) %ptr) {
+; SI-LABEL: global_atomic_or_0_i32_ret:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_mov_b32 s4, s6
+; SI-NEXT:    s_mov_b32 s5, s6
+; SI-NEXT:    buffer_atomic_add v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_wbinvl1
+; SI-NEXT:    v_mov_b32_e32 v0, v2
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: global_atomic_or_0_i32_ret:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, 0
+; VI-NEXT:    flat_atomic_add v0, v[0:1], v2 glc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: global_atomic_or_0_i32_ret:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    global_atomic_add v0, v[0:1], v2, off glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = atomicrmw or ptr addrspace(1) %ptr, i32 0 seq_cst
+  ret i32 %result
+}
+
 ; ---------------------------------------------------------------------
 ; atomicrmw xor
 ; ---------------------------------------------------------------------
@@ -3152,6 +3226,43 @@ define amdgpu_gfx i32 @global_atomic_xor_i32_ret_offset_scalar(ptr addrspace(1)
   ret i32 %result
 }
 
+define i32 @global_atomic_xor_0_i32_ret(ptr addrspace(1) %ptr) {
+; SI-LABEL: global_atomic_xor_0_i32_ret:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_mov_b32_e32 v2, 0
+; SI-NEXT:    s_mov_b32 s4, s6
+; SI-NEXT:    s_mov_b32 s5, s6
+; SI-NEXT:    buffer_atomic_add v2, v[0:1], s[4:7], 0 addr64 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_wbinvl1
+; SI-NEXT:    v_mov_b32_e32 v0, v2
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    s_setpc_b64 s[30:31]
+;
+; VI-LABEL: global_atomic_xor_0_i32_ret:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v2, 0
+; VI-NEXT:    flat_atomic_add v0, v[0:1], v2 glc
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    buffer_wbinvl1_vol
+; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: global_atomic_xor_0_i32_ret:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    global_atomic_add v0, v[0:1], v2, off glc
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    buffer_wbinvl1_vol
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %result = atomicrmw xor ptr addrspace(1) %ptr, i32 0 seq_cst
+  ret i32 %result
+}
+
 ; ---------------------------------------------------------------------
 ; atomicrmw max
 ; ---------------------------------------------------------------------
@@ -3166,7 +3277,7 @@ define void @global_atomic_max_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; SI-NEXT:    s_mov_b32 s5, s6
 ; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
 ; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB64_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB67_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_max_i32_e32 v3, v4, v2
@@ -3180,7 +3291,7 @@ define void @global_atomic_max_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
 ; SI-NEXT:    v_mov_b32_e32 v4, v5
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB64_1
+; SI-NEXT:    s_cbranch_execnz .LBB67_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -3191,7 +3302,7 @@ define void @global_atomic_max_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    flat_load_dword v4, v[0:1]
 ; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB64_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB67_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_max_i32_e32 v3, v4, v2
@@ -3202,7 +3313,7 @@ define void @global_atomic_max_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; VI-NEXT:    v_mov_b32_e32 v4, v3
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB64_1
+; VI-NEXT:    s_cbranch_execnz .LBB67_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -3212,7 +3323,7 @@ define void @global_atomic_max_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    global_load_dword v4, v[0:1], off
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB64_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB67_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_max_i32_e32 v3, v4, v2
@@ -3223,7 +3334,7 @@ define void @global_atomic_max_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v4, v3
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB64_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB67_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -3241,7 +3352,7 @@ define void @global_atomic_max_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; SI-NEXT:    s_mov_b32 s5, s6
 ; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:16
 ; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB65_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB68_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_max_i32_e32 v3, v4, v2
@@ -3255,7 +3366,7 @@ define void @global_atomic_max_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
 ; SI-NEXT:    v_mov_b32_e32 v4, v5
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB65_1
+; SI-NEXT:    s_cbranch_execnz .LBB68_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -3268,7 +3379,7 @@ define void @global_atomic_max_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dword v4, v[0:1]
 ; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB65_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB68_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_max_i32_e32 v3, v4, v2
@@ -3279,7 +3390,7 @@ define void @global_atomic_max_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; VI-NEXT:    v_mov_b32_e32 v4, v3
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB65_1
+; VI-NEXT:    s_cbranch_execnz .LBB68_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -3289,7 +3400,7 @@ define void @global_atomic_max_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:16
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB65_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB68_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_max_i32_e32 v3, v4, v2
@@ -3300,7 +3411,7 @@ define void @global_atomic_max_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v4, v3
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB65_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB68_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -3319,7 +3430,7 @@ define i32 @global_atomic_max_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; SI-NEXT:    s_mov_b32 s5, s6
 ; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
 ; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB66_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB69_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v5, v3
@@ -3333,7 +3444,7 @@ define i32 @global_atomic_max_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
 ; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB66_1
+; SI-NEXT:    s_cbranch_execnz .LBB69_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    v_mov_b32_e32 v0, v3
@@ -3345,7 +3456,7 @@ define i32 @global_atomic_max_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    flat_load_dword v3, v[0:1]
 ; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB66_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB69_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v4, v3
@@ -3356,7 +3467,7 @@ define i32 @global_atomic_max_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
 ; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB66_1
+; VI-NEXT:    s_cbranch_execnz .LBB69_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    v_mov_b32_e32 v0, v3
@@ -3367,7 +3478,7 @@ define i32 @global_atomic_max_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    global_load_dword v3, v[0:1], off
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB66_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB69_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v4, v3
@@ -3378,7 +3489,7 @@ define i32 @global_atomic_max_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
 ; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB66_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB69_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, v3
@@ -3397,7 +3508,7 @@ define i32 @global_atomic_max_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; SI-NEXT:    s_mov_b32 s5, s6
 ; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:16
 ; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB67_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB70_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v5, v3
@@ -3411,7 +3522,7 @@ define i32 @global_atomic_max_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
 ; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB67_1
+; SI-NEXT:    s_cbranch_execnz .LBB70_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    v_mov_b32_e32 v0, v3
@@ -3425,7 +3536,7 @@ define i32 @global_atomic_max_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dword v0, v[3:4]
 ; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB67_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB70_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, v0
@@ -3436,7 +3547,7 @@ define i32 @global_atomic_max_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
 ; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB67_1
+; VI-NEXT:    s_cbranch_execnz .LBB70_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -3446,7 +3557,7 @@ define i32 @global_atomic_max_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    global_load_dword v3, v[0:1], off offset:16
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB67_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB70_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v4, v3
@@ -3457,7 +3568,7 @@ define i32 @global_atomic_max_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
 ; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB67_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB70_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, v3
@@ -3482,7 +3593,7 @@ define amdgpu_gfx void @global_atomic_max_i32_noret_scalar(ptr addrspace(1) inre
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0
 ; SI-NEXT:    s_mov_b64 s[36:37], 0
-; SI-NEXT:  .LBB68_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB71_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_max_i32_e32 v0, s34, v1
@@ -3496,7 +3607,7 @@ define amdgpu_gfx void @global_atomic_max_i32_noret_scalar(ptr addrspace(1) inre
 ; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
 ; SI-NEXT:    v_mov_b32_e32 v1, v2
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT:    s_cbranch_execnz .LBB68_1
+; SI-NEXT:    s_cbranch_execnz .LBB71_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; SI-NEXT:    v_readlane_b32 s7, v4, 1
@@ -3514,7 +3625,7 @@ define amdgpu_gfx void @global_atomic_max_i32_noret_scalar(ptr addrspace(1) inre
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
 ; VI-NEXT:    flat_load_dword v3, v[0:1]
 ; VI-NEXT:    s_mov_b64 s[34:35], 0
-; VI-NEXT:  .LBB68_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB71_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_max_i32_e32 v2, s6, v3
@@ -3525,7 +3636,7 @@ define amdgpu_gfx void @global_atomic_max_i32_noret_scalar(ptr addrspace(1) inre
 ; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; VI-NEXT:    v_mov_b32_e32 v3, v2
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT:    s_cbranch_execnz .LBB68_1
+; VI-NEXT:    s_cbranch_execnz .LBB71_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -3536,7 +3647,7 @@ define amdgpu_gfx void @global_atomic_max_i32_noret_scalar(ptr addrspace(1) inre
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    global_load_dword v1, v2, s[4:5]
 ; GFX9-NEXT:    s_mov_b64 s[34:35], 0
-; GFX9-NEXT:  .LBB68_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB71_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_max_i32_e32 v0, s6, v1
@@ -3547,7 +3658,7 @@ define amdgpu_gfx void @global_atomic_max_i32_noret_scalar(ptr addrspace(1) inre
 ; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; GFX9-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT:    s_cbranch_execnz .LBB68_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB71_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -3570,7 +3681,7 @@ define amdgpu_gfx void @global_atomic_max_i32_noret_offset_scalar(ptr addrspace(
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0 offset:16
 ; SI-NEXT:    s_mov_b64 s[36:37], 0
-; SI-NEXT:  .LBB69_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB72_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_max_i32_e32 v0, s34, v1
@@ -3584,7 +3695,7 @@ define amdgpu_gfx void @global_atomic_max_i32_noret_offset_scalar(ptr addrspace(
 ; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
 ; SI-NEXT:    v_mov_b32_e32 v1, v2
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT:    s_cbranch_execnz .LBB69_1
+; SI-NEXT:    s_cbranch_execnz .LBB72_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; SI-NEXT:    v_readlane_b32 s7, v4, 1
@@ -3604,7 +3715,7 @@ define amdgpu_gfx void @global_atomic_max_i32_noret_offset_scalar(ptr addrspace(
 ; VI-NEXT:    v_mov_b32_e32 v1, s35
 ; VI-NEXT:    flat_load_dword v3, v[0:1]
 ; VI-NEXT:    s_mov_b64 s[34:35], 0
-; VI-NEXT:  .LBB69_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB72_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_max_i32_e32 v2, s6, v3
@@ -3615,7 +3726,7 @@ define amdgpu_gfx void @global_atomic_max_i32_noret_offset_scalar(ptr addrspace(
 ; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; VI-NEXT:    v_mov_b32_e32 v3, v2
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT:    s_cbranch_execnz .LBB69_1
+; VI-NEXT:    s_cbranch_execnz .LBB72_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -3626,7 +3737,7 @@ define amdgpu_gfx void @global_atomic_max_i32_noret_offset_scalar(ptr addrspace(
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    global_load_dword v1, v2, s[4:5] offset:16
 ; GFX9-NEXT:    s_mov_b64 s[34:35], 0
-; GFX9-NEXT:  .LBB69_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB72_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_max_i32_e32 v0, s6, v1
@@ -3637,7 +3748,7 @@ define amdgpu_gfx void @global_atomic_max_i32_noret_offset_scalar(ptr addrspace(
 ; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; GFX9-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT:    s_cbranch_execnz .LBB69_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB72_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -3661,7 +3772,7 @@ define amdgpu_gfx i32 @global_atomic_max_i32_ret_scalar(ptr addrspace(1) inreg %
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0
 ; SI-NEXT:    s_mov_b64 s[36:37], 0
-; SI-NEXT:  .LBB70_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB73_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v2, v0
@@ -3675,7 +3786,7 @@ define amdgpu_gfx i32 @global_atomic_max_i32_ret_scalar(ptr addrspace(1) inreg %
 ; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
 ; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT:    s_cbranch_execnz .LBB70_1
+; SI-NEXT:    s_cbranch_execnz .LBB73_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; SI-NEXT:    v_readlane_b32 s7, v3, 1
@@ -3695,7 +3806,7 @@ define amdgpu_gfx i32 @global_atomic_max_i32_ret_scalar(ptr addrspace(1) inreg %
 ; VI-NEXT:    v_mov_b32_e32 v1, s4
 ; VI-NEXT:    s_mov_b64 s[34:35], 0
 ; VI-NEXT:    v_mov_b32_e32 v2, s5
-; VI-NEXT:  .LBB70_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB73_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v4, v0
@@ -3706,7 +3817,7 @@ define amdgpu_gfx i32 @global_atomic_max_i32_ret_scalar(ptr addrspace(1) inreg %
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
 ; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT:    s_cbranch_execnz .LBB70_1
+; VI-NEXT:    s_cbranch_execnz .LBB73_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -3717,7 +3828,7 @@ define amdgpu_gfx i32 @global_atomic_max_i32_ret_scalar(ptr addrspace(1) inreg %
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    global_load_dword v0, v1, s[4:5]
 ; GFX9-NEXT:    s_mov_b64 s[34:35], 0
-; GFX9-NEXT:  .LBB70_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB73_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v3, v0
@@ -3728,7 +3839,7 @@ define amdgpu_gfx i32 @global_atomic_max_i32_ret_scalar(ptr addrspace(1) inreg %
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
 ; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT:    s_cbranch_execnz .LBB70_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB73_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -3751,7 +3862,7 @@ define amdgpu_gfx i32 @global_atomic_max_i32_ret_offset_scalar(ptr addrspace(1)
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0 offset:16
 ; SI-NEXT:    s_mov_b64 s[36:37], 0
-; SI-NEXT:  .LBB71_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB74_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v2, v0
@@ -3765,7 +3876,7 @@ define amdgpu_gfx i32 @global_atomic_max_i32_ret_offset_scalar(ptr addrspace(1)
 ; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
 ; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT:    s_cbranch_execnz .LBB71_1
+; SI-NEXT:    s_cbranch_execnz .LBB74_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; SI-NEXT:    v_readlane_b32 s7, v3, 1
@@ -3785,7 +3896,7 @@ define amdgpu_gfx i32 @global_atomic_max_i32_ret_offset_scalar(ptr addrspace(1)
 ; VI-NEXT:    v_mov_b32_e32 v2, s35
 ; VI-NEXT:    flat_load_dword v0, v[1:2]
 ; VI-NEXT:    s_mov_b64 s[34:35], 0
-; VI-NEXT:  .LBB71_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB74_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v4, v0
@@ -3796,7 +3907,7 @@ define amdgpu_gfx i32 @global_atomic_max_i32_ret_offset_scalar(ptr addrspace(1)
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
 ; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT:    s_cbranch_execnz .LBB71_1
+; VI-NEXT:    s_cbranch_execnz .LBB74_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -3807,7 +3918,7 @@ define amdgpu_gfx i32 @global_atomic_max_i32_ret_offset_scalar(ptr addrspace(1)
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    global_load_dword v0, v1, s[4:5] offset:16
 ; GFX9-NEXT:    s_mov_b64 s[34:35], 0
-; GFX9-NEXT:  .LBB71_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB74_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v3, v0
@@ -3818,7 +3929,7 @@ define amdgpu_gfx i32 @global_atomic_max_i32_ret_offset_scalar(ptr addrspace(1)
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
 ; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT:    s_cbranch_execnz .LBB71_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB74_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -3843,7 +3954,7 @@ define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr addrspace(1) %out, i
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v1, s3
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:  .LBB72_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB75_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    v_max_i32_e32 v0, s2, v1
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -3856,7 +3967,7 @@ define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr addrspace(1) %out, i
 ; SI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
 ; SI-NEXT:    v_mov_b32_e32 v1, v2
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[0:1]
-; SI-NEXT:    s_cbranch_execnz .LBB72_1
+; SI-NEXT:    s_cbranch_execnz .LBB75_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_endpgm
 ;
@@ -3877,7 +3988,7 @@ define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr addrspace(1) %out, i
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v3, s3
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
-; VI-NEXT:  .LBB72_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB75_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    v_max_i32_e32 v2, s2, v3
 ; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
@@ -3887,7 +3998,7 @@ define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr addrspace(1) %out, i
 ; VI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v3, v2
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[0:1]
-; VI-NEXT:    s_cbranch_execnz .LBB72_1
+; VI-NEXT:    s_cbranch_execnz .LBB75_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_endpgm
 ;
@@ -3905,7 +4016,7 @@ define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr addrspace(1) %out, i
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s3
-; GFX9-NEXT:  .LBB72_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB75_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    v_max_i32_e32 v0, s2, v1
 ; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
@@ -3915,7 +4026,7 @@ define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr addrspace(1) %out, i
 ; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB72_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB75_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_endpgm
 entry:
@@ -3942,7 +4053,7 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64_offset(ptr addrspace(1) %ou
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v1, s6
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:  .LBB73_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB76_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    v_max_i32_e32 v0, s8, v1
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -3955,7 +4066,7 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64_offset(ptr addrspace(1) %ou
 ; SI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
 ; SI-NEXT:    v_mov_b32_e32 v1, v2
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[0:1]
-; SI-NEXT:    s_cbranch_execnz .LBB73_1
+; SI-NEXT:    s_cbranch_execnz .LBB76_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
@@ -3983,7 +4094,7 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64_offset(ptr addrspace(1) %ou
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v2, s5
 ; VI-NEXT:    v_mov_b32_e32 v1, s7
-; VI-NEXT:  .LBB73_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB76_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    v_mov_b32_e32 v3, v2
 ; VI-NEXT:    v_max_i32_e32 v2, s4, v3
@@ -3993,7 +4104,7 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64_offset(ptr addrspace(1) %ou
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
 ; VI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[0:1]
-; VI-NEXT:    s_cbranch_execnz .LBB73_1
+; VI-NEXT:    s_cbranch_execnz .LBB76_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
@@ -4016,7 +4127,7 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64_offset(ptr addrspace(1) %ou
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s3
-; GFX9-NEXT:  .LBB73_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB76_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    v_mov_b32_e32 v3, v0
 ; GFX9-NEXT:    v_max_i32_e32 v2, s2, v3
@@ -4026,7 +4137,7 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64_offset(ptr addrspace(1) %ou
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
 ; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB73_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB76_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
@@ -4056,7 +4167,7 @@ define amdgpu_kernel void @atomic_max_i32_addr64(ptr addrspace(1) %out, i32 %in,
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v1, s3
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:  .LBB74_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB77_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    v_max_i32_e32 v0, s2, v1
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -4069,7 +4180,7 @@ define amdgpu_kernel void @atomic_max_i32_addr64(ptr addrspace(1) %out, i32 %in,
 ; SI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
 ; SI-NEXT:    v_mov_b32_e32 v1, v2
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[0:1]
-; SI-NEXT:    s_cbranch_execnz .LBB74_1
+; SI-NEXT:    s_cbranch_execnz .LBB77_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_endpgm
 ;
@@ -4088,7 +4199,7 @@ define amdgpu_kernel void @atomic_max_i32_addr64(ptr addrspace(1) %out, i32 %in,
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v3, s3
-; VI-NEXT:  .LBB74_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB77_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    v_max_i32_e32 v2, s2, v3
 ; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
@@ -4098,7 +4209,7 @@ define amdgpu_kernel void @atomic_max_i32_addr64(ptr addrspace(1) %out, i32 %in,
 ; VI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v3, v2
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[0:1]
-; VI-NEXT:    s_cbranch_execnz .LBB74_1
+; VI-NEXT:    s_cbranch_execnz .LBB77_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_endpgm
 ;
@@ -4116,7 +4227,7 @@ define amdgpu_kernel void @atomic_max_i32_addr64(ptr addrspace(1) %out, i32 %in,
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s3
-; GFX9-NEXT:  .LBB74_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB77_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    v_max_i32_e32 v0, s2, v1
 ; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
@@ -4126,7 +4237,7 @@ define amdgpu_kernel void @atomic_max_i32_addr64(ptr addrspace(1) %out, i32 %in,
 ; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB74_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB77_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_endpgm
 entry:
@@ -4152,7 +4263,7 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v1, s6
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:  .LBB75_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB78_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    v_max_i32_e32 v0, s8, v1
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -4165,7 +4276,7 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; SI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
 ; SI-NEXT:    v_mov_b32_e32 v1, v2
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[0:1]
-; SI-NEXT:    s_cbranch_execnz .LBB75_1
+; SI-NEXT:    s_cbranch_execnz .LBB78_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
@@ -4191,7 +4302,7 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; VI-NEXT:    v_mov_b32_e32 v1, s7
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v2, s5
-; VI-NEXT:  .LBB75_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB78_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    v_mov_b32_e32 v3, v2
 ; VI-NEXT:    v_max_i32_e32 v2, s4, v3
@@ -4201,7 +4312,7 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
 ; VI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[0:1]
-; VI-NEXT:    s_cbranch_execnz .LBB75_1
+; VI-NEXT:    s_cbranch_execnz .LBB78_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
@@ -4224,7 +4335,7 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s3
-; GFX9-NEXT:  .LBB75_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB78_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    v_mov_b32_e32 v3, v0
 ; GFX9-NEXT:    v_max_i32_e32 v2, s2, v3
@@ -4234,7 +4345,7 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
 ; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB75_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB78_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
@@ -4261,7 +4372,7 @@ define void @global_atomic_umax_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; SI-NEXT:    s_mov_b32 s5, s6
 ; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
 ; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB76_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB79_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_max_u32_e32 v3, v4, v2
@@ -4275,7 +4386,7 @@ define void @global_atomic_umax_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
 ; SI-NEXT:    v_mov_b32_e32 v4, v5
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB76_1
+; SI-NEXT:    s_cbranch_execnz .LBB79_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -4286,7 +4397,7 @@ define void @global_atomic_umax_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    flat_load_dword v4, v[0:1]
 ; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB76_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB79_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_max_u32_e32 v3, v4, v2
@@ -4297,7 +4408,7 @@ define void @global_atomic_umax_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; VI-NEXT:    v_mov_b32_e32 v4, v3
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB76_1
+; VI-NEXT:    s_cbranch_execnz .LBB79_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -4307,7 +4418,7 @@ define void @global_atomic_umax_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    global_load_dword v4, v[0:1], off
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB76_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB79_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_max_u32_e32 v3, v4, v2
@@ -4318,7 +4429,7 @@ define void @global_atomic_umax_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v4, v3
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB76_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB79_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -4336,7 +4447,7 @@ define void @global_atomic_umax_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; SI-NEXT:    s_mov_b32 s5, s6
 ; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:16
 ; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB77_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB80_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_max_u32_e32 v3, v4, v2
@@ -4350,7 +4461,7 @@ define void @global_atomic_umax_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
 ; SI-NEXT:    v_mov_b32_e32 v4, v5
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB77_1
+; SI-NEXT:    s_cbranch_execnz .LBB80_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -4363,7 +4474,7 @@ define void @global_atomic_umax_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dword v4, v[0:1]
 ; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB77_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB80_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_max_u32_e32 v3, v4, v2
@@ -4374,7 +4485,7 @@ define void @global_atomic_umax_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; VI-NEXT:    v_mov_b32_e32 v4, v3
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB77_1
+; VI-NEXT:    s_cbranch_execnz .LBB80_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -4384,7 +4495,7 @@ define void @global_atomic_umax_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:16
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB77_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB80_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_max_u32_e32 v3, v4, v2
@@ -4395,7 +4506,7 @@ define void @global_atomic_umax_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v4, v3
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB77_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB80_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -4414,7 +4525,7 @@ define i32 @global_atomic_umax_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; SI-NEXT:    s_mov_b32 s5, s6
 ; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
 ; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB78_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB81_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v5, v3
@@ -4428,7 +4539,7 @@ define i32 @global_atomic_umax_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
 ; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB78_1
+; SI-NEXT:    s_cbranch_execnz .LBB81_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    v_mov_b32_e32 v0, v3
@@ -4440,7 +4551,7 @@ define i32 @global_atomic_umax_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    flat_load_dword v3, v[0:1]
 ; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB78_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB81_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v4, v3
@@ -4451,7 +4562,7 @@ define i32 @global_atomic_umax_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
 ; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB78_1
+; VI-NEXT:    s_cbranch_execnz .LBB81_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    v_mov_b32_e32 v0, v3
@@ -4462,7 +4573,7 @@ define i32 @global_atomic_umax_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    global_load_dword v3, v[0:1], off
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB78_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB81_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v4, v3
@@ -4473,7 +4584,7 @@ define i32 @global_atomic_umax_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
 ; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB78_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB81_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, v3
@@ -4492,7 +4603,7 @@ define i32 @global_atomic_umax_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; SI-NEXT:    s_mov_b32 s5, s6
 ; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:16
 ; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB79_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB82_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v5, v3
@@ -4506,7 +4617,7 @@ define i32 @global_atomic_umax_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
 ; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB79_1
+; SI-NEXT:    s_cbranch_execnz .LBB82_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    v_mov_b32_e32 v0, v3
@@ -4520,7 +4631,7 @@ define i32 @global_atomic_umax_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dword v0, v[3:4]
 ; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB79_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB82_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, v0
@@ -4531,7 +4642,7 @@ define i32 @global_atomic_umax_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
 ; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB79_1
+; VI-NEXT:    s_cbranch_execnz .LBB82_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -4541,7 +4652,7 @@ define i32 @global_atomic_umax_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    global_load_dword v3, v[0:1], off offset:16
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB79_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB82_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v4, v3
@@ -4552,7 +4663,7 @@ define i32 @global_atomic_umax_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
 ; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB79_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB82_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, v3
@@ -4577,7 +4688,7 @@ define amdgpu_gfx void @global_atomic_umax_i32_noret_scalar(ptr addrspace(1) inr
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0
 ; SI-NEXT:    s_mov_b64 s[36:37], 0
-; SI-NEXT:  .LBB80_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB83_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_max_u32_e32 v0, s34, v1
@@ -4591,7 +4702,7 @@ define amdgpu_gfx void @global_atomic_umax_i32_noret_scalar(ptr addrspace(1) inr
 ; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
 ; SI-NEXT:    v_mov_b32_e32 v1, v2
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT:    s_cbranch_execnz .LBB80_1
+; SI-NEXT:    s_cbranch_execnz .LBB83_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; SI-NEXT:    v_readlane_b32 s7, v4, 1
@@ -4609,7 +4720,7 @@ define amdgpu_gfx void @global_atomic_umax_i32_noret_scalar(ptr addrspace(1) inr
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
 ; VI-NEXT:    flat_load_dword v3, v[0:1]
 ; VI-NEXT:    s_mov_b64 s[34:35], 0
-; VI-NEXT:  .LBB80_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB83_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_max_u32_e32 v2, s6, v3
@@ -4620,7 +4731,7 @@ define amdgpu_gfx void @global_atomic_umax_i32_noret_scalar(ptr addrspace(1) inr
 ; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; VI-NEXT:    v_mov_b32_e32 v3, v2
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT:    s_cbranch_execnz .LBB80_1
+; VI-NEXT:    s_cbranch_execnz .LBB83_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -4631,7 +4742,7 @@ define amdgpu_gfx void @global_atomic_umax_i32_noret_scalar(ptr addrspace(1) inr
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    global_load_dword v1, v2, s[4:5]
 ; GFX9-NEXT:    s_mov_b64 s[34:35], 0
-; GFX9-NEXT:  .LBB80_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB83_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_max_u32_e32 v0, s6, v1
@@ -4642,7 +4753,7 @@ define amdgpu_gfx void @global_atomic_umax_i32_noret_scalar(ptr addrspace(1) inr
 ; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; GFX9-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT:    s_cbranch_execnz .LBB80_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB83_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -4665,7 +4776,7 @@ define amdgpu_gfx void @global_atomic_umax_i32_noret_offset_scalar(ptr addrspace
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0 offset:16
 ; SI-NEXT:    s_mov_b64 s[36:37], 0
-; SI-NEXT:  .LBB81_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB84_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_max_u32_e32 v0, s34, v1
@@ -4679,7 +4790,7 @@ define amdgpu_gfx void @global_atomic_umax_i32_noret_offset_scalar(ptr addrspace
 ; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
 ; SI-NEXT:    v_mov_b32_e32 v1, v2
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT:    s_cbranch_execnz .LBB81_1
+; SI-NEXT:    s_cbranch_execnz .LBB84_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; SI-NEXT:    v_readlane_b32 s7, v4, 1
@@ -4699,7 +4810,7 @@ define amdgpu_gfx void @global_atomic_umax_i32_noret_offset_scalar(ptr addrspace
 ; VI-NEXT:    v_mov_b32_e32 v1, s35
 ; VI-NEXT:    flat_load_dword v3, v[0:1]
 ; VI-NEXT:    s_mov_b64 s[34:35], 0
-; VI-NEXT:  .LBB81_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB84_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_max_u32_e32 v2, s6, v3
@@ -4710,7 +4821,7 @@ define amdgpu_gfx void @global_atomic_umax_i32_noret_offset_scalar(ptr addrspace
 ; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; VI-NEXT:    v_mov_b32_e32 v3, v2
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT:    s_cbranch_execnz .LBB81_1
+; VI-NEXT:    s_cbranch_execnz .LBB84_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -4721,7 +4832,7 @@ define amdgpu_gfx void @global_atomic_umax_i32_noret_offset_scalar(ptr addrspace
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    global_load_dword v1, v2, s[4:5] offset:16
 ; GFX9-NEXT:    s_mov_b64 s[34:35], 0
-; GFX9-NEXT:  .LBB81_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB84_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_max_u32_e32 v0, s6, v1
@@ -4732,7 +4843,7 @@ define amdgpu_gfx void @global_atomic_umax_i32_noret_offset_scalar(ptr addrspace
 ; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; GFX9-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT:    s_cbranch_execnz .LBB81_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB84_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -4756,7 +4867,7 @@ define amdgpu_gfx i32 @global_atomic_umax_i32_ret_scalar(ptr addrspace(1) inreg
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0
 ; SI-NEXT:    s_mov_b64 s[36:37], 0
-; SI-NEXT:  .LBB82_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB85_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v2, v0
@@ -4770,7 +4881,7 @@ define amdgpu_gfx i32 @global_atomic_umax_i32_ret_scalar(ptr addrspace(1) inreg
 ; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
 ; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT:    s_cbranch_execnz .LBB82_1
+; SI-NEXT:    s_cbranch_execnz .LBB85_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; SI-NEXT:    v_readlane_b32 s7, v3, 1
@@ -4790,7 +4901,7 @@ define amdgpu_gfx i32 @global_atomic_umax_i32_ret_scalar(ptr addrspace(1) inreg
 ; VI-NEXT:    v_mov_b32_e32 v1, s4
 ; VI-NEXT:    s_mov_b64 s[34:35], 0
 ; VI-NEXT:    v_mov_b32_e32 v2, s5
-; VI-NEXT:  .LBB82_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB85_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v4, v0
@@ -4801,7 +4912,7 @@ define amdgpu_gfx i32 @global_atomic_umax_i32_ret_scalar(ptr addrspace(1) inreg
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
 ; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT:    s_cbranch_execnz .LBB82_1
+; VI-NEXT:    s_cbranch_execnz .LBB85_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -4812,7 +4923,7 @@ define amdgpu_gfx i32 @global_atomic_umax_i32_ret_scalar(ptr addrspace(1) inreg
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    global_load_dword v0, v1, s[4:5]
 ; GFX9-NEXT:    s_mov_b64 s[34:35], 0
-; GFX9-NEXT:  .LBB82_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB85_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v3, v0
@@ -4823,7 +4934,7 @@ define amdgpu_gfx i32 @global_atomic_umax_i32_ret_scalar(ptr addrspace(1) inreg
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
 ; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT:    s_cbranch_execnz .LBB82_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB85_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -4846,7 +4957,7 @@ define amdgpu_gfx i32 @global_atomic_umax_i32_ret_offset_scalar(ptr addrspace(1)
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0 offset:16
 ; SI-NEXT:    s_mov_b64 s[36:37], 0
-; SI-NEXT:  .LBB83_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB86_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v2, v0
@@ -4860,7 +4971,7 @@ define amdgpu_gfx i32 @global_atomic_umax_i32_ret_offset_scalar(ptr addrspace(1)
 ; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
 ; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT:    s_cbranch_execnz .LBB83_1
+; SI-NEXT:    s_cbranch_execnz .LBB86_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; SI-NEXT:    v_readlane_b32 s7, v3, 1
@@ -4880,7 +4991,7 @@ define amdgpu_gfx i32 @global_atomic_umax_i32_ret_offset_scalar(ptr addrspace(1)
 ; VI-NEXT:    v_mov_b32_e32 v2, s35
 ; VI-NEXT:    flat_load_dword v0, v[1:2]
 ; VI-NEXT:    s_mov_b64 s[34:35], 0
-; VI-NEXT:  .LBB83_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB86_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v4, v0
@@ -4891,7 +5002,7 @@ define amdgpu_gfx i32 @global_atomic_umax_i32_ret_offset_scalar(ptr addrspace(1)
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
 ; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT:    s_cbranch_execnz .LBB83_1
+; VI-NEXT:    s_cbranch_execnz .LBB86_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -4902,7 +5013,7 @@ define amdgpu_gfx i32 @global_atomic_umax_i32_ret_offset_scalar(ptr addrspace(1)
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    global_load_dword v0, v1, s[4:5] offset:16
 ; GFX9-NEXT:    s_mov_b64 s[34:35], 0
-; GFX9-NEXT:  .LBB83_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB86_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v3, v0
@@ -4913,7 +5024,7 @@ define amdgpu_gfx i32 @global_atomic_umax_i32_ret_offset_scalar(ptr addrspace(1)
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
 ; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT:    s_cbranch_execnz .LBB83_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB86_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -4938,7 +5049,7 @@ define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr addrspace(1) %out,
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v1, s3
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:  .LBB84_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB87_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    v_max_u32_e32 v0, s2, v1
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -4951,7 +5062,7 @@ define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr addrspace(1) %out,
 ; SI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
 ; SI-NEXT:    v_mov_b32_e32 v1, v2
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[0:1]
-; SI-NEXT:    s_cbranch_execnz .LBB84_1
+; SI-NEXT:    s_cbranch_execnz .LBB87_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_endpgm
 ;
@@ -4972,7 +5083,7 @@ define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr addrspace(1) %out,
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v3, s3
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
-; VI-NEXT:  .LBB84_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB87_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    v_max_u32_e32 v2, s2, v3
 ; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
@@ -4982,7 +5093,7 @@ define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr addrspace(1) %out,
 ; VI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v3, v2
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[0:1]
-; VI-NEXT:    s_cbranch_execnz .LBB84_1
+; VI-NEXT:    s_cbranch_execnz .LBB87_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_endpgm
 ;
@@ -5000,7 +5111,7 @@ define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr addrspace(1) %out,
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s3
-; GFX9-NEXT:  .LBB84_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB87_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    v_max_u32_e32 v0, s2, v1
 ; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
@@ -5010,7 +5121,7 @@ define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr addrspace(1) %out,
 ; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB84_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB87_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_endpgm
 entry:
@@ -5037,7 +5148,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(ptr addrspace(1) %o
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v1, s6
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:  .LBB85_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB88_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    v_max_u32_e32 v0, s8, v1
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -5050,7 +5161,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(ptr addrspace(1) %o
 ; SI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
 ; SI-NEXT:    v_mov_b32_e32 v1, v2
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[0:1]
-; SI-NEXT:    s_cbranch_execnz .LBB85_1
+; SI-NEXT:    s_cbranch_execnz .LBB88_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
@@ -5078,7 +5189,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(ptr addrspace(1) %o
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v2, s5
 ; VI-NEXT:    v_mov_b32_e32 v1, s7
-; VI-NEXT:  .LBB85_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB88_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    v_mov_b32_e32 v3, v2
 ; VI-NEXT:    v_max_u32_e32 v2, s4, v3
@@ -5088,7 +5199,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(ptr addrspace(1) %o
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
 ; VI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[0:1]
-; VI-NEXT:    s_cbranch_execnz .LBB85_1
+; VI-NEXT:    s_cbranch_execnz .LBB88_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
@@ -5111,7 +5222,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(ptr addrspace(1) %o
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s3
-; GFX9-NEXT:  .LBB85_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB88_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    v_mov_b32_e32 v3, v0
 ; GFX9-NEXT:    v_max_u32_e32 v2, s2, v3
@@ -5121,7 +5232,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(ptr addrspace(1) %o
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
 ; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB85_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB88_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
@@ -5152,7 +5263,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v1, s6
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:  .LBB86_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB89_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    v_max_u32_e32 v0, s8, v1
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -5165,7 +5276,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; SI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
 ; SI-NEXT:    v_mov_b32_e32 v1, v2
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[0:1]
-; SI-NEXT:    s_cbranch_execnz .LBB86_1
+; SI-NEXT:    s_cbranch_execnz .LBB89_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
@@ -5191,7 +5302,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; VI-NEXT:    v_mov_b32_e32 v1, s7
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v2, s5
-; VI-NEXT:  .LBB86_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB89_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    v_mov_b32_e32 v3, v2
 ; VI-NEXT:    v_max_u32_e32 v2, s4, v3
@@ -5201,7 +5312,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
 ; VI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[0:1]
-; VI-NEXT:    s_cbranch_execnz .LBB86_1
+; VI-NEXT:    s_cbranch_execnz .LBB89_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
@@ -5224,7 +5335,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s3
-; GFX9-NEXT:  .LBB86_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB89_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    v_mov_b32_e32 v3, v0
 ; GFX9-NEXT:    v_max_u32_e32 v2, s2, v3
@@ -5234,7 +5345,7 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
 ; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB86_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB89_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
@@ -5261,7 +5372,7 @@ define void @global_atomic_umin_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; SI-NEXT:    s_mov_b32 s5, s6
 ; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
 ; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB87_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB90_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_min_u32_e32 v3, v4, v2
@@ -5275,7 +5386,7 @@ define void @global_atomic_umin_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
 ; SI-NEXT:    v_mov_b32_e32 v4, v5
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB87_1
+; SI-NEXT:    s_cbranch_execnz .LBB90_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -5286,7 +5397,7 @@ define void @global_atomic_umin_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    flat_load_dword v4, v[0:1]
 ; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB87_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB90_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_min_u32_e32 v3, v4, v2
@@ -5297,7 +5408,7 @@ define void @global_atomic_umin_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; VI-NEXT:    v_mov_b32_e32 v4, v3
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB87_1
+; VI-NEXT:    s_cbranch_execnz .LBB90_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -5307,7 +5418,7 @@ define void @global_atomic_umin_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    global_load_dword v4, v[0:1], off
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB87_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB90_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_min_u32_e32 v3, v4, v2
@@ -5318,7 +5429,7 @@ define void @global_atomic_umin_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v4, v3
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB87_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB90_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -5336,7 +5447,7 @@ define void @global_atomic_umin_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; SI-NEXT:    s_mov_b32 s5, s6
 ; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:16
 ; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB88_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB91_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_min_u32_e32 v3, v4, v2
@@ -5350,7 +5461,7 @@ define void @global_atomic_umin_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
 ; SI-NEXT:    v_mov_b32_e32 v4, v5
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB88_1
+; SI-NEXT:    s_cbranch_execnz .LBB91_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -5363,7 +5474,7 @@ define void @global_atomic_umin_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dword v4, v[0:1]
 ; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB88_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB91_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_min_u32_e32 v3, v4, v2
@@ -5374,7 +5485,7 @@ define void @global_atomic_umin_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; VI-NEXT:    v_mov_b32_e32 v4, v3
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB88_1
+; VI-NEXT:    s_cbranch_execnz .LBB91_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -5384,7 +5495,7 @@ define void @global_atomic_umin_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:16
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB88_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB91_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_min_u32_e32 v3, v4, v2
@@ -5395,7 +5506,7 @@ define void @global_atomic_umin_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v4, v3
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB88_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB91_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -5414,7 +5525,7 @@ define i32 @global_atomic_umin_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; SI-NEXT:    s_mov_b32 s5, s6
 ; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
 ; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB89_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB92_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v5, v3
@@ -5428,7 +5539,7 @@ define i32 @global_atomic_umin_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
 ; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB89_1
+; SI-NEXT:    s_cbranch_execnz .LBB92_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    v_mov_b32_e32 v0, v3
@@ -5440,7 +5551,7 @@ define i32 @global_atomic_umin_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    flat_load_dword v3, v[0:1]
 ; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB89_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB92_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v4, v3
@@ -5451,7 +5562,7 @@ define i32 @global_atomic_umin_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
 ; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB89_1
+; VI-NEXT:    s_cbranch_execnz .LBB92_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    v_mov_b32_e32 v0, v3
@@ -5462,7 +5573,7 @@ define i32 @global_atomic_umin_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    global_load_dword v3, v[0:1], off
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB89_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB92_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v4, v3
@@ -5473,7 +5584,7 @@ define i32 @global_atomic_umin_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
 ; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB89_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB92_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, v3
@@ -5492,7 +5603,7 @@ define i32 @global_atomic_umin_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; SI-NEXT:    s_mov_b32 s5, s6
 ; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:16
 ; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB90_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB93_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v5, v3
@@ -5506,7 +5617,7 @@ define i32 @global_atomic_umin_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
 ; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB90_1
+; SI-NEXT:    s_cbranch_execnz .LBB93_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    v_mov_b32_e32 v0, v3
@@ -5520,7 +5631,7 @@ define i32 @global_atomic_umin_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dword v0, v[3:4]
 ; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB90_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB93_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, v0
@@ -5531,7 +5642,7 @@ define i32 @global_atomic_umin_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
 ; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB90_1
+; VI-NEXT:    s_cbranch_execnz .LBB93_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -5541,7 +5652,7 @@ define i32 @global_atomic_umin_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    global_load_dword v3, v[0:1], off offset:16
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB90_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB93_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v4, v3
@@ -5552,7 +5663,7 @@ define i32 @global_atomic_umin_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
 ; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB90_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB93_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, v3
@@ -5577,7 +5688,7 @@ define amdgpu_gfx void @global_atomic_umin_i32_noret_scalar(ptr addrspace(1) inr
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0
 ; SI-NEXT:    s_mov_b64 s[36:37], 0
-; SI-NEXT:  .LBB91_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB94_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_min_u32_e32 v0, s34, v1
@@ -5591,7 +5702,7 @@ define amdgpu_gfx void @global_atomic_umin_i32_noret_scalar(ptr addrspace(1) inr
 ; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
 ; SI-NEXT:    v_mov_b32_e32 v1, v2
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT:    s_cbranch_execnz .LBB91_1
+; SI-NEXT:    s_cbranch_execnz .LBB94_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; SI-NEXT:    v_readlane_b32 s7, v4, 1
@@ -5609,7 +5720,7 @@ define amdgpu_gfx void @global_atomic_umin_i32_noret_scalar(ptr addrspace(1) inr
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
 ; VI-NEXT:    flat_load_dword v3, v[0:1]
 ; VI-NEXT:    s_mov_b64 s[34:35], 0
-; VI-NEXT:  .LBB91_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB94_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_min_u32_e32 v2, s6, v3
@@ -5620,7 +5731,7 @@ define amdgpu_gfx void @global_atomic_umin_i32_noret_scalar(ptr addrspace(1) inr
 ; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; VI-NEXT:    v_mov_b32_e32 v3, v2
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT:    s_cbranch_execnz .LBB91_1
+; VI-NEXT:    s_cbranch_execnz .LBB94_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -5631,7 +5742,7 @@ define amdgpu_gfx void @global_atomic_umin_i32_noret_scalar(ptr addrspace(1) inr
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    global_load_dword v1, v2, s[4:5]
 ; GFX9-NEXT:    s_mov_b64 s[34:35], 0
-; GFX9-NEXT:  .LBB91_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB94_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_min_u32_e32 v0, s6, v1
@@ -5642,7 +5753,7 @@ define amdgpu_gfx void @global_atomic_umin_i32_noret_scalar(ptr addrspace(1) inr
 ; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; GFX9-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT:    s_cbranch_execnz .LBB91_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB94_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -5665,7 +5776,7 @@ define amdgpu_gfx void @global_atomic_umin_i32_noret_offset_scalar(ptr addrspace
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0 offset:16
 ; SI-NEXT:    s_mov_b64 s[36:37], 0
-; SI-NEXT:  .LBB92_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB95_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_min_u32_e32 v0, s34, v1
@@ -5679,7 +5790,7 @@ define amdgpu_gfx void @global_atomic_umin_i32_noret_offset_scalar(ptr addrspace
 ; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
 ; SI-NEXT:    v_mov_b32_e32 v1, v2
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT:    s_cbranch_execnz .LBB92_1
+; SI-NEXT:    s_cbranch_execnz .LBB95_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; SI-NEXT:    v_readlane_b32 s7, v4, 1
@@ -5699,7 +5810,7 @@ define amdgpu_gfx void @global_atomic_umin_i32_noret_offset_scalar(ptr addrspace
 ; VI-NEXT:    v_mov_b32_e32 v1, s35
 ; VI-NEXT:    flat_load_dword v3, v[0:1]
 ; VI-NEXT:    s_mov_b64 s[34:35], 0
-; VI-NEXT:  .LBB92_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB95_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_min_u32_e32 v2, s6, v3
@@ -5710,7 +5821,7 @@ define amdgpu_gfx void @global_atomic_umin_i32_noret_offset_scalar(ptr addrspace
 ; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; VI-NEXT:    v_mov_b32_e32 v3, v2
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT:    s_cbranch_execnz .LBB92_1
+; VI-NEXT:    s_cbranch_execnz .LBB95_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -5721,7 +5832,7 @@ define amdgpu_gfx void @global_atomic_umin_i32_noret_offset_scalar(ptr addrspace
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    global_load_dword v1, v2, s[4:5] offset:16
 ; GFX9-NEXT:    s_mov_b64 s[34:35], 0
-; GFX9-NEXT:  .LBB92_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB95_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_min_u32_e32 v0, s6, v1
@@ -5732,7 +5843,7 @@ define amdgpu_gfx void @global_atomic_umin_i32_noret_offset_scalar(ptr addrspace
 ; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; GFX9-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT:    s_cbranch_execnz .LBB92_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB95_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -5756,7 +5867,7 @@ define amdgpu_gfx i32 @global_atomic_umin_i32_ret_scalar(ptr addrspace(1) inreg
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0
 ; SI-NEXT:    s_mov_b64 s[36:37], 0
-; SI-NEXT:  .LBB93_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB96_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v2, v0
@@ -5770,7 +5881,7 @@ define amdgpu_gfx i32 @global_atomic_umin_i32_ret_scalar(ptr addrspace(1) inreg
 ; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
 ; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT:    s_cbranch_execnz .LBB93_1
+; SI-NEXT:    s_cbranch_execnz .LBB96_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; SI-NEXT:    v_readlane_b32 s7, v3, 1
@@ -5790,7 +5901,7 @@ define amdgpu_gfx i32 @global_atomic_umin_i32_ret_scalar(ptr addrspace(1) inreg
 ; VI-NEXT:    v_mov_b32_e32 v1, s4
 ; VI-NEXT:    s_mov_b64 s[34:35], 0
 ; VI-NEXT:    v_mov_b32_e32 v2, s5
-; VI-NEXT:  .LBB93_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB96_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v4, v0
@@ -5801,7 +5912,7 @@ define amdgpu_gfx i32 @global_atomic_umin_i32_ret_scalar(ptr addrspace(1) inreg
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
 ; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT:    s_cbranch_execnz .LBB93_1
+; VI-NEXT:    s_cbranch_execnz .LBB96_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -5812,7 +5923,7 @@ define amdgpu_gfx i32 @global_atomic_umin_i32_ret_scalar(ptr addrspace(1) inreg
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    global_load_dword v0, v1, s[4:5]
 ; GFX9-NEXT:    s_mov_b64 s[34:35], 0
-; GFX9-NEXT:  .LBB93_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB96_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v3, v0
@@ -5823,7 +5934,7 @@ define amdgpu_gfx i32 @global_atomic_umin_i32_ret_scalar(ptr addrspace(1) inreg
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
 ; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT:    s_cbranch_execnz .LBB93_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB96_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -5846,7 +5957,7 @@ define amdgpu_gfx i32 @global_atomic_umin_i32_ret_offset_scalar(ptr addrspace(1)
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0 offset:16
 ; SI-NEXT:    s_mov_b64 s[36:37], 0
-; SI-NEXT:  .LBB94_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB97_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v2, v0
@@ -5860,7 +5971,7 @@ define amdgpu_gfx i32 @global_atomic_umin_i32_ret_offset_scalar(ptr addrspace(1)
 ; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
 ; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT:    s_cbranch_execnz .LBB94_1
+; SI-NEXT:    s_cbranch_execnz .LBB97_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; SI-NEXT:    v_readlane_b32 s7, v3, 1
@@ -5880,7 +5991,7 @@ define amdgpu_gfx i32 @global_atomic_umin_i32_ret_offset_scalar(ptr addrspace(1)
 ; VI-NEXT:    v_mov_b32_e32 v2, s35
 ; VI-NEXT:    flat_load_dword v0, v[1:2]
 ; VI-NEXT:    s_mov_b64 s[34:35], 0
-; VI-NEXT:  .LBB94_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB97_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v4, v0
@@ -5891,7 +6002,7 @@ define amdgpu_gfx i32 @global_atomic_umin_i32_ret_offset_scalar(ptr addrspace(1)
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
 ; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT:    s_cbranch_execnz .LBB94_1
+; VI-NEXT:    s_cbranch_execnz .LBB97_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -5902,7 +6013,7 @@ define amdgpu_gfx i32 @global_atomic_umin_i32_ret_offset_scalar(ptr addrspace(1)
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    global_load_dword v0, v1, s[4:5] offset:16
 ; GFX9-NEXT:    s_mov_b64 s[34:35], 0
-; GFX9-NEXT:  .LBB94_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB97_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v3, v0
@@ -5913,7 +6024,7 @@ define amdgpu_gfx i32 @global_atomic_umin_i32_ret_offset_scalar(ptr addrspace(1)
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
 ; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT:    s_cbranch_execnz .LBB94_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB97_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -5936,7 +6047,7 @@ define void @global_atomic_min_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; SI-NEXT:    s_mov_b32 s5, s6
 ; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64
 ; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB95_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB98_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_min_i32_e32 v3, v4, v2
@@ -5950,7 +6061,7 @@ define void @global_atomic_min_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
 ; SI-NEXT:    v_mov_b32_e32 v4, v5
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB95_1
+; SI-NEXT:    s_cbranch_execnz .LBB98_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -5961,7 +6072,7 @@ define void @global_atomic_min_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    flat_load_dword v4, v[0:1]
 ; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB95_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB98_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_min_i32_e32 v3, v4, v2
@@ -5972,7 +6083,7 @@ define void @global_atomic_min_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; VI-NEXT:    v_mov_b32_e32 v4, v3
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB95_1
+; VI-NEXT:    s_cbranch_execnz .LBB98_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -5982,7 +6093,7 @@ define void @global_atomic_min_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    global_load_dword v4, v[0:1], off
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB95_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB98_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_min_i32_e32 v3, v4, v2
@@ -5993,7 +6104,7 @@ define void @global_atomic_min_i32_noret(ptr addrspace(1) %ptr, i32 %in) {
 ; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v4, v3
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB95_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB98_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -6011,7 +6122,7 @@ define void @global_atomic_min_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; SI-NEXT:    s_mov_b32 s5, s6
 ; SI-NEXT:    buffer_load_dword v4, v[0:1], s[4:7], 0 addr64 offset:16
 ; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB96_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB99_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_min_i32_e32 v3, v4, v2
@@ -6025,7 +6136,7 @@ define void @global_atomic_min_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
 ; SI-NEXT:    v_mov_b32_e32 v4, v5
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB96_1
+; SI-NEXT:    s_cbranch_execnz .LBB99_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -6038,7 +6149,7 @@ define void @global_atomic_min_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dword v4, v[0:1]
 ; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB96_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB99_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_min_i32_e32 v3, v4, v2
@@ -6049,7 +6160,7 @@ define void @global_atomic_min_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; VI-NEXT:    v_mov_b32_e32 v4, v3
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB96_1
+; VI-NEXT:    s_cbranch_execnz .LBB99_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -6059,7 +6170,7 @@ define void @global_atomic_min_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    global_load_dword v4, v[0:1], off offset:16
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB96_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB99_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_min_i32_e32 v3, v4, v2
@@ -6070,7 +6181,7 @@ define void @global_atomic_min_i32_noret_offset(ptr addrspace(1) %out, i32 %in)
 ; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v4, v3
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB96_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB99_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -6089,7 +6200,7 @@ define i32 @global_atomic_min_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; SI-NEXT:    s_mov_b32 s5, s6
 ; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64
 ; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB97_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB100_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v5, v3
@@ -6103,7 +6214,7 @@ define i32 @global_atomic_min_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
 ; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB97_1
+; SI-NEXT:    s_cbranch_execnz .LBB100_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    v_mov_b32_e32 v0, v3
@@ -6115,7 +6226,7 @@ define i32 @global_atomic_min_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; VI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; VI-NEXT:    flat_load_dword v3, v[0:1]
 ; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB97_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB100_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v4, v3
@@ -6126,7 +6237,7 @@ define i32 @global_atomic_min_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
 ; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB97_1
+; VI-NEXT:    s_cbranch_execnz .LBB100_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    v_mov_b32_e32 v0, v3
@@ -6137,7 +6248,7 @@ define i32 @global_atomic_min_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    global_load_dword v3, v[0:1], off
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB97_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB100_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v4, v3
@@ -6148,7 +6259,7 @@ define i32 @global_atomic_min_i32_ret(ptr addrspace(1) %ptr, i32 %in) {
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
 ; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB97_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB100_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, v3
@@ -6167,7 +6278,7 @@ define i32 @global_atomic_min_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; SI-NEXT:    s_mov_b32 s5, s6
 ; SI-NEXT:    buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:16
 ; SI-NEXT:    s_mov_b64 s[8:9], 0
-; SI-NEXT:  .LBB98_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB101_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v5, v3
@@ -6181,7 +6292,7 @@ define i32 @global_atomic_min_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v5
 ; SI-NEXT:    s_or_b64 s[8:9], vcc, s[8:9]
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[8:9]
-; SI-NEXT:    s_cbranch_execnz .LBB98_1
+; SI-NEXT:    s_cbranch_execnz .LBB101_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; SI-NEXT:    v_mov_b32_e32 v0, v3
@@ -6195,7 +6306,7 @@ define i32 @global_atomic_min_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; VI-NEXT:    v_addc_u32_e32 v4, vcc, 0, v1, vcc
 ; VI-NEXT:    flat_load_dword v0, v[3:4]
 ; VI-NEXT:    s_mov_b64 s[4:5], 0
-; VI-NEXT:  .LBB98_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB101_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, v0
@@ -6206,7 +6317,7 @@ define i32 @global_atomic_min_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v1
 ; VI-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; VI-NEXT:    s_cbranch_execnz .LBB98_1
+; VI-NEXT:    s_cbranch_execnz .LBB101_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -6216,7 +6327,7 @@ define i32 @global_atomic_min_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    global_load_dword v3, v[0:1], off offset:16
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
-; GFX9-NEXT:  .LBB98_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB101_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v4, v3
@@ -6227,7 +6338,7 @@ define i32 @global_atomic_min_i32_ret_offset(ptr addrspace(1) %out, i32 %in) {
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v4
 ; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB98_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB101_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, v3
@@ -6252,7 +6363,7 @@ define amdgpu_gfx void @global_atomic_min_i32_noret_scalar(ptr addrspace(1) inre
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0
 ; SI-NEXT:    s_mov_b64 s[36:37], 0
-; SI-NEXT:  .LBB99_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB102_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_min_i32_e32 v0, s34, v1
@@ -6266,7 +6377,7 @@ define amdgpu_gfx void @global_atomic_min_i32_noret_scalar(ptr addrspace(1) inre
 ; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
 ; SI-NEXT:    v_mov_b32_e32 v1, v2
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT:    s_cbranch_execnz .LBB99_1
+; SI-NEXT:    s_cbranch_execnz .LBB102_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; SI-NEXT:    v_readlane_b32 s7, v4, 1
@@ -6284,7 +6395,7 @@ define amdgpu_gfx void @global_atomic_min_i32_noret_scalar(ptr addrspace(1) inre
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
 ; VI-NEXT:    flat_load_dword v3, v[0:1]
 ; VI-NEXT:    s_mov_b64 s[34:35], 0
-; VI-NEXT:  .LBB99_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB102_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_min_i32_e32 v2, s6, v3
@@ -6295,7 +6406,7 @@ define amdgpu_gfx void @global_atomic_min_i32_noret_scalar(ptr addrspace(1) inre
 ; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; VI-NEXT:    v_mov_b32_e32 v3, v2
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT:    s_cbranch_execnz .LBB99_1
+; VI-NEXT:    s_cbranch_execnz .LBB102_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -6306,7 +6417,7 @@ define amdgpu_gfx void @global_atomic_min_i32_noret_scalar(ptr addrspace(1) inre
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    global_load_dword v1, v2, s[4:5]
 ; GFX9-NEXT:    s_mov_b64 s[34:35], 0
-; GFX9-NEXT:  .LBB99_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB102_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_min_i32_e32 v0, s6, v1
@@ -6317,7 +6428,7 @@ define amdgpu_gfx void @global_atomic_min_i32_noret_scalar(ptr addrspace(1) inre
 ; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; GFX9-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT:    s_cbranch_execnz .LBB99_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB102_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -6340,7 +6451,7 @@ define amdgpu_gfx void @global_atomic_min_i32_noret_offset_scalar(ptr addrspace(
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    buffer_load_dword v1, off, s[4:7], 0 offset:16
 ; SI-NEXT:    s_mov_b64 s[36:37], 0
-; SI-NEXT:  .LBB100_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB103_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_min_i32_e32 v0, s34, v1
@@ -6354,7 +6465,7 @@ define amdgpu_gfx void @global_atomic_min_i32_noret_offset_scalar(ptr addrspace(
 ; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
 ; SI-NEXT:    v_mov_b32_e32 v1, v2
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT:    s_cbranch_execnz .LBB100_1
+; SI-NEXT:    s_cbranch_execnz .LBB103_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; SI-NEXT:    v_readlane_b32 s7, v4, 1
@@ -6374,7 +6485,7 @@ define amdgpu_gfx void @global_atomic_min_i32_noret_offset_scalar(ptr addrspace(
 ; VI-NEXT:    v_mov_b32_e32 v1, s35
 ; VI-NEXT:    flat_load_dword v3, v[0:1]
 ; VI-NEXT:    s_mov_b64 s[34:35], 0
-; VI-NEXT:  .LBB100_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB103_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_min_i32_e32 v2, s6, v3
@@ -6385,7 +6496,7 @@ define amdgpu_gfx void @global_atomic_min_i32_noret_offset_scalar(ptr addrspace(
 ; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; VI-NEXT:    v_mov_b32_e32 v3, v2
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT:    s_cbranch_execnz .LBB100_1
+; VI-NEXT:    s_cbranch_execnz .LBB103_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -6396,7 +6507,7 @@ define amdgpu_gfx void @global_atomic_min_i32_noret_offset_scalar(ptr addrspace(
 ; GFX9-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-NEXT:    global_load_dword v1, v2, s[4:5] offset:16
 ; GFX9-NEXT:    s_mov_b64 s[34:35], 0
-; GFX9-NEXT:  .LBB100_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB103_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_min_i32_e32 v0, s6, v1
@@ -6407,7 +6518,7 @@ define amdgpu_gfx void @global_atomic_min_i32_noret_offset_scalar(ptr addrspace(
 ; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; GFX9-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT:    s_cbranch_execnz .LBB100_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB103_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -6431,7 +6542,7 @@ define amdgpu_gfx i32 @global_atomic_min_i32_ret_scalar(ptr addrspace(1) inreg %
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0
 ; SI-NEXT:    s_mov_b64 s[36:37], 0
-; SI-NEXT:  .LBB101_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB104_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v2, v0
@@ -6445,7 +6556,7 @@ define amdgpu_gfx i32 @global_atomic_min_i32_ret_scalar(ptr addrspace(1) inreg %
 ; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
 ; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT:    s_cbranch_execnz .LBB101_1
+; SI-NEXT:    s_cbranch_execnz .LBB104_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; SI-NEXT:    v_readlane_b32 s7, v3, 1
@@ -6465,7 +6576,7 @@ define amdgpu_gfx i32 @global_atomic_min_i32_ret_scalar(ptr addrspace(1) inreg %
 ; VI-NEXT:    v_mov_b32_e32 v1, s4
 ; VI-NEXT:    s_mov_b64 s[34:35], 0
 ; VI-NEXT:    v_mov_b32_e32 v2, s5
-; VI-NEXT:  .LBB101_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB104_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v4, v0
@@ -6476,7 +6587,7 @@ define amdgpu_gfx i32 @global_atomic_min_i32_ret_scalar(ptr addrspace(1) inreg %
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
 ; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT:    s_cbranch_execnz .LBB101_1
+; VI-NEXT:    s_cbranch_execnz .LBB104_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -6487,7 +6598,7 @@ define amdgpu_gfx i32 @global_atomic_min_i32_ret_scalar(ptr addrspace(1) inreg %
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    global_load_dword v0, v1, s[4:5]
 ; GFX9-NEXT:    s_mov_b64 s[34:35], 0
-; GFX9-NEXT:  .LBB101_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB104_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v3, v0
@@ -6498,7 +6609,7 @@ define amdgpu_gfx i32 @global_atomic_min_i32_ret_scalar(ptr addrspace(1) inreg %
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
 ; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT:    s_cbranch_execnz .LBB101_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB104_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -6521,7 +6632,7 @@ define amdgpu_gfx i32 @global_atomic_min_i32_ret_offset_scalar(ptr addrspace(1)
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    buffer_load_dword v0, off, s[4:7], 0 offset:16
 ; SI-NEXT:    s_mov_b64 s[36:37], 0
-; SI-NEXT:  .LBB102_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB105_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v2, v0
@@ -6535,7 +6646,7 @@ define amdgpu_gfx i32 @global_atomic_min_i32_ret_offset_scalar(ptr addrspace(1)
 ; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v2
 ; SI-NEXT:    s_or_b64 s[36:37], vcc, s[36:37]
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[36:37]
-; SI-NEXT:    s_cbranch_execnz .LBB102_1
+; SI-NEXT:    s_cbranch_execnz .LBB105_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[36:37]
 ; SI-NEXT:    v_readlane_b32 s7, v3, 1
@@ -6555,7 +6666,7 @@ define amdgpu_gfx i32 @global_atomic_min_i32_ret_offset_scalar(ptr addrspace(1)
 ; VI-NEXT:    v_mov_b32_e32 v2, s35
 ; VI-NEXT:    flat_load_dword v0, v[1:2]
 ; VI-NEXT:    s_mov_b64 s[34:35], 0
-; VI-NEXT:  .LBB102_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB105_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v4, v0
@@ -6566,7 +6677,7 @@ define amdgpu_gfx i32 @global_atomic_min_i32_ret_offset_scalar(ptr addrspace(1)
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v4
 ; VI-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; VI-NEXT:    s_cbranch_execnz .LBB102_1
+; VI-NEXT:    s_cbranch_execnz .LBB105_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
@@ -6577,7 +6688,7 @@ define amdgpu_gfx i32 @global_atomic_min_i32_ret_offset_scalar(ptr addrspace(1)
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX9-NEXT:    global_load_dword v0, v1, s[4:5] offset:16
 ; GFX9-NEXT:    s_mov_b64 s[34:35], 0
-; GFX9-NEXT:  .LBB102_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB105_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v3, v0
@@ -6588,7 +6699,7 @@ define amdgpu_gfx i32 @global_atomic_min_i32_ret_offset_scalar(ptr addrspace(1)
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
 ; GFX9-NEXT:    s_or_b64 s[34:35], vcc, s[34:35]
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[34:35]
-; GFX9-NEXT:    s_cbranch_execnz .LBB102_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB105_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[34:35]
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
@@ -6613,7 +6724,7 @@ define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr addrspace(1) %out, i
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v1, s3
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:  .LBB103_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB106_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    v_min_i32_e32 v0, s2, v1
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -6626,7 +6737,7 @@ define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr addrspace(1) %out, i
 ; SI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
 ; SI-NEXT:    v_mov_b32_e32 v1, v2
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[0:1]
-; SI-NEXT:    s_cbranch_execnz .LBB103_1
+; SI-NEXT:    s_cbranch_execnz .LBB106_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_endpgm
 ;
@@ -6647,7 +6758,7 @@ define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr addrspace(1) %out, i
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v3, s3
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
-; VI-NEXT:  .LBB103_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB106_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    v_min_i32_e32 v2, s2, v3
 ; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
@@ -6657,7 +6768,7 @@ define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr addrspace(1) %out, i
 ; VI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v3, v2
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[0:1]
-; VI-NEXT:    s_cbranch_execnz .LBB103_1
+; VI-NEXT:    s_cbranch_execnz .LBB106_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_endpgm
 ;
@@ -6675,7 +6786,7 @@ define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr addrspace(1) %out, i
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s3
-; GFX9-NEXT:  .LBB103_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB106_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    v_min_i32_e32 v0, s2, v1
 ; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
@@ -6685,7 +6796,7 @@ define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr addrspace(1) %out, i
 ; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB103_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB106_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_endpgm
 entry:
@@ -6712,7 +6823,7 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64_offset(ptr addrspace(1) %ou
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v1, s6
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:  .LBB104_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB107_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    v_min_i32_e32 v0, s8, v1
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -6725,7 +6836,7 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64_offset(ptr addrspace(1) %ou
 ; SI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
 ; SI-NEXT:    v_mov_b32_e32 v1, v2
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[0:1]
-; SI-NEXT:    s_cbranch_execnz .LBB104_1
+; SI-NEXT:    s_cbranch_execnz .LBB107_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
@@ -6753,7 +6864,7 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64_offset(ptr addrspace(1) %ou
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v2, s5
 ; VI-NEXT:    v_mov_b32_e32 v1, s7
-; VI-NEXT:  .LBB104_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB107_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    v_mov_b32_e32 v3, v2
 ; VI-NEXT:    v_min_i32_e32 v2, s4, v3
@@ -6763,7 +6874,7 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64_offset(ptr addrspace(1) %ou
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
 ; VI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[0:1]
-; VI-NEXT:    s_cbranch_execnz .LBB104_1
+; VI-NEXT:    s_cbranch_execnz .LBB107_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
@@ -6786,7 +6897,7 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64_offset(ptr addrspace(1) %ou
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s3
-; GFX9-NEXT:  .LBB104_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB107_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    v_mov_b32_e32 v3, v0
 ; GFX9-NEXT:    v_min_i32_e32 v2, s2, v3
@@ -6796,7 +6907,7 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64_offset(ptr addrspace(1) %ou
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
 ; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB104_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB107_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
@@ -6822,7 +6933,7 @@ define amdgpu_kernel void @atomic_min_i32(ptr addrspace(1) %out, i32 %in) {
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v1, s3
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:  .LBB105_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB108_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    v_min_i32_e32 v0, s2, v1
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -6835,7 +6946,7 @@ define amdgpu_kernel void @atomic_min_i32(ptr addrspace(1) %out, i32 %in) {
 ; SI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
 ; SI-NEXT:    v_mov_b32_e32 v1, v2
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[0:1]
-; SI-NEXT:    s_cbranch_execnz .LBB105_1
+; SI-NEXT:    s_cbranch_execnz .LBB108_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_endpgm
 ;
@@ -6850,7 +6961,7 @@ define amdgpu_kernel void @atomic_min_i32(ptr addrspace(1) %out, i32 %in) {
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v3, s3
-; VI-NEXT:  .LBB105_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB108_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    v_min_i32_e32 v2, s2, v3
 ; VI-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
@@ -6860,7 +6971,7 @@ define amdgpu_kernel void @atomic_min_i32(ptr addrspace(1) %out, i32 %in) {
 ; VI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v3, v2
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[0:1]
-; VI-NEXT:    s_cbranch_execnz .LBB105_1
+; VI-NEXT:    s_cbranch_execnz .LBB108_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_endpgm
 ;
@@ -6874,7 +6985,7 @@ define amdgpu_kernel void @atomic_min_i32(ptr addrspace(1) %out, i32 %in) {
 ; GFX9-NEXT:    s_load_dword s5, s[2:3], 0x0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s5
-; GFX9-NEXT:  .LBB105_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB108_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    v_min_i32_e32 v0, s4, v1
 ; GFX9-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] glc
@@ -6884,7 +6995,7 @@ define amdgpu_kernel void @atomic_min_i32(ptr addrspace(1) %out, i32 %in) {
 ; GFX9-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
 ; GFX9-NEXT:    v_mov_b32_e32 v1, v0
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[0:1]
-; GFX9-NEXT:    s_cbranch_execnz .LBB105_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB108_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_endpgm
 entry:
@@ -6909,7 +7020,7 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v1, s6
 ; SI-NEXT:    s_mov_b32 s6, -1
-; SI-NEXT:  .LBB106_1: ; %atomicrmw.start
+; SI-NEXT:  .LBB109_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    v_min_i32_e32 v0, s8, v1
 ; SI-NEXT:    s_waitcnt expcnt(0)
@@ -6922,7 +7033,7 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; SI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
 ; SI-NEXT:    v_mov_b32_e32 v1, v2
 ; SI-NEXT:    s_andn2_b64 exec, exec, s[0:1]
-; SI-NEXT:    s_cbranch_execnz .LBB106_1
+; SI-NEXT:    s_cbranch_execnz .LBB109_1
 ; SI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; SI-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
@@ -6948,7 +7059,7 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; VI-NEXT:    v_mov_b32_e32 v1, s7
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v2, s5
-; VI-NEXT:  .LBB106_1: ; %atomicrmw.start
+; VI-NEXT:  .LBB109_1: ; %atomicrmw.start
 ; VI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; VI-NEXT:    v_mov_b32_e32 v3, v2
 ; VI-NEXT:    v_min_i32_e32 v2, s4, v3
@@ -6958,7 +7069,7 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v2, v3
 ; VI-NEXT:    s_or_b64 s[0:1], vcc, s[0:1]
 ; VI-NEXT:    s_andn2_b64 exec, exec, s[0:1]
-; VI-NEXT:    s_cbranch_execnz .LBB106_1
+; VI-NEXT:    s_cbranch_execnz .LBB109_1
 ; VI-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; VI-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
@@ -6981,7 +7092,7 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; GFX9-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s3
-; GFX9-NEXT:  .LBB106_1: ; %atomicrmw.start
+; GFX9-NEXT:  .LBB109_1: ; %atomicrmw.start
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX9-NEXT:    v_mov_b32_e32 v3, v0
 ; GFX9-NEXT:    v_min_i32_e32 v2, s2, v3
@@ -6991,7 +7102,7 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64(ptr addrspace(1) %out, ptr
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v0, v3
 ; GFX9-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GFX9-NEXT:    s_andn2_b64 exec, exec, s[4:5]
-; GFX9-NEXT:    s_cbranch_execnz .LBB106_1
+; GFX9-NEXT:    s_cbranch_execnz .LBB109_1
 ; GFX9-NEXT:  ; %bb.2: ; %atomicrmw.end
 ; GFX9-NEXT:    s_or_b64 exec, exec, s[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v1, 0
diff --git a/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomicrmw-integer-ops-0-to-add-0.ll b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomicrmw-integer-ops-0-to-add-0.ll
new file mode 100644
index 000000000000..e32c218e9fe5
--- /dev/null
+++ b/llvm/test/Transforms/AtomicExpand/AMDGPU/expand-atomicrmw-integer-ops-0-to-add-0.ll
@@ -0,0 +1,130 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=atomic-expand %s | FileCheck %s
+
+; Test that system scoped atomicrmw or 0 is transformed to add 0.
+
+; Transform to add
+define i32 @test_atomicrmw_or_0_global_system(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define i32 @test_atomicrmw_or_0_global_system(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    [[RES:%.*]] = atomicrmw add ptr addrspace(1) [[PTR]], i32 0 seq_cst, align 4, !foo.md [[META0:![0-9]+]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = atomicrmw or ptr addrspace(1) %ptr, i32 0 seq_cst, !foo.md !0
+  ret i32 %res
+}
+
+; Transform to add
+define i32 @test_atomicrmw_or_0_global_one_as(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define i32 @test_atomicrmw_or_0_global_one_as(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    [[RES:%.*]] = atomicrmw add ptr addrspace(1) [[PTR]], i32 0 syncscope("one-as") seq_cst, align 4
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = atomicrmw or ptr addrspace(1) %ptr, i32 0 syncscope("one-as") seq_cst
+  ret i32 %res
+}
+
+; Transform to add
+define i32 @test_atomicrmw_or_0_flat_system(ptr %ptr) {
+; CHECK-LABEL: define i32 @test_atomicrmw_or_0_flat_system(
+; CHECK-SAME: ptr [[PTR:%.*]]) {
+; CHECK-NEXT:    [[RES:%.*]] = atomicrmw add ptr [[PTR]], i32 0 seq_cst, align 4
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = atomicrmw or ptr %ptr, i32 0 seq_cst
+  ret i32 %res
+}
+
+; Transform to add
+define i32 @test_atomicrmw_or_0_as999_system(ptr addrspace(999) %ptr) {
+; CHECK-LABEL: define i32 @test_atomicrmw_or_0_as999_system(
+; CHECK-SAME: ptr addrspace(999) [[PTR:%.*]]) {
+; CHECK-NEXT:    [[RES:%.*]] = atomicrmw add ptr addrspace(999) [[PTR]], i32 0 seq_cst, align 4
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = atomicrmw or ptr addrspace(999) %ptr, i32 0 seq_cst
+  ret i32 %res
+}
+
+; Leave as-is, only system scope should be changed.
+define i32 @test_atomicrmw_or_0_global_agent(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define i32 @test_atomicrmw_or_0_global_agent(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    [[RES:%.*]] = atomicrmw or ptr addrspace(1) [[PTR]], i32 0 syncscope("agent") seq_cst, align 4
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = atomicrmw or ptr addrspace(1) %ptr, i32 0 syncscope("agent") seq_cst
+  ret i32 %res
+}
+
+; Leave as-is, LDS atomics aren't relevant.
+define i32 @test_atomicrmw_or_0_local(ptr addrspace(3) %ptr) {
+; CHECK-LABEL: define i32 @test_atomicrmw_or_0_local(
+; CHECK-SAME: ptr addrspace(3) [[PTR:%.*]]) {
+; CHECK-NEXT:    [[RES:%.*]] = atomicrmw or ptr addrspace(3) [[PTR]], i32 0 seq_cst, align 4
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = atomicrmw or ptr addrspace(3) %ptr, i32 0 seq_cst
+  ret i32 %res
+}
+
+; Leave non-0 values alone.
+define i32 @test_atomicrmw_or_1_global_system(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define i32 @test_atomicrmw_or_1_global_system(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    [[RES:%.*]] = atomicrmw or ptr addrspace(1) [[PTR]], i32 1 seq_cst, align 4
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = atomicrmw or ptr addrspace(1) %ptr, i32 1 seq_cst
+  ret i32 %res
+}
+
+define i32 @test_atomicrmw_or_var_global_system(ptr addrspace(1) %ptr, i32 %val) {
+; CHECK-LABEL: define i32 @test_atomicrmw_or_var_global_system(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT:    [[RES:%.*]] = atomicrmw or ptr addrspace(1) [[PTR]], i32 [[VAL]] seq_cst, align 4
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = atomicrmw or ptr addrspace(1) %ptr, i32 %val seq_cst
+  ret i32 %res
+}
+
+; Leave as-is
+define i32 @test_atomicrmw_add_0_global_system(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define i32 @test_atomicrmw_add_0_global_system(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    [[RES:%.*]] = atomicrmw add ptr addrspace(1) [[PTR]], i32 0 seq_cst, align 4
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = atomicrmw add ptr addrspace(1) %ptr, i32 0 seq_cst
+  ret i32 %res
+}
+
+; Transform to add
+define i32 @test_atomicrmw_sub_0_global_system(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define i32 @test_atomicrmw_sub_0_global_system(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    [[RES:%.*]] = atomicrmw add ptr addrspace(1) [[PTR]], i32 0 seq_cst, align 4, !foo.md [[META0]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = atomicrmw sub ptr addrspace(1) %ptr, i32 0 seq_cst, !foo.md !0
+  ret i32 %res
+}
+
+; Transform to add
+define i32 @test_atomicrmw_xor_0_global_system(ptr addrspace(1) %ptr) {
+; CHECK-LABEL: define i32 @test_atomicrmw_xor_0_global_system(
+; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
+; CHECK-NEXT:    [[RES:%.*]] = atomicrmw add ptr addrspace(1) [[PTR]], i32 0 seq_cst, align 4, !foo.md [[META0]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %res = atomicrmw xor ptr addrspace(1) %ptr, i32 0 seq_cst, !foo.md !0
+  ret i32 %res
+}
+
+
+!0 = !{}
+;.
+; CHECK: [[META0]] = !{}
+;.
-- 
GitLab


From b9bed1f1106e3116e8bea38806c511b57b956929 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Fri, 12 Apr 2024 16:44:46 -0700
Subject: [PATCH 126/464] [test] Improve tests for alias/ifunc attributes

---
 clang/test/Sema/alias-unused-win.cpp | 31 +++++++++++++++++
 clang/test/Sema/alias-unused.c       |  7 ----
 clang/test/Sema/alias-unused.cpp     | 50 ++++++++++++++++++++++++++++
 3 files changed, 81 insertions(+), 7 deletions(-)
 create mode 100644 clang/test/Sema/alias-unused-win.cpp
 delete mode 100644 clang/test/Sema/alias-unused.c
 create mode 100644 clang/test/Sema/alias-unused.cpp

diff --git a/clang/test/Sema/alias-unused-win.cpp b/clang/test/Sema/alias-unused-win.cpp
new file mode 100644
index 000000000000..47c96d411751
--- /dev/null
+++ b/clang/test/Sema/alias-unused-win.cpp
@@ -0,0 +1,31 @@
+// RUN: %clang_cc1 -triple %ms_abi_triple -Wunused -x c -verify %s
+// RUN: %clang_cc1 -triple %ms_abi_triple -Wunused -verify=expected,cxx %s
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+static int f(void) { return 42; } // cxx-warning{{unused function 'f'}}
+int g(void) __attribute__((alias("f")));
+
+static int foo [] = { 42, 0xDEAD }; // cxx-warning{{variable 'foo' is not needed and will not be emitted}}
+extern typeof(foo) bar __attribute__((unused, alias("foo")));
+
+static int __attribute__((overloadable)) f0(int x) { return x; } // expected-warning{{unused function 'f0'}}
+static float __attribute__((overloadable)) f0(float x) { return x; } // expected-warning{{unused function 'f0'}}
+int g0(void) __attribute__((alias("?f0@@YAHH@Z")));
+
+#ifdef __cplusplus
+/// https://github.com/llvm/llvm-project/issues/88593
+/// We report a warning in C++ mode because the internal linkage `resolver` gets
+/// mangled as it does not have a language linkage. GCC does not mangle
+/// `resolver` or report a warning.
+static int f1() { return 42; } // cxx-warning{{unused function 'f1'}}
+int g1(void) __attribute__((alias("?f1@@YAHXZ")));
+}
+
+namespace ns {
+static int f3(int) { return 42; } // cxx-warning{{unused function 'f3'}}
+static int f3() { return 42; } // cxx-warning{{unused function 'f3'}}
+int g3() __attribute__((alias("?f3@ns@@YAHXZ")));
+}
+#endif
diff --git a/clang/test/Sema/alias-unused.c b/clang/test/Sema/alias-unused.c
deleted file mode 100644
index de9fc8cc7376..000000000000
--- a/clang/test/Sema/alias-unused.c
+++ /dev/null
@@ -1,7 +0,0 @@
-// RUN: %clang_cc1 -triple x86_64-linux-gnu -Wunneeded-internal-declaration -x c -verify %s
-// expected-no-diagnostics
-static int f(void) { return 42; }
-int g(void) __attribute__((alias("f")));
-
-static int foo [] = { 42, 0xDEAD };
-extern typeof(foo) bar __attribute__((unused, alias("foo")));
diff --git a/clang/test/Sema/alias-unused.cpp b/clang/test/Sema/alias-unused.cpp
new file mode 100644
index 000000000000..509d3e26225d
--- /dev/null
+++ b/clang/test/Sema/alias-unused.cpp
@@ -0,0 +1,50 @@
+// RUN: %clang_cc1 -triple %itanium_abi_triple -Wunused -x c -verify %s
+// RUN: %clang_cc1 -triple %itanium_abi_triple -Wunused -verify=expected,cxx %s
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+static int f(void) { return 42; }
+int g(void) __attribute__((alias("f")));
+
+static int foo [] = { 42, 0xDEAD }; // cxx-warning{{variable 'foo' is not needed and will not be emitted}}
+extern typeof(foo) bar __attribute__((unused, alias("foo")));
+
+/// https://github.com/llvm/llvm-project/issues/88593
+/// We report a warning in C++ mode because the internal linkage `resolver` gets
+/// mangled as it does not have a language linkage. GCC does not mangle
+/// `resolver` or report a warning.
+static int (*resolver(void))(void) { return f; } // expected-warning{{unused function 'resolver'}}
+int ifunc(void) __attribute__((ifunc("resolver")));
+
+static int __attribute__((overloadable)) f0(int x) { return x; } // expected-warning{{unused function 'f0'}}
+static float __attribute__((overloadable)) f0(float x) { return x; } // expected-warning{{unused function 'f0'}}
+int g0(void) __attribute__((alias("_ZL2f0i")));
+
+#ifdef __cplusplus
+static int f1() { return 42; } // expected-warning{{unused function 'f1'}}
+int g1(void) __attribute__((alias("_ZL2f1v")));
+}
+
+static int f2(int) { return 42; } // expected-warning{{unused function 'f2'}}
+static int f2() { return 42; } // expected-warning{{unused function 'f2'}}
+int g2() __attribute__((alias("_ZL2f2v")));
+
+static int (*resolver1())() { return f; } // expected-warning{{unused function 'resolver1'}}
+static int (*resolver1(int))() { return f; } // expected-warning{{unused function 'resolver1'}}
+int ifunc1() __attribute__((ifunc("_ZL9resolver1i")));
+
+/// TODO: We should report "unused function" for f3(int).
+namespace ns {
+static int f3(int) { return 42; } // cxx-warning{{unused function 'f3'}}
+static int f3() { return 42; } // cxx-warning{{unused function 'f3'}}
+int g3() __attribute__((alias("_ZN2nsL2f3Ev")));
+}
+
+template <class T>
+static void *f4(T) { return nullptr; }
+static void *f4() { return nullptr; } // cxx-warning{{unused function 'f4'}}
+extern void g4_int() __attribute__((ifunc("_ZL2f4IiEPvT_")));
+extern void g4_char() __attribute__((ifunc("_ZL2f4IcEPcT_"))); // rejected by CodeGen
+void *use4 = f4(0);
+#endif
-- 
GitLab


From 9e95c4947d31670ddd3abad1051eda06729c96b3 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Sat, 13 Apr 2024 09:01:17 +0900
Subject: [PATCH 127/464] [DSE] Fix non-determinism due to address reuse
 (#84943)

The malloc->calloc fold creates a new MemoryAccess, which may end of at
the same address as a previously deleted access inside SkipStores.

To the most part, this is not a problem, because SkipStores is normally
only used together with MemDefs. Neither the old malloc access nor the
new calloc access will be part of MemDefs, so there is no problem here.

However, SkipStores is also used in one more place: In the main DSE
loop, ToCheck entries are checked against it. Fix this by not using
SkipStores here, and instead using a separate set to track deletions
inside this loop. This way it is not affected by the calloc optimization
that happens outside it.

This is all pretty ugly, but I haven't found another good way to fix it.
Suggestions welcome.

No test case as I don't have a reliable DSE-only test-case for this.

Fixes https://github.com/llvm/llvm-project/issues/84458.
---
 .../Scalar/DeadStoreElimination.cpp           | 22 ++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
index bfc8bd5970bf..ed4212d29cef 100644
--- a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
@@ -1699,7 +1699,9 @@ struct DSEState {
 
   /// Delete dead memory defs and recursively add their operands to ToRemove if
   /// they became dead.
-  void deleteDeadInstruction(Instruction *SI) {
+  void
+  deleteDeadInstruction(Instruction *SI,
+                        SmallPtrSetImpl<MemoryAccess *> *Deleted = nullptr) {
     MemorySSAUpdater Updater(&MSSA);
     SmallVector<Instruction *, 32> NowDeadInsts;
     NowDeadInsts.push_back(SI);
@@ -1720,6 +1722,8 @@ struct DSEState {
         if (IsMemDef) {
           auto *MD = cast<MemoryDef>(MA);
           SkipStores.insert(MD);
+          if (Deleted)
+            Deleted->insert(MD);
           if (auto *SI = dyn_cast<StoreInst>(MD->getMemoryInst())) {
             if (SI->getValueOperand()->getType()->isPointerTy()) {
               const Value *UO = getUnderlyingObject(SI->getValueOperand());
@@ -2168,6 +2172,11 @@ static bool eliminateDeadStores(Function &F, AliasAnalysis &AA, MemorySSA &MSSA,
     unsigned PartialLimit = MemorySSAPartialStoreLimit;
     // Worklist of MemoryAccesses that may be killed by KillingDef.
     SmallSetVector<MemoryAccess *, 8> ToCheck;
+    // Track MemoryAccesses that have been deleted in the loop below, so we can
+    // skip them. Don't use SkipStores for this, which may contain reused
+    // MemoryAccess addresses.
+    SmallPtrSet<MemoryAccess *, 8> Deleted;
+    [[maybe_unused]] unsigned OrigNumSkipStores = State.SkipStores.size();
     ToCheck.insert(KillingDef->getDefiningAccess());
 
     bool Shortend = false;
@@ -2175,7 +2184,7 @@ static bool eliminateDeadStores(Function &F, AliasAnalysis &AA, MemorySSA &MSSA,
     // Check if MemoryAccesses in the worklist are killed by KillingDef.
     for (unsigned I = 0; I < ToCheck.size(); I++) {
       MemoryAccess *Current = ToCheck[I];
-      if (State.SkipStores.count(Current))
+      if (Deleted.contains(Current))
         continue;
 
       std::optional<MemoryAccess *> MaybeDeadAccess = State.getDomMemoryDef(
@@ -2222,7 +2231,7 @@ static bool eliminateDeadStores(Function &F, AliasAnalysis &AA, MemorySSA &MSSA,
           continue;
         LLVM_DEBUG(dbgs() << "DSE: Remove Dead Store:\n  DEAD: " << *DeadI
                           << "\n  KILLER: " << *KillingI << '\n');
-        State.deleteDeadInstruction(DeadI);
+        State.deleteDeadInstruction(DeadI, &Deleted);
         ++NumFastStores;
         MadeChange = true;
       } else {
@@ -2259,7 +2268,7 @@ static bool eliminateDeadStores(Function &F, AliasAnalysis &AA, MemorySSA &MSSA,
               Shortend = true;
               // Remove killing store and remove any outstanding overlap
               // intervals for the updated store.
-              State.deleteDeadInstruction(KillingSI);
+              State.deleteDeadInstruction(KillingSI, &Deleted);
               auto I = State.IOLs.find(DeadSI->getParent());
               if (I != State.IOLs.end())
                 I->second.erase(DeadSI);
@@ -2271,13 +2280,16 @@ static bool eliminateDeadStores(Function &F, AliasAnalysis &AA, MemorySSA &MSSA,
         if (OR == OW_Complete) {
           LLVM_DEBUG(dbgs() << "DSE: Remove Dead Store:\n  DEAD: " << *DeadI
                             << "\n  KILLER: " << *KillingI << '\n');
-          State.deleteDeadInstruction(DeadI);
+          State.deleteDeadInstruction(DeadI, &Deleted);
           ++NumFastStores;
           MadeChange = true;
         }
       }
     }
 
+    assert(State.SkipStores.size() - OrigNumSkipStores == Deleted.size() &&
+           "SkipStores and Deleted out of sync?");
+
     // Check if the store is a no-op.
     if (!Shortend && State.storeIsNoop(KillingDef, KillingUndObj)) {
       LLVM_DEBUG(dbgs() << "DSE: Remove No-Op Store:\n  DEAD: " << *KillingI
-- 
GitLab


From f172bfcdc229ed6f3e8a0298dab3713fb763ff28 Mon Sep 17 00:00:00 2001
From: Volodymyr Sapsai <vsapsai@apple.com>
Date: Fri, 12 Apr 2024 18:27:40 -0700
Subject: [PATCH 128/464] [unused-includes][ADT] Don't include DJB.h as it's
 not used. NFC. (#88211)

StringMap uses xxHash instead of DJB.
---
 llvm/include/llvm/ADT/StringMap.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/llvm/include/llvm/ADT/StringMap.h b/llvm/include/llvm/ADT/StringMap.h
index 32e611467ac5..453d91349e35 100644
--- a/llvm/include/llvm/ADT/StringMap.h
+++ b/llvm/include/llvm/ADT/StringMap.h
@@ -17,7 +17,6 @@
 #include "llvm/ADT/StringMapEntry.h"
 #include "llvm/ADT/iterator.h"
 #include "llvm/Support/AllocatorBase.h"
-#include "llvm/Support/DJB.h"
 #include "llvm/Support/PointerLikeTypeTraits.h"
 #include <initializer_list>
 #include <iterator>
-- 
GitLab


From f22c30063ddcb7f8e8c4fb0496f515be7f9408b8 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Fri, 12 Apr 2024 20:30:15 -0700
Subject: [PATCH 129/464] [test] alias-unused.cpp: set triple to Linux

Otherwise the test would fail on Darwin and other platforms that use
Itanium ABI but do not support alias/ifunc.
---
 clang/test/Sema/alias-unused.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/test/Sema/alias-unused.cpp b/clang/test/Sema/alias-unused.cpp
index 509d3e26225d..dc8e46f072d7 100644
--- a/clang/test/Sema/alias-unused.cpp
+++ b/clang/test/Sema/alias-unused.cpp
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -triple %itanium_abi_triple -Wunused -x c -verify %s
-// RUN: %clang_cc1 -triple %itanium_abi_triple -Wunused -verify=expected,cxx %s
+// RUN: %clang_cc1 -triple x86_64-linux -Wunused -x c -verify %s
+// RUN: %clang_cc1 -triple x86_64-linux -Wunused -verify=expected,cxx %s
 
 #ifdef __cplusplus
 extern "C" {
-- 
GitLab


From aef28100294d04c2253a1cb8bbf552fa296dea4e Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Fri, 12 Apr 2024 20:53:44 -0700
Subject: [PATCH 130/464] [ELF] Relocations: Avoid MIPS check for ELFCLASS64
 with if constexpr. NFC

After inlining, `scanSection` is significantly longer (more than 100+
instructions on x86-64 built with Clang) when `i` does not always
increment by one (MIPS).
---
 lld/ELF/Relocations.cpp | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp
index 55274344f881..04db413a6609 100644
--- a/lld/ELF/Relocations.cpp
+++ b/lld/ELF/Relocations.cpp
@@ -1419,11 +1419,16 @@ template <class ELFT, class RelTy> void RelocationScanner::scanOne(RelTy *&i) {
   uint32_t symIndex = rel.getSymbol(config->isMips64EL);
   Symbol &sym = sec->getFile<ELFT>()->getSymbol(symIndex);
   RelType type;
-  if (config->mipsN32Abi) {
-    type = getMipsN32RelType(i);
-  } else {
+  if constexpr (ELFT::Is64Bits) {
     type = rel.getType(config->isMips64EL);
     ++i;
+  } else {
+    if (config->mipsN32Abi) {
+      type = getMipsN32RelType(i);
+    } else {
+      type = rel.getType(config->isMips64EL);
+      ++i;
+    }
   }
   // Get an offset in an output section this relocation is applied to.
   uint64_t offset = getter.get(rel.r_offset);
-- 
GitLab


From 94b3c192d3468752e6bc2f20957d433bccdb8ff3 Mon Sep 17 00:00:00 2001
From: Jim Lin <jim@andestech.com>
Date: Fri, 12 Apr 2024 23:39:30 -0500
Subject: [PATCH 131/464] [M68k][clang] Enable frame pointer optimization by
 default (#87264)

Enable frame pointer optimization by default to match it with gcc.

Fixes: https://github.com/llvm/llvm-project/issues/75013
---
 clang/lib/Driver/ToolChains/CommonArgs.cpp | 1 +
 clang/test/Driver/frame-pointer-elim.c     | 6 ++++++
 2 files changed, 7 insertions(+)

diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 62a53b85ce09..f10aa4dfaa9d 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -114,6 +114,7 @@ static bool useFramePointerForTargetByDefault(const llvm::opt::ArgList &Args,
   case llvm::Triple::csky:
   case llvm::Triple::loongarch32:
   case llvm::Triple::loongarch64:
+  case llvm::Triple::m68k:
     return !clang::driver::tools::areOptimizationsEnabled(Args);
   default:
     break;
diff --git a/clang/test/Driver/frame-pointer-elim.c b/clang/test/Driver/frame-pointer-elim.c
index 847c5b7b0b01..e1b0a468ab82 100644
--- a/clang/test/Driver/frame-pointer-elim.c
+++ b/clang/test/Driver/frame-pointer-elim.c
@@ -125,6 +125,12 @@
 // RUN: %clang -### -target sparc64 -S -O1 %s 2>&1 | \
 // RUN:   FileCheck --check-prefix=KEEP-NONE %s
 
+// M68k targets omit the frame pointer when optimizations are enabled.
+// RUN: %clang -### -target m68k -S %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=KEEP-ALL %s
+// RUN: %clang -### -target m68k -S -O1 %s 2>&1 | \
+// RUN:   FileCheck --check-prefix=KEEP-NONE %s
+
 // For AAarch32 (A32, T32) linux targets, default omit frame pointer when
 // optimizations are enabled.
 // RUN: %clang -### -target arm-linux-gnueabihf- -marm -S %s 2>&1 | \
-- 
GitLab


From 0a6f6df5b0c3d0f2a42f013bf5cafb9b5020dcac Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Sat, 13 Apr 2024 08:54:25 +0400
Subject: [PATCH 132/464] [clang] Introduce `SemaCUDA` (#88559)

This patch moves CUDA-related `Sema` function into new `SemaCUDA` class,
following the recent example of SYCL, OpenACC, and HLSL. This is a part
of the effort to split Sema. Additional context can be found in
https://github.com/llvm/llvm-project/pull/82217,
https://github.com/llvm/llvm-project/pull/84184,
https://github.com/llvm/llvm-project/pull/87634.
---
 clang/include/clang/Basic/Cuda.h              |   8 +
 clang/include/clang/Sema/Sema.h               | 304 +-----------------
 clang/include/clang/Sema/SemaBase.h           |   2 +-
 clang/include/clang/Sema/SemaCUDA.h           | 304 ++++++++++++++++++
 clang/include/clang/Serialization/ASTReader.h |   2 +-
 clang/lib/Parse/ParseDecl.cpp                 |   4 +-
 clang/lib/Parse/ParseExpr.cpp                 |   7 +-
 clang/lib/Parse/ParsePragma.cpp               |   5 +-
 clang/lib/Sema/Sema.cpp                       |  18 +-
 clang/lib/Sema/SemaBase.cpp                   |   5 +-
 clang/lib/Sema/SemaCUDA.cpp                   | 277 ++++++++--------
 clang/lib/Sema/SemaDecl.cpp                   |  19 +-
 clang/lib/Sema/SemaDeclAttr.cpp               |  12 +-
 clang/lib/Sema/SemaDeclCXX.cpp                |  21 +-
 clang/lib/Sema/SemaExpr.cpp                   |  17 +-
 clang/lib/Sema/SemaExprCXX.cpp                |  27 +-
 clang/lib/Sema/SemaLambda.cpp                 |   9 +-
 clang/lib/Sema/SemaOverload.cpp               |  60 ++--
 clang/lib/Sema/SemaStmt.cpp                   |   5 +-
 clang/lib/Sema/SemaTemplate.cpp               |  15 +-
 .../lib/Sema/SemaTemplateInstantiateDecl.cpp  |   3 +-
 clang/lib/Sema/SemaType.cpp                   |   9 +-
 clang/lib/Serialization/ASTReader.cpp         |   5 +-
 clang/lib/Serialization/ASTWriter.cpp         |   5 +-
 24 files changed, 604 insertions(+), 539 deletions(-)
 create mode 100644 clang/include/clang/Sema/SemaCUDA.h

diff --git a/clang/include/clang/Basic/Cuda.h b/clang/include/clang/Basic/Cuda.h
index 3e77a74c7c00..acc6bb6581d8 100644
--- a/clang/include/clang/Basic/Cuda.h
+++ b/clang/include/clang/Basic/Cuda.h
@@ -126,6 +126,14 @@ enum class CudaArch {
   HIPDefault = CudaArch::GFX906,
 };
 
+enum class CUDAFunctionTarget {
+  Device,
+  Global,
+  Host,
+  HostDevice,
+  InvalidTarget
+};
+
 static inline bool IsNVIDIAGpuArch(CudaArch A) {
   return A >= CudaArch::SM_20 && A < CudaArch::GFX600;
 }
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 00888b7f7a73..6b9789334811 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -38,6 +38,7 @@
 #include "clang/AST/TypeOrdering.h"
 #include "clang/Basic/BitmaskEnum.h"
 #include "clang/Basic/Builtins.h"
+#include "clang/Basic/Cuda.h"
 #include "clang/Basic/DarwinSDKInfo.h"
 #include "clang/Basic/ExpressionTraits.h"
 #include "clang/Basic/Module.h"
@@ -183,6 +184,7 @@ class Preprocessor;
 class PseudoDestructorTypeStorage;
 class PseudoObjectExpr;
 class QualType;
+class SemaCUDA;
 class SemaHLSL;
 class SemaOpenACC;
 class SemaSYCL;
@@ -435,14 +437,6 @@ enum class CXXSpecialMemberKind {
   Invalid
 };
 
-enum class CUDAFunctionTarget {
-  Device,
-  Global,
-  Host,
-  HostDevice,
-  InvalidTarget
-};
-
 /// Sema - This implements semantic analysis and AST building for C.
 /// \nosubgrouping
 class Sema final : public SemaBase {
@@ -486,8 +480,7 @@ class Sema final : public SemaBase {
   // 35. Code Completion (SemaCodeComplete.cpp)
   // 36. FixIt Helpers (SemaFixItUtils.cpp)
   // 37. Name Lookup for RISC-V Vector Intrinsic (SemaRISCVVectorLookup.cpp)
-  // 38. CUDA (SemaCUDA.cpp)
-  // 39. OpenMP Directives and Clauses (SemaOpenMP.cpp)
+  // 38. OpenMP Directives and Clauses (SemaOpenMP.cpp)
 
   /// \name Semantic Analysis
   /// Implementations are in Sema.cpp
@@ -981,9 +974,19 @@ public:
     return DelayedDiagnostics.push(pool);
   }
 
+  /// Diagnostics that are emitted only if we discover that the given function
+  /// must be codegen'ed.  Because handling these correctly adds overhead to
+  /// compilation, this is currently only enabled for CUDA compilations.
+  SemaDiagnosticBuilder::DeferredDiagnosticsType DeviceDeferredDiags;
+
   /// CurContext - This is the current declaration context of parsing.
   DeclContext *CurContext;
 
+  SemaCUDA &CUDA() {
+    assert(CUDAPtr);
+    return *CUDAPtr;
+  }
+
   SemaHLSL &HLSL() {
     assert(HLSLPtr);
     return *HLSLPtr;
@@ -1029,6 +1032,7 @@ private:
 
   mutable IdentifierInfo *Ident_super;
 
+  std::unique_ptr<SemaCUDA> CUDAPtr;
   std::unique_ptr<SemaHLSL> HLSLPtr;
   std::unique_ptr<SemaOpenACC> OpenACCPtr;
   std::unique_ptr<SemaSYCL> SYCLPtr;
@@ -12908,258 +12912,6 @@ private:
   //
   //
 
-  /// \name CUDA
-  /// Implementations are in SemaCUDA.cpp
-  ///@{
-
-public:
-  /// Increments our count of the number of times we've seen a pragma forcing
-  /// functions to be __host__ __device__.  So long as this count is greater
-  /// than zero, all functions encountered will be __host__ __device__.
-  void PushForceCUDAHostDevice();
-
-  /// Decrements our count of the number of times we've seen a pragma forcing
-  /// functions to be __host__ __device__.  Returns false if the count is 0
-  /// before incrementing, so you can emit an error.
-  bool PopForceCUDAHostDevice();
-
-  ExprResult ActOnCUDAExecConfigExpr(Scope *S, SourceLocation LLLLoc,
-                                     MultiExprArg ExecConfig,
-                                     SourceLocation GGGLoc);
-
-  /// Diagnostics that are emitted only if we discover that the given function
-  /// must be codegen'ed.  Because handling these correctly adds overhead to
-  /// compilation, this is currently only enabled for CUDA compilations.
-  SemaDiagnosticBuilder::DeferredDiagnosticsType DeviceDeferredDiags;
-
-  /// A pair of a canonical FunctionDecl and a SourceLocation.  When used as the
-  /// key in a hashtable, both the FD and location are hashed.
-  struct FunctionDeclAndLoc {
-    CanonicalDeclPtr<const FunctionDecl> FD;
-    SourceLocation Loc;
-  };
-
-  /// FunctionDecls and SourceLocations for which CheckCUDACall has emitted a
-  /// (maybe deferred) "bad call" diagnostic.  We use this to avoid emitting the
-  /// same deferred diag twice.
-  llvm::DenseSet<FunctionDeclAndLoc> LocsWithCUDACallDiags;
-
-  /// An inverse call graph, mapping known-emitted functions to one of their
-  /// known-emitted callers (plus the location of the call).
-  ///
-  /// Functions that we can tell a priori must be emitted aren't added to this
-  /// map.
-  llvm::DenseMap</* Callee = */ CanonicalDeclPtr<const FunctionDecl>,
-                 /* Caller = */ FunctionDeclAndLoc>
-      DeviceKnownEmittedFns;
-
-  /// Creates a SemaDiagnosticBuilder that emits the diagnostic if the current
-  /// context is "used as device code".
-  ///
-  /// - If CurContext is a __host__ function, does not emit any diagnostics
-  ///   unless \p EmitOnBothSides is true.
-  /// - If CurContext is a __device__ or __global__ function, emits the
-  ///   diagnostics immediately.
-  /// - If CurContext is a __host__ __device__ function and we are compiling for
-  ///   the device, creates a diagnostic which is emitted if and when we realize
-  ///   that the function will be codegen'ed.
-  ///
-  /// Example usage:
-  ///
-  ///  // Variable-length arrays are not allowed in CUDA device code.
-  ///  if (CUDADiagIfDeviceCode(Loc, diag::err_cuda_vla)
-  ///     << llvm::to_underlying(CurrentCUDATarget()))
-  ///    return ExprError();
-  ///  // Otherwise, continue parsing as normal.
-  SemaDiagnosticBuilder CUDADiagIfDeviceCode(SourceLocation Loc,
-                                             unsigned DiagID);
-
-  /// Creates a SemaDiagnosticBuilder that emits the diagnostic if the current
-  /// context is "used as host code".
-  ///
-  /// Same as CUDADiagIfDeviceCode, with "host" and "device" switched.
-  SemaDiagnosticBuilder CUDADiagIfHostCode(SourceLocation Loc, unsigned DiagID);
-
-  /// Determines whether the given function is a CUDA device/host/kernel/etc.
-  /// function.
-  ///
-  /// Use this rather than examining the function's attributes yourself -- you
-  /// will get it wrong.  Returns CUDAFunctionTarget::Host if D is null.
-  CUDAFunctionTarget IdentifyCUDATarget(const FunctionDecl *D,
-                                        bool IgnoreImplicitHDAttr = false);
-  CUDAFunctionTarget IdentifyCUDATarget(const ParsedAttributesView &Attrs);
-
-  enum CUDAVariableTarget {
-    CVT_Device,  /// Emitted on device side with a shadow variable on host side
-    CVT_Host,    /// Emitted on host side only
-    CVT_Both,    /// Emitted on both sides with different addresses
-    CVT_Unified, /// Emitted as a unified address, e.g. managed variables
-  };
-  /// Determines whether the given variable is emitted on host or device side.
-  CUDAVariableTarget IdentifyCUDATarget(const VarDecl *D);
-
-  /// Defines kinds of CUDA global host/device context where a function may be
-  /// called.
-  enum CUDATargetContextKind {
-    CTCK_Unknown,       /// Unknown context
-    CTCK_InitGlobalVar, /// Function called during global variable
-                        /// initialization
-  };
-
-  /// Define the current global CUDA host/device context where a function may be
-  /// called. Only used when a function is called outside of any functions.
-  struct CUDATargetContext {
-    CUDAFunctionTarget Target = CUDAFunctionTarget::HostDevice;
-    CUDATargetContextKind Kind = CTCK_Unknown;
-    Decl *D = nullptr;
-  } CurCUDATargetCtx;
-
-  struct CUDATargetContextRAII {
-    Sema &S;
-    CUDATargetContext SavedCtx;
-    CUDATargetContextRAII(Sema &S_, CUDATargetContextKind K, Decl *D);
-    ~CUDATargetContextRAII() { S.CurCUDATargetCtx = SavedCtx; }
-  };
-
-  /// Gets the CUDA target for the current context.
-  CUDAFunctionTarget CurrentCUDATarget() {
-    return IdentifyCUDATarget(dyn_cast<FunctionDecl>(CurContext));
-  }
-
-  static bool isCUDAImplicitHostDeviceFunction(const FunctionDecl *D);
-
-  // CUDA function call preference. Must be ordered numerically from
-  // worst to best.
-  enum CUDAFunctionPreference {
-    CFP_Never,      // Invalid caller/callee combination.
-    CFP_WrongSide,  // Calls from host-device to host or device
-                    // function that do not match current compilation
-                    // mode.
-    CFP_HostDevice, // Any calls to host/device functions.
-    CFP_SameSide,   // Calls from host-device to host or device
-                    // function matching current compilation mode.
-    CFP_Native,     // host-to-host or device-to-device calls.
-  };
-
-  /// Identifies relative preference of a given Caller/Callee
-  /// combination, based on their host/device attributes.
-  /// \param Caller function which needs address of \p Callee.
-  ///               nullptr in case of global context.
-  /// \param Callee target function
-  ///
-  /// \returns preference value for particular Caller/Callee combination.
-  CUDAFunctionPreference IdentifyCUDAPreference(const FunctionDecl *Caller,
-                                                const FunctionDecl *Callee);
-
-  /// Determines whether Caller may invoke Callee, based on their CUDA
-  /// host/device attributes.  Returns false if the call is not allowed.
-  ///
-  /// Note: Will return true for CFP_WrongSide calls.  These may appear in
-  /// semantically correct CUDA programs, but only if they're never codegen'ed.
-  bool IsAllowedCUDACall(const FunctionDecl *Caller,
-                         const FunctionDecl *Callee) {
-    return IdentifyCUDAPreference(Caller, Callee) != CFP_Never;
-  }
-
-  /// May add implicit CUDAHostAttr and CUDADeviceAttr attributes to FD,
-  /// depending on FD and the current compilation settings.
-  void maybeAddCUDAHostDeviceAttrs(FunctionDecl *FD,
-                                   const LookupResult &Previous);
-
-  /// May add implicit CUDAConstantAttr attribute to VD, depending on VD
-  /// and current compilation settings.
-  void MaybeAddCUDAConstantAttr(VarDecl *VD);
-
-  /// Check whether we're allowed to call Callee from the current context.
-  ///
-  /// - If the call is never allowed in a semantically-correct program
-  ///   (CFP_Never), emits an error and returns false.
-  ///
-  /// - If the call is allowed in semantically-correct programs, but only if
-  ///   it's never codegen'ed (CFP_WrongSide), creates a deferred diagnostic to
-  ///   be emitted if and when the caller is codegen'ed, and returns true.
-  ///
-  ///   Will only create deferred diagnostics for a given SourceLocation once,
-  ///   so you can safely call this multiple times without generating duplicate
-  ///   deferred errors.
-  ///
-  /// - Otherwise, returns true without emitting any diagnostics.
-  bool CheckCUDACall(SourceLocation Loc, FunctionDecl *Callee);
-
-  void CUDACheckLambdaCapture(CXXMethodDecl *D, const sema::Capture &Capture);
-
-  /// Set __device__ or __host__ __device__ attributes on the given lambda
-  /// operator() method.
-  ///
-  /// CUDA lambdas by default is host device function unless it has explicit
-  /// host or device attribute.
-  void CUDASetLambdaAttrs(CXXMethodDecl *Method);
-
-  /// Record \p FD if it is a CUDA/HIP implicit host device function used on
-  /// device side in device compilation.
-  void CUDARecordImplicitHostDeviceFuncUsedByDevice(const FunctionDecl *FD);
-
-  /// Finds a function in \p Matches with highest calling priority
-  /// from \p Caller context and erases all functions with lower
-  /// calling priority.
-  void EraseUnwantedCUDAMatches(
-      const FunctionDecl *Caller,
-      SmallVectorImpl<std::pair<DeclAccessPair, FunctionDecl *>> &Matches);
-
-  /// Given a implicit special member, infer its CUDA target from the
-  /// calls it needs to make to underlying base/field special members.
-  /// \param ClassDecl the class for which the member is being created.
-  /// \param CSM the kind of special member.
-  /// \param MemberDecl the special member itself.
-  /// \param ConstRHS true if this is a copy operation with a const object on
-  ///        its RHS.
-  /// \param Diagnose true if this call should emit diagnostics.
-  /// \return true if there was an error inferring.
-  /// The result of this call is implicit CUDA target attribute(s) attached to
-  /// the member declaration.
-  bool inferCUDATargetForImplicitSpecialMember(CXXRecordDecl *ClassDecl,
-                                               CXXSpecialMemberKind CSM,
-                                               CXXMethodDecl *MemberDecl,
-                                               bool ConstRHS, bool Diagnose);
-
-  /// \return true if \p CD can be considered empty according to CUDA
-  /// (E.2.3.1 in CUDA 7.5 Programming guide).
-  bool isEmptyCudaConstructor(SourceLocation Loc, CXXConstructorDecl *CD);
-  bool isEmptyCudaDestructor(SourceLocation Loc, CXXDestructorDecl *CD);
-
-  // \brief Checks that initializers of \p Var satisfy CUDA restrictions. In
-  // case of error emits appropriate diagnostic and invalidates \p Var.
-  //
-  // \details CUDA allows only empty constructors as initializers for global
-  // variables (see E.2.3.1, CUDA 7.5). The same restriction also applies to all
-  // __shared__ variables whether they are local or not (they all are implicitly
-  // static in CUDA). One exception is that CUDA allows constant initializers
-  // for __constant__ and __device__ variables.
-  void checkAllowedCUDAInitializer(VarDecl *VD);
-
-  /// Check whether NewFD is a valid overload for CUDA. Emits
-  /// diagnostics and invalidates NewFD if not.
-  void checkCUDATargetOverload(FunctionDecl *NewFD,
-                               const LookupResult &Previous);
-  /// Copies target attributes from the template TD to the function FD.
-  void inheritCUDATargetAttrs(FunctionDecl *FD, const FunctionTemplateDecl &TD);
-
-  /// Returns the name of the launch configuration function.  This is the name
-  /// of the function that will be called to configure kernel call, with the
-  /// parameters specified via <<<>>>.
-  std::string getCudaConfigureFuncName() const;
-
-private:
-  unsigned ForceCUDAHostDeviceDepth = 0;
-
-  ///@}
-
-  //
-  //
-  // -------------------------------------------------------------------------
-  //
-  //
-
   /// \name OpenMP Directives and Clauses
   /// Implementations are in SemaOpenMP.cpp
   ///@{
@@ -14546,32 +14298,4 @@ std::unique_ptr<sema::RISCVIntrinsicManager>
 CreateRISCVIntrinsicManager(Sema &S);
 } // end namespace clang
 
-namespace llvm {
-// Hash a FunctionDeclAndLoc by looking at both its FunctionDecl and its
-// SourceLocation.
-template <> struct DenseMapInfo<clang::Sema::FunctionDeclAndLoc> {
-  using FunctionDeclAndLoc = clang::Sema::FunctionDeclAndLoc;
-  using FDBaseInfo =
-      DenseMapInfo<clang::CanonicalDeclPtr<const clang::FunctionDecl>>;
-
-  static FunctionDeclAndLoc getEmptyKey() {
-    return {FDBaseInfo::getEmptyKey(), clang::SourceLocation()};
-  }
-
-  static FunctionDeclAndLoc getTombstoneKey() {
-    return {FDBaseInfo::getTombstoneKey(), clang::SourceLocation()};
-  }
-
-  static unsigned getHashValue(const FunctionDeclAndLoc &FDL) {
-    return hash_combine(FDBaseInfo::getHashValue(FDL.FD),
-                        FDL.Loc.getHashValue());
-  }
-
-  static bool isEqual(const FunctionDeclAndLoc &LHS,
-                      const FunctionDeclAndLoc &RHS) {
-    return LHS.FD == RHS.FD && LHS.Loc == RHS.Loc;
-  }
-};
-} // namespace llvm
-
 #endif
diff --git a/clang/include/clang/Sema/SemaBase.h b/clang/include/clang/Sema/SemaBase.h
index ff718022fca0..3220f71dd797 100644
--- a/clang/include/clang/Sema/SemaBase.h
+++ b/clang/include/clang/Sema/SemaBase.h
@@ -146,7 +146,7 @@ public:
     ///   if (SemaDiagnosticBuilder(...) << foo << bar)
     ///     return ExprError();
     ///
-    /// But see CUDADiagIfDeviceCode() and CUDADiagIfHostCode() -- you probably
+    /// But see DiagIfDeviceCode() and DiagIfHostCode() -- you probably
     /// want to use these instead of creating a SemaDiagnosticBuilder yourself.
     operator bool() const { return isImmediate(); }
 
diff --git a/clang/include/clang/Sema/SemaCUDA.h b/clang/include/clang/Sema/SemaCUDA.h
new file mode 100644
index 000000000000..63dc3f4da240
--- /dev/null
+++ b/clang/include/clang/Sema/SemaCUDA.h
@@ -0,0 +1,304 @@
+//===----- SemaCUDA.h ----- Semantic Analysis for CUDA constructs ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+/// This file declares semantic analysis for CUDA constructs.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_SEMA_SEMACUDA_H
+#define LLVM_CLANG_SEMA_SEMACUDA_H
+
+#include "clang/AST/Decl.h"
+#include "clang/AST/DeclCXX.h"
+#include "clang/AST/Redeclarable.h"
+#include "clang/Basic/Cuda.h"
+#include "clang/Basic/SourceLocation.h"
+#include "clang/Sema/Lookup.h"
+#include "clang/Sema/Ownership.h"
+#include "clang/Sema/ParsedAttr.h"
+#include "clang/Sema/Scope.h"
+#include "clang/Sema/ScopeInfo.h"
+#include "clang/Sema/SemaBase.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallVector.h"
+#include <string>
+
+namespace clang {
+
+enum class CUDAFunctionTarget;
+
+class SemaCUDA : public SemaBase {
+public:
+  SemaCUDA(Sema &S);
+
+  /// Increments our count of the number of times we've seen a pragma forcing
+  /// functions to be __host__ __device__.  So long as this count is greater
+  /// than zero, all functions encountered will be __host__ __device__.
+  void PushForceHostDevice();
+
+  /// Decrements our count of the number of times we've seen a pragma forcing
+  /// functions to be __host__ __device__.  Returns false if the count is 0
+  /// before incrementing, so you can emit an error.
+  bool PopForceHostDevice();
+
+  ExprResult ActOnExecConfigExpr(Scope *S, SourceLocation LLLLoc,
+                                 MultiExprArg ExecConfig,
+                                 SourceLocation GGGLoc);
+
+  /// A pair of a canonical FunctionDecl and a SourceLocation.  When used as the
+  /// key in a hashtable, both the FD and location are hashed.
+  struct FunctionDeclAndLoc {
+    CanonicalDeclPtr<const FunctionDecl> FD;
+    SourceLocation Loc;
+  };
+
+  /// FunctionDecls and SourceLocations for which CheckCall has emitted a
+  /// (maybe deferred) "bad call" diagnostic.  We use this to avoid emitting the
+  /// same deferred diag twice.
+  llvm::DenseSet<FunctionDeclAndLoc> LocsWithCUDACallDiags;
+
+  /// An inverse call graph, mapping known-emitted functions to one of their
+  /// known-emitted callers (plus the location of the call).
+  ///
+  /// Functions that we can tell a priori must be emitted aren't added to this
+  /// map.
+  llvm::DenseMap</* Callee = */ CanonicalDeclPtr<const FunctionDecl>,
+                 /* Caller = */ FunctionDeclAndLoc>
+      DeviceKnownEmittedFns;
+
+  /// Creates a SemaDiagnosticBuilder that emits the diagnostic if the current
+  /// context is "used as device code".
+  ///
+  /// - If CurContext is a __host__ function, does not emit any diagnostics
+  ///   unless \p EmitOnBothSides is true.
+  /// - If CurContext is a __device__ or __global__ function, emits the
+  ///   diagnostics immediately.
+  /// - If CurContext is a __host__ __device__ function and we are compiling for
+  ///   the device, creates a diagnostic which is emitted if and when we realize
+  ///   that the function will be codegen'ed.
+  ///
+  /// Example usage:
+  ///
+  ///  // Variable-length arrays are not allowed in CUDA device code.
+  ///  if (DiagIfDeviceCode(Loc, diag::err_cuda_vla) << CurrentTarget())
+  ///    return ExprError();
+  ///  // Otherwise, continue parsing as normal.
+  SemaDiagnosticBuilder DiagIfDeviceCode(SourceLocation Loc, unsigned DiagID);
+
+  /// Creates a SemaDiagnosticBuilder that emits the diagnostic if the current
+  /// context is "used as host code".
+  ///
+  /// Same as DiagIfDeviceCode, with "host" and "device" switched.
+  SemaDiagnosticBuilder DiagIfHostCode(SourceLocation Loc, unsigned DiagID);
+
+  /// Determines whether the given function is a CUDA device/host/kernel/etc.
+  /// function.
+  ///
+  /// Use this rather than examining the function's attributes yourself -- you
+  /// will get it wrong.  Returns CUDAFunctionTarget::Host if D is null.
+  CUDAFunctionTarget IdentifyTarget(const FunctionDecl *D,
+                                    bool IgnoreImplicitHDAttr = false);
+  CUDAFunctionTarget IdentifyTarget(const ParsedAttributesView &Attrs);
+
+  enum CUDAVariableTarget {
+    CVT_Device,  /// Emitted on device side with a shadow variable on host side
+    CVT_Host,    /// Emitted on host side only
+    CVT_Both,    /// Emitted on both sides with different addresses
+    CVT_Unified, /// Emitted as a unified address, e.g. managed variables
+  };
+  /// Determines whether the given variable is emitted on host or device side.
+  CUDAVariableTarget IdentifyTarget(const VarDecl *D);
+
+  /// Defines kinds of CUDA global host/device context where a function may be
+  /// called.
+  enum CUDATargetContextKind {
+    CTCK_Unknown,       /// Unknown context
+    CTCK_InitGlobalVar, /// Function called during global variable
+                        /// initialization
+  };
+
+  /// Define the current global CUDA host/device context where a function may be
+  /// called. Only used when a function is called outside of any functions.
+  struct CUDATargetContext {
+    CUDAFunctionTarget Target = CUDAFunctionTarget::HostDevice;
+    CUDATargetContextKind Kind = CTCK_Unknown;
+    Decl *D = nullptr;
+  } CurCUDATargetCtx;
+
+  struct CUDATargetContextRAII {
+    SemaCUDA &S;
+    SemaCUDA::CUDATargetContext SavedCtx;
+    CUDATargetContextRAII(SemaCUDA &S_, SemaCUDA::CUDATargetContextKind K,
+                          Decl *D);
+    ~CUDATargetContextRAII() { S.CurCUDATargetCtx = SavedCtx; }
+  };
+
+  /// Gets the CUDA target for the current context.
+  CUDAFunctionTarget CurrentTarget() {
+    return IdentifyTarget(dyn_cast<FunctionDecl>(SemaRef.CurContext));
+  }
+
+  static bool isImplicitHostDeviceFunction(const FunctionDecl *D);
+
+  // CUDA function call preference. Must be ordered numerically from
+  // worst to best.
+  enum CUDAFunctionPreference {
+    CFP_Never,      // Invalid caller/callee combination.
+    CFP_WrongSide,  // Calls from host-device to host or device
+                    // function that do not match current compilation
+                    // mode.
+    CFP_HostDevice, // Any calls to host/device functions.
+    CFP_SameSide,   // Calls from host-device to host or device
+                    // function matching current compilation mode.
+    CFP_Native,     // host-to-host or device-to-device calls.
+  };
+
+  /// Identifies relative preference of a given Caller/Callee
+  /// combination, based on their host/device attributes.
+  /// \param Caller function which needs address of \p Callee.
+  ///               nullptr in case of global context.
+  /// \param Callee target function
+  ///
+  /// \returns preference value for particular Caller/Callee combination.
+  CUDAFunctionPreference IdentifyPreference(const FunctionDecl *Caller,
+                                            const FunctionDecl *Callee);
+
+  /// Determines whether Caller may invoke Callee, based on their CUDA
+  /// host/device attributes.  Returns false if the call is not allowed.
+  ///
+  /// Note: Will return true for CFP_WrongSide calls.  These may appear in
+  /// semantically correct CUDA programs, but only if they're never codegen'ed.
+  bool IsAllowedCall(const FunctionDecl *Caller, const FunctionDecl *Callee) {
+    return IdentifyPreference(Caller, Callee) != CFP_Never;
+  }
+
+  /// May add implicit CUDAHostAttr and CUDADeviceAttr attributes to FD,
+  /// depending on FD and the current compilation settings.
+  void maybeAddHostDeviceAttrs(FunctionDecl *FD, const LookupResult &Previous);
+
+  /// May add implicit CUDAConstantAttr attribute to VD, depending on VD
+  /// and current compilation settings.
+  void MaybeAddConstantAttr(VarDecl *VD);
+
+  /// Check whether we're allowed to call Callee from the current context.
+  ///
+  /// - If the call is never allowed in a semantically-correct program
+  ///   (CFP_Never), emits an error and returns false.
+  ///
+  /// - If the call is allowed in semantically-correct programs, but only if
+  ///   it's never codegen'ed (CFP_WrongSide), creates a deferred diagnostic to
+  ///   be emitted if and when the caller is codegen'ed, and returns true.
+  ///
+  ///   Will only create deferred diagnostics for a given SourceLocation once,
+  ///   so you can safely call this multiple times without generating duplicate
+  ///   deferred errors.
+  ///
+  /// - Otherwise, returns true without emitting any diagnostics.
+  bool CheckCall(SourceLocation Loc, FunctionDecl *Callee);
+
+  void CheckLambdaCapture(CXXMethodDecl *D, const sema::Capture &Capture);
+
+  /// Set __device__ or __host__ __device__ attributes on the given lambda
+  /// operator() method.
+  ///
+  /// CUDA lambdas by default is host device function unless it has explicit
+  /// host or device attribute.
+  void SetLambdaAttrs(CXXMethodDecl *Method);
+
+  /// Record \p FD if it is a CUDA/HIP implicit host device function used on
+  /// device side in device compilation.
+  void RecordImplicitHostDeviceFuncUsedByDevice(const FunctionDecl *FD);
+
+  /// Finds a function in \p Matches with highest calling priority
+  /// from \p Caller context and erases all functions with lower
+  /// calling priority.
+  void EraseUnwantedMatches(
+      const FunctionDecl *Caller,
+      llvm::SmallVectorImpl<std::pair<DeclAccessPair, FunctionDecl *>>
+          &Matches);
+
+  /// Given a implicit special member, infer its CUDA target from the
+  /// calls it needs to make to underlying base/field special members.
+  /// \param ClassDecl the class for which the member is being created.
+  /// \param CSM the kind of special member.
+  /// \param MemberDecl the special member itself.
+  /// \param ConstRHS true if this is a copy operation with a const object on
+  ///        its RHS.
+  /// \param Diagnose true if this call should emit diagnostics.
+  /// \return true if there was an error inferring.
+  /// The result of this call is implicit CUDA target attribute(s) attached to
+  /// the member declaration.
+  bool inferTargetForImplicitSpecialMember(CXXRecordDecl *ClassDecl,
+                                           CXXSpecialMemberKind CSM,
+                                           CXXMethodDecl *MemberDecl,
+                                           bool ConstRHS, bool Diagnose);
+
+  /// \return true if \p CD can be considered empty according to CUDA
+  /// (E.2.3.1 in CUDA 7.5 Programming guide).
+  bool isEmptyConstructor(SourceLocation Loc, CXXConstructorDecl *CD);
+  bool isEmptyDestructor(SourceLocation Loc, CXXDestructorDecl *CD);
+
+  // \brief Checks that initializers of \p Var satisfy CUDA restrictions. In
+  // case of error emits appropriate diagnostic and invalidates \p Var.
+  //
+  // \details CUDA allows only empty constructors as initializers for global
+  // variables (see E.2.3.1, CUDA 7.5). The same restriction also applies to all
+  // __shared__ variables whether they are local or not (they all are implicitly
+  // static in CUDA). One exception is that CUDA allows constant initializers
+  // for __constant__ and __device__ variables.
+  void checkAllowedInitializer(VarDecl *VD);
+
+  /// Check whether NewFD is a valid overload for CUDA. Emits
+  /// diagnostics and invalidates NewFD if not.
+  void checkTargetOverload(FunctionDecl *NewFD, const LookupResult &Previous);
+  /// Copies target attributes from the template TD to the function FD.
+  void inheritTargetAttrs(FunctionDecl *FD, const FunctionTemplateDecl &TD);
+
+  /// Returns the name of the launch configuration function.  This is the name
+  /// of the function that will be called to configure kernel call, with the
+  /// parameters specified via <<<>>>.
+  std::string getConfigureFuncName() const;
+
+private:
+  unsigned ForceHostDeviceDepth = 0;
+
+  friend class ASTReader;
+  friend class ASTWriter;
+};
+
+} // namespace clang
+
+namespace llvm {
+// Hash a FunctionDeclAndLoc by looking at both its FunctionDecl and its
+// SourceLocation.
+template <> struct DenseMapInfo<clang::SemaCUDA::FunctionDeclAndLoc> {
+  using FunctionDeclAndLoc = clang::SemaCUDA::FunctionDeclAndLoc;
+  using FDBaseInfo =
+      DenseMapInfo<clang::CanonicalDeclPtr<const clang::FunctionDecl>>;
+
+  static FunctionDeclAndLoc getEmptyKey() {
+    return {FDBaseInfo::getEmptyKey(), clang::SourceLocation()};
+  }
+
+  static FunctionDeclAndLoc getTombstoneKey() {
+    return {FDBaseInfo::getTombstoneKey(), clang::SourceLocation()};
+  }
+
+  static unsigned getHashValue(const FunctionDeclAndLoc &FDL) {
+    return hash_combine(FDBaseInfo::getHashValue(FDL.FD),
+                        FDL.Loc.getHashValue());
+  }
+
+  static bool isEqual(const FunctionDeclAndLoc &LHS,
+                      const FunctionDeclAndLoc &RHS) {
+    return LHS.FD == RHS.FD && LHS.Loc == RHS.Loc;
+  }
+};
+} // namespace llvm
+
+#endif // LLVM_CLANG_SEMA_SEMACUDA_H
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index 6656c1c58dec..e3fde887f99c 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -873,7 +873,7 @@ private:
 
   /// Our current depth in #pragma cuda force_host_device begin/end
   /// macros.
-  unsigned ForceCUDAHostDeviceDepth = 0;
+  unsigned ForceHostDeviceDepth = 0;
 
   /// The IDs of the declarations Sema stores directly.
   ///
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index 583232f2d610..951e2210031a 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -26,6 +26,7 @@
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/ParsedTemplate.h"
 #include "clang/Sema/Scope.h"
+#include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaDiagnostic.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallString.h"
@@ -2664,7 +2665,8 @@ Decl *Parser::ParseDeclarationAfterDeclaratorAndAttributes(
     }
   }
 
-  Sema::CUDATargetContextRAII X(Actions, Sema::CTCK_InitGlobalVar, ThisDecl);
+  SemaCUDA::CUDATargetContextRAII X(Actions.CUDA(),
+                                    SemaCUDA::CTCK_InitGlobalVar, ThisDecl);
   switch (TheInitKind) {
   // Parse declarator '=' initializer.
   case InitKind::Equal: {
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index d08e675604d1..473ec9afd601 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -30,6 +30,7 @@
 #include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/ParsedTemplate.h"
 #include "clang/Sema/Scope.h"
+#include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaSYCL.h"
 #include "clang/Sema/TypoCorrection.h"
 #include "llvm/ADT/SmallVector.h"
@@ -2129,10 +2130,8 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
         }
 
         if (!LHS.isInvalid()) {
-          ExprResult ECResult = Actions.ActOnCUDAExecConfigExpr(getCurScope(),
-                                    OpenLoc,
-                                    ExecConfigExprs,
-                                    CloseLoc);
+          ExprResult ECResult = Actions.CUDA().ActOnExecConfigExpr(
+              getCurScope(), OpenLoc, ExecConfigExprs, CloseLoc);
           if (ECResult.isInvalid())
             LHS = ExprError();
           else
diff --git a/clang/lib/Parse/ParsePragma.cpp b/clang/lib/Parse/ParsePragma.cpp
index 0f692e2146a4..3979f75b6020 100644
--- a/clang/lib/Parse/ParsePragma.cpp
+++ b/clang/lib/Parse/ParsePragma.cpp
@@ -21,6 +21,7 @@
 #include "clang/Parse/RAIIObjectsForParser.h"
 #include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/Scope.h"
+#include "clang/Sema/SemaCUDA.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringSwitch.h"
 #include <optional>
@@ -3900,8 +3901,8 @@ void PragmaForceCUDAHostDeviceHandler::HandlePragma(
   }
 
   if (Info->isStr("begin"))
-    Actions.PushForceCUDAHostDevice();
-  else if (!Actions.PopForceCUDAHostDevice())
+    Actions.CUDA().PushForceHostDevice();
+  else if (!Actions.CUDA().PopForceHostDevice())
     PP.Diag(FirstTok.getLocation(),
             diag::err_pragma_cannot_end_force_cuda_host_device);
 
diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index a2ea66f339c8..8de202f4f7a0 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -41,6 +41,7 @@
 #include "clang/Sema/RISCVIntrinsicManager.h"
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/ScopeInfo.h"
+#include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaConsumer.h"
 #include "clang/Sema/SemaHLSL.h"
 #include "clang/Sema/SemaInternal.h"
@@ -199,6 +200,7 @@ Sema::Sema(Preprocessor &pp, ASTContext &ctxt, ASTConsumer &consumer,
       LateTemplateParser(nullptr), LateTemplateParserCleanup(nullptr),
       OpaqueParser(nullptr), CurContext(nullptr), ExternalSource(nullptr),
       CurScope(nullptr), Ident_super(nullptr),
+      CUDAPtr(std::make_unique<SemaCUDA>(*this)),
       HLSLPtr(std::make_unique<SemaHLSL>(*this)),
       OpenACCPtr(std::make_unique<SemaOpenACC>(*this)),
       SYCLPtr(std::make_unique<SemaSYCL>(*this)),
@@ -1635,15 +1637,15 @@ bool Sema::hasUncompilableErrorOccurred() const {
 // Print notes showing how we can reach FD starting from an a priori
 // known-callable function.
 static void emitCallStackNotes(Sema &S, const FunctionDecl *FD) {
-  auto FnIt = S.DeviceKnownEmittedFns.find(FD);
-  while (FnIt != S.DeviceKnownEmittedFns.end()) {
+  auto FnIt = S.CUDA().DeviceKnownEmittedFns.find(FD);
+  while (FnIt != S.CUDA().DeviceKnownEmittedFns.end()) {
     // Respect error limit.
     if (S.Diags.hasFatalErrorOccurred())
       return;
     DiagnosticBuilder Builder(
         S.Diags.Report(FnIt->second.Loc, diag::note_called_by));
     Builder << FnIt->second.FD;
-    FnIt = S.DeviceKnownEmittedFns.find(FnIt->second.FD);
+    FnIt = S.CUDA().DeviceKnownEmittedFns.find(FnIt->second.FD);
   }
 }
 
@@ -1747,7 +1749,7 @@ public:
         (ShouldEmitRootNode || InOMPDeviceContext))
       S.finalizeOpenMPDelayedAnalysis(Caller, FD, Loc);
     if (Caller)
-      S.DeviceKnownEmittedFns[FD] = {Caller, Loc};
+      S.CUDA().DeviceKnownEmittedFns[FD] = {Caller, Loc};
     // Always emit deferred diagnostics for the direct users. This does not
     // lead to explosion of diagnostics since each user is visited at most
     // twice.
@@ -1836,8 +1838,8 @@ void Sema::emitDeferredDiags() {
 // which other not-known-emitted functions.
 //
 // When we see something which is illegal if the current function is emitted
-// (usually by way of CUDADiagIfDeviceCode, CUDADiagIfHostCode, or
-// CheckCUDACall), we first check if the current function is known-emitted.  If
+// (usually by way of DiagIfDeviceCode, DiagIfHostCode, or
+// CheckCall), we first check if the current function is known-emitted.  If
 // so, we immediately output the diagnostic.
 //
 // Otherwise, we "defer" the diagnostic.  It sits in Sema::DeviceDeferredDiags
@@ -1900,8 +1902,8 @@ Sema::targetDiag(SourceLocation Loc, unsigned DiagID, const FunctionDecl *FD) {
                ? diagIfOpenMPDeviceCode(Loc, DiagID, FD)
                : diagIfOpenMPHostCode(Loc, DiagID, FD);
   if (getLangOpts().CUDA)
-    return getLangOpts().CUDAIsDevice ? CUDADiagIfDeviceCode(Loc, DiagID)
-                                      : CUDADiagIfHostCode(Loc, DiagID);
+    return getLangOpts().CUDAIsDevice ? CUDA().DiagIfDeviceCode(Loc, DiagID)
+                                      : CUDA().DiagIfHostCode(Loc, DiagID);
 
   if (getLangOpts().SYCLIsDevice)
     return SYCL().DiagIfDeviceCode(Loc, DiagID);
diff --git a/clang/lib/Sema/SemaBase.cpp b/clang/lib/Sema/SemaBase.cpp
index 95c0cfbe283b..0442fb2929e3 100644
--- a/clang/lib/Sema/SemaBase.cpp
+++ b/clang/lib/Sema/SemaBase.cpp
@@ -1,5 +1,6 @@
 #include "clang/Sema/SemaBase.h"
 #include "clang/Sema/Sema.h"
+#include "clang/Sema/SemaCUDA.h"
 
 namespace clang {
 
@@ -70,8 +71,8 @@ Sema::SemaDiagnosticBuilder SemaBase::Diag(SourceLocation Loc, unsigned DiagID,
   }
 
   SemaDiagnosticBuilder DB = getLangOpts().CUDAIsDevice
-                                 ? SemaRef.CUDADiagIfDeviceCode(Loc, DiagID)
-                                 : SemaRef.CUDADiagIfHostCode(Loc, DiagID);
+                                 ? SemaRef.CUDA().DiagIfDeviceCode(Loc, DiagID)
+                                 : SemaRef.CUDA().DiagIfHostCode(Loc, DiagID);
   SetIsLastErrorImmediate(DB.isImmediate());
   return DB;
 }
diff --git a/clang/lib/Sema/SemaCUDA.cpp b/clang/lib/Sema/SemaCUDA.cpp
index 9d6d709e262a..80ea43dc5316 100644
--- a/clang/lib/Sema/SemaCUDA.cpp
+++ b/clang/lib/Sema/SemaCUDA.cpp
@@ -10,6 +10,7 @@
 ///
 //===----------------------------------------------------------------------===//
 
+#include "clang/Sema/SemaCUDA.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/ExprCXX.h"
@@ -27,6 +28,8 @@
 #include <optional>
 using namespace clang;
 
+SemaCUDA::SemaCUDA(Sema &S) : SemaBase(S) {}
+
 template <typename AttrT> static bool hasExplicitAttr(const VarDecl *D) {
   if (!D)
     return false;
@@ -35,37 +38,37 @@ template <typename AttrT> static bool hasExplicitAttr(const VarDecl *D) {
   return false;
 }
 
-void Sema::PushForceCUDAHostDevice() {
+void SemaCUDA::PushForceHostDevice() {
   assert(getLangOpts().CUDA && "Should only be called during CUDA compilation");
-  ForceCUDAHostDeviceDepth++;
+  ForceHostDeviceDepth++;
 }
 
-bool Sema::PopForceCUDAHostDevice() {
+bool SemaCUDA::PopForceHostDevice() {
   assert(getLangOpts().CUDA && "Should only be called during CUDA compilation");
-  if (ForceCUDAHostDeviceDepth == 0)
+  if (ForceHostDeviceDepth == 0)
     return false;
-  ForceCUDAHostDeviceDepth--;
+  ForceHostDeviceDepth--;
   return true;
 }
 
-ExprResult Sema::ActOnCUDAExecConfigExpr(Scope *S, SourceLocation LLLLoc,
+ExprResult SemaCUDA::ActOnExecConfigExpr(Scope *S, SourceLocation LLLLoc,
                                          MultiExprArg ExecConfig,
                                          SourceLocation GGGLoc) {
-  FunctionDecl *ConfigDecl = Context.getcudaConfigureCallDecl();
+  FunctionDecl *ConfigDecl = getASTContext().getcudaConfigureCallDecl();
   if (!ConfigDecl)
     return ExprError(Diag(LLLLoc, diag::err_undeclared_var_use)
-                     << getCudaConfigureFuncName());
+                     << getConfigureFuncName());
   QualType ConfigQTy = ConfigDecl->getType();
 
-  DeclRefExpr *ConfigDR = new (Context)
-      DeclRefExpr(Context, ConfigDecl, false, ConfigQTy, VK_LValue, LLLLoc);
-  MarkFunctionReferenced(LLLLoc, ConfigDecl);
+  DeclRefExpr *ConfigDR = new (getASTContext()) DeclRefExpr(
+      getASTContext(), ConfigDecl, false, ConfigQTy, VK_LValue, LLLLoc);
+  SemaRef.MarkFunctionReferenced(LLLLoc, ConfigDecl);
 
-  return BuildCallExpr(S, ConfigDR, LLLLoc, ExecConfig, GGGLoc, nullptr,
-                       /*IsExecConfig=*/true);
+  return SemaRef.BuildCallExpr(S, ConfigDR, LLLLoc, ExecConfig, GGGLoc, nullptr,
+                               /*IsExecConfig=*/true);
 }
 
-CUDAFunctionTarget Sema::IdentifyCUDATarget(const ParsedAttributesView &Attrs) {
+CUDAFunctionTarget SemaCUDA::IdentifyTarget(const ParsedAttributesView &Attrs) {
   bool HasHostAttr = false;
   bool HasDeviceAttr = false;
   bool HasGlobalAttr = false;
@@ -112,12 +115,11 @@ static bool hasAttr(const Decl *D, bool IgnoreImplicitAttr) {
          });
 }
 
-Sema::CUDATargetContextRAII::CUDATargetContextRAII(Sema &S_,
-                                                   CUDATargetContextKind K,
-                                                   Decl *D)
+SemaCUDA::CUDATargetContextRAII::CUDATargetContextRAII(
+    SemaCUDA &S_, SemaCUDA::CUDATargetContextKind K, Decl *D)
     : S(S_) {
   SavedCtx = S.CurCUDATargetCtx;
-  assert(K == CTCK_InitGlobalVar);
+  assert(K == SemaCUDA::CTCK_InitGlobalVar);
   auto *VD = dyn_cast_or_null<VarDecl>(D);
   if (VD && VD->hasGlobalStorage() && !VD->isStaticLocal()) {
     auto Target = CUDAFunctionTarget::Host;
@@ -130,8 +132,8 @@ Sema::CUDATargetContextRAII::CUDATargetContextRAII(Sema &S_,
   }
 }
 
-/// IdentifyCUDATarget - Determine the CUDA compilation target for this function
-CUDAFunctionTarget Sema::IdentifyCUDATarget(const FunctionDecl *D,
+/// IdentifyTarget - Determine the CUDA compilation target for this function
+CUDAFunctionTarget SemaCUDA::IdentifyTarget(const FunctionDecl *D,
                                             bool IgnoreImplicitHDAttr) {
   // Code that lives outside a function gets the target from CurCUDATargetCtx.
   if (D == nullptr)
@@ -160,7 +162,7 @@ CUDAFunctionTarget Sema::IdentifyCUDATarget(const FunctionDecl *D,
 }
 
 /// IdentifyTarget - Determine the CUDA compilation target for this variable.
-Sema::CUDAVariableTarget Sema::IdentifyCUDATarget(const VarDecl *Var) {
+SemaCUDA::CUDAVariableTarget SemaCUDA::IdentifyTarget(const VarDecl *Var) {
   if (Var->hasAttr<HIPManagedAttr>())
     return CVT_Unified;
   // Only constexpr and const variabless with implicit constant attribute
@@ -180,7 +182,7 @@ Sema::CUDAVariableTarget Sema::IdentifyCUDATarget(const VarDecl *Var) {
   //  - on both sides in host device functions
   //  - on device side in device or global functions
   if (auto *FD = dyn_cast<FunctionDecl>(Var->getDeclContext())) {
-    switch (IdentifyCUDATarget(FD)) {
+    switch (IdentifyTarget(FD)) {
     case CUDAFunctionTarget::HostDevice:
       return CVT_Both;
     case CUDAFunctionTarget::Device:
@@ -221,21 +223,21 @@ Sema::CUDAVariableTarget Sema::IdentifyCUDATarget(const VarDecl *Var) {
 // | hd | h  | SS  | WS  | (d) |
 // | hd | hd | HD  | HD  | (b) |
 
-Sema::CUDAFunctionPreference
-Sema::IdentifyCUDAPreference(const FunctionDecl *Caller,
+SemaCUDA::CUDAFunctionPreference
+SemaCUDA::IdentifyPreference(const FunctionDecl *Caller,
                              const FunctionDecl *Callee) {
   assert(Callee && "Callee must be valid.");
 
   // Treat ctor/dtor as host device function in device var initializer to allow
   // trivial ctor/dtor without device attr to be used. Non-trivial ctor/dtor
-  // will be diagnosed by checkAllowedCUDAInitializer.
+  // will be diagnosed by checkAllowedInitializer.
   if (Caller == nullptr && CurCUDATargetCtx.Kind == CTCK_InitGlobalVar &&
       CurCUDATargetCtx.Target == CUDAFunctionTarget::Device &&
       (isa<CXXConstructorDecl>(Callee) || isa<CXXDestructorDecl>(Callee)))
     return CFP_HostDevice;
 
-  CUDAFunctionTarget CallerTarget = IdentifyCUDATarget(Caller);
-  CUDAFunctionTarget CalleeTarget = IdentifyCUDATarget(Callee);
+  CUDAFunctionTarget CallerTarget = IdentifyTarget(Caller);
+  CUDAFunctionTarget CalleeTarget = IdentifyTarget(Callee);
 
   // If one of the targets is invalid, the check always fails, no matter what
   // the other target is.
@@ -309,13 +311,13 @@ template <typename AttrT> static bool hasImplicitAttr(const FunctionDecl *D) {
   return D->isImplicit();
 }
 
-bool Sema::isCUDAImplicitHostDeviceFunction(const FunctionDecl *D) {
+bool SemaCUDA::isImplicitHostDeviceFunction(const FunctionDecl *D) {
   bool IsImplicitDevAttr = hasImplicitAttr<CUDADeviceAttr>(D);
   bool IsImplicitHostAttr = hasImplicitAttr<CUDAHostAttr>(D);
   return IsImplicitDevAttr && IsImplicitHostAttr;
 }
 
-void Sema::EraseUnwantedCUDAMatches(
+void SemaCUDA::EraseUnwantedMatches(
     const FunctionDecl *Caller,
     SmallVectorImpl<std::pair<DeclAccessPair, FunctionDecl *>> &Matches) {
   if (Matches.size() <= 1)
@@ -325,7 +327,7 @@ void Sema::EraseUnwantedCUDAMatches(
 
   // Gets the CUDA function preference for a call from Caller to Match.
   auto GetCFP = [&](const Pair &Match) {
-    return IdentifyCUDAPreference(Caller, Match.second);
+    return IdentifyPreference(Caller, Match.second);
   };
 
   // Find the best call preference among the functions in Matches.
@@ -367,7 +369,7 @@ resolveCalleeCUDATargetConflict(CUDAFunctionTarget Target1,
   return false;
 }
 
-bool Sema::inferCUDATargetForImplicitSpecialMember(CXXRecordDecl *ClassDecl,
+bool SemaCUDA::inferTargetForImplicitSpecialMember(CXXRecordDecl *ClassDecl,
                                                    CXXSpecialMemberKind CSM,
                                                    CXXMethodDecl *MemberDecl,
                                                    bool ConstRHS,
@@ -388,7 +390,7 @@ bool Sema::inferCUDATargetForImplicitSpecialMember(CXXRecordDecl *ClassDecl,
 
   // We're going to invoke special member lookup; mark that these special
   // members are called from this one, and not from its caller.
-  ContextRAII MethodContext(*this, MemberDecl);
+  Sema::ContextRAII MethodContext(SemaRef, MemberDecl);
 
   // Look for special members in base classes that should be invoked from here.
   // Infer the target of this member base on the ones it should call.
@@ -412,17 +414,17 @@ bool Sema::inferCUDATargetForImplicitSpecialMember(CXXRecordDecl *ClassDecl,
 
     CXXRecordDecl *BaseClassDecl = cast<CXXRecordDecl>(BaseType->getDecl());
     Sema::SpecialMemberOverloadResult SMOR =
-        LookupSpecialMember(BaseClassDecl, CSM,
-                            /* ConstArg */ ConstRHS,
-                            /* VolatileArg */ false,
-                            /* RValueThis */ false,
-                            /* ConstThis */ false,
-                            /* VolatileThis */ false);
+        SemaRef.LookupSpecialMember(BaseClassDecl, CSM,
+                                    /* ConstArg */ ConstRHS,
+                                    /* VolatileArg */ false,
+                                    /* RValueThis */ false,
+                                    /* ConstThis */ false,
+                                    /* VolatileThis */ false);
 
     if (!SMOR.getMethod())
       continue;
 
-    CUDAFunctionTarget BaseMethodTarget = IdentifyCUDATarget(SMOR.getMethod());
+    CUDAFunctionTarget BaseMethodTarget = IdentifyTarget(SMOR.getMethod());
     if (!InferredTarget) {
       InferredTarget = BaseMethodTarget;
     } else {
@@ -435,7 +437,8 @@ bool Sema::inferCUDATargetForImplicitSpecialMember(CXXRecordDecl *ClassDecl,
               << (unsigned)CSM << llvm::to_underlying(*InferredTarget)
               << llvm::to_underlying(BaseMethodTarget);
         }
-        MemberDecl->addAttr(CUDAInvalidTargetAttr::CreateImplicit(Context));
+        MemberDecl->addAttr(
+            CUDAInvalidTargetAttr::CreateImplicit(getASTContext()));
         return true;
       }
     }
@@ -448,25 +451,24 @@ bool Sema::inferCUDATargetForImplicitSpecialMember(CXXRecordDecl *ClassDecl,
     }
 
     const RecordType *FieldType =
-        Context.getBaseElementType(F->getType())->getAs<RecordType>();
+        getASTContext().getBaseElementType(F->getType())->getAs<RecordType>();
     if (!FieldType) {
       continue;
     }
 
     CXXRecordDecl *FieldRecDecl = cast<CXXRecordDecl>(FieldType->getDecl());
     Sema::SpecialMemberOverloadResult SMOR =
-        LookupSpecialMember(FieldRecDecl, CSM,
-                            /* ConstArg */ ConstRHS && !F->isMutable(),
-                            /* VolatileArg */ false,
-                            /* RValueThis */ false,
-                            /* ConstThis */ false,
-                            /* VolatileThis */ false);
+        SemaRef.LookupSpecialMember(FieldRecDecl, CSM,
+                                    /* ConstArg */ ConstRHS && !F->isMutable(),
+                                    /* VolatileArg */ false,
+                                    /* RValueThis */ false,
+                                    /* ConstThis */ false,
+                                    /* VolatileThis */ false);
 
     if (!SMOR.getMethod())
       continue;
 
-    CUDAFunctionTarget FieldMethodTarget =
-        IdentifyCUDATarget(SMOR.getMethod());
+    CUDAFunctionTarget FieldMethodTarget = IdentifyTarget(SMOR.getMethod());
     if (!InferredTarget) {
       InferredTarget = FieldMethodTarget;
     } else {
@@ -479,7 +481,8 @@ bool Sema::inferCUDATargetForImplicitSpecialMember(CXXRecordDecl *ClassDecl,
               << (unsigned)CSM << llvm::to_underlying(*InferredTarget)
               << llvm::to_underlying(FieldMethodTarget);
         }
-        MemberDecl->addAttr(CUDAInvalidTargetAttr::CreateImplicit(Context));
+        MemberDecl->addAttr(
+            CUDAInvalidTargetAttr::CreateImplicit(getASTContext()));
         return true;
       }
     }
@@ -499,16 +502,16 @@ bool Sema::inferCUDATargetForImplicitSpecialMember(CXXRecordDecl *ClassDecl,
   // We either setting attributes first time, or the inferred ones must match
   // previously set ones.
   if (NeedsD && !HasD)
-    MemberDecl->addAttr(CUDADeviceAttr::CreateImplicit(Context));
+    MemberDecl->addAttr(CUDADeviceAttr::CreateImplicit(getASTContext()));
   if (NeedsH && !HasH)
-    MemberDecl->addAttr(CUDAHostAttr::CreateImplicit(Context));
+    MemberDecl->addAttr(CUDAHostAttr::CreateImplicit(getASTContext()));
 
   return false;
 }
 
-bool Sema::isEmptyCudaConstructor(SourceLocation Loc, CXXConstructorDecl *CD) {
+bool SemaCUDA::isEmptyConstructor(SourceLocation Loc, CXXConstructorDecl *CD) {
   if (!CD->isDefined() && CD->isTemplateInstantiation())
-    InstantiateFunctionDefinition(Loc, CD->getFirstDecl());
+    SemaRef.InstantiateFunctionDefinition(Loc, CD->getFirstDecl());
 
   // (E.2.3.1, CUDA 7.5) A constructor for a class type is considered
   // empty at a point in the translation unit, if it is either a
@@ -536,7 +539,7 @@ bool Sema::isEmptyCudaConstructor(SourceLocation Loc, CXXConstructorDecl *CD) {
   if (!llvm::all_of(CD->inits(), [&](const CXXCtorInitializer *CI) {
         if (const CXXConstructExpr *CE =
                 dyn_cast<CXXConstructExpr>(CI->getInit()))
-          return isEmptyCudaConstructor(Loc, CE->getConstructor());
+          return isEmptyConstructor(Loc, CE->getConstructor());
         return false;
       }))
     return false;
@@ -544,13 +547,13 @@ bool Sema::isEmptyCudaConstructor(SourceLocation Loc, CXXConstructorDecl *CD) {
   return true;
 }
 
-bool Sema::isEmptyCudaDestructor(SourceLocation Loc, CXXDestructorDecl *DD) {
+bool SemaCUDA::isEmptyDestructor(SourceLocation Loc, CXXDestructorDecl *DD) {
   // No destructor -> no problem.
   if (!DD)
     return true;
 
   if (!DD->isDefined() && DD->isTemplateInstantiation())
-    InstantiateFunctionDefinition(Loc, DD->getFirstDecl());
+    SemaRef.InstantiateFunctionDefinition(Loc, DD->getFirstDecl());
 
   // (E.2.3.1, CUDA 7.5) A destructor for a class type is considered
   // empty at a point in the translation unit, if it is either a
@@ -579,7 +582,7 @@ bool Sema::isEmptyCudaDestructor(SourceLocation Loc, CXXDestructorDecl *DD) {
   // destructors for all base classes...
   if (!llvm::all_of(ClassDecl->bases(), [&](const CXXBaseSpecifier &BS) {
         if (CXXRecordDecl *RD = BS.getType()->getAsCXXRecordDecl())
-          return isEmptyCudaDestructor(Loc, RD->getDestructor());
+          return isEmptyDestructor(Loc, RD->getDestructor());
         return true;
       }))
     return false;
@@ -589,7 +592,7 @@ bool Sema::isEmptyCudaDestructor(SourceLocation Loc, CXXDestructorDecl *DD) {
         if (CXXRecordDecl *RD = Field->getType()
                                     ->getBaseElementTypeUnsafe()
                                     ->getAsCXXRecordDecl())
-          return isEmptyCudaDestructor(Loc, RD->getDestructor());
+          return isEmptyDestructor(Loc, RD->getDestructor());
         return true;
       }))
     return false;
@@ -620,7 +623,7 @@ bool IsDependentVar(VarDecl *VD) {
 // __shared__ variables whether they are local or not (they all are implicitly
 // static in CUDA). One exception is that CUDA allows constant initializers
 // for __constant__ and __device__ variables.
-bool HasAllowedCUDADeviceStaticInitializer(Sema &S, VarDecl *VD,
+bool HasAllowedCUDADeviceStaticInitializer(SemaCUDA &S, VarDecl *VD,
                                            CUDAInitializerCheckKind CheckKind) {
   assert(!VD->isInvalidDecl() && VD->hasGlobalStorage());
   assert(!IsDependentVar(VD) && "do not check dependent var");
@@ -629,30 +632,30 @@ bool HasAllowedCUDADeviceStaticInitializer(Sema &S, VarDecl *VD,
     if (!Init)
       return true;
     if (const auto *CE = dyn_cast<CXXConstructExpr>(Init)) {
-      return S.isEmptyCudaConstructor(VD->getLocation(), CE->getConstructor());
+      return S.isEmptyConstructor(VD->getLocation(), CE->getConstructor());
     }
     return false;
   };
   auto IsConstantInit = [&](const Expr *Init) {
     assert(Init);
-    ASTContext::CUDAConstantEvalContextRAII EvalCtx(S.Context,
+    ASTContext::CUDAConstantEvalContextRAII EvalCtx(S.getASTContext(),
                                                     /*NoWronSidedVars=*/true);
-    return Init->isConstantInitializer(S.Context,
+    return Init->isConstantInitializer(S.getASTContext(),
                                        VD->getType()->isReferenceType());
   };
   auto HasEmptyDtor = [&](VarDecl *VD) {
     if (const auto *RD = VD->getType()->getAsCXXRecordDecl())
-      return S.isEmptyCudaDestructor(VD->getLocation(), RD->getDestructor());
+      return S.isEmptyDestructor(VD->getLocation(), RD->getDestructor());
     return true;
   };
   if (CheckKind == CICK_Shared)
     return IsEmptyInit(Init) && HasEmptyDtor(VD);
-  return S.LangOpts.GPUAllowDeviceInit ||
+  return S.getLangOpts().GPUAllowDeviceInit ||
          ((IsEmptyInit(Init) || IsConstantInit(Init)) && HasEmptyDtor(VD));
 }
 } // namespace
 
-void Sema::checkAllowedCUDAInitializer(VarDecl *VD) {
+void SemaCUDA::checkAllowedInitializer(VarDecl *VD) {
   // Return early if VD is inside a non-instantiated template function since
   // the implicit constructor is not defined yet.
   if (const FunctionDecl *FD =
@@ -688,7 +691,7 @@ void Sema::checkAllowedCUDAInitializer(VarDecl *VD) {
       InitFn = CE->getDirectCallee();
     }
     if (InitFn) {
-      CUDAFunctionTarget InitFnTarget = IdentifyCUDATarget(InitFn);
+      CUDAFunctionTarget InitFnTarget = IdentifyTarget(InitFn);
       if (InitFnTarget != CUDAFunctionTarget::Host &&
           InitFnTarget != CUDAFunctionTarget::HostDevice) {
         Diag(VD->getLocation(), diag::err_ref_bad_target_global_initializer)
@@ -700,22 +703,22 @@ void Sema::checkAllowedCUDAInitializer(VarDecl *VD) {
   }
 }
 
-void Sema::CUDARecordImplicitHostDeviceFuncUsedByDevice(
+void SemaCUDA::RecordImplicitHostDeviceFuncUsedByDevice(
     const FunctionDecl *Callee) {
-  FunctionDecl *Caller = getCurFunctionDecl(/*AllowLambda=*/true);
+  FunctionDecl *Caller = SemaRef.getCurFunctionDecl(/*AllowLambda=*/true);
   if (!Caller)
     return;
 
-  if (!isCUDAImplicitHostDeviceFunction(Callee))
+  if (!isImplicitHostDeviceFunction(Callee))
     return;
 
-  CUDAFunctionTarget CallerTarget = IdentifyCUDATarget(Caller);
+  CUDAFunctionTarget CallerTarget = IdentifyTarget(Caller);
 
   // Record whether an implicit host device function is used on device side.
   if (CallerTarget != CUDAFunctionTarget::Device &&
       CallerTarget != CUDAFunctionTarget::Global &&
       (CallerTarget != CUDAFunctionTarget::HostDevice ||
-       (isCUDAImplicitHostDeviceFunction(Caller) &&
+       (isImplicitHostDeviceFunction(Caller) &&
         !getASTContext().CUDAImplicitHostDeviceFunUsedByDevice.count(Caller))))
     return;
 
@@ -731,18 +734,18 @@ void Sema::CUDARecordImplicitHostDeviceFuncUsedByDevice(
 //    system header, in which case we leave the constexpr function unattributed.
 //
 // In addition, all function decls are treated as __host__ __device__ when
-// ForceCUDAHostDeviceDepth > 0 (corresponding to code within a
+// ForceHostDeviceDepth > 0 (corresponding to code within a
 //   #pragma clang force_cuda_host_device_begin/end
 // pair).
-void Sema::maybeAddCUDAHostDeviceAttrs(FunctionDecl *NewD,
+void SemaCUDA::maybeAddHostDeviceAttrs(FunctionDecl *NewD,
                                        const LookupResult &Previous) {
   assert(getLangOpts().CUDA && "Should only be called during CUDA compilation");
 
-  if (ForceCUDAHostDeviceDepth > 0) {
+  if (ForceHostDeviceDepth > 0) {
     if (!NewD->hasAttr<CUDAHostAttr>())
-      NewD->addAttr(CUDAHostAttr::CreateImplicit(Context));
+      NewD->addAttr(CUDAHostAttr::CreateImplicit(getASTContext()));
     if (!NewD->hasAttr<CUDADeviceAttr>())
-      NewD->addAttr(CUDADeviceAttr::CreateImplicit(Context));
+      NewD->addAttr(CUDADeviceAttr::CreateImplicit(getASTContext()));
     return;
   }
 
@@ -753,8 +756,8 @@ void Sema::maybeAddCUDAHostDeviceAttrs(FunctionDecl *NewD,
       !NewD->hasAttr<CUDAGlobalAttr>() &&
       (NewD->getDescribedFunctionTemplate() ||
        NewD->isFunctionTemplateSpecialization())) {
-    NewD->addAttr(CUDAHostAttr::CreateImplicit(Context));
-    NewD->addAttr(CUDADeviceAttr::CreateImplicit(Context));
+    NewD->addAttr(CUDAHostAttr::CreateImplicit(getASTContext()));
+    NewD->addAttr(CUDADeviceAttr::CreateImplicit(getASTContext()));
     return;
   }
 
@@ -771,8 +774,9 @@ void Sema::maybeAddCUDAHostDeviceAttrs(FunctionDecl *NewD,
     FunctionDecl *OldD = D->getAsFunction();
     return OldD && OldD->hasAttr<CUDADeviceAttr>() &&
            !OldD->hasAttr<CUDAHostAttr>() &&
-           !IsOverload(NewD, OldD, /* UseMemberUsingDeclRules = */ false,
-                       /* ConsiderCudaAttrs = */ false);
+           !SemaRef.IsOverload(NewD, OldD,
+                               /* UseMemberUsingDeclRules = */ false,
+                               /* ConsiderCudaAttrs = */ false);
   };
   auto It = llvm::find_if(Previous, IsMatchingDeviceFn);
   if (It != Previous.end()) {
@@ -781,7 +785,7 @@ void Sema::maybeAddCUDAHostDeviceAttrs(FunctionDecl *NewD,
     // in a system header, in which case we simply return without making NewD
     // host+device.
     NamedDecl *Match = *It;
-    if (!getSourceManager().isInSystemHeader(Match->getLocation())) {
+    if (!SemaRef.getSourceManager().isInSystemHeader(Match->getLocation())) {
       Diag(NewD->getLocation(),
            diag::err_cuda_unattributed_constexpr_cannot_overload_device)
           << NewD;
@@ -791,14 +795,14 @@ void Sema::maybeAddCUDAHostDeviceAttrs(FunctionDecl *NewD,
     return;
   }
 
-  NewD->addAttr(CUDAHostAttr::CreateImplicit(Context));
-  NewD->addAttr(CUDADeviceAttr::CreateImplicit(Context));
+  NewD->addAttr(CUDAHostAttr::CreateImplicit(getASTContext()));
+  NewD->addAttr(CUDADeviceAttr::CreateImplicit(getASTContext()));
 }
 
 // TODO: `__constant__` memory may be a limited resource for certain targets.
 // A safeguard may be needed at the end of compilation pipeline if
 // `__constant__` memory usage goes beyond limit.
-void Sema::MaybeAddCUDAConstantAttr(VarDecl *VD) {
+void SemaCUDA::MaybeAddConstantAttr(VarDecl *VD) {
   // Do not promote dependent variables since the cotr/dtor/initializer are
   // not determined. Do it after instantiation.
   if (getLangOpts().CUDAIsDevice && !VD->hasAttr<CUDAConstantAttr>() &&
@@ -812,14 +816,15 @@ void Sema::MaybeAddCUDAConstantAttr(VarDecl *VD) {
   }
 }
 
-Sema::SemaDiagnosticBuilder Sema::CUDADiagIfDeviceCode(SourceLocation Loc,
-                                                       unsigned DiagID) {
+SemaBase::SemaDiagnosticBuilder SemaCUDA::DiagIfDeviceCode(SourceLocation Loc,
+                                                           unsigned DiagID) {
   assert(getLangOpts().CUDA && "Should only be called during CUDA compilation");
-  FunctionDecl *CurFunContext = getCurFunctionDecl(/*AllowLambda=*/true);
+  FunctionDecl *CurFunContext =
+      SemaRef.getCurFunctionDecl(/*AllowLambda=*/true);
   SemaDiagnosticBuilder::Kind DiagKind = [&] {
     if (!CurFunContext)
       return SemaDiagnosticBuilder::K_Nop;
-    switch (CurrentCUDATarget()) {
+    switch (CurrentTarget()) {
     case CUDAFunctionTarget::Global:
     case CUDAFunctionTarget::Device:
       return SemaDiagnosticBuilder::K_Immediate;
@@ -829,27 +834,29 @@ Sema::SemaDiagnosticBuilder Sema::CUDADiagIfDeviceCode(SourceLocation Loc,
       // mode until the function is known-emitted.
       if (!getLangOpts().CUDAIsDevice)
         return SemaDiagnosticBuilder::K_Nop;
-      if (IsLastErrorImmediate && Diags.getDiagnosticIDs()->isBuiltinNote(DiagID))
+      if (SemaRef.IsLastErrorImmediate &&
+          getDiagnostics().getDiagnosticIDs()->isBuiltinNote(DiagID))
         return SemaDiagnosticBuilder::K_Immediate;
-      return (getEmissionStatus(CurFunContext) ==
-              FunctionEmissionStatus::Emitted)
+      return (SemaRef.getEmissionStatus(CurFunContext) ==
+              Sema::FunctionEmissionStatus::Emitted)
                  ? SemaDiagnosticBuilder::K_ImmediateWithCallStack
                  : SemaDiagnosticBuilder::K_Deferred;
     default:
       return SemaDiagnosticBuilder::K_Nop;
     }
   }();
-  return SemaDiagnosticBuilder(DiagKind, Loc, DiagID, CurFunContext, *this);
+  return SemaDiagnosticBuilder(DiagKind, Loc, DiagID, CurFunContext, SemaRef);
 }
 
-Sema::SemaDiagnosticBuilder Sema::CUDADiagIfHostCode(SourceLocation Loc,
+Sema::SemaDiagnosticBuilder SemaCUDA::DiagIfHostCode(SourceLocation Loc,
                                                      unsigned DiagID) {
   assert(getLangOpts().CUDA && "Should only be called during CUDA compilation");
-  FunctionDecl *CurFunContext = getCurFunctionDecl(/*AllowLambda=*/true);
+  FunctionDecl *CurFunContext =
+      SemaRef.getCurFunctionDecl(/*AllowLambda=*/true);
   SemaDiagnosticBuilder::Kind DiagKind = [&] {
     if (!CurFunContext)
       return SemaDiagnosticBuilder::K_Nop;
-    switch (CurrentCUDATarget()) {
+    switch (CurrentTarget()) {
     case CUDAFunctionTarget::Host:
       return SemaDiagnosticBuilder::K_Immediate;
     case CUDAFunctionTarget::HostDevice:
@@ -858,40 +865,41 @@ Sema::SemaDiagnosticBuilder Sema::CUDADiagIfHostCode(SourceLocation Loc,
       // mode until the function is known-emitted.
       if (getLangOpts().CUDAIsDevice)
         return SemaDiagnosticBuilder::K_Nop;
-      if (IsLastErrorImmediate && Diags.getDiagnosticIDs()->isBuiltinNote(DiagID))
+      if (SemaRef.IsLastErrorImmediate &&
+          getDiagnostics().getDiagnosticIDs()->isBuiltinNote(DiagID))
         return SemaDiagnosticBuilder::K_Immediate;
-      return (getEmissionStatus(CurFunContext) ==
-              FunctionEmissionStatus::Emitted)
+      return (SemaRef.getEmissionStatus(CurFunContext) ==
+              Sema::FunctionEmissionStatus::Emitted)
                  ? SemaDiagnosticBuilder::K_ImmediateWithCallStack
                  : SemaDiagnosticBuilder::K_Deferred;
     default:
       return SemaDiagnosticBuilder::K_Nop;
     }
   }();
-  return SemaDiagnosticBuilder(DiagKind, Loc, DiagID, CurFunContext, *this);
+  return SemaDiagnosticBuilder(DiagKind, Loc, DiagID, CurFunContext, SemaRef);
 }
 
-bool Sema::CheckCUDACall(SourceLocation Loc, FunctionDecl *Callee) {
+bool SemaCUDA::CheckCall(SourceLocation Loc, FunctionDecl *Callee) {
   assert(getLangOpts().CUDA && "Should only be called during CUDA compilation");
   assert(Callee && "Callee may not be null.");
 
-  const auto &ExprEvalCtx = currentEvaluationContext();
+  const auto &ExprEvalCtx = SemaRef.currentEvaluationContext();
   if (ExprEvalCtx.isUnevaluated() || ExprEvalCtx.isConstantEvaluated())
     return true;
 
   // FIXME: Is bailing out early correct here?  Should we instead assume that
   // the caller is a global initializer?
-  FunctionDecl *Caller = getCurFunctionDecl(/*AllowLambda=*/true);
+  FunctionDecl *Caller = SemaRef.getCurFunctionDecl(/*AllowLambda=*/true);
   if (!Caller)
     return true;
 
   // If the caller is known-emitted, mark the callee as known-emitted.
   // Otherwise, mark the call in our call graph so we can traverse it later.
-  bool CallerKnownEmitted =
-      getEmissionStatus(Caller) == FunctionEmissionStatus::Emitted;
+  bool CallerKnownEmitted = SemaRef.getEmissionStatus(Caller) ==
+                            Sema::FunctionEmissionStatus::Emitted;
   SemaDiagnosticBuilder::Kind DiagKind = [this, Caller, Callee,
                                           CallerKnownEmitted] {
-    switch (IdentifyCUDAPreference(Caller, Callee)) {
+    switch (IdentifyPreference(Caller, Callee)) {
     case CFP_Never:
     case CFP_WrongSide:
       assert(Caller && "Never/wrongSide calls require a non-null caller");
@@ -908,7 +916,7 @@ bool Sema::CheckCUDACall(SourceLocation Loc, FunctionDecl *Callee) {
 
   if (DiagKind == SemaDiagnosticBuilder::K_Nop) {
     // For -fgpu-rdc, keep track of external kernels used by host functions.
-    if (LangOpts.CUDAIsDevice && LangOpts.GPURelocatableDeviceCode &&
+    if (getLangOpts().CUDAIsDevice && getLangOpts().GPURelocatableDeviceCode &&
         Callee->hasAttr<CUDAGlobalAttr>() && !Callee->isDefined() &&
         (!Caller || (!Caller->getDescribedFunctionTemplate() &&
                      getASTContext().GetGVALinkageForFunction(Caller) ==
@@ -924,12 +932,13 @@ bool Sema::CheckCUDACall(SourceLocation Loc, FunctionDecl *Callee) {
   if (!LocsWithCUDACallDiags.insert({Caller, Loc}).second)
     return true;
 
-  SemaDiagnosticBuilder(DiagKind, Loc, diag::err_ref_bad_target, Caller, *this)
-      << llvm::to_underlying(IdentifyCUDATarget(Callee)) << /*function*/ 0
-      << Callee << llvm::to_underlying(IdentifyCUDATarget(Caller));
+  SemaDiagnosticBuilder(DiagKind, Loc, diag::err_ref_bad_target, Caller,
+                        SemaRef)
+      << llvm::to_underlying(IdentifyTarget(Callee)) << /*function*/ 0 << Callee
+      << llvm::to_underlying(IdentifyTarget(Caller));
   if (!Callee->getBuiltinID())
     SemaDiagnosticBuilder(DiagKind, Callee->getLocation(),
-                          diag::note_previous_decl, Caller, *this)
+                          diag::note_previous_decl, Caller, SemaRef)
         << Callee;
   return DiagKind != SemaDiagnosticBuilder::K_Immediate &&
          DiagKind != SemaDiagnosticBuilder::K_ImmediateWithCallStack;
@@ -940,7 +949,7 @@ bool Sema::CheckCUDACall(SourceLocation Loc, FunctionDecl *Callee) {
 // defined and uses the capture by reference when the lambda is called. When
 // the capture and use happen on different sides, the capture is invalid and
 // should be diagnosed.
-void Sema::CUDACheckLambdaCapture(CXXMethodDecl *Callee,
+void SemaCUDA::CheckLambdaCapture(CXXMethodDecl *Callee,
                                   const sema::Capture &Capture) {
   // In host compilation we only need to check lambda functions emitted on host
   // side. In such lambda functions, a reference capture is invalid only
@@ -950,12 +959,12 @@ void Sema::CUDACheckLambdaCapture(CXXMethodDecl *Callee,
   // kernel cannot pass a lambda back to a host function since we cannot
   // define a kernel argument type which can hold the lambda before the lambda
   // itself is defined.
-  if (!LangOpts.CUDAIsDevice)
+  if (!getLangOpts().CUDAIsDevice)
     return;
 
   // File-scope lambda can only do init captures for global variables, which
   // results in passing by value for these global variables.
-  FunctionDecl *Caller = getCurFunctionDecl(/*AllowLambda=*/true);
+  FunctionDecl *Caller = SemaRef.getCurFunctionDecl(/*AllowLambda=*/true);
   if (!Caller)
     return;
 
@@ -972,7 +981,7 @@ void Sema::CUDACheckLambdaCapture(CXXMethodDecl *Callee,
   auto DiagKind = SemaDiagnosticBuilder::K_Deferred;
   if (Capture.isVariableCapture() && !getLangOpts().HIPStdPar) {
     SemaDiagnosticBuilder(DiagKind, Capture.getLocation(),
-                          diag::err_capture_bad_target, Callee, *this)
+                          diag::err_capture_bad_target, Callee, SemaRef)
         << Capture.getVariable();
   } else if (Capture.isThisCapture()) {
     // Capture of this pointer is allowed since this pointer may be pointing to
@@ -981,28 +990,28 @@ void Sema::CUDACheckLambdaCapture(CXXMethodDecl *Callee,
     // accessible on device side.
     SemaDiagnosticBuilder(DiagKind, Capture.getLocation(),
                           diag::warn_maybe_capture_bad_target_this_ptr, Callee,
-                          *this);
+                          SemaRef);
   }
 }
 
-void Sema::CUDASetLambdaAttrs(CXXMethodDecl *Method) {
+void SemaCUDA::SetLambdaAttrs(CXXMethodDecl *Method) {
   assert(getLangOpts().CUDA && "Should only be called during CUDA compilation");
   if (Method->hasAttr<CUDAHostAttr>() || Method->hasAttr<CUDADeviceAttr>())
     return;
-  Method->addAttr(CUDADeviceAttr::CreateImplicit(Context));
-  Method->addAttr(CUDAHostAttr::CreateImplicit(Context));
+  Method->addAttr(CUDADeviceAttr::CreateImplicit(getASTContext()));
+  Method->addAttr(CUDAHostAttr::CreateImplicit(getASTContext()));
 }
 
-void Sema::checkCUDATargetOverload(FunctionDecl *NewFD,
+void SemaCUDA::checkTargetOverload(FunctionDecl *NewFD,
                                    const LookupResult &Previous) {
   assert(getLangOpts().CUDA && "Should only be called during CUDA compilation");
-  CUDAFunctionTarget NewTarget = IdentifyCUDATarget(NewFD);
+  CUDAFunctionTarget NewTarget = IdentifyTarget(NewFD);
   for (NamedDecl *OldND : Previous) {
     FunctionDecl *OldFD = OldND->getAsFunction();
     if (!OldFD)
       continue;
 
-    CUDAFunctionTarget OldTarget = IdentifyCUDATarget(OldFD);
+    CUDAFunctionTarget OldTarget = IdentifyTarget(OldFD);
     // Don't allow HD and global functions to overload other functions with the
     // same signature.  We allow overloading based on CUDA attributes so that
     // functions can have different implementations on the host and device, but
@@ -1010,17 +1019,17 @@ void Sema::checkCUDATargetOverload(FunctionDecl *NewFD,
     // should have the same implementation on both sides.
     if (NewTarget != OldTarget &&
         ((NewTarget == CUDAFunctionTarget::HostDevice &&
-          !(LangOpts.OffloadImplicitHostDeviceTemplates &&
-            isCUDAImplicitHostDeviceFunction(NewFD) &&
+          !(getLangOpts().OffloadImplicitHostDeviceTemplates &&
+            isImplicitHostDeviceFunction(NewFD) &&
             OldTarget == CUDAFunctionTarget::Device)) ||
          (OldTarget == CUDAFunctionTarget::HostDevice &&
-          !(LangOpts.OffloadImplicitHostDeviceTemplates &&
-            isCUDAImplicitHostDeviceFunction(OldFD) &&
+          !(getLangOpts().OffloadImplicitHostDeviceTemplates &&
+            isImplicitHostDeviceFunction(OldFD) &&
             NewTarget == CUDAFunctionTarget::Device)) ||
          (NewTarget == CUDAFunctionTarget::Global) ||
          (OldTarget == CUDAFunctionTarget::Global)) &&
-        !IsOverload(NewFD, OldFD, /* UseMemberUsingDeclRules = */ false,
-                    /* ConsiderCudaAttrs = */ false)) {
+        !SemaRef.IsOverload(NewFD, OldFD, /* UseMemberUsingDeclRules = */ false,
+                            /* ConsiderCudaAttrs = */ false)) {
       Diag(NewFD->getLocation(), diag::err_cuda_ovl_target)
           << llvm::to_underlying(NewTarget) << NewFD->getDeclName()
           << llvm::to_underlying(OldTarget) << OldFD;
@@ -1041,21 +1050,21 @@ static void copyAttrIfPresent(Sema &S, FunctionDecl *FD,
   }
 }
 
-void Sema::inheritCUDATargetAttrs(FunctionDecl *FD,
+void SemaCUDA::inheritTargetAttrs(FunctionDecl *FD,
                                   const FunctionTemplateDecl &TD) {
   const FunctionDecl &TemplateFD = *TD.getTemplatedDecl();
-  copyAttrIfPresent<CUDAGlobalAttr>(*this, FD, TemplateFD);
-  copyAttrIfPresent<CUDAHostAttr>(*this, FD, TemplateFD);
-  copyAttrIfPresent<CUDADeviceAttr>(*this, FD, TemplateFD);
+  copyAttrIfPresent<CUDAGlobalAttr>(SemaRef, FD, TemplateFD);
+  copyAttrIfPresent<CUDAHostAttr>(SemaRef, FD, TemplateFD);
+  copyAttrIfPresent<CUDADeviceAttr>(SemaRef, FD, TemplateFD);
 }
 
-std::string Sema::getCudaConfigureFuncName() const {
+std::string SemaCUDA::getConfigureFuncName() const {
   if (getLangOpts().HIP)
     return getLangOpts().HIPUseNewLaunchAPI ? "__hipPushCallConfiguration"
                                             : "hipConfigureCall";
 
   // New CUDA kernel launch sequence.
-  if (CudaFeatureEnabled(Context.getTargetInfo().getSDKVersion(),
+  if (CudaFeatureEnabled(getASTContext().getTargetInfo().getSDKVersion(),
                          CudaFeature::CUDA_USES_NEW_LAUNCH))
     return "__cudaPushCallConfiguration";
 
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 720e56692359..17032d137052 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -45,6 +45,7 @@
 #include "clang/Sema/ParsedTemplate.h"
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/ScopeInfo.h"
+#include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaHLSL.h"
 #include "clang/Sema/SemaInternal.h"
 #include "clang/Sema/Template.h"
@@ -10595,12 +10596,12 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
 
     // We do not add HD attributes to specializations here because
     // they may have different constexpr-ness compared to their
-    // templates and, after maybeAddCUDAHostDeviceAttrs() is applied,
+    // templates and, after maybeAddHostDeviceAttrs() is applied,
     // may end up with different effective targets. Instead, a
     // specialization inherits its target attributes from its template
     // in the CheckFunctionTemplateSpecialization() call below.
     if (getLangOpts().CUDA && !isFunctionTemplateSpecialization)
-      maybeAddCUDAHostDeviceAttrs(NewFD, Previous);
+      CUDA().maybeAddHostDeviceAttrs(NewFD, Previous);
 
     // Handle explict specializations of function templates
     // and friend function declarations with an explicit
@@ -10898,12 +10899,12 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
 
   if (getLangOpts().CUDA) {
     IdentifierInfo *II = NewFD->getIdentifier();
-    if (II && II->isStr(getCudaConfigureFuncName()) &&
+    if (II && II->isStr(CUDA().getConfigureFuncName()) &&
         !NewFD->isInvalidDecl() &&
         NewFD->getDeclContext()->getRedeclContext()->isTranslationUnit()) {
       if (!R->castAs<FunctionType>()->getReturnType()->isScalarType())
         Diag(NewFD->getLocation(), diag::err_config_scalar_return)
-            << getCudaConfigureFuncName();
+            << CUDA().getConfigureFuncName();
       Context.setcudaConfigureCallDecl(NewFD);
     }
 
@@ -12398,7 +12399,7 @@ bool Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD,
     }
 
     if (!Redeclaration && LangOpts.CUDA)
-      checkCUDATargetOverload(NewFD, Previous);
+      CUDA().checkTargetOverload(NewFD, Previous);
   }
 
   // Check if the function definition uses any AArch64 SME features without
@@ -14415,7 +14416,7 @@ StmtResult Sema::ActOnCXXForRangeIdentifier(Scope *S, SourceLocation IdentLoc,
 void Sema::CheckCompleteVariableDeclaration(VarDecl *var) {
   if (var->isInvalidDecl()) return;
 
-  MaybeAddCUDAConstantAttr(var);
+  CUDA().MaybeAddConstantAttr(var);
 
   if (getLangOpts().OpenCL) {
     // OpenCL v2.0 s6.12.5 - Every block variable declaration must have an
@@ -14829,7 +14830,7 @@ void Sema::FinalizeDeclaration(Decl *ThisDecl) {
   // variables whether they are local or not. CUDA also allows
   // constant initializers for __constant__ and __device__ variables.
   if (getLangOpts().CUDA)
-    checkAllowedCUDAInitializer(VD);
+    CUDA().checkAllowedInitializer(VD);
 
   // Grab the dllimport or dllexport attribute off of the VarDecl.
   const InheritableAttr *DLLAttr = getDLLAttr(VD);
@@ -20666,7 +20667,7 @@ Sema::FunctionEmissionStatus Sema::getEmissionStatus(const FunctionDecl *FD,
     // when compiling for host, device and global functions are never emitted.
     // (Technically, we do emit a host-side stub for global functions, but this
     // doesn't count for our purposes here.)
-    CUDAFunctionTarget T = IdentifyCUDATarget(FD);
+    CUDAFunctionTarget T = CUDA().IdentifyTarget(FD);
     if (LangOpts.CUDAIsDevice && T == CUDAFunctionTarget::Host)
       return FunctionEmissionStatus::CUDADiscarded;
     if (!LangOpts.CUDAIsDevice &&
@@ -20691,5 +20692,5 @@ bool Sema::shouldIgnoreInHostDeviceCheck(FunctionDecl *Callee) {
   // for host, only HD functions actually called from the host get marked as
   // known-emitted.
   return LangOpts.CUDA && !LangOpts.CUDAIsDevice &&
-         IdentifyCUDATarget(Callee) == CUDAFunctionTarget::Global;
+         CUDA().IdentifyTarget(Callee) == CUDAFunctionTarget::Global;
 }
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 56c9d90c9b52..b7b1fbc625a1 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -39,6 +39,7 @@
 #include "clang/Sema/ParsedAttr.h"
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/ScopeInfo.h"
+#include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaHLSL.h"
 #include "clang/Sema/SemaInternal.h"
 #include "llvm/ADT/STLExtras.h"
@@ -5099,8 +5100,8 @@ static void handleSharedAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
     return;
   }
   if (S.getLangOpts().CUDA && VD->hasLocalStorage() &&
-      S.CUDADiagIfHostCode(AL.getLoc(), diag::err_cuda_host_shared)
-          << llvm::to_underlying(S.CurrentCUDATarget()))
+      S.CUDA().DiagIfHostCode(AL.getLoc(), diag::err_cuda_host_shared)
+          << llvm::to_underlying(S.CUDA().CurrentTarget()))
     return;
   D->addAttr(::new (S.Context) CUDASharedAttr(S.Context, AL));
 }
@@ -5189,8 +5190,9 @@ static void handleCallConvAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   // Diagnostic is emitted elsewhere: here we store the (valid) AL
   // in the Decl node for syntactic reasoning, e.g., pretty-printing.
   CallingConv CC;
-  if (S.CheckCallingConvAttr(AL, CC, /*FD*/ nullptr,
-                             S.IdentifyCUDATarget(dyn_cast<FunctionDecl>(D))))
+  if (S.CheckCallingConvAttr(
+          AL, CC, /*FD*/ nullptr,
+          S.CUDA().IdentifyTarget(dyn_cast<FunctionDecl>(D))))
     return;
 
   if (!isa<ObjCMethodDecl>(D)) {
@@ -5495,7 +5497,7 @@ bool Sema::CheckCallingConvAttr(const ParsedAttr &Attrs, CallingConv &CC,
   if (LangOpts.CUDA) {
     auto *Aux = Context.getAuxTargetInfo();
     assert(FD || CFT != CUDAFunctionTarget::InvalidTarget);
-    auto CudaTarget = FD ? IdentifyCUDATarget(FD) : CFT;
+    auto CudaTarget = FD ? CUDA().IdentifyTarget(FD) : CFT;
     bool CheckHost = false, CheckDevice = false;
     switch (CudaTarget) {
     case CUDAFunctionTarget::HostDevice:
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index 51c14443d2d8..1fe10375222c 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -42,6 +42,7 @@
 #include "clang/Sema/ParsedTemplate.h"
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/ScopeInfo.h"
+#include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaInternal.h"
 #include "clang/Sema/Template.h"
 #include "llvm/ADT/ArrayRef.h"
@@ -9876,15 +9877,15 @@ bool Sema::ShouldDeleteSpecialMember(CXXMethodDecl *MD,
     // failed.
     // For inherited constructors (non-null ICI), CSM may be passed so that MD
     // is treated as certain special member, which may not reflect what special
-    // member MD really is. However inferCUDATargetForImplicitSpecialMember
+    // member MD really is. However inferTargetForImplicitSpecialMember
     // expects CSM to match MD, therefore recalculate CSM.
     assert(ICI || CSM == getSpecialMember(MD));
     auto RealCSM = CSM;
     if (ICI)
       RealCSM = getSpecialMember(MD);
 
-    return inferCUDATargetForImplicitSpecialMember(RD, RealCSM, MD,
-                                                   SMI.ConstArg, Diagnose);
+    return CUDA().inferTargetForImplicitSpecialMember(RD, RealCSM, MD,
+                                                      SMI.ConstArg, Diagnose);
   }
 
   return false;
@@ -14055,7 +14056,7 @@ CXXConstructorDecl *Sema::DeclareImplicitDefaultConstructor(
   setupImplicitSpecialMemberType(DefaultCon, Context.VoidTy, std::nullopt);
 
   if (getLangOpts().CUDA)
-    inferCUDATargetForImplicitSpecialMember(
+    CUDA().inferTargetForImplicitSpecialMember(
         ClassDecl, CXXSpecialMemberKind::DefaultConstructor, DefaultCon,
         /* ConstRHS */ false,
         /* Diagnose */ false);
@@ -14341,7 +14342,7 @@ CXXDestructorDecl *Sema::DeclareImplicitDestructor(CXXRecordDecl *ClassDecl) {
   setupImplicitSpecialMemberType(Destructor, Context.VoidTy, std::nullopt);
 
   if (getLangOpts().CUDA)
-    inferCUDATargetForImplicitSpecialMember(
+    CUDA().inferTargetForImplicitSpecialMember(
         ClassDecl, CXXSpecialMemberKind::Destructor, Destructor,
         /* ConstRHS */ false,
         /* Diagnose */ false);
@@ -14983,7 +14984,7 @@ CXXMethodDecl *Sema::DeclareImplicitCopyAssignment(CXXRecordDecl *ClassDecl) {
   setupImplicitSpecialMemberType(CopyAssignment, RetType, ArgType);
 
   if (getLangOpts().CUDA)
-    inferCUDATargetForImplicitSpecialMember(
+    CUDA().inferTargetForImplicitSpecialMember(
         ClassDecl, CXXSpecialMemberKind::CopyAssignment, CopyAssignment,
         /* ConstRHS */ Const,
         /* Diagnose */ false);
@@ -15335,7 +15336,7 @@ CXXMethodDecl *Sema::DeclareImplicitMoveAssignment(CXXRecordDecl *ClassDecl) {
   setupImplicitSpecialMemberType(MoveAssignment, RetType, ArgType);
 
   if (getLangOpts().CUDA)
-    inferCUDATargetForImplicitSpecialMember(
+    CUDA().inferTargetForImplicitSpecialMember(
         ClassDecl, CXXSpecialMemberKind::MoveAssignment, MoveAssignment,
         /* ConstRHS */ false,
         /* Diagnose */ false);
@@ -15733,7 +15734,7 @@ CXXConstructorDecl *Sema::DeclareImplicitCopyConstructor(
   setupImplicitSpecialMemberType(CopyConstructor, Context.VoidTy, ArgType);
 
   if (getLangOpts().CUDA)
-    inferCUDATargetForImplicitSpecialMember(
+    CUDA().inferTargetForImplicitSpecialMember(
         ClassDecl, CXXSpecialMemberKind::CopyConstructor, CopyConstructor,
         /* ConstRHS */ Const,
         /* Diagnose */ false);
@@ -15878,7 +15879,7 @@ CXXConstructorDecl *Sema::DeclareImplicitMoveConstructor(
   setupImplicitSpecialMemberType(MoveConstructor, Context.VoidTy, ArgType);
 
   if (getLangOpts().CUDA)
-    inferCUDATargetForImplicitSpecialMember(
+    CUDA().inferTargetForImplicitSpecialMember(
         ClassDecl, CXXSpecialMemberKind::MoveConstructor, MoveConstructor,
         /* ConstRHS */ false,
         /* Diagnose */ false);
@@ -16184,7 +16185,7 @@ ExprResult Sema::BuildCXXConstructExpr(
              DeclInitType->getBaseElementTypeUnsafe()->getAsCXXRecordDecl()) &&
          "given constructor for wrong type");
   MarkFunctionReferenced(ConstructLoc, Constructor);
-  if (getLangOpts().CUDA && !CheckCUDACall(ConstructLoc, Constructor))
+  if (getLangOpts().CUDA && !CUDA().CheckCall(ConstructLoc, Constructor))
     return ExprError();
 
   return CheckForImmediateInvocation(
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index b294d2bd9f53..823bf36d88bc 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -49,6 +49,7 @@
 #include "clang/Sema/ParsedTemplate.h"
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/ScopeInfo.h"
+#include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaFixItUtils.h"
 #include "clang/Sema/SemaInternal.h"
 #include "clang/Sema/Template.h"
@@ -308,7 +309,7 @@ bool Sema::DiagnoseUseOfDecl(NamedDecl *D, ArrayRef<SourceLocation> Locs,
         DeduceReturnType(FD, Loc))
       return true;
 
-    if (getLangOpts().CUDA && !CheckCUDACall(Loc, FD))
+    if (getLangOpts().CUDA && !CUDA().CheckCall(Loc, FD))
       return true;
 
   }
@@ -17307,7 +17308,7 @@ ExprResult Sema::BuildVAArgExpr(SourceLocation BuiltinLoc,
   // CUDA device code does not support varargs.
   if (getLangOpts().CUDA && getLangOpts().CUDAIsDevice) {
     if (const FunctionDecl *F = dyn_cast<FunctionDecl>(CurContext)) {
-      CUDAFunctionTarget T = IdentifyCUDATarget(F);
+      CUDAFunctionTarget T = CUDA().IdentifyTarget(F);
       if (T == CUDAFunctionTarget::Global || T == CUDAFunctionTarget::Device ||
           T == CUDAFunctionTarget::HostDevice)
         return ExprError(Diag(E->getBeginLoc(), diag::err_va_arg_in_device));
@@ -18961,7 +18962,7 @@ void Sema::MarkFunctionReferenced(SourceLocation Loc, FunctionDecl *Func,
     checkSpecializationReachability(Loc, Func);
 
   if (getLangOpts().CUDA)
-    CheckCUDACall(Loc, Func);
+    CUDA().CheckCall(Loc, Func);
 
   // If we need a definition, try to create one.
   if (NeedDefinition && !Func->getBody()) {
@@ -19108,7 +19109,7 @@ void Sema::MarkFunctionReferenced(SourceLocation Loc, FunctionDecl *Func,
   // side. Therefore keep trying until it is recorded.
   if (LangOpts.OffloadImplicitHostDeviceTemplates && LangOpts.CUDAIsDevice &&
       !getASTContext().CUDAImplicitHostDeviceFunUsedByDevice.count(Func))
-    CUDARecordImplicitHostDeviceFuncUsedByDevice(Func);
+    CUDA().RecordImplicitHostDeviceFuncUsedByDevice(Func);
 
   // If this is the first "real" use, act on that.
   if (OdrUse == OdrUseContext::Used && !Func->isUsed(/*CheckUsedAttr=*/false)) {
@@ -19181,9 +19182,9 @@ MarkVarDeclODRUsed(ValueDecl *V, SourceLocation Loc, Sema &SemaRef,
 
   if (SemaRef.LangOpts.CUDA && Var->hasGlobalStorage()) {
     auto *FD = dyn_cast_or_null<FunctionDecl>(SemaRef.CurContext);
-    auto VarTarget = SemaRef.IdentifyCUDATarget(Var);
-    auto UserTarget = SemaRef.IdentifyCUDATarget(FD);
-    if (VarTarget == Sema::CVT_Host &&
+    auto VarTarget = SemaRef.CUDA().IdentifyTarget(Var);
+    auto UserTarget = SemaRef.CUDA().IdentifyTarget(FD);
+    if (VarTarget == SemaCUDA::CVT_Host &&
         (UserTarget == CUDAFunctionTarget::Device ||
          UserTarget == CUDAFunctionTarget::HostDevice ||
          UserTarget == CUDAFunctionTarget::Global)) {
@@ -19199,7 +19200,7 @@ MarkVarDeclODRUsed(ValueDecl *V, SourceLocation Loc, Sema &SemaRef,
                                ? diag::note_cuda_const_var_unpromoted
                                : diag::note_cuda_host_var);
       }
-    } else if (VarTarget == Sema::CVT_Device &&
+    } else if (VarTarget == SemaCUDA::CVT_Device &&
                !Var->hasAttr<CUDASharedAttr>() &&
                (UserTarget == CUDAFunctionTarget::Host ||
                 UserTarget == CUDAFunctionTarget::HostDevice)) {
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index ce9d5c26e218..8911257a6f61 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -38,6 +38,7 @@
 #include "clang/Sema/ParsedTemplate.h"
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/ScopeInfo.h"
+#include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaInternal.h"
 #include "clang/Sema/SemaLambda.h"
 #include "clang/Sema/Template.h"
@@ -884,8 +885,8 @@ ExprResult Sema::BuildCXXThrow(SourceLocation OpLoc, Expr *Ex,
 
   // Exceptions aren't allowed in CUDA device code.
   if (getLangOpts().CUDA)
-    CUDADiagIfDeviceCode(OpLoc, diag::err_cuda_device_exceptions)
-        << "throw" << llvm::to_underlying(CurrentCUDATarget());
+    CUDA().DiagIfDeviceCode(OpLoc, diag::err_cuda_device_exceptions)
+        << "throw" << llvm::to_underlying(CUDA().CurrentTarget());
 
   if (getCurScope() && getCurScope()->isOpenMPSimdDirectiveScope())
     Diag(OpLoc, diag::err_omp_simd_region_cannot_use_stmt) << "throw";
@@ -1708,17 +1709,17 @@ bool Sema::isUsualDeallocationFunction(const CXXMethodDecl *Method) {
   // [CUDA] Ignore this function, if we can't call it.
   const FunctionDecl *Caller = getCurFunctionDecl(/*AllowLambda=*/true);
   if (getLangOpts().CUDA) {
-    auto CallPreference = IdentifyCUDAPreference(Caller, Method);
+    auto CallPreference = CUDA().IdentifyPreference(Caller, Method);
     // If it's not callable at all, it's not the right function.
-    if (CallPreference < CFP_WrongSide)
+    if (CallPreference < SemaCUDA::CFP_WrongSide)
       return false;
-    if (CallPreference == CFP_WrongSide) {
+    if (CallPreference == SemaCUDA::CFP_WrongSide) {
       // Maybe. We have to check if there are better alternatives.
       DeclContext::lookup_result R =
           Method->getDeclContext()->lookup(Method->getDeclName());
       for (const auto *D : R) {
         if (const auto *FD = dyn_cast<FunctionDecl>(D)) {
-          if (IdentifyCUDAPreference(Caller, FD) > CFP_WrongSide)
+          if (CUDA().IdentifyPreference(Caller, FD) > SemaCUDA::CFP_WrongSide)
             return false;
         }
       }
@@ -1737,7 +1738,7 @@ bool Sema::isUsualDeallocationFunction(const CXXMethodDecl *Method) {
   return llvm::none_of(PreventedBy, [&](const FunctionDecl *FD) {
     assert(FD->getNumParams() == 1 &&
            "Only single-operand functions should be in PreventedBy");
-    return IdentifyCUDAPreference(Caller, FD) >= CFP_HostDevice;
+    return CUDA().IdentifyPreference(Caller, FD) >= SemaCUDA::CFP_HostDevice;
   });
 }
 
@@ -1774,7 +1775,7 @@ namespace {
     UsualDeallocFnInfo(Sema &S, DeclAccessPair Found)
         : Found(Found), FD(dyn_cast<FunctionDecl>(Found->getUnderlyingDecl())),
           Destroying(false), HasSizeT(false), HasAlignValT(false),
-          CUDAPref(Sema::CFP_Native) {
+          CUDAPref(SemaCUDA::CFP_Native) {
       // A function template declaration is never a usual deallocation function.
       if (!FD)
         return;
@@ -1800,7 +1801,7 @@ namespace {
 
       // In CUDA, determine how much we'd like / dislike to call this.
       if (S.getLangOpts().CUDA)
-        CUDAPref = S.IdentifyCUDAPreference(
+        CUDAPref = S.CUDA().IdentifyPreference(
             S.getCurFunctionDecl(/*AllowLambda=*/true), FD);
     }
 
@@ -1831,7 +1832,7 @@ namespace {
     DeclAccessPair Found;
     FunctionDecl *FD;
     bool Destroying, HasSizeT, HasAlignValT;
-    Sema::CUDAFunctionPreference CUDAPref;
+    SemaCUDA::CUDAFunctionPreference CUDAPref;
   };
 }
 
@@ -1855,7 +1856,7 @@ static UsualDeallocFnInfo resolveDeallocationOverload(
   for (auto I = R.begin(), E = R.end(); I != E; ++I) {
     UsualDeallocFnInfo Info(S, I.getPair());
     if (!Info || !isNonPlacementDeallocationFunction(S, Info.FD) ||
-        Info.CUDAPref == Sema::CFP_Never)
+        Info.CUDAPref == SemaCUDA::CFP_Never)
       continue;
 
     if (!Best) {
@@ -2956,8 +2957,8 @@ bool Sema::FindAllocationFunctions(SourceLocation StartLoc, SourceRange Range,
     }
 
     if (getLangOpts().CUDA)
-      EraseUnwantedCUDAMatches(getCurFunctionDecl(/*AllowLambda=*/true),
-                               Matches);
+      CUDA().EraseUnwantedMatches(getCurFunctionDecl(/*AllowLambda=*/true),
+                                  Matches);
   } else {
     // C++1y [expr.new]p22:
     //   For a non-placement allocation function, the normal deallocation
diff --git a/clang/lib/Sema/SemaLambda.cpp b/clang/lib/Sema/SemaLambda.cpp
index 5b95bae567b7..35a51c6c2328 100644
--- a/clang/lib/Sema/SemaLambda.cpp
+++ b/clang/lib/Sema/SemaLambda.cpp
@@ -9,17 +9,18 @@
 //  This file implements semantic analysis for C++ lambda expressions.
 //
 //===----------------------------------------------------------------------===//
-#include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/SemaLambda.h"
 #include "TypeLocBuilder.h"
 #include "clang/AST/ASTLambda.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/Basic/TargetInfo.h"
+#include "clang/Sema/DeclSpec.h"
 #include "clang/Sema/Initialization.h"
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/ScopeInfo.h"
+#include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaInternal.h"
-#include "clang/Sema/SemaLambda.h"
 #include "clang/Sema/Template.h"
 #include "llvm/ADT/STLExtras.h"
 #include <optional>
@@ -1393,7 +1394,7 @@ void Sema::ActOnStartOfLambdaDefinition(LambdaIntroducer &Intro,
 
   // CUDA lambdas get implicit host and device attributes.
   if (getLangOpts().CUDA)
-    CUDASetLambdaAttrs(Method);
+    CUDA().SetLambdaAttrs(Method);
 
   // OpenMP lambdas might get assumumption attributes.
   if (LangOpts.OpenMP)
@@ -2136,7 +2137,7 @@ ExprResult Sema::BuildLambdaExpr(SourceLocation StartLoc, SourceLocation EndLoc,
       CaptureInits.push_back(Init.get());
 
       if (LangOpts.CUDA)
-        CUDACheckLambdaCapture(CallOperator, From);
+        CUDA().CheckLambdaCapture(CallOperator, From);
     }
 
     Class->setCaptures(Context, Captures);
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index e1155dc2d5d2..397e7681828f 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -31,6 +31,7 @@
 #include "clang/Sema/Initialization.h"
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/Overload.h"
+#include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaInternal.h"
 #include "clang/Sema/Template.h"
 #include "clang/Sema/TemplateDeduction.h"
@@ -1549,8 +1550,8 @@ static bool IsOverloadOrOverrideImpl(Sema &SemaRef, FunctionDecl *New,
     // Don't allow overloading of destructors.  (In theory we could, but it
     // would be a giant change to clang.)
     if (!isa<CXXDestructorDecl>(New)) {
-      CUDAFunctionTarget NewTarget = SemaRef.IdentifyCUDATarget(New),
-                         OldTarget = SemaRef.IdentifyCUDATarget(Old);
+      CUDAFunctionTarget NewTarget = SemaRef.CUDA().IdentifyTarget(New),
+                         OldTarget = SemaRef.CUDA().IdentifyTarget(Old);
       if (NewTarget != CUDAFunctionTarget::InvalidTarget) {
         assert((OldTarget != CUDAFunctionTarget::InvalidTarget) &&
                "Unexpected invalid target.");
@@ -7100,7 +7101,7 @@ void Sema::AddOverloadCandidate(
     // inferred for the member automatically, based on the bases and fields of
     // the class.
     if (!(Caller && Caller->isImplicit()) &&
-        !IsAllowedCUDACall(Caller, Function)) {
+        !CUDA().IsAllowedCall(Caller, Function)) {
       Candidate.Viable = false;
       Candidate.FailureKind = ovl_fail_bad_target;
       return;
@@ -7618,7 +7619,8 @@ Sema::AddMethodCandidate(CXXMethodDecl *Method, DeclAccessPair FoundDecl,
 
   // (CUDA B.1): Check for invalid calls between targets.
   if (getLangOpts().CUDA)
-    if (!IsAllowedCUDACall(getCurFunctionDecl(/*AllowLambda=*/true), Method)) {
+    if (!CUDA().IsAllowedCall(getCurFunctionDecl(/*AllowLambda=*/true),
+                              Method)) {
       Candidate.Viable = false;
       Candidate.FailureKind = ovl_fail_bad_target;
       return;
@@ -10440,7 +10442,7 @@ bool clang::isBetterOverloadCandidate(
   // If other rules cannot determine which is better, CUDA preference will be
   // used again to determine which is better.
   //
-  // TODO: Currently IdentifyCUDAPreference does not return correct values
+  // TODO: Currently IdentifyPreference does not return correct values
   // for functions called in global variable initializers due to missing
   // correct context about device/host. Therefore we can only enforce this
   // rule when there is a caller. We should enforce this rule for functions
@@ -10452,14 +10454,14 @@ bool clang::isBetterOverloadCandidate(
   if (S.getLangOpts().CUDA && Cand1.Function && Cand2.Function &&
       S.getLangOpts().GPUExcludeWrongSideOverloads) {
     if (FunctionDecl *Caller = S.getCurFunctionDecl(/*AllowLambda=*/true)) {
-      bool IsCallerImplicitHD = Sema::isCUDAImplicitHostDeviceFunction(Caller);
+      bool IsCallerImplicitHD = SemaCUDA::isImplicitHostDeviceFunction(Caller);
       bool IsCand1ImplicitHD =
-          Sema::isCUDAImplicitHostDeviceFunction(Cand1.Function);
+          SemaCUDA::isImplicitHostDeviceFunction(Cand1.Function);
       bool IsCand2ImplicitHD =
-          Sema::isCUDAImplicitHostDeviceFunction(Cand2.Function);
-      auto P1 = S.IdentifyCUDAPreference(Caller, Cand1.Function);
-      auto P2 = S.IdentifyCUDAPreference(Caller, Cand2.Function);
-      assert(P1 != Sema::CFP_Never && P2 != Sema::CFP_Never);
+          SemaCUDA::isImplicitHostDeviceFunction(Cand2.Function);
+      auto P1 = S.CUDA().IdentifyPreference(Caller, Cand1.Function);
+      auto P2 = S.CUDA().IdentifyPreference(Caller, Cand2.Function);
+      assert(P1 != SemaCUDA::CFP_Never && P2 != SemaCUDA::CFP_Never);
       // The implicit HD function may be a function in a system header which
       // is forced by pragma. In device compilation, if we prefer HD candidates
       // over wrong-sided candidates, overloading resolution may change, which
@@ -10473,8 +10475,8 @@ bool clang::isBetterOverloadCandidate(
       auto EmitThreshold =
           (S.getLangOpts().CUDAIsDevice && IsCallerImplicitHD &&
            (IsCand1ImplicitHD || IsCand2ImplicitHD))
-              ? Sema::CFP_Never
-              : Sema::CFP_WrongSide;
+              ? SemaCUDA::CFP_Never
+              : SemaCUDA::CFP_WrongSide;
       auto Cand1Emittable = P1 > EmitThreshold;
       auto Cand2Emittable = P2 > EmitThreshold;
       if (Cand1Emittable && !Cand2Emittable)
@@ -10758,8 +10760,8 @@ bool clang::isBetterOverloadCandidate(
   // to determine which is better.
   if (S.getLangOpts().CUDA && Cand1.Function && Cand2.Function) {
     FunctionDecl *Caller = S.getCurFunctionDecl(/*AllowLambda=*/true);
-    return S.IdentifyCUDAPreference(Caller, Cand1.Function) >
-           S.IdentifyCUDAPreference(Caller, Cand2.Function);
+    return S.CUDA().IdentifyPreference(Caller, Cand1.Function) >
+           S.CUDA().IdentifyPreference(Caller, Cand2.Function);
   }
 
   // General member function overloading is handled above, so this only handles
@@ -10891,15 +10893,15 @@ OverloadCandidateSet::BestViableFunction(Sema &S, SourceLocation Loc,
         llvm::any_of(Candidates, [&](OverloadCandidate *Cand) {
           // Check viable function only.
           return Cand->Viable && Cand->Function &&
-                 S.IdentifyCUDAPreference(Caller, Cand->Function) ==
-                     Sema::CFP_SameSide;
+                 S.CUDA().IdentifyPreference(Caller, Cand->Function) ==
+                     SemaCUDA::CFP_SameSide;
         });
     if (ContainsSameSideCandidate) {
       auto IsWrongSideCandidate = [&](OverloadCandidate *Cand) {
         // Check viable function only to avoid unnecessary data copying/moving.
         return Cand->Viable && Cand->Function &&
-               S.IdentifyCUDAPreference(Caller, Cand->Function) ==
-                   Sema::CFP_WrongSide;
+               S.CUDA().IdentifyPreference(Caller, Cand->Function) ==
+                   SemaCUDA::CFP_WrongSide;
       };
       llvm::erase_if(Candidates, IsWrongSideCandidate);
     }
@@ -11938,8 +11940,8 @@ static void DiagnoseBadTarget(Sema &S, OverloadCandidate *Cand) {
   FunctionDecl *Caller = S.getCurFunctionDecl(/*AllowLambda=*/true);
   FunctionDecl *Callee = Cand->Function;
 
-  CUDAFunctionTarget CallerTarget = S.IdentifyCUDATarget(Caller),
-                     CalleeTarget = S.IdentifyCUDATarget(Callee);
+  CUDAFunctionTarget CallerTarget = S.CUDA().IdentifyTarget(Caller),
+                     CalleeTarget = S.CUDA().IdentifyTarget(Callee);
 
   std::string FnDesc;
   std::pair<OverloadCandidateKind, OverloadCandidateSelect> FnKindPair =
@@ -11986,9 +11988,9 @@ static void DiagnoseBadTarget(Sema &S, OverloadCandidate *Cand) {
       }
     }
 
-    S.inferCUDATargetForImplicitSpecialMember(ParentClass, CSM, Meth,
-                                              /* ConstRHS */ ConstRHS,
-                                              /* Diagnose */ true);
+    S.CUDA().inferTargetForImplicitSpecialMember(ParentClass, CSM, Meth,
+                                                 /* ConstRHS */ ConstRHS,
+                                                 /* Diagnose */ true);
   }
 }
 
@@ -13060,7 +13062,7 @@ private:
       if (S.getLangOpts().CUDA) {
         FunctionDecl *Caller = S.getCurFunctionDecl(/*AllowLambda=*/true);
         if (!(Caller && Caller->isImplicit()) &&
-            !S.IsAllowedCUDACall(Caller, FunDecl))
+            !S.CUDA().IsAllowedCall(Caller, FunDecl))
           return false;
       }
       if (FunDecl->isMultiVersion()) {
@@ -13180,8 +13182,8 @@ private:
   }
 
   void EliminateSuboptimalCudaMatches() {
-    S.EraseUnwantedCUDAMatches(S.getCurFunctionDecl(/*AllowLambda=*/true),
-                               Matches);
+    S.CUDA().EraseUnwantedMatches(S.getCurFunctionDecl(/*AllowLambda=*/true),
+                                  Matches);
   }
 
 public:
@@ -13335,8 +13337,8 @@ Sema::resolveAddressOfSingleOverloadCandidate(Expr *E, DeclAccessPair &Pair) {
   // Return positive for better, negative for worse, 0 for equal preference.
   auto CheckCUDAPreference = [&](FunctionDecl *FD1, FunctionDecl *FD2) {
     FunctionDecl *Caller = getCurFunctionDecl(/*AllowLambda=*/true);
-    return static_cast<int>(IdentifyCUDAPreference(Caller, FD1)) -
-           static_cast<int>(IdentifyCUDAPreference(Caller, FD2));
+    return static_cast<int>(CUDA().IdentifyPreference(Caller, FD1)) -
+           static_cast<int>(CUDA().IdentifyPreference(Caller, FD2));
   };
 
   auto CheckMoreConstrained = [&](FunctionDecl *FD1,
diff --git a/clang/lib/Sema/SemaStmt.cpp b/clang/lib/Sema/SemaStmt.cpp
index 1c2f6120f621..d28c24cfdfd3 100644
--- a/clang/lib/Sema/SemaStmt.cpp
+++ b/clang/lib/Sema/SemaStmt.cpp
@@ -33,6 +33,7 @@
 #include "clang/Sema/Ownership.h"
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/ScopeInfo.h"
+#include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaInternal.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
@@ -4574,8 +4575,8 @@ StmtResult Sema::ActOnCXXTryBlock(SourceLocation TryLoc, Stmt *TryBlock,
 
   // Exceptions aren't allowed in CUDA device code.
   if (getLangOpts().CUDA)
-    CUDADiagIfDeviceCode(TryLoc, diag::err_cuda_device_exceptions)
-        << "try" << llvm::to_underlying(CurrentCUDATarget());
+    CUDA().DiagIfDeviceCode(TryLoc, diag::err_cuda_device_exceptions)
+        << "try" << llvm::to_underlying(CUDA().CurrentTarget());
 
   if (getCurScope() && getCurScope()->isOpenMPSimdDirectiveScope())
     Diag(TryLoc, diag::err_omp_simd_region_cannot_use_stmt) << "try";
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index e0f5e53dc248..95171359f0ab 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -33,6 +33,7 @@
 #include "clang/Sema/Overload.h"
 #include "clang/Sema/ParsedTemplate.h"
 #include "clang/Sema/Scope.h"
+#include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaInternal.h"
 #include "clang/Sema/Template.h"
 #include "clang/Sema/TemplateDeduction.h"
@@ -10155,9 +10156,9 @@ bool Sema::CheckFunctionTemplateSpecialization(
       // take target attributes into account, we reject candidates
       // here that have a different target.
       if (LangOpts.CUDA &&
-          IdentifyCUDATarget(Specialization,
-                             /* IgnoreImplicitHDAttr = */ true) !=
-              IdentifyCUDATarget(FD, /* IgnoreImplicitHDAttr = */ true)) {
+          CUDA().IdentifyTarget(Specialization,
+                                /* IgnoreImplicitHDAttr = */ true) !=
+              CUDA().IdentifyTarget(FD, /* IgnoreImplicitHDAttr = */ true)) {
         FailedCandidates.addCandidate().set(
             I.getPair(), FunTmpl->getTemplatedDecl(),
             MakeDeductionFailureInfo(
@@ -10328,7 +10329,7 @@ bool Sema::CheckFunctionTemplateSpecialization(
   // virtue e.g. of being constexpr, and it passes these implicit
   // attributes on to its specializations.)
   if (LangOpts.CUDA)
-    inheritCUDATargetAttrs(FD, *Specialization->getPrimaryTemplate());
+    CUDA().inheritTargetAttrs(FD, *Specialization->getPrimaryTemplate());
 
   // The "previous declaration" for this function template specialization is
   // the prior function template specialization.
@@ -11364,9 +11365,9 @@ DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
     // target attributes into account, we reject candidates here that
     // have a different target.
     if (LangOpts.CUDA &&
-        IdentifyCUDATarget(Specialization,
-                           /* IgnoreImplicitHDAttr = */ true) !=
-            IdentifyCUDATarget(D.getDeclSpec().getAttributes())) {
+        CUDA().IdentifyTarget(Specialization,
+                              /* IgnoreImplicitHDAttr = */ true) !=
+            CUDA().IdentifyTarget(D.getDeclSpec().getAttributes())) {
       FailedCandidates.addCandidate().set(
           P.getPair(), FunTmpl->getTemplatedDecl(),
           MakeDeductionFailureInfo(
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index 7f510d34d671..c0469a47ab8b 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -26,6 +26,7 @@
 #include "clang/Sema/Initialization.h"
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/ScopeInfo.h"
+#include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaInternal.h"
 #include "clang/Sema/Template.h"
 #include "clang/Sema/TemplateInstCallback.h"
@@ -5537,7 +5538,7 @@ void Sema::InstantiateVariableInitializer(
   }
 
   if (getLangOpts().CUDA)
-    checkAllowedCUDAInitializer(Var);
+    CUDA().checkAllowedInitializer(Var);
 }
 
 /// Instantiate the definition of the given variable from its
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index b3f6078952f6..404c4e8e31b5 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -33,6 +33,7 @@
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/ParsedTemplate.h"
 #include "clang/Sema/ScopeInfo.h"
+#include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaInternal.h"
 #include "clang/Sema/Template.h"
 #include "clang/Sema/TemplateInstCallback.h"
@@ -2735,7 +2736,7 @@ QualType Sema::BuildArrayType(QualType T, ArraySizeModifier ASM,
       bool IsCUDADevice = (getLangOpts().CUDA && getLangOpts().CUDAIsDevice);
       targetDiag(Loc,
                  IsCUDADevice ? diag::err_cuda_vla : diag::err_vla_unsupported)
-          << (IsCUDADevice ? llvm::to_underlying(CurrentCUDATarget()) : 0);
+          << (IsCUDADevice ? llvm::to_underlying(CUDA().CurrentTarget()) : 0);
     } else if (sema::FunctionScopeInfo *FSI = getCurFunction()) {
       // VLAs are supported on this target, but we may need to do delayed
       // checking that the VLA is not being used within a coroutine.
@@ -3618,7 +3619,7 @@ static QualType GetDeclSpecTypeForDeclarator(TypeProcessingState &state,
   // D.getDeclarationAttributes()) because those are always C++11 attributes,
   // and those don't get distributed.
   distributeTypeAttrsFromDeclarator(
-      state, T, SemaRef.IdentifyCUDATarget(D.getAttributes()));
+      state, T, SemaRef.CUDA().IdentifyTarget(D.getAttributes()));
 
   // Find the deduced type in this type. Look in the trailing return type if we
   // have one, otherwise in the DeclSpec type.
@@ -4139,7 +4140,7 @@ static CallingConv getCCForDeclaratorChunk(
       // handleFunctionTypeAttr.
       CallingConv CC;
       if (!S.CheckCallingConvAttr(AL, CC, /*FunctionDecl=*/nullptr,
-                                  S.IdentifyCUDATarget(D.getAttributes())) &&
+                                  S.CUDA().IdentifyTarget(D.getAttributes())) &&
           (!FTI.isVariadic || supportsVariadicCall(CC))) {
         return CC;
       }
@@ -5825,7 +5826,7 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state,
 
     // See if there are any attributes on this declarator chunk.
     processTypeAttrs(state, T, TAL_DeclChunk, DeclType.getAttrs(),
-                     S.IdentifyCUDATarget(D.getAttributes()));
+                     S.CUDA().IdentifyTarget(D.getAttributes()));
 
     if (DeclType.Kind != DeclaratorChunk::Paren) {
       if (ExpectNoDerefChunk && !IsNoDerefableChunk(DeclType))
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index a47c7a393d63..8c4b460970ad 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -78,6 +78,7 @@
 #include "clang/Sema/ObjCMethodList.h"
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/Sema.h"
+#include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/Weak.h"
 #include "clang/Serialization/ASTBitCodes.h"
 #include "clang/Serialization/ASTDeserializationListener.h"
@@ -3995,7 +3996,7 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
       if (Record.size() != 1)
         return llvm::createStringError(std::errc::illegal_byte_sequence,
                                        "invalid cuda pragma options record");
-      ForceCUDAHostDeviceDepth = Record[0];
+      ForceHostDeviceDepth = Record[0];
       break;
 
     case ALIGN_PACK_PRAGMA_OPTIONS: {
@@ -8274,7 +8275,7 @@ void ASTReader::UpdateSema() {
             PragmaMSPointersToMembersState,
         PointersToMembersPragmaLocation);
   }
-  SemaObj->ForceCUDAHostDeviceDepth = ForceCUDAHostDeviceDepth;
+  SemaObj->CUDA().ForceHostDeviceDepth = ForceHostDeviceDepth;
 
   if (PragmaAlignPackCurrentValue) {
     // The bottom of the stack might have a default value. It must be adjusted
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index d45695784354..85b7fd5535a1 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -65,6 +65,7 @@
 #include "clang/Sema/IdentifierResolver.h"
 #include "clang/Sema/ObjCMethodList.h"
 #include "clang/Sema/Sema.h"
+#include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/Weak.h"
 #include "clang/Serialization/ASTBitCodes.h"
 #include "clang/Serialization/ASTReader.h"
@@ -4335,8 +4336,8 @@ void ASTWriter::WriteOpenCLExtensions(Sema &SemaRef) {
   Stream.EmitRecord(OPENCL_EXTENSIONS, Record);
 }
 void ASTWriter::WriteCUDAPragmas(Sema &SemaRef) {
-  if (SemaRef.ForceCUDAHostDeviceDepth > 0) {
-    RecordData::value_type Record[] = {SemaRef.ForceCUDAHostDeviceDepth};
+  if (SemaRef.CUDA().ForceHostDeviceDepth > 0) {
+    RecordData::value_type Record[] = {SemaRef.CUDA().ForceHostDeviceDepth};
     Stream.EmitRecord(CUDA_PRAGMA_FORCE_HOST_DEVICE_DEPTH, Record);
   }
 }
-- 
GitLab


From 6dcb60481323c871556a5a14ed887a5867ece114 Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Sat, 13 Apr 2024 08:55:09 +0400
Subject: [PATCH 133/464] [clang] Implement
 `__is_pointer_interconvertible_base_of()` (#88473)

This patch implements intrinsic that supports
`std::is_pointer_interconvertible_base_of` type trait from
[P0466R5](https://wg21.link/p0466r5) "Layout-compatibility and
Pointer-interconvertibility Traits".

Normative wording:
> Comment: If `Base` and Derived are non-union class types and are not
(possibly _cv_-qualified) versions of the same type, `Derived` is a
complete type.
> Condition: `Derived` is unambiguously derived from `Base` without
regard to _cv_-qualifiers, and each object of type `Derived` is
pointer-interconvertible (6.7.2 [basic.compound]) with its `Base`
subobject, or `Base` and `Derived` are not unions and name the same
class type without regard to _cv_-qualifiers.

The paper also express the following intent:
> Note that `is_pointer_interconvertible_base_of_v<T,T>` is always true
under this wording, even though `T` is not derived from itself.

I find the treatment of unions in the wording contradictory to this
intent, and I'm not able to find anything relevant in minutes or on the
reflector. That said, this patch implements what the wording says, since
it's very explicit about unions.
---
 clang/docs/ReleaseNotes.rst              |  3 +-
 clang/include/clang/Basic/TokenKinds.def |  1 +
 clang/include/clang/Sema/Sema.h          |  2 +
 clang/lib/Sema/SemaChecking.cpp          | 21 ++++++
 clang/lib/Sema/SemaExprCXX.cpp           | 16 +++++
 clang/test/SemaCXX/type-traits.cpp       | 91 +++++++++++++++++++++++-
 6 files changed, 132 insertions(+), 2 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 45a9a79739a4..399d5ba796d1 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -98,7 +98,8 @@ C++20 Feature Support
   behavior can use the flag '-Xclang -fno-skip-odr-check-in-gmf'.
   (#GH79240).
 
-- Implemented the `__is_layout_compatible` intrinsic to support
+- Implemented the `__is_layout_compatible` and `__is_pointer_interconvertible_base_of`
+  intrinsics to support
   `P0466R5: Layout-compatibility and Pointer-interconvertibility Traits <https://wg21.link/P0466R5>`_.
 
 - Clang now implements [module.import]p7 fully. Clang now will import module
diff --git a/clang/include/clang/Basic/TokenKinds.def b/clang/include/clang/Basic/TokenKinds.def
index 800af0e6d044..a27fbed358a6 100644
--- a/clang/include/clang/Basic/TokenKinds.def
+++ b/clang/include/clang/Basic/TokenKinds.def
@@ -521,6 +521,7 @@ TYPE_TRAIT_1(__is_union, IsUnion, KEYCXX)
 TYPE_TRAIT_1(__has_unique_object_representations,
              HasUniqueObjectRepresentations, KEYCXX)
 TYPE_TRAIT_2(__is_layout_compatible, IsLayoutCompatible, KEYCXX)
+TYPE_TRAIT_2(__is_pointer_interconvertible_base_of, IsPointerInterconvertibleBaseOf, KEYCXX)
 
 #define TRANSFORM_TYPE_TRAIT_DEF(_, Trait) KEYWORD(__##Trait, KEYCXX)
 #include "clang/Basic/TransformTypeTraits.def"
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 6b9789334811..3b252f94dcbe 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -1999,6 +1999,8 @@ public:
   };
 
   bool IsLayoutCompatible(QualType T1, QualType T2) const;
+  bool IsPointerInterconvertibleBaseOf(const TypeSourceInfo *Base,
+                                       const TypeSourceInfo *Derived);
 
   bool CheckFunctionCall(FunctionDecl *FDecl, CallExpr *TheCall,
                          const FunctionProtoType *Proto);
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index abfd9a303157..8e21811b67d9 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -19710,6 +19710,27 @@ bool Sema::IsLayoutCompatible(QualType T1, QualType T2) const {
   return isLayoutCompatible(getASTContext(), T1, T2);
 }
 
+//===-------------- Pointer interconvertibility ----------------------------//
+
+bool Sema::IsPointerInterconvertibleBaseOf(const TypeSourceInfo *Base,
+                                           const TypeSourceInfo *Derived) {
+  QualType BaseT = Base->getType()->getCanonicalTypeUnqualified();
+  QualType DerivedT = Derived->getType()->getCanonicalTypeUnqualified();
+
+  if (BaseT->isStructureOrClassType() && DerivedT->isStructureOrClassType() &&
+      getASTContext().hasSameType(BaseT, DerivedT))
+    return true;
+
+  if (!IsDerivedFrom(Derived->getTypeLoc().getBeginLoc(), DerivedT, BaseT))
+    return false;
+
+  // Per [basic.compound]/4.3, containing object has to be standard-layout.
+  if (DerivedT->getAsCXXRecordDecl()->isStandardLayout())
+    return true;
+
+  return false;
+}
+
 //===--- CHECK: pointer_with_type_tag attribute: datatypes should match ----//
 
 /// Given a type tag expression find the type tag itself.
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index 8911257a6f61..1cfd3e56a583 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -6100,6 +6100,22 @@ static bool EvaluateBinaryTypeTrait(Sema &Self, TypeTrait BTT, const TypeSourceI
       Self.Diag(Rhs->getTypeLoc().getBeginLoc(), diag::err_vla_unsupported)
           << 1 << tok::kw___is_layout_compatible;
     return Self.IsLayoutCompatible(LhsT, RhsT);
+  }
+  case BTT_IsPointerInterconvertibleBaseOf: {
+    if (LhsT->isStructureOrClassType() && RhsT->isStructureOrClassType() &&
+        !Self.getASTContext().hasSameUnqualifiedType(LhsT, RhsT)) {
+      Self.RequireCompleteType(Rhs->getTypeLoc().getBeginLoc(), RhsT,
+                               diag::err_incomplete_type);
+    }
+
+    if (LhsT->isVariableArrayType())
+      Self.Diag(Lhs->getTypeLoc().getBeginLoc(), diag::err_vla_unsupported)
+          << 1 << tok::kw___is_pointer_interconvertible_base_of;
+    if (RhsT->isVariableArrayType())
+      Self.Diag(Rhs->getTypeLoc().getBeginLoc(), diag::err_vla_unsupported)
+          << 1 << tok::kw___is_pointer_interconvertible_base_of;
+
+    return Self.IsPointerInterconvertibleBaseOf(Lhs, Rhs);
   }
     default: llvm_unreachable("not a BTT");
   }
diff --git a/clang/test/SemaCXX/type-traits.cpp b/clang/test/SemaCXX/type-traits.cpp
index 421d3007d27f..d43701c3d976 100644
--- a/clang/test/SemaCXX/type-traits.cpp
+++ b/clang/test/SemaCXX/type-traits.cpp
@@ -1740,7 +1740,7 @@ void is_layout_compatible(int n)
   static_assert(!__is_layout_compatible(void, int));
   static_assert(__is_layout_compatible(void, const void));
   static_assert(__is_layout_compatible(void, volatile void));
-  static_assert(__is_layout_compatible(const int, volatile int));
+  static_assert(__is_layout_compatible(const void, volatile void));
   static_assert(__is_layout_compatible(int, int));
   static_assert(__is_layout_compatible(int, const int));
   static_assert(__is_layout_compatible(int, volatile int));
@@ -1839,6 +1839,95 @@ void is_layout_compatible(int n)
   static_assert(!__is_layout_compatible(EnumClassForward, int));
 }
 
+namespace IPIBO {
+struct Base {};
+struct Base2 {};
+struct Base3 : Base {};
+struct Base3Virtual : virtual Base {};
+struct Derived : Base {};
+struct DerivedIndirect : Base3 {};
+struct DerivedMultiple : Base, Base2 {};
+struct DerivedAmbiguous : Base, Base3 {};
+/* expected-warning@-1 {{direct base 'Base' is inaccessible due to ambiguity:
+    struct IPIBO::DerivedAmbiguous -> Base
+    struct IPIBO::DerivedAmbiguous -> Base3 -> Base}} */
+struct DerivedPrivate : private Base {};
+struct DerivedVirtual : virtual Base {};
+
+union Union {};
+union UnionIncomplete;
+struct StructIncomplete; // #StructIncomplete
+
+void is_pointer_interconvertible_base_of(int n)
+{
+  static_assert(__is_pointer_interconvertible_base_of(Base, Derived));
+  static_assert(!__is_pointer_interconvertible_base_of(Base2, Derived));
+  static_assert(__is_pointer_interconvertible_base_of(Base, DerivedIndirect));
+  static_assert(__is_pointer_interconvertible_base_of(Base, DerivedMultiple));
+  static_assert(!__is_pointer_interconvertible_base_of(Base3, DerivedMultiple));
+  static_assert(!__is_pointer_interconvertible_base_of(Base, DerivedAmbiguous));
+  static_assert(__is_pointer_interconvertible_base_of(Base, DerivedPrivate));
+  static_assert(!__is_pointer_interconvertible_base_of(Base, DerivedVirtual));
+  static_assert(!__is_pointer_interconvertible_base_of(Union, Union));
+  static_assert(!__is_pointer_interconvertible_base_of(UnionIncomplete, UnionIncomplete));
+  static_assert(__is_pointer_interconvertible_base_of(StructIncomplete, StructIncomplete));
+  static_assert(__is_pointer_interconvertible_base_of(StructIncomplete, const StructIncomplete));
+  static_assert(__is_pointer_interconvertible_base_of(StructIncomplete, volatile StructIncomplete));
+  static_assert(__is_pointer_interconvertible_base_of(const StructIncomplete, volatile StructIncomplete));
+  static_assert(!__is_pointer_interconvertible_base_of(StructIncomplete, Derived));
+  static_assert(!__is_pointer_interconvertible_base_of(Base, StructIncomplete));
+  // expected-error@-1 {{incomplete type 'StructIncomplete' where a complete type is required}}
+  //   expected-note@#StructIncomplete {{forward declaration of 'IPIBO::StructIncomplete'}}
+  static_assert(!__is_pointer_interconvertible_base_of(CStruct2, CppStructNonStandardByBase2));
+  static_assert(!__is_pointer_interconvertible_base_of(void, void));
+  static_assert(!__is_pointer_interconvertible_base_of(void, int));
+  static_assert(!__is_pointer_interconvertible_base_of(void, const void));
+  static_assert(!__is_pointer_interconvertible_base_of(void, volatile void));
+  static_assert(!__is_pointer_interconvertible_base_of(const void, volatile void));
+  static_assert(!__is_pointer_interconvertible_base_of(int, int));
+  static_assert(!__is_pointer_interconvertible_base_of(int, const int));
+  static_assert(!__is_pointer_interconvertible_base_of(int, volatile int));
+  static_assert(!__is_pointer_interconvertible_base_of(const int, volatile int));
+  static_assert(!__is_pointer_interconvertible_base_of(int *, int * __restrict));
+  static_assert(!__is_pointer_interconvertible_base_of(int, _Atomic int));
+  static_assert(!__is_pointer_interconvertible_base_of(_Atomic(int), _Atomic int));
+  static_assert(!__is_pointer_interconvertible_base_of(int, unsigned int));
+  static_assert(!__is_pointer_interconvertible_base_of(char, unsigned char));
+  static_assert(!__is_pointer_interconvertible_base_of(char, signed char));
+  static_assert(!__is_pointer_interconvertible_base_of(unsigned char, signed char));
+  using function_type = void();
+  using function_type2 = void(char);
+  static_assert(!__is_pointer_interconvertible_base_of(const function_type, const function_type));
+  // expected-warning@-1 {{'const' qualifier on function type 'function_type' (aka 'void ()') has no effect}}
+  // expected-warning@-2 {{'const' qualifier on function type 'function_type' (aka 'void ()') has no effect}}
+  static_assert(!__is_pointer_interconvertible_base_of(function_type, const function_type));
+  // expected-warning@-1 {{'const' qualifier on function type 'function_type' (aka 'void ()') has no effect}}
+  static_assert(!__is_pointer_interconvertible_base_of(const function_type, const function_type2));
+  // expected-warning@-1 {{'const' qualifier on function type 'function_type' (aka 'void ()') has no effect}}
+  // expected-warning@-2 {{'const' qualifier on function type 'function_type2' (aka 'void (char)') has no effect}}
+  static_assert(!__is_pointer_interconvertible_base_of(int CStruct2::*, int CStruct2::*));
+  static_assert(!__is_pointer_interconvertible_base_of(int CStruct2::*, char CStruct2::*));
+  static_assert(!__is_pointer_interconvertible_base_of(void(CStruct2::*)(int), void(CStruct2::*)(int)));
+  static_assert(!__is_pointer_interconvertible_base_of(void(CStruct2::*)(int), void(CStruct2::*)(char)));
+  static_assert(!__is_pointer_interconvertible_base_of(int[], int[]));
+  static_assert(!__is_pointer_interconvertible_base_of(int[], double[]));
+  static_assert(!__is_pointer_interconvertible_base_of(int[2], int[2]));
+  static_assert(!__is_pointer_interconvertible_base_of(int[n], int[2]));
+  // expected-error@-1 {{variable length arrays are not supported in '__is_pointer_interconvertible_base_of'}}
+  static_assert(!__is_pointer_interconvertible_base_of(int[n], int[n]));
+  // expected-error@-1 {{variable length arrays are not supported in '__is_pointer_interconvertible_base_of'}}
+  // expected-error@-2 {{variable length arrays are not supported in '__is_pointer_interconvertible_base_of'}}
+  static_assert(!__is_pointer_interconvertible_base_of(int&, int&));
+  static_assert(!__is_pointer_interconvertible_base_of(int&, char&));
+  static_assert(!__is_pointer_interconvertible_base_of(void(int), void(int)));
+  static_assert(!__is_pointer_interconvertible_base_of(void(int), void(char)));
+  static_assert(!__is_pointer_interconvertible_base_of(void(&)(int), void(&)(int)));
+  static_assert(!__is_pointer_interconvertible_base_of(void(&)(int), void(&)(char)));
+  static_assert(!__is_pointer_interconvertible_base_of(void(*)(int), void(*)(int)));
+  static_assert(!__is_pointer_interconvertible_base_of(void(*)(int), void(*)(char)));
+}
+}
+
 void is_signed()
 {
   //static_assert(__is_signed(char));
-- 
GitLab


From 79dca25f4a0fefd47e9e37d9ce47d84dc0b3bedb Mon Sep 17 00:00:00 2001
From: Cyndy Ishida <cyndy_ishida@apple.com>
Date: Fri, 12 Apr 2024 22:05:12 -0700
Subject: [PATCH 134/464] [InstallAPI] Replace std::string -> StringRef

---
 clang/tools/clang-installapi/Options.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/tools/clang-installapi/Options.cpp b/clang/tools/clang-installapi/Options.cpp
index 120ff3da8998..3dc61476ce09 100644
--- a/clang/tools/clang-installapi/Options.cpp
+++ b/clang/tools/clang-installapi/Options.cpp
@@ -372,7 +372,7 @@ bool Options::addFilePaths(InputArgList &Args, PathSeq &Headers,
       }
       // Sort headers to ensure deterministic behavior.
       sort(*InputHeadersOrErr);
-      for (std::string &H : *InputHeadersOrErr)
+      for (StringRef H : *InputHeadersOrErr)
         Headers.emplace_back(std::move(H));
     } else
       Headers.emplace_back(Path);
@@ -690,7 +690,7 @@ InstallAPIContext Options::createContext() {
     FrameworkName = getFrameworkNameFromInstallName(LinkerOpts.InstallName);
 
   // Process inputs.
-  for (const std::string &ListPath : DriverOpts.FileLists) {
+  for (const StringRef ListPath : DriverOpts.FileLists) {
     auto Buffer = FM->getBufferForFile(ListPath);
     if (auto Err = Buffer.getError()) {
       Diags->Report(diag::err_cannot_open_file) << ListPath << Err.message();
-- 
GitLab


From ed128c7df9b4e60bfd814dc9fd22de1dde4a2c1c Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Sat, 13 Apr 2024 08:28:02 +0300
Subject: [PATCH 135/464] [clang][NFC] Replace `dr` with `cwg` in DR test suite

This patch is breaking `dr` to `cwg` equivalence in our terminology, making room for tests for LWG issues that concern compiler intrinsics.
---
 clang/test/CXX/drs/{dr118.cpp => cwg118.cpp}  |   2 +-
 clang/test/CXX/drs/{dr124.cpp => cwg124.cpp}  |  18 +-
 clang/test/CXX/drs/{dr158.cpp => cwg158.cpp}  |   2 +-
 .../test/CXX/drs/{dr1748.cpp => cwg1748.cpp}  |   2 +-
 .../test/CXX/drs/{dr177x.cpp => cwg177x.cpp}  |   8 +-
 .../test/CXX/drs/{dr1807.cpp => cwg1807.cpp}  |  10 +-
 clang/test/CXX/drs/{dr185.cpp => cwg185.cpp}  |  12 +-
 clang/test/CXX/drs/{dr193.cpp => cwg193.cpp}  |  18 +-
 clang/test/CXX/drs/{dr199.cpp => cwg199.cpp}  |  12 +-
 clang/test/CXX/drs/{dr201.cpp => cwg201.cpp}  |  16 +-
 clang/test/CXX/drs/{dr210.cpp => cwg210.cpp}  |   8 +-
 .../test/CXX/drs/{dr2335.cpp => cwg2335.cpp}  |  26 +-
 .../test/CXX/drs/{dr2390.cpp => cwg2390.cpp}  |   2 +-
 .../test/CXX/drs/{dr2504.cpp => cwg2504.cpp}  |   8 +-
 clang/test/CXX/drs/{dr292.cpp => cwg292.cpp}  |   8 +-
 clang/test/CXX/drs/{dr392.cpp => cwg392.cpp}  |  10 +-
 clang/test/CXX/drs/{dr412.cpp => cwg412.cpp}  |   2 +-
 clang/test/CXX/drs/{dr438.cpp => cwg438.cpp}  |   6 +-
 clang/test/CXX/drs/{dr439.cpp => cwg439.cpp}  |   6 +-
 clang/test/CXX/drs/{dr441.cpp => cwg441.cpp}  |  18 +-
 clang/test/CXX/drs/{dr462.cpp => cwg462.cpp}  |  10 +-
 clang/test/CXX/drs/{dr492.cpp => cwg492.cpp}  |   6 +-
 clang/test/CXX/drs/{dr519.cpp => cwg519.cpp}  |   6 +-
 clang/test/CXX/drs/{dr571.cpp => cwg571.cpp}  |   6 +-
 clang/test/CXX/drs/{dr605.cpp => cwg605.cpp}  |   6 +-
 clang/test/CXX/drs/{dr650.cpp => cwg650.cpp}  |  12 +-
 clang/test/CXX/drs/{dr653.cpp => cwg653.cpp}  |   6 +-
 clang/test/CXX/drs/{dr658.cpp => cwg658.cpp}  |   6 +-
 clang/test/CXX/drs/{dr661.cpp => cwg661.cpp}  |   8 +-
 clang/test/CXX/drs/{dr672.cpp => cwg672.cpp}  |   8 +-
 clang/test/CXX/drs/{dr593.cpp => cwgr593.cpp} |   6 +-
 clang/test/CXX/drs/dr0xx.cpp                  | 842 +++++++++---------
 clang/test/CXX/drs/dr10xx.cpp                 |  36 +-
 clang/test/CXX/drs/dr11xx.cpp                 |  24 +-
 clang/test/CXX/drs/dr12xx.cpp                 |  34 +-
 clang/test/CXX/drs/dr13xx.cpp                 | 234 ++---
 clang/test/CXX/drs/dr14xx.cpp                 | 238 ++---
 clang/test/CXX/drs/dr15xx.cpp                 | 370 ++++----
 clang/test/CXX/drs/dr16xx.cpp                 | 290 +++---
 clang/test/CXX/drs/dr17xx.cpp                 |  64 +-
 clang/test/CXX/drs/dr18xx.cpp                 | 100 +--
 clang/test/CXX/drs/dr19xx.cpp                 |  68 +-
 clang/test/CXX/drs/dr1xx.cpp                  | 614 ++++++-------
 clang/test/CXX/drs/dr20xx.cpp                 | 152 ++--
 clang/test/CXX/drs/dr21xx.cpp                 | 104 +--
 clang/test/CXX/drs/dr22xx.cpp                 |  46 +-
 clang/test/CXX/drs/dr23xx.cpp                 |  80 +-
 clang/test/CXX/drs/dr24xx.cpp                 |   8 +-
 clang/test/CXX/drs/dr25xx.cpp                 |  66 +-
 clang/test/CXX/drs/dr26xx.cpp                 |  76 +-
 clang/test/CXX/drs/dr27xx.cpp                 |  18 +-
 clang/test/CXX/drs/dr28xx.cpp                 |   8 +-
 clang/test/CXX/drs/dr2xx.cpp                  | 510 +++++------
 clang/test/CXX/drs/dr3xx.cpp                  | 774 ++++++++--------
 clang/test/CXX/drs/dr4xx.cpp                  | 558 ++++++------
 clang/test/CXX/drs/dr5xx.cpp                  | 444 ++++-----
 clang/test/CXX/drs/dr6xx.cpp                  | 446 +++++-----
 clang/test/CXX/drs/dr7xx.cpp                  | 120 +--
 clang/test/CXX/drs/dr8xx.cpp                  |  12 +-
 clang/test/CXX/drs/dr9xx.cpp                  |  62 +-
 clang/www/make_cxx_dr_status                  |   6 +-
 61 files changed, 3339 insertions(+), 3339 deletions(-)
 rename clang/test/CXX/drs/{dr118.cpp => cwg118.cpp} (98%)
 rename clang/test/CXX/drs/{dr124.cpp => cwg124.cpp} (83%)
 rename clang/test/CXX/drs/{dr158.cpp => cwg158.cpp} (98%)
 rename clang/test/CXX/drs/{dr1748.cpp => cwg1748.cpp} (98%)
 rename clang/test/CXX/drs/{dr177x.cpp => cwg177x.cpp} (95%)
 rename clang/test/CXX/drs/{dr1807.cpp => cwg1807.cpp} (83%)
 rename clang/test/CXX/drs/{dr185.cpp => cwg185.cpp} (85%)
 rename clang/test/CXX/drs/{dr193.cpp => cwg193.cpp} (79%)
 rename clang/test/CXX/drs/{dr199.cpp => cwg199.cpp} (85%)
 rename clang/test/CXX/drs/{dr201.cpp => cwg201.cpp} (82%)
 rename clang/test/CXX/drs/{dr210.cpp => cwg210.cpp} (91%)
 rename clang/test/CXX/drs/{dr2335.cpp => cwg2335.cpp} (56%)
 rename clang/test/CXX/drs/{dr2390.cpp => cwg2390.cpp} (98%)
 rename clang/test/CXX/drs/{dr2504.cpp => cwg2504.cpp} (90%)
 rename clang/test/CXX/drs/{dr292.cpp => cwg292.cpp} (91%)
 rename clang/test/CXX/drs/{dr392.cpp => cwg392.cpp} (88%)
 rename clang/test/CXX/drs/{dr412.cpp => cwg412.cpp} (99%)
 rename clang/test/CXX/drs/{dr438.cpp => cwg438.cpp} (94%)
 rename clang/test/CXX/drs/{dr439.cpp => cwg439.cpp} (94%)
 rename clang/test/CXX/drs/{dr441.cpp => cwg441.cpp} (76%)
 rename clang/test/CXX/drs/{dr462.cpp => cwg462.cpp} (87%)
 rename clang/test/CXX/drs/{dr492.cpp => cwg492.cpp} (94%)
 rename clang/test/CXX/drs/{dr519.cpp => cwg519.cpp} (95%)
 rename clang/test/CXX/drs/{dr571.cpp => cwg571.cpp} (91%)
 rename clang/test/CXX/drs/{dr605.cpp => cwg605.cpp} (91%)
 rename clang/test/CXX/drs/{dr650.cpp => cwg650.cpp} (86%)
 rename clang/test/CXX/drs/{dr653.cpp => cwg653.cpp} (92%)
 rename clang/test/CXX/drs/{dr658.cpp => cwg658.cpp} (92%)
 rename clang/test/CXX/drs/{dr661.cpp => cwg661.cpp} (91%)
 rename clang/test/CXX/drs/{dr672.cpp => cwg672.cpp} (91%)
 rename clang/test/CXX/drs/{dr593.cpp => cwgr593.cpp} (93%)

diff --git a/clang/test/CXX/drs/dr118.cpp b/clang/test/CXX/drs/cwg118.cpp
similarity index 98%
rename from clang/test/CXX/drs/dr118.cpp
rename to clang/test/CXX/drs/cwg118.cpp
index 58aa3912c801..04e19ce05078 100644
--- a/clang/test/CXX/drs/dr118.cpp
+++ b/clang/test/CXX/drs/cwg118.cpp
@@ -3,7 +3,7 @@
 // RUN: %clang_cc1 -triple x86_64-linux -std=c++14 %s -pedantic-errors -emit-llvm -o - | FileCheck %s --implicit-check-not " call "
 // RUN: %clang_cc1 -triple x86_64-linux -std=c++1z %s -pedantic-errors -emit-llvm -o - | FileCheck %s --implicit-check-not " call "
 
-// dr118: yes
+// cwg118: yes
 
 struct S {
   virtual void f();
diff --git a/clang/test/CXX/drs/dr124.cpp b/clang/test/CXX/drs/cwg124.cpp
similarity index 83%
rename from clang/test/CXX/drs/dr124.cpp
rename to clang/test/CXX/drs/cwg124.cpp
index c07beb11709c..fef3c6085c37 100644
--- a/clang/test/CXX/drs/dr124.cpp
+++ b/clang/test/CXX/drs/cwg124.cpp
@@ -12,7 +12,7 @@
 #define NOTHROW noexcept(true)
 #endif
 
-namespace dr124 { // dr124: 2.7
+namespace cwg124 { // cwg124: 2.7
 
 extern void full_expr_fence() NOTHROW;
 
@@ -32,20 +32,20 @@ void f() {
   full_expr_fence();
 }
 
-// CHECK-LABEL: define {{.*}} void @dr124::f()()
-// CHECK:         call void @dr124::full_expr_fence()
+// CHECK-LABEL: define {{.*}} void @cwg124::f()()
+// CHECK:         call void @cwg124::full_expr_fence()
 // CHECK:         br label %arrayctor.loop
 // CHECK-LABEL: arrayctor.loop:
-// CHECK:         call void @dr124::A::A()
-// CHECK:         call void @dr124::B::B(dr124::A)
-// CHECK:         call void @dr124::A::~A()
+// CHECK:         call void @cwg124::A::A()
+// CHECK:         call void @cwg124::B::B(cwg124::A)
+// CHECK:         call void @cwg124::A::~A()
 // CHECK:         br {{.*}}, label %arrayctor.cont, label %arrayctor.loop
 // CHECK-LABEL: arrayctor.cont:
-// CHECK:         call void @dr124::full_expr_fence()
+// CHECK:         call void @cwg124::full_expr_fence()
 // CHECK:         br label %arraydestroy.body
 // CHECK-LABEL: arraydestroy.body:
-// CHECK:         call void @dr124::B::~B()
+// CHECK:         call void @cwg124::B::~B()
 // CHECK-LABEL: }
 
 
-} // namespace dr124
+} // namespace cwg124
diff --git a/clang/test/CXX/drs/dr158.cpp b/clang/test/CXX/drs/cwg158.cpp
similarity index 98%
rename from clang/test/CXX/drs/dr158.cpp
rename to clang/test/CXX/drs/cwg158.cpp
index a0a8bd05baee..9301c790297e 100644
--- a/clang/test/CXX/drs/dr158.cpp
+++ b/clang/test/CXX/drs/cwg158.cpp
@@ -3,7 +3,7 @@
 // RUN: %clang_cc1 -triple x86_64-linux -std=c++14 %s -O3 -disable-llvm-passes -pedantic-errors -emit-llvm -o - | FileCheck %s
 // RUN: %clang_cc1 -triple x86_64-linux -std=c++1z %s -O3 -disable-llvm-passes -pedantic-errors -emit-llvm -o - | FileCheck %s
 
-// dr158: yes
+// cwg158: yes
 
 // CHECK-LABEL: define {{.*}} @_Z1f
 const int *f(const int * const *p, int **q) {
diff --git a/clang/test/CXX/drs/dr1748.cpp b/clang/test/CXX/drs/cwg1748.cpp
similarity index 98%
rename from clang/test/CXX/drs/dr1748.cpp
rename to clang/test/CXX/drs/cwg1748.cpp
index 7e04f402d266..f216963d69f2 100644
--- a/clang/test/CXX/drs/dr1748.cpp
+++ b/clang/test/CXX/drs/cwg1748.cpp
@@ -3,7 +3,7 @@
 // RUN: %clang_cc1 -std=c++14 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | FileCheck %s
 // RUN: %clang_cc1 -std=c++1z %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | FileCheck %s
 
-// dr1748: 3.7
+// cwg1748: 3.7
 
 // FIXME: __SIZE_TYPE__ expands to 'long long' on some targets.
 __extension__ typedef __SIZE_TYPE__ size_t;
diff --git a/clang/test/CXX/drs/dr177x.cpp b/clang/test/CXX/drs/cwg177x.cpp
similarity index 95%
rename from clang/test/CXX/drs/dr177x.cpp
rename to clang/test/CXX/drs/cwg177x.cpp
index 7b96ff0996c7..cc62bdac4cf0 100644
--- a/clang/test/CXX/drs/dr177x.cpp
+++ b/clang/test/CXX/drs/cwg177x.cpp
@@ -4,9 +4,9 @@
 // RUN: %clang_cc1 -std=c++1z %s -fexceptions -fcxx-exceptions -pedantic-errors -ast-dump | FileCheck %s --check-prefixes=CHECK,CXX11,CXX14
 // RUN: %clang_cc1 -std=c++1z %s -fexceptions -fcxx-exceptions -pedantic-errors -triple i386-windows-pc -ast-dump | FileCheck %s --check-prefixes=CHECK,CXX11,CXX14
 
-namespace dr1772 { // dr1772: 14
+namespace cwg1772 { // cwg1772: 14
   // __func__ in a lambda should name operator(), not the containing function.
-  // CHECK: NamespaceDecl{{.+}}dr1772
+  // CHECK: NamespaceDecl{{.+}}cwg1772
 #if __cplusplus >= 201103L
   auto x = []() { __func__; };
   // CXX11: LambdaExpr
@@ -30,10 +30,10 @@ namespace dr1772 { // dr1772: 14
 #endif // __cplusplus >= 201103L
 }
 
-namespace dr1779 { // dr1779: 14
+namespace cwg1779 { // cwg1779: 14
   // __func__ in a function template, member function template, or generic
   //  lambda should have a dependent type.
-  // CHECK: NamespaceDecl{{.+}}dr1779
+  // CHECK: NamespaceDecl{{.+}}cwg1779
 
   template<typename T>
   void FuncTemplate() {
diff --git a/clang/test/CXX/drs/dr1807.cpp b/clang/test/CXX/drs/cwg1807.cpp
similarity index 83%
rename from clang/test/CXX/drs/dr1807.cpp
rename to clang/test/CXX/drs/cwg1807.cpp
index 81e1e8ca640e..59edacc49658 100644
--- a/clang/test/CXX/drs/dr1807.cpp
+++ b/clang/test/CXX/drs/cwg1807.cpp
@@ -6,7 +6,7 @@
 // RUN: %clang_cc1 -std=c++23 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK,SINCE-CXX11
 // RUN: %clang_cc1 -std=c++2c %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK,SINCE-CXX11
 
-namespace dr1807 { // dr1807: 3.0
+namespace cwg1807 { // cwg1807: 3.0
 struct S {
   S() {}
   ~S() {}
@@ -17,12 +17,12 @@ void f() {
 }
 }
 
-// CHECK-LABEL:      define dso_local void @dr1807::f()
-// CHECK:              invoke void @dr1807::S::S(){{.+}}
+// CHECK-LABEL:      define dso_local void @cwg1807::f()
+// CHECK:              invoke void @cwg1807::S::S(){{.+}}
 // CHECK-NEXT:         {{.+}} unwind label %lpad
 // CHECK-LABEL:      lpad:
 // CHECK:              br {{.+}}, label {{.+}}, label %arraydestroy.body
 // CHECK-LABEL:      arraydestroy.body:         
 // CHECK:              [[ARRAYDESTROY_ELEMENT:%.*]] = getelementptr {{.+}}, i64 -1
-// CXX98-NEXT:         invoke void @dr1807::S::~S()({{.*}}[[ARRAYDESTROY_ELEMENT]])
-// SINCE-CXX11-NEXT:   call void @dr1807::S::~S()({{.*}}[[ARRAYDESTROY_ELEMENT]])
+// CXX98-NEXT:         invoke void @cwg1807::S::~S()({{.*}}[[ARRAYDESTROY_ELEMENT]])
+// SINCE-CXX11-NEXT:   call void @cwg1807::S::~S()({{.*}}[[ARRAYDESTROY_ELEMENT]])
diff --git a/clang/test/CXX/drs/dr185.cpp b/clang/test/CXX/drs/cwg185.cpp
similarity index 85%
rename from clang/test/CXX/drs/dr185.cpp
rename to clang/test/CXX/drs/cwg185.cpp
index aff00f1a8764..8ab5bc5d28f8 100644
--- a/clang/test/CXX/drs/dr185.cpp
+++ b/clang/test/CXX/drs/cwg185.cpp
@@ -6,7 +6,7 @@
 // RUN: %clang_cc1 -std=c++23 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK
 // RUN: %clang_cc1 -std=c++2c %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK
 
-namespace dr185 { // dr185: 2.7
+namespace cwg185 { // cwg185: 2.7
 struct A {
   mutable int value;
   explicit A(int i) : value(i) {}
@@ -20,11 +20,11 @@ int foo() {
   return n.value;
 }
 
-// CHECK-LABEL: define {{.*}} i32 @dr185::foo()
-// CHECK:         call void @dr185::A::A(int)(ptr {{[^,]*}} %ref.tmp, {{.*}})
+// CHECK-LABEL: define {{.*}} i32 @cwg185::foo()
+// CHECK:         call void @cwg185::A::A(int)(ptr {{[^,]*}} %ref.tmp, {{.*}})
 // CHECK:         store ptr %ref.tmp, ptr %t
 // CHECK-NOT:     %t =
-// CHECK:         [[DR185_T:%.+]] = load ptr, ptr %t
-// CHECK:         call void @llvm.memcpy.p0.p0.i64(ptr {{[^,]*}} %n, ptr {{[^,]*}} [[DR185_T]], {{.*}})
+// CHECK:         [[CWG185_T:%.+]] = load ptr, ptr %t
+// CHECK:         call void @llvm.memcpy.p0.p0.i64(ptr {{[^,]*}} %n, ptr {{[^,]*}} [[CWG185_T]], {{.*}})
 // CHECK-LABEL: }
-} // namespace dr185
+} // namespace cwg185
diff --git a/clang/test/CXX/drs/dr193.cpp b/clang/test/CXX/drs/cwg193.cpp
similarity index 79%
rename from clang/test/CXX/drs/dr193.cpp
rename to clang/test/CXX/drs/cwg193.cpp
index c010dad50e40..0a6ac38941d2 100644
--- a/clang/test/CXX/drs/dr193.cpp
+++ b/clang/test/CXX/drs/cwg193.cpp
@@ -12,7 +12,7 @@
 #define NOTHROW noexcept(true)
 #endif
 
-namespace dr193 { // dr193: 2.7
+namespace cwg193 { // cwg193: 2.7
 struct A {
   ~A() NOTHROW {}
 };
@@ -35,12 +35,12 @@ void foo() {
 }
 
 // skipping over D1 (complete object destructor)
-// CHECK-LABEL: define {{.*}} void @dr193::D::~D(){{.*}}
-// CHECK-LABEL: define {{.*}} void @dr193::D::~D(){{.*}}
-// CHECK-NOT:     call void @dr193::A::~A()
-// CHECK-NOT:     call void @dr193::B::~B()
-// CHECK:         call void @dr193::C::~C()
-// CHECK:         call void @dr193::B::~B()
-// CHECK:         call void @dr193::A::~A()
+// CHECK-LABEL: define {{.*}} void @cwg193::D::~D(){{.*}}
+// CHECK-LABEL: define {{.*}} void @cwg193::D::~D(){{.*}}
+// CHECK-NOT:     call void @cwg193::A::~A()
+// CHECK-NOT:     call void @cwg193::B::~B()
+// CHECK:         call void @cwg193::C::~C()
+// CHECK:         call void @cwg193::B::~B()
+// CHECK:         call void @cwg193::A::~A()
 // CHECK-LABEL: }
-} // namespace dr193
+} // namespace cwg193
diff --git a/clang/test/CXX/drs/dr199.cpp b/clang/test/CXX/drs/cwg199.cpp
similarity index 85%
rename from clang/test/CXX/drs/dr199.cpp
rename to clang/test/CXX/drs/cwg199.cpp
index 7517d79680c6..5d2e5110786f 100644
--- a/clang/test/CXX/drs/dr199.cpp
+++ b/clang/test/CXX/drs/cwg199.cpp
@@ -12,7 +12,7 @@
 #define NOTHROW noexcept(true)
 #endif
 
-namespace dr199 { // dr199: 2.8
+namespace cwg199 { // cwg199: 2.8
 struct A {
   ~A() NOTHROW {}
 };
@@ -25,9 +25,9 @@ void foo() {
   A(), B();
 }
 
-// CHECK-LABEL: define {{.*}} void @dr199::foo()
-// CHECK-NOT:     call void @dr199::A::~A()
-// CHECK:         call void @dr199::B::~B()
-// CHECK:         call void @dr199::A::~A()
+// CHECK-LABEL: define {{.*}} void @cwg199::foo()
+// CHECK-NOT:     call void @cwg199::A::~A()
+// CHECK:         call void @cwg199::B::~B()
+// CHECK:         call void @cwg199::A::~A()
 // CHECK-LABEL: }
-} // namespace dr199
+} // namespace cwg199
diff --git a/clang/test/CXX/drs/dr201.cpp b/clang/test/CXX/drs/cwg201.cpp
similarity index 82%
rename from clang/test/CXX/drs/dr201.cpp
rename to clang/test/CXX/drs/cwg201.cpp
index 7e864981e13b..b6cf92a1fc74 100644
--- a/clang/test/CXX/drs/dr201.cpp
+++ b/clang/test/CXX/drs/cwg201.cpp
@@ -12,7 +12,7 @@
 #define NOTHROW noexcept(true)
 #endif
 
-namespace dr201 { // dr201: 2.8
+namespace cwg201 { // cwg201: 2.8
 
 extern void full_expr_fence() NOTHROW;
 
@@ -31,12 +31,12 @@ void foo() {
   full_expr_fence();
 }
 
-// CHECK-LABEL: define {{.*}} void @dr201::foo()
-// CHECK:         call void @dr201::full_expr_fence()
-// CHECK:         call void @dr201::B::B(dr201::A)
-// CHECK:         call void @dr201::A::~A()
-// CHECK:         call void @dr201::full_expr_fence()
-// CHECK:         call void @dr201::B::~B()
+// CHECK-LABEL: define {{.*}} void @cwg201::foo()
+// CHECK:         call void @cwg201::full_expr_fence()
+// CHECK:         call void @cwg201::B::B(cwg201::A)
+// CHECK:         call void @cwg201::A::~A()
+// CHECK:         call void @cwg201::full_expr_fence()
+// CHECK:         call void @cwg201::B::~B()
 // CHECK-LABEL: }
 
-} // namespace dr201
+} // namespace cwg201
diff --git a/clang/test/CXX/drs/dr210.cpp b/clang/test/CXX/drs/cwg210.cpp
similarity index 91%
rename from clang/test/CXX/drs/dr210.cpp
rename to clang/test/CXX/drs/cwg210.cpp
index 156ee81093b4..2c3cf61a6a5b 100644
--- a/clang/test/CXX/drs/dr210.cpp
+++ b/clang/test/CXX/drs/cwg210.cpp
@@ -13,7 +13,7 @@
 #pragma clang diagnostic pop
 #endif
 
-namespace dr210 { // dr210: 2.7
+namespace cwg210 { // cwg210: 2.7
 struct B {
   long i;
   B();
@@ -33,9 +33,9 @@ void toss(const B* b) {
   throw *b;
 }
 
-// CHECK-LABEL: define {{.*}} void @dr210::toss(dr210::B const*)
+// CHECK-LABEL: define {{.*}} void @cwg210::toss(cwg210::B const*)
 // CHECK:         %[[EXCEPTION:.*]] = call ptr @__cxa_allocate_exception(i64 16)
-// CHECK:         call void @__cxa_throw(ptr %[[EXCEPTION]], ptr @typeinfo for dr210::B, ptr @dr210::B::~B())
+// CHECK:         call void @__cxa_throw(ptr %[[EXCEPTION]], ptr @typeinfo for cwg210::B, ptr @cwg210::B::~B())
 // CHECK-LABEL: }
 
-} // namespace dr210
+} // namespace cwg210
diff --git a/clang/test/CXX/drs/dr2335.cpp b/clang/test/CXX/drs/cwg2335.cpp
similarity index 56%
rename from clang/test/CXX/drs/dr2335.cpp
rename to clang/test/CXX/drs/cwg2335.cpp
index 33eebb2c4a5c..8b00a9d2d98a 100644
--- a/clang/test/CXX/drs/dr2335.cpp
+++ b/clang/test/CXX/drs/cwg2335.cpp
@@ -10,7 +10,7 @@
 // expected-no-diagnostics
 #endif
 
-namespace dr2335 { // dr2335: no drafting 2018-06
+namespace cwg2335 { // cwg2335: no drafting 2018-06
 // FIXME: current consensus is that the examples are well-formed.
 #if __cplusplus >= 201402L
 namespace ex1 {
@@ -25,24 +25,24 @@ namespace ex2 {
 template <int> struct X {};
 template <class T> struct partition_indices {
   static auto compute_right() { return X<I>(); }
-  // since-cxx14-error@-1 {{no member 'I' in 'dr2335::ex2::partition_indices<int>'; it has not yet been instantiated}}
-  //   since-cxx14-note@#dr2335-ex2-right {{in instantiation of member function 'dr2335::ex2::partition_indices<int>::compute_right' requested here}}
-  //   since-cxx14-note@#dr2335-ex2-inst {{in instantiation of template class 'dr2335::ex2::partition_indices<int>' requested here}}
-  //   since-cxx14-note@#dr2335-ex2-I {{not-yet-instantiated member is declared here}}
-  static constexpr auto right = compute_right; // #dr2335-ex2-right
-  static constexpr int I = sizeof(T); // #dr2335-ex2-I
+  // since-cxx14-error@-1 {{no member 'I' in 'cwg2335::ex2::partition_indices<int>'; it has not yet been instantiated}}
+  //   since-cxx14-note@#cwg2335-ex2-right {{in instantiation of member function 'cwg2335::ex2::partition_indices<int>::compute_right' requested here}}
+  //   since-cxx14-note@#cwg2335-ex2-inst {{in instantiation of template class 'cwg2335::ex2::partition_indices<int>' requested here}}
+  //   since-cxx14-note@#cwg2335-ex2-I {{not-yet-instantiated member is declared here}}
+  static constexpr auto right = compute_right; // #cwg2335-ex2-right
+  static constexpr int I = sizeof(T); // #cwg2335-ex2-I
 };
-template struct partition_indices<int>; // #dr2335-ex2-inst
+template struct partition_indices<int>; // #cwg2335-ex2-inst
 } // namespace ex2
 
 namespace ex3 {
 struct partition_indices {
-  static auto compute_right() {} // #dr2335-compute_right
-  static constexpr auto right = compute_right; // #dr2335-ex3-right
+  static auto compute_right() {} // #cwg2335-compute_right
+  static constexpr auto right = compute_right; // #cwg2335-ex3-right
   // since-cxx14-error@-1 {{function 'compute_right' with deduced return type cannot be used before it is defined}}
-  //   since-cxx14-note@#dr2335-compute_right {{'compute_right' declared here}}
-  // since-cxx14-error@#dr2335-ex3-right {{declaration of variable 'right' with deduced type 'const auto' requires an initializer}}
+  //   since-cxx14-note@#cwg2335-compute_right {{'compute_right' declared here}}
+  // since-cxx14-error@#cwg2335-ex3-right {{declaration of variable 'right' with deduced type 'const auto' requires an initializer}}
 };
 } // namespace ex3
 #endif
-} // namespace dr2335
+} // namespace cwg2335
diff --git a/clang/test/CXX/drs/dr2390.cpp b/clang/test/CXX/drs/cwg2390.cpp
similarity index 98%
rename from clang/test/CXX/drs/dr2390.cpp
rename to clang/test/CXX/drs/cwg2390.cpp
index 3931365b568c..41bbd0d1c549 100644
--- a/clang/test/CXX/drs/dr2390.cpp
+++ b/clang/test/CXX/drs/cwg2390.cpp
@@ -1,6 +1,6 @@
 // RUN: %clang_cc1 -E -P %s -o - | FileCheck %s
 
-// dr2390: 14
+// cwg2390: 14
 
 namespace PR48462 {
 // Test that macro expansion of the builtin argument works.
diff --git a/clang/test/CXX/drs/dr2504.cpp b/clang/test/CXX/drs/cwg2504.cpp
similarity index 90%
rename from clang/test/CXX/drs/dr2504.cpp
rename to clang/test/CXX/drs/cwg2504.cpp
index 686ea73cd6a0..fa775df327cb 100644
--- a/clang/test/CXX/drs/dr2504.cpp
+++ b/clang/test/CXX/drs/cwg2504.cpp
@@ -6,7 +6,7 @@
 // RUN: %clang_cc1 -std=c++23 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK,SINCE-CXX11
 // RUN: %clang_cc1 -std=c++2c %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK,SINCE-CXX11
 
-namespace dr2504 { // dr2504: no
+namespace cwg2504 { // cwg2504: no
 #if __cplusplus >= 201103L
 struct V { V() = default; V(int); };
 struct Q { Q(); };
@@ -27,11 +27,11 @@ void foo() { C c; } // bar is not invoked, because the V subobject is not initia
 //        we are not supposed to unconditionally call `bar()` and call a constructor
 //        inherited from `V`.
 
-// SINCE-CXX11-LABEL: define linkonce_odr void @dr2504::B::B()
+// SINCE-CXX11-LABEL: define linkonce_odr void @cwg2504::B::B()
 // SINCE-CXX11-NOT:     br
-// SINCE-CXX11:         call noundef i32 @dr2504::bar()
+// SINCE-CXX11:         call noundef i32 @cwg2504::bar()
 // SINCE-CXX11-NOT:     br
-// SINCE-CXX11:         call void @dr2504::A::A(int)
+// SINCE-CXX11:         call void @cwg2504::A::A(int)
 // SINCE-CXX11-LABEL: }
 
 // CHECK: {{.*}}
diff --git a/clang/test/CXX/drs/dr292.cpp b/clang/test/CXX/drs/cwg292.cpp
similarity index 91%
rename from clang/test/CXX/drs/dr292.cpp
rename to clang/test/CXX/drs/cwg292.cpp
index 19caeef291fa..b05d3b92d627 100644
--- a/clang/test/CXX/drs/dr292.cpp
+++ b/clang/test/CXX/drs/cwg292.cpp
@@ -6,7 +6,7 @@
 // RUN: %clang_cc1 -std=c++23 %s -triple x86_64-linux-gnu -emit-llvm -disable-llvm-passes -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK
 // RUN: %clang_cc1 -std=c++2c %s -triple x86_64-linux-gnu -emit-llvm -disable-llvm-passes -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK
 
-namespace dr292 { // dr292: 2.9
+namespace cwg292 { // cwg292: 2.9
 
 extern int g();
 
@@ -18,13 +18,13 @@ void f() {
   new A(g());
 }
 
-// CHECK-LABEL: define {{.*}} void @dr292::f()()
+// CHECK-LABEL: define {{.*}} void @cwg292::f()()
 // CHECK:         %[[CALL:.+]] = call {{.*}} @operator new(unsigned long)({{.*}})
-// CHECK:         invoke {{.*}} i32 @dr292::g()()
+// CHECK:         invoke {{.*}} i32 @cwg292::g()()
 // CHECK-NEXT:           to {{.*}} unwind label %lpad
 // CHECK-LABEL: lpad:
 // CHECK:         call void @operator delete(void*)(ptr {{.*}} %[[CALL]])
 // CHECK-LABEL: eh.resume:
 // CHECK-LABEL: }
 
-} // namespace dr292
+} // namespace cwg292
diff --git a/clang/test/CXX/drs/dr392.cpp b/clang/test/CXX/drs/cwg392.cpp
similarity index 88%
rename from clang/test/CXX/drs/dr392.cpp
rename to clang/test/CXX/drs/cwg392.cpp
index 26e6259f7196..e118dd7bdb5c 100644
--- a/clang/test/CXX/drs/dr392.cpp
+++ b/clang/test/CXX/drs/cwg392.cpp
@@ -12,7 +12,7 @@
 #define NOTHROW noexcept(true)
 #endif
 
-namespace dr392 { // dr392: 2.8
+namespace cwg392 { // cwg392: 2.8
 
 struct A {
   operator bool() NOTHROW;
@@ -32,9 +32,9 @@ void f()
   if (C().get()) {}
 }
 
-} // namespace dr392
+} // namespace cwg392
 
-// CHECK-LABEL: define {{.*}} void @dr392::f()()
-// CHECK:         call {{.*}} i1 @dr392::A::operator bool()
-// CHECK:         call void @dr392::C::~C()
+// CHECK-LABEL: define {{.*}} void @cwg392::f()()
+// CHECK:         call {{.*}} i1 @cwg392::A::operator bool()
+// CHECK:         call void @cwg392::C::~C()
 // CHECK-LABEL: }
diff --git a/clang/test/CXX/drs/dr412.cpp b/clang/test/CXX/drs/cwg412.cpp
similarity index 99%
rename from clang/test/CXX/drs/dr412.cpp
rename to clang/test/CXX/drs/cwg412.cpp
index 8ea29135d1df..7e75bececac8 100644
--- a/clang/test/CXX/drs/dr412.cpp
+++ b/clang/test/CXX/drs/cwg412.cpp
@@ -6,7 +6,7 @@
 // RUN: %clang_cc1 -std=c++23 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -DNOEXCEPT=noexcept -DBAD_ALLOC=
 // RUN: %clang_cc1 -std=c++2c %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -DNOEXCEPT=noexcept -DBAD_ALLOC=
 
-// dr412: 3.4
+// cwg412: 3.4
 // lwg404: yes
 // lwg2340: yes
 
diff --git a/clang/test/CXX/drs/dr438.cpp b/clang/test/CXX/drs/cwg438.cpp
similarity index 94%
rename from clang/test/CXX/drs/dr438.cpp
rename to clang/test/CXX/drs/cwg438.cpp
index a6ed39b88c24..5f2fb7c70d87 100644
--- a/clang/test/CXX/drs/dr438.cpp
+++ b/clang/test/CXX/drs/cwg438.cpp
@@ -6,7 +6,7 @@
 // RUN: %clang_cc1 -std=c++23 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK
 // RUN: %clang_cc1 -std=c++2c %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK
 
-namespace dr438 { // dr438: 2.7
+namespace cwg438 { // cwg438: 2.7
 
 void f() {
   long A[2];
@@ -14,9 +14,9 @@ void f() {
   A[A[0]] = 1;
 }
 
-} // namespace dr438
+} // namespace cwg438
 
-// CHECK-LABEL: define {{.*}} void @dr438::f()()
+// CHECK-LABEL: define {{.*}} void @cwg438::f()()
 // CHECK:         [[A:%.+]] = alloca [2 x i64]
 // CHECK:         {{.+}} = getelementptr inbounds [2 x i64], ptr [[A]], i64 0, i64 0
 // CHECK:         [[ARRAYIDX1:%.+]] = getelementptr inbounds [2 x i64], ptr [[A]], i64 0, i64 0
diff --git a/clang/test/CXX/drs/dr439.cpp b/clang/test/CXX/drs/cwg439.cpp
similarity index 94%
rename from clang/test/CXX/drs/dr439.cpp
rename to clang/test/CXX/drs/cwg439.cpp
index 46960af93bb9..e409b803797f 100644
--- a/clang/test/CXX/drs/dr439.cpp
+++ b/clang/test/CXX/drs/cwg439.cpp
@@ -6,7 +6,7 @@
 // RUN: %clang_cc1 -std=c++23 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK
 // RUN: %clang_cc1 -std=c++2c %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK
 
-namespace dr439 { // dr439: 2.7
+namespace cwg439 { // cwg439: 2.7
 
 void f() {
   int* p1 = new int;
@@ -14,12 +14,12 @@ void f() {
   bool b = p1 == p2; // b will have the value true.
 }
 
-} // namespace dr439
+} // namespace cwg439
 
 // We're checking that p2 was copied from p1, and then was carried over
 // to the comparison without change.
 
-// CHECK-LABEL: define {{.*}} void @dr439::f()()
+// CHECK-LABEL: define {{.*}} void @cwg439::f()()
 // CHECK:         [[P1:%.+]] = alloca ptr, align 8
 // CHECK-NEXT:    [[P2:%.+]] = alloca ptr, align 8
 // CHECK:         [[TEMP0:%.+]] = load ptr, ptr [[P1]]
diff --git a/clang/test/CXX/drs/dr441.cpp b/clang/test/CXX/drs/cwg441.cpp
similarity index 76%
rename from clang/test/CXX/drs/dr441.cpp
rename to clang/test/CXX/drs/cwg441.cpp
index 6504bba689d2..5f566f230193 100644
--- a/clang/test/CXX/drs/dr441.cpp
+++ b/clang/test/CXX/drs/cwg441.cpp
@@ -6,7 +6,7 @@
 // RUN: %clang_cc1 -std=c++23 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK
 // RUN: %clang_cc1 -std=c++2c %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK
 
-namespace dr441 { // dr441: 2.7
+namespace cwg441 { // cwg441: 2.7
 
 struct A {
   A() {}
@@ -17,21 +17,21 @@ int i;
 int& ir = i;
 int* ip = &i;
 
-} // namespace dr441
+} // namespace cwg441
 
-// CHECK-DAG:   @dr441::dynamic_init = global %"struct.dr441::A" zeroinitializer
-// CHECK-DAG:   @dr441::i = global i32 0
-// CHECK-DAG:   @dr441::ir = constant ptr @dr441::i
-// CHECK-DAG:   @dr441::ip = global ptr @dr441::i
-// CHECK-DAG:   @llvm.global_ctors = appending global [{{.+}}] [{ {{.+}} } { {{.+}}, ptr @_GLOBAL__sub_I_dr441.cpp, {{.+}} }]
+// CHECK-DAG:   @cwg441::dynamic_init = global %"struct.cwg441::A" zeroinitializer
+// CHECK-DAG:   @cwg441::i = global i32 0
+// CHECK-DAG:   @cwg441::ir = constant ptr @cwg441::i
+// CHECK-DAG:   @cwg441::ip = global ptr @cwg441::i
+// CHECK-DAG:   @llvm.global_ctors = appending global [{{.+}}] [{ {{.+}} } { {{.+}}, ptr @_GLOBAL__sub_I_cwg441.cpp, {{.+}} }]
 
 // CHECK-LABEL: define {{.*}} void @__cxx_global_var_init()
 // CHECK-NEXT:  entry:
-// CHECK-NEXT:    call void @dr441::A::A()({{.*}} @dr441::dynamic_init)
+// CHECK-NEXT:    call void @cwg441::A::A()({{.*}} @cwg441::dynamic_init)
 // CHECK-NEXT:    ret void
 // CHECK-NEXT:  }
 
-// CHECK-LABEL: define {{.*}} void @_GLOBAL__sub_I_dr441.cpp()
+// CHECK-LABEL: define {{.*}} void @_GLOBAL__sub_I_cwg441.cpp()
 // CHECK-NEXT:  entry:
 // CHECK-NEXT:    call void @__cxx_global_var_init()
 // CHECK-NEXT:    ret void
diff --git a/clang/test/CXX/drs/dr462.cpp b/clang/test/CXX/drs/cwg462.cpp
similarity index 87%
rename from clang/test/CXX/drs/dr462.cpp
rename to clang/test/CXX/drs/cwg462.cpp
index 2b268778ea10..bdbcacd733bb 100644
--- a/clang/test/CXX/drs/dr462.cpp
+++ b/clang/test/CXX/drs/cwg462.cpp
@@ -12,7 +12,7 @@
 #define NOTHROW noexcept(true)
 #endif
 
-namespace dr462 { // dr462: 2.7
+namespace cwg462 { // cwg462: 2.7
 
 struct A {
   ~A() NOTHROW {}
@@ -25,9 +25,9 @@ void f() {
   full_expr_fence();
 }
 
-} // namespace dr462
+} // namespace cwg462
 
-// CHECK-LABEL: define {{.*}} void @dr462::f()()
-// CHECK:         call void @dr462::full_expr_fence()()
-// CHECK:         call void @dr462::A::~A()
+// CHECK-LABEL: define {{.*}} void @cwg462::f()()
+// CHECK:         call void @cwg462::full_expr_fence()()
+// CHECK:         call void @cwg462::A::~A()
 // CHECK-LABEL: }
diff --git a/clang/test/CXX/drs/dr492.cpp b/clang/test/CXX/drs/cwg492.cpp
similarity index 94%
rename from clang/test/CXX/drs/dr492.cpp
rename to clang/test/CXX/drs/cwg492.cpp
index f53f1cb54124..7fc46b04d72b 100644
--- a/clang/test/CXX/drs/dr492.cpp
+++ b/clang/test/CXX/drs/cwg492.cpp
@@ -18,7 +18,7 @@ struct type_info {
 }; 
 }
 
-namespace dr492 { // dr492: 2.7
+namespace cwg492 { // cwg492: 2.7
 
 void f() {
   typeid(int).name();
@@ -27,9 +27,9 @@ void f() {
   typeid(const volatile int).name();
 }
 
-} // namespace dr492
+} // namespace cwg492
 
-// CHECK-LABEL: define {{.*}} void @dr492::f()()
+// CHECK-LABEL: define {{.*}} void @cwg492::f()()
 // CHECK:         {{.*}} = call {{.*}} @std::type_info::name() const({{.*}} @typeinfo for int)
 // CHECK-NEXT:    {{.*}} = call {{.*}} @std::type_info::name() const({{.*}} @typeinfo for int)
 // CHECK-NEXT:    {{.*}} = call {{.*}} @std::type_info::name() const({{.*}} @typeinfo for int)
diff --git a/clang/test/CXX/drs/dr519.cpp b/clang/test/CXX/drs/cwg519.cpp
similarity index 95%
rename from clang/test/CXX/drs/dr519.cpp
rename to clang/test/CXX/drs/cwg519.cpp
index 67c01d95ef7c..ce8a1cc95f60 100644
--- a/clang/test/CXX/drs/dr519.cpp
+++ b/clang/test/CXX/drs/cwg519.cpp
@@ -6,7 +6,7 @@
 // RUN: %clang_cc1 -std=c++23 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK
 // RUN: %clang_cc1 -std=c++2c %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK
 
-namespace dr519 { // dr519: 2.7
+namespace cwg519 { // cwg519: 2.7
 void f() {
   int *a = 0;
   void *v = a;
@@ -16,12 +16,12 @@ void f() {
   int *b = static_cast<int*>(w);
   bool c2 = b == static_cast<int *>(0);
 }
-} // namespace dr519
+} // namespace cwg519
 
 // We're checking that `null`s that were initially stored in `a` and `w`
 // are simply copied over all the way to respective comparisons with `null`.
 
-// CHECK-LABEL: define {{.*}} void @dr519::f()()
+// CHECK-LABEL: define {{.*}} void @cwg519::f()()
 // CHECK:         store ptr null, ptr [[A:%.+]],
 // CHECK-NEXT:    [[TEMP_A:%.+]] = load ptr, ptr [[A]] 
 // CHECK-NEXT:    store ptr [[TEMP_A]], ptr [[V:%.+]],
diff --git a/clang/test/CXX/drs/dr571.cpp b/clang/test/CXX/drs/cwg571.cpp
similarity index 91%
rename from clang/test/CXX/drs/dr571.cpp
rename to clang/test/CXX/drs/cwg571.cpp
index 19a85b7ddc35..9f0f455fb727 100644
--- a/clang/test/CXX/drs/dr571.cpp
+++ b/clang/test/CXX/drs/cwg571.cpp
@@ -6,7 +6,7 @@
 // RUN: %clang_cc1 -std=c++23 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK
 // RUN: %clang_cc1 -std=c++2c %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK
 
-namespace dr571 { // dr571: 2.7
+namespace cwg571 { // cwg571: 2.7
   typedef int &ir;
   int n;
   const ir r = n;
@@ -16,5 +16,5 @@ namespace dr571 { // dr571: 2.7
 
 // Entities have external linkage by default.
 
-// CHECK: @dr571::r = constant ptr @dr571::n
-// CHECK: @dr571::r2 = constant ptr @dr571::n
+// CHECK: @cwg571::r = constant ptr @cwg571::n
+// CHECK: @cwg571::r2 = constant ptr @cwg571::n
diff --git a/clang/test/CXX/drs/dr605.cpp b/clang/test/CXX/drs/cwg605.cpp
similarity index 91%
rename from clang/test/CXX/drs/dr605.cpp
rename to clang/test/CXX/drs/cwg605.cpp
index 6c212d8dabc0..2fd9e8155bf7 100644
--- a/clang/test/CXX/drs/dr605.cpp
+++ b/clang/test/CXX/drs/cwg605.cpp
@@ -6,7 +6,7 @@
 // RUN: %clang_cc1 -std=c++23 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK
 // RUN: %clang_cc1 -std=c++2c %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK
 
-namespace dr605 { // dr605: 2.7
+namespace cwg605 { // cwg605: 2.7
 
 template <class T>
 static T f(T t) {}
@@ -18,6 +18,6 @@ void g(int a) {
   f(a);
 }
 
-} // namespace dr605
+} // namespace cwg605
 
-// CHECK: define internal {{.*}} i32 @int dr605::f<int>(int)
+// CHECK: define internal {{.*}} i32 @int cwg605::f<int>(int)
diff --git a/clang/test/CXX/drs/dr650.cpp b/clang/test/CXX/drs/cwg650.cpp
similarity index 86%
rename from clang/test/CXX/drs/dr650.cpp
rename to clang/test/CXX/drs/cwg650.cpp
index 715b4fdf04a7..dcb844095b05 100644
--- a/clang/test/CXX/drs/dr650.cpp
+++ b/clang/test/CXX/drs/cwg650.cpp
@@ -12,7 +12,7 @@
 #define NOTHROW noexcept(true)
 #endif
 
-namespace dr650 { // dr650: 2.8
+namespace cwg650 { // cwg650: 2.8
 
 struct Q {
   ~Q() NOTHROW;
@@ -31,10 +31,10 @@ const S& f() {
   return (R(), S());
 }
 
-} // namespace dr650
+} // namespace cwg650
 
-// CHECK-LABEL: define {{.*}} @dr650::f()()
-// CHECK:         call void @dr650::S::~S()
-// CHECK:         call void @dr650::R::~R()
-// CHECK:         call void @dr650::Q::~Q()
+// CHECK-LABEL: define {{.*}} @cwg650::f()()
+// CHECK:         call void @cwg650::S::~S()
+// CHECK:         call void @cwg650::R::~R()
+// CHECK:         call void @cwg650::Q::~Q()
 // CHECK-LABEL: }
diff --git a/clang/test/CXX/drs/dr653.cpp b/clang/test/CXX/drs/cwg653.cpp
similarity index 92%
rename from clang/test/CXX/drs/dr653.cpp
rename to clang/test/CXX/drs/cwg653.cpp
index fd1f0153bfb7..3aeb394347ea 100644
--- a/clang/test/CXX/drs/dr653.cpp
+++ b/clang/test/CXX/drs/cwg653.cpp
@@ -6,7 +6,7 @@
 // RUN: %clang_cc1 -std=c++23 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK
 // RUN: %clang_cc1 -std=c++2c %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK
 
-namespace dr653 { // dr653: 2.7
+namespace cwg653 { // cwg653: 2.7
 
 union U {
   int a;
@@ -18,8 +18,8 @@ void f(U u) {
   v = u;
 }
 
-} // namespace dr653
+} // namespace cwg653
 
-// CHECK-LABEL: define {{.*}} void @dr653::f(dr653::U)
+// CHECK-LABEL: define {{.*}} void @cwg653::f(cwg653::U)
 // CHECK:         call void @llvm.memcpy.p0.p0.i64(ptr {{.*}} %v, ptr {{.*}} %u, {{.*}})
 // CHECK-LABEL: }
diff --git a/clang/test/CXX/drs/dr658.cpp b/clang/test/CXX/drs/cwg658.cpp
similarity index 92%
rename from clang/test/CXX/drs/dr658.cpp
rename to clang/test/CXX/drs/cwg658.cpp
index 51034c2af3bf..2f7f1ad7deda 100644
--- a/clang/test/CXX/drs/dr658.cpp
+++ b/clang/test/CXX/drs/cwg658.cpp
@@ -6,17 +6,17 @@
 // RUN: %clang_cc1 -std=c++23 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK
 // RUN: %clang_cc1 -std=c++2c %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK
 
-namespace dr658 { // dr658: 2.7
+namespace cwg658 { // cwg658: 2.7
 
 void f(int* p1) {
   char* p2 = reinterpret_cast<char*>(p1);
 }
 
-} // namespace dr658
+} // namespace cwg658
 
 // We're checking that p1 is stored into p2 without changes.
 
-// CHECK-LABEL: define {{.*}} void @dr658::f(int*)(ptr noundef %p1)
+// CHECK-LABEL: define {{.*}} void @cwg658::f(int*)(ptr noundef %p1)
 // CHECK:         [[P1_ADDR:%.+]] = alloca ptr, align 8
 // CHECK-NEXT:    [[P2:%.+]] = alloca ptr, align 8
 // CHECK:         store ptr %p1, ptr [[P1_ADDR]]
diff --git a/clang/test/CXX/drs/dr661.cpp b/clang/test/CXX/drs/cwg661.cpp
similarity index 91%
rename from clang/test/CXX/drs/dr661.cpp
rename to clang/test/CXX/drs/cwg661.cpp
index 4e97bb708847..557213171070 100644
--- a/clang/test/CXX/drs/dr661.cpp
+++ b/clang/test/CXX/drs/cwg661.cpp
@@ -6,9 +6,9 @@
 // RUN: %clang_cc1 -std=c++23 %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK
 // RUN: %clang_cc1 -std=c++2c %s -triple x86_64-linux-gnu -emit-llvm -o - -fexceptions -fcxx-exceptions -pedantic-errors | llvm-cxxfilt -n | FileCheck %s --check-prefixes CHECK
 
-namespace dr661 {
+namespace cwg661 {
 
-void f(int a, int b) { // dr661: 2.7
+void f(int a, int b) { // cwg661: 2.7
   a == b;
   a != b;
   a < b;
@@ -17,9 +17,9 @@ void f(int a, int b) { // dr661: 2.7
   a >= b;
 }
 
-} // namespace dr661
+} // namespace cwg661
 
-// CHECK-LABEL: define {{.*}} void @dr661::f(int, int)
+// CHECK-LABEL: define {{.*}} void @cwg661::f(int, int)
 // CHECK:         icmp eq
 // CHECK:         icmp ne
 // CHECK:         icmp slt
diff --git a/clang/test/CXX/drs/dr672.cpp b/clang/test/CXX/drs/cwg672.cpp
similarity index 91%
rename from clang/test/CXX/drs/dr672.cpp
rename to clang/test/CXX/drs/cwg672.cpp
index d5f0530ecbc9..07a64eaaf826 100644
--- a/clang/test/CXX/drs/dr672.cpp
+++ b/clang/test/CXX/drs/cwg672.cpp
@@ -12,7 +12,7 @@
 #define NOTHROW noexcept(true)
 #endif
 
-namespace dr672 { // dr672: 2.7
+namespace cwg672 { // cwg672: 2.7
 
 struct A {
   A() NOTHROW;
@@ -22,11 +22,11 @@ void f() {
   A *a = new A;
 }
 
-} // namespace dr672
+} // namespace cwg672
 
-// CHECK-LABEL: define {{.*}} void @dr672::f()()
+// CHECK-LABEL: define {{.*}} void @cwg672::f()()
 // CHECK:         [[A:%.+]] = alloca ptr
 // CHECK:         [[CALL:%.+]] = call {{.*}} ptr @operator new(unsigned long)
-// CHECK:         call void @dr672::A::A()
+// CHECK:         call void @cwg672::A::A()
 // CHECK:         store ptr [[CALL]], ptr [[A]]
 // CHECK-LABEL: }
diff --git a/clang/test/CXX/drs/dr593.cpp b/clang/test/CXX/drs/cwgr593.cpp
similarity index 93%
rename from clang/test/CXX/drs/dr593.cpp
rename to clang/test/CXX/drs/cwgr593.cpp
index 4998af966ebb..d747f4e4a161 100644
--- a/clang/test/CXX/drs/dr593.cpp
+++ b/clang/test/CXX/drs/cwgr593.cpp
@@ -12,7 +12,7 @@
 #define NOTHROW noexcept(true)
 #endif
 
-namespace dr593 { // dr593: 2.8
+namespace cwg593 { // cwg593: 2.8
 
 void f();
 void fence() NOTHROW;
@@ -29,7 +29,7 @@ void g() {
   A();
 }
 
-} // namespace dr593
+} // namespace cwg593
 
-// CHECK:      call void @dr593::fence()()
+// CHECK:      call void @cwg593::fence()()
 // CHECK-NEXT: invoke void @__cxa_rethrow()
diff --git a/clang/test/CXX/drs/dr0xx.cpp b/clang/test/CXX/drs/dr0xx.cpp
index 5959f0a0c8dd..a304862885c6 100644
--- a/clang/test/CXX/drs/dr0xx.cpp
+++ b/clang/test/CXX/drs/dr0xx.cpp
@@ -5,44 +5,44 @@
 // RUN: %clang_cc1 -std=c++20 %s -verify=expected,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors -triple %itanium_abi_triple
 // RUN: %clang_cc1 -std=c++23 %s -verify=expected,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors -triple %itanium_abi_triple
 
-namespace dr1 { // dr1: no
-  namespace X { extern "C" void dr1_f(int a = 1); }
-  namespace Y { extern "C" void dr1_f(int a = 1); }
-  using X::dr1_f; using Y::dr1_f;
+namespace cwg1 { // cwg1: no
+  namespace X { extern "C" void cwg1_f(int a = 1); }
+  namespace Y { extern "C" void cwg1_f(int a = 1); }
+  using X::cwg1_f; using Y::cwg1_f;
   void g() {
-    dr1_f(0);
+    cwg1_f(0);
     // FIXME: This should be rejected, due to the ambiguous default argument.
-    dr1_f();
+    cwg1_f();
   }
   namespace X {
-    using Y::dr1_f;
+    using Y::cwg1_f;
     void h() {
-      dr1_f(0);
+      cwg1_f(0);
       // FIXME: This should be rejected, due to the ambiguous default argument.
-      dr1_f();
+      cwg1_f();
     }
   }
 
   namespace X {
     void z(int);
   }
-  void X::z(int = 1) {} // #dr1-z
+  void X::z(int = 1) {} // #cwg1-z
   namespace X {
     void z(int = 1);
     // expected-error@-1 {{redefinition of default argument}}
-    //   expected-note@#dr1-z {{previous definition is here}}
+    //   expected-note@#cwg1-z {{previous definition is here}}
   }
 
   void i(int = 1);
   void j() {
     void i(int = 1);
-    using dr1::i;
+    using cwg1::i;
     i(0);
     // FIXME: This should be rejected, due to the ambiguous default argument.
     i();
   }
   void k() {
-    using dr1::i;
+    using cwg1::i;
     void i(int = 1);
     i(0);
     // FIXME: This should be rejected, due to the ambiguous default argument.
@@ -50,27 +50,27 @@ namespace dr1 { // dr1: no
   }
 }
 
-namespace dr3 { // dr3: yes
+namespace cwg3 { // cwg3: yes
   template<typename T> struct A {};
-  template<typename T> void f(T) { A<T> a; } // #dr3-f-T
+  template<typename T> void f(T) { A<T> a; } // #cwg3-f-T
   template void f(int);
   template<> struct A<int> {};
-  // expected-error@-1 {{explicit specialization of 'dr3::A<int>' after instantiation}}
-  //   expected-note@#dr3-f-T {{implicit instantiation first required here}}
+  // expected-error@-1 {{explicit specialization of 'cwg3::A<int>' after instantiation}}
+  //   expected-note@#cwg3-f-T {{implicit instantiation first required here}}
 }
 
-namespace dr4 { // dr4: 2.8
+namespace cwg4 { // cwg4: 2.8
   extern "C" {
-    static void dr4_f(int) {}
-    static void dr4_f(float) {}
-    void dr4_g(int) {} // #dr4-g-int
-    void dr4_g(float) {}
-    // expected-error@-1 {{conflicting types for 'dr4_g'}}
-    //   expected-note@#dr4-g-int {{previous definition is here}}
+    static void cwg4_f(int) {}
+    static void cwg4_f(float) {}
+    void cwg4_g(int) {} // #cwg4-g-int
+    void cwg4_g(float) {}
+    // expected-error@-1 {{conflicting types for 'cwg4_g'}}
+    //   expected-note@#cwg4-g-int {{previous definition is here}}
   }
 }
 
-namespace dr5 { // dr5: 3.1
+namespace cwg5 { // cwg5: 3.1
   struct A {} a;
   struct B {
     B(const A&);
@@ -84,24 +84,24 @@ namespace dr5 { // dr5: 3.1
   const C c = e;
 }
 
-namespace dr7 { // dr7: 3.4
+namespace cwg7 { // cwg7: 3.4
   class A { public: ~A(); };
-  class B : virtual private A {}; // #dr7-B
-  class C : public B {} c; // #dr7-C
-  // expected-error@#dr7-C {{inherited virtual base class 'A' has private destructor}}
-  //   expected-note@#dr7-C {{in implicit default constructor for 'dr7::C' first required here}}
-  //   expected-note@#dr7-B {{declared private here}}
-  // expected-error@#dr7-C {{inherited virtual base class 'A' has private destructor}}
-  //   expected-note@#dr7-C {{in implicit destructor for 'dr7::C' first required here}}
-  //   expected-note@#dr7-B {{declared private here}}
+  class B : virtual private A {}; // #cwg7-B
+  class C : public B {} c; // #cwg7-C
+  // expected-error@#cwg7-C {{inherited virtual base class 'A' has private destructor}}
+  //   expected-note@#cwg7-C {{in implicit default constructor for 'cwg7::C' first required here}}
+  //   expected-note@#cwg7-B {{declared private here}}
+  // expected-error@#cwg7-C {{inherited virtual base class 'A' has private destructor}}
+  //   expected-note@#cwg7-C {{in implicit destructor for 'cwg7::C' first required here}}
+  //   expected-note@#cwg7-B {{declared private here}}
   class VeryDerivedC : public B, virtual public A {} vdc;
 
-  class X { ~X(); }; // #dr7-X
+  class X { ~X(); }; // #cwg7-X
   class Y : X { ~Y() {} };
   // expected-error@-1 {{base class 'X' has private destructor}}
-  //   expected-note@#dr7-X {{implicitly declared private here}}
+  //   expected-note@#cwg7-X {{implicitly declared private here}}
 
-  namespace PR16370 { // This regressed the first time DR7 was fixed.
+  namespace PR16370 { // This regressed the first time CWG7 was fixed.
     struct S1 { virtual ~S1(); };
     struct S2 : S1 {};
     struct S3 : S2 {};
@@ -114,7 +114,7 @@ namespace dr7 { // dr7: 3.4
   }
 }
 
-namespace dr8 { // dr8: dup 45
+namespace cwg8 { // cwg8: dup 45
   class A {
     struct U;
     static const int k = 5;
@@ -126,23 +126,23 @@ namespace dr8 { // dr8: dup 45
   A::T<A::U, A::k, &A::f> *A::g() { return 0; }
 }
 
-namespace dr9 { // dr9: 2.8
+namespace cwg9 { // cwg9: 2.8
   struct B {
   protected:
-    int m; // #dr9-m
+    int m; // #cwg9-m
     friend int R1();
   };
-  struct N : protected B { // #dr9-N
+  struct N : protected B { // #cwg9-N
     friend int R2();
   } n;
   int R1() { return n.m; }
-  // expected-error@-1 {{'m' is a protected member of 'dr9::B'}}
-  //   expected-note@#dr9-N {{constrained by protected inheritance here}}
-  //   expected-note@#dr9-m {{member is declared here}}
+  // expected-error@-1 {{'m' is a protected member of 'cwg9::B'}}
+  //   expected-note@#cwg9-N {{constrained by protected inheritance here}}
+  //   expected-note@#cwg9-m {{member is declared here}}
   int R2() { return n.m; }
 }
 
-namespace dr10 { // dr10: dup 45
+namespace cwg10 { // cwg10: dup 45
   class A {
     struct B {
       A::B *p;
@@ -150,7 +150,7 @@ namespace dr10 { // dr10: dup 45
   };
 }
 
-namespace dr11 { // dr11: yes
+namespace cwg11 { // cwg11: yes
   template<typename T> struct A : T {
     using typename T::U;
     U u;
@@ -164,19 +164,19 @@ namespace dr11 { // dr11: yes
   A<X> ax;
 }
 
-namespace dr12 { // dr12: sup 239
+namespace cwg12 { // cwg12: sup 239
   enum E { e };
   E &f(E, E = e);
   void g() {
     int &f(int, E = e);
-    // Under DR12, these call two different functions.
-    // Under DR239, they call the same function.
+    // Under CWG12, these call two different functions.
+    // Under CWG239, they call the same function.
     int &b = f(e);
     int &c = f(1);
   }
 }
 
-namespace dr13 { // dr13: no
+namespace cwg13 { // cwg13: no
   extern "C" void f(int);
   void g(char);
 
@@ -191,60 +191,60 @@ namespace dr13 { // dr13: no
   int a4 = h(g);
 }
 
-namespace dr14 { // dr14: 3.4
-  namespace X { extern "C" int dr14_f(); }
-  namespace Y { extern "C" int dr14_f(); }
+namespace cwg14 { // cwg14: 3.4
+  namespace X { extern "C" int cwg14_f(); }
+  namespace Y { extern "C" int cwg14_f(); }
   using namespace X;
   using namespace Y;
-  int k = dr14_f();
+  int k = cwg14_f();
 
   class C {
     int k;
-    friend int Y::dr14_f();
+    friend int Y::cwg14_f();
   } c;
   namespace Z {
-    extern "C" int dr14_f() { return c.k; }
+    extern "C" int cwg14_f() { return c.k; }
   }
 
-  namespace X { typedef int T; typedef int U; } // #dr14-X-U
-  namespace Y { typedef int T; typedef long U; } // #dr14-Y-U
+  namespace X { typedef int T; typedef int U; } // #cwg14-X-U
+  namespace Y { typedef int T; typedef long U; } // #cwg14-Y-U
   T t; // ok, same type both times
   U u;
   // expected-error@-1 {{reference to 'U' is ambiguous}}
-  //   expected-note@#dr14-X-U {{candidate found by name lookup is 'dr14::X::U'}}
-  //   expected-note@#dr14-Y-U {{candidate found by name lookup is 'dr14::Y::U'}}
+  //   expected-note@#cwg14-X-U {{candidate found by name lookup is 'cwg14::X::U'}}
+  //   expected-note@#cwg14-Y-U {{candidate found by name lookup is 'cwg14::Y::U'}}
 }
 
-namespace dr15 { // dr15: yes
-  template<typename T> void f(int); // #dr15-f-decl-first
+namespace cwg15 { // cwg15: yes
+  template<typename T> void f(int); // #cwg15-f-decl-first
   template<typename T> void f(int = 0);
   // expected-error@-1 {{default arguments cannot be added to a function template that has already been declared}}
-  //   expected-note@#dr15-f-decl-first {{previous template declaration is here}}
+  //   expected-note@#cwg15-f-decl-first {{previous template declaration is here}}
 }
 
-namespace dr16 { // dr16: 2.8
-  class A { // #dr16-A
-    void f(); // #dr16-A-f-decl
+namespace cwg16 { // cwg16: 2.8
+  class A { // #cwg16-A
+    void f(); // #cwg16-A-f-decl
     friend class C;
   };
-  class B : A {}; // #dr16-B
+  class B : A {}; // #cwg16-B
   class C : B {
     void g() {
       f();
-      // expected-error@-1 {{'f' is a private member of 'dr16::A'}}
-      //   expected-note@#dr16-B {{constrained by implicitly private inheritance here}}
-      //   expected-note@#dr16-A-f-decl {{member is declared here}}
-      A::f(); // #dr16-A-f-call
-      // expected-error@#dr16-A-f-call {{'A' is a private member of 'dr16::A'}}
-      //   expected-note@#dr16-B {{constrained by implicitly private inheritance here}}
-      //   expected-note@#dr16-A {{member is declared here}}
-      // expected-error@#dr16-A-f-call {{cannot cast 'dr16::C' to its private base class 'dr16::A'}}
-      //   expected-note@#dr16-B {{implicitly declared private here}}
+      // expected-error@-1 {{'f' is a private member of 'cwg16::A'}}
+      //   expected-note@#cwg16-B {{constrained by implicitly private inheritance here}}
+      //   expected-note@#cwg16-A-f-decl {{member is declared here}}
+      A::f(); // #cwg16-A-f-call
+      // expected-error@#cwg16-A-f-call {{'A' is a private member of 'cwg16::A'}}
+      //   expected-note@#cwg16-B {{constrained by implicitly private inheritance here}}
+      //   expected-note@#cwg16-A {{member is declared here}}
+      // expected-error@#cwg16-A-f-call {{cannot cast 'cwg16::C' to its private base class 'cwg16::A'}}
+      //   expected-note@#cwg16-B {{implicitly declared private here}}
     }
   };
 }
 
-namespace dr17 { // dr17: yes
+namespace cwg17 { // cwg17: yes
   class A {
     int n;
     int f();
@@ -257,38 +257,38 @@ namespace dr17 { // dr17: yes
   };
 }
 
-// dr18: sup 577
+// cwg18: sup 577
 
-namespace dr19 { // dr19: 3.1
+namespace cwg19 { // cwg19: 3.1
   struct A {
-    int n; // #dr19-n
+    int n; // #cwg19-n
   };
-  struct B : protected A { // #dr19-B
+  struct B : protected A { // #cwg19-B
   };
   struct C : B {} c;
   struct D : B {
     int get1() { return c.n; }
-    // expected-error@-1 {{'n' is a protected member of 'dr19::A'}}
-    //   expected-note@#dr19-B {{constrained by protected inheritance here}}
-    //   expected-note@#dr19-n {{member is declared here}}
+    // expected-error@-1 {{'n' is a protected member of 'cwg19::A'}}
+    //   expected-note@#cwg19-B {{constrained by protected inheritance here}}
+    //   expected-note@#cwg19-n {{member is declared here}}
     int get2() { return ((A&)c).n; } // ok, A is an accessible base of B from here
   };
 }
 
-namespace dr20 { // dr20: 2.8
+namespace cwg20 { // cwg20: 2.8
   class X {
   public:
     X();
   private:
-    X(const X&); // #dr20-X-ctor
+    X(const X&); // #cwg20-X-ctor
   };
   X &f();
   X x = f();
-  // expected-error@-1 {{calling a private constructor of class 'dr20::X'}}
-  //   expected-note@#dr20-X-ctor {{declared private here}}
+  // expected-error@-1 {{calling a private constructor of class 'cwg20::X'}}
+  //   expected-note@#cwg20-X-ctor {{declared private here}}
 }
 
-namespace dr21 { // dr21: 3.4
+namespace cwg21 { // cwg21: 3.4
   template<typename T> struct A;
   struct X {
     template<typename T = int> friend struct A;
@@ -298,25 +298,25 @@ namespace dr21 { // dr21: 3.4
   };
 }
 
-namespace dr22 { // dr22: sup 481
-  template<typename dr22_T = dr22_T> struct X;
-  // expected-error@-1 {{unknown type name 'dr22_T'}}
+namespace cwg22 { // cwg22: sup 481
+  template<typename cwg22_T = cwg22_T> struct X;
+  // expected-error@-1 {{unknown type name 'cwg22_T'}}
   typedef int T;
   template<typename T = T> struct Y;
 }
 
-namespace dr23 { // dr23: yes
-  template<typename T> void f(T, T); // #dr23-f-T-T
-  template<typename T> void f(T, int); // #dr23-f-T-int
+namespace cwg23 { // cwg23: yes
+  template<typename T> void f(T, T); // #cwg23-f-T-T
+  template<typename T> void f(T, int); // #cwg23-f-T-int
   void g() { f(0, 0); }
   // expected-error@-1 {{call to 'f' is ambiguous}}
-  //   expected-note@#dr23-f-T-T {{candidate function [with T = int]}}
-  //   expected-note@#dr23-f-T-int {{candidate function [with T = int]}}
+  //   expected-note@#cwg23-f-T-T {{candidate function [with T = int]}}
+  //   expected-note@#cwg23-f-T-int {{candidate function [with T = int]}}
 }
 
-// dr24: na
+// cwg24: na
 
-namespace dr25 { // dr25: yes
+namespace cwg25 { // cwg25: yes
   struct A {
     void f() throw(int);
     // since-cxx17-error@-1 {{ISO C++17 does not allow dynamic exception specifications}}
@@ -351,7 +351,7 @@ namespace dr25 { // dr25: yes
   }
 }
 
-namespace dr26 { // dr26: yes
+namespace cwg26 { // cwg26: yes
   struct A { A(A, const A & = A()); };
   // expected-error@-1 {{copy constructor must pass its first argument by reference}}
   struct B {
@@ -371,77 +371,77 @@ namespace dr26 { // dr26: yes
   };
 }
 
-namespace dr27 { // dr27: yes
+namespace cwg27 { // cwg27: yes
   enum E { e } n;
   E &m = true ? n : n;
 }
 
-// dr28: na lib
+// cwg28: na lib
 
-namespace dr29 { // dr29: 3.4
-  void dr29_f0(); // #dr29-f0
-  void g0() { void dr29_f0(); }
-  extern "C++" void g0_cxx() { void dr29_f0(); }
-  extern "C" void g0_c() { void dr29_f0(); }
-  // expected-error@-1 {{declaration of 'dr29_f0' has a different language linkage}}
-  //   expected-note@#dr29-f0 {{previous declaration is here}}
+namespace cwg29 { // cwg29: 3.4
+  void cwg29_f0(); // #cwg29-f0
+  void g0() { void cwg29_f0(); }
+  extern "C++" void g0_cxx() { void cwg29_f0(); }
+  extern "C" void g0_c() { void cwg29_f0(); }
+  // expected-error@-1 {{declaration of 'cwg29_f0' has a different language linkage}}
+  //   expected-note@#cwg29-f0 {{previous declaration is here}}
 
-  extern "C" void dr29_f1(); // #dr29-f1
-  void g1() { void dr29_f1(); }
-  extern "C" void g1_c() { void dr29_f1(); }
-  extern "C++" void g1_cxx() { void dr29_f1(); }
-  // expected-error@-1 {{declaration of 'dr29_f1' has a different language linkage}}
-  //   expected-note@#dr29-f1 {{previous declaration is here}}
+  extern "C" void cwg29_f1(); // #cwg29-f1
+  void g1() { void cwg29_f1(); }
+  extern "C" void g1_c() { void cwg29_f1(); }
+  extern "C++" void g1_cxx() { void cwg29_f1(); }
+  // expected-error@-1 {{declaration of 'cwg29_f1' has a different language linkage}}
+  //   expected-note@#cwg29-f1 {{previous declaration is here}}
 
-  void g2() { void dr29_f2(); } // #dr29-f2
-  extern "C" void dr29_f2();
-  // expected-error@-1 {{declaration of 'dr29_f2' has a different language linkage}}
-  //   expected-note@#dr29-f2 {{previous declaration is here}}
+  void g2() { void cwg29_f2(); } // #cwg29-f2
+  extern "C" void cwg29_f2();
+  // expected-error@-1 {{declaration of 'cwg29_f2' has a different language linkage}}
+  //   expected-note@#cwg29-f2 {{previous declaration is here}}
 
-  extern "C" void g3() { void dr29_f3(); } // #dr29-f3
-  extern "C++" void dr29_f3();
-  // expected-error@-1 {{declaration of 'dr29_f3' has a different language linkage}}
-  //   expected-note@#dr29-f3 {{previous declaration is here}}
+  extern "C" void g3() { void cwg29_f3(); } // #cwg29-f3
+  extern "C++" void cwg29_f3();
+  // expected-error@-1 {{declaration of 'cwg29_f3' has a different language linkage}}
+  //   expected-note@#cwg29-f3 {{previous declaration is here}}
 
-  extern "C++" void g4() { void dr29_f4(); } // #dr29-f4
-  extern "C" void dr29_f4();
-  // expected-error@-1 {{declaration of 'dr29_f4' has a different language linkage}}
-  //   expected-note@#dr29-f4 {{previous declaration is here}}
+  extern "C++" void g4() { void cwg29_f4(); } // #cwg29-f4
+  extern "C" void cwg29_f4();
+  // expected-error@-1 {{declaration of 'cwg29_f4' has a different language linkage}}
+  //   expected-note@#cwg29-f4 {{previous declaration is here}}
 
   extern "C" void g5();
-  extern "C++" void dr29_f5();
+  extern "C++" void cwg29_f5();
   void g5() {
-    void dr29_f5(); // ok, g5 is extern "C" but we're not inside the linkage-specification here.
+    void cwg29_f5(); // ok, g5 is extern "C" but we're not inside the linkage-specification here.
   }
 
   extern "C++" void g6();
-  extern "C" void dr29_f6();
+  extern "C" void cwg29_f6();
   void g6() {
-    void dr29_f6(); // ok, g6 is extern "C" but we're not inside the linkage-specification here.
+    void cwg29_f6(); // ok, g6 is extern "C" but we're not inside the linkage-specification here.
   }
 
   extern "C" void g7();
-  extern "C++" void dr29_f7(); // #dr29-f7
+  extern "C++" void cwg29_f7(); // #cwg29-f7
   extern "C" void g7() {
-    void dr29_f7();
-    // expected-error@-1 {{declaration of 'dr29_f7' has a different language linkage}}
-    //   expected-note@#dr29-f7 {{previous declaration is here}}
+    void cwg29_f7();
+    // expected-error@-1 {{declaration of 'cwg29_f7' has a different language linkage}}
+    //   expected-note@#cwg29-f7 {{previous declaration is here}}
   }
 
   extern "C++" void g8();
-  extern "C" void dr29_f8(); // #dr29-f8
+  extern "C" void cwg29_f8(); // #cwg29-f8
   extern "C++" void g8() {
-    void dr29_f8();
-    // expected-error@-1 {{declaration of 'dr29_f8' has a different language linkage}}
-    //   expected-note@#dr29-f8 {{previous declaration is here}}
+    void cwg29_f8();
+    // expected-error@-1 {{declaration of 'cwg29_f8' has a different language linkage}}
+    //   expected-note@#cwg29-f8 {{previous declaration is here}}
   }
 }
 
-namespace dr30 { // dr30: sup 468 c++11
+namespace cwg30 { // cwg30: sup 468 c++11
   struct A {
     template<int> static int f();
   } a, *p = &a;
-  // FIXME: It's not clear whether DR468 applies to C++98 too.
+  // FIXME: It's not clear whether CWG468 applies to C++98 too.
   int x = A::template f<0>();
   // cxx98-error@-1 {{'template' keyword outside of a template}}
   int y = a.template f<0>();
@@ -450,29 +450,29 @@ namespace dr30 { // dr30: sup 468 c++11
   // cxx98-error@-1 {{'template' keyword outside of a template}}
 }
 
-namespace dr31 { // dr31: 2.8
+namespace cwg31 { // cwg31: 2.8
   class X {
   private:
-    void operator delete(void*); // #dr31-delete
+    void operator delete(void*); // #cwg31-delete
   };
   // We would call X::operator delete if X() threw (even though it can't,
   // and even though we allocated the X using ::operator delete).
   X *p = new X;
-  // expected-error@-1 {{'operator delete' is a private member of 'dr31::X'}}
-  //   expected-note@#dr31-delete {{declared private here}}
+  // expected-error@-1 {{'operator delete' is a private member of 'cwg31::X'}}
+  //   expected-note@#cwg31-delete {{declared private here}}
 }
 
-// dr32: na
+// cwg32: na
 
-namespace dr33 { // dr33: 9
-  namespace X { struct S; void f(void (*)(S)); } // #dr33-f-S
-  namespace Y { struct T; void f(void (*)(T)); } // #dr33-f-T
+namespace cwg33 { // cwg33: 9
+  namespace X { struct S; void f(void (*)(S)); } // #cwg33-f-S
+  namespace Y { struct T; void f(void (*)(T)); } // #cwg33-f-T
   void g(X::S);
   template<typename Z> Z g(Y::T);
   void h() { f(&g); }
   // expected-error@-1 {{call to 'f' is ambiguous}}
-  //   expected-note@#dr33-f-S {{candidate function}}
-  //   expected-note@#dr33-f-T {{candidate function}}
+  //   expected-note@#cwg33-f-S {{candidate function}}
+  //   expected-note@#cwg33-f-T {{candidate function}}
 
   template<typename T> void t(X::S);
   template<typename T, typename U = void> void u(X::S);
@@ -507,10 +507,10 @@ namespace dr33 { // dr33: 9
   }
 }
 
-// dr34: na
-// dr35: dup 178
+// cwg34: na
+// cwg35: dup 178
 
-namespace dr36 { // dr36: 2.8
+namespace cwg36 { // cwg36: 2.8
 namespace example1 {
   namespace A {
     int i;
@@ -540,25 +540,25 @@ namespace example2 {
 
   struct D : virtual B, virtual C
   {
-    using B::i; // #dr36-ex2-B-i-first
+    using B::i; // #cwg36-ex2-B-i-first
     using B::i;
     // expected-error@-1 {{redeclaration of using declaration}}
-    //   expected-note@#dr36-ex2-B-i-first {{previous using declaration}}
+    //   expected-note@#cwg36-ex2-B-i-first {{previous using declaration}}
 
-    using C::i; // #dr36-ex2-C-i-first
+    using C::i; // #cwg36-ex2-C-i-first
     using C::i;
     // expected-error@-1 {{redeclaration of using declaration}}
-    //   expected-note@#dr36-ex2-C-i-first {{previous using declaration}}
+    //   expected-note@#cwg36-ex2-C-i-first {{previous using declaration}}
 
-    using B::j; // #dr36-ex2-B-j-first
+    using B::j; // #cwg36-ex2-B-j-first
     using B::j;
     // expected-error@-1 {{redeclaration of using declaration}}
-    //   expected-note@#dr36-ex2-B-j-first {{previous using declaration}}
+    //   expected-note@#cwg36-ex2-B-j-first {{previous using declaration}}
 
-    using C::j; // #dr36-ex2-C-j-first
+    using C::j; // #cwg36-ex2-C-j-first
     using C::j;
     // expected-error@-1 {{redeclaration of using declaration}}
-    //   expected-note@#dr36-ex2-C-j-first {{previous using declaration}}
+    //   expected-note@#cwg36-ex2-C-j-first {{previous using declaration}}
   };
 }
 
@@ -578,25 +578,25 @@ namespace example3 {
   template<typename T>
   struct D : virtual B<T>, virtual C<T>
   {
-    using B<T>::i; // #dr36-ex3-B-i-first
+    using B<T>::i; // #cwg36-ex3-B-i-first
     using B<T>::i;
     // expected-error@-1 {{redeclaration of using declaration}}
-    //   expected-note@#dr36-ex3-B-i-first {{previous using declaration}}
+    //   expected-note@#cwg36-ex3-B-i-first {{previous using declaration}}
 
-    using C<T>::i; // #dr36-ex3-C-i-first
+    using C<T>::i; // #cwg36-ex3-C-i-first
     using C<T>::i;
     // expected-error@-1 {{redeclaration of using declaration}}
-    //   expected-note@#dr36-ex3-C-i-first {{previous using declaration}}
+    //   expected-note@#cwg36-ex3-C-i-first {{previous using declaration}}
 
-    using B<T>::j; // #dr36-ex3-B-j-first
+    using B<T>::j; // #cwg36-ex3-B-j-first
     using B<T>::j;
     // expected-error@-1 {{redeclaration of using declaration}}
-    //   expected-note@#dr36-ex3-B-j-first {{previous using declaration}}
+    //   expected-note@#cwg36-ex3-B-j-first {{previous using declaration}}
 
-    using C<T>::j; // #dr36-ex3-C-j-first
+    using C<T>::j; // #cwg36-ex3-C-j-first
     using C<T>::j;
     // expected-error@-1 {{redeclaration of using declaration}}
-    //   expected-note@#dr36-ex3-C-j-first {{previous using declaration}}
+    //   expected-note@#cwg36-ex3-C-j-first {{previous using declaration}}
   };
 }
 namespace example4 {
@@ -607,23 +607,23 @@ namespace example4 {
 
   template<typename T>
   struct G : E<T> {
-    using E<T>::k; // #dr36-E-k-first
+    using E<T>::k; // #cwg36-E-k-first
     using E<T>::k;
     // expected-error@-1 {{redeclaration of using declaration}}
-    //   expected-note@#dr36-E-k-first {{previous using declaration}}
+    //   expected-note@#cwg36-E-k-first {{previous using declaration}}
   };
 }
 }
 
-// dr37: sup 475
+// cwg37: sup 475
 
-namespace dr38 { // dr38: yes
+namespace cwg38 { // cwg38: yes
   template<typename T> struct X {};
   template<typename T> X<T> operator+(X<T> a, X<T> b) { return a; }
   template X<int> operator+<int>(X<int>, X<int>);
 }
 
-namespace dr39 { // dr39: no
+namespace cwg39 { // cwg39: no
   namespace example1 {
     struct A { int &f(int); };
     struct B : A {
@@ -635,16 +635,16 @@ namespace dr39 { // dr39: no
 
   namespace example2 {
     struct A {
-      int &x(int); // #dr39-A-x-decl
-      static int &y(int); // #dr39-A-y-decl
+      int &x(int); // #cwg39-A-x-decl
+      static int &y(int); // #cwg39-A-y-decl
     };
     struct V {
       int &z(int);
     };
     struct B : A, virtual V {
-      using A::x; // #dr39-using-A-x
+      using A::x; // #cwg39-using-A-x
       float &x(float);
-      using A::y; // #dr39-using-A-y
+      using A::y; // #cwg39-using-A-y
       static float &y(float);
       using V::z;
       float &z(float);
@@ -652,18 +652,18 @@ namespace dr39 { // dr39: no
     struct C : A, B, virtual V {} c;
     /* expected-warning@-1
     {{direct base 'A' is inaccessible due to ambiguity:
-    struct dr39::example2::C -> A
-    struct dr39::example2::C -> B -> A}} */
+    struct cwg39::example2::C -> A
+    struct cwg39::example2::C -> B -> A}} */
     int &x = c.x(0);
     // expected-error@-1 {{member 'x' found in multiple base classes of different types}}
-    //   expected-note@#dr39-A-x-decl {{member found by ambiguous name lookup}}
-    //   expected-note@#dr39-using-A-x {{member found by ambiguous name lookup}}
+    //   expected-note@#cwg39-A-x-decl {{member found by ambiguous name lookup}}
+    //   expected-note@#cwg39-using-A-x {{member found by ambiguous name lookup}}
 
     // FIXME: This is valid, because we find the same static data member either way.
     int &y = c.y(0);
     // expected-error@-1 {{member 'y' found in multiple base classes of different types}}
-    //   expected-note@#dr39-A-y-decl {{member found by ambiguous name lookup}}
-    //   expected-note@#dr39-using-A-y {{member found by ambiguous name lookup}}
+    //   expected-note@#cwg39-A-y-decl {{member found by ambiguous name lookup}}
+    //   expected-note@#cwg39-using-A-y {{member found by ambiguous name lookup}}
     int &z = c.z(0);
   }
 
@@ -676,63 +676,63 @@ namespace dr39 { // dr39: no
   }
 
   namespace example4 {
-    struct A { int n; }; // #dr39-ex4-A-n
+    struct A { int n; }; // #cwg39-ex4-A-n
     struct B : A {};
     struct C : A {};
     struct D : B, C { int f() { return n; } };
     /* expected-error@-1
     {{non-static member 'n' found in multiple base-class subobjects of type 'A':
-    struct dr39::example4::D -> B -> A
-    struct dr39::example4::D -> C -> A}} */
-    //   expected-note@#dr39-ex4-A-n {{member found by ambiguous name lookup}}
+    struct cwg39::example4::D -> B -> A
+    struct cwg39::example4::D -> C -> A}} */
+    //   expected-note@#cwg39-ex4-A-n {{member found by ambiguous name lookup}}
   }
 
   namespace PR5916 {
     // FIXME: This is valid.
-    struct A { int n; }; // #dr39-A-n
+    struct A { int n; }; // #cwg39-A-n
     struct B : A {};
     struct C : A {};
     struct D : B, C {};
-    int k = sizeof(D::n); // #dr39-sizeof
-    /* expected-error@#dr39-sizeof
+    int k = sizeof(D::n); // #cwg39-sizeof
+    /* expected-error@#cwg39-sizeof
     {{non-static member 'n' found in multiple base-class subobjects of type 'A':
-    struct dr39::PR5916::D -> B -> A
-    struct dr39::PR5916::D -> C -> A}} */
-    //   expected-note@#dr39-A-n {{member found by ambiguous name lookup}}
+    struct cwg39::PR5916::D -> B -> A
+    struct cwg39::PR5916::D -> C -> A}} */
+    //   expected-note@#cwg39-A-n {{member found by ambiguous name lookup}}
 
-    // expected-error@#dr39-sizeof {{unknown type name}}
+    // expected-error@#cwg39-sizeof {{unknown type name}}
 #if __cplusplus >= 201103L
     decltype(D::n) n;
     /* expected-error@-1
     {{non-static member 'n' found in multiple base-class subobjects of type 'A':
-    struct dr39::PR5916::D -> B -> A
-    struct dr39::PR5916::D -> C -> A}} */
-    //   expected-note@#dr39-A-n {{member found by ambiguous name lookup}}
+    struct cwg39::PR5916::D -> B -> A
+    struct cwg39::PR5916::D -> C -> A}} */
+    //   expected-note@#cwg39-A-n {{member found by ambiguous name lookup}}
 #endif
   }
 }
 
-// dr40: na
+// cwg40: na
 
-namespace dr41 { // dr41: yes
+namespace cwg41 { // cwg41: yes
   struct S f(S);
 }
 
-namespace dr42 { // dr42: yes
+namespace cwg42 { // cwg42: yes
   struct A { static const int k = 0; };
   struct B : A { static const int k = A::k; };
 }
 
-// dr43: na
+// cwg43: na
 
-namespace dr44 { // dr44: sup 727
+namespace cwg44 { // cwg44: sup 727
   struct A {
     template<int> void f();
     template<> void f<0>();
   };
 }
 
-namespace dr45 { // dr45: yes
+namespace cwg45 { // cwg45: yes
   class A {
     class B {};
     class C : B {};
@@ -740,27 +740,27 @@ namespace dr45 { // dr45: yes
   };
 }
 
-namespace dr46 { // dr46: yes
+namespace cwg46 { // cwg46: yes
   template<typename> struct A { template<typename> struct B {}; };
   template template struct A<int>::B<int>;
   // expected-error@-1 {{expected unqualified-id}}
 }
 
-namespace dr47 { // dr47: sup 329
+namespace cwg47 { // cwg47: sup 329
   template<typename T> struct A {
-    friend void f() { T t; } // #dr47-f
+    friend void f() { T t; } // #cwg47-f
     // expected-error@-1 {{redefinition of 'f'}}
-    //   expected-note@#dr47-b {{in instantiation of template class 'dr47::A<float>' requested here}}
-    //   expected-note@#dr47-f {{previous definition is here}}
+    //   expected-note@#cwg47-b {{in instantiation of template class 'cwg47::A<float>' requested here}}
+    //   expected-note@#cwg47-f {{previous definition is here}}
   };
   A<int> a;
-  A<float> b; // #dr47-b
+  A<float> b; // #cwg47-b
 
   void f();
   void g() { f(); }
 }
 
-namespace dr48 { // dr48: yes
+namespace cwg48 { // cwg48: yes
   namespace {
     struct S {
       static const int m = 0;
@@ -776,45 +776,45 @@ namespace dr48 { // dr48: yes
   const int &c = S::o;
 }
 
-namespace dr49 { // dr49: 2.8
-  template<int*> struct A {}; // #dr49-A
+namespace cwg49 { // cwg49: 2.8
+  template<int*> struct A {}; // #cwg49-A
   int k;
 #if __has_feature(cxx_constexpr)
   constexpr
 #endif
-  int *const p = &k; // #dr49-p
+  int *const p = &k; // #cwg49-p
   A<&k> a;
-  A<p> b; // #dr49-b
-  // cxx98-error@#dr49-b {{non-type template argument referring to object 'p' with internal linkage is a C++11 extension}}
-  //   cxx98-note@#dr49-p {{non-type template argument refers to object here}}
-  // cxx98-14-error@#dr49-b {{non-type template argument for template parameter of pointer type 'int *' must have its address taken}}
-  //   cxx98-14-note@#dr49-A {{template parameter is declared here}}
-  int *q = &k; // #dr49-q
-  A<q> c; // #dr49-c
-  // cxx98-error@#dr49-c {{non-type template argument for template parameter of pointer type 'int *' must have its address taken}}
-  //   cxx98-note@#dr49-A {{template parameter is declared here}}
-  // cxx11-14-error@#dr49-c {{non-type template argument of type 'int *' is not a constant expression}}
-  //   cxx11-14-note@#dr49-c {{read of non-constexpr variable 'q' is not allowed in a constant expression}}
-  //   cxx11-14-note@#dr49-q {{declared here}}
-  //   cxx11-14-note@#dr49-A {{template parameter is declared here}}
-  // since-cxx17-error@#dr49-c {{non-type template argument is not a constant expression}}
-  //   since-cxx17-note@#dr49-c {{read of non-constexpr variable 'q' is not allowed in a constant expression}}
-  //   since-cxx17-note@#dr49-q {{declared here}}
-}
-
-namespace dr50 { // dr50: yes
-  struct X; // #dr50-X
+  A<p> b; // #cwg49-b
+  // cxx98-error@#cwg49-b {{non-type template argument referring to object 'p' with internal linkage is a C++11 extension}}
+  //   cxx98-note@#cwg49-p {{non-type template argument refers to object here}}
+  // cxx98-14-error@#cwg49-b {{non-type template argument for template parameter of pointer type 'int *' must have its address taken}}
+  //   cxx98-14-note@#cwg49-A {{template parameter is declared here}}
+  int *q = &k; // #cwg49-q
+  A<q> c; // #cwg49-c
+  // cxx98-error@#cwg49-c {{non-type template argument for template parameter of pointer type 'int *' must have its address taken}}
+  //   cxx98-note@#cwg49-A {{template parameter is declared here}}
+  // cxx11-14-error@#cwg49-c {{non-type template argument of type 'int *' is not a constant expression}}
+  //   cxx11-14-note@#cwg49-c {{read of non-constexpr variable 'q' is not allowed in a constant expression}}
+  //   cxx11-14-note@#cwg49-q {{declared here}}
+  //   cxx11-14-note@#cwg49-A {{template parameter is declared here}}
+  // since-cxx17-error@#cwg49-c {{non-type template argument is not a constant expression}}
+  //   since-cxx17-note@#cwg49-c {{read of non-constexpr variable 'q' is not allowed in a constant expression}}
+  //   since-cxx17-note@#cwg49-q {{declared here}}
+}
+
+namespace cwg50 { // cwg50: yes
+  struct X; // #cwg50-X
   extern X *p;
   X *q = (X*)p;
   X *r = static_cast<X*>(p);
   X *s = const_cast<X*>(p);
   X *t = reinterpret_cast<X*>(p);
   X *u = dynamic_cast<X*>(p);
-  // expected-error@-1 {{'dr50::X' is an incomplete type}}
-  //   expected-note@#dr50-X {{forward declaration of 'dr50::X'}}
+  // expected-error@-1 {{'cwg50::X' is an incomplete type}}
+  //   expected-note@#cwg50-X {{forward declaration of 'cwg50::X'}}
 }
 
-namespace dr51 { // dr51: 2.8
+namespace cwg51 { // cwg51: 2.8
   struct A {};
   struct B : A {};
   struct S {
@@ -824,57 +824,57 @@ namespace dr51 { // dr51: 2.8
   A &a = s;
 }
 
-namespace dr52 { // dr52: 2.8
-  struct A { int n; }; // #dr52-A
-  struct B : private A {} b; // #dr52-B
-  int k = b.A::n; // #dr52-k
+namespace cwg52 { // cwg52: 2.8
+  struct A { int n; }; // #cwg52-A
+  struct B : private A {} b; // #cwg52-B
+  int k = b.A::n; // #cwg52-k
   // FIXME: This first diagnostic is very strangely worded, and seems to be bogus.
-  // expected-error@#dr52-k {{'A' is a private member of 'dr52::A'}}
-  //   expected-note@#dr52-B {{constrained by private inheritance here}}
-  //   expected-note@#dr52-A {{member is declared here}}
-  // expected-error@#dr52-k {{cannot cast 'struct B' to its private base class 'dr52::A'}}
-  //   expected-note@#dr52-B {{declared private here}}
+  // expected-error@#cwg52-k {{'A' is a private member of 'cwg52::A'}}
+  //   expected-note@#cwg52-B {{constrained by private inheritance here}}
+  //   expected-note@#cwg52-A {{member is declared here}}
+  // expected-error@#cwg52-k {{cannot cast 'struct B' to its private base class 'cwg52::A'}}
+  //   expected-note@#cwg52-B {{declared private here}}
 }
 
-namespace dr53 { // dr53: yes
+namespace cwg53 { // cwg53: yes
   int n = 0;
   enum E { e } x = static_cast<E>(n);
 }
 
-namespace dr54 { // dr54: 2.8
+namespace cwg54 { // cwg54: 2.8
   struct A { int a; } a;
   struct V { int v; } v;
-  struct B : private A, virtual V { int b; } b; // #dr54-B
+  struct B : private A, virtual V { int b; } b; // #cwg54-B
 
   A &sab = static_cast<A&>(b);
   // expected-error@-1 {{cannot cast 'struct B' to its private base class 'A'}}
-  //   expected-note@#dr54-B {{declared private here}}
+  //   expected-note@#cwg54-B {{declared private here}}
   A *spab = static_cast<A*>(&b);
   // expected-error@-1 {{cannot cast 'struct B' to its private base class 'A'}}
-  //   expected-note@#dr54-B {{declared private here}}
+  //   expected-note@#cwg54-B {{declared private here}}
   int A::*smab = static_cast<int A::*>(&B::b);
-  // expected-error@-1 {{cannot cast 'dr54::B' to its private base class 'dr54::A'}}
-  //   expected-note@#dr54-B {{declared private here}}
+  // expected-error@-1 {{cannot cast 'cwg54::B' to its private base class 'cwg54::A'}}
+  //   expected-note@#cwg54-B {{declared private here}}
   B &sba = static_cast<B&>(a);
-  // expected-error@-1 {{cannot cast private base class 'dr54::A' to 'dr54::B'}}
-  //   expected-note@#dr54-B {{declared private here}}
+  // expected-error@-1 {{cannot cast private base class 'cwg54::A' to 'cwg54::B'}}
+  //   expected-note@#cwg54-B {{declared private here}}
   B *spba = static_cast<B*>(&a);
-  // expected-error@-1 {{cannot cast private base class 'dr54::A' to 'dr54::B'}}
-  //   expected-note@#dr54-B {{declared private here}}
+  // expected-error@-1 {{cannot cast private base class 'cwg54::A' to 'cwg54::B'}}
+  //   expected-note@#cwg54-B {{declared private here}}
   int B::*smba = static_cast<int B::*>(&A::a);
-  // expected-error@-1 {{cannot cast private base class 'dr54::A' to 'dr54::B'}}
-  //   expected-note@#dr54-B {{declared private here}}
+  // expected-error@-1 {{cannot cast private base class 'cwg54::A' to 'cwg54::B'}}
+  //   expected-note@#cwg54-B {{declared private here}}
 
   V &svb = static_cast<V&>(b);
   V *spvb = static_cast<V*>(&b);
   int V::*smvb = static_cast<int V::*>(&B::b);
-  // expected-error@-1 {{conversion from pointer to member of class 'dr54::B' to pointer to member of class 'dr54::V' via virtual base 'dr54::V' is not allowed}}
+  // expected-error@-1 {{conversion from pointer to member of class 'cwg54::B' to pointer to member of class 'cwg54::V' via virtual base 'cwg54::V' is not allowed}}
   B &sbv = static_cast<B&>(v);
-  // expected-error@-1 {{cannot cast 'struct V' to 'B &' via virtual base 'dr54::V'}}
+  // expected-error@-1 {{cannot cast 'struct V' to 'B &' via virtual base 'cwg54::V'}}
   B *spbv = static_cast<B*>(&v);
-  // expected-error@-1 {{cannot cast 'dr54::V *' to 'B *' via virtual base 'dr54::V'}}
+  // expected-error@-1 {{cannot cast 'cwg54::V *' to 'B *' via virtual base 'cwg54::V'}}
   int B::*smbv = static_cast<int B::*>(&V::v);
-  // expected-error@-1 {{conversion from pointer to member of class 'dr54::V' to pointer to member of class 'dr54::B' via virtual base 'dr54::V' is not allowed}}
+  // expected-error@-1 {{conversion from pointer to member of class 'cwg54::V' to pointer to member of class 'cwg54::B' via virtual base 'cwg54::V' is not allowed}}
 
   A &cab = (A&)(b);
   A *cpab = (A*)(&b);
@@ -886,37 +886,37 @@ namespace dr54 { // dr54: 2.8
   V &cvb = (V&)(b);
   V *cpvb = (V*)(&b);
   int V::*cmvb = (int V::*)(&B::b);
-  // expected-error@-1 {{conversion from pointer to member of class 'dr54::B' to pointer to member of class 'dr54::V' via virtual base 'dr54::V' is not allowed}}
+  // expected-error@-1 {{conversion from pointer to member of class 'cwg54::B' to pointer to member of class 'cwg54::V' via virtual base 'cwg54::V' is not allowed}}
   B &cbv = (B&)(v);
-  // expected-error@-1 {{cannot cast 'struct V' to 'B &' via virtual base 'dr54::V'}}
+  // expected-error@-1 {{cannot cast 'struct V' to 'B &' via virtual base 'cwg54::V'}}
   B *cpbv = (B*)(&v);
-  // expected-error@-1 {{cannot cast 'dr54::V *' to 'B *' via virtual base 'dr54::V'}}
+  // expected-error@-1 {{cannot cast 'cwg54::V *' to 'B *' via virtual base 'cwg54::V'}}
   int B::*cmbv = (int B::*)(&V::v);
-  // expected-error@-1 {{conversion from pointer to member of class 'dr54::V' to pointer to member of class 'dr54::B' via virtual base 'dr54::V' is not allowed}}
+  // expected-error@-1 {{conversion from pointer to member of class 'cwg54::V' to pointer to member of class 'cwg54::B' via virtual base 'cwg54::V' is not allowed}}
 }
 
-namespace dr55 { // dr55: yes
+namespace cwg55 { // cwg55: yes
   enum E { e = 5 };
   int test[(e + 1 == 6) ? 1 : -1];
 }
 
-namespace dr56 { // dr56: yes
+namespace cwg56 { // cwg56: yes
   struct A {
-    typedef int T; // #dr56-typedef-int-T-first
+    typedef int T; // #cwg56-typedef-int-T-first
     typedef int T;
     // expected-error@-1 {{redefinition of 'T'}}
-    //   expected-note@#dr56-typedef-int-T-first {{previous definition is here}}
+    //   expected-note@#cwg56-typedef-int-T-first {{previous definition is here}}
   };
   struct B {
     struct X;
-    typedef X X; // #dr56-typedef-X-X-first
+    typedef X X; // #cwg56-typedef-X-X-first
     typedef X X;
     // expected-error@-1 {{redefinition of 'X'}}
-    //   expected-note@#dr56-typedef-X-X-first {{previous definition is here}}
+    //   expected-note@#cwg56-typedef-X-X-first {{previous definition is here}}
   };
 }
 
-namespace dr58 { // dr58: 3.1
+namespace cwg58 { // cwg58: 3.1
   // FIXME: Ideally, we should have a CodeGen test for this.
 #if __cplusplus >= 201103L
   enum E1 { E1_0 = 0, E1_1 = 1 };
@@ -927,54 +927,54 @@ namespace dr58 { // dr58: 3.1
 #endif
 }
 
-namespace dr59 { // dr59: yes
+namespace cwg59 { // cwg59: yes
 #pragma clang diagnostic push
 #pragma clang diagnostic ignored "-Wdeprecated-volatile"
   template<typename T> struct convert_to { operator T() const; };
-  struct A {}; // #dr59-A
-  struct B : A {}; // #dr59-B
+  struct A {}; // #cwg59-A
+  struct B : A {}; // #cwg59-B
 
   A a1 = convert_to<A>();
   A a2 = convert_to<A&>();
   A a3 = convert_to<const A>();
   A a4 = convert_to<const volatile A>();
-  // cxx98-14-error@-1 {{no viable constructor copying variable of type 'const volatile dr59::A'}}
-  //   cxx98-14-note@#dr59-A {{candidate constructor (the implicit copy constructor) not viable: 1st argument ('const volatile dr59::A') would lose volatile qualifier}}
-  //   cxx11-14-note@#dr59-A {{candidate constructor (the implicit move constructor) not viable: 1st argument ('const volatile dr59::A') would lose const and volatile qualifiers}}
-  //   cxx98-14-note@#dr59-A {{candidate constructor (the implicit default constructor) not viable: requires 0 arguments, but 1 was provided}}
+  // cxx98-14-error@-1 {{no viable constructor copying variable of type 'const volatile cwg59::A'}}
+  //   cxx98-14-note@#cwg59-A {{candidate constructor (the implicit copy constructor) not viable: 1st argument ('const volatile cwg59::A') would lose volatile qualifier}}
+  //   cxx11-14-note@#cwg59-A {{candidate constructor (the implicit move constructor) not viable: 1st argument ('const volatile cwg59::A') would lose const and volatile qualifiers}}
+  //   cxx98-14-note@#cwg59-A {{candidate constructor (the implicit default constructor) not viable: requires 0 arguments, but 1 was provided}}
   A a5 = convert_to<const volatile A&>();
-  // expected-error@-1 {{no viable constructor copying variable of type 'const volatile dr59::A'}}
-  //   expected-note@#dr59-A {{candidate constructor (the implicit copy constructor) not viable: 1st argument ('const volatile dr59::A') would lose volatile qualifier}}
-  //   since-cxx11-note@#dr59-A {{candidate constructor (the implicit move constructor) not viable: 1st argument ('const volatile dr59::A') would lose const and volatile qualifiers}}
-  //   expected-note@#dr59-A {{candidate constructor (the implicit default constructor) not viable: requires 0 arguments, but 1 was provided}}
+  // expected-error@-1 {{no viable constructor copying variable of type 'const volatile cwg59::A'}}
+  //   expected-note@#cwg59-A {{candidate constructor (the implicit copy constructor) not viable: 1st argument ('const volatile cwg59::A') would lose volatile qualifier}}
+  //   since-cxx11-note@#cwg59-A {{candidate constructor (the implicit move constructor) not viable: 1st argument ('const volatile cwg59::A') would lose const and volatile qualifiers}}
+  //   expected-note@#cwg59-A {{candidate constructor (the implicit default constructor) not viable: requires 0 arguments, but 1 was provided}}
 
   B b1 = convert_to<B>();
   B b2 = convert_to<B&>();
   B b3 = convert_to<const B>();
   B b4 = convert_to<const volatile B>();
-  // cxx98-14-error@-1 {{no viable constructor copying variable of type 'const volatile dr59::B'}}
-  //   cxx98-14-note@#dr59-B {{candidate constructor (the implicit copy constructor) not viable: 1st argument ('const volatile dr59::B') would lose volatile qualifier}}
-  //   cxx11-14-note@#dr59-B {{candidate constructor (the implicit move constructor) not viable: 1st argument ('const volatile dr59::B') would lose const and volatile qualifiers}}
-  //   cxx98-14-note@#dr59-B {{candidate constructor (the implicit default constructor) not viable: requires 0 arguments, but 1 was provided}}
+  // cxx98-14-error@-1 {{no viable constructor copying variable of type 'const volatile cwg59::B'}}
+  //   cxx98-14-note@#cwg59-B {{candidate constructor (the implicit copy constructor) not viable: 1st argument ('const volatile cwg59::B') would lose volatile qualifier}}
+  //   cxx11-14-note@#cwg59-B {{candidate constructor (the implicit move constructor) not viable: 1st argument ('const volatile cwg59::B') would lose const and volatile qualifiers}}
+  //   cxx98-14-note@#cwg59-B {{candidate constructor (the implicit default constructor) not viable: requires 0 arguments, but 1 was provided}}
   B b5 = convert_to<const volatile B&>();
-  // expected-error@-1 {{no viable constructor copying variable of type 'const volatile dr59::B'}}
-  //   expected-note@#dr59-B {{candidate constructor (the implicit copy constructor) not viable: 1st argument ('const volatile dr59::B') would lose volatile qualifier}}
-  //   since-cxx11-note@#dr59-B {{candidate constructor (the implicit move constructor) not viable: 1st argument ('const volatile dr59::B') would lose const and volatile qualifiers}}
-  //   expected-note@#dr59-B {{candidate constructor (the implicit default constructor) not viable: requires 0 arguments, but 1 was provided}}
+  // expected-error@-1 {{no viable constructor copying variable of type 'const volatile cwg59::B'}}
+  //   expected-note@#cwg59-B {{candidate constructor (the implicit copy constructor) not viable: 1st argument ('const volatile cwg59::B') would lose volatile qualifier}}
+  //   since-cxx11-note@#cwg59-B {{candidate constructor (the implicit move constructor) not viable: 1st argument ('const volatile cwg59::B') would lose const and volatile qualifiers}}
+  //   expected-note@#cwg59-B {{candidate constructor (the implicit default constructor) not viable: requires 0 arguments, but 1 was provided}}
 
   A c1 = convert_to<B>();
   A c2 = convert_to<B&>();
   A c3 = convert_to<const B>();
   A c4 = convert_to<const volatile B>();
-  // expected-error@-1 {{no viable constructor copying variable of type 'const volatile dr59::B'}}
-  //   expected-note@#dr59-A {{candidate constructor (the implicit copy constructor) not viable: no known conversion from 'const volatile dr59::B' to 'const A &' for 1st argument}}
-  //   since-cxx11-note@#dr59-A {{candidate constructor (the implicit move constructor) not viable: no known conversion from 'const volatile dr59::B' to 'A &&' for 1st argument}}
-  //   expected-note@#dr59-A {{candidate constructor (the implicit default constructor) not viable: requires 0 arguments, but 1 was provided}}
+  // expected-error@-1 {{no viable constructor copying variable of type 'const volatile cwg59::B'}}
+  //   expected-note@#cwg59-A {{candidate constructor (the implicit copy constructor) not viable: no known conversion from 'const volatile cwg59::B' to 'const A &' for 1st argument}}
+  //   since-cxx11-note@#cwg59-A {{candidate constructor (the implicit move constructor) not viable: no known conversion from 'const volatile cwg59::B' to 'A &&' for 1st argument}}
+  //   expected-note@#cwg59-A {{candidate constructor (the implicit default constructor) not viable: requires 0 arguments, but 1 was provided}}
   A c5 = convert_to<const volatile B&>();
-  // expected-error@-1 {{no viable constructor copying variable of type 'const volatile dr59::B'}}
-  //   expected-note@#dr59-A {{candidate constructor (the implicit copy constructor) not viable: no known conversion from 'const volatile dr59::B' to 'const A &' for 1st argument}}
-  //   since-cxx11-note@#dr59-A {{candidate constructor (the implicit move constructor) not viable: no known conversion from 'const volatile dr59::B' to 'A &&' for 1st argument}}
-  //   expected-note@#dr59-A {{candidate constructor (the implicit default constructor) not viable: requires 0 arguments, but 1 was provided}}
+  // expected-error@-1 {{no viable constructor copying variable of type 'const volatile cwg59::B'}}
+  //   expected-note@#cwg59-A {{candidate constructor (the implicit copy constructor) not viable: no known conversion from 'const volatile cwg59::B' to 'const A &' for 1st argument}}
+  //   since-cxx11-note@#cwg59-A {{candidate constructor (the implicit move constructor) not viable: no known conversion from 'const volatile cwg59::B' to 'A &&' for 1st argument}}
+  //   expected-note@#cwg59-A {{candidate constructor (the implicit default constructor) not viable: requires 0 arguments, but 1 was provided}}
 
   int n1 = convert_to<int>();
   int n2 = convert_to<int&>();
@@ -984,14 +984,14 @@ namespace dr59 { // dr59: yes
 #pragma clang diagnostic pop
 }
 
-namespace dr60 { // dr60: yes
+namespace cwg60 { // cwg60: yes
   void f(int &);
   int &f(...);
   const int k = 0;
   int &n = f(k);
 }
 
-namespace dr61 { // dr61: 3.4
+namespace cwg61 { // cwg61: 3.4
   struct X {
     static void f();
   } x;
@@ -1008,7 +1008,7 @@ namespace dr61 { // dr61: 3.4
   // expected-error@-1 {{cannot create a non-constant pointer to member function}}
 }
 
-namespace dr62 { // dr62: 2.9
+namespace cwg62 { // cwg62: 2.9
   struct A {
     struct { int n; } b;
   };
@@ -1019,7 +1019,7 @@ namespace dr62 { // dr62: 2.9
   X<A> x1;
   A a = get<A>();
 
-  typedef struct { } *NoNameForLinkagePtr; // #dr62-unnamed
+  typedef struct { } *NoNameForLinkagePtr; // #cwg62-unnamed
   NoNameForLinkagePtr noNameForLinkagePtr;
 
   struct Danger {
@@ -1028,19 +1028,19 @@ namespace dr62 { // dr62: 2.9
 
   X<NoNameForLinkagePtr> x2;
   // cxx98-error@-1 {{template argument uses unnamed type}}
-  //   cxx98-note@#dr62-unnamed {{unnamed type used in template argument was declared here}}
+  //   cxx98-note@#cwg62-unnamed {{unnamed type used in template argument was declared here}}
   X<const NoNameForLinkagePtr> x3;
   // cxx98-error@-1 {{template argument uses unnamed type}}
-  //   cxx98-note@#dr62-unnamed {{unnamed type used in template argument was declared here}}
+  //   cxx98-note@#cwg62-unnamed {{unnamed type used in template argument was declared here}}
   NoNameForLinkagePtr p1 = get<NoNameForLinkagePtr>();
   // cxx98-error@-1 {{template argument uses unnamed type}}
-  //   cxx98-note@#dr62-unnamed {{unnamed type used in template argument was declared here}}
+  //   cxx98-note@#cwg62-unnamed {{unnamed type used in template argument was declared here}}
   NoNameForLinkagePtr p2 = get<const NoNameForLinkagePtr>();
   // cxx98-error@-1 {{template argument uses unnamed type}}
-  //   cxx98-note@#dr62-unnamed {{unnamed type used in template argument was declared here}}
+  //   cxx98-note@#cwg62-unnamed {{unnamed type used in template argument was declared here}}
   int n1 = take(noNameForLinkagePtr);
   // cxx98-error@-1 {{template argument uses unnamed type}}
-  //   cxx98-note@#dr62-unnamed {{unnamed type used in template argument was declared here}}
+  //   cxx98-note@#cwg62-unnamed {{unnamed type used in template argument was declared here}}
 
   X<Danger> x4;
 
@@ -1061,13 +1061,13 @@ namespace dr62 { // dr62: 2.9
   }
 }
 
-namespace dr63 { // dr63: yes
+namespace cwg63 { // cwg63: yes
   template<typename T> struct S { typename T::error e; };
   extern S<int> *p;
   void *q = p;
 }
 
-namespace dr64 { // dr64: yes
+namespace cwg64 { // cwg64: yes
   template<class T> void f(T);
   template<class T> void f(T*);
   template<> void f(int*);
@@ -1075,11 +1075,11 @@ namespace dr64 { // dr64: yes
   template<> void f(int);
 }
 
-// dr65: na
+// cwg65: na
 
-namespace dr66 { // dr66: no
+namespace cwg66 { // cwg66: no
   namespace X {
-    int f(int n); // #dr66-f-first
+    int f(int n); // #cwg66-f-first
   }
   using X::f;
   namespace X {
@@ -1089,36 +1089,36 @@ namespace dr66 { // dr66: no
   // FIXME: The first two calls here should be accepted.
   int a = f();
   // expected-error@-1 {{no matching function for call to 'f'}}
-  //   expected-note@#dr66-f-first {{candidate function not viable: requires single argument 'n', but no arguments were provided}}
+  //   expected-note@#cwg66-f-first {{candidate function not viable: requires single argument 'n', but no arguments were provided}}
   int b = f(1);
   int c = f(1, 2);
   // expected-error@-1 {{no matching function for call to 'f'}}
-  //   expected-note@#dr66-f-first {{candidate function not viable: requires single argument 'n', but 2 arguments were provided}}
+  //   expected-note@#cwg66-f-first {{candidate function not viable: requires single argument 'n', but 2 arguments were provided}}
 }
 
-// dr67: na
+// cwg67: na
 
-namespace dr68 { // dr68: 2.8
+namespace cwg68 { // cwg68: 2.8
   template<typename T> struct X {};
-  struct ::dr68::X<int> x1;
-  struct ::dr68::template X<int> x2;
+  struct ::cwg68::X<int> x1;
+  struct ::cwg68::template X<int> x2;
   // cxx98-error@-1 {{'template' keyword outside of a template}}
   struct Y {
     friend struct X<int>;
-    friend struct ::dr68::X<char>;
-    friend struct ::dr68::template X<double>;
+    friend struct ::cwg68::X<char>;
+    friend struct ::cwg68::template X<double>;
     // cxx98-error@-1 {{'template' keyword outside of a template}}
   };
   template<typename>
   struct Z {
-    friend struct ::dr68::template X<double>;
-    friend typename ::dr68::X<double>;
-    // cxx98-error@-1 {{unelaborated friend declaration is a C++11 extension; specify 'struct' to befriend 'typename ::dr68::X<double>'}}
+    friend struct ::cwg68::template X<double>;
+    friend typename ::cwg68::X<double>;
+    // cxx98-error@-1 {{unelaborated friend declaration is a C++11 extension; specify 'struct' to befriend 'typename ::cwg68::X<double>'}}
   };
 }
 
-namespace dr69 { // dr69: 9
-  template<typename T> static void f() {} // #dr69-f
+namespace cwg69 { // cwg69: 9
+  template<typename T> static void f() {} // #cwg69-f
   // FIXME: Should we warn here?
   inline void g() { f<int>(); }
   extern template void f<char>();
@@ -1127,21 +1127,21 @@ namespace dr69 { // dr69: 9
   template<void(*)()> struct Q {};
   Q<&f<int> > q;
   // cxx98-error@-1 {{non-type template argument referring to function 'f<int>' with internal linkage is a C++11 extension}}
-  //   cxx98-note@#dr69-f {{non-type template argument refers to function here}}
+  //   cxx98-note@#cwg69-f {{non-type template argument refers to function here}}
 }
 
-namespace dr70 { // dr70: yes
+namespace cwg70 { // cwg70: yes
   template<int> struct A {};
   template<int I, int J> int f(int (&)[I + J], A<I>, A<J>);
   int arr[7];
   int k = f(arr, A<3>(), A<4>());
 }
 
-// dr71: na
-// dr72: dup 69
+// cwg71: na
+// cwg72: dup 69
 
 #if __cplusplus >= 201103L
-namespace dr73 { // dr73: sup 1652
+namespace cwg73 { // cwg73: sup 1652
   int a, b;
   static_assert(&a + 1 != &b, "");
   // expected-error@-1 {{static assertion expression is not an integral constant expression}}
@@ -1149,43 +1149,43 @@ namespace dr73 { // dr73: sup 1652
 }
 #endif
 
-namespace dr74 { // dr74: yes
+namespace cwg74 { // cwg74: yes
   enum E { k = 5 };
   int (*p)[k] = new int[k][k];
 }
 
-namespace dr75 { // dr75: yes
+namespace cwg75 { // cwg75: yes
   struct S {
     static int n = 0;
     // expected-error@-1 {{non-const static data member must be initialized out of line}}
   };
 }
 
-namespace dr76 { // dr76: yes
+namespace cwg76 { // cwg76: yes
   const volatile int n = 1;
-  int arr[n]; // #dr76-vla
-  // expected-error@#dr76-vla {{variable length arrays in C++ are a Clang extension}}
-  //   expected-note@#dr76-vla {{read of volatile-qualified type 'const volatile int' is not allowed in a constant expression}}
-  // expected-error@#dr76-vla {{variable length array declaration not allowed at file scope}}
+  int arr[n]; // #cwg76-vla
+  // expected-error@#cwg76-vla {{variable length arrays in C++ are a Clang extension}}
+  //   expected-note@#cwg76-vla {{read of volatile-qualified type 'const volatile int' is not allowed in a constant expression}}
+  // expected-error@#cwg76-vla {{variable length array declaration not allowed at file scope}}
 }
 
-namespace dr77 { // dr77: yes
+namespace cwg77 { // cwg77: yes
   struct A {
     struct B {};
     friend struct B;
   };
 }
 
-namespace dr78 { // dr78: sup ????
-  // Under DR78, this is valid, because 'k' has static storage duration, so is
+namespace cwg78 { // cwg78: sup ????
+  // Under CWG78, this is valid, because 'k' has static storage duration, so is
   // zero-initialized.
   const int k;
   // expected-error@-1 {{default initialization of an object of const type 'const int'}}
 }
 
-// dr79: na
+// cwg79: na
 
-namespace dr80 { // dr80: 2.9
+namespace cwg80 { // cwg80: 2.9
   struct A {
     int A;
   };
@@ -1205,22 +1205,22 @@ namespace dr80 { // dr80: 2.9
   };
 }
 
-// dr81: na
-// dr82: dup 48
+// cwg81: na
+// cwg82: dup 48
 
-namespace dr83 { // dr83: yes
+namespace cwg83 { // cwg83: yes
   int &f(const char*);
   char &f(char *);
   int &k = f("foo");
 }
 
-namespace dr84 { // dr84: yes
+namespace cwg84 { // cwg84: yes
   struct B;
   struct A { operator B() const; };
   struct C {};
   struct B {
-    B(B&); // #dr84-copy-ctor
-    B(C); // #dr84-ctor-from-C
+    B(B&); // #cwg84-copy-ctor
+    B(C); // #cwg84-ctor-from-C
     operator C() const;
   };
   A a;
@@ -1228,78 +1228,78 @@ namespace dr84 { // dr84: yes
   // here. In C++17, we initialize the B object directly using 'A::operator B()'.
   B b = a;
   // cxx98-14-error@-1 {{no viable constructor copying variable of type 'B'}}
-  //   cxx98-14-note@#dr84-copy-ctor {{candidate constructor not viable: expects an lvalue for 1st argument}}
-  //   cxx98-14-note@#dr84-ctor-from-C {{candidate constructor not viable: no known conversion from 'B' to 'C' for 1st argument}}
+  //   cxx98-14-note@#cwg84-copy-ctor {{candidate constructor not viable: expects an lvalue for 1st argument}}
+  //   cxx98-14-note@#cwg84-ctor-from-C {{candidate constructor not viable: no known conversion from 'B' to 'C' for 1st argument}}
 }
 
-namespace dr85 { // dr85: 3.4
+namespace cwg85 { // cwg85: 3.4
   struct A {
     struct B;
-    struct B {}; // #dr85-B-def
+    struct B {}; // #cwg85-B-def
     struct B;
     // expected-error@-1 {{class member cannot be redeclared}}
-    //   expected-note@#dr85-B-def {{previous declaration is here}}
+    //   expected-note@#cwg85-B-def {{previous declaration is here}}
 
     union U;
-    union U {}; // #dr85-U-def
+    union U {}; // #cwg85-U-def
     union U;
     // expected-error@-1 {{class member cannot be redeclared}}
-    //   expected-note@#dr85-U-def {{previous declaration is here}}
+    //   expected-note@#cwg85-U-def {{previous declaration is here}}
 
 #if __cplusplus >= 201103L
     enum E1 : int;
-    enum E1 : int { e1 }; // #dr85-E1-def
+    enum E1 : int { e1 }; // #cwg85-E1-def
     enum E1 : int;
     // expected-error@-1 {{class member cannot be redeclared}}
-    //   expected-note@#dr85-E1-def {{previous declaration is here}}
+    //   expected-note@#cwg85-E1-def {{previous declaration is here}}
 
     enum class E2;
-    enum class E2 { e2 }; // #dr85-E2-def
+    enum class E2 { e2 }; // #cwg85-E2-def
     enum class E2;
     // expected-error@-1 {{class member cannot be redeclared}}
-    //   expected-note@#dr85-E2-def {{previous declaration is here}}
+    //   expected-note@#cwg85-E2-def {{previous declaration is here}}
 #endif
   };
 
   template <typename T>
   struct C {
-    struct B {}; // #dr85-C-B-def
+    struct B {}; // #cwg85-C-B-def
     struct B;
     // expected-error@-1 {{class member cannot be redeclared}}
-    //   expected-note@#dr85-C-B-def {{previous declaration is here}}
+    //   expected-note@#cwg85-C-B-def {{previous declaration is here}}
   };
 }
 
-// dr86: dup 446
+// cwg86: dup 446
 
-namespace dr87 { // dr87: no
-  // FIXME: Superseded by dr1975
+namespace cwg87 { // cwg87: no
+  // FIXME: Superseded by cwg1975
   template<typename T> struct X {};
   // FIXME: This is invalid.
   X<void() throw()> x;
-  // This is valid under dr87 but not under dr1975.
+  // This is valid under cwg87 but not under cwg1975.
   X<void(void() throw())> y;
 }
 
-namespace dr88 { // dr88: 2.8
+namespace cwg88 { // cwg88: 2.8
   template<typename T> struct S {
-    static const int a = 1; // #dr88-a
+    static const int a = 1; // #cwg88-a
     static const int b;
   };
   template<> const int S<int>::a = 4;
   // expected-error@-1 {{static data member 'a' already has an initializer}}
-  //   expected-note@#dr88-a {{previous initialization is here}}
+  //   expected-note@#cwg88-a {{previous initialization is here}}
   template<> const int S<int>::b = 4;
 }
 
-// dr89: na
+// cwg89: na
 
-namespace dr90 { // dr90: yes
+namespace cwg90 { // cwg90: yes
   struct A {
-    template<typename T> friend void dr90_f(T);
+    template<typename T> friend void cwg90_f(T);
   };
   struct B : A {
-    template<typename T> friend void dr90_g(T);
+    template<typename T> friend void cwg90_g(T);
     struct C {};
     union D {};
   };
@@ -1307,41 +1307,41 @@ namespace dr90 { // dr90: yes
   struct F : B::C {};
 
   void test() {
-    dr90_f(A());
-    dr90_f(B());
-    dr90_f(B::C());
-    // expected-error@-1 {{use of undeclared identifier 'dr90_f'}}
-    dr90_f(B::D());
-    // expected-error@-1 {{use of undeclared identifier 'dr90_f'}}
-    dr90_f(E());
-    dr90_f(F());
-    // expected-error@-1 {{use of undeclared identifier 'dr90_f'}}
-
-    dr90_g(A());
-    // expected-error@-1 {{use of undeclared identifier 'dr90_g'}}
-    dr90_g(B());
-    dr90_g(B::C());
-    dr90_g(B::D());
-    dr90_g(E());
-    dr90_g(F());
-    // expected-error@-1 {{use of undeclared identifier 'dr90_g'}}
+    cwg90_f(A());
+    cwg90_f(B());
+    cwg90_f(B::C());
+    // expected-error@-1 {{use of undeclared identifier 'cwg90_f'}}
+    cwg90_f(B::D());
+    // expected-error@-1 {{use of undeclared identifier 'cwg90_f'}}
+    cwg90_f(E());
+    cwg90_f(F());
+    // expected-error@-1 {{use of undeclared identifier 'cwg90_f'}}
+
+    cwg90_g(A());
+    // expected-error@-1 {{use of undeclared identifier 'cwg90_g'}}
+    cwg90_g(B());
+    cwg90_g(B::C());
+    cwg90_g(B::D());
+    cwg90_g(E());
+    cwg90_g(F());
+    // expected-error@-1 {{use of undeclared identifier 'cwg90_g'}}
   }
 }
 
-namespace dr91 { // dr91: yes
+namespace cwg91 { // cwg91: yes
   union U { friend int f(U); };
   int k = f(U());
 }
 
-namespace dr92 { // dr92: 4 c++17
+namespace cwg92 { // cwg92: 4 c++17
   void f() throw(int, float);
   // since-cxx17-error@-1 {{ISO C++17 does not allow dynamic exception specifications}}
   //   since-cxx17-note@-2 {{use 'noexcept(false)' instead}}
-  void (*p)() throw(int) = &f; // #dr92-p
-  // since-cxx17-error@#dr92-p {{ISO C++17 does not allow dynamic exception specifications}}
-  //   since-cxx17-note@#dr92-p {{use 'noexcept(false)' instead}}
-  // cxx98-14-error@#dr92-p {{target exception specification is not superset of source}}
-  // since-cxx17-warning@#dr92-p {{target exception specification is not superset of source}}
+  void (*p)() throw(int) = &f; // #cwg92-p
+  // since-cxx17-error@#cwg92-p {{ISO C++17 does not allow dynamic exception specifications}}
+  //   since-cxx17-note@#cwg92-p {{use 'noexcept(false)' instead}}
+  // cxx98-14-error@#cwg92-p {{target exception specification is not superset of source}}
+  // since-cxx17-warning@#cwg92-p {{target exception specification is not superset of source}}
   void (*q)() throw(int);
   // since-cxx17-error@-1 {{ISO C++17 does not allow dynamic exception specifications}}
   //   since-cxx17-note@-2 {{use 'noexcept(false)' instead}}
@@ -1349,17 +1349,17 @@ namespace dr92 { // dr92: 4 c++17
   // cxx98-14-error@-1 {{exception specifications are not allowed beyond a single level of indirection}}
   // since-cxx17-error@-2 {{cannot initialize a variable of type 'void (**)() throw()' with an rvalue of type 'void (**)() throw(int)'}}
 
-  void g(void() throw()); // #dr92-g
+  void g(void() throw()); // #cwg92-g
   // cxx98-14-warning@-1 {{mangled name of 'g' will change in C++17 due to non-throwing exception specification in function signature}}
   void h() throw() {
     g(f);
     // cxx98-14-error@-1 {{target exception specification is not superset of source}}
     // since-cxx17-error@-2 {{no matching function for call to 'g'}}
-    //   since-cxx17-note@#dr92-g {{candidate function not viable: no known conversion from 'void () throw(int, float)' to 'void (*)() throw()' for 1st argument}}
+    //   since-cxx17-note@#cwg92-g {{candidate function not viable: no known conversion from 'void () throw(int, float)' to 'void (*)() throw()' for 1st argument}}
     g(q);
     // cxx98-14-error@-1 {{target exception specification is not superset of source}}
     // since-cxx17-error@-2 {{no matching function for call to 'g'}}
-    //   since-cxx17-note@#dr92-g {{candidate function not viable: no known conversion from 'void (*)() throw(int)' to 'void (*)() throw()' for 1st argument}}
+    //   since-cxx17-note@#cwg92-g {{candidate function not viable: no known conversion from 'void (*)() throw(int)' to 'void (*)() throw()' for 1st argument}}
   }
 
   // Prior to C++17, this is OK because the exception specification is not
@@ -1376,31 +1376,31 @@ namespace dr92 { // dr92: 4 c++17
   Y<&h> yp; // ok
 }
 
-// dr93: na
+// cwg93: na
 
-namespace dr94 { // dr94: yes
+namespace cwg94 { // cwg94: yes
   struct A { static const int n = 5; };
   int arr[A::n];
 }
 
-namespace dr95 { // dr95: 3.3
+namespace cwg95 { // cwg95: 3.3
   struct A;
   struct B;
   namespace N {
     class C {
       friend struct A;
       friend struct B;
-      static void f(); // #dr95-C-f
+      static void f(); // #cwg95-C-f
     };
-    struct A *p; // dr95::A, not dr95::N::A.
+    struct A *p; // cwg95::A, not cwg95::N::A.
   }
   A *q = N::p; // ok, same type
   struct B { void f() { N::C::f(); } };
-  // expected-error@-1 {{'f' is a private member of 'dr95::N::C'}}
-  //   expected-note@#dr95-C-f {{implicitly declared private here}}
+  // expected-error@-1 {{'f' is a private member of 'cwg95::N::C'}}
+  //   expected-note@#cwg95-C-f {{implicitly declared private here}}
 }
 
-namespace dr96 { // dr96: no
+namespace cwg96 { // cwg96: no
   struct A {
     void f(int);
     template<typename T> int f(T);
@@ -1420,42 +1420,42 @@ namespace dr96 { // dr96: no
   }
 }
 
-namespace dr97 { // dr97: yes
+namespace cwg97 { // cwg97: yes
   struct A {
     static const int a = false;
     static const int b = !a;
   };
 }
 
-namespace dr98 { // dr98: yes
+namespace cwg98 { // cwg98: yes
   void test(int n) {
     switch (n) {
-      try { // #dr98-try
+      try { // #cwg98-try
         case 0:
         // expected-error@-1 {{cannot jump from switch statement to this case label}}
-        //   expected-note@#dr98-try {{jump bypasses initialization of try block}}
+        //   expected-note@#cwg98-try {{jump bypasses initialization of try block}}
         x:
           throw n;
-      } catch (...) { // #dr98-catch
+      } catch (...) { // #cwg98-catch
         case 1:
         // expected-error@-1 {{cannot jump from switch statement to this case label}}
-        //   expected-note@#dr98-catch {{jump bypasses initialization of catch block}}
+        //   expected-note@#cwg98-catch {{jump bypasses initialization of catch block}}
         y:
           throw n;
       }
       case 2:
         goto x;
         // expected-error@-1 {{cannot jump from this goto statement to its label}}
-        //   expected-note@#dr98-try {{jump bypasses initialization of try block}}
+        //   expected-note@#cwg98-try {{jump bypasses initialization of try block}}
       case 3:
         goto y;
         // expected-error@-1 {{cannot jump from this goto statement to its label}}
-        //   expected-note@#dr98-catch {{jump bypasses initialization of catch block}}
+        //   expected-note@#cwg98-catch {{jump bypasses initialization of catch block}}
     }
   }
 }
 
-namespace dr99 { // dr99: sup 214
+namespace cwg99 { // cwg99: sup 214
   template<typename T> void f(T&);
   template<typename T> int &f(const T&);
   const int n = 0;
diff --git a/clang/test/CXX/drs/dr10xx.cpp b/clang/test/CXX/drs/dr10xx.cpp
index 77c59078414c..58d552942c77 100644
--- a/clang/test/CXX/drs/dr10xx.cpp
+++ b/clang/test/CXX/drs/dr10xx.cpp
@@ -14,26 +14,26 @@ namespace std {
   };
 }
 
-namespace dr1004 { // dr1004: 5
+namespace cwg1004 { // cwg1004: 5
   template<typename> struct A {};
   template<typename> struct B1 {};
   template<template<typename> class> struct B2 {};
-  template<typename X> void f(); // #dr1004-f-1
-  template<template<typename> class X> void f(); // #dr1004-f-2
-  template<template<typename> class X> void g(); // #dr1004-g-1
-  template<typename X> void g(); // #dr1004-g-2
+  template<typename X> void f(); // #cwg1004-f-1
+  template<template<typename> class X> void f(); // #cwg1004-f-2
+  template<template<typename> class X> void g(); // #cwg1004-g-1
+  template<typename X> void g(); // #cwg1004-g-2
   struct C : A<int> {
     B1<A> b1a;
     B2<A> b2a;
     void h() {
       f<A>();
       // expected-error@-1 {{call to 'f' is ambiguous}}
-      //   expected-note@#dr1004-f-1 {{candidate function [with X = dr1004::A<int>]}}
-      //   expected-note@#dr1004-f-2 {{candidate function [with X = dr1004::A]}}
+      //   expected-note@#cwg1004-f-1 {{candidate function [with X = cwg1004::A<int>]}}
+      //   expected-note@#cwg1004-f-2 {{candidate function [with X = cwg1004::A]}}
       g<A>();
       // expected-error@-1 {{call to 'g' is ambiguous}}
-      //   expected-note@#dr1004-g-1 {{candidate function [with X = dr1004::A]}}
-      //   expected-note@#dr1004-g-2 {{candidate function [with X = dr1004::A<int>]}}
+      //   expected-note@#cwg1004-g-1 {{candidate function [with X = cwg1004::A]}}
+      //   expected-note@#cwg1004-g-2 {{candidate function [with X = cwg1004::A<int>]}}
     }
   };
 
@@ -41,17 +41,17 @@ namespace dr1004 { // dr1004: 5
   // name lookup of "T::template A" names the constructor.
   template<class T, template<class> class U = T::template A> struct Third { };
   // expected-error@-1 {{is a constructor name}}
-  //   expected-note@#dr1004-t {{in instantiation of default argument}}
-  Third<A<int> > t; // #dr1004-t
+  //   expected-note@#cwg1004-t {{in instantiation of default argument}}
+  Third<A<int> > t; // #cwg1004-t
 }
 
-namespace dr1042 { // dr1042: 3.5
+namespace cwg1042 { // cwg1042: 3.5
 #if __cplusplus >= 201402L
   // C++14 added an attribute that we can test the semantics of.
-  using foo [[deprecated]] = int; // #dr1042-using
+  using foo [[deprecated]] = int; // #cwg1042-using
   foo f = 12;
   // since-cxx14-warning@-1 {{'foo' is deprecated}}
-  //   since-cxx14-note@#dr1042-using {{'foo' has been explicitly marked deprecated here}}
+  //   since-cxx14-note@#cwg1042-using {{'foo' has been explicitly marked deprecated here}}
 #elif __cplusplus >= 201103L
   // C++11 did not have any attributes that could be applied to an alias
   // declaration, so the best we can test is that we accept an empty attribute
@@ -60,7 +60,7 @@ namespace dr1042 { // dr1042: 3.5
 #endif
 }
 
-namespace dr1048 { // dr1048: 3.6
+namespace cwg1048 { // cwg1048: 3.6
   struct A {};
   const A f();
   A g();
@@ -78,20 +78,20 @@ namespace dr1048 { // dr1048: 3.6
 #endif
 }
 
-namespace dr1054 { // dr1054: no
+namespace cwg1054 { // cwg1054: no
   // FIXME: Test is incomplete.
   struct A {} volatile a;
   void f() {
     // FIXME: This is wrong: an lvalue-to-rvalue conversion is applied here,
     // which copy-initializes a temporary from 'a'. Therefore this is
     // ill-formed because A does not have a volatile copy constructor.
-    // (We might want to track this aspect under dr1383 instead?)
+    // (We might want to track this aspect under cwg1383 instead?)
     a;
     // expected-warning@-1 {{expression result unused; assign into a variable to force a volatile load}}
   }
 }
 
-namespace dr1070 { // dr1070: 3.5
+namespace cwg1070 { // cwg1070: 3.5
 #if __cplusplus >= 201103L
   struct A {
     A(std::initializer_list<int>);
diff --git a/clang/test/CXX/drs/dr11xx.cpp b/clang/test/CXX/drs/dr11xx.cpp
index a71a105c7eb2..46a0e526be39 100644
--- a/clang/test/CXX/drs/dr11xx.cpp
+++ b/clang/test/CXX/drs/dr11xx.cpp
@@ -4,27 +4,27 @@
 // RUN: %clang_cc1 -std=c++17 %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++2a %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors
 
-namespace dr1111 { // dr1111: 3.2
+namespace cwg1111 { // cwg1111: 3.2
 namespace example1 {
-template <typename> struct set; // #dr1111-struct-set
+template <typename> struct set; // #cwg1111-struct-set
 
 struct X {
-  template <typename T> void set(const T &value); // #dr1111-func-set
+  template <typename T> void set(const T &value); // #cwg1111-func-set
 };
 void foo() {
   X x;
   // FIXME: should we backport C++11 behavior?
   x.set<double>(3.2);
   // cxx98-error@-1 {{lookup of 'set' in member access expression is ambiguous; using member of 'X'}}
-  //   cxx98-note@#dr1111-func-set {{lookup in the object type 'X' refers here}}
-  //   cxx98-note@#dr1111-struct-set {{lookup from the current scope refers here}}
+  //   cxx98-note@#cwg1111-func-set {{lookup in the object type 'X' refers here}}
+  //   cxx98-note@#cwg1111-struct-set {{lookup from the current scope refers here}}
 }
 
 struct Y {};
 void bar() {
   Y y;
   y.set<double>(3.2);
-  // expected-error@-1 {{no member named 'set' in 'dr1111::example1::Y'}}
+  // expected-error@-1 {{no member named 'set' in 'cwg1111::example1::Y'}}
 }
 } // namespace example1
 
@@ -42,14 +42,14 @@ void baz() {
   a.operator A();
 }
 } // namespace example2
-} // namespace dr1111
+} // namespace cwg1111
 
-namespace dr1113 { // dr1113: partial
+namespace cwg1113 { // cwg1113: partial
   namespace named {
-    extern int a; // #dr1113-a
+    extern int a; // #cwg1113-a
     static int a;
     // expected-error@-1 {{static declaration of 'a' follows non-static}}
-    //   expected-note@#dr1113-a {{previous declaration is here}}
+    //   expected-note@#cwg1113-a {{previous declaration is here}}
   }
   namespace {
     extern int a;
@@ -57,7 +57,7 @@ namespace dr1113 { // dr1113: partial
     int b = a;
   }
 
-  // FIXME: Per DR1113 and DR4, this is ill-formed due to ambiguity: the second
+  // FIXME: Per CWG1113 and CWG4, this is ill-formed due to ambiguity: the second
   // 'f' has internal linkage, and so does not have C language linkage, so is
   // not a redeclaration of the first 'f'.
   //
@@ -71,4 +71,4 @@ namespace dr1113 { // dr1113: partial
   void g() { f(); }
 }
 
-// dr1150: na
+// cwg1150: na
diff --git a/clang/test/CXX/drs/dr12xx.cpp b/clang/test/CXX/drs/dr12xx.cpp
index da5dd02a0067..cdfbc6d67265 100644
--- a/clang/test/CXX/drs/dr12xx.cpp
+++ b/clang/test/CXX/drs/dr12xx.cpp
@@ -5,9 +5,9 @@
 // RUN: %clang_cc1 -std=c++20 %s -verify=expected,since-cxx17,since-cxx14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++23 %s -verify=expected,since-cxx17,since-cxx14,since-cxx11,since-cxx23 -fexceptions -fcxx-exceptions -pedantic-errors
 
-// dr1200: na
+// cwg1200: na
 
-namespace dr1213 { // dr1213: 7
+namespace cwg1213 { // cwg1213: 7
 #if __cplusplus >= 201103L
   using T = int[3];
   int &&r = T{}[1];
@@ -32,7 +32,7 @@ namespace dr1213 { // dr1213: 7
 }
 
 #if __cplusplus >= 201103L
-namespace dr1223 { // dr1223: 17 drafting 2023-05-12
+namespace cwg1223 { // cwg1223: 17 drafting 2023-05-12
 struct M;
 template <typename T>
 struct V;
@@ -80,31 +80,31 @@ void g() {
     A b(auto ()->C);
     static_assert(sizeof(B ()->C[1] == sizeof(int)), "");
     sizeof(auto () -> C[1]);
-    // since-cxx11-error@-1 {{function cannot return array type 'C[1]' (aka 'dr1223::BB[1]')}}
+    // since-cxx11-error@-1 {{function cannot return array type 'C[1]' (aka 'cwg1223::BB[1]')}}
 }
 
 }
 #endif
 
 #if __cplusplus >= 201103L
-namespace dr1227 { // dr1227: 3.0
+namespace cwg1227 { // cwg1227: 3.0
 template <class T> struct A { using X = typename T::X; };
 // since-cxx11-error@-1 {{type 'int' cannot be used prior to '::' because it has no members}}
-//   since-cxx11-note@#dr1227-g {{in instantiation of template class 'dr1227::A<int>' requested here}}
-//   since-cxx11-note@#dr1227-g-int {{while substituting explicitly-specified template arguments into function template 'g'}}
+//   since-cxx11-note@#cwg1227-g {{in instantiation of template class 'cwg1227::A<int>' requested here}}
+//   since-cxx11-note@#cwg1227-g-int {{while substituting explicitly-specified template arguments into function template 'g'}}
 template <class T> typename T::X f(typename A<T>::X);
 template <class T> void f(...) { }
-template <class T> auto g(typename A<T>::X) -> typename T::X; // #dr1227-g
+template <class T> auto g(typename A<T>::X) -> typename T::X; // #cwg1227-g
 template <class T> void g(...) { }
 
 void h() {
   f<int>(0); // OK, substituting return type causes deduction to fail
-  g<int>(0); // #dr1227-g-int
+  g<int>(0); // #cwg1227-g-int
 }
 }
 #endif
 
-namespace dr1250 { // dr1250: 3.9
+namespace cwg1250 { // cwg1250: 3.9
 struct Incomplete;
 
 struct Base {
@@ -116,7 +116,7 @@ struct Derived : Base {
 };
 }
 
-namespace dr1265 { // dr1265: 5
+namespace cwg1265 { // cwg1265: 5
 #if __cplusplus >= 201103L
   auto a = 0, b() -> int;
   // since-cxx11-error@-1 {{declaration with trailing return type must be the only declaration in its group}}
@@ -136,9 +136,9 @@ namespace dr1265 { // dr1265: 5
 #endif
 }
 
-// dr1291: na
+// cwg1291: na
 
-namespace dr1295 { // dr1295: 4
+namespace cwg1295 { // cwg1295: 4
   struct X {
     unsigned bitfield : 4;
   };
@@ -150,11 +150,11 @@ namespace dr1295 { // dr1295: 4
   unsigned const &r2 = static_cast<unsigned &&>(x.bitfield);
   // cxx98-error@-1 {{rvalue references are a C++11 extension}}
 
-  template<unsigned &r> struct Y {}; // #dr1295-Y
-  Y<x.bitfield> y; // #dr1295-y
+  template<unsigned &r> struct Y {}; // #cwg1295-Y
+  Y<x.bitfield> y; // #cwg1295-y
   // cxx98-14-error@-1 {{non-type template argument does not refer to any declaration}}
-  //   cxx98-14-note@#dr1295-Y {{template parameter is declared here}}
-  // since-cxx17-error@#dr1295-y {{reference cannot bind to bit-field in converted constant expression}}
+  //   cxx98-14-note@#cwg1295-Y {{template parameter is declared here}}
+  // since-cxx17-error@#cwg1295-y {{reference cannot bind to bit-field in converted constant expression}}
 
 #if __cplusplus >= 201103L
   const unsigned other = 0;
diff --git a/clang/test/CXX/drs/dr13xx.cpp b/clang/test/CXX/drs/dr13xx.cpp
index d8e3b5d87bd1..dad82c4e2829 100644
--- a/clang/test/CXX/drs/dr13xx.cpp
+++ b/clang/test/CXX/drs/dr13xx.cpp
@@ -17,18 +17,18 @@ namespace std {
 }
 
 #if __cplusplus >= 201103L
-namespace dr1305 { // dr1305: 3.0
-struct Incomplete; // #dr1305-Incomplete
+namespace cwg1305 { // cwg1305: 3.0
+struct Incomplete; // #cwg1305-Incomplete
 struct Complete {};
 
 int incomplete = alignof(Incomplete(&)[]);
 // since-cxx11-error@-1 {{invalid application of 'alignof' to an incomplete type 'Incomplete'}}
-//   since-cxx11-note@#dr1305-Incomplete {{forward declaration of 'dr1305::Incomplete'}}
+//   since-cxx11-note@#cwg1305-Incomplete {{forward declaration of 'cwg1305::Incomplete'}}
 int complete = alignof(Complete(&)[]);
 }
 #endif
 
-namespace dr1307 { // dr1307: 14
+namespace cwg1307 { // cwg1307: 14
 #if __cplusplus >= 201103L
 void f(int const (&)[2]);
 void f(int const (&)[3]);
@@ -38,11 +38,11 @@ void caller() {
   f({1, 2, 3});
 }
 #endif // __cplusplus >= 201103L
-} // namespace dr1307
+} // namespace cwg1307
 
-// dr1308: sup 1330
+// cwg1308: sup 1330
 
-namespace dr1310 { // dr1310: 5
+namespace cwg1310 { // cwg1310: 5
   struct S {} * sp = new S::S;
   // expected-error@-1 {{qualified reference to 'S' is a constructor name rather than a type in this context}}
   void f() {
@@ -126,8 +126,8 @@ namespace dr1310 { // dr1310: 5
   void wt_test() {
     typename W::W w2a;
     // expected-error@-1 {{ISO C++ specifies that qualified reference to 'W' is a constructor name rather than a type in this context, despite preceding 'typename' keyword}}
-    //   cxx98-note@#dr1310-W-int {{in instantiation of function template specialization 'dr1310::wt_test<dr1310::W<int> >' requested here}}
-    //   since-cxx11-note@#dr1310-W-int {{in instantiation of function template specialization 'dr1310::wt_test<dr1310::W<int>>' requested here}}
+    //   cxx98-note@#cwg1310-W-int {{in instantiation of function template specialization 'cwg1310::wt_test<cwg1310::W<int> >' requested here}}
+    //   since-cxx11-note@#cwg1310-W-int {{in instantiation of function template specialization 'cwg1310::wt_test<cwg1310::W<int>>' requested here}}
     typename W::template W<int> w4;
     // expected-error@-1 {{ISO C++ specifies that qualified reference to 'W' is a constructor name rather than a template name in this context, despite preceding 'template' keyword}}
     TTy<typename W::W> tt2;
@@ -148,11 +148,11 @@ namespace dr1310 { // dr1310: 5
     (void)w.template W<int>::W::n;
     (void)w.template W<int>::template W<int>::n;
   }
-  template void wt_test<W<int> >(); // #dr1310-W-int
+  template void wt_test<W<int> >(); // #cwg1310-W-int
   template void wt_test_good<W<int> >();
 }
 
-namespace dr1315 { // dr1315: partial
+namespace cwg1315 { // cwg1315: partial
   template <int I, int J> struct A {};
   template <int I> struct A<I + 5, I * 2> {};
   // expected-error@-1 {{class template partial specialization contains a template parameter that cannot be deduced; this partial specialization will never be used}}
@@ -160,7 +160,7 @@ namespace dr1315 { // dr1315: partial
   template <int I> struct A<I, I> {};
 
   template <int I, int J, int K> struct B;
-  template <int I, int K> struct B<I, I * 2, K> {}; // #dr1315-B-1
+  template <int I, int K> struct B<I, I * 2, K> {}; // #cwg1315-B-1
   B<1, 2, 3> b1;
 
   // Multiple declarations with the same dependent expression are equivalent
@@ -169,13 +169,13 @@ namespace dr1315 { // dr1315: partial
   B<1, 2, 2>::type b2;
 
   // Multiple declarations with differing dependent expressions are unordered.
-  template <int I, int K> struct B<I, I + 1, K> {}; // #dr1315-B-2
+  template <int I, int K> struct B<I, I + 1, K> {}; // #cwg1315-B-2
   B<1, 2, 4> b3;
   // expected-error@-1 {{ambiguous partial specializations of 'B<1, 2, 4>'}}
-  //   expected-note@#dr1315-B-1 {{partial specialization matches [with I = 1, K = 4]}}
-  //   expected-note@#dr1315-B-2 {{partial specialization matches [with I = 1, K = 4]}}
+  //   expected-note@#cwg1315-B-1 {{partial specialization matches [with I = 1, K = 4]}}
+  //   expected-note@#cwg1315-B-2 {{partial specialization matches [with I = 1, K = 4]}}
 
-  // FIXME: Under dr1315, this is perhaps valid, but that is not clear: this
+  // FIXME: Under cwg1315, this is perhaps valid, but that is not clear: this
   // fails the "more specialized than the primary template" test because the
   // dependent type of T::value is not the same as 'int'.
   // A core issue will be opened to decide what is supposed to happen here.
@@ -184,7 +184,7 @@ namespace dr1315 { // dr1315: partial
   // expected-error@-1 {{type of specialized non-type template argument depends on a template parameter of the partial specialization}}
 }
 
-namespace dr1330 { // dr1330: 4 c++11
+namespace cwg1330 { // cwg1330: 4 c++11
   // exception-specifications are parsed in a context where the class is complete.
   struct A {
     void f() throw(T) {}
@@ -204,7 +204,7 @@ namespace dr1330 { // dr1330: 4 c++11
   //   since-cxx17-note@-2 {{use 'noexcept(false)' instead}}
   void (A::*af2)() throw() = &A::f;
   // cxx98-14-error@-1 {{target exception specification is not superset of source}}
-  // since-cxx17-error@-2 {{cannot initialize a variable of type 'void (dr1330::A::*)() throw()' with an rvalue of type 'void (dr1330::A::*)() throw(T)': different exception specifications}}
+  // since-cxx17-error@-2 {{cannot initialize a variable of type 'void (cwg1330::A::*)() throw()' with an rvalue of type 'void (cwg1330::A::*)() throw(T)': different exception specifications}}
 
 #if __cplusplus >= 201103L
   static_assert(noexcept(A().g()), "");
@@ -252,7 +252,7 @@ namespace dr1330 { // dr1330: 4 c++11
   void (B<P>::*bpf3)() = &B<P>::f;
   void (B<P>::*bpf4)() throw() = &B<P>::f;
   // cxx98-14-error@-1 {{target exception specification is not superset of source}}
-  // since-cxx17-error@-2 {{cannot initialize a variable of type 'void (B<P>::*)() throw()' with an rvalue of type 'void (dr1330::B<dr1330::P>::*)() throw(T, typename P::type)': different exception specifications}}
+  // since-cxx17-error@-2 {{cannot initialize a variable of type 'void (B<P>::*)() throw()' with an rvalue of type 'void (cwg1330::B<cwg1330::P>::*)() throw(T, typename P::type)': different exception specifications}}
 
 #if __cplusplus >= 201103L
   static_assert(noexcept(B<P>().g()), "");
@@ -260,73 +260,73 @@ namespace dr1330 { // dr1330: 4 c++11
   static_assert(!noexcept(B<Q>().g()), "");
 #endif
 
-  template<typename T> int f() throw(typename T::error) { return 0; } // #dr1330-f
-  // expected-error@#dr1330-f {{type 'int' cannot be used prior to '::' because it has no members}}
-  //   cxx98-note@#dr1330-f-int {{in instantiation of function template specialization 'dr1330::f<int>' requested here}}
-  //   since-cxx11-note@#dr1330-f-int {{in instantiation of exception specification for 'f<int>' requested here}}
-  // cxx98-14-error@#dr1330-f {{type 'short' cannot be used prior to '::' because it has no members}}
-  //   cxx98-14-note@#dr1330-f-short {{in instantiation of function template specialization 'dr1330::f<short>' requested here}}
-  //   cxx11-14-note@#dr1330-f {{in instantiation of exception specification for 'f<short>' requested here}}
-  // since-cxx11-error@#dr1330-f {{type 'char' cannot be used prior to '::' because it has no members}}
-  //   since-cxx11-note@#dr1330-f-char {{in instantiation of exception specification for 'f<char>' requested here}}
-  // since-cxx11-error@#dr1330-f {{type 'float' cannot be used prior to '::' because it has no members}}
-  //   since-cxx11-note@#dr1330-f-float {{in instantiation of exception specification for 'f<float>' requested here}}
-  // since-cxx17-error@#dr1330-f {{ISO C++17 does not allow dynamic exception specifications}}
-  //   since-cxx17-note@#dr1330-f {{use 'noexcept(false)' instead}}
+  template<typename T> int f() throw(typename T::error) { return 0; } // #cwg1330-f
+  // expected-error@#cwg1330-f {{type 'int' cannot be used prior to '::' because it has no members}}
+  //   cxx98-note@#cwg1330-f-int {{in instantiation of function template specialization 'cwg1330::f<int>' requested here}}
+  //   since-cxx11-note@#cwg1330-f-int {{in instantiation of exception specification for 'f<int>' requested here}}
+  // cxx98-14-error@#cwg1330-f {{type 'short' cannot be used prior to '::' because it has no members}}
+  //   cxx98-14-note@#cwg1330-f-short {{in instantiation of function template specialization 'cwg1330::f<short>' requested here}}
+  //   cxx11-14-note@#cwg1330-f {{in instantiation of exception specification for 'f<short>' requested here}}
+  // since-cxx11-error@#cwg1330-f {{type 'char' cannot be used prior to '::' because it has no members}}
+  //   since-cxx11-note@#cwg1330-f-char {{in instantiation of exception specification for 'f<char>' requested here}}
+  // since-cxx11-error@#cwg1330-f {{type 'float' cannot be used prior to '::' because it has no members}}
+  //   since-cxx11-note@#cwg1330-f-float {{in instantiation of exception specification for 'f<float>' requested here}}
+  // since-cxx17-error@#cwg1330-f {{ISO C++17 does not allow dynamic exception specifications}}
+  //   since-cxx17-note@#cwg1330-f {{use 'noexcept(false)' instead}}
 
   // An exception-specification is needed even if the function is only used in
   // an unevaluated operand.
-  int f1 = sizeof(f<int>()); // #dr1330-f-int
+  int f1 = sizeof(f<int>()); // #cwg1330-f-int
 #if __cplusplus >= 201103L
-  decltype(f<char>()) f2; // #dr1330-f-char
-  bool f3 = noexcept(f<float>()); /// #dr1330-f-float
+  decltype(f<char>()) f2; // #cwg1330-f-char
+  bool f3 = noexcept(f<float>()); /// #cwg1330-f-float
 #endif
   // In C++17 onwards, substituting explicit template arguments into the
   // function type substitutes into the exception specification (because it's
   // part of the type). In earlier languages, we don't notice there's a problem
   // until we've already started to instantiate.
-  template int f<short>(); // #dr1330-f-short
+  template int f<short>(); // #cwg1330-f-short
   // since-cxx17-error@-1 {{explicit instantiation of 'f' does not refer to a function template, variable template, member function, member class, or static data member}}
-  //   since-cxx17-note@#dr1330-f {{candidate template ignored: substitution failure [with T = short]: type 'short' cannot be used prior to '::' because it has no members}}
+  //   since-cxx17-note@#cwg1330-f {{candidate template ignored: substitution failure [with T = short]: type 'short' cannot be used prior to '::' because it has no members}}
 
   template<typename T> struct C {
-    C() throw(typename T::type); // #dr1330-C
+    C() throw(typename T::type); // #cwg1330-C
     // since-cxx17-error@-1 {{ISO C++17 does not allow dynamic exception specifications}}
     //   since-cxx17-note@-2 {{use 'noexcept(false)' instead}}
-    // cxx98-error@#dr1330-C {{type 'void' cannot be used prior to '::' because it has no members}}
-    //   cxx98-note@#dr1330-C-void {{in instantiation of template class 'dr1330::C<void>' requested here}}
-    // expected-error@#dr1330-C {{type 'int' cannot be used prior to '::' because it has no members}}
-    //   cxx98-note@#dr1330-C-int {{in instantiation of template class 'dr1330::C<int>' requested here}}
-    //   since-cxx11-note@#dr1330-C-int {{in instantiation of exception specification for 'C' requested here}}
-    //   since-cxx11-note@#dr1330-e {{in evaluation of exception specification for 'dr1330::E::E' needed here}}
+    // cxx98-error@#cwg1330-C {{type 'void' cannot be used prior to '::' because it has no members}}
+    //   cxx98-note@#cwg1330-C-void {{in instantiation of template class 'cwg1330::C<void>' requested here}}
+    // expected-error@#cwg1330-C {{type 'int' cannot be used prior to '::' because it has no members}}
+    //   cxx98-note@#cwg1330-C-int {{in instantiation of template class 'cwg1330::C<int>' requested here}}
+    //   since-cxx11-note@#cwg1330-C-int {{in instantiation of exception specification for 'C' requested here}}
+    //   since-cxx11-note@#cwg1330-e {{in evaluation of exception specification for 'cwg1330::E::E' needed here}}
   };
-  struct D : C<void> {}; // #dr1330-C-void
+  struct D : C<void> {}; // #cwg1330-C-void
   void f(D &d) { d = d; } // ok
 
-  struct E : C<int> {}; // #dr1330-C-int
-  E e; // #dr1330-e
+  struct E : C<int> {}; // #cwg1330-C-int
+  E e; // #cwg1330-e
 }
 
-// dr1334: sup 1719
+// cwg1334: sup 1719
 
-namespace dr1341 { // dr1341: sup P0683R1
+namespace cwg1341 { // cwg1341: sup P0683R1
 #if __cplusplus >= 202002L
 int a;
-const int b = 0; // #dr1341-b
+const int b = 0; // #cwg1341-b
 struct S {
   int x1 : 8 = 42;
   int x2 : 8 { 42 };
   int y1 : true ? 8 : a = 42;
   int y2 : true ? 8 : b = 42;
   // since-cxx20-error@-1 {{cannot assign to variable 'b' with const-qualified type 'const int'}}
-  //   since-cxx20-note@#dr1341-b {{variable 'b' declared const here}}
+  //   since-cxx20-note@#cwg1341-b {{variable 'b' declared const here}}
   int y3 : (true ? 8 : b) = 42;
   int z : 1 || new int { 0 };
 };
 #endif
 }
 
-namespace dr1346 { // dr1346: 3.5
+namespace cwg1346 { // cwg1346: 3.5
   auto a(1);
   // cxx98-error@-1 {{'auto' type specifier is a C++11 extension}}
   auto b(1, 2);
@@ -345,9 +345,9 @@ namespace dr1346 { // dr1346: 3.5
     auto x(ts...);
     // cxx98-error@-1 {{'auto' type specifier is a C++11 extension}}
     // expected-error@-2 {{initializer for variable 'x' with type 'auto' is empty}}
-    //   expected-note@#dr1346-f {{in instantiation of function template specialization 'dr1346::f<>' requested here}}
+    //   expected-note@#cwg1346-f {{in instantiation of function template specialization 'cwg1346::f<>' requested here}}
   }
-  template void f(); // #dr1346-f
+  template void f(); // #cwg1346-f
 
 #if __cplusplus >= 201103L
   void init_capture() {
@@ -369,7 +369,7 @@ namespace dr1346 { // dr1346: 3.5
 #endif
 }
 
-namespace dr1347 { // dr1347: 3.1
+namespace cwg1347 { // cwg1347: 3.1
   auto x = 5, *y = &x;
   // cxx98-error@-1 {{'auto' type specifier is a C++11 extension}}
   auto z = y, *q = y;
@@ -383,7 +383,7 @@ namespace dr1347 { // dr1347: 3.1
 #endif
 }
 
-namespace dr1350 { // dr1350: 3.5
+namespace cwg1350 { // cwg1350: 3.5
 #if __cplusplus >= 201103L
 struct NoexceptCtor {
   NoexceptCtor(int) noexcept {}
@@ -452,12 +452,12 @@ struct D4 : NoexceptCtor, ThrowingDefaultArgTemplate {
 
 static_assert(!__is_nothrow_constructible(D4, int), "");
 #endif
-} // namespace dr1350
+} // namespace cwg1350
 
-namespace dr1358 { // dr1358: 3.1
+namespace cwg1358 { // cwg1358: 3.1
 #if __cplusplus >= 201103L
   struct Lit { constexpr operator int() const { return 0; } };
-  struct NonLit { NonLit(); operator int(); }; // #dr1358-NonLit
+  struct NonLit { NonLit(); operator int(); }; // #cwg1358-NonLit
   struct NonConstexprConv { constexpr operator int() const; };
   struct Virt { virtual int f(int) const; };
 
@@ -486,83 +486,83 @@ namespace dr1358 { // dr1358: 3.1
     int member;
     constexpr B(NonLit u) : member(u) {}
     // cxx11-20-error@-1 {{constexpr constructor's 1st parameter type 'NonLit' is not a literal type}}
-    //   cxx11-20-note@#dr1358-NonLit {{'NonLit' is not literal because it is not an aggregate and has no constexpr constructors other than copy or move constructors}}
+    //   cxx11-20-note@#cwg1358-NonLit {{'NonLit' is not literal because it is not an aggregate and has no constexpr constructors other than copy or move constructors}}
     constexpr NonLit f(NonLit u) const { return NonLit(); }
     // cxx11-20-error@-1 {{constexpr function's return type 'NonLit' is not a literal type}}
-    //   cxx11-20-note@#dr1358-NonLit {{'NonLit' is not literal because it is not an aggregate and has no constexpr constructors other than copy or move constructors}}
+    //   cxx11-20-note@#cwg1358-NonLit {{'NonLit' is not literal because it is not an aggregate and has no constexpr constructors other than copy or move constructors}}
   };
 #endif
 }
 
-namespace dr1359 { // dr1359: 3.5
+namespace cwg1359 { // cwg1359: 3.5
 #if __cplusplus >= 201103L
   union A { constexpr A() = default; };
-  union B { constexpr B() = default; int a; }; // #dr1359-B
+  union B { constexpr B() = default; int a; }; // #cwg1359-B
   // cxx11-17-error@-1 {{defaulted definition of default constructor cannot be marked constexpr before C++23}}
-  union C { constexpr C() = default; int a, b; }; // #dr1359-C
+  union C { constexpr C() = default; int a, b; }; // #cwg1359-C
   // cxx11-17-error@-1 {{defaulted definition of default constructor cannot be marked constexpr}} 
   struct X { constexpr X() = default; union {}; };
   // since-cxx11-error@-1 {{declaration does not declare anything}}
-  struct Y { constexpr Y() = default; union { int a; }; }; // #dr1359-Y
+  struct Y { constexpr Y() = default; union { int a; }; }; // #cwg1359-Y
   // cxx11-17-error@-1 {{defaulted definition of default constructor cannot be marked constexpr}}
 
   constexpr A a = A();
   constexpr B b = B();
   // cxx11-17-error@-1 {{no matching constructor for initialization of 'B'}}
-  //   cxx11-17-note@#dr1359-B {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 0 were provided}}
-  //   cxx11-17-note@#dr1359-B {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 0 were provided}}
+  //   cxx11-17-note@#cwg1359-B {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 0 were provided}}
+  //   cxx11-17-note@#cwg1359-B {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 0 were provided}}
   constexpr C c = C();
   // cxx11-17-error@-1 {{no matching constructor for initialization of 'C'}}
-  //   cxx11-17-note@#dr1359-C {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 0 were provided}}
-  //   cxx11-17-note@#dr1359-C {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 0 were provided}}
+  //   cxx11-17-note@#cwg1359-C {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 0 were provided}}
+  //   cxx11-17-note@#cwg1359-C {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 0 were provided}}
   constexpr X x = X();
   constexpr Y y = Y();
   // cxx11-17-error@-1 {{no matching constructor for initialization of 'Y'}}
-  //   cxx11-17-note@#dr1359-Y {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 0 were provided}}
-  //   cxx11-17-note@#dr1359-Y {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 0 were provided}}
+  //   cxx11-17-note@#cwg1359-Y {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 0 were provided}}
+  //   cxx11-17-note@#cwg1359-Y {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 0 were provided}}
 #endif
 }
 
-namespace dr1388 { // dr1388: 4
-  template<typename A, typename ...T> void f(T..., A); // #dr1388-f
+namespace cwg1388 { // cwg1388: 4
+  template<typename A, typename ...T> void f(T..., A); // #cwg1388-f
   // cxx98-error@-1 {{variadic templates are a C++11 extension}}
-  template<typename ...T> void g(T..., int); // #dr1388-g
+  template<typename ...T> void g(T..., int); // #cwg1388-g
   // cxx98-error@-1 {{variadic templates are a C++11 extension}}
-  template<typename ...T, typename A> void h(T..., A); // #dr1388-h
+  template<typename ...T, typename A> void h(T..., A); // #cwg1388-h
   // cxx98-error@-1 {{variadic templates are a C++11 extension}}
 
   void test_f() {
     f(0); // ok, trailing parameter pack deduced to empty
     f(0, 0);
     // expected-error@-1 {{no matching function for call to 'f'}}
-    //   expected-note@#dr1388-f {{candidate function [with A = int, T = <>] not viable: requires 1 argument, but 2 were provided}}
+    //   expected-note@#cwg1388-f {{candidate function [with A = int, T = <>] not viable: requires 1 argument, but 2 were provided}}
     f<int>(0);
     f<int>(0, 0);
     // expected-error@-1 {{no matching function for call to 'f'}}
-    //   expected-note@#dr1388-f {{candidate function [with A = int, T = <>] not viable: requires 1 argument, but 2 were provided}}
+    //   expected-note@#cwg1388-f {{candidate function [with A = int, T = <>] not viable: requires 1 argument, but 2 were provided}}
     f<int, int>(0, 0);
     f<int, int, int>(0, 0);
     // expected-error@-1 {{no matching function for call to 'f'}}
-    //   expected-note@#dr1388-f {{candidate function [with A = int, T = <int, int>] not viable: requires 3 arguments, but 2 were provided}}
+    //   expected-note@#cwg1388-f {{candidate function [with A = int, T = <int, int>] not viable: requires 3 arguments, but 2 were provided}}
 
     g(0);
     g(0, 0);
     // expected-error@-1 {{no matching function for call to 'g'}}
-    //   expected-note@#dr1388-g {{candidate function [with T = <>] not viable: requires 1 argument, but 2 were provided}}
+    //   expected-note@#cwg1388-g {{candidate function [with T = <>] not viable: requires 1 argument, but 2 were provided}}
     g<>(0);
     g<int>(0);
     // expected-error@-1 {{no matching function for call to 'g'}}
-    //   expected-note@#dr1388-g {{candidate function [with T = <int>] not viable: requires 2 arguments, but 1 was provided}}
+    //   expected-note@#cwg1388-g {{candidate function [with T = <int>] not viable: requires 2 arguments, but 1 was provided}}
     g<int>(0, 0);
 
     h(0);
     h(0, 0);
     // expected-error@-1 {{no matching function for call to 'h'}}
-    //   expected-note@#dr1388-h {{candidate function [with T = <>, A = int] not viable: requires 1 argument, but 2 were provided}}
+    //   expected-note@#cwg1388-h {{candidate function [with T = <>, A = int] not viable: requires 1 argument, but 2 were provided}}
     h<int>(0, 0);
     h<int, int>(0, 0);
     // expected-error@-1 {{no matching function for call to 'h'}}
-    //   expected-note@#dr1388-h {{candidate template ignored: couldn't infer template argument 'A'}}
+    //   expected-note@#cwg1388-h {{candidate template ignored: couldn't infer template argument 'A'}}
   }
 
   // A non-trailing parameter pack is still a non-deduced context, even though
@@ -570,40 +570,40 @@ namespace dr1388 { // dr1388: 4
   template<typename T, typename U> struct pair {};
   template<typename ...T> struct tuple { typedef char type; }; //
   // cxx98-error@-1 {{variadic templates are a C++11 extension}}
-  template<typename ...T, typename ...U> void f_pair_1(pair<T, U>..., int); // #dr1388-f-1
+  template<typename ...T, typename ...U> void f_pair_1(pair<T, U>..., int); // #cwg1388-f-1
   // cxx98-error@-1 {{variadic templates are a C++11 extension}}
   // cxx98-error@-2 {{variadic templates are a C++11 extension}}
   template<typename ...T, typename U> void f_pair_2(pair<T, char>..., U);
   // cxx98-error@-1 {{variadic templates are a C++11 extension}}
-  template<typename ...T, typename ...U> void f_pair_3(pair<T, U>..., tuple<U...>); // #dr1388-f-3
+  template<typename ...T, typename ...U> void f_pair_3(pair<T, U>..., tuple<U...>); // #cwg1388-f-3
   // cxx98-error@-1 {{variadic templates are a C++11 extension}}
   // cxx98-error@-2 {{variadic templates are a C++11 extension}}
-  template<typename ...T> void f_pair_4(pair<T, char>..., T...); // #dr1388-f-4
+  template<typename ...T> void f_pair_4(pair<T, char>..., T...); // #cwg1388-f-4
   // cxx98-error@-1 {{variadic templates are a C++11 extension}}
   void g(pair<int, char> a, pair<long, char> b, tuple<char, char> c) {
     f_pair_1<int, long>(a, b, 0);
     // expected-error@-1 {{no matching function for call to 'f_pair_1'}}
-    //   expected-note@#dr1388-f-1 {{candidate template ignored: substitution failure [with T = <int, long>]: deduced incomplete pack <(no value), (no value)> for template parameter 'U'}}
+    //   expected-note@#cwg1388-f-1 {{candidate template ignored: substitution failure [with T = <int, long>]: deduced incomplete pack <(no value), (no value)> for template parameter 'U'}}
     f_pair_2<int, long>(a, b, 0);
     f_pair_3<int, long>(a, b, c);
     f_pair_3<int, long>(a, b, tuple<char>());
     // expected-error@-1 {{no matching function for call to 'f_pair_3'}}
-    //   expected-note@#dr1388-f-3 {{candidate template ignored: deduced packs of different lengths for parameter 'U' (<(no value), (no value)> vs. <char>)}}
+    //   expected-note@#cwg1388-f-3 {{candidate template ignored: deduced packs of different lengths for parameter 'U' (<(no value), (no value)> vs. <char>)}}
     f_pair_4<int, long>(a, b, 0, 0L);
     f_pair_4<int, long>(a, b, 0, 0L, "foo");
     // expected-error@-1 {{no matching function for call to 'f_pair_4'}}
-    //   expected-note@#dr1388-f-4 {{candidate template ignored: deduced packs of different lengths for parameter 'T' (<int, long> vs. <int, long, const char *>)}}
+    //   expected-note@#cwg1388-f-4 {{candidate template ignored: deduced packs of different lengths for parameter 'T' (<int, long> vs. <int, long, const char *>)}}
   }
 }
 
-namespace dr1391 { // dr1391: partial
+namespace cwg1391 { // cwg1391: partial
   struct A {}; struct B : A {};
-  template<typename T> struct C { C(int); typename T::error error; }; // #dr1391-C
-  // expected-error@#dr1391-C {{type 'int' cannot be used prior to '::' because it has no members}}
-  //   expected-note@#dr1391-b {{in instantiation of template class 'dr1391::C<int>' requested here}}
-  //   expected-note@#dr1391-b {{while substituting deduced template arguments into function template 'b' [with T = int]}}
-  // expected-error@#dr1391-C {{type 'double' cannot be used prior to '::' because it has no members}}
-  //   expected-note@#dr1391-c {{in instantiation of template class 'dr1391::C<double>' requested here}}
+  template<typename T> struct C { C(int); typename T::error error; }; // #cwg1391-C
+  // expected-error@#cwg1391-C {{type 'int' cannot be used prior to '::' because it has no members}}
+  //   expected-note@#cwg1391-b {{in instantiation of template class 'cwg1391::C<int>' requested here}}
+  //   expected-note@#cwg1391-b {{while substituting deduced template arguments into function template 'b' [with T = int]}}
+  // expected-error@#cwg1391-C {{type 'double' cannot be used prior to '::' because it has no members}}
+  //   expected-note@#cwg1391-c {{in instantiation of template class 'cwg1391::C<double>' requested here}}
   template<typename T> struct D {};
 
   // No deduction is performed for parameters with no deducible template-parameters, therefore types do not need to match.
@@ -644,42 +644,42 @@ namespace dr1391 { // dr1391: partial
   void test_b() {
     b(0, 0); // ok, deduction fails prior to forming a conversion sequence and instantiating C<int>
     // FIXME: The "while substituting" note should point at the overload candidate.
-    b<int>(0, 0); // #dr1391-b
+    b<int>(0, 0); // #cwg1391-b
   }
 
   template<typename T> struct Id { typedef T type; };
   template<typename T> void c(T, typename Id<C<T> >::type);
   void test_c() {
     // Implicit conversion sequences for dependent types are checked later.
-    c(0.0, 0); // #dr1391-c
+    c(0.0, 0); // #cwg1391-c
   }
 
   namespace partial_ordering {
     // FIXME: Second template should be considered more specialized because non-dependent parameter is ignored.
-    template<typename T> int a(T, short) = delete; // #dr1391-a-short
+    template<typename T> int a(T, short) = delete; // #cwg1391-a-short
     // cxx98-error@-1 {{deleted function definitions are a C++11 extension}}
-    template<typename T> int a(T*, char); // #dr1391-a-char
+    template<typename T> int a(T*, char); // #cwg1391-a-char
     int test_a = a((int*)0, 0);
     // expected-error@-1 {{call to 'a' is ambiguous}} FIXME
-    //   expected-note@#dr1391-a-short {{candidate function [with T = int *] has been explicitly deleted}}
-    //   expected-note@#dr1391-a-char {{candidate function [with T = int]}}
+    //   expected-note@#cwg1391-a-short {{candidate function [with T = int *] has been explicitly deleted}}
+    //   expected-note@#cwg1391-a-char {{candidate function [with T = int]}}
 
     // FIXME: Second template should be considered more specialized:
     // deducing #1 from #2 ignores the second P/A pair, so deduction succeeds,
     // deducing #2 from #1 fails to deduce T, so deduction fails.
-    template<typename T> int b(T, int) = delete; // #dr1391-b-int
+    template<typename T> int b(T, int) = delete; // #cwg1391-b-int
     // cxx98-error@-1 {{deleted function definitions are a C++11 extension}}
-    template<typename T, typename U> int b(T*, U); // #dr1391-b-U
+    template<typename T, typename U> int b(T*, U); // #cwg1391-b-U
     int test_b = b((int*)0, 0);
     // expected-error@-1 {{call to 'b' is ambiguous}} FIXME
-    //   expected-note@#dr1391-b-int {{candidate function [with T = int *] has been explicitly deleted}}
-    //   expected-note@#dr1391-b-U {{candidate function [with T = int, U = int]}}
+    //   expected-note@#cwg1391-b-int {{candidate function [with T = int *] has been explicitly deleted}}
+    //   expected-note@#cwg1391-b-U {{candidate function [with T = int, U = int]}}
 
     // Unintended consequences: because partial ordering does not consider
     // explicit template arguments, and deduction from a non-dependent type
     // vacuously succeeds, a non-dependent template is less specialized than
     // anything else!
-    // According to DR1391, this is ambiguous!
+    // According to CWG1391, this is ambiguous!
     template<typename T> int c(int);
     template<typename T> int c(T);
     int test_c1 = c(0); // ok
@@ -687,46 +687,46 @@ namespace dr1391 { // dr1391: partial
   }
 }
 
-namespace dr1394 { // dr1394: 15
+namespace cwg1394 { // cwg1394: 15
 #if __cplusplus >= 201103L
 struct Incomplete;
 Incomplete f(Incomplete) = delete; // well-formed
 #endif
 }
 
-namespace dr1395 { // dr1395: 16
+namespace cwg1395 { // cwg1395: 16
 #if __cplusplus >= 201103L
   template <typename T, typename... U> void f(T, U...);
   template <typename T> void f(T);
   void h(int i) {
-    // This is made ambiguous by dr692, but made valid again by dr1395.
+    // This is made ambiguous by cwg692, but made valid again by cwg1395.
     f(&i);
   }
 #endif
 }
 
-namespace dr1397 { // dr1397: 3.2
+namespace cwg1397 { // cwg1397: 3.2
 #if __cplusplus >= 201103L
 struct A {
 // cxx11-error@-1 {{default member initializer for 'p' needed within definition of enclosing class 'A' outside of member functions}}
-//   cxx11-note@#dr1397-p {{in evaluation of exception specification for 'dr1397::A::A' needed here}}
-//   cxx11-note@#dr1397-p {{default member initializer declared here}}
-  void *p = A{}; // #dr1397-p
+//   cxx11-note@#cwg1397-p {{in evaluation of exception specification for 'cwg1397::A::A' needed here}}
+//   cxx11-note@#cwg1397-p {{default member initializer declared here}}
+  void *p = A{}; // #cwg1397-p
   // since-cxx14-error@-1 {{default member initializer for 'p' needed within definition of enclosing class 'A' outside of member functions}}
   //   since-cxx14-note@-2 {{default member initializer declared here}}
   operator void*() const { return nullptr; }
 };
 #endif
-} // namespace dr1397
+} // namespace cwg1397
 
-namespace dr1399 { // dr1399: dup 1388
-  template<typename ...T> void f(T..., int, T...) {} // #dr1399-f 
+namespace cwg1399 { // cwg1399: dup 1388
+  template<typename ...T> void f(T..., int, T...) {} // #cwg1399-f 
   // cxx98-error@-1 {{variadic templates are a C++11 extension}}
   void g() {
     f(0);
     f<int>(0, 0, 0);
     f(0, 0, 0);
     // expected-error@-1 {{no matching function for call to 'f'}}
-    //   expected-note@#dr1399-f {{candidate template ignored: deduced packs of different lengths for parameter 'T' (<> vs. <int, int>)}}
+    //   expected-note@#cwg1399-f {{candidate template ignored: deduced packs of different lengths for parameter 'T' (<> vs. <int, int>)}}
   }
 }
diff --git a/clang/test/CXX/drs/dr14xx.cpp b/clang/test/CXX/drs/dr14xx.cpp
index ed6dda731fd5..9ff9a68dc13c 100644
--- a/clang/test/CXX/drs/dr14xx.cpp
+++ b/clang/test/CXX/drs/dr14xx.cpp
@@ -6,7 +6,7 @@
 // RUN: %clang_cc1 -std=c++23 %s -verify=expected,since-cxx11,since-cxx14,since-cxx20 -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++2c %s -verify=expected,since-cxx11,since-cxx14,since-cxx20 -fexceptions -fcxx-exceptions -pedantic-errors
 
-namespace dr1413 { // dr1413: 12
+namespace cwg1413 { // cwg1413: 12
   template<int> struct Check {
     typedef int type;
   };
@@ -21,28 +21,28 @@ namespace dr1413 { // dr1413: 12
       // expected-error@-1 {{use of undeclared identifier 'var1'}}
 
       // ok, variable declaration
-      Check<true ? 0 : a>::type *var2; // #dr1413-var2
+      Check<true ? 0 : a>::type *var2; // #cwg1413-var2
       Check<true ? 0 : b>::type *var3;
       // expected-error@-1 {{use of undeclared identifier 'var3'}}
-      //   expected-note@#dr1413-var2 {{'var2' declared here}}
+      //   expected-note@#cwg1413-var2 {{'var2' declared here}}
       Check<true ? 0 : ((void)c, 0)>::type *var4;
       // expected-error@-1 {{use of undeclared identifier 'var4'}}
-      //   expected-note@#dr1413-var2 {{'var2' declared here}}
+      //   expected-note@#cwg1413-var2 {{'var2' declared here}}
 
       // value-dependent because of the implied type-dependent 'this->', not because of 'd'
       Check<true ? 0 : (d(), 0)>::type *var5;
       // expected-error@-1 {{use of undeclared identifier 'var5'}}
-      //   expected-note@#dr1413-var2 {{'var2' declared here}}
+      //   expected-note@#cwg1413-var2 {{'var2' declared here}}
 
       // value-dependent because of the value-dependent '&' operator, not because of 'A::d'
       Check<true ? 0 : (&A::d(), 0)>::type *var5;
       // expected-error@-1 {{use of undeclared identifier 'var5'}}
-      //   expected-note@#dr1413-var2 {{'var2' declared here}}
+      //   expected-note@#cwg1413-var2 {{'var2' declared here}}
     }
   };
 }
 
-namespace dr1423 { // dr1423: 11
+namespace cwg1423 { // cwg1423: 11
 #if __cplusplus >= 201103L
   bool b1 = nullptr;
   // since-cxx11-error@-1 {{cannot initialize a variable of type 'bool' with an rvalue of type 'std::nullptr_t'}}
@@ -55,9 +55,9 @@ namespace dr1423 { // dr1423: 11
 #endif
 }
 
-// dr1425: na abi
+// cwg1425: na abi
 
-namespace dr1432 { // dr1432: 16
+namespace cwg1432 { // cwg1432: 16
 #if __cplusplus >= 201103L
   template<typename T> T declval();
 
@@ -78,7 +78,7 @@ namespace dr1432 { // dr1432: 16
 #endif
 }
 
-namespace dr1443 { // dr1443: yes
+namespace cwg1443 { // cwg1443: yes
 struct A {
   int i;
   A() { void foo(int=i); }
@@ -86,7 +86,7 @@ struct A {
 };
 }
 
-namespace dr1460 { // dr1460: 3.5
+namespace cwg1460 { // cwg1460: 3.5
 #if __cplusplus >= 201103L
   namespace DRExample {
     union A {
@@ -121,23 +121,23 @@ namespace dr1460 { // dr1460: 3.5
   }
 
   union A {};
-  union B { int n; }; // #dr1460-B
+  union B { int n; }; // #cwg1460-B
   union C { int n = 0; };
   struct D { union {}; };
   // expected-error@-1 {{declaration does not declare anything}}
-  struct E { union { int n; }; }; // #dr1460-E
+  struct E { union { int n; }; }; // #cwg1460-E
   struct F { union { int n = 0; }; };
 
   struct X {
     friend constexpr A::A() noexcept;
     friend constexpr B::B() noexcept;
     // cxx11-17-error@-1 {{constexpr declaration of 'B' follows non-constexpr declaration}}
-    //   cxx11-17-note@#dr1460-B {{previous declaration is here}}
+    //   cxx11-17-note@#cwg1460-B {{previous declaration is here}}
     friend constexpr C::C() noexcept;
     friend constexpr D::D() noexcept;
     friend constexpr E::E() noexcept;
     // cxx11-17-error@-1 {{constexpr declaration of 'E' follows non-constexpr declaration}}
-    //   cxx11-17-note@#dr1460-E {{previous declaration is here}}
+    //   cxx11-17-note@#cwg1460-E {{previous declaration is here}}
     friend constexpr F::F() noexcept;
   };
 
@@ -167,63 +167,63 @@ namespace dr1460 { // dr1460: 3.5
       union {
         int n = 0;
       };
-      union { // #dr1460-H-union
+      union { // #cwg1460-H-union
         int m;
       };
       constexpr H() {}
       // cxx11-17-error@-1 {{constexpr constructor that does not initialize all members is a C++20 extension}}
-      //   cxx11-17-note@#dr1460-H-union {{member not initialized by constructor}}
+      //   cxx11-17-note@#cwg1460-H-union {{member not initialized by constructor}}
       constexpr H(bool) : m(1) {}
       constexpr H(char) : n(1) {}
       // cxx11-17-error@-1 {{constexpr constructor that does not initialize all members is a C++20 extension}}
-      //   cxx11-17-note@#dr1460-H-union {{member not initialized by constructor}}
+      //   cxx11-17-note@#cwg1460-H-union {{member not initialized by constructor}}
       constexpr H(double) : m(1), n(1) {}
     };
   }
 
 #if __cplusplus >= 201403L
   template<typename T> constexpr bool check() {
-    T t; // #dr1460-t
+    T t; // #cwg1460-t
     return true;
   }
   static_assert(check<A>(), "");
-  static_assert(check<B>(), ""); // #dr1460-check-B
+  static_assert(check<B>(), ""); // #cwg1460-check-B
   // cxx14-17-error@-1 {{static assertion expression is not an integral constant expression}}
-  //   cxx14-17-note@#dr1460-t {{non-constexpr constructor 'B' cannot be used in a constant expression}}
-  //   cxx14-17-note@#dr1460-check-B {{in call to 'check<dr1460::B>()'}}
-  //   cxx14-17-note@#dr1460-B {{declared here}}
+  //   cxx14-17-note@#cwg1460-t {{non-constexpr constructor 'B' cannot be used in a constant expression}}
+  //   cxx14-17-note@#cwg1460-check-B {{in call to 'check<cwg1460::B>()'}}
+  //   cxx14-17-note@#cwg1460-B {{declared here}}
   static_assert(check<C>(), "");
   static_assert(check<D>(), "");
-  static_assert(check<E>(), ""); // #dr1460-check-E
+  static_assert(check<E>(), ""); // #cwg1460-check-E
   // cxx14-17-error@-1 {{static assertion expression is not an integral constant expression}}
-  //   cxx14-17-note@#dr1460-t {{non-constexpr constructor 'E' cannot be used in a constant expression}}
-  //   cxx14-17-note@#dr1460-check-E {{in call to 'check<dr1460::E>()'}}
-  //   cxx14-17-note@#dr1460-E {{declared here}}
+  //   cxx14-17-note@#cwg1460-t {{non-constexpr constructor 'E' cannot be used in a constant expression}}
+  //   cxx14-17-note@#cwg1460-check-E {{in call to 'check<cwg1460::E>()'}}
+  //   cxx14-17-note@#cwg1460-E {{declared here}}
   static_assert(check<F>(), "");
 #endif
 
   union G {
-    int a = 0; // #dr1460-G-a
+    int a = 0; // #cwg1460-G-a
     int b = 0;
     // expected-error@-1 {{initializing multiple members of union}}
-    //   expected-note@#dr1460-G-a {{previous initialization is here}}
+    //   expected-note@#cwg1460-G-a {{previous initialization is here}}
   };
   union H {
     union {
-      int a = 0; // #dr1460-H-a
+      int a = 0; // #cwg1460-H-a
     };
     union {
       int b = 0;
       // expected-error@-1 {{initializing multiple members of union}}
-      //   expected-note@#dr1460-H-a {{previous initialization is here}}
+      //   expected-note@#cwg1460-H-a {{previous initialization is here}}
     };
   };
   struct I {
     union {
-      int a = 0; // #dr1460-I-a
+      int a = 0; // #cwg1460-I-a
       int b = 0;
       // expected-error@-1 {{initializing multiple members of union}}
-      //   expected-note@#dr1460-I-a {{previous initialization is here}}
+      //   expected-note@#cwg1460-I-a {{previous initialization is here}}
     };
   };
   struct J {
@@ -374,9 +374,9 @@ namespace std {
 } // std
 #endif
 
-namespace dr1467 {  // dr1467: 3.7 c++11
+namespace cwg1467 {  // cwg1467: 3.7 c++11
 #if __cplusplus >= 201103L
-  // Note that the change to [over.best.ics] was partially undone by DR2076;
+  // Note that the change to [over.best.ics] was partially undone by CWG2076;
   // the resulting rule is tested with the tests for that change.
 
   // List-initialization of aggregate from same-type object
@@ -441,12 +441,12 @@ namespace dr1467 {  // dr1467: 3.7 c++11
     X x;
     X x2{x};
 
-    void f1(int); // #dr1467-f1
-    void f1(std::initializer_list<long>) = delete; // #dr1467-f1-deleted
+    void f1(int); // #cwg1467-f1
+    void f1(std::initializer_list<long>) = delete; // #cwg1467-f1-deleted
     void g1() { f1({42}); }
     // since-cxx11-error@-1 {{call to deleted function 'f1'}}
-    //   since-cxx11-note@#dr1467-f1 {{candidate function}}
-    //   since-cxx11-note@#dr1467-f1-deleted {{candidate function has been explicitly deleted}}
+    //   since-cxx11-note@#cwg1467-f1 {{candidate function}}
+    //   since-cxx11-note@#cwg1467-f1-deleted {{candidate function has been explicitly deleted}}
 
     template <class T, class U>
     struct Pair {
@@ -456,12 +456,12 @@ namespace dr1467 {  // dr1467: 3.7 c++11
       String(const char *);
     };
 
-    void f2(Pair<const char *, const char *>); // #dr1467-f2
-    void f2(std::initializer_list<String>) = delete; // #dr1467-f2-deleted
+    void f2(Pair<const char *, const char *>); // #cwg1467-f2
+    void f2(std::initializer_list<String>) = delete; // #cwg1467-f2-deleted
     void g2() { f2({"foo", "bar"}); }
     // since-cxx11-error@-1 {{call to deleted function 'f2'}}
-    //   since-cxx11-note@#dr1467-f2 {{candidate function}}
-    //   since-cxx11-note@#dr1467-f2-deleted {{candidate function has been explicitly deleted}}
+    //   since-cxx11-note@#cwg1467-f2 {{candidate function}}
+    //   since-cxx11-note@#cwg1467-f2-deleted {{candidate function has been explicitly deleted}}
   } // dr_example
 
   namespace nonaggregate {
@@ -522,88 +522,88 @@ namespace dr1467 {  // dr1467: 3.7 c++11
   // When the array size is 4 the call will attempt to bind an lvalue to an
   // rvalue and fail. Therefore #2 will be called. (rsmith will bring this
   // issue to CWG)
-  void f(const char(&&)[4]);              // #dr1467-f-char-4
-  void f(const char(&&)[5]) = delete;     // #dr1467-f-char-5
-  void f(const wchar_t(&&)[4]);           // #dr1467-f-wchar-4
-  void f(const wchar_t(&&)[5]) = delete;  // #dr1467-f-wchar-5
+  void f(const char(&&)[4]);              // #cwg1467-f-char-4
+  void f(const char(&&)[5]) = delete;     // #cwg1467-f-char-5
+  void f(const wchar_t(&&)[4]);           // #cwg1467-f-wchar-4
+  void f(const wchar_t(&&)[5]) = delete;  // #cwg1467-f-wchar-5
 #if __cplusplus >= 202002L
-  void f2(const char8_t(&&)[4]);          // #dr1467-f2-char8-4
-  void f2(const char8_t(&&)[5]) = delete; // #dr1467-f2-char8-5
+  void f2(const char8_t(&&)[4]);          // #cwg1467-f2-char8-4
+  void f2(const char8_t(&&)[5]) = delete; // #cwg1467-f2-char8-5
 #endif
-  void f(const char16_t(&&)[4]);          // #dr1467-f-char16-4
-  void f(const char16_t(&&)[5]) = delete; // #dr1467-f-char16-5
-  void f(const char32_t(&&)[4]);          // #dr1467-f-char32-4
-  void f(const char32_t(&&)[5]) = delete; // #dr1467-f-char32-5
+  void f(const char16_t(&&)[4]);          // #cwg1467-f-char16-4
+  void f(const char16_t(&&)[5]) = delete; // #cwg1467-f-char16-5
+  void f(const char32_t(&&)[4]);          // #cwg1467-f-char32-4
+  void f(const char32_t(&&)[5]) = delete; // #cwg1467-f-char32-5
   void g() {
     f({"abc"});
     // since-cxx11-error@-1 {{call to deleted function 'f'}}
-    //   since-cxx11-note@#dr1467-f-char-5 {{candidate function has been explicitly deleted}}
-    //   since-cxx11-note@#dr1467-f-char-4 {{candidate function not viable: expects an rvalue for 1st argument}}
-    //   since-cxx11-note@#dr1467-f-wchar-4 {{candidate function not viable: no known conversion from 'const char[4]' to 'const wchar_t' for 1st argument}}
-    //   since-cxx11-note@#dr1467-f-wchar-5 {{candidate function not viable: no known conversion from 'const char[4]' to 'const wchar_t' for 1st argument}}
-    //   since-cxx11-note@#dr1467-f-char16-4 {{candidate function not viable: no known conversion from 'const char[4]' to 'const char16_t' for 1st argument}}
-    //   since-cxx11-note@#dr1467-f-char16-5 {{candidate function not viable: no known conversion from 'const char[4]' to 'const char16_t' for 1st argument}}
-    //   since-cxx11-note@#dr1467-f-char32-4 {{candidate function not viable: no known conversion from 'const char[4]' to 'const char32_t' for 1st argument}}
-    //   since-cxx11-note@#dr1467-f-char32-5 {{candidate function not viable: no known conversion from 'const char[4]' to 'const char32_t' for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-char-5 {{candidate function has been explicitly deleted}}
+    //   since-cxx11-note@#cwg1467-f-char-4 {{candidate function not viable: expects an rvalue for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-wchar-4 {{candidate function not viable: no known conversion from 'const char[4]' to 'const wchar_t' for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-wchar-5 {{candidate function not viable: no known conversion from 'const char[4]' to 'const wchar_t' for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-char16-4 {{candidate function not viable: no known conversion from 'const char[4]' to 'const char16_t' for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-char16-5 {{candidate function not viable: no known conversion from 'const char[4]' to 'const char16_t' for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-char32-4 {{candidate function not viable: no known conversion from 'const char[4]' to 'const char32_t' for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-char32-5 {{candidate function not viable: no known conversion from 'const char[4]' to 'const char32_t' for 1st argument}}
     f({((("abc")))});
     // since-cxx11-error@-1 {{call to deleted function 'f'}}
-    //   since-cxx11-note@#dr1467-f-char-5 {{candidate function has been explicitly deleted}}
-    //   since-cxx11-note@#dr1467-f-char-4 {{candidate function not viable: expects an rvalue for 1st argument}}
-    //   since-cxx11-note@#dr1467-f-wchar-4 {{candidate function not viable: no known conversion from 'const char[4]' to 'const wchar_t' for 1st argument}}
-    //   since-cxx11-note@#dr1467-f-wchar-5 {{candidate function not viable: no known conversion from 'const char[4]' to 'const wchar_t' for 1st argument}}
-    //   since-cxx11-note@#dr1467-f-char16-4 {{candidate function not viable: no known conversion from 'const char[4]' to 'const char16_t' for 1st argument}}
-    //   since-cxx11-note@#dr1467-f-char16-5 {{candidate function not viable: no known conversion from 'const char[4]' to 'const char16_t' for 1st argument}}
-    //   since-cxx11-note@#dr1467-f-char32-4 {{candidate function not viable: no known conversion from 'const char[4]' to 'const char32_t' for 1st argument}}
-    //   since-cxx11-note@#dr1467-f-char32-5 {{candidate function not viable: no known conversion from 'const char[4]' to 'const char32_t' for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-char-5 {{candidate function has been explicitly deleted}}
+    //   since-cxx11-note@#cwg1467-f-char-4 {{candidate function not viable: expects an rvalue for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-wchar-4 {{candidate function not viable: no known conversion from 'const char[4]' to 'const wchar_t' for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-wchar-5 {{candidate function not viable: no known conversion from 'const char[4]' to 'const wchar_t' for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-char16-4 {{candidate function not viable: no known conversion from 'const char[4]' to 'const char16_t' for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-char16-5 {{candidate function not viable: no known conversion from 'const char[4]' to 'const char16_t' for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-char32-4 {{candidate function not viable: no known conversion from 'const char[4]' to 'const char32_t' for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-char32-5 {{candidate function not viable: no known conversion from 'const char[4]' to 'const char32_t' for 1st argument}}
     f({L"abc"});
     // since-cxx11-error@-1 {{call to deleted function 'f'}}
-    //   since-cxx11-note@#dr1467-f-wchar-5 {{candidate function has been explicitly deleted}}
-    //   since-cxx11-note@#dr1467-f-char-4 {{candidate function not viable: no known conversion from 'const wchar_t[4]' to 'const char' for 1st argument}}
-    //   since-cxx11-note@#dr1467-f-char-5 {{candidate function not viable: no known conversion from 'const wchar_t[4]' to 'const char' for 1st argument}}
-    //   since-cxx11-note@#dr1467-f-wchar-4 {{candidate function not viable: expects an rvalue for 1st argument}}
-    //   since-cxx11-note@#dr1467-f-char16-4 {{candidate function not viable: no known conversion from 'const wchar_t[4]' to 'const char16_t' for 1st argument}}
-    //   since-cxx11-note@#dr1467-f-char16-5 {{candidate function not viable: no known conversion from 'const wchar_t[4]' to 'const char16_t' for 1st argument}}
-    //   since-cxx11-note@#dr1467-f-char32-4 {{candidate function not viable: no known conversion from 'const wchar_t[4]' to 'const char32_t' for 1st argument}}
-    //   since-cxx11-note@#dr1467-f-char32-5 {{candidate function not viable: no known conversion from 'const wchar_t[4]' to 'const char32_t' for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-wchar-5 {{candidate function has been explicitly deleted}}
+    //   since-cxx11-note@#cwg1467-f-char-4 {{candidate function not viable: no known conversion from 'const wchar_t[4]' to 'const char' for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-char-5 {{candidate function not viable: no known conversion from 'const wchar_t[4]' to 'const char' for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-wchar-4 {{candidate function not viable: expects an rvalue for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-char16-4 {{candidate function not viable: no known conversion from 'const wchar_t[4]' to 'const char16_t' for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-char16-5 {{candidate function not viable: no known conversion from 'const wchar_t[4]' to 'const char16_t' for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-char32-4 {{candidate function not viable: no known conversion from 'const wchar_t[4]' to 'const char32_t' for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-char32-5 {{candidate function not viable: no known conversion from 'const wchar_t[4]' to 'const char32_t' for 1st argument}}
 #if __cplusplus >= 202002L
     f2({u8"abc"});
     // since-cxx20-error@-1 {{call to deleted function 'f2'}}
-    //   since-cxx20-note@#dr1467-f2-char8-5 {{candidate function has been explicitly deleted}}
-    //   since-cxx20-note@#dr1467-f2-char8-4 {{candidate function not viable: expects an rvalue for 1st argument}}
+    //   since-cxx20-note@#cwg1467-f2-char8-5 {{candidate function has been explicitly deleted}}
+    //   since-cxx20-note@#cwg1467-f2-char8-4 {{candidate function not viable: expects an rvalue for 1st argument}}
 #endif
     f({uR"(abc)"});
     // since-cxx11-error@-1 {{call to deleted function 'f'}}
-    //   since-cxx11-note@#dr1467-f-char16-5 {{candidate function has been explicitly deleted}}
-    //   since-cxx11-note@#dr1467-f-char-4 {{candidate function not viable: no known conversion from 'const char16_t[4]' to 'const char' for 1st argument}}
-    //   since-cxx11-note@#dr1467-f-char-5 {{candidate function not viable: no known conversion from 'const char16_t[4]' to 'const char' for 1st argument}}
-    //   since-cxx11-note@#dr1467-f-wchar-4 {{candidate function not viable: no known conversion from 'const char16_t[4]' to 'const wchar_t' for 1st argument}}
-    //   since-cxx11-note@#dr1467-f-wchar-5 {{candidate function not viable: no known conversion from 'const char16_t[4]' to 'const wchar_t' for 1st argument}}
-    //   since-cxx11-note@#dr1467-f-char16-4 {{candidate function not viable: expects an rvalue for 1st argument}}
-    //   since-cxx11-note@#dr1467-f-char32-4 {{candidate function not viable: no known conversion from 'const char16_t[4]' to 'const char32_t' for 1st argument}}
-    //   since-cxx11-note@#dr1467-f-char32-5 {{candidate function not viable: no known conversion from 'const char16_t[4]' to 'const char32_t' for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-char16-5 {{candidate function has been explicitly deleted}}
+    //   since-cxx11-note@#cwg1467-f-char-4 {{candidate function not viable: no known conversion from 'const char16_t[4]' to 'const char' for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-char-5 {{candidate function not viable: no known conversion from 'const char16_t[4]' to 'const char' for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-wchar-4 {{candidate function not viable: no known conversion from 'const char16_t[4]' to 'const wchar_t' for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-wchar-5 {{candidate function not viable: no known conversion from 'const char16_t[4]' to 'const wchar_t' for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-char16-4 {{candidate function not viable: expects an rvalue for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-char32-4 {{candidate function not viable: no known conversion from 'const char16_t[4]' to 'const char32_t' for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-char32-5 {{candidate function not viable: no known conversion from 'const char16_t[4]' to 'const char32_t' for 1st argument}}
     f({(UR"(abc)")});
     // since-cxx11-error@-1 {{call to deleted function 'f'}}
-    //   since-cxx11-note@#dr1467-f-char32-5 {{candidate function has been explicitly deleted}}
-    //   since-cxx11-note@#dr1467-f-char-4 {{candidate function not viable: no known conversion from 'const char32_t[4]' to 'const char' for 1st argument}}
-    //   since-cxx11-note@#dr1467-f-char-5 {{candidate function not viable: no known conversion from 'const char32_t[4]' to 'const char' for 1st argument}}
-    //   since-cxx11-note@#dr1467-f-wchar-4 {{candidate function not viable: no known conversion from 'const char32_t[4]' to 'const wchar_t' for 1st argument}}
-    //   since-cxx11-note@#dr1467-f-wchar-5 {{candidate function not viable: no known conversion from 'const char32_t[4]' to 'const wchar_t' for 1st argument}}
-    //   since-cxx11-note@#dr1467-f-char16-4 {{candidate function not viable: no known conversion from 'const char32_t[4]' to 'const char16_t' for 1st argument}}
-    //   since-cxx11-note@#dr1467-f-char16-5 {{candidate function not viable: no known conversion from 'const char32_t[4]' to 'const char16_t' for 1st argument}}
-    //   since-cxx11-note@#dr1467-f-char32-4 {{candidate function not viable: expects an rvalue for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-char32-5 {{candidate function has been explicitly deleted}}
+    //   since-cxx11-note@#cwg1467-f-char-4 {{candidate function not viable: no known conversion from 'const char32_t[4]' to 'const char' for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-char-5 {{candidate function not viable: no known conversion from 'const char32_t[4]' to 'const char' for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-wchar-4 {{candidate function not viable: no known conversion from 'const char32_t[4]' to 'const wchar_t' for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-wchar-5 {{candidate function not viable: no known conversion from 'const char32_t[4]' to 'const wchar_t' for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-char16-4 {{candidate function not viable: no known conversion from 'const char32_t[4]' to 'const char16_t' for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-char16-5 {{candidate function not viable: no known conversion from 'const char32_t[4]' to 'const char16_t' for 1st argument}}
+    //   since-cxx11-note@#cwg1467-f-char32-4 {{candidate function not viable: expects an rvalue for 1st argument}}
   }
   } // namespace StringLiterals
 #endif
-} // dr1467
+} // cwg1467
 
-namespace dr1479 { // dr1479: 3.1
+namespace cwg1479 { // cwg1479: 3.1
 #if __cplusplus >= 201103L
   int operator"" _a(const char*, std::size_t = 0);
   // since-cxx11-error@-1 {{literal operator cannot have a default argument}}
 #endif
 }
 
-namespace dr1482 { // dr1482: 3.0
+namespace cwg1482 { // cwg1482: 3.0
                    // NB: sup 2516, test reused there
 #if __cplusplus >= 201103L
 template <typename T> struct S {
@@ -612,11 +612,11 @@ template <typename T> struct S {
 enum E2 : S<E2>::I { e };
 // since-cxx11-error@-1 {{use of undeclared identifier 'E2'}}
 #endif
-} // namespace dr1482
+} // namespace cwg1482
 
-namespace dr1487 { // dr1487: 3.3
+namespace cwg1487 { // cwg1487: 3.3
 #if __cplusplus >= 201103L
-struct A { // #dr1482-A
+struct A { // #cwg1482-A
   struct B {
     using A::A;
     // since-cxx11-error@-1 {{using declaration refers into 'A::', which is not a base class of 'B'}}
@@ -624,7 +624,7 @@ struct A { // #dr1482-A
 
   struct C : A {
   // since-cxx11-error@-1 {{base class has incomplete type}}
-  //   since-cxx11-note@#dr1482-A {{definition of 'dr1487::A' is not complete until the closing '}'}}
+  //   since-cxx11-note@#cwg1482-A {{definition of 'cwg1487::A' is not complete until the closing '}'}}
     using A::A;
     // since-cxx11-error@-1 {{using declaration refers into 'A::', which is not a base class of 'C'}}
   };
@@ -636,9 +636,9 @@ struct D : A {
   using A::A;
 };
 #endif
-} // namespace dr1487
+} // namespace cwg1487
 
-namespace dr1490 {  // dr1490: 3.7 c++11
+namespace cwg1490 {  // cwg1490: 3.7 c++11
 #if __cplusplus >= 201103L
   // List-initialization from a string literal
 
@@ -646,51 +646,51 @@ namespace dr1490 {  // dr1490: 3.7 c++11
   std::initializer_list<char>{"abc"};
   // since-cxx11-error@-1 {{expected unqualified-id}}}
 #endif
-} // dr1490
+} // cwg1490
 
-namespace dr1495 { // dr1495: 4
+namespace cwg1495 { // cwg1495: 4
 #if __cplusplus >= 201103L
   // Deduction succeeds in both directions.
-  template<typename T, typename U> struct A {}; // #dr1495-A
+  template<typename T, typename U> struct A {}; // #cwg1495-A
   template<typename T, typename U> struct A<U, T> {};
   // since-cxx11-error@-1 {{class template partial specialization is not more specialized than the primary template}}
-  //   since-cxx11-note@#dr1495-A {{template is declared here}}
+  //   since-cxx11-note@#cwg1495-A {{template is declared here}}
 
   // Primary template is more specialized.
-  template<typename, typename...> struct B {}; // #dr1495-B
+  template<typename, typename...> struct B {}; // #cwg1495-B
   template<typename ...Ts> struct B<Ts...> {};
   // since-cxx11-error@-1 {{class template partial specialization is not more specialized than the primary template}}
-  //   since-cxx11-note@#dr1495-B {{template is declared here}}
+  //   since-cxx11-note@#cwg1495-B {{template is declared here}}
 
   // Deduction fails in both directions.
-  template<int, typename, typename ...> struct C {}; // #dr1495-C
+  template<int, typename, typename ...> struct C {}; // #cwg1495-C
   template<typename ...Ts> struct C<0, Ts...> {};
   // since-cxx11-error@-1 {{class template partial specialization is not more specialized than the primary template}}
-  //   since-cxx11-note@#dr1495-C {{template is declared here}}
+  //   since-cxx11-note@#cwg1495-C {{template is declared here}}
 
 #if __cplusplus >= 201402L
   // Deduction succeeds in both directions.
-  template<typename T, typename U> int a; // #dr1495-a
+  template<typename T, typename U> int a; // #cwg1495-a
   template<typename T, typename U> int a<U, T>;
   // since-cxx14-error@-1 {{variable template partial specialization is not more specialized than the primary template}}
-  //   since-cxx14-note@#dr1495-a {{template is declared here}}
+  //   since-cxx14-note@#cwg1495-a {{template is declared here}}
 
   // Primary template is more specialized.
-  template<typename, typename...> int b; // #dr1495-b
+  template<typename, typename...> int b; // #cwg1495-b
   template<typename ...Ts> int b<Ts...>;
   // since-cxx14-error@-1 {{variable template partial specialization is not more specialized than the primary template}}
-  //   since-cxx14-note@#dr1495-b {{template is declared here}}
+  //   since-cxx14-note@#cwg1495-b {{template is declared here}}
 
   // Deduction fails in both directions.
-  template<int, typename, typename ...> int c; // #dr1495-c
+  template<int, typename, typename ...> int c; // #cwg1495-c
   template<typename ...Ts> int c<0, Ts...>;
   // since-cxx14-error@-1 {{variable template partial specialization is not more specialized than the primary template}}
-  //   since-cxx14-note@#dr1495-c {{template is declared here}}
+  //   since-cxx14-note@#cwg1495-c {{template is declared here}}
 #endif
 #endif
 }
 
-namespace dr1496 { // dr1496: no
+namespace cwg1496 { // cwg1496: no
 #if __cplusplus >= 201103L
 struct A {
     A() = delete;
diff --git a/clang/test/CXX/drs/dr15xx.cpp b/clang/test/CXX/drs/dr15xx.cpp
index 6e3ad41c748f..21a392a5141e 100644
--- a/clang/test/CXX/drs/dr15xx.cpp
+++ b/clang/test/CXX/drs/dr15xx.cpp
@@ -6,7 +6,7 @@
 // RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx23,since-cxx20,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify=expected,since-cxx23,since-cxx20,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors
 
-namespace dr1512 { // dr1512: 4
+namespace cwg1512 { // cwg1512: 4
   void f(char *p) {
     if (p > 0) {}
     // expected-error@-1 {{ordered comparison between pointer and zero ('char *' and 'int')}}
@@ -33,11 +33,11 @@ namespace dr1512 { // dr1512: 4
   template<typename A, typename B, typename C> void composite_pointer_type_is_ord() {
     composite_pointer_type_is_base<A, B, C>();
 
-    typedef __typeof(val<A>() < val<B>()) cmp; // #dr1512-lt 
-    // since-cxx17-warning@#dr1512-lt {{ordered comparison of function pointers ('int (*)() noexcept' and 'int (*)()')}}
-    //   since-cxx17-note@#dr1512-noexcept-1st {{in instantiation of function template specialization 'dr1512::composite_pointer_type_is_ord<int (*)() noexcept, int (*)(), int (*)()>' requested here}}
-    // since-cxx17-warning@#dr1512-lt {{ordered comparison of function pointers ('int (*)()' and 'int (*)() noexcept')}}
-    //   since-cxx17-note@#dr1512-noexcept-2nd {{in instantiation of function template specialization 'dr1512::composite_pointer_type_is_ord<int (*)(), int (*)() noexcept, int (*)()>' requested here}}
+    typedef __typeof(val<A>() < val<B>()) cmp; // #cwg1512-lt 
+    // since-cxx17-warning@#cwg1512-lt {{ordered comparison of function pointers ('int (*)() noexcept' and 'int (*)()')}}
+    //   since-cxx17-note@#cwg1512-noexcept-1st {{in instantiation of function template specialization 'cwg1512::composite_pointer_type_is_ord<int (*)() noexcept, int (*)(), int (*)()>' requested here}}
+    // since-cxx17-warning@#cwg1512-lt {{ordered comparison of function pointers ('int (*)()' and 'int (*)() noexcept')}}
+    //   since-cxx17-note@#cwg1512-noexcept-2nd {{in instantiation of function template specialization 'cwg1512::composite_pointer_type_is_ord<int (*)(), int (*)() noexcept, int (*)()>' requested here}}
     typedef __typeof(val<A>() <= val<B>()) cmp;
     // since-cxx17-warning@-1 {{ordered comparison of function pointers ('int (*)() noexcept' and 'int (*)()')}}
     // since-cxx17-warning@-2 {{ordered comparison of function pointers ('int (*)()' and 'int (*)() noexcept')}}
@@ -95,8 +95,8 @@ namespace dr1512 { // dr1512: 4
     // since-cxx20-warning@-1 {{volatile-qualified return type 'volatile int' is deprecated}}
 
 #if __cplusplus >= 201703L
-    composite_pointer_type_is_ord<int (*)() noexcept, int (*)(), int (*)()>(); // #dr1512-noexcept-1st
-    composite_pointer_type_is_ord<int (*)(), int (*)() noexcept, int (*)()>(); // #dr1512-noexcept-2nd
+    composite_pointer_type_is_ord<int (*)() noexcept, int (*)(), int (*)()>(); // #cwg1512-noexcept-1st
+    composite_pointer_type_is_ord<int (*)(), int (*)() noexcept, int (*)()>(); // #cwg1512-noexcept-2nd
     composite_pointer_type_is_unord<int (A::*)() noexcept, int (A::*)(), int (A::*)()>();
     composite_pointer_type_is_unord<int (A::*)(), int (A::*)() noexcept, int (A::*)()>();
     // FIXME: This looks like a standard defect; these should probably all have type 'int (B::*)()'.
@@ -129,7 +129,7 @@ namespace dr1512 { // dr1512: 4
   }
 
 #if __cplusplus >= 201103L
-  template<typename T> struct Wrap { operator T(); }; // #dr1512-Wrap
+  template<typename T> struct Wrap { operator T(); }; // #cwg1512-Wrap
   void test_overload() {
     using nullptr_t = decltype(nullptr);
     void(Wrap<nullptr_t>() == Wrap<nullptr_t>());
@@ -143,75 +143,75 @@ namespace dr1512 { // dr1512: 4
     void(Wrap<nullptr_t>() >= Wrap<nullptr_t>());
     // since-cxx11-error@-1 {{invalid operands to binary expression ('Wrap<nullptr_t>' (aka 'Wrap<std::nullptr_t>') and 'Wrap<nullptr_t>' (aka 'Wrap<std::nullptr_t>'))}}
 
-    // Under dr1213, this is ill-formed: we select the builtin operator<(int*, int*)
+    // Under cwg1213, this is ill-formed: we select the builtin operator<(int*, int*)
     // but then only convert as far as 'nullptr_t', which we then can't convert to 'int*'.
     void(Wrap<nullptr_t>() == Wrap<int*>());
     void(Wrap<nullptr_t>() != Wrap<int*>());
     void(Wrap<nullptr_t>() < Wrap<int*>());
     // since-cxx11-error@-1 {{invalid operands to binary expression ('Wrap<nullptr_t>' (aka 'Wrap<std::nullptr_t>') and 'Wrap<int *>')}}
-    //   since-cxx11-note@#dr1512-Wrap {{first operand was implicitly converted to type 'std::nullptr_t'}}
-    //   since-cxx11-note@#dr1512-Wrap {{second operand was implicitly converted to type 'int *'}}
+    //   since-cxx11-note@#cwg1512-Wrap {{first operand was implicitly converted to type 'std::nullptr_t'}}
+    //   since-cxx11-note@#cwg1512-Wrap {{second operand was implicitly converted to type 'int *'}}
     void(Wrap<nullptr_t>() > Wrap<int*>());
     // since-cxx11-error@-1 {{invalid operands}}
-    //   since-cxx11-note@#dr1512-Wrap {{first operand was implicitly converted to type 'std::nullptr_t'}}
-    //   since-cxx11-note@#dr1512-Wrap{{second operand was implicitly converted to type 'int *'}}
+    //   since-cxx11-note@#cwg1512-Wrap {{first operand was implicitly converted to type 'std::nullptr_t'}}
+    //   since-cxx11-note@#cwg1512-Wrap{{second operand was implicitly converted to type 'int *'}}
     void(Wrap<nullptr_t>() <= Wrap<int*>());
     // since-cxx11-error@-1 {{invalid operands}}
-    //   since-cxx11-note@#dr1512-Wrap {{first operand was implicitly converted to type 'std::nullptr_t'}}
-    //   since-cxx11-note@#dr1512-Wrap {{second operand was implicitly converted to type 'int *'}}
+    //   since-cxx11-note@#cwg1512-Wrap {{first operand was implicitly converted to type 'std::nullptr_t'}}
+    //   since-cxx11-note@#cwg1512-Wrap {{second operand was implicitly converted to type 'int *'}}
     void(Wrap<nullptr_t>() >= Wrap<int*>());
     // since-cxx11-error@-1 {{invalid operands}}
-    //   since-cxx11-note@#dr1512-Wrap {{first operand was implicitly converted to type 'std::nullptr_t'}}
-    //   since-cxx11-note@#dr1512-Wrap {{second operand was implicitly converted to type 'int *'}}
+    //   since-cxx11-note@#cwg1512-Wrap {{first operand was implicitly converted to type 'std::nullptr_t'}}
+    //   since-cxx11-note@#cwg1512-Wrap {{second operand was implicitly converted to type 'int *'}}
   }
 #endif
 }
 
-namespace dr1514 { // dr1514: 11
+namespace cwg1514 { // cwg1514: 11
 #if __cplusplus >= 201103L
   struct S {
-    enum E : int {}; // #dr1514-E
+    enum E : int {}; // #cwg1514-E
     enum E : int {};
     // since-cxx11-error@-1 {{redefinition of 'E'}}
-    //   since-cxx11-note@#dr1514-E {{previous definition is here}}
+    //   since-cxx11-note@#cwg1514-E {{previous definition is here}}
   };
   S::E se; // OK, complete type, not zero-width bitfield.
 
-  // The behavior in other contexts is superseded by DR1966.
+  // The behavior in other contexts is superseded by CWG1966.
 #endif
 }
 
-namespace dr1518 { // dr1518: 4
+namespace cwg1518 { // cwg1518: 4
 #if __cplusplus >= 201103L
-struct Z0 { // #dr1518-Z0
-  explicit Z0() = default; // #dr1518-Z0-ctor
+struct Z0 { // #cwg1518-Z0
+  explicit Z0() = default; // #cwg1518-Z0-ctor
 };
-struct Z { // #dr1518-Z
-  explicit Z(); // #dr1518-Z-ctor
-  explicit Z(int); // #dr1518-Z-int
-  explicit Z(int, int); // #dr1518-Z-int-int
+struct Z { // #cwg1518-Z
+  explicit Z(); // #cwg1518-Z-ctor
+  explicit Z(int); // #cwg1518-Z-int
+  explicit Z(int, int); // #cwg1518-Z-int-int
 };
-template <class T> int Eat(T); // #dr1518-Eat
+template <class T> int Eat(T); // #cwg1518-Eat
 Z0 a;
 Z0 b{};
 Z0 c = {};
 // since-cxx11-error@-1 {{chosen constructor is explicit in copy-initialization}}
-//   since-cxx11-note@#dr1518-Z0-ctor {{explicit constructor declared here}}
+//   since-cxx11-note@#cwg1518-Z0-ctor {{explicit constructor declared here}}
 int i = Eat<Z0>({});
 // since-cxx11-error@-1 {{no matching function for call to 'Eat'}}
-//   since-cxx11-note@#dr1518-Eat {{candidate function template not viable: cannot convert initializer list argument to 'Z0'}}
+//   since-cxx11-note@#cwg1518-Eat {{candidate function template not viable: cannot convert initializer list argument to 'Z0'}}
 
 Z c2 = {};
 // since-cxx11-error@-1 {{chosen constructor is explicit in copy-initialization}}
-//   since-cxx11-note@#dr1518-Z-ctor {{explicit constructor declared here}}
+//   since-cxx11-note@#cwg1518-Z-ctor {{explicit constructor declared here}}
 int i2 = Eat<Z>({});
 // since-cxx11-error@-1 {{no matching function for call to 'Eat'}}
-//   since-cxx11-note@#dr1518-Eat {{candidate function template not viable: cannot convert initializer list argument to 'Z'}}
+//   since-cxx11-note@#cwg1518-Eat {{candidate function template not viable: cannot convert initializer list argument to 'Z'}}
 Z a1 = 1;
 // since-cxx11-error@-1 {{no viable conversion from 'int' to 'Z'}}
-//   since-cxx11-note@#dr1518-Z {{candidate constructor (the implicit copy constructor) not viable: no known conversion from 'int' to 'const Z &' for 1st argument}}
-//   since-cxx11-note@#dr1518-Z {{candidate constructor (the implicit move constructor) not viable: no known conversion from 'int' to 'Z &&' for 1st argument}}
-//   since-cxx11-note@#dr1518-Z-int {{explicit constructor is not a candidate}}
+//   since-cxx11-note@#cwg1518-Z {{candidate constructor (the implicit copy constructor) not viable: no known conversion from 'int' to 'const Z &' for 1st argument}}
+//   since-cxx11-note@#cwg1518-Z {{candidate constructor (the implicit move constructor) not viable: no known conversion from 'int' to 'Z &&' for 1st argument}}
+//   since-cxx11-note@#cwg1518-Z-int {{explicit constructor is not a candidate}}
 Z a3 = Z(1);
 Z a2(1);
 Z *p = new Z(1);
@@ -219,129 +219,129 @@ Z a4 = (Z)1;
 Z a5 = static_cast<Z>(1);
 Z a6 = {4, 3};
 // since-cxx11-error@-1 {{chosen constructor is explicit in copy-initialization}}
-//   since-cxx11-note@#dr1518-Z-int-int {{explicit constructor declared here}}
+//   since-cxx11-note@#cwg1518-Z-int-int {{explicit constructor declared here}}
 
-struct UserProvidedBaseCtor { // #dr1518-U
+struct UserProvidedBaseCtor { // #cwg1518-U
   UserProvidedBaseCtor() {}
 };
-struct DoesntInheritCtor : UserProvidedBaseCtor { // #dr1518-D-U
+struct DoesntInheritCtor : UserProvidedBaseCtor { // #cwg1518-D-U
   int x;
 };
 DoesntInheritCtor I{{}, 42};
 // cxx11-14-error@-1 {{no matching constructor for initialization of 'DoesntInheritCtor'}}
-//   cxx11-14-note@#dr1518-D-U {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 2 were provided}}
-//   cxx11-14-note@#dr1518-D-U {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 2 were provided}}
-//   cxx11-14-note@#dr1518-D-U {{candidate constructor (the implicit default constructor) not viable: requires 0 arguments, but 2 were provided}}
+//   cxx11-14-note@#cwg1518-D-U {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 2 were provided}}
+//   cxx11-14-note@#cwg1518-D-U {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 2 were provided}}
+//   cxx11-14-note@#cwg1518-D-U {{candidate constructor (the implicit default constructor) not viable: requires 0 arguments, but 2 were provided}}
 
-struct BaseCtor { BaseCtor() = default; }; // #dr1518-BC
-struct InheritsCtor : BaseCtor { // #dr1518-I
-  using BaseCtor::BaseCtor;      // #dr1518-I-using
+struct BaseCtor { BaseCtor() = default; }; // #cwg1518-BC
+struct InheritsCtor : BaseCtor { // #cwg1518-I
+  using BaseCtor::BaseCtor;      // #cwg1518-I-using
   int x;
 };
 InheritsCtor II = {{}, 42};
 // since-cxx11-error@-1 {{no matching constructor for initialization of 'InheritsCtor'}}
-//   since-cxx11-note@#dr1518-BC {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 2 were provided}}
-//   since-cxx11-note@#dr1518-I-using {{constructor from base class 'BaseCtor' inherited here}}
-//   since-cxx11-note@#dr1518-BC {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 2 were provided}}
-//   since-cxx11-note@#dr1518-I-using {{constructor from base class 'BaseCtor' inherited here}}
-//   since-cxx11-note@#dr1518-I {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 2 were provided}}
-//   since-cxx11-note@#dr1518-I {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 2 were provided}}
-//   since-cxx11-note@#dr1518-I {{candidate constructor (the implicit default constructor) not viable: requires 0 arguments, but 2 were provided}}
+//   since-cxx11-note@#cwg1518-BC {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 2 were provided}}
+//   since-cxx11-note@#cwg1518-I-using {{constructor from base class 'BaseCtor' inherited here}}
+//   since-cxx11-note@#cwg1518-BC {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 2 were provided}}
+//   since-cxx11-note@#cwg1518-I-using {{constructor from base class 'BaseCtor' inherited here}}
+//   since-cxx11-note@#cwg1518-I {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 2 were provided}}
+//   since-cxx11-note@#cwg1518-I {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 2 were provided}}
+//   since-cxx11-note@#cwg1518-I {{candidate constructor (the implicit default constructor) not viable: requires 0 arguments, but 2 were provided}}
 
 namespace std_example {
   struct A {
-    explicit A() = default; // #dr1518-A
+    explicit A() = default; // #cwg1518-A
   };
 
   struct B : A {
-    explicit B() = default; // #dr1518-B
+    explicit B() = default; // #cwg1518-B
   };
 
   struct C {
-    explicit C(); // #dr1518-C
+    explicit C(); // #cwg1518-C
   };
 
   struct D : A {
     C c;
-    explicit D() = default; // #dr1518-D
+    explicit D() = default; // #cwg1518-D
   };
 
   template <typename T> void f() {
     T t; // ok
     T u{}; // ok
-    T v = {}; // #dr1518-v
-    // since-cxx11-error@#dr1518-v {{chosen constructor is explicit in copy-initialization}}
-    //   since-cxx11-note@#dr1518-f-A {{in instantiation of function template specialization 'dr1518::std_example::f<dr1518::std_example::A>' requested here}}
-    //   since-cxx11-note@#dr1518-A {{explicit constructor declared here}}
-    // since-cxx11-error@#dr1518-v {{chosen constructor is explicit in copy-initialization}}
-    //   since-cxx11-note@#dr1518-f-B {{in instantiation of function template specialization 'dr1518::std_example::f<dr1518::std_example::B>' requested here}}
-    //   since-cxx11-note@#dr1518-B {{explicit constructor declared here}}
-    // since-cxx11-error@#dr1518-v {{chosen constructor is explicit in copy-initialization}}
-    //   since-cxx11-note@#dr1518-f-C {{in instantiation of function template specialization 'dr1518::std_example::f<dr1518::std_example::C>' requested here}}
-    //   since-cxx11-note@#dr1518-C {{explicit constructor declared here}}
-    // since-cxx11-error@#dr1518-v {{chosen constructor is explicit in copy-initialization}}
-    //   since-cxx11-note@#dr1518-f-D {{in instantiation of function template specialization 'dr1518::std_example::f<dr1518::std_example::D>' requested here}}
-    //   since-cxx11-note@#dr1518-D {{explicit constructor declared here}}
+    T v = {}; // #cwg1518-v
+    // since-cxx11-error@#cwg1518-v {{chosen constructor is explicit in copy-initialization}}
+    //   since-cxx11-note@#cwg1518-f-A {{in instantiation of function template specialization 'cwg1518::std_example::f<cwg1518::std_example::A>' requested here}}
+    //   since-cxx11-note@#cwg1518-A {{explicit constructor declared here}}
+    // since-cxx11-error@#cwg1518-v {{chosen constructor is explicit in copy-initialization}}
+    //   since-cxx11-note@#cwg1518-f-B {{in instantiation of function template specialization 'cwg1518::std_example::f<cwg1518::std_example::B>' requested here}}
+    //   since-cxx11-note@#cwg1518-B {{explicit constructor declared here}}
+    // since-cxx11-error@#cwg1518-v {{chosen constructor is explicit in copy-initialization}}
+    //   since-cxx11-note@#cwg1518-f-C {{in instantiation of function template specialization 'cwg1518::std_example::f<cwg1518::std_example::C>' requested here}}
+    //   since-cxx11-note@#cwg1518-C {{explicit constructor declared here}}
+    // since-cxx11-error@#cwg1518-v {{chosen constructor is explicit in copy-initialization}}
+    //   since-cxx11-note@#cwg1518-f-D {{in instantiation of function template specialization 'cwg1518::std_example::f<cwg1518::std_example::D>' requested here}}
+    //   since-cxx11-note@#cwg1518-D {{explicit constructor declared here}}
   }
   template <typename T> void g() {
-    void x(T t); // #dr1518-x
-    x({}); // #dr1518-x-call
-    // since-cxx11-error@#dr1518-x-call {{chosen constructor is explicit in copy-initialization}}
-    //   since-cxx11-note@#dr1518-g-A {{in instantiation of function template specialization 'dr1518::std_example::g<dr1518::std_example::A>' requested here}}
-    //   since-cxx11-note@#dr1518-A {{explicit constructor declared here}}
-    //   since-cxx11-note@#dr1518-x {{passing argument to parameter 't' here}}
-    // since-cxx11-error@#dr1518-x-call {{chosen constructor is explicit in copy-initialization}}
-    //   since-cxx11-note@#dr1518-g-B {{in instantiation of function template specialization 'dr1518::std_example::g<dr1518::std_example::B>' requested here}}
-    //   since-cxx11-note@#dr1518-B {{explicit constructor declared here}}
-    //   since-cxx11-note@#dr1518-x {{passing argument to parameter 't' here}}
-    // since-cxx11-error@#dr1518-x-call {{chosen constructor is explicit in copy-initialization}}
-    //   since-cxx11-note@#dr1518-g-C {{in instantiation of function template specialization 'dr1518::std_example::g<dr1518::std_example::C>' requested here}}
-    //   since-cxx11-note@#dr1518-C {{explicit constructor declared here}}
-    //   since-cxx11-note@#dr1518-x {{passing argument to parameter 't' here}}
-    // since-cxx11-error@#dr1518-x-call {{chosen constructor is explicit in copy-initialization}}
-    //   since-cxx11-note@#dr1518-g-D {{in instantiation of function template specialization 'dr1518::std_example::g<dr1518::std_example::D>' requested here}}
-    //   since-cxx11-note@#dr1518-D {{explicit constructor declared here}}
-    //   since-cxx11-note@#dr1518-x {{passing argument to parameter 't' here}}
+    void x(T t); // #cwg1518-x
+    x({}); // #cwg1518-x-call
+    // since-cxx11-error@#cwg1518-x-call {{chosen constructor is explicit in copy-initialization}}
+    //   since-cxx11-note@#cwg1518-g-A {{in instantiation of function template specialization 'cwg1518::std_example::g<cwg1518::std_example::A>' requested here}}
+    //   since-cxx11-note@#cwg1518-A {{explicit constructor declared here}}
+    //   since-cxx11-note@#cwg1518-x {{passing argument to parameter 't' here}}
+    // since-cxx11-error@#cwg1518-x-call {{chosen constructor is explicit in copy-initialization}}
+    //   since-cxx11-note@#cwg1518-g-B {{in instantiation of function template specialization 'cwg1518::std_example::g<cwg1518::std_example::B>' requested here}}
+    //   since-cxx11-note@#cwg1518-B {{explicit constructor declared here}}
+    //   since-cxx11-note@#cwg1518-x {{passing argument to parameter 't' here}}
+    // since-cxx11-error@#cwg1518-x-call {{chosen constructor is explicit in copy-initialization}}
+    //   since-cxx11-note@#cwg1518-g-C {{in instantiation of function template specialization 'cwg1518::std_example::g<cwg1518::std_example::C>' requested here}}
+    //   since-cxx11-note@#cwg1518-C {{explicit constructor declared here}}
+    //   since-cxx11-note@#cwg1518-x {{passing argument to parameter 't' here}}
+    // since-cxx11-error@#cwg1518-x-call {{chosen constructor is explicit in copy-initialization}}
+    //   since-cxx11-note@#cwg1518-g-D {{in instantiation of function template specialization 'cwg1518::std_example::g<cwg1518::std_example::D>' requested here}}
+    //   since-cxx11-note@#cwg1518-D {{explicit constructor declared here}}
+    //   since-cxx11-note@#cwg1518-x {{passing argument to parameter 't' here}}
   }
 
   void test() {
-    f<A>(); // #dr1518-f-A
-    f<B>(); // #dr1518-f-B
-    f<C>(); // #dr1518-f-C
-    f<D>(); // #dr1518-f-D
-    g<A>(); // #dr1518-g-A
-    g<B>(); // #dr1518-g-B
-    g<C>(); // #dr1518-g-C
-    g<D>(); // #dr1518-g-D
+    f<A>(); // #cwg1518-f-A
+    f<B>(); // #cwg1518-f-B
+    f<C>(); // #cwg1518-f-C
+    f<D>(); // #cwg1518-f-D
+    g<A>(); // #cwg1518-g-A
+    g<B>(); // #cwg1518-g-B
+    g<C>(); // #cwg1518-g-C
+    g<D>(); // #cwg1518-g-D
   }
 }
 #endif // __cplusplus >= 201103L
 }
 
-namespace dr1550 { // dr1550: 3.4
+namespace cwg1550 { // cwg1550: 3.4
   int f(bool b, int n) {
     return (b ? (throw 0) : n) + (b ? n : (throw 0));
   }
 }
 
-namespace dr1558 { // dr1558: 12
+namespace cwg1558 { // cwg1558: 12
 #if __cplusplus >= 201103L
   template<class T, class...> using first_of = T;
-  template<class T> first_of<void, typename T::type> f(int); // #dr1558-f 
-  template<class T> void f(...) = delete; // #dr1558-f-deleted
+  template<class T> first_of<void, typename T::type> f(int); // #cwg1558-f 
+  template<class T> void f(...) = delete; // #cwg1558-f-deleted
 
   struct X { typedef void type; };
   void test() {
     f<X>(0);
     f<int>(0);
     // since-cxx11-error@-1 {{call to deleted function 'f'}}
-    //   since-cxx11-note@#dr1558-f-deleted {{candidate function [with T = int] has been explicitly deleted}}
-    //   since-cxx11-note@#dr1558-f {{candidate template ignored: substitution failure [with T = int]: type 'int' cannot be used prior to '::' because it has no members}}
+    //   since-cxx11-note@#cwg1558-f-deleted {{candidate function [with T = int] has been explicitly deleted}}
+    //   since-cxx11-note@#cwg1558-f {{candidate template ignored: substitution failure [with T = int]: type 'int' cannot be used prior to '::' because it has no members}}
   }
 #endif
 }
 
-namespace dr1560 { // dr1560: 3.5
+namespace cwg1560 { // cwg1560: 3.5
   void f(bool b, int n) {
     (b ? throw 0 : n) = (b ? n : throw 0) = 0;
   }
@@ -350,7 +350,7 @@ namespace dr1560 { // dr1560: 3.5
   const X &x = true ? get() : throw 0;
 }
 
-namespace dr1563 { // dr1563: yes
+namespace cwg1563 { // cwg1563: yes
 #if __cplusplus >= 201103L
   double bar(double) { return 0.0; }
   float bar(float) { return 0.0f; }
@@ -360,7 +360,7 @@ namespace dr1563 { // dr1563: yes
 #endif
 }
 
-namespace dr1567 { // dr1567: 3.3
+namespace cwg1567 { // cwg1567: 3.3
 #if __cplusplus >= 201103L
 struct B;
 struct A {
@@ -368,12 +368,12 @@ struct A {
   A(const B&) = delete;
   A(A&&);
   A(B&&) = delete;
-  A(int); // #dr1567-A-int
+  A(int); // #cwg1567-A-int
 };
 
-struct B: A { // #dr1567-B
-  using A::A; // #dr1567-using-A
-  B(double); // #dr1567-B-double
+struct B: A { // #cwg1567-B
+  using A::A; // #cwg1567-using-A
+  B(double); // #cwg1567-B-double
 };
 
 A a{0};
@@ -384,22 +384,22 @@ B b3{B{1.0}};
 // Good, copy/move ctors are not inherited
 B b4{a};
 // since-cxx11-error@-1 {{no matching constructor for initialization of 'B'}}
-//   since-cxx11-note@#dr1567-A-int {{candidate inherited constructor not viable: no known conversion from 'A' to 'int' for 1st argument}}
-//   since-cxx11-note@#dr1567-using-A {{constructor from base class 'A' inherited here}}
-//   since-cxx11-note@#dr1567-B {{candidate constructor (the implicit copy constructor) not viable: no known conversion from 'A' to 'const B' for 1st argument}}
-//   since-cxx11-note@#dr1567-B {{candidate constructor (the implicit move constructor) not viable: no known conversion from 'A' to 'B' for 1st argument}}
-//   since-cxx11-note@#dr1567-B-double {{candidate constructor not viable: no known conversion from 'A' to 'double' for 1st argument}}
+//   since-cxx11-note@#cwg1567-A-int {{candidate inherited constructor not viable: no known conversion from 'A' to 'int' for 1st argument}}
+//   since-cxx11-note@#cwg1567-using-A {{constructor from base class 'A' inherited here}}
+//   since-cxx11-note@#cwg1567-B {{candidate constructor (the implicit copy constructor) not viable: no known conversion from 'A' to 'const B' for 1st argument}}
+//   since-cxx11-note@#cwg1567-B {{candidate constructor (the implicit move constructor) not viable: no known conversion from 'A' to 'B' for 1st argument}}
+//   since-cxx11-note@#cwg1567-B-double {{candidate constructor not viable: no known conversion from 'A' to 'double' for 1st argument}}
 B b5{A{0}};
 // since-cxx11-error@-1 {{no matching constructor for initialization of 'B'}}
-//   since-cxx11-note@#dr1567-A-int {{candidate inherited constructor not viable: no known conversion from 'A' to 'int' for 1st argument}}
-//   since-cxx11-note@#dr1567-using-A {{constructor from base class 'A' inherited here}}
-//   since-cxx11-note@#dr1567-B {{candidate constructor (the implicit copy constructor) not viable: no known conversion from 'A' to 'const B' for 1st argument}}
-//   since-cxx11-note@#dr1567-B {{candidate constructor (the implicit move constructor) not viable: no known conversion from 'A' to 'B' for 1st argument}}
-//   since-cxx11-note@#dr1567-B-double {{candidate constructor not viable: no known conversion from 'A' to 'double' for 1st argument}}
+//   since-cxx11-note@#cwg1567-A-int {{candidate inherited constructor not viable: no known conversion from 'A' to 'int' for 1st argument}}
+//   since-cxx11-note@#cwg1567-using-A {{constructor from base class 'A' inherited here}}
+//   since-cxx11-note@#cwg1567-B {{candidate constructor (the implicit copy constructor) not viable: no known conversion from 'A' to 'const B' for 1st argument}}
+//   since-cxx11-note@#cwg1567-B {{candidate constructor (the implicit move constructor) not viable: no known conversion from 'A' to 'B' for 1st argument}}
+//   since-cxx11-note@#cwg1567-B-double {{candidate constructor not viable: no known conversion from 'A' to 'double' for 1st argument}}
 #endif
 }
 
-namespace dr1573 { // dr1573: 3.9
+namespace cwg1573 { // cwg1573: 3.9
 #if __cplusplus >= 201103L
   // ellipsis is inherited (p0136r1 supersedes this part).
   struct A { A(); A(int, char, ...); };
@@ -407,38 +407,38 @@ namespace dr1573 { // dr1573: 3.9
   B b(1, 'x', 4.0, "hello"); // ok
 
   // inherited constructor is effectively constexpr if the user-written constructor would be
-  struct C { C(); constexpr C(int) {} }; // #dr1573-C
+  struct C { C(); constexpr C(int) {} }; // #cwg1573-C
   struct D : C { using C::C; };
   constexpr D d = D(0); // ok
-  struct E : C { using C::C; A a; }; // #dr1573-E
+  struct E : C { using C::C; A a; }; // #cwg1573-E
   constexpr E e = E(0);
   // since-cxx11-error@-1 {{constexpr variable cannot have non-literal type 'const E'}}
-  //   since-cxx11-note@#dr1573-E {{'E' is not literal because it has data member 'a' of non-literal type 'A'}}
+  //   since-cxx11-note@#cwg1573-E {{'E' is not literal because it has data member 'a' of non-literal type 'A'}}
 
   // FIXME: This diagnostic is pretty bad; we should explain that the problem
   // is that F::c would be initialized by a non-constexpr constructor.
-  struct F : C { using C::C; C c; }; // #dr1573-F
+  struct F : C { using C::C; C c; }; // #cwg1573-F
   constexpr F f = F(0);
   // since-cxx11-error@-1 {{constexpr variable 'f' must be initialized by a constant expression}}
   //   cxx11-20-note@-2 {{constructor inherited from base class 'C' cannot be used in a constant expression; derived class cannot be implicitly initialized}}
   //   since-cxx23-note@-3 {{in implicit initialization for inherited constructor of 'F'}}
-  //   since-cxx23-note@#dr1573-F {{non-constexpr constructor 'C' cannot be used in a constant expression}}
-  //   cxx11-20-note@#dr1573-F {{declared here}}
-  //   since-cxx23-note@#dr1573-C {{declared here}}
+  //   since-cxx23-note@#cwg1573-F {{non-constexpr constructor 'C' cannot be used in a constant expression}}
+  //   cxx11-20-note@#cwg1573-F {{declared here}}
+  //   since-cxx23-note@#cwg1573-C {{declared here}}
 
   // inherited constructor is effectively deleted if the user-written constructor would be
   struct G { G(int); };
-  struct H : G { using G::G; G g; }; // #dr1573-H
+  struct H : G { using G::G; G g; }; // #cwg1573-H
   H h(0);
   // since-cxx11-error@-1 {{constructor inherited by 'H' from base class 'G' is implicitly deleted}}
-  //   since-cxx11-note@#dr1573-H {{constructor inherited by 'H' is implicitly deleted because field 'g' has no default constructor}}
+  //   since-cxx11-note@#cwg1573-H {{constructor inherited by 'H' is implicitly deleted because field 'g' has no default constructor}}
 
   // deleted definition of constructor is inherited
-  struct I { I(int) = delete; }; // #dr1573-I
+  struct I { I(int) = delete; }; // #cwg1573-I
   struct J : I { using I::I; };
   J j(0);
   // since-cxx11-error@-1 {{call to deleted constructor of 'J'}}
-  //   since-cxx11-note@#dr1573-I {{'I' has been explicitly marked deleted here}}
+  //   since-cxx11-note@#cwg1573-I {{'I' has been explicitly marked deleted here}}
 #endif
 }
 
@@ -483,18 +483,18 @@ namespace std {
 } // std
 #endif
 
-namespace dr1579 { // dr1579: 3.9
+namespace cwg1579 { // cwg1579: 3.9
 #if __cplusplus >= 201103L
 template<class T>
 struct GenericMoveOnly {
   GenericMoveOnly();
-  template<class U> GenericMoveOnly(const GenericMoveOnly<U> &) = delete; // #dr1579-deleted-U
-  GenericMoveOnly(const int &) = delete; // #dr1579-deleted-int
+  template<class U> GenericMoveOnly(const GenericMoveOnly<U> &) = delete; // #cwg1579-deleted-U
+  GenericMoveOnly(const int &) = delete; // #cwg1579-deleted-int
   template<class U> GenericMoveOnly(GenericMoveOnly<U> &&);
   GenericMoveOnly(int &&);
 };
 
-GenericMoveOnly<float> DR1579_Eligible(GenericMoveOnly<char> CharMO) {
+GenericMoveOnly<float> CWG1579_Eligible(GenericMoveOnly<char> CharMO) {
   int i;
   GenericMoveOnly<char> GMO;
 
@@ -510,7 +510,7 @@ GenericMoveOnly<float> DR1579_Eligible(GenericMoveOnly<char> CharMO) {
 
 GenericMoveOnly<char> GlobalMO;
 
-GenericMoveOnly<float> DR1579_Ineligible(int &AnInt,
+GenericMoveOnly<float> CWG1579_Ineligible(int &AnInt,
                                           GenericMoveOnly<char> &CharMO) {
   static GenericMoveOnly<char> StaticMove;
   extern GenericMoveOnly<char> ExternMove;
@@ -518,63 +518,63 @@ GenericMoveOnly<float> DR1579_Ineligible(int &AnInt,
   if (0)
     return AnInt;
     // since-cxx11-error@-1 {{conversion function from 'int' to 'GenericMoveOnly<float>' invokes a deleted function}}
-    //   since-cxx11-note@#dr1579-deleted-int {{'GenericMoveOnly' has been explicitly marked deleted here}}
+    //   since-cxx11-note@#cwg1579-deleted-int {{'GenericMoveOnly' has been explicitly marked deleted here}}
   else if (0)
     return GlobalMO;
     // since-cxx11-error@-1 {{conversion function from 'GenericMoveOnly<char>' to 'GenericMoveOnly<float>' invokes a deleted function}}
-    //   since-cxx11-note@#dr1579-deleted-U {{'GenericMoveOnly<char>' has been explicitly marked deleted here}}
+    //   since-cxx11-note@#cwg1579-deleted-U {{'GenericMoveOnly<char>' has been explicitly marked deleted here}}
   else if (0)
     return StaticMove;
     // since-cxx11-error@-1 {{conversion function from 'GenericMoveOnly<char>' to 'GenericMoveOnly<float>' invokes a deleted function}}
-    //   since-cxx11-note@#dr1579-deleted-U {{'GenericMoveOnly<char>' has been explicitly marked deleted here}}
+    //   since-cxx11-note@#cwg1579-deleted-U {{'GenericMoveOnly<char>' has been explicitly marked deleted here}}
   else if (0)
     return ExternMove;
     // since-cxx11-error@-1 {{conversion function from 'GenericMoveOnly<char>' to 'GenericMoveOnly<float>' invokes a deleted function}}
-    //   since-cxx11-note@#dr1579-deleted-U {{'GenericMoveOnly<char>' has been explicitly marked deleted here}}
+    //   since-cxx11-note@#cwg1579-deleted-U {{'GenericMoveOnly<char>' has been explicitly marked deleted here}}
   else if (0)
     return AnInt;
     // since-cxx11-error@-1 {{conversion function from 'int' to 'GenericMoveOnly<float>' invokes a deleted function}}
-    //   since-cxx11-note@#dr1579-deleted-int {{'GenericMoveOnly' has been explicitly marked deleted here}}
+    //   since-cxx11-note@#cwg1579-deleted-int {{'GenericMoveOnly' has been explicitly marked deleted here}}
   else
     return CharMO;
     // since-cxx11-error@-1 {{conversion function from 'GenericMoveOnly<char>' to 'GenericMoveOnly<float>' invokes a deleted function}}
-    //   since-cxx11-note@#dr1579-deleted-U {{'GenericMoveOnly<char>' has been explicitly marked deleted here}}
+    //   since-cxx11-note@#cwg1579-deleted-U {{'GenericMoveOnly<char>' has been explicitly marked deleted here}}
 }
 
-auto DR1579_lambda_valid = [](GenericMoveOnly<float> mo) ->
+auto CWG1579_lambda_valid = [](GenericMoveOnly<float> mo) ->
   GenericMoveOnly<char> {
   return mo;
 };
 
-auto DR1579_lambda_invalid = []() -> GenericMoveOnly<char> {
+auto CWG1579_lambda_invalid = []() -> GenericMoveOnly<char> {
   static GenericMoveOnly<float> mo;
   return mo;
   // since-cxx11-error@-1 {{conversion function from 'GenericMoveOnly<float>' to 'GenericMoveOnly<char>' invokes a deleted function}}
-  //   since-cxx11-note@#dr1579-deleted-U {{'GenericMoveOnly<float>' has been explicitly marked deleted here}}
+  //   since-cxx11-note@#cwg1579-deleted-U {{'GenericMoveOnly<float>' has been explicitly marked deleted here}}
 };
 #endif
-} // end namespace dr1579
+} // end namespace cwg1579
 
-namespace dr1584 { // dr1584: 7 drafting 2015-05
+namespace cwg1584 { // cwg1584: 7 drafting 2015-05
 #if __cplusplus >= 201103L
   // Deducing function types from cv-qualified types
-  template<typename T> void f(const T *); // #dr1584-f
+  template<typename T> void f(const T *); // #cwg1584-f
   template<typename T> void g(T *, const T * = 0);
   template<typename T> void h(T *) { T::error; }
   // since-cxx11-error@-1 {{type 'void ()' cannot be used prior to '::' because it has no members}}
-  //   since-cxx11-note@#dr1584-h {{in instantiation of function template specialization 'dr1584::h<void ()>' requested here}}
+  //   since-cxx11-note@#cwg1584-h {{in instantiation of function template specialization 'cwg1584::h<void ()>' requested here}}
   template<typename T> void h(const T *);
   void i() {
     f(&i);
     // since-cxx11-error@-1 {{no matching function for call to 'f'}}
-    //   since-cxx11-note@#dr1584-f {{candidate template ignored: could not match 'const T *' against 'void (*)()'}}
+    //   since-cxx11-note@#cwg1584-f {{candidate template ignored: could not match 'const T *' against 'void (*)()'}}
     g(&i);
-    h(&i); // #dr1584-h
+    h(&i); // #cwg1584-h
   }
 #endif
 }
 
-namespace dr1589 {   // dr1589: 3.7 c++11
+namespace cwg1589 {   // cwg1589: 3.7 c++11
 #if __cplusplus >= 201103L
   // Ambiguous ranking of list-initialization sequences
 
@@ -595,33 +595,33 @@ namespace dr1589 {   // dr1589: 3.7 c++11
 
   namespace with_error {
     void f0(long);
-    void f0(std::initializer_list<int>);          // #dr1589-f0-ilist
-    void f0(std::initializer_list<int>, int = 0); // #dr1589-f0-ilist-int
+    void f0(std::initializer_list<int>);          // #cwg1589-f0-ilist
+    void f0(std::initializer_list<int>, int = 0); // #cwg1589-f0-ilist-int
     void g0() { f0({1L}); }
     // since-cxx11-error@-1 {{call to 'f0' is ambiguous}}
-    //   since-cxx11-note@#dr1589-f0-ilist {{candidate function}}
-    //   since-cxx11-note@#dr1589-f0-ilist-int {{candidate function}}
+    //   since-cxx11-note@#cwg1589-f0-ilist {{candidate function}}
+    //   since-cxx11-note@#cwg1589-f0-ilist-int {{candidate function}}
 
     void f1(int);
-    void f1(std::initializer_list<long>);          // #dr1589-f1-ilist
-    void f1(std::initializer_list<long>, int = 0); // #dr1589-f1-ilist-long
+    void f1(std::initializer_list<long>);          // #cwg1589-f1-ilist
+    void f1(std::initializer_list<long>, int = 0); // #cwg1589-f1-ilist-long
     void g1() { f1({42}); }
     // since-cxx11-error@-1 {{call to 'f1' is ambiguous}}
-    //   since-cxx11-note@#dr1589-f1-ilist {{candidate function}}
-    //   since-cxx11-note@#dr1589-f1-ilist-long {{candidate function}}
+    //   since-cxx11-note@#cwg1589-f1-ilist {{candidate function}}
+    //   since-cxx11-note@#cwg1589-f1-ilist-long {{candidate function}}
 
     void f2(std::pair<const char*, const char*>);
-    void f2(std::initializer_list<std::string>);          // #dr1589-f2-ilist
-    void f2(std::initializer_list<std::string>, int = 0); // #dr1589-f2-ilist-int
+    void f2(std::initializer_list<std::string>);          // #cwg1589-f2-ilist
+    void f2(std::initializer_list<std::string>, int = 0); // #cwg1589-f2-ilist-int
     void g2() { f2({"foo","bar"}); }
     // since-cxx11-error@-1 {{call to 'f2' is ambiguous}}
-    //   since-cxx11-note@#dr1589-f2-ilist {{candidate function}}
-    //   since-cxx11-note@#dr1589-f2-ilist-int {{candidate function}}
+    //   since-cxx11-note@#cwg1589-f2-ilist {{candidate function}}
+    //   since-cxx11-note@#cwg1589-f2-ilist-int {{candidate function}}
   }
 #endif
-} // dr1589
+} // cwg1589
 
-namespace dr1591 {  //dr1591. Deducing array bound and element type from initializer list
+namespace cwg1591 {  //cwg1591. Deducing array bound and element type from initializer list
 #if __cplusplus >= 201103L
   template<class T, int N> int h(T const(&)[N]);
   int X = h({1,2,3});              // T deduced to int, N deduced to 3
@@ -630,10 +630,10 @@ namespace dr1591 {  //dr1591. Deducing array bound and element type from initial
   int Y = j({42});                 // T deduced to int, array bound not considered
 
   struct Aggr { int i; int j; };
-  template<int N> int k(Aggr const(&)[N]); // #dr1591-k
+  template<int N> int k(Aggr const(&)[N]); // #cwg1591-k
   int Y0 = k({1,2,3});
   // since-cxx11-error@-1 {{no matching function for call to 'k'}}
-  //   since-cxx11-note@#dr1591-k {{candidate function [with N = 3] not viable: no known conversion from 'int' to 'const Aggr' for 1st argument}}
+  //   since-cxx11-note@#cwg1591-k {{candidate function [with N = 3] not viable: no known conversion from 'int' to 'const Aggr' for 1st argument}}
   int Z = k({{1},{2},{3}});        // OK, N deduced to 3
 
   template<int M, int N> int m(int const(&)[M][N]);
@@ -644,31 +644,31 @@ namespace dr1591 {  //dr1591. Deducing array bound and element type from initial
   
   
   namespace check_multi_dim_arrays {
-    template<class T, int N, int M, int O> int ***f(const T (&a)[N][M][O]); // #dr1591-f-3
-    template<class T, int N, int M> int **f(const T (&a)[N][M]); // #dr1591-f-2
+    template<class T, int N, int M, int O> int ***f(const T (&a)[N][M][O]); // #cwg1591-f-3
+    template<class T, int N, int M> int **f(const T (&a)[N][M]); // #cwg1591-f-2
    
-   template<class T, int N> int *f(const T (&a)[N]); // #dr1591-f-1
+   template<class T, int N> int *f(const T (&a)[N]); // #cwg1591-f-1
     int ***p3 = f({  {  {1,2}, {3, 4}  }, {  {5,6}, {7, 8}  }, {  {9,10}, {11, 12}  } });
     int ***p33 = f({  {  {1,2}, {3, 4}  }, {  {5,6}, {7, 8}  }, {  {9,10}, {11, 12, 13}  } });
     // since-cxx11-error@-1 {{no matching function for call to 'f'}}
-    //   since-cxx11-note@#dr1591-f-2 {{candidate template ignored: couldn't infer template argument 'T'}}
-    //   since-cxx11-note@#dr1591-f-1 {{candidate template ignored: couldn't infer template argument 'T'}}
-    //   since-cxx11-note@#dr1591-f-3 {{candidate template ignored: deduced conflicting values for parameter 'O' (2 vs. 3)}}
+    //   since-cxx11-note@#cwg1591-f-2 {{candidate template ignored: couldn't infer template argument 'T'}}
+    //   since-cxx11-note@#cwg1591-f-1 {{candidate template ignored: couldn't infer template argument 'T'}}
+    //   since-cxx11-note@#cwg1591-f-3 {{candidate template ignored: deduced conflicting values for parameter 'O' (2 vs. 3)}}
     int **p2 = f({  {1,2,3}, {3, 4, 5}  });
     int **p22 = f({  {1,2}, {3, 4}  });
     int *p1 = f({1, 2, 3});
   }
   namespace check_multi_dim_arrays_rref {
-    template<class T, int N, int M, int O> int ***g(T (&&a)[N][M][O]); // #dr1591-g-3
-    template<class T, int N, int M> int **g(T (&&a)[N][M]); // #dr1591-g-2
+    template<class T, int N, int M, int O> int ***g(T (&&a)[N][M][O]); // #cwg1591-g-3
+    template<class T, int N, int M> int **g(T (&&a)[N][M]); // #cwg1591-g-2
    
-    template<class T, int N> int *g(T (&&a)[N]); // #dr1591-g-1
+    template<class T, int N> int *g(T (&&a)[N]); // #cwg1591-g-1
     int ***p3 = g({  {  {1,2}, {3, 4}  }, {  {5,6}, {7, 8}  }, {  {9,10}, {11, 12}  } });
     int ***p33 = g({  {  {1,2}, {3, 4}  }, {  {5,6}, {7, 8}  }, {  {9,10}, {11, 12, 13}  } });
     // since-cxx11-error@-1 {{no matching function for call to 'g'}}
-    //   since-cxx11-note@#dr1591-g-2 {{candidate template ignored: couldn't infer template argument 'T'}}
-    //   since-cxx11-note@#dr1591-g-1 {{candidate template ignored: couldn't infer template argument 'T'}}
-    //   since-cxx11-note@#dr1591-g-3 {{candidate template ignored: deduced conflicting values for parameter 'O' (2 vs. 3)}}
+    //   since-cxx11-note@#cwg1591-g-2 {{candidate template ignored: couldn't infer template argument 'T'}}
+    //   since-cxx11-note@#cwg1591-g-1 {{candidate template ignored: couldn't infer template argument 'T'}}
+    //   since-cxx11-note@#cwg1591-g-3 {{candidate template ignored: deduced conflicting values for parameter 'O' (2 vs. 3)}}
     int **p2 = g({  {1,2,3}, {3, 4, 5}  });
     int **p22 = g({  {1,2}, {3, 4}  });
     int *p1 = g({1, 2, 3});
@@ -684,8 +684,8 @@ namespace dr1591 {  //dr1591. Deducing array bound and element type from initial
     
     template<class T, int N> int *i(T (&&)[N]);  // #1
     template<class T> char *i(std::initializer_list<T> &&);  // #2
-    template<class T, int N, int M> int **i(T (&&)[N][M]); // #3 #dr1591-i-2
-    template<class T, int N> char **i(std::initializer_list<T> (&&)[N]); // #4 #dr1591-i-1
+    template<class T, int N, int M> int **i(T (&&)[N][M]); // #3 #cwg1591-i-2
+    template<class T, int N> char **i(std::initializer_list<T> (&&)[N]); // #4 #cwg1591-i-1
 
     template<class T> short *i(T (&&)[2]);  // #5
 
@@ -697,11 +697,11 @@ namespace dr1591 {  //dr1591. Deducing array bound and element type from initial
 
     void *pv1 = i({ {1, 2, 3}, {4, 5, 6} }); // ambiguous btw 3 & 4
     // since-cxx11-error@-1 {{call to 'i' is ambiguous}} 
-    //   since-cxx11-note@#dr1591-i-2 {{candidate function [with T = int, N = 2, M = 3]}}
-    //   since-cxx11-note@#dr1591-i-1 {{candidate function [with T = int, N = 2]}}
+    //   since-cxx11-note@#cwg1591-i-2 {{candidate function [with T = int, N = 2, M = 3]}}
+    //   since-cxx11-note@#cwg1591-i-1 {{candidate function [with T = int, N = 2]}}
     char **pcc = i({ {1}, {2, 3} }); // OK #4
 
     short *ps = i(Arr<int>{1, 2});  // OK #5
   }
 #endif
-} // dr1591
+} // cwg1591
diff --git a/clang/test/CXX/drs/dr16xx.cpp b/clang/test/CXX/drs/dr16xx.cpp
index f4d6c04fb8e0..6d7bb7619f8b 100644
--- a/clang/test/CXX/drs/dr16xx.cpp
+++ b/clang/test/CXX/drs/dr16xx.cpp
@@ -25,7 +25,7 @@ namespace std {
 } // std
 #endif
 
-namespace dr1601 { // dr1601: 10
+namespace cwg1601 { // cwg1601: 10
 enum E : char { e };
 // cxx98-error@-1 {{enumeration types with a fixed underlying type are a C++11 extension}}
 void f(char);
@@ -33,9 +33,9 @@ void f(int);
 void g() {
   f(e);
 }
-} // namespace dr1601
+} // namespace cwg1601
 
-namespace dr1606 { // dr1606: 3.1
+namespace cwg1606 { // cwg1606: 3.1
 #if __cplusplus >= 201103L
   std::size_t test() {
     int i = 1;
@@ -44,16 +44,16 @@ namespace dr1606 { // dr1606: 3.1
     return sizeof(f);
   }
 #endif
-} // namespace dr1606
+} // namespace cwg1606
 
-namespace dr1611 { // dr1611: dup 1658
+namespace cwg1611 { // cwg1611: dup 1658
   struct A { A(int); };
   struct B : virtual A { virtual void f() = 0; };
   struct C : B { C() : A(0) {} void f(); };
   C c;
 }
 
-namespace dr1631 {  // dr1631: 3.7
+namespace cwg1631 {  // cwg1631: 3.7
 #if __cplusplus >= 201103L
   // Incorrect overload resolution for single-element initializer-list
 
@@ -70,24 +70,24 @@ namespace dr1631 {  // dr1631: 3.7
 
   namespace with_error {
     void f(B, int);           // TODO: expected- note {{candidate function}}
-    void f(int, A);           // #dr1631-f
-    void f(int, A, int = 0);  // #dr1631-f-int
+    void f(int, A);           // #cwg1631-f
+    void f(int, A, int = 0);  // #cwg1631-f-int
 
     void test() {
       f({0}, {{1}});
       // since-cxx11-error@-1 {{call to 'f' is ambiguous}}
-      //   since-cxx11-note@#dr1631-f {{candidate function}}
-      //   since-cxx11-note@#dr1631-f-int {{candidate function}}
+      //   since-cxx11-note@#cwg1631-f {{candidate function}}
+      //   since-cxx11-note@#cwg1631-f-int {{candidate function}}
     }
   }
 #endif
 }
 
-namespace dr1638 { // dr1638: 3.1
+namespace cwg1638 { // cwg1638: 3.1
 #if __cplusplus >= 201103L
   template<typename T> struct A {
-    enum class E; // #dr1638-E
-    enum class F : T; // #dr1638-F
+    enum class E; // #cwg1638-E
+    enum class F : T; // #cwg1638-F
   };
 
   template<> enum class A<int>::E;
@@ -100,13 +100,13 @@ namespace dr1638 { // dr1638: 3.1
 
   template<> enum class A<short>::F;
   // since-cxx11-error@-1 {{enumeration redeclared with different underlying type 'int' (was 'short')}}
-  //   since-cxx11-note@#dr1638-F {{previous declaration is here}}
+  //   since-cxx11-note@#cwg1638-F {{previous declaration is here}}
   template<> enum class A<char>::E : char;
   // since-cxx11-error@-1 {{enumeration redeclared with different underlying type 'char' (was 'int')}}
-  //   since-cxx11-note@#dr1638-E {{previous declaration is here}}
+  //   since-cxx11-note@#cwg1638-E {{previous declaration is here}}
   template<> enum class A<char>::F : int;
   // since-cxx11-error@-1 {{enumeration redeclared with different underlying type 'int' (was 'char')}}
-  //   since-cxx11-note@#dr1638-F {{previous declaration is here}}
+  //   since-cxx11-note@#cwg1638-F {{previous declaration is here}}
 
   enum class A<unsigned>::E;
   // since-cxx11-error@-1 {{template specialization requires 'template<>'}}
@@ -124,34 +124,34 @@ namespace dr1638 { // dr1638: 3.1
 #endif
 }
 
-namespace dr1645 { // dr1645: 3.9
+namespace cwg1645 { // cwg1645: 3.9
 #if __cplusplus >= 201103L
   struct A {
-    constexpr A(int, float = 0); // #dr1645-int-float
-    explicit A(int, int = 0); // #dr1645-int-int
-    A(int, int, int = 0) = delete; // #dr1645-int-int-int
+    constexpr A(int, float = 0); // #cwg1645-int-float
+    explicit A(int, int = 0); // #cwg1645-int-int
+    A(int, int, int = 0) = delete; // #cwg1645-int-int-int
   };
 
   struct B : A {
-    using A::A; // #dr1645-using
+    using A::A; // #cwg1645-using
   };
 
   constexpr B a(0);
   // since-cxx11-error@-1 {{call to constructor of 'const B' is ambiguous}}
-  //   since-cxx11-note@#dr1645-int-float {{candidate inherited constructor}}
-  //   since-cxx11-note@#dr1645-using {{constructor from base class 'A' inherited here}}
-  //   since-cxx11-note@#dr1645-int-int {{candidate inherited constructor}}
-  //   since-cxx11-note@#dr1645-using {{constructor from base class 'A' inherited here}}
+  //   since-cxx11-note@#cwg1645-int-float {{candidate inherited constructor}}
+  //   since-cxx11-note@#cwg1645-using {{constructor from base class 'A' inherited here}}
+  //   since-cxx11-note@#cwg1645-int-int {{candidate inherited constructor}}
+  //   since-cxx11-note@#cwg1645-using {{constructor from base class 'A' inherited here}}
   constexpr B b(0, 0);
   // since-cxx11-error@-1 {{call to constructor of 'const B' is ambiguous}}
-  //   since-cxx11-note@#dr1645-int-int {{candidate inherited constructor}}
-  //   since-cxx11-note@#dr1645-using {{constructor from base class 'A' inherited here}}
-  //   since-cxx11-note@#dr1645-int-int-int {{candidate inherited constructor has been explicitly deleted}}
-  //   since-cxx11-note@#dr1645-using {{constructor from base class 'A' inherited here}}
+  //   since-cxx11-note@#cwg1645-int-int {{candidate inherited constructor}}
+  //   since-cxx11-note@#cwg1645-using {{constructor from base class 'A' inherited here}}
+  //   since-cxx11-note@#cwg1645-int-int-int {{candidate inherited constructor has been explicitly deleted}}
+  //   since-cxx11-note@#cwg1645-using {{constructor from base class 'A' inherited here}}
 #endif
 }
 
-namespace dr1652 { // dr1652: 3.6
+namespace cwg1652 { // cwg1652: 3.6
   int a, b;
   int arr[&a + 1 == &b ? 1 : 2];
   // expected-error@-1 {{variable length arrays in C++ are a Clang extension}}
@@ -159,7 +159,7 @@ namespace dr1652 { // dr1652: 3.6
   // expected-error@-3 {{variable length array declaration not allowed at file scope}}
 }
 
-namespace dr1653 { // dr1653: 4 c++17
+namespace cwg1653 { // cwg1653: 4 c++17
   void f(bool b) {
     ++b;
     // cxx98-14-warning@-1 {{incrementing expression of type bool is deprecated and incompatible with C++17}}
@@ -176,10 +176,10 @@ namespace dr1653 { // dr1653: 4 c++17
   }
 }
 
-namespace dr1658 { // dr1658: 5
+namespace cwg1658 { // cwg1658: 5
   namespace DefCtor {
-    class A { A(); }; // #dr1658-A1
-    class B { ~B(); }; // #dr1658-B1
+    class A { A(); }; // #cwg1658-A1
+    class B { ~B(); }; // #cwg1658-B1
 
     // The stars align! An abstract class does not construct its virtual bases.
     struct C : virtual A { C(); virtual void foo() = 0; };
@@ -190,76 +190,76 @@ namespace dr1658 { // dr1658: 5
     // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}}
 
     // In all other cases, we are not so lucky.
-    struct E : A { E(); virtual void foo() = 0; }; // #dr1658-E1
-    E::E() = default; // #dr1658-E1-ctor
+    struct E : A { E(); virtual void foo() = 0; }; // #cwg1658-E1
+    E::E() = default; // #cwg1658-E1-ctor
     // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}}
     // cxx98-error@-2 {{base class 'A' has private default constructor}}
-    //   cxx98-note@-3 {{in defaulted default constructor for 'dr1658::DefCtor::E' first required here}}
-    //   cxx98-note@#dr1658-A1 {{implicitly declared private here}}
-    // since-cxx11-error@#dr1658-E1-ctor {{defaulting this default constructor would delete it after its first declaration}}
-    //   since-cxx11-note@#dr1658-E1 {{default constructor of 'E' is implicitly deleted because base class 'A' has an inaccessible default constructor}}
-    struct F : virtual A { F(); }; // #dr1658-F1
-    F::F() = default; // #dr1658-F1-ctor
+    //   cxx98-note@-3 {{in defaulted default constructor for 'cwg1658::DefCtor::E' first required here}}
+    //   cxx98-note@#cwg1658-A1 {{implicitly declared private here}}
+    // since-cxx11-error@#cwg1658-E1-ctor {{defaulting this default constructor would delete it after its first declaration}}
+    //   since-cxx11-note@#cwg1658-E1 {{default constructor of 'E' is implicitly deleted because base class 'A' has an inaccessible default constructor}}
+    struct F : virtual A { F(); }; // #cwg1658-F1
+    F::F() = default; // #cwg1658-F1-ctor
     // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}}
     // cxx98-error@-2 {{inherited virtual base class 'A' has private default constructor}}
-    //   cxx98-note@-3 {{in defaulted default constructor for 'dr1658::DefCtor::F' first required here}}
-    //   cxx98-note@#dr1658-A1 {{implicitly declared private here}}
-    // since-cxx11-error@#dr1658-F1-ctor {{defaulting this default constructor would delete it after its first declaration}}
-    //   since-cxx11-note@#dr1658-F1 {{default constructor of 'F' is implicitly deleted because base class 'A' has an inaccessible default constructor}}
+    //   cxx98-note@-3 {{in defaulted default constructor for 'cwg1658::DefCtor::F' first required here}}
+    //   cxx98-note@#cwg1658-A1 {{implicitly declared private here}}
+    // since-cxx11-error@#cwg1658-F1-ctor {{defaulting this default constructor would delete it after its first declaration}}
+    //   since-cxx11-note@#cwg1658-F1 {{default constructor of 'F' is implicitly deleted because base class 'A' has an inaccessible default constructor}}
 
-    struct G : B { G(); virtual void foo() = 0; }; // #dr1658-G1
-    G::G() = default; // #dr1658-G1-ctor
+    struct G : B { G(); virtual void foo() = 0; }; // #cwg1658-G1
+    G::G() = default; // #cwg1658-G1-ctor
     // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}}
-    // cxx98-error@#dr1658-G1 {{base class 'B' has private destructor}}
-    //   cxx98-note@#dr1658-G1-ctor {{in defaulted default constructor for 'dr1658::DefCtor::G' first required here}}
-    //   cxx98-note@#dr1658-B1 {{implicitly declared private here}}
-    // since-cxx11-error@#dr1658-G1-ctor {{defaulting this default constructor would delete it after its first declaration}}
-    //   since-cxx11-note@#dr1658-G1 {{default constructor of 'G' is implicitly deleted because base class 'B' has an inaccessible destructor}}
-    struct H : virtual B { H(); }; // #dr1658-H1
-    H::H() = default; // #dr1658-H1-ctor
+    // cxx98-error@#cwg1658-G1 {{base class 'B' has private destructor}}
+    //   cxx98-note@#cwg1658-G1-ctor {{in defaulted default constructor for 'cwg1658::DefCtor::G' first required here}}
+    //   cxx98-note@#cwg1658-B1 {{implicitly declared private here}}
+    // since-cxx11-error@#cwg1658-G1-ctor {{defaulting this default constructor would delete it after its first declaration}}
+    //   since-cxx11-note@#cwg1658-G1 {{default constructor of 'G' is implicitly deleted because base class 'B' has an inaccessible destructor}}
+    struct H : virtual B { H(); }; // #cwg1658-H1
+    H::H() = default; // #cwg1658-H1-ctor
     // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}}
-    // cxx98-error@#dr1658-H1 {{base class 'B' has private destructor}}
-    //   cxx98-note@#dr1658-H1-ctor {{in defaulted default constructor for 'dr1658::DefCtor::H' first required here}}
-    //   cxx98-note@#dr1658-B1 {{implicitly declared private here}}
-    // since-cxx11-error@#dr1658-H1-ctor {{defaulting this default constructor would delete it after its first declaration}}
-    //   since-cxx11-note@#dr1658-H1 {{default constructor of 'H' is implicitly deleted because base class 'B' has an inaccessible destructor}}
+    // cxx98-error@#cwg1658-H1 {{base class 'B' has private destructor}}
+    //   cxx98-note@#cwg1658-H1-ctor {{in defaulted default constructor for 'cwg1658::DefCtor::H' first required here}}
+    //   cxx98-note@#cwg1658-B1 {{implicitly declared private here}}
+    // since-cxx11-error@#cwg1658-H1-ctor {{defaulting this default constructor would delete it after its first declaration}}
+    //   since-cxx11-note@#cwg1658-H1 {{default constructor of 'H' is implicitly deleted because base class 'B' has an inaccessible destructor}}
   }
 
   namespace Dtor {
-    class B { ~B(); }; // #dr1658-B2
+    class B { ~B(); }; // #cwg1658-B2
 
     struct D : virtual B { ~D(); virtual void foo() = 0; };
     D::~D() = default; // ok, not deleted
     // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}}
 
-    struct G : B { ~G(); virtual void foo() = 0; }; // #dr1658-G2
-    G::~G() = default; // #dr1658-G2-dtor
+    struct G : B { ~G(); virtual void foo() = 0; }; // #cwg1658-G2
+    G::~G() = default; // #cwg1658-G2-dtor
     // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}}
-    // cxx98-error@#dr1658-G2 {{base class 'B' has private destructor}}
-    //   cxx98-note@#dr1658-G2-dtor {{in defaulted destructor for 'dr1658::Dtor::G' first required here}}
-    //   cxx98-note@#dr1658-B2 {{implicitly declared private here}}
-    // since-cxx11-error@#dr1658-G2-dtor {{defaulting this destructor would delete it after its first declaration}}
-    //   since-cxx11-note@#dr1658-G2 {{destructor of 'G' is implicitly deleted because base class 'B' has an inaccessible destructor}}
-    struct H : virtual B { ~H(); }; // #dr1658-H2
-    H::~H() = default; // #dr1658-H2-dtor
+    // cxx98-error@#cwg1658-G2 {{base class 'B' has private destructor}}
+    //   cxx98-note@#cwg1658-G2-dtor {{in defaulted destructor for 'cwg1658::Dtor::G' first required here}}
+    //   cxx98-note@#cwg1658-B2 {{implicitly declared private here}}
+    // since-cxx11-error@#cwg1658-G2-dtor {{defaulting this destructor would delete it after its first declaration}}
+    //   since-cxx11-note@#cwg1658-G2 {{destructor of 'G' is implicitly deleted because base class 'B' has an inaccessible destructor}}
+    struct H : virtual B { ~H(); }; // #cwg1658-H2
+    H::~H() = default; // #cwg1658-H2-dtor
     // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}}
-    // cxx98-error@#dr1658-H2 {{base class 'B' has private destructor}}
-    //   cxx98-note@#dr1658-H2-dtor {{in defaulted destructor for 'dr1658::Dtor::H' first required here}}
-    //   cxx98-note@#dr1658-B2 {{implicitly declared private here}}
-    // since-cxx11-error@#dr1658-H2-dtor {{defaulting this destructor would delete it after its first declaration}}
-    //   since-cxx11-note@#dr1658-H2 {{destructor of 'H' is implicitly deleted because base class 'B' has an inaccessible destructor}}
+    // cxx98-error@#cwg1658-H2 {{base class 'B' has private destructor}}
+    //   cxx98-note@#cwg1658-H2-dtor {{in defaulted destructor for 'cwg1658::Dtor::H' first required here}}
+    //   cxx98-note@#cwg1658-B2 {{implicitly declared private here}}
+    // since-cxx11-error@#cwg1658-H2-dtor {{defaulting this destructor would delete it after its first declaration}}
+    //   since-cxx11-note@#cwg1658-H2 {{destructor of 'H' is implicitly deleted because base class 'B' has an inaccessible destructor}}
   }
 
   namespace MemInit {
-    struct A { A(int); }; // #dr1658-A3
+    struct A { A(int); }; // #cwg1658-A3
     struct B : virtual A {
       B() {}
       virtual void f() = 0;
     };
     struct C : virtual A {
       C() {}
-      // expected-error@-1 {{constructor for 'dr1658::MemInit::C' must explicitly initialize the base class 'A' which does not have a default constructor}}
-      //   expected-note@#dr1658-A3 {{'dr1658::MemInit::A' declared here}}
+      // expected-error@-1 {{constructor for 'cwg1658::MemInit::C' must explicitly initialize the base class 'A' which does not have a default constructor}}
+      //   expected-note@#cwg1658-A3 {{'cwg1658::MemInit::A' declared here}}
     };
   }
 
@@ -277,7 +277,7 @@ namespace dr1658 { // dr1658: 5
   }
 
   namespace CopyCtor {
-    class A { A(const A&); A(A&&); }; // #dr1658-A5
+    class A { A(const A&); A(A&&); }; // #cwg1658-A5
     // cxx98-error@-1 {{rvalue references are a C++11 extension}}
 
     struct C : virtual A { C(const C&); C(C&&); virtual void foo() = 0; };
@@ -288,46 +288,46 @@ namespace dr1658 { // dr1658: 5
     // cxx98-error@-1 {{rvalue references are a C++11 extension}}
     // cxx98-error@-2 {{defaulted function definitions are a C++11 extension}}
 
-    struct E : A { E(const E&); E(E&&); virtual void foo() = 0; }; // #dr1658-E5
+    struct E : A { E(const E&); E(E&&); virtual void foo() = 0; }; // #cwg1658-E5
     // cxx98-error@-1 {{rvalue references are a C++11 extension}}
-    E::E(const E&) = default; // #dr1658-E5-copy-ctor
+    E::E(const E&) = default; // #cwg1658-E5-copy-ctor
     // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}}
     // cxx98-error@-2 {{base class 'A' has private copy constructor}}
-    //   cxx98-note@-3 {{in defaulted copy constructor for 'dr1658::CopyCtor::E' first required here}}
-    //   cxx98-note@#dr1658-A5 {{implicitly declared private here}}
-    // since-cxx11-error@#dr1658-E5-copy-ctor {{defaulting this copy constructor would delete it after its first declaration}}
-    //   since-cxx11-note@#dr1658-E5 {{copy constructor of 'E' is implicitly deleted because base class 'A' has an inaccessible copy constructor}}
-    E::E(E&&) = default; // #dr1658-E5-move-ctor
+    //   cxx98-note@-3 {{in defaulted copy constructor for 'cwg1658::CopyCtor::E' first required here}}
+    //   cxx98-note@#cwg1658-A5 {{implicitly declared private here}}
+    // since-cxx11-error@#cwg1658-E5-copy-ctor {{defaulting this copy constructor would delete it after its first declaration}}
+    //   since-cxx11-note@#cwg1658-E5 {{copy constructor of 'E' is implicitly deleted because base class 'A' has an inaccessible copy constructor}}
+    E::E(E&&) = default; // #cwg1658-E5-move-ctor
     // cxx98-error@-1 {{rvalue references are a C++11 extension}}
     // cxx98-error@-2 {{defaulted function definitions are a C++11 extension}}
     // cxx98-error@-3 {{base class 'A' has private move constructor}}
-    //   cxx98-note@-4 {{in defaulted move constructor for 'dr1658::CopyCtor::E' first required here}}
-    //   cxx98-note@#dr1658-A5 {{implicitly declared private here}}
-    // since-cxx11-error@#dr1658-E5-move-ctor {{defaulting this move constructor would delete it after its first declaration}}
-    //   since-cxx11-note@#dr1658-E5 {{move constructor of 'E' is implicitly deleted because base class 'A' has an inaccessible move constructor}}
-    struct F : virtual A { F(const F&); F(F&&); }; // #dr1658-F5
+    //   cxx98-note@-4 {{in defaulted move constructor for 'cwg1658::CopyCtor::E' first required here}}
+    //   cxx98-note@#cwg1658-A5 {{implicitly declared private here}}
+    // since-cxx11-error@#cwg1658-E5-move-ctor {{defaulting this move constructor would delete it after its first declaration}}
+    //   since-cxx11-note@#cwg1658-E5 {{move constructor of 'E' is implicitly deleted because base class 'A' has an inaccessible move constructor}}
+    struct F : virtual A { F(const F&); F(F&&); }; // #cwg1658-F5
     // cxx98-error@-1 {{rvalue references are a C++11 extension}}
-    F::F(const F&) = default; // #dr1658-F5-copy-ctor
+    F::F(const F&) = default; // #cwg1658-F5-copy-ctor
     // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}}
     // cxx98-error@-2 {{inherited virtual base class 'A' has private copy constructor}}
-    //   cxx98-note@-3 {{in defaulted copy constructor for 'dr1658::CopyCtor::F' first required here}}
-    //   cxx98-note@#dr1658-A5 {{implicitly declared private here}}
-    // since-cxx11-error@#dr1658-F5-copy-ctor {{defaulting this copy constructor would delete it after its first declaration}}
-    //   since-cxx11-note@#dr1658-F5 {{copy constructor of 'F' is implicitly deleted because base class 'A' has an inaccessible copy constructor}}
-    F::F(F&&) = default; // #dr1658-F5-move-ctor
+    //   cxx98-note@-3 {{in defaulted copy constructor for 'cwg1658::CopyCtor::F' first required here}}
+    //   cxx98-note@#cwg1658-A5 {{implicitly declared private here}}
+    // since-cxx11-error@#cwg1658-F5-copy-ctor {{defaulting this copy constructor would delete it after its first declaration}}
+    //   since-cxx11-note@#cwg1658-F5 {{copy constructor of 'F' is implicitly deleted because base class 'A' has an inaccessible copy constructor}}
+    F::F(F&&) = default; // #cwg1658-F5-move-ctor
     // cxx98-error@-1 {{rvalue references are a C++11 extension}}
     // cxx98-error@-2 {{defaulted function definitions are a C++11 extension}}
     // cxx98-error@-3 {{inherited virtual base class 'A' has private move constructor}}
-    //   cxx98-note@-4 {{in defaulted move constructor for 'dr1658::CopyCtor::F' first required here}}
-    //   cxx98-note@#dr1658-A5 {{implicitly declared private here}}
-    // since-cxx11-error@#dr1658-F5-move-ctor {{defaulting this move constructor would delete it after its first declaration}}
-    //   since-cxx11-note@#dr1658-F5 {{move constructor of 'F' is implicitly deleted because base class 'A' has an inaccessible move constructor}}
+    //   cxx98-note@-4 {{in defaulted move constructor for 'cwg1658::CopyCtor::F' first required here}}
+    //   cxx98-note@#cwg1658-A5 {{implicitly declared private here}}
+    // since-cxx11-error@#cwg1658-F5-move-ctor {{defaulting this move constructor would delete it after its first declaration}}
+    //   since-cxx11-note@#cwg1658-F5 {{move constructor of 'F' is implicitly deleted because base class 'A' has an inaccessible move constructor}}
   }
 
-  // assignment case is superseded by dr2180
+  // assignment case is superseded by cwg2180
 }
 
-namespace dr1672 { // dr1672: 7
+namespace cwg1672 { // cwg1672: 7
   struct Empty {};
   struct A : Empty {};
   struct B { Empty e; };
@@ -352,9 +352,9 @@ namespace dr1672 { // dr1672: 7
   static_assert(!__is_standard_layout(Y<X>), "");
 }
 
-namespace dr1684 { // dr1684: 3.6
+namespace cwg1684 { // cwg1684: 3.6
 #if __cplusplus >= 201103L
-  struct NonLiteral { // #dr1684-struct
+  struct NonLiteral { // #cwg1684-struct
     NonLiteral();
     constexpr int f() { return 0; }
     // cxx11-warning@-1 {{'constexpr' non-static member function will not be implicitly 'const' in C++14; add 'const' to avoid a change in behavior}}
@@ -362,34 +362,34 @@ namespace dr1684 { // dr1684: 3.6
   constexpr int f(NonLiteral &) { return 0; }
   constexpr int f(NonLiteral) { return 0; }
   // cxx11-20-error@-1 {{constexpr function's 1st parameter type 'NonLiteral' is not a literal type}}
-  //   cxx11-20-note@#dr1684-struct {{'NonLiteral' is not literal because it is not an aggregate and has no constexpr constructors other than copy or move constructors}}
+  //   cxx11-20-note@#cwg1684-struct {{'NonLiteral' is not literal because it is not an aggregate and has no constexpr constructors other than copy or move constructors}}
 #endif
 }
 
-namespace dr1687 { // dr1687: 7
+namespace cwg1687 { // cwg1687: 7
   template<typename T> struct To {
-    operator T(); // #dr1687-op-T
+    operator T(); // #cwg1687-op-T
   };
 
   int *a = To<int*>() + 100.0;
   // expected-error@-1 {{invalid operands to binary expression ('To<int *>' and 'double')}}
-  //   expected-note@#dr1687-op-T {{first operand was implicitly converted to type 'int *'}}
-  //   since-cxx20-note@#dr1687-op-T {{second operand was implicitly converted to type 'dr1687::E2'}}
+  //   expected-note@#cwg1687-op-T {{first operand was implicitly converted to type 'int *'}}
+  //   since-cxx20-note@#cwg1687-op-T {{second operand was implicitly converted to type 'cwg1687::E2'}}
   int *b = To<int*>() + To<double>();
   // expected-error@-1 {{invalid operands to binary expression ('To<int *>' and 'To<double>')}}
-  //   expected-note@#dr1687-op-T {{first operand was implicitly converted to type 'int *'}}
-  //   expected-note@#dr1687-op-T {{second operand was implicitly converted to type 'double'}}
+  //   expected-note@#cwg1687-op-T {{first operand was implicitly converted to type 'int *'}}
+  //   expected-note@#cwg1687-op-T {{second operand was implicitly converted to type 'double'}}
 
 #if __cplusplus >= 202002L
   enum E1 {};
   enum E2 {};
   auto c = To<E1>() <=> To<E2>();
   // since-cxx20-error@-1 {{invalid operands to binary expression ('To<E1>' and 'To<E2>')}}
-  //   since-cxx20-note@#dr1687-op-T {{operand was implicitly converted to type 'dr1687::E}}
+  //   since-cxx20-note@#cwg1687-op-T {{operand was implicitly converted to type 'cwg1687::E}}
 #endif
 }
 
-namespace dr1690 { // dr1690: 9
+namespace cwg1690 { // cwg1690: 9
   // See also the various tests in "CXX/basic/basic.lookup/basic.lookup.argdep".
 #if __cplusplus >= 201103L
   namespace N {
@@ -404,7 +404,7 @@ namespace dr1690 { // dr1690: 9
 #endif
 }
 
-namespace dr1691 { // dr1691: 9
+namespace cwg1691 { // cwg1691: 9
 #if __cplusplus >= 201103L
   namespace N {
     namespace M {
@@ -412,19 +412,19 @@ namespace dr1691 { // dr1691: 9
       void f(E);
     }
     enum M::E : int {};
-    void g(M::E); // #dr1691-g
+    void g(M::E); // #cwg1691-g
   }
   void test() {
     N::M::E e;
     f(e); // ok
     g(e);
     // since-cxx11-error@-1 {{use of undeclared identifier 'g'; did you mean 'N::g'?}}
-    //   since-cxx11-note@#dr1691-g {{'N::g' declared here}}
+    //   since-cxx11-note@#cwg1691-g {{'N::g' declared here}}
   }
 #endif
 }
 
-namespace dr1692 { // dr1692: 9
+namespace cwg1692 { // cwg1692: 9
   namespace N {
     struct A {
       struct B {
@@ -439,7 +439,7 @@ namespace dr1692 { // dr1692: 9
   }
 }
 
-namespace dr1696 { // dr1696: 7
+namespace cwg1696 { // cwg1696: 7
   namespace std_examples {
 #if __cplusplus >= 201402L
     extern struct A a;
@@ -456,66 +456,66 @@ namespace dr1696 { // dr1696: 7
   struct A { A(); ~A(); };
 #if __cplusplus >= 201103L
   struct B {
-    A &&a; // #dr1696-a
+    A &&a; // #cwg1696-a
     B() : a{} {}
     // since-cxx11-error@-1 {{reference member 'a' binds to a temporary object whose lifetime would be shorter than the lifetime of the constructed object}}
-    //   since-cxx11-note@#dr1696-a {{reference member declared here}}
+    //   since-cxx11-note@#cwg1696-a {{reference member declared here}}
   } b;
 #endif
 
   struct C {
     C();
-    const A &a; // #dr1696-C-a
+    const A &a; // #cwg1696-C-a
   };
   C::C() : a(A()) {}
   // expected-error@-1 {{reference member 'a' binds to a temporary object whose lifetime would be shorter than the lifetime of the constructed object}}
-  //   expected-note@#dr1696-C-a {{reference member declared here}}
+  //   expected-note@#cwg1696-C-a {{reference member declared here}}
 
 #if __cplusplus >= 201103L
-  // This is OK in C++14 onwards, per DR1815, though we don't support that yet:
+  // This is OK in C++14 onwards, per CWG1815, though we don't support that yet:
   //   D1 d1 = {};
   // is equivalent to
   //   D1 d1 = {A()};
   // ... which lifetime-extends the A temporary.
   struct D1 {
   // cxx11-error@-1 {{reference member 'a' binds to a temporary object whose lifetime would be shorter than the lifetime of the constructed object}}
-  //   cxx11-note@#dr1696-d1 {{in implicit default constructor for 'dr1696::D1' first required here}}
-  //   cxx11-note@#dr1696-D1-a {{initializing field 'a' with default member initializer}}
-    const A &a = A(); // #dr1696-D1-a
+  //   cxx11-note@#cwg1696-d1 {{in implicit default constructor for 'cwg1696::D1' first required here}}
+  //   cxx11-note@#cwg1696-D1-a {{initializing field 'a' with default member initializer}}
+    const A &a = A(); // #cwg1696-D1-a
   };
-  D1 d1 = {}; // #dr1696-d1
+  D1 d1 = {}; // #cwg1696-d1
   // since-cxx14-warning@-1 {{lifetime extension of temporary created by aggregate initialization using a default member initializer is not yet supported; lifetime of temporary will end at the end of the full-expression}}
-  //   since-cxx14-note@#dr1696-D1-a {{initializing field 'a' with default member initializer}}
+  //   since-cxx14-note@#cwg1696-D1-a {{initializing field 'a' with default member initializer}}
 
   struct D2 {
-    const A &a = A(); // #dr1696-D2-a
+    const A &a = A(); // #cwg1696-D2-a
     D2() {}
     // since-cxx11-error@-1 {{reference member 'a' binds to a temporary object whose lifetime would be shorter than the lifetime of the constructed object}}
-    //   since-cxx11-note@#dr1696-D2-a {{initializing field 'a' with default member initializer}}
+    //   since-cxx11-note@#cwg1696-D2-a {{initializing field 'a' with default member initializer}}
   };
 
   struct D3 {
   // since-cxx11-error@-1 {{reference member 'a' binds to a temporary object whose lifetime would be shorter than the lifetime of the constructed object}}
-  //   since-cxx11-note@#dr1696-d3 {{in implicit default constructor for 'dr1696::D3' first required here}}
-  //   since-cxx11-note@#dr1696-D3-a {{initializing field 'a' with default member initializer}}
-    const A &a = A(); // #dr1696-D3-a
+  //   since-cxx11-note@#cwg1696-d3 {{in implicit default constructor for 'cwg1696::D3' first required here}}
+  //   since-cxx11-note@#cwg1696-D3-a {{initializing field 'a' with default member initializer}}
+    const A &a = A(); // #cwg1696-D3-a
   };
-  D3 d3; // #dr1696-d3
+  D3 d3; // #cwg1696-d3
 
   struct haslist1 {
-    std::initializer_list<int> il; // #dr1696-il-1
+    std::initializer_list<int> il; // #cwg1696-il-1
     haslist1(int i) : il{i, 2, 3} {}
     // since-cxx11-error@-1 {{backing array for 'std::initializer_list' member 'il' is a temporary object whose lifetime would be shorter than the lifetime of the constructed object}}
-    //   since-cxx11-note@#dr1696-il-1 {{'std::initializer_list' member declared here}}
+    //   since-cxx11-note@#cwg1696-il-1 {{'std::initializer_list' member declared here}}
   };
 
   struct haslist2 {
-    std::initializer_list<int> il; // #dr1696-il-2
+    std::initializer_list<int> il; // #cwg1696-il-2
     haslist2();
   };
   haslist2::haslist2() : il{1, 2} {}
   // since-cxx11-error@-1 {{backing array for 'std::initializer_list' member 'il' is a temporary object whose lifetime would be shorter than the lifetime of the constructed object}}
-  //   since-cxx11-note@#dr1696-il-2 {{'std::initializer_list' member declared here}}
+  //   since-cxx11-note@#cwg1696-il-2 {{'std::initializer_list' member declared here}}
 
   struct haslist3 {
     std::initializer_list<int> il = {1, 2, 3};
@@ -523,17 +523,17 @@ namespace dr1696 { // dr1696: 7
 
   struct haslist4 {
   // since-cxx11-error@-1 {{backing array for 'std::initializer_list' member 'il' is a temporary object whose lifetime would be shorter than the lifetime of the constructed object}}
-  //   since-cxx11-note@#dr1696-hl4 {{in implicit default constructor for 'dr1696::haslist4' first required here}}
-  //   since-cxx11-note@#dr1696-il-4 {{initializing field 'il' with default member initializer}}
-    std::initializer_list<int> il = {1, 2, 3}; // #dr1696-il-4
+  //   since-cxx11-note@#cwg1696-hl4 {{in implicit default constructor for 'cwg1696::haslist4' first required here}}
+  //   since-cxx11-note@#cwg1696-il-4 {{initializing field 'il' with default member initializer}}
+    std::initializer_list<int> il = {1, 2, 3}; // #cwg1696-il-4
   };
-  haslist4 hl4; // #dr1696-hl4
+  haslist4 hl4; // #cwg1696-hl4
 
   struct haslist5 {
-    std::initializer_list<int> il = {1, 2, 3}; // #dr1696-il-5
+    std::initializer_list<int> il = {1, 2, 3}; // #cwg1696-il-5
     haslist5() {}
     // since-cxx11-error@-1 {{backing array for 'std::initializer_list' member 'il' is a temporary object whose lifetime would be shorter than the lifetime of the constructed object}}
-    //   since-cxx11-note@#dr1696-il-5 {{nitializing field 'il' with default member initializer}}
+    //   since-cxx11-note@#cwg1696-il-5 {{nitializing field 'il' with default member initializer}}
   };
 #endif
 }
diff --git a/clang/test/CXX/drs/dr17xx.cpp b/clang/test/CXX/drs/dr17xx.cpp
index d3cb5e58f06b..fb53a56923b1 100644
--- a/clang/test/CXX/drs/dr17xx.cpp
+++ b/clang/test/CXX/drs/dr17xx.cpp
@@ -6,7 +6,7 @@
 // RUN: %clang_cc1 -std=c++23 %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++2c %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors
 
-namespace dr1710 { // dr1710: no
+namespace cwg1710 { // cwg1710: no
 // FIXME: all of the following is well-formed
 template <typename T> struct D1 : T::template B<int>::template C<int> {};
 template <typename T> struct D2 : T::B<int>::template C<int> {};
@@ -16,9 +16,9 @@ template <typename T> struct D3 : T::template B<int>::C<int> {};
 template <typename T> struct D4 : T::B<int>::C<int> {};
 // expected-error@-1 {{use 'template' keyword to treat 'B' as a dependent template name}}
 // expected-error@-2 {{use 'template' keyword to treat 'C' as a dependent template name}}
-} // namespace dr1710
+} // namespace cwg1710
 
-namespace dr1715 { // dr1715: 3.9
+namespace cwg1715 { // cwg1715: 3.9
 #if __cplusplus >= 201103L
   struct B {
     template<class T> B(T, typename T::Q);
@@ -32,21 +32,21 @@ namespace dr1715 { // dr1715: 3.9
   struct D : B {
     using B::B;
   };
-  struct E : B { // #dr1715-E
-    template<class T> E(T t, typename T::Q q) : B(t, q) {} // #dr1715-E-ctor
+  struct E : B { // #cwg1715-E
+    template<class T> E(T t, typename T::Q q) : B(t, q) {} // #cwg1715-E-ctor
   };
 
   B b(S(), 1);
   D d(S(), 2);
   E e(S(), 3);
   // since-cxx11-error@-1 {{no matching constructor for initialization of 'E'}}
-  //   since-cxx11-note@#dr1715-E-ctor {{candidate template ignored: substitution failure [with T = S]: 'Q' is a private member of 'dr1715::S'}}
-  //   since-cxx11-note@#dr1715-E {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 2 were provided}}
-  //   since-cxx11-note@#dr1715-E {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 2 were provided}}
+  //   since-cxx11-note@#cwg1715-E-ctor {{candidate template ignored: substitution failure [with T = S]: 'Q' is a private member of 'cwg1715::S'}}
+  //   since-cxx11-note@#cwg1715-E {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 2 were provided}}
+  //   since-cxx11-note@#cwg1715-E {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 2 were provided}}
 #endif
 }
 
-namespace dr1719 { // dr1719: 19
+namespace cwg1719 { // cwg1719: 19
 #if __cplusplus >= 201103L
 struct CStruct {
   int one;
@@ -72,9 +72,9 @@ static_assert(__is_layout_compatible(const int, volatile int), "");
 static_assert(__is_layout_compatible(CStruct, CStructWithQualifiers), "");
 static_assert(__is_layout_compatible(int[], const volatile int[]), "");
 #endif
-} // namespace dr1719
+} // namespace cwg1719
 
-namespace dr1722 { // dr1722: 9
+namespace cwg1722 { // cwg1722: 9
 #if __cplusplus >= 201103L
 void f() {
   const auto lambda = [](int x) { return x + 1; };
@@ -84,9 +84,9 @@ void f() {
       "Lambda-to-function-pointer conversion is expected to be noexcept");
 }
 #endif
-} // namespace dr1722
+} // namespace cwg1722
 
-namespace dr1734 { // dr1734: no
+namespace cwg1734 { // cwg1734: no
 #if __cplusplus >= 201103L
 struct A {
     A(const A&) = delete;
@@ -98,7 +98,7 @@ static_assert(__is_trivially_copyable(A), "");
 #endif
 }
 
-namespace dr1736 { // dr1736: 3.9
+namespace cwg1736 { // cwg1736: 3.9
 #if __cplusplus >= 201103L
 struct S {
   template <class T> S(T t) {
@@ -107,17 +107,17 @@ struct S {
     };
     typename T::type value;
     // since-cxx11-error@-1 {{type 'int' cannot be used prior to '::' because it has no members}}
-    //   since-cxx11-note@#dr1736-l {{in instantiation of function template specialization 'dr1736::S::S<int>' requested here}}
-    //   since-cxx11-note@#dr1736-s {{in instantiation of function template specialization 'dr1736::S::S<dr1736::Q>' requested here}}
-    L l(value); // #dr1736-l
+    //   since-cxx11-note@#cwg1736-l {{in instantiation of function template specialization 'cwg1736::S::S<int>' requested here}}
+    //   since-cxx11-note@#cwg1736-s {{in instantiation of function template specialization 'cwg1736::S::S<cwg1736::Q>' requested here}}
+    L l(value); // #cwg1736-l
   }
 };
 struct Q { typedef int type; } q;
-S s(q); // #dr1736-s
+S s(q); // #cwg1736-s
 #endif
 }
 
-namespace dr1738 { // dr1738: sup P0136R1
+namespace cwg1738 { // cwg1738: sup P0136R1
 #if __cplusplus >= 201103L
 struct A {
   template <typename T>
@@ -134,9 +134,9 @@ template B::B(int, double);
 #endif
 }
 
-// dr1748 is in dr1748.cpp
+// cwg1748 is in cwg1748.cpp
 
-namespace dr1753 { // dr1753: 11
+namespace cwg1753 { // cwg1753: 11
   typedef int T;
   struct A { typedef int T; };
   namespace B { typedef int T; }
@@ -145,9 +145,9 @@ namespace dr1753 { // dr1753: 11
     n.~T();
     n.T::~T();
 
-    n.dr1753::~T();
-    // expected-error@-1 {{'dr1753' does not refer to a type name in pseudo-destructor expression; expected the name of type 'T' (aka 'int')}}
-    n.dr1753::T::~T();
+    n.cwg1753::~T();
+    // expected-error@-1 {{'cwg1753' does not refer to a type name in pseudo-destructor expression; expected the name of type 'T' (aka 'int')}}
+    n.cwg1753::T::~T();
 
     n.A::~T();
     // expected-error@-1 {{the type of object expression ('T' (aka 'int')) does not match the type being destroyed ('A') in pseudo-destructor expression}}
@@ -167,7 +167,7 @@ namespace dr1753 { // dr1753: 11
   }
 }
 
-namespace dr1756 { // dr1756: 3.7
+namespace cwg1756 { // cwg1756: 3.7
 #if __cplusplus >= 201103L
   // Direct-list-initialization of a non-class object
   
@@ -178,7 +178,7 @@ namespace dr1756 { // dr1756: 3.7
 #endif
 }
 
-namespace dr1758 { // dr1758: 3.7
+namespace cwg1758 { // cwg1758: 3.7
 #if __cplusplus >= 201103L
   // Explicit conversion in copy/move list initialization
 
@@ -197,7 +197,7 @@ namespace dr1758 { // dr1758: 3.7
 #endif
 }
 
-namespace dr1762 { // dr1762: 14
+namespace cwg1762 { // cwg1762: 14
 #if __cplusplus >= 201103L
   float operator ""_E(const char *);
   float operator ""E(const char *);
@@ -206,9 +206,9 @@ namespace dr1762 { // dr1762: 14
 #endif
 }
 
-// dr1772 is in dr177x.cpp
+// cwg1772 is in cwg177x.cpp
 
-namespace dr1778 { // dr1778: 9
+namespace cwg1778 { // cwg1778: 9
   // Superseded by P1286R2.
 #if __cplusplus >= 201103L
   struct A { A() noexcept(true) = default; };
@@ -223,9 +223,9 @@ namespace dr1778 { // dr1778: 9
 #endif
 }
 
-// dr1779 is in dr177x.cpp
+// cwg1779 is in cwg177x.cpp
 
-namespace dr1794 { // dr1794: yes
+namespace cwg1794 { // cwg1794: yes
                    // NB: dup 1710
 #if __cplusplus >= 201103L
 template <template <typename> class Template> struct Internal {
@@ -238,4 +238,4 @@ using Instantiate = Template<Arg>;
 template <template <typename> class Template, typename Argument>
 using Bind = Instantiate<Internal<Template>::template Bind, Argument>;
 #endif
-} // namespace dr1794
+} // namespace cwg1794
diff --git a/clang/test/CXX/drs/dr18xx.cpp b/clang/test/CXX/drs/dr18xx.cpp
index e78730e8992c..3a2248a1af55 100644
--- a/clang/test/CXX/drs/dr18xx.cpp
+++ b/clang/test/CXX/drs/dr18xx.cpp
@@ -11,24 +11,24 @@
 // cxx98-error@-1 {{variadic macros are a C99 feature}}
 #endif
 
-namespace dr1800 { // dr1800: 2.9
+namespace cwg1800 { // cwg1800: 2.9
 struct A { union { int n; }; };
 static_assert(__is_same(__decltype(&A::n), int A::*), "");
-} // namespace dr1800
+} // namespace cwg1800
 
-namespace dr1801 { // dr1801: 2.8
+namespace cwg1801 { // cwg1801: 2.8
 static union {
   int i;
 };
 
-template <int &> struct S {}; // #dr1801-S
-S<i> V; // #dr1801-S-i
+template <int &> struct S {}; // #cwg1801-S
+S<i> V; // #cwg1801-S-i
 // cxx98-14-error@-1 {{non-type template argument does not refer to any declaration}}
-//   cxx98-14-note@#dr1801-S {{template parameter is declared here}}
-// cxx17-error@#dr1801-S-i {{non-type template argument refers to subobject '.i'}}
+//   cxx98-14-note@#cwg1801-S {{template parameter is declared here}}
+// cxx17-error@#cwg1801-S-i {{non-type template argument refers to subobject '.i'}}
 }
 
-namespace dr1802 { // dr1802: 3.1
+namespace cwg1802 { // cwg1802: 3.1
 #if __cplusplus >= 201103L
 // Using a Wikipedia example of surrogate pair:
 // https://en.wikipedia.org/wiki/UTF-16#Examples
@@ -37,9 +37,9 @@ static_assert(a[0] == 0xD801, "");
 static_assert(a[1] == 0xDC37, "");
 static_assert(a[2] == 0x0, "");
 #endif
-} // namespace dr1802
+} // namespace cwg1802
 
-namespace dr1803 { // dr1803: 2.9
+namespace cwg1803 { // cwg1803: 2.9
 #if __cplusplus >= 201103L
 struct A {
   enum E : int;
@@ -50,9 +50,9 @@ struct A {
   enum struct ES {};
 };
 #endif
-} // namespace dr1803
+} // namespace cwg1803
 
-namespace dr1804 { // dr1804: 2.7
+namespace cwg1804 { // cwg1804: 2.7
 template <typename, typename>
 struct A {
   void f1();
@@ -162,11 +162,11 @@ void A<double, U>::C<V>::f4() {
   D d;
   d.i = 0;
 }
-} // namespace dr1804
+} // namespace cwg1804
 
-// dr1807 is in dr1807.cpp
+// cwg1807 is in cwg1807.cpp
 
-namespace dr1812 { // dr1812: no
+namespace cwg1812 { // cwg1812: no
                    // NB: dup 1710
 #if __cplusplus >= 201103L
 template <typename T> struct A {
@@ -174,9 +174,9 @@ template <typename T> struct A {
   // since-cxx11-error@-1 {{use 'template' keyword to treat 'C' as a dependent template name}}
 };
 #endif
-} // namespace dr1812
+} // namespace cwg1812
 
-namespace dr1813 { // dr1813: 7
+namespace cwg1813 { // cwg1813: 7
   struct B { int i; };
   struct C : B {};
   struct D : C {};
@@ -198,7 +198,7 @@ namespace dr1813 { // dr1813: 7
   static_assert(!__is_standard_layout(U), "");
 }
 
-namespace dr1814 { // dr1814: yes
+namespace cwg1814 { // cwg1814: yes
 #if __cplusplus >= 201103L
   void test() {
     auto lam = [](int x = 42) { return x; };
@@ -206,23 +206,23 @@ namespace dr1814 { // dr1814: yes
 #endif
 }
 
-namespace dr1815 { // dr1815: no
+namespace cwg1815 { // cwg1815: no
 #if __cplusplus >= 201402L
   // FIXME: needs codegen test
-  struct A { int &&r = 0; }; // #dr1815-A 
+  struct A { int &&r = 0; }; // #cwg1815-A 
   A a = {};
   // since-cxx14-warning@-1 {{lifetime extension of temporary created by aggregate initialization using a default member initializer is not yet supported; lifetime of temporary will end at the end of the full-expression}} FIXME
-  //   since-cxx14-note@#dr1815-A {{initializing field 'r' with default member initializer}}
+  //   since-cxx14-note@#cwg1815-A {{initializing field 'r' with default member initializer}}
 
-  struct B { int &&r = 0; }; // #dr1815-B
+  struct B { int &&r = 0; }; // #cwg1815-B
   // since-cxx14-error@-1 {{reference member 'r' binds to a temporary object whose lifetime would be shorter than the lifetime of the constructed object}}
-  //   since-cxx14-note@#dr1815-B {{initializing field 'r' with default member initializer}}
-  //   since-cxx14-note@#dr1815-b {{in implicit default constructor for 'dr1815::B' first required here}}
-  B b; // #dr1815-b
+  //   since-cxx14-note@#cwg1815-B {{initializing field 'r' with default member initializer}}
+  //   since-cxx14-note@#cwg1815-b {{in implicit default constructor for 'cwg1815::B' first required here}}
+  B b; // #cwg1815-b
 #endif
 }
 
-namespace dr1821 { // dr1821: 2.9
+namespace cwg1821 { // cwg1821: 2.9
 struct A {
   template <typename> struct B {
     void f();
@@ -236,9 +236,9 @@ struct A {
   void C::f() {}
   // expected-error@-1 {{non-friend class member 'f' cannot have a qualified name}}
 };
-} // namespace dr1821
+} // namespace cwg1821
 
-namespace dr1822 { // dr1822: yes
+namespace cwg1822 { // cwg1822: yes
 #if __cplusplus >= 201103L
   double a;
   auto x = [] (int a) {
@@ -247,7 +247,7 @@ namespace dr1822 { // dr1822: yes
 #endif
 }
 
-namespace dr1837 { // dr1837: 3.3
+namespace cwg1837 { // cwg1837: 3.3
 #if __cplusplus >= 201103L
   template <typename T>
   struct Fish { static const bool value = true; };
@@ -291,7 +291,7 @@ namespace dr1837 { // dr1837: 3.3
 #endif
 }
 
-namespace dr1872 { // dr1872: 9
+namespace cwg1872 { // cwg1872: 9
 #if __cplusplus >= 201103L
   template<typename T> struct A : T {
     constexpr int f() const { return 0; }
@@ -320,7 +320,7 @@ namespace dr1872 { // dr1872: 9
 #endif
 }
 
-namespace dr1878 { // dr1878: 18
+namespace cwg1878 { // cwg1878: 18
 #if __cplusplus >= 201402L
 #if __cplusplus >= 202002L
 template <typename T>
@@ -355,7 +355,7 @@ struct S {
 #endif
 }
 
-namespace dr1881 { // dr1881: 7
+namespace cwg1881 { // cwg1881: 7
   struct A { int a : 4; };
   struct B : A { int b : 3; };
   static_assert(__is_standard_layout(A), "");
@@ -367,17 +367,17 @@ namespace dr1881 { // dr1881: 7
   static_assert(!__is_standard_layout(D), "");
 }
 
-namespace dr1890 { // dr1890: no drafting 2018-06-04
+namespace cwg1890 { // cwg1890: no drafting 2018-06-04
 // FIXME: current consensus for CWG2335 is that the examples are well-formed.
 namespace ex1 {
 #if __cplusplus >= 201402L
 struct A {
   struct B {
-    auto foo() { return 0; } // #dr1890-foo
+    auto foo() { return 0; } // #cwg1890-foo
   };
   decltype(B().foo()) x;
   // since-cxx14-error@-1 {{function 'foo' with deduced return type cannot be used before it is defined}}
-  //   since-cxx14-note@#dr1890-foo {{'foo' declared here}}
+  //   since-cxx14-note@#cwg1890-foo {{'foo' declared here}}
 };
 #endif
 } // namespace ex1
@@ -389,17 +389,17 @@ struct Bar {
     int a = 0;
   };
   static_assert(__is_constructible(Baz), "");
-  // since-cxx11-error@-1 {{static assertion failed due to requirement '__is_constructible(dr1890::ex2::Bar::Baz)'}}
+  // since-cxx11-error@-1 {{static assertion failed due to requirement '__is_constructible(cwg1890::ex2::Bar::Baz)'}}
 };
 #endif
 } // namespace ex2
-} // namespace dr1890
+} // namespace cwg1890
 
-void dr1891() { // dr1891: 4
+void cwg1891() { // cwg1891: 4
 #if __cplusplus >= 201103L
   int n;
-  auto a = []{}; // #dr1891-a
-  auto b = [=]{ return n; }; // #dr1891-b
+  auto a = []{}; // #cwg1891-a
+  auto b = [=]{ return n; }; // #cwg1891-b
   typedef decltype(a) A;
   typedef decltype(b) B;
 
@@ -410,31 +410,31 @@ void dr1891() { // dr1891: 4
   // C++20 allows default construction for non-capturing lambdas (P0624R2).
   A x;
   // cxx11-17-error@-1 {{no matching constructor for initialization of 'A' (aka '(lambda at}}
-  //   cxx11-17-note@#dr1891-a {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 0 were provided}}
-  //   cxx11-17-note@#dr1891-a {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 0 were provided}}
+  //   cxx11-17-note@#cwg1891-a {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 0 were provided}}
+  //   cxx11-17-note@#cwg1891-a {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 0 were provided}}
   B y;
   // since-cxx11-error@-1 {{no matching constructor for initialization of 'B' (aka '(lambda at}}
-  //   since-cxx11-note@#dr1891-b {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 0 were provided}}
-  //   since-cxx11-note@#dr1891-b {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 0 were provided}}
+  //   since-cxx11-note@#cwg1891-b {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 0 were provided}}
+  //   since-cxx11-note@#cwg1891-b {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 0 were provided}}
 
   // C++20 allows assignment for non-capturing lambdas (P0624R2).
   a = a;
   // cxx11-17-error-re@-1 {{{{object of type '\(lambda at .+\)' cannot be assigned because its copy assignment operator is implicitly deleted}}}}
-  //   cxx11-17-note@#dr1891-a {{lambda expression begins here}}
+  //   cxx11-17-note@#cwg1891-a {{lambda expression begins here}}
   a = static_cast<A&&>(a);
   // cxx11-17-error-re@-1 {{{{object of type '\(lambda at .+\)' cannot be assigned because its copy assignment operator is implicitly deleted}}}}
-  //   cxx11-17-note@#dr1891-a {{lambda expression begins here}}
+  //   cxx11-17-note@#cwg1891-a {{lambda expression begins here}}
   b = b;
   // since-cxx11-error-re@-1 {{{{object of type '\(lambda at .+\)' cannot be assigned because its copy assignment operator is implicitly deleted}}}}
-  //   since-cxx11-note@#dr1891-b {{lambda expression begins here}}
+  //   since-cxx11-note@#cwg1891-b {{lambda expression begins here}}
   b = static_cast<B&&>(b);
   // since-cxx11-error-re@-1 {{{{object of type '\(lambda at .+\)' cannot be assigned because its copy assignment operator is implicitly deleted}}}}
-  //   since-cxx11-note@#dr1891-b {{lambda expression begins here}}
+  //   since-cxx11-note@#cwg1891-b {{lambda expression begins here}}
 #endif
 }
 
-namespace dr1894 { // dr1894: 3.8
-                   // NB: reusing part of dr407 test
+namespace cwg1894 { // cwg1894: 3.8
+                   // NB: reusing part of cwg407 test
 namespace A {
   struct S {};
 }
diff --git a/clang/test/CXX/drs/dr19xx.cpp b/clang/test/CXX/drs/dr19xx.cpp
index f8c1581f0854..3e4f82813f51 100644
--- a/clang/test/CXX/drs/dr19xx.cpp
+++ b/clang/test/CXX/drs/dr19xx.cpp
@@ -8,13 +8,13 @@
 
 namespace std { struct type_info; }
 
-namespace dr1902 { // dr1902: 3.7
+namespace cwg1902 { // cwg1902: 3.7
   struct A {};
   struct B {
-    B(A); // #dr1902-B-A
-    B() = delete; // #dr1902-B-ctor
+    B(A); // #cwg1902-B-A
+    B() = delete; // #cwg1902-B-ctor
     // cxx98-error@-1 {{deleted function definitions are a C++11 extension}}
-    B(const B&) = delete; // #dr1902-B-copy-ctor
+    B(const B&) = delete; // #cwg1902-B-copy-ctor
     // cxx98-error@-1 {{deleted function definitions are a C++11 extension}}
     operator A();
   };
@@ -22,19 +22,19 @@ namespace dr1902 { // dr1902: 3.7
   extern B b1;
   B b2(b1);
   // expected-error@-1 {{call to deleted constructor of 'B'}}
-  //   expected-note@#dr1902-B-copy-ctor {{'B' has been explicitly marked deleted here}}
+  //   expected-note@#cwg1902-B-copy-ctor {{'B' has been explicitly marked deleted here}}
 
 #if __cplusplus >= 201103L
   // This is ambiguous, even though calling the B(const B&) constructor would
   // both directly and indirectly call a deleted function.
   B b({});
   // since-cxx11-error@-1 {{call to constructor of 'B' is ambiguous}}
-  //   since-cxx11-note@#dr1902-B-A {{candidate constructor}}
-  //   since-cxx11-note@#dr1902-B-copy-ctor {{candidate constructor has been explicitly deleted}}
+  //   since-cxx11-note@#cwg1902-B-A {{candidate constructor}}
+  //   since-cxx11-note@#cwg1902-B-copy-ctor {{candidate constructor has been explicitly deleted}}
 #endif
 }
 
-namespace dr1903 { // dr1903: 2.7
+namespace cwg1903 { // cwg1903: 2.7
   namespace A {
     struct a {};
     int a;
@@ -59,7 +59,7 @@ namespace dr1903 { // dr1903: 2.7
   }
 }
 
-namespace dr1909 { // dr1909: 3.7
+namespace cwg1909 { // cwg1909: 3.7
   struct A {
     template<typename T> struct A {};
     // expected-error@-1 {{member 'A' has the same name as its class}}
@@ -80,7 +80,7 @@ namespace dr1909 { // dr1909: 3.7
   };
 }
 
-namespace dr1940 { // dr1940: 3.5
+namespace cwg1940 { // cwg1940: 3.5
 #if __cplusplus >= 201103L
 static union {
   static_assert(true, "");  // ok
@@ -91,7 +91,7 @@ static union {
 #endif
 }
 
-namespace dr1941 { // dr1941: 3.9
+namespace cwg1941 { // cwg1941: 3.9
 #if __cplusplus >= 201402L
 template<typename X>
 struct base {
@@ -121,7 +121,7 @@ derived d2(42, 9);
 #endif
 }
 
-namespace dr1947 { // dr1947: 3.5
+namespace cwg1947 { // cwg1947: 3.5
 #if __cplusplus >= 201402L
 unsigned o = 0'01;  // ok
 unsigned b = 0b'01;
@@ -132,26 +132,26 @@ unsigned x = 0x'01;
 }
 
 #if __cplusplus >= 201103L
-// dr1948: 3.5
+// cwg1948: 3.5
 // FIXME: This diagnostic could be improved.
 void *operator new(__SIZE_TYPE__) noexcept { return nullptr; }
 // since-cxx11-error@-1 {{exception specification in declaration does not match previous declaration}}
 #endif
 
-namespace dr1959 { // dr1959: 3.9
+namespace cwg1959 { // cwg1959: 3.9
 #if __cplusplus >= 201103L
   struct b;
   struct c;
   struct a {
     a() = default;
-    a(const a &) = delete; // #dr1959-copy-ctor
+    a(const a &) = delete; // #cwg1959-copy-ctor
     a(const b &) = delete; // not inherited
-    a(c &&) = delete; // #dr1959-move-ctor
-    template<typename T> a(T) = delete; // #dr1959-temp-ctor
+    a(c &&) = delete; // #cwg1959-move-ctor
+    template<typename T> a(T) = delete; // #cwg1959-temp-ctor
   };
 
-  struct b : a { // #dr1959-b
-    using a::a; // #dr1959-using-a
+  struct b : a { // #cwg1959-b
+    using a::a; // #cwg1959-using-a
   };
 
   a x;
@@ -160,15 +160,15 @@ namespace dr1959 { // dr1959: 3.9
   // where the base class is reference-related to the argument type.
   b y = x;
   // since-cxx11-error@-1 {{no viable conversion from 'a' to 'b'}}
-  //   since-cxx11-note@#dr1959-move-ctor {{candidate inherited constructor not viable: no known conversion from 'a' to 'c &&' for 1st argument}}
-  //   since-cxx11-note@#dr1959-using-a {{constructor from base class 'a' inherited here}}
-  //   since-cxx11-note@#dr1959-b {{candidate constructor (the implicit copy constructor) not viable: cannot bind base class object of type 'a' to derived class reference 'const b &' for 1st argument}}
-  //   since-cxx11-note@#dr1959-temp-ctor {{candidate template ignored: instantiation would take its own class type by value}}
-  //   since-cxx11-note@#dr1959-using-a {{constructor from base class 'a' inherited here}}
+  //   since-cxx11-note@#cwg1959-move-ctor {{candidate inherited constructor not viable: no known conversion from 'a' to 'c &&' for 1st argument}}
+  //   since-cxx11-note@#cwg1959-using-a {{constructor from base class 'a' inherited here}}
+  //   since-cxx11-note@#cwg1959-b {{candidate constructor (the implicit copy constructor) not viable: cannot bind base class object of type 'a' to derived class reference 'const b &' for 1st argument}}
+  //   since-cxx11-note@#cwg1959-temp-ctor {{candidate template ignored: instantiation would take its own class type by value}}
+  //   since-cxx11-note@#cwg1959-using-a {{constructor from base class 'a' inherited here}}
   b z = z;
   // since-cxx11-error@-1 {{call to implicitly-deleted copy constructor of 'b'}}
-  //   since-cxx11-note@#dr1959-b {{copy constructor of 'b' is implicitly deleted because base class 'a' has a deleted copy constructor}}
-  //   since-cxx11-note@#dr1959-copy-ctor {{'a' has been explicitly marked deleted here}}
+  //   since-cxx11-note@#cwg1959-b {{copy constructor of 'b' is implicitly deleted because base class 'a' has a deleted copy constructor}}
+  //   since-cxx11-note@#cwg1959-copy-ctor {{'a' has been explicitly marked deleted here}}
 
   struct c : a {
     using a::a;
@@ -181,7 +181,7 @@ namespace dr1959 { // dr1959: 3.9
 #endif
 }
 
-namespace dr1960 { // dr1960: no
+namespace cwg1960 { // cwg1960: no
 struct A {
 void f() {}
 protected:
@@ -202,7 +202,7 @@ using A::g;
 };
 }
 
-namespace dr1966 { // dr1966: 11
+namespace cwg1966 { // cwg1966: 11
 #if __cplusplus >= 201103L
   struct A {
     enum E : int {1};
@@ -215,12 +215,12 @@ namespace dr1966 { // dr1966: 11
   auto *p3 = true ? new enum G : int {};
   // since-cxx11-error@-1 {{ISO C++ forbids forward references to 'enum' types}}
   // since-cxx11-error@-2 {{allocation of incomplete type 'enum G'}}
-  //   since-cxx11-note@-3 {{forward declaration of 'dr1966::G'}}
+  //   since-cxx11-note@-3 {{forward declaration of 'cwg1966::G'}}
   auto h() -> enum E : int {};
   // since-cxx11-error@-1 {{non-defining declaration of enumeration with a fixed underlying type is only permitted as a standalone declaration}}
 
   enum X : enum Y : int {} {};
-  // since-cxx11-error@-1 {{'dr1966::Y' cannot be defined in a type specifier}}
+  // since-cxx11-error@-1 {{'cwg1966::Y' cannot be defined in a type specifier}}
   struct Q {
     // FIXME: can we emit something nicer than that?
     enum X : enum Y : int {} {};
@@ -231,9 +231,9 @@ namespace dr1966 { // dr1966: 11
 #endif
 }
 
-namespace dr1968 { // dr1968: no
+namespace cwg1968 { // cwg1968: no
 #if __cplusplus >= 201103L
-  // FIXME: According to DR1968, both of these should be considered
+  // FIXME: According to CWG1968, both of these should be considered
   // non-constant.
   static_assert(&typeid(int) == &typeid(int), "");
 
@@ -242,7 +242,7 @@ namespace dr1968 { // dr1968: no
 #endif
 }
 
-namespace dr1991 { // dr1991: 3.9
+namespace cwg1991 { // cwg1991: 3.9
 #if __cplusplus >= 201103L
   struct A {
     A(int, int) = delete;
@@ -260,4 +260,4 @@ namespace dr1991 { // dr1991: 3.9
 #endif
 }
 
-// dr1994: dup 529
+// cwg1994: dup 529
diff --git a/clang/test/CXX/drs/dr1xx.cpp b/clang/test/CXX/drs/dr1xx.cpp
index d55033cef1b6..5b497dda047d 100644
--- a/clang/test/CXX/drs/dr1xx.cpp
+++ b/clang/test/CXX/drs/dr1xx.cpp
@@ -5,61 +5,61 @@
 // RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors
 
-namespace dr100 { // dr100: yes
-  template<const char (*)[4]> struct A {}; // #dr100-A
-  template<const char (&)[4]> struct B {}; // #dr100-B
-  template<const char *> struct C {}; // #dr100-C
-  template<const char &> struct D {}; // #dr100-D
-  A<&"foo"> a; // #dr100-a
-  // cxx98-14-error@#dr100-a {{non-type template argument does not refer to any declaration}}
-  //   cxx98-14-note@#dr100-A {{template parameter is declared here}}
-  // since-cxx17-error@#dr100-a {{pointer to string literal is not allowed in a template argument}}
-  B<"bar"> b; // #dr100-b
-  // cxx98-14-error@#dr100-b {{non-type template argument does not refer to any declaration}}
-  //   cxx98-14-note@#dr100-B {{template parameter is declared here}}
-  // since-cxx17-error@#dr100-b {{reference to string literal is not allowed in a template argument}}
-  C<"baz"> c; // #dr100-c
-  // cxx98-14-error@#dr100-c {{non-type template argument does not refer to any declaration}}
-  //   cxx98-14-note@#dr100-C {{template parameter is declared here}}
-  // since-cxx17-error@#dr100-c {{pointer to subobject of string literal is not allowed in a template argument}}
-  D<*"quux"> d; // #dr100-d
-  // cxx98-14-error@#dr100-d {{non-type template argument does not refer to any declaration}}
-  //   cxx98-14-note@#dr100-D {{template parameter is declared here}}
-  // since-cxx17-error@#dr100-d {{reference to subobject of string literal is not allowed in a template argument}}
-}
-
-namespace dr101 { // dr101: 3.5
-  extern "C" void dr101_f();
+namespace cwg100 { // cwg100: yes
+  template<const char (*)[4]> struct A {}; // #cwg100-A
+  template<const char (&)[4]> struct B {}; // #cwg100-B
+  template<const char *> struct C {}; // #cwg100-C
+  template<const char &> struct D {}; // #cwg100-D
+  A<&"foo"> a; // #cwg100-a
+  // cxx98-14-error@#cwg100-a {{non-type template argument does not refer to any declaration}}
+  //   cxx98-14-note@#cwg100-A {{template parameter is declared here}}
+  // since-cxx17-error@#cwg100-a {{pointer to string literal is not allowed in a template argument}}
+  B<"bar"> b; // #cwg100-b
+  // cxx98-14-error@#cwg100-b {{non-type template argument does not refer to any declaration}}
+  //   cxx98-14-note@#cwg100-B {{template parameter is declared here}}
+  // since-cxx17-error@#cwg100-b {{reference to string literal is not allowed in a template argument}}
+  C<"baz"> c; // #cwg100-c
+  // cxx98-14-error@#cwg100-c {{non-type template argument does not refer to any declaration}}
+  //   cxx98-14-note@#cwg100-C {{template parameter is declared here}}
+  // since-cxx17-error@#cwg100-c {{pointer to subobject of string literal is not allowed in a template argument}}
+  D<*"quux"> d; // #cwg100-d
+  // cxx98-14-error@#cwg100-d {{non-type template argument does not refer to any declaration}}
+  //   cxx98-14-note@#cwg100-D {{template parameter is declared here}}
+  // since-cxx17-error@#cwg100-d {{reference to subobject of string literal is not allowed in a template argument}}
+}
+
+namespace cwg101 { // cwg101: 3.5
+  extern "C" void cwg101_f();
   typedef unsigned size_t;
   namespace X {
-    extern "C" void dr101_f();
+    extern "C" void cwg101_f();
     typedef unsigned size_t;
   }
-  using X::dr101_f;
+  using X::cwg101_f;
   using X::size_t;
-  extern "C" void dr101_f();
+  extern "C" void cwg101_f();
   typedef unsigned size_t;
 }
 
-namespace dr102 { // dr102: yes
+namespace cwg102 { // cwg102: yes
   namespace A {
     template<typename T> T f(T a, T b) { return a + b; }
     // expected-error@-1 {{call to function 'operator+' that is neither visible in the template definition nor found by argument-dependent lookup}}
-    //   expected-note@#dr102-instantiation {{in instantiation of function template specialization 'dr102::A::f<dr102::B::S>' requested here}}
-    //   expected-note@#dr102-operator-plus {{'operator+' should be declared prior to the call site or in namespace 'dr102::B'}}
+    //   expected-note@#cwg102-instantiation {{in instantiation of function template specialization 'cwg102::A::f<cwg102::B::S>' requested here}}
+    //   expected-note@#cwg102-operator-plus {{'operator+' should be declared prior to the call site or in namespace 'cwg102::B'}}
   }
   namespace B {
     struct S {};
   }
-  B::S operator+(B::S, B::S); // #dr102-operator-plus
-  template B::S A::f(B::S, B::S); // #dr102-instantiation
+  B::S operator+(B::S, B::S); // #cwg102-operator-plus
+  template B::S A::f(B::S, B::S); // #cwg102-instantiation
 }
 
-// dr103: na
-// dr104: na lib
-// dr105: na
+// cwg103: na
+// cwg104: na lib
+// cwg105: na
 
-namespace dr106 { // dr106: sup 540
+namespace cwg106 { // cwg106: sup 540
   typedef int &r1;
   typedef r1 &r1;
   typedef const r1 r1;
@@ -75,12 +75,12 @@ namespace dr106 { // dr106: sup 540
   // expected-warning@-1 {{'const' qualifier on reference type 'r2' (aka 'const int &') has no effect}}
 }
 
-namespace dr107 { // dr107: yes
+namespace cwg107 { // cwg107: yes
   struct S {};
   extern "C" S operator+(S, S) { return S(); }
 }
 
-namespace dr108 { // dr108: 2.9
+namespace cwg108 { // cwg108: 2.9
   template<typename T> struct A {
     struct B { typedef int X; };
     B::X x;
@@ -91,7 +91,7 @@ namespace dr108 { // dr108: 2.9
   template<> struct A<int>::B { int X; };
 }
 
-namespace dr109 { // dr109: yes
+namespace cwg109 { // cwg109: yes
   struct A { template<typename T> void f(T); };
   template<typename T> struct B : T {
     using T::template f;
@@ -108,37 +108,37 @@ namespace dr109 { // dr109: yes
   };
 }
 
-namespace dr111 { // dr111: dup 535
+namespace cwg111 { // cwg111: dup 535
   struct A { A(); A(volatile A&, int = 0); A(A&, const char * = "foo"); };
-  struct B : A { B(); }; // #dr111-B
+  struct B : A { B(); }; // #cwg111-B
   const B b1;
   B b2(b1);
   // expected-error@-1 {{no matching constructor for initialization of 'B'}}
-  //   expected-note@#dr111-B {{candidate constructor (the implicit copy constructor) not viable: 1st argument ('const B') would lose const qualifier}}
-  //   expected-note@#dr111-B {{candidate constructor not viable: requires 0 arguments, but 1 was provided}}
+  //   expected-note@#cwg111-B {{candidate constructor (the implicit copy constructor) not viable: 1st argument ('const B') would lose const qualifier}}
+  //   expected-note@#cwg111-B {{candidate constructor not viable: requires 0 arguments, but 1 was provided}}
 }
 
-namespace dr112 { // dr112: yes
+namespace cwg112 { // cwg112: yes
   struct T { int n; };
   typedef T Arr[1];
 
-  const T a1[1] = {}; // #dr112-a1
+  const T a1[1] = {}; // #cwg112-a1
   volatile T a2[1] = {};
-  const Arr a3 = {}; // #dr112-a3
+  const Arr a3 = {}; // #cwg112-a3
   volatile Arr a4 = {};
   template<const volatile T*> struct X {};
   // FIXME: Test this somehow in C++11 and on.
   X<a1> x1;
   // cxx98-error@-1 {{non-type template argument referring to object 'a1' with internal linkage is a C++11 extension}}
-  //   cxx98-note@#dr112-a1 {{non-type template argument refers to object here}}
+  //   cxx98-note@#cwg112-a1 {{non-type template argument refers to object here}}
   X<a2> x2;
   X<a3> x3;
   // cxx98-error@-1 {{non-type template argument referring to object 'a3' with internal linkage is a C++11 extension}}
-  //   cxx98-note@#dr112-a3 {{non-type template argument refers to object here}}
+  //   cxx98-note@#cwg112-a3 {{non-type template argument refers to object here}}
   X<a4> x4;
 }
 
-namespace dr113 { // dr113: yes
+namespace cwg113 { // cwg113: yes
   extern void (*p)();
   void f() {
     no_such_function();
@@ -149,50 +149,50 @@ namespace dr113 { // dr113: yes
   void (*p)() = &g;
 }
 
-namespace dr114 { // dr114: yes
+namespace cwg114 { // cwg114: yes
   struct A {
-    virtual void f(int) = 0; // #dr114-A-f
+    virtual void f(int) = 0; // #cwg114-A-f
   };
   struct B : A {
     template<typename T> void f(T);
     void g() { f(0); }
   } b;
   // expected-error@-1 {{variable type 'struct B' is an abstract class}}
-  //   expected-note@#dr114-A-f {{unimplemented pure virtual method 'f' in 'B'}}
+  //   expected-note@#cwg114-A-f {{unimplemented pure virtual method 'f' in 'B'}}
 }
 
-namespace dr115 { // dr115: 3.0
-  template<typename T> int f(T); // #dr115-f
-  template<typename T> int g(T); // #dr115-g
-  template<typename T> int g(T, int); // #dr115-g-int
+namespace cwg115 { // cwg115: 3.0
+  template<typename T> int f(T); // #cwg115-f
+  template<typename T> int g(T); // #cwg115-g
+  template<typename T> int g(T, int); // #cwg115-g-int
 
   int k1 = f(&f);
   // expected-error@-1 {{no matching function for call to 'f'}}
-  //   expected-note@#dr115-f {{candidate template ignored: couldn't infer template argument 'T'}}
+  //   expected-note@#cwg115-f {{candidate template ignored: couldn't infer template argument 'T'}}
   int k2 = f(&f<int>);
   int k3 = f(&g<int>);
   // expected-error@-1 {{no matching function for call to 'f'}}
-  //   expected-note@#dr115-f {{candidate template ignored: couldn't infer template argument 'T'}}
+  //   expected-note@#cwg115-f {{candidate template ignored: couldn't infer template argument 'T'}}
 
   void h() {
     (void)&f;
     // expected-error@-1 {{address of overloaded function 'f' cannot be cast to type 'void'}}
-    //   expected-note@#dr115-f {{candidate function template}}
+    //   expected-note@#cwg115-f {{candidate function template}}
     (void)&f<int>;
     (void)&g<int>;
     // expected-error@-1 {{address of overloaded function 'g' cannot be cast to type 'void'}}
-    //   expected-note@#dr115-g-int {{candidate function template}}
-    //   expected-note@#dr115-g {{candidate function template}}
+    //   expected-note@#cwg115-g-int {{candidate function template}}
+    //   expected-note@#cwg115-g {{candidate function template}}
 
     &f;
     // expected-error@-1 {{reference to overloaded function could not be resolved; did you mean to call it?}}
-    //   expected-note@#dr115-f {{possible target for call}}
+    //   expected-note@#cwg115-f {{possible target for call}}
     &f<int>;
     // expected-warning@-1 {{expression result unused}}
     &g<int>;
     // expected-error@-1 {{reference to overloaded function could not be resolved; did you mean to call it?}}
-    //   expected-note@#dr115-g-int {{possible target for call}}
-    //   expected-note@#dr115-g {{possible target for call}}
+    //   expected-note@#cwg115-g-int {{possible target for call}}
+    //   expected-note@#cwg115-g {{possible target for call}}
   }
 
   struct S {
@@ -251,43 +251,43 @@ namespace dr115 { // dr115: 3.0
 
 #if __cplusplus >= 201103L
   // Special case kicks in only if a template argument list is specified.
-  template<typename T=int> void with_default(); // #dr115-with-default
+  template<typename T=int> void with_default(); // #cwg115-with-default
   int k10 = f(&with_default);
   // expected-error@-1 {{no matching function for call to 'f'}}
-  //   expected-note@#dr115-f {{candidate template ignored: couldn't infer template argument 'T'}}
+  //   expected-note@#cwg115-f {{candidate template ignored: couldn't infer template argument 'T'}}
   int k11 = f(&with_default<>);
   void k() {
     (void)&with_default;
     // expected-error@-1 {{address of overloaded function 'with_default' cannot be cast to type 'void'}}
-    //   expected-note@#dr115-with-default {{candidate function template}}
+    //   expected-note@#cwg115-with-default {{candidate function template}}
     (void)&with_default<>;
     &with_default;
     // expected-error@-1 {{reference to overloaded function could not be resolved; did you mean to call it?}}
-    //   expected-note@#dr115-with-default {{possible target for call}}
+    //   expected-note@#cwg115-with-default {{possible target for call}}
     &with_default<>;
     // expected-warning@-1 {{expression result unused}}
   }
 #endif
 }
 
-namespace dr116 { // dr116: yes
+namespace cwg116 { // cwg116: yes
   template<int> struct A {};
-  template<int N> void f(A<N>) {} // #dr116-f-N
+  template<int N> void f(A<N>) {} // #cwg116-f-N
   template<int M> void f(A<M>) {}
   // expected-error@-1 {{redefinition of 'f'}}
-  //   expected-note@#dr116-f-N {{previous definition is here}}
-  template<typename T> void f(A<sizeof(T)>) {} // #dr116-f-T
+  //   expected-note@#cwg116-f-N {{previous definition is here}}
+  template<typename T> void f(A<sizeof(T)>) {} // #cwg116-f-T
   template<typename U> void f(A<sizeof(U)>) {}
   // expected-error@-1 {{redefinition of 'f'}}
-  //   expected-note@#dr116-f-T {{previous definition is here}}
+  //   expected-note@#cwg116-f-T {{previous definition is here}}
 }
 
-// dr117: na
-// dr118 is in dr118.cpp
-// dr119: na
-// dr120: na
+// cwg117: na
+// cwg118 is in cwg118.cpp
+// cwg119: na
+// cwg120: na
 
-namespace dr121 { // dr121: yes
+namespace cwg121 { // cwg121: yes
   struct X {
     template<typename T> struct Y {};
   };
@@ -300,32 +300,32 @@ namespace dr121 { // dr121: yes
   Z<X> z;
 }
 
-namespace dr122 { // dr122: yes
+namespace cwg122 { // cwg122: yes
   template<typename T> void f();
   void g() { f<int>(); }
 }
 
-// dr123: na
-// dr124 is in dr124.cpp
+// cwg123: na
+// cwg124 is in cwg124.cpp
 
-// dr125: yes
-struct dr125_A { struct dr125_B {}; }; // #dr125_B
-dr125_A::dr125_B dr125_C();
-namespace dr125_B { dr125_A dr125_C(); }
-namespace dr125 {
+// cwg125: yes
+struct cwg125_A { struct cwg125_B {}; }; // #cwg125_B
+cwg125_A::cwg125_B cwg125_C();
+namespace cwg125_B { cwg125_A cwg125_C(); }
+namespace cwg125 {
   struct X {
-    friend dr125_A::dr125_B (::dr125_C)(); // ok
-    friend dr125_A (::dr125_B::dr125_C)(); // ok
-    friend dr125_A::dr125_B::dr125_C(); // #dr125_C
-    // expected-error@#dr125_C {{missing return type for function 'dr125_C'; did you mean the constructor name 'dr125_B'?}}
-    // cxx98-error@#dr125_C {{'dr125_B' is missing exception specification 'throw()'}}
-    //   cxx98-note@#dr125_B {{previous declaration is here}}
-    // since-cxx11-error@#dr125_C {{'dr125_B' is missing exception specification 'noexcept'}}
-    //   since-cxx11-note@#dr125_B {{previous declaration is here}}
+    friend cwg125_A::cwg125_B (::cwg125_C)(); // ok
+    friend cwg125_A (::cwg125_B::cwg125_C)(); // ok
+    friend cwg125_A::cwg125_B::cwg125_C(); // #cwg125_C
+    // expected-error@#cwg125_C {{missing return type for function 'cwg125_C'; did you mean the constructor name 'cwg125_B'?}}
+    // cxx98-error@#cwg125_C {{'cwg125_B' is missing exception specification 'throw()'}}
+    //   cxx98-note@#cwg125_B {{previous declaration is here}}
+    // since-cxx11-error@#cwg125_C {{'cwg125_B' is missing exception specification 'noexcept'}}
+    //   since-cxx11-note@#cwg125_B {{previous declaration is here}}
   };
 }
 
-namespace dr126 { // dr126: partial
+namespace cwg126 { // cwg126: partial
   // FIXME: We do not yet generate correct code for this change:
   // eg:
   //   catch (void*&) should catch void* but not int*
@@ -350,23 +350,23 @@ namespace dr126 { // dr126: partial
   struct A {
     virtual void cp() throw(C*);
     virtual void dp() throw(C*);
-    virtual void ep() throw(C*); // #dr126-ep
-    virtual void fp() throw(C*); // #dr126-fp
+    virtual void ep() throw(C*); // #cwg126-ep
+    virtual void fp() throw(C*); // #cwg126-fp
     virtual void gp() throw(C*);
-    virtual void hp() throw(C*); // #dr126-hp
+    virtual void hp() throw(C*); // #cwg126-hp
 
     virtual void cr() throw(C&);
     virtual void dr() throw(C&);
-    virtual void er() throw(C&); // #dr126-er
-    virtual void fr() throw(C&); // #dr126-fr
+    virtual void er() throw(C&); // #cwg126-er
+    virtual void fr() throw(C&); // #cwg126-fr
     virtual void gr() throw(C&);
-    virtual void hr() throw(C&); // #dr126-hr
+    virtual void hr() throw(C&); // #cwg126-hr
 
     virtual void pv() throw(void*);
 
     virtual void np() throw(C*);
     virtual void npm() throw(int C::*);
-    virtual void nr() throw(C*&); // #dr126-nr
+    virtual void nr() throw(C*&); // #cwg126-nr
     virtual void ncr() throw(C*const&);
 
     virtual void ref1() throw(C *const&);
@@ -374,36 +374,36 @@ namespace dr126 { // dr126: partial
 
     virtual void v() throw(int);
     virtual void w() throw(const int);
-    virtual void x() throw(int*); // #dr126-x
+    virtual void x() throw(int*); // #cwg126-x
     virtual void y() throw(const int*);
-    virtual void z() throw(int); // #dr126-z
+    virtual void z() throw(int); // #cwg126-z
   };
   struct B : A {
     virtual void cp() throw(C*);
     virtual void dp() throw(D*);
     virtual void ep() throw(E*);
     // cxx98-14-error@-1 {{exception specification of overriding function is more lax than base version}}
-    //   cxx98-14-note@#dr126-ep {{overridden virtual function is here}}
+    //   cxx98-14-note@#cwg126-ep {{overridden virtual function is here}}
     virtual void fp() throw(F*);
     // cxx98-14-error@-1 {{exception specification of overriding function is more lax than base version}}
-    //   cxx98-14-note@#dr126-fp {{overridden virtual function is here}}
+    //   cxx98-14-note@#cwg126-fp {{overridden virtual function is here}}
     virtual void gp() throw(G*);
     virtual void hp() throw(H*);
     // cxx98-14-error@-1 {{exception specification of overriding function is more lax than base version}}
-    //   cxx98-14-note@#dr126-hp {{overridden virtual function is here}}
+    //   cxx98-14-note@#cwg126-hp {{overridden virtual function is here}}
 
     virtual void cr() throw(C&);
     virtual void dr() throw(D&);
     virtual void er() throw(E&);
     // cxx98-14-error@-1 {{exception specification of overriding function is more lax than base version}}
-    //   cxx98-14-note@#dr126-er {{overridden virtual function is here}}
+    //   cxx98-14-note@#cwg126-er {{overridden virtual function is here}}
     virtual void fr() throw(F&);
     // cxx98-14-error@-1 {{exception specification of overriding function is more lax than base version}}
-    //   cxx98-14-note@#dr126-fr {{overridden virtual function is here}}
+    //   cxx98-14-note@#cwg126-fr {{overridden virtual function is here}}
     virtual void gr() throw(G&);
     virtual void hr() throw(H&);
     // cxx98-14-error@-1 {{exception specification of overriding function is more lax than base version}}
-    //   cxx98-14-note@#dr126-hr {{overridden virtual function is here}}
+    //   cxx98-14-note@#cwg126-hr {{overridden virtual function is here}}
 
     virtual void pv() throw(C*);
 
@@ -413,7 +413,7 @@ namespace dr126 { // dr126: partial
     virtual void npm() throw(nullptr_t&);
     virtual void nr() throw(nullptr_t);
     // cxx11-14-error@-1 {{exception specification of overriding function is more lax than base version}}
-    //   cxx11-14-note@#dr126-nr {{overridden virtual function is here}}
+    //   cxx11-14-note@#cwg126-nr {{overridden virtual function is here}}
     virtual void ncr() throw(nullptr_t);
 #endif // __cplusplus >= 201103L
 
@@ -424,11 +424,11 @@ namespace dr126 { // dr126: partial
     virtual void w() throw(int);
     virtual void x() throw(const int*);
     // cxx98-14-error@-1 {{exception specification of overriding function is more lax than base version}}
-    //   cxx98-14-note@#dr126-x {{overridden virtual function is here}}
+    //   cxx98-14-note@#cwg126-x {{overridden virtual function is here}}
     virtual void y() throw(int*); // ok
     virtual void z() throw(long);
     // cxx98-14-error@-1 {{exception specification of overriding function is more lax than base version}}
-    //   cxx98-14-note@#dr126-z {{overridden virtual function is here}}
+    //   cxx98-14-note@#cwg126-z {{overridden virtual function is here}}
   };
 #endif // __cplusplus <= 201402L
   void f() throw(int);
@@ -436,37 +436,37 @@ namespace dr126 { // dr126: partial
   //   since-cxx17-note@-2 {{use 'noexcept(false)' instead}}
 }
 
-namespace dr127 { // dr127: 2.9
+namespace cwg127 { // cwg127: 2.9
   __extension__ typedef __decltype(sizeof(0)) size_t;
   template<typename T> struct A {
     A() { throw 0; }
     void *operator new(size_t, const char * = 0);
-    void operator delete(void *, const char *) { T::error; } // #dr127-delete-const-char
-    // expected-error@#dr127-delete-const-char {{type 'void' cannot be used prior to '::' because it has no members}}
-    //   expected-note@#dr127-p {{in instantiation of member function 'dr127::A<void>::operator delete' requested here}}
-    // expected-error@#dr127-delete-const-char {{type 'int' cannot be used prior to '::' because it has no members}}
-    //   expected-note@#dr127-q {{in instantiation of member function 'dr127::A<int>::operator delete' requested here}}
+    void operator delete(void *, const char *) { T::error; } // #cwg127-delete-const-char
+    // expected-error@#cwg127-delete-const-char {{type 'void' cannot be used prior to '::' because it has no members}}
+    //   expected-note@#cwg127-p {{in instantiation of member function 'cwg127::A<void>::operator delete' requested here}}
+    // expected-error@#cwg127-delete-const-char {{type 'int' cannot be used prior to '::' because it has no members}}
+    //   expected-note@#cwg127-q {{in instantiation of member function 'cwg127::A<int>::operator delete' requested here}}
     void operator delete(void *) { T::error; }
   };
-  A<void> *p = new A<void>; // #dr127-p
-  A<int> *q = new ("") A<int>; // #dr127-q
+  A<void> *p = new A<void>; // #cwg127-p
+  A<int> *q = new ("") A<int>; // #cwg127-q
 }
 
-namespace dr128 { // dr128: yes
+namespace cwg128 { // cwg128: yes
   enum E1 { e1 } x = e1;
   enum E2 { e2 } y = static_cast<E2>(x), z = static_cast<E2>(e1);
 }
 
-// dr129: dup 616
-// dr130: na
+// cwg129: dup 616
+// cwg130: na
 
-namespace dr131 { // dr131: sup P1949
+namespace cwg131 { // cwg131: sup P1949
   const char *a_with_\u0e8c = "\u0e8c";
   const char *b_with_\u0e8d = "\u0e8d";
   const char *c_with_\u0e8e = "\u0e8e";
 }
 
-namespace dr132 { // dr132: no
+namespace cwg132 { // cwg132: no
   void f() {
     extern struct {} x; // ok
     extern struct S {} y; // FIXME: This is invalid.
@@ -474,10 +474,10 @@ namespace dr132 { // dr132: no
   static enum { E } e;
 }
 
-// dr133: dup 87
-// dr134: na
+// cwg133: dup 87
+// cwg134: na
 
-namespace dr135 { // dr135: yes
+namespace cwg135 { // cwg135: yes
   struct A {
     A f(A a) { return a; }
     friend A g(A a) { return a; }
@@ -485,55 +485,55 @@ namespace dr135 { // dr135: yes
   };
 }
 
-namespace dr136 { // dr136: 3.4
-  void f(int, int, int = 0); // #dr136-f
-  void g(int, int, int); // #dr136-g
+namespace cwg136 { // cwg136: 3.4
+  void f(int, int, int = 0); // #cwg136-f
+  void g(int, int, int); // #cwg136-g
   struct A {
     friend void f(int, int = 0, int);
     // expected-error@-1 {{friend declaration specifying a default argument must be the only declaration}}
-    //   expected-note@#dr136-f {{previous declaration is here}}
+    //   expected-note@#cwg136-f {{previous declaration is here}}
     friend void g(int, int, int = 0);
     // expected-error@-1 {{friend declaration specifying a default argument must be the only declaration}}
-    //   expected-note@#dr136-g {{previous declaration is here}}
+    //   expected-note@#cwg136-g {{previous declaration is here}}
     friend void h(int, int, int = 0);
     // expected-error@-1 {{friend declaration specifying a default argument must be a definition}}
-    friend void i(int, int, int = 0) {} // #dr136-A-i
+    friend void i(int, int, int = 0) {} // #cwg136-A-i
     friend void j(int, int, int = 0) {}
     operator int();
   };
   void i(int, int, int);
   // expected-error@-1 {{friend declaration specifying a default argument must be the only declaration}}
-  //   expected-note@#dr136-A-i {{previous declaration is here}}
+  //   expected-note@#cwg136-A-i {{previous declaration is here}}
   void q() {
     j(A(), A()); // ok, has default argument
   }
-  extern "C" void k(int, int, int, int); // #dr136-k 
+  extern "C" void k(int, int, int, int); // #cwg136-k 
   namespace NSA {
   struct A {
-    friend void dr136::k(int, int, int, int = 0);
+    friend void cwg136::k(int, int, int, int = 0);
     // expected-error@-1 {{friend declaration specifying a default argument must be the only declaration}}
-    //   expected-note@#dr136-k {{previous declaration is here}}
+    //   expected-note@#cwg136-k {{previous declaration is here}}
   };
   }
   namespace NSB {
   struct A {
-    friend void dr136::k(int, int, int = 0, int); // #dr136-friend-k
-    // expected-error@#dr136-friend-k {{friend declaration specifying a default argument must be the only declaration}}
-    //   expected-note@#dr136-k {{previous declaration is here}}
-    // expected-error@#dr136-friend-k {{missing default argument on parameter}}
+    friend void cwg136::k(int, int, int = 0, int); // #cwg136-friend-k
+    // expected-error@#cwg136-friend-k {{friend declaration specifying a default argument must be the only declaration}}
+    //   expected-note@#cwg136-k {{previous declaration is here}}
+    // expected-error@#cwg136-friend-k {{missing default argument on parameter}}
   };
   }
   struct B {
-    void f(int); // #dr136-B-f
+    void f(int); // #cwg136-B-f
   };
   struct C {
     friend void B::f(int = 0);
     // expected-error@-1 {{friend declaration specifying a default argument must be the only declaration}}
-    //   expected-note@#dr136-B-f {{previous declaration is here}}
+    //   expected-note@#cwg136-B-f {{previous declaration is here}}
   };
 }
 
-namespace dr137 { // dr137: yes
+namespace cwg137 { // cwg137: yes
   extern void *p;
   extern const void *cp;
   extern volatile void *vp;
@@ -557,13 +557,13 @@ namespace dr137 { // dr137: yes
   const volatile int *cvqcv = static_cast<const volatile int*>(cvp);
 }
 
-namespace dr139 { // dr139: yes
+namespace cwg139 { // cwg139: yes
   namespace example1 {
-    typedef int f; // #dr139-typedef-f
+    typedef int f; // #cwg139-typedef-f
     struct A {
       friend void f(A &);
       // expected-error@-1 {{redefinition of 'f' as different kind of symbol}}
-      //   expected-note@#dr139-typedef-f {{previous definition is here}}
+      //   expected-note@#cwg139-typedef-f {{previous definition is here}}
     };
   }
 
@@ -579,34 +579,34 @@ namespace dr139 { // dr139: yes
   }
 }
 
-namespace dr140 { // dr140: yes
-  void f(int *const) {} // #dr140-f-first
+namespace cwg140 { // cwg140: yes
+  void f(int *const) {} // #cwg140-f-first
   void f(int[3]) {}
   // expected-error@-1 {{redefinition of 'f'}}
-  //   expected-note@#dr140-f-first {{previous definition is here}}
+  //   expected-note@#cwg140-f-first {{previous definition is here}}
   void g(const int);
   void g(int n) { n = 2; }
 }
 
-namespace dr141 { // dr141: 3.1
+namespace cwg141 { // cwg141: 3.1
   template<typename T> void f();
-  template<typename T> struct S { int n; }; // #dr141-S
+  template<typename T> struct S { int n; }; // #cwg141-S
   struct A : S<int> {
     template<typename T> void f();
-    template<typename T> struct S {}; // #dr141-A-S
+    template<typename T> struct S {}; // #cwg141-A-S
   } a;
   struct B : S<int> {} b;
   void g() {
     a.f<int>();
-    (void)a.S<int>::n; // #dr141-a
-    // cxx98-error@#dr141-a {{lookup of 'S' in member access expression is ambiguous; using member of 'struct A'}}
-    //   cxx98-note@#dr141-A-S {{lookup in the object type 'struct A' refers here}}
-    //   cxx98-note@#dr141-S {{lookup from the current scope refers here}}
-    // expected-error@#dr141-a {{no member named 'n' in 'dr141::A::S<int>'; did you mean '::dr141::S<int>::n'?}}
-    //   expected-note@#dr141-S {{'::dr141::S<int>::n' declared here}}
+    (void)a.S<int>::n; // #cwg141-a
+    // cxx98-error@#cwg141-a {{lookup of 'S' in member access expression is ambiguous; using member of 'struct A'}}
+    //   cxx98-note@#cwg141-A-S {{lookup in the object type 'struct A' refers here}}
+    //   cxx98-note@#cwg141-S {{lookup from the current scope refers here}}
+    // expected-error@#cwg141-a {{no member named 'n' in 'cwg141::A::S<int>'; did you mean '::cwg141::S<int>::n'?}}
+    //   expected-note@#cwg141-S {{'::cwg141::S<int>::n' declared here}}
     // FIXME: we issue a useful diagnostic first, then some bogus ones.
     b.f<int>();
-    // expected-error@-1 {{no member named 'f' in 'dr141::B'}}
+    // expected-error@-1 {{no member named 'f' in 'cwg141::B'}}
     // expected-error@-2 +{{}}
     (void)b.S<int>::n;
   }
@@ -630,60 +630,60 @@ namespace dr141 { // dr141: 3.1
   void i() { C<X>().i(); } // ok!!
 }
 
-namespace dr142 { // dr142: 2.8
-  class B { // #dr142-B
+namespace cwg142 { // cwg142: 2.8
+  class B { // #cwg142-B
   public:
-    int mi; // #dr142-B-mi
-    static int si; // #dr142-B-si
+    int mi; // #cwg142-B-mi
+    static int si; // #cwg142-B-si
   };
-  class D : private B { // #dr142-D
+  class D : private B { // #cwg142-D
   };
   class DD : public D {
     void f();
   };
   void DD::f() {
     mi = 3;
-    // expected-error@-1 {{'mi' is a private member of 'dr142::B'}}
-    //   expected-note@#dr142-D {{constrained by private inheritance here}}
-    //   expected-note@#dr142-B-mi {{member is declared here}}
+    // expected-error@-1 {{'mi' is a private member of 'cwg142::B'}}
+    //   expected-note@#cwg142-D {{constrained by private inheritance here}}
+    //   expected-note@#cwg142-B-mi {{member is declared here}}
     si = 3;
-    // expected-error@-1 {{'si' is a private member of 'dr142::B'}}
-    //   expected-note@#dr142-D {{constrained by private inheritance here}}
-    //   expected-note@#dr142-B-si {{member is declared here}}
+    // expected-error@-1 {{'si' is a private member of 'cwg142::B'}}
+    //   expected-note@#cwg142-D {{constrained by private inheritance here}}
+    //   expected-note@#cwg142-B-si {{member is declared here}}
     B b_old;
-    // expected-error@-1 {{'B' is a private member of 'dr142::B'}}
-    //   expected-note@#dr142-D {{constrained by private inheritance here}}
-    //   expected-note@#dr142-B {{member is declared here}}
-    dr142::B b;
+    // expected-error@-1 {{'B' is a private member of 'cwg142::B'}}
+    //   expected-note@#cwg142-D {{constrained by private inheritance here}}
+    //   expected-note@#cwg142-B {{member is declared here}}
+    cwg142::B b;
     b.mi = 3;
     b.si = 3;
     B::si = 3;
-    // expected-error@-1 {{'B' is a private member of 'dr142::B'}}
-    //   expected-note@#dr142-D {{constrained by private inheritance here}}
-    //   expected-note@#dr142-B {{member is declared here}}
-    dr142::B::si = 3;
-    B *bp1_old = this; // #dr142-bp1_old
-    // expected-error@#dr142-bp1_old {{'B' is a private member of 'dr142::B'}}
-    //   expected-note@#dr142-D {{constrained by private inheritance here}}
-    //   expected-note@#dr142-B {{member is declared here}}
-    // expected-error@#dr142-bp1_old {{cannot cast 'dr142::DD' to its private base class 'B'}}
-    //   expected-note@#dr142-D {{declared private here}}
-    dr142::B *bp1 = this;
-    // expected-error@-1 {{cannot cast 'dr142::DD' to its private base class 'dr142::B'}}
-    //   expected-note@#dr142-D {{declared private here}}
-    B *bp2_old = (B*)this; // #dr142-bp2_old
-    // expected-error@#dr142-bp2_old {{'B' is a private member of 'dr142::B'}}
-    //   expected-note@#dr142-D {{constrained by private inheritance here}}
-    //   expected-note@#dr142-B {{member is declared here}}
-    // expected-error@#dr142-bp2_old {{'B' is a private member of 'dr142::B'}}
-    //   expected-note@#dr142-D {{constrained by private inheritance here}}
-    //   expected-note@#dr142-B {{member is declared here}}
-    dr142::B *bp2 = (dr142::B*)this;
+    // expected-error@-1 {{'B' is a private member of 'cwg142::B'}}
+    //   expected-note@#cwg142-D {{constrained by private inheritance here}}
+    //   expected-note@#cwg142-B {{member is declared here}}
+    cwg142::B::si = 3;
+    B *bp1_old = this; // #cwg142-bp1_old
+    // expected-error@#cwg142-bp1_old {{'B' is a private member of 'cwg142::B'}}
+    //   expected-note@#cwg142-D {{constrained by private inheritance here}}
+    //   expected-note@#cwg142-B {{member is declared here}}
+    // expected-error@#cwg142-bp1_old {{cannot cast 'cwg142::DD' to its private base class 'B'}}
+    //   expected-note@#cwg142-D {{declared private here}}
+    cwg142::B *bp1 = this;
+    // expected-error@-1 {{cannot cast 'cwg142::DD' to its private base class 'cwg142::B'}}
+    //   expected-note@#cwg142-D {{declared private here}}
+    B *bp2_old = (B*)this; // #cwg142-bp2_old
+    // expected-error@#cwg142-bp2_old {{'B' is a private member of 'cwg142::B'}}
+    //   expected-note@#cwg142-D {{constrained by private inheritance here}}
+    //   expected-note@#cwg142-B {{member is declared here}}
+    // expected-error@#cwg142-bp2_old {{'B' is a private member of 'cwg142::B'}}
+    //   expected-note@#cwg142-D {{constrained by private inheritance here}}
+    //   expected-note@#cwg142-B {{member is declared here}}
+    cwg142::B *bp2 = (cwg142::B*)this;
     bp2->mi = 3;
   }
 }
 
-namespace dr143 { // dr143: yes
+namespace cwg143 { // cwg143: yes
   namespace A { struct X; }
   namespace B { void f(A::X); }
   namespace A {
@@ -695,7 +695,7 @@ namespace dr143 { // dr143: yes
   }
 }
 
-namespace dr145 { // dr145: yes
+namespace cwg145 { // cwg145: yes
   void f(bool b) {
     ++b;
     // cxx98-14-warning@-1 {{incrementing expression of type bool is deprecated and incompatible with C++17}}
@@ -706,7 +706,7 @@ namespace dr145 { // dr145: yes
   }
 }
 
-namespace dr147 { // dr147: yes
+namespace cwg147 { // cwg147: yes
   namespace example1 {
     template<typename> struct A {
       template<typename T> A(T);
@@ -734,15 +734,15 @@ namespace dr147 { // dr147: yes
   }
 }
 
-namespace dr148 { // dr148: yes
+namespace cwg148 { // cwg148: yes
   struct A { int A::*p; };
   int check1[__is_pod(int(A::*)) ? 1 : -1];
   int check2[__is_pod(A) ? 1 : -1];
 }
 
-// dr149: na
+// cwg149: na
 
-namespace dr151 { // dr151: 3.1
+namespace cwg151 { // cwg151: 3.1
   struct X {};
   typedef int X::*p;
 #if __cplusplus < 201103L
@@ -754,28 +754,28 @@ namespace dr151 { // dr151: 3.1
 #undef fold
 }
 
-namespace dr152 { // dr152: yes
+namespace cwg152 { // cwg152: yes
   struct A {
-    A(); // #dr152-A-ctor
-    explicit A(const A&); // #dr152-A-explicit-ctor
+    A(); // #cwg152-A-ctor
+    explicit A(const A&); // #cwg152-A-explicit-ctor
   };
   A a1 = A();
   // cxx98-14-error@-1 {{no matching constructor for initialization of 'A'}}
-  //   cxx98-14-note@#dr152-A-explicit-ctor {{explicit constructor is not a candidate}}
-  //   cxx98-14-note@#dr152-A-ctor {{candidate constructor not viable: requires 0 arguments, but 1 was provided}}
+  //   cxx98-14-note@#cwg152-A-explicit-ctor {{explicit constructor is not a candidate}}
+  //   cxx98-14-note@#cwg152-A-ctor {{candidate constructor not viable: requires 0 arguments, but 1 was provided}}
   A a2((A()));
 
   A &f();
   A a3 = f();
   // expected-error@-1 {{no matching constructor for initialization of 'A'}}
-  //   expected-note@#dr152-A-explicit-ctor {{explicit constructor is not a candidate}}
-  //   expected-note@#dr152-A-ctor {{candidate constructor not viable: requires 0 arguments, but 1 was provided}}
+  //   expected-note@#cwg152-A-explicit-ctor {{explicit constructor is not a candidate}}
+  //   expected-note@#cwg152-A-ctor {{candidate constructor not viable: requires 0 arguments, but 1 was provided}}
   A a4(f());
 }
 
-// dr153: na
+// cwg153: na
 
-namespace dr154 { // dr154: yes
+namespace cwg154 { // cwg154: yes
   union { int a; };
   // expected-error@-1 {{nonymous unions at namespace or global scope must be declared 'static'}}
   namespace {
@@ -784,29 +784,29 @@ namespace dr154 { // dr154: yes
   static union { int c; };
 }
 
-namespace dr155 { // dr155: dup 632
+namespace cwg155 { // cwg155: dup 632
   struct S { int n; } s = { { 1 } };
   // expected-warning@-1 {{braces around scalar initializer}}
 }
 
-// dr158 is in dr158.cpp
+// cwg158 is in cwg158.cpp
 
-namespace dr159 { // dr159: 3.5
+namespace cwg159 { // cwg159: 3.5
   namespace X { void f(); }
   void f();
-  void dr159::f() {}
+  void cwg159::f() {}
   // expected-warning@-1 {{extra qualification on member 'f'}}
-  void dr159::X::f() {}
+  void cwg159::X::f() {}
 }
 
-// dr160: na
+// cwg160: na
 
-namespace dr161 { // dr161: 3.1
+namespace cwg161 { // cwg161: 3.1
   class A {
   protected:
-    struct B { int n; } b; // #dr161-B
+    struct B { int n; } b; // #cwg161-B
     static B bs;
-    void f(); // #dr161-f
+    void f(); // #cwg161-f
     static void sf();
   };
   struct C : A {};
@@ -816,18 +816,18 @@ namespace dr161 { // dr161: 3.1
       B b1;
       C::B b2; // ok, accessible as a member of A
       (void)&C::b;
-      // expected-error@-1 {{'b' is a protected member of 'dr161::A'}}
-      //   expected-note@#dr161-B {{declared protected here}}
+      // expected-error@-1 {{'b' is a protected member of 'cwg161::A'}}
+      //   expected-note@#cwg161-B {{declared protected here}}
       (void)&C::bs;
       (void)c.b;
-      // expected-error@-1 {{'b' is a protected member of 'dr161::A'}}
-      //   expected-note@#dr161-B {{declared protected here}}
+      // expected-error@-1 {{'b' is a protected member of 'cwg161::A'}}
+      //   expected-note@#cwg161-B {{declared protected here}}
       (void)c.bs;
       f();
       sf();
       c.f();
       // expected-error@-1 {{protected}}
-      //   expected-note@#dr161-f {{declared protected here}}
+      //   expected-note@#cwg161-f {{declared protected here}}
       c.sf();
       A::f();
       D::f();
@@ -838,7 +838,7 @@ namespace dr161 { // dr161: 3.1
   };
 }
 
-namespace dr162 { // dr162: no
+namespace cwg162 { // cwg162: no
   struct A {
     char &f(char);
     static int &f(int);
@@ -857,9 +857,9 @@ namespace dr162 { // dr162: no
   // expected-error@-1 {{reference to overloaded function could not be resolved; did you mean to call it?}}
 }
 
-// dr163: na
+// cwg163: na
 
-namespace dr164 { // dr164: yes
+namespace cwg164 { // cwg164: yes
   void f(int);
   template <class T> int g(T t) { return f(t); }
 
@@ -869,35 +869,35 @@ namespace dr164 { // dr164: yes
   int k = g(e);
 }
 
-namespace dr165 { // dr165: no
+namespace cwg165 { // cwg165: no
   namespace N {
     struct A { friend struct B; };
     void f() { void g(); }
   }
-  // FIXME: dr1477 says this is ok, dr165 says it's ill-formed
+  // FIXME: cwg1477 says this is ok, cwg165 says it's ill-formed
   struct N::B {};
-  // FIXME: dr165 says this is ill-formed, but the argument in dr1477 says it's ok
+  // FIXME: cwg165 says this is ill-formed, but the argument in cwg1477 says it's ok
   void N::g() {}
 }
 
-namespace dr166 { // dr166: 2.9
+namespace cwg166 { // cwg166: 2.9
   namespace A { class X; }
 
   template<typename T> int f(T t) { return t.n; }
   int g(A::X);
   template<typename T> int h(T t) { return t.n; }
-  // expected-error@-1 {{'n' is a private member of 'dr166::A::X'}}
-  //   expected-note@#dr166-h-instantiation {{in instantiation of function template specialization 'dr166::h<dr166::A::X>' requested here}}
-  //   expected-note@#dr166-X-n {{implicitly declared private here}}
+  // expected-error@-1 {{'n' is a private member of 'cwg166::A::X'}}
+  //   expected-note@#cwg166-h-instantiation {{in instantiation of function template specialization 'cwg166::h<cwg166::A::X>' requested here}}
+  //   expected-note@#cwg166-X-n {{implicitly declared private here}}
   int i(A::X);
 
   namespace A {
     class X {
       friend int f<X>(X);
-      friend int dr166::g(X);
+      friend int cwg166::g(X);
       friend int h(X);
       friend int i(X);
-      int n; // #dr166-X-n
+      int n; // #cwg166-X-n
     };
 
     int h(X x) { return x.n; }
@@ -906,15 +906,15 @@ namespace dr166 { // dr166: 2.9
 
   template int f(A::X);
   int g(A::X x) { return x.n; }
-  template int h(A::X); // #dr166-h-instantiation
+  template int h(A::X); // #cwg166-h-instantiation
   int i(A::X x) { return x.n; }
-  // expected-error@-1 {{'n' is a private member of 'dr166::A::X'}}
-  //   expected-note@#dr166-X-n {{implicitly declared private here}}
+  // expected-error@-1 {{'n' is a private member of 'cwg166::A::X'}}
+  //   expected-note@#cwg166-X-n {{implicitly declared private here}}
 }
 
-// dr167: sup 1012
+// cwg167: sup 1012
 
-namespace dr168 { // dr168: no
+namespace cwg168 { // cwg168: no
   extern "C" typedef int (*p)();
   extern "C++" typedef int (*q)();
   struct S {
@@ -924,7 +924,7 @@ namespace dr168 { // dr168: no
   q b = &S::f;
 }
 
-namespace dr169 { // dr169: yes
+namespace cwg169 { // cwg169: yes
   template<typename> struct A { int n; };
   struct B {
     template<typename> struct C;
@@ -943,18 +943,18 @@ namespace dr169 { // dr169: yes
   };
 }
 
-namespace { // dr171: 3.4
-  int dr171a;
+namespace { // cwg171: 3.4
+  int cwg171a;
 }
-int dr171b; // #dr171b-int
-namespace dr171 {
-  extern "C" void dr171a();
-  extern "C" void dr171b();
-  // expected-error@-1 {{declaration of 'dr171b' with C language linkage conflicts with declaration in global scope}}
-  //   expected-note@#dr171b-int {{declared in global scope here}}
+int cwg171b; // #cwg171b-int
+namespace cwg171 {
+  extern "C" void cwg171a();
+  extern "C" void cwg171b();
+  // expected-error@-1 {{declaration of 'cwg171b' with C language linkage conflicts with declaration in global scope}}
+  //   expected-note@#cwg171b-int {{declared in global scope here}}
 }
 
-namespace dr172 { // dr172: yes
+namespace cwg172 { // cwg172: yes
   enum { zero };
   int check1[-1 < zero ? 1 : -1];
 
@@ -988,38 +988,38 @@ namespace dr172 { // dr172: yes
   int check8b[-f > 0 ? 1 : -1];
 }
 
-namespace dr173 { // dr173: yes
+namespace cwg173 { // cwg173: yes
   int check[('0' + 1 == '1' && '0' + 2 == '2' && '0' + 3 == '3' &&
              '0' + 4 == '4' && '0' + 5 == '5' && '0' + 6 == '6' &&
              '0' + 7 == '7' && '0' + 8 == '8' && '0' + 9 == '9') ? 1 : -1];
 }
 
-// dr174: sup 1012
+// cwg174: sup 1012
 
-namespace dr175 { // dr175: 2.8
-  struct A {}; // #dr175-A
-  struct B : private A {}; // #dr175-B
+namespace cwg175 { // cwg175: 2.8
+  struct A {}; // #cwg175-A
+  struct B : private A {}; // #cwg175-B
   struct C : B {
     A a;
-    // expected-error@-1 {{'A' is a private member of 'dr175::A'}}
-    //   expected-note@#dr175-B {{constrained by private inheritance here}}
-    //   expected-note@#dr175-A {{member is declared here}}
-    dr175::A b;
+    // expected-error@-1 {{'A' is a private member of 'cwg175::A'}}
+    //   expected-note@#cwg175-B {{constrained by private inheritance here}}
+    //   expected-note@#cwg175-A {{member is declared here}}
+    cwg175::A b;
   };
 }
 
-namespace dr176 { // dr176: 3.1
+namespace cwg176 { // cwg176: 3.1
   template<typename T> class Y;
   template<> class Y<int> {
     void f() {
-      typedef Y A; // #dr176-A-first
+      typedef Y A; // #cwg176-A-first
       typedef Y<char> A;
       // expected-error@-1 {{typedef redefinition with different types ('Y<char>' vs 'Y<int>')}}
-      //   expected-note@#dr176-A-first {{previous definition is here}}
+      //   expected-note@#cwg176-A-first {{previous definition is here}}
     }
   };
 
-  template<typename T> struct Base {}; // #dr176-Base
+  template<typename T> struct Base {}; // #cwg176-Base
   template<typename T> struct Derived : public Base<T> {
     void f() {
       typedef typename Derived::template Base<T> A;
@@ -1031,45 +1031,45 @@ namespace dr176 { // dr176: 3.1
   template<typename T> struct Derived2 : Base<int>, Base<char> {
     typename Derived2::Base b;
     // expected-error@-1 {{member 'Base' found in multiple base classes of different types}}
-    //   expected-note@#dr176-Base {{member type 'dr176::Base<int>' found by ambiguous name lookup}}
-    //   expected-note@#dr176-Base {{member type 'dr176::Base<char>' found by ambiguous name lookup}}
+    //   expected-note@#cwg176-Base {{member type 'cwg176::Base<int>' found by ambiguous name lookup}}
+    //   expected-note@#cwg176-Base {{member type 'cwg176::Base<char>' found by ambiguous name lookup}}
     typename Derived2::Base<double> d;
   };
 
-  template<typename T> class X { // #dr176-X
+  template<typename T> class X { // #cwg176-X
     X *p1;
     X<T> *p2;
     X<int> *p3;
-    dr176::X *p4; // #dr176-p4
-    // cxx98-14-error@#dr176-p4 {{use of class template 'dr176::X' requires template arguments}}
-    //  cxx98-14-note@#dr176-X {{template is declared here}}
-    // since-cxx17-error@#dr176-p4 {{use of class template 'X' requires template arguments; argument deduction not allowed in non-static class member}}
-    //  since-cxx17-note@#dr176-X {{template is declared here}}
+    cwg176::X *p4; // #cwg176-p4
+    // cxx98-14-error@#cwg176-p4 {{use of class template 'cwg176::X' requires template arguments}}
+    //  cxx98-14-note@#cwg176-X {{template is declared here}}
+    // since-cxx17-error@#cwg176-p4 {{use of class template 'X' requires template arguments; argument deduction not allowed in non-static class member}}
+    //  since-cxx17-note@#cwg176-X {{template is declared here}}
   };
 }
 
-namespace dr177 { // dr177: yes
+namespace cwg177 { // cwg177: yes
   struct B {};
   struct A {
-    A(A &); // #dr177-A-copy-ctor
-    A(const B &); // #dr177-A-ctor-from-B
+    A(A &); // #cwg177-A-copy-ctor
+    A(const B &); // #cwg177-A-ctor-from-B
   };
   B b;
   A a = b;
   // cxx98-14-error@-1 {{no viable constructor copying variable of type 'A'}}
-  //   cxx98-14-note@#dr177-A-copy-ctor {{candidate constructor not viable: expects an lvalue for 1st argument}}
-  //   cxx98-14-note@#dr177-A-ctor-from-B {{candidate constructor not viable: no known conversion from 'A' to 'const B &' for 1st argument}}
+  //   cxx98-14-note@#cwg177-A-copy-ctor {{candidate constructor not viable: expects an lvalue for 1st argument}}
+  //   cxx98-14-note@#cwg177-A-ctor-from-B {{candidate constructor not viable: no known conversion from 'A' to 'const B &' for 1st argument}}
 
-  struct C { C(C&); }; // #dr177-C-copy-ctor
+  struct C { C(C&); }; // #cwg177-C-copy-ctor
   struct D : C {};
   struct E { operator D(); };
   E e;
   C c = e;
   // expected-error@-1 {{no viable constructor copying variable of type 'D'}}
-  //   expected-note@#dr177-C-copy-ctor {{candidate constructor not viable: expects an lvalue for 1st argument}}
+  //   expected-note@#cwg177-C-copy-ctor {{candidate constructor not viable: expects an lvalue for 1st argument}}
 }
 
-namespace dr178 { // dr178: yes
+namespace cwg178 { // cwg178: yes
   int check[int() == 0 ? 1 : -1];
 #if __cplusplus >= 201103L
   static_assert(int{} == 0, "");
@@ -1082,13 +1082,13 @@ namespace dr178 { // dr178: yes
 #endif
 }
 
-namespace dr179 { // dr179: yes
+namespace cwg179 { // cwg179: yes
   void f();
   int n = &f - &f;
   // expected-error@-1 {{arithmetic on pointers to the function type 'void ()'}}
 }
 
-namespace dr180 { // dr180: 2.8
+namespace cwg180 { // cwg180: 2.8
   template<typename T> struct X : T, T::some_base {
     X() : T::some_type_that_might_be_T(), T::some_base() {}
     friend class T::some_class;
@@ -1098,7 +1098,7 @@ namespace dr180 { // dr180: 2.8
   };
 }
 
-namespace dr181 { // dr181: yes
+namespace cwg181 { // cwg181: yes
   namespace X {
     template <template X<class T> > struct A { };
     // expected-error@-1 +{{}}
@@ -1112,7 +1112,7 @@ namespace dr181 { // dr181: yes
   }
 }
 
-namespace dr182 { // dr182: 14
+namespace cwg182 { // cwg182: 14
   template <class T> struct C {
     void f();
     void g();
@@ -1135,7 +1135,7 @@ namespace dr182 { // dr182: 14
   }
 }
 
-namespace dr183 { // dr183: sup 382
+namespace cwg183 { // cwg183: sup 382
   template<typename T> struct A {};
   template<typename T> struct B {
     typedef int X;
@@ -1146,7 +1146,7 @@ namespace dr183 { // dr183: sup 382
   };
 }
 
-namespace dr184 { // dr184: yes
+namespace cwg184 { // cwg184: yes
   template<typename T = float> struct B {};
 
   template<template<typename TT = float> class T> struct A {
@@ -1154,10 +1154,10 @@ namespace dr184 { // dr184: yes
     void g();
   };
 
-  template<template<typename TT> class T> void A<T>::f() { // #dr184-T
+  template<template<typename TT> class T> void A<T>::f() { // #cwg184-T
     T<> t;
     // expected-error@-1 {{too few template arguments for template template parameter 'T'}}
-    //   expected-note@#dr184-T {{template is declared here}}
+    //   expected-note@#cwg184-T {{template is declared here}}
   }
 
   template<template<typename TT = char> class T> void A<T>::g() {
@@ -1169,25 +1169,25 @@ namespace dr184 { // dr184: yes
   void h() { A<B>().g(); }
 }
 
-// dr185 is in dr185.cpp
+// cwg185 is in cwg185.cpp
 
-namespace dr187 { // dr187: sup 481
+namespace cwg187 { // cwg187: sup 481
   const int Z = 1;
   template<int X = Z, int Z = X> struct A;
   typedef A<> T;
   typedef A<1, 1> T;
 }
 
-namespace dr188 { // dr188: yes
+namespace cwg188 { // cwg188: yes
   char c[10];
   int check[sizeof(0, c) == 10 ? 1 : -1];
 }
 
-// dr190 FIXME: add codegen test for tbaa
+// cwg190 FIXME: add codegen test for tbaa
 //              or implement C++20 std::is_layout_compatible and test it this way
 
-int dr191_j;
-namespace dr191 { // dr191: yes
+int cwg191_j;
+namespace cwg191 { // cwg191: yes
   namespace example1 {
     struct outer {
       static int i;
@@ -1208,7 +1208,7 @@ namespace dr191 { // dr191: yes
       void f() {
         struct local2 {
           void g() {
-            dr191_j = 5;
+            cwg191_j = 5;
           }
         };
       }
@@ -1216,9 +1216,9 @@ namespace dr191 { // dr191: yes
   }
 }
 
-// dr193 is in dr193.cpp
+// cwg193 is in cwg193.cpp
 
-namespace dr194 { // dr194: yes
+namespace cwg194 { // cwg194: yes
   struct A {
     A();
     void A();
@@ -1234,7 +1234,7 @@ namespace dr194 { // dr194: yes
   };
 }
 
-namespace dr195 { // dr195: yes
+namespace cwg195 { // cwg195: yes
   void f();
   int *p = (int*)&f;
   // cxx98-error@-1 {{cast between pointer-to-function and pointer-to-object is an extension}}
@@ -1242,14 +1242,14 @@ namespace dr195 { // dr195: yes
   // cxx98-error@-1 {{cast between pointer-to-function and pointer-to-object is an extension}}
 }
 
-namespace dr197 { // dr197: yes
+namespace cwg197 { // cwg197: yes
   char &f(char);
 
   template <class T> void g(T t) {
     char &a = f(1);
     char &b = f(T(1));
     // expected-error@-1 {{non-const lvalue reference to type 'char' cannot bind to a value of unrelated type 'int'}}
-    //   expected-note@#dr197-g-e-call {{in instantiation of function template specialization 'dr197::g<dr197::E>' requested here}}
+    //   expected-note@#cwg197-g-e-call {{in instantiation of function template specialization 'cwg197::g<cwg197::E>' requested here}}
     char &c = f(t);
     // expected-error@-1 {{non-const lvalue reference to type 'char' cannot bind to a value of unrelated type 'int'}}
   }
@@ -1262,11 +1262,11 @@ namespace dr197 { // dr197: yes
   void h() {
     g('a');
     g(2);
-    g(e); // #dr197-g-e-call
+    g(e); // #cwg197-g-e-call
   }
 }
 
-namespace dr198 { // dr198: yes
+namespace cwg198 { // cwg198: yes
   struct A {
     int n;
     struct B {
@@ -1291,4 +1291,4 @@ namespace dr198 { // dr198: yes
   };
 }
 
-// dr199 is in dr199.cpp
+// cwg199 is in cwg199.cpp
diff --git a/clang/test/CXX/drs/dr20xx.cpp b/clang/test/CXX/drs/dr20xx.cpp
index 96f2572cfb52..291a77e0cc71 100644
--- a/clang/test/CXX/drs/dr20xx.cpp
+++ b/clang/test/CXX/drs/dr20xx.cpp
@@ -11,7 +11,7 @@
 // cxx98-error@-1 {{variadic macros are a C99 feature}}
 #endif
 
-namespace dr2007 { // dr2007: 3.4
+namespace cwg2007 { // cwg2007: 3.4
 template<typename T> struct A { typename T::error e; };
 template<typename T> struct B { };
 B<A<void> > b1;
@@ -20,24 +20,24 @@ int a = b2[0];
 // cxx98-error@-1 {{type 'B<A<void> >' does not provide a subscript operator}}
 // since-cxx11-error@-2 {{type 'B<A<void>>' does not provide a subscript operator}}
 int b = __builtin_addressof(b2)->foo;
-// cxx98-error@-1 {{no member named 'foo' in 'dr2007::B<dr2007::A<void> >'}}
-// since-cxx11-error@-2 {{no member named 'foo' in 'dr2007::B<dr2007::A<void>>'}}
+// cxx98-error@-1 {{no member named 'foo' in 'cwg2007::B<cwg2007::A<void> >'}}
+// since-cxx11-error@-2 {{no member named 'foo' in 'cwg2007::B<cwg2007::A<void>>'}}
 }
 
-// dr2009: na
+// cwg2009: na
 
-namespace dr2026 { // dr2026: 11
+namespace cwg2026 { // cwg2026: 11
   template<int> struct X {};
 
-  const int a = a + 1; // #dr2026-a
+  const int a = a + 1; // #cwg2026-a
   // expected-warning@-1 {{variable 'a' is uninitialized when used within its own initialization}}
-  X<a> xa; // #dr2026-xa
+  X<a> xa; // #cwg2026-xa
   // cxx98-error@-1 {{non-type template argument of type 'int' is not an integral constant expression}}
   //   cxx98-note@-2 {{initializer of 'a' is not a constant expression}}
-  //   cxx98-note@#dr2026-a {{declared here}}
-  // since-cxx11-error@#dr2026-xa {{non-type template argument is not a constant expression}}
-  //   since-cxx11-note@#dr2026-xa {{initializer of 'a' is not a constant expression}}
-  //   since-cxx11-note@#dr2026-a {{declared here}}
+  //   cxx98-note@#cwg2026-a {{declared here}}
+  // since-cxx11-error@#cwg2026-xa {{non-type template argument is not a constant expression}}
+  //   since-cxx11-note@#cwg2026-xa {{initializer of 'a' is not a constant expression}}
+  //   since-cxx11-note@#cwg2026-a {{declared here}}
 
 #if __cplusplus >= 201103L
   constexpr int b = b;
@@ -59,15 +59,15 @@ namespace dr2026 { // dr2026: 11
 #endif
 
   void f() {
-    static const int e = e + 1; // #dr2026-e
+    static const int e = e + 1; // #cwg2026-e
     // expected-warning@-1 {{static variable 'e' is suspiciously used within its own initialization}}
-    X<e> xe; // #dr2026-xe
+    X<e> xe; // #cwg2026-xe
     // cxx98-error@-1 {{non-type template argument of type 'int' is not an integral constant expression}}
     //   cxx98-note@-2 {{initializer of 'e' is not a constant expression}}
-    //   cxx98-note@#dr2026-e {{declared here}}
-    // since-cxx11-error@#dr2026-xe {{non-type template argument is not a constant expression}}
-    //   since-cxx11-note@#dr2026-xe {{initializer of 'e' is not a constant expression}}
-    //   since-cxx11-note@#dr2026-e {{declared here}}
+    //   cxx98-note@#cwg2026-e {{declared here}}
+    // since-cxx11-error@#cwg2026-xe {{non-type template argument is not a constant expression}}
+    //   since-cxx11-note@#cwg2026-xe {{initializer of 'e' is not a constant expression}}
+    //   since-cxx11-note@#cwg2026-e {{declared here}}
 
 #if __cplusplus >= 201103L
     static constexpr int f = f;
@@ -90,7 +90,7 @@ namespace dr2026 { // dr2026: 11
   }
 }
 
-namespace dr2049 { // dr2049: 18 drafting P2308R1
+namespace cwg2049 { // cwg2049: 18 drafting P2308R1
 #if __cplusplus >= 202302L
 template <int* x = {}> struct X {};
 X<> a;
@@ -99,7 +99,7 @@ static_assert(__is_same(decltype(a), decltype(b)));
 #endif
 }
 
-namespace dr2061 { // dr2061: yes
+namespace cwg2061 { // cwg2061: yes
 #if __cplusplus >= 201103L
   namespace A {
     inline namespace b {
@@ -127,7 +127,7 @@ namespace dr2061 { // dr2061: yes
 #endif // C++11
 }
 
-namespace dr2076 { // dr2076: 13
+namespace cwg2076 { // cwg2076: 13
 #if __cplusplus >= 201103L
   namespace std_example {
     struct A { A(int); };
@@ -149,8 +149,8 @@ namespace dr2076 { // dr2076: 13
     operator string_view() const;
   };
 
-  void foo(const string &); // #dr2076-foo 
-  void bar(string_view); // #dr2076-bar
+  void foo(const string &); // #cwg2076-foo 
+  void bar(string_view); // #cwg2076-bar
 
   void func(const string &arg) {
     // An argument in one set of braces is subject to user-defined conversions;
@@ -161,27 +161,27 @@ namespace dr2076 { // dr2076: 13
     foo({{arg}});
     foo({{{arg}}});
     // since-cxx11-error@-1 {{no matching function}}
-    //   since-cxx11-note@#dr2076-foo  {{cannot convert initializer list}}
+    //   since-cxx11-note@#cwg2076-foo  {{cannot convert initializer list}}
     bar(arg);
     bar({arg});
     bar({{arg}});
     // since-cxx11-error@-1 {{no matching function}}
-    //   since-cxx11-note@#dr2076-bar {{cannot convert initializer list}}
+    //   since-cxx11-note@#cwg2076-bar {{cannot convert initializer list}}
     bar({{{arg}}});
     // since-cxx11-error@-1 {{no matching function}}
-    //   since-cxx11-note@#dr2076-bar {{cannot convert initializer list}}
+    //   since-cxx11-note@#cwg2076-bar {{cannot convert initializer list}}
   }
 #endif
 }
 
-namespace dr2082 { // dr2082: 11
+namespace cwg2082 { // cwg2082: 11
   void test1(int x, int = sizeof(x)); // ok
 #if __cplusplus >= 201103L
   void test2(int x, int = decltype(x){}); // ok
 #endif
 }
 
-namespace dr2083 { // dr2083: partial
+namespace cwg2083 { // cwg2083: partial
 #if __cplusplus >= 201103L
   void non_const_mem_ptr() {
     struct A {
@@ -204,23 +204,23 @@ namespace dr2083 { // dr2083: partial
   const int a = 1;
   int b;
   // Note, references only get special odr-use / constant initializxer
-  // treatment in C++11 onwards. We continue to apply that even after DR2083.
+  // treatment in C++11 onwards. We continue to apply that even after CWG2083.
   void ref_to_non_const() {
     int c;
-    const int &ra = a; // #dr2083-ra
-    int &rb = b; // #dr2083-rb
-    int &rc = c; // #dr2083-rc
+    const int &ra = a; // #cwg2083-ra
+    int &rb = b; // #cwg2083-rb
+    int &rc = c; // #cwg2083-rc
     struct A {
       int f() {
         int a = ra;
-        // cxx98-error@-1 {{reference to local variable 'ra' declared in enclosing function 'dr2083::ref_to_non_const'}}
-        //   cxx98-note@#dr2083-ra {{'ra' declared here}}
+        // cxx98-error@-1 {{reference to local variable 'ra' declared in enclosing function 'cwg2083::ref_to_non_const'}}
+        //   cxx98-note@#cwg2083-ra {{'ra' declared here}}
         int b = rb;
-        // cxx98-error@-1 {{reference to local variable 'rb' declared in enclosing function 'dr2083::ref_to_non_const'}}
-        //   cxx98-note@#dr2083-rb {{'rb' declared here}}
+        // cxx98-error@-1 {{reference to local variable 'rb' declared in enclosing function 'cwg2083::ref_to_non_const'}}
+        //   cxx98-note@#cwg2083-rb {{'rb' declared here}}
         int c = rc;
-        // expected-error@-1 {{reference to local variable 'rc' declared in enclosing function 'dr2083::ref_to_non_const'}}
-        //   expected-note@#dr2083-rc {{'rc' declared here}}
+        // expected-error@-1 {{reference to local variable 'rc' declared in enclosing function 'cwg2083::ref_to_non_const'}}
+        //   expected-note@#cwg2083-rc {{'rc' declared here}}
         return a + b + c;
       }
     };
@@ -244,9 +244,9 @@ namespace dr2083 { // dr2083: partial
     constexpr NoMut1 nm1 = {1, 2};
     constexpr NoMut2 nm2 = {1, 2};
     constexpr NoMut3 nm3 = {1, 2};
-    constexpr Mut1 m1 = {1, 2}; // #dr2083-m1
-    constexpr Mut2 m2 = {1, 2}; // #dr2083-m2
-    constexpr Mut3 m3 = {1, 2}; // #dr2083-m3
+    constexpr Mut1 m1 = {1, 2}; // #cwg2083-m1
+    constexpr Mut2 m2 = {1, 2}; // #cwg2083-m2
+    constexpr Mut3 m3 = {1, 2}; // #cwg2083-m3
     struct A {
       void f() {
         static_assert(nm1.a == 1, "");
@@ -254,14 +254,14 @@ namespace dr2083 { // dr2083: partial
         static_assert(nm3.a == 1, "");
         // Can't even access a non-mutable member of a variable containing mutable fields.
         static_assert(m1.a == 1, "");
-        // since-cxx11-error@-1 {{reference to local variable 'm1' declared in enclosing function 'dr2083::mutable_subobjects'}}
-        //   since-cxx11-note@#dr2083-m1 {{'m1' declared here}}
+        // since-cxx11-error@-1 {{reference to local variable 'm1' declared in enclosing function 'cwg2083::mutable_subobjects'}}
+        //   since-cxx11-note@#cwg2083-m1 {{'m1' declared here}}
         static_assert(m2.m.a == 1, "");
-        // since-cxx11-error@-1 {{reference to local variable 'm2' declared in enclosing function 'dr2083::mutable_subobjects'}}
-        //   since-cxx11-note@#dr2083-m2 {{'m2' declared here}}
+        // since-cxx11-error@-1 {{reference to local variable 'm2' declared in enclosing function 'cwg2083::mutable_subobjects'}}
+        //   since-cxx11-note@#cwg2083-m2 {{'m2' declared here}}
         static_assert(m3.a == 1, "");
-        // since-cxx11-error@-1 {{reference to local variable 'm3' declared in enclosing function 'dr2083::mutable_subobjects'}}
-        //   since-cxx11-note@#dr2083-m3 {{'m3' declared here}}
+        // since-cxx11-error@-1 {{reference to local variable 'm3' declared in enclosing function 'cwg2083::mutable_subobjects'}}
+        //   since-cxx11-note@#cwg2083-m3 {{'m3' declared here}}
       }
     };
   }
@@ -274,7 +274,7 @@ namespace dr2083 { // dr2083: partial
 #if __cplusplus >= 201103L
     constexpr
 #endif
-      A a = {}; // #dr2083-a
+      A a = {}; // #cwg2083-a
     struct B {
       void f() {
         ellipsis(n);
@@ -282,8 +282,8 @@ namespace dr2083 { // dr2083: partial
         // conversion, it calls a constructor and binds 'a' to a reference, so
         // it results in an odr-use.
         ellipsis(a);
-        // expected-error@-1 {{reference to local variable 'a' declared in enclosing function 'dr2083::ellipsis'}}
-        //   expected-note@#dr2083-a {{'a' declared here}}
+        // expected-error@-1 {{reference to local variable 'a' declared in enclosing function 'cwg2083::ellipsis'}}
+        //   expected-note@#cwg2083-a {{'a' declared here}}
       }
     };
   }
@@ -291,7 +291,7 @@ namespace dr2083 { // dr2083: partial
 #if __cplusplus >= 201103L
   void volatile_lval() {
     struct A { int n; };
-    constexpr A a = {0}; // #dr2083-a2
+    constexpr A a = {0}; // #cwg2083-a2
     struct B {
       void f() {
         // An lvalue-to-rvalue conversion of a volatile lvalue always results
@@ -300,8 +300,8 @@ namespace dr2083 { // dr2083: partial
         int x = a.*p;
         volatile int A::*q = p;
         int y = a.*q;
-        // since-cxx11-error@-1 {{reference to local variable 'a' declared in enclosing function 'dr2083::volatile_lval'}}
-        //   since-cxx11-note@#dr2083-a2 {{'a' declared here}}
+        // since-cxx11-error@-1 {{reference to local variable 'a' declared in enclosing function 'cwg2083::volatile_lval'}}
+        //   since-cxx11-note@#cwg2083-a2 {{'a' declared here}}
       }
     };
   }
@@ -309,45 +309,45 @@ namespace dr2083 { // dr2083: partial
 
   void discarded_lval() {
     struct A { int x; mutable int y; volatile int z; };
-    A a; // #dr2083-a-3
-    int &r = a.x; // #dr2083-r
+    A a; // #cwg2083-a-3
+    int &r = a.x; // #cwg2083-r
     struct B {
       void f() {
         // FIXME: We emit more errors than we should be. They are explictly marked below.
         a.x;
         // expected-warning@-1 {{expression result unused}}
-        // expected-error@-2 {{reference to local variable 'a' declared in enclosing function 'dr2083::discarded_lval'}} FIXME
-        //   expected-note@#dr2083-a-3 {{'a' declared here}}
+        // expected-error@-2 {{reference to local variable 'a' declared in enclosing function 'cwg2083::discarded_lval'}} FIXME
+        //   expected-note@#cwg2083-a-3 {{'a' declared here}}
         a.*&A::x;
         // expected-warning@-1 {{expression result unused}}
-        // expected-error@-2 {{reference to local variable 'a' declared in enclosing function 'dr2083::discarded_lval'}} FIXME
-        //   expected-note@#dr2083-a-3 {{'a' declared here}}
-        true ? a.x : a.y; // #dr2083-ternary
+        // expected-error@-2 {{reference to local variable 'a' declared in enclosing function 'cwg2083::discarded_lval'}} FIXME
+        //   expected-note@#cwg2083-a-3 {{'a' declared here}}
+        true ? a.x : a.y; // #cwg2083-ternary
         // expected-warning@-1 {{expression result unused}}
-        // expected-error@#dr2083-ternary {{reference to local variable 'a' declared in enclosing function 'dr2083::discarded_lval'}} FIXME
-        //   expected-note@#dr2083-a-3 {{'a' declared here}}
-        // expected-error@#dr2083-ternary {{reference to local variable 'a' declared in enclosing function 'dr2083::discarded_lval'}} FIXME
-        //   expected-note@#dr2083-a-3 {{'a' declared here}}
+        // expected-error@#cwg2083-ternary {{reference to local variable 'a' declared in enclosing function 'cwg2083::discarded_lval'}} FIXME
+        //   expected-note@#cwg2083-a-3 {{'a' declared here}}
+        // expected-error@#cwg2083-ternary {{reference to local variable 'a' declared in enclosing function 'cwg2083::discarded_lval'}} FIXME
+        //   expected-note@#cwg2083-a-3 {{'a' declared here}}
         (void)a.x;
-        // expected-error@-1 {{reference to local variable 'a' declared in enclosing function 'dr2083::discarded_lval'}} FIXME
-        //   expected-note@#dr2083-a-3 {{'a' declared here}}
+        // expected-error@-1 {{reference to local variable 'a' declared in enclosing function 'cwg2083::discarded_lval'}} FIXME
+        //   expected-note@#cwg2083-a-3 {{'a' declared here}}
         a.x, discarded_lval();
         // expected-warning@-1 {{left operand of comma operator has no effect}}
-        // expected-error@-2 {{reference to local variable 'a' declared in enclosing function 'dr2083::discarded_lval'}} FIXME
-        //   expected-note@#dr2083-a-3 {{'a' declared here}}
+        // expected-error@-2 {{reference to local variable 'a' declared in enclosing function 'cwg2083::discarded_lval'}} FIXME
+        //   expected-note@#cwg2083-a-3 {{'a' declared here}}
 
         // 'volatile' qualifier triggers an lvalue-to-rvalue conversion.
         a.z;
         // cxx98-warning@-1 {{expression result unused; assign into a variable to force a volatile load}}
-        // expected-error@-2 {{reference to local variable 'a' declared in enclosing function 'dr2083::discarded_lval'}}
-        //   expected-note@#dr2083-a-3 {{'a' declared here}}
+        // expected-error@-2 {{reference to local variable 'a' declared in enclosing function 'cwg2083::discarded_lval'}}
+        //   expected-note@#cwg2083-a-3 {{'a' declared here}}
 
         // References always get "loaded" to determine what they reference,
         // even if the result is discarded.
         r;
         // expected-warning@-1 {{expression result unused}}
-        // expected-error@-2 {{reference to local variable 'r' declared in enclosing function 'dr2083::discarded_lval'}}
-        //   expected-note@#dr2083-r {{'r' declared here}}
+        // expected-error@-2 {{reference to local variable 'r' declared in enclosing function 'cwg2083::discarded_lval'}}
+        //   expected-note@#cwg2083-r {{'r' declared here}}
       }
     };
   }
@@ -355,11 +355,11 @@ namespace dr2083 { // dr2083: partial
   namespace dr_example_1 {
     extern int globx;
     int main() {
-      const int &x = globx; // #dr2083-x
+      const int &x = globx; // #cwg2083-x
       struct A {
         const int *foo() { return &x; }
-        // cxx98-error@-1 {{reference to local variable 'x' declared in enclosing function 'dr2083::dr_example_1::main'}}
-        //   cxx98-note@#dr2083-x {{'x' declared here}}
+        // cxx98-error@-1 {{reference to local variable 'x' declared in enclosing function 'cwg2083::dr_example_1::main'}}
+        //   cxx98-note@#cwg2083-x {{'x' declared here}}
       } a;
       return *a.foo();
     }
@@ -400,7 +400,7 @@ namespace dr2083 { // dr2083: partial
 #endif
 }
 
-namespace dr2094 { // dr2094: 5
+namespace cwg2094 { // cwg2094: 5
   struct A { int n; };
   struct B { volatile int n; };
   static_assert(__is_trivially_copyable(volatile int), "");
@@ -419,4 +419,4 @@ namespace dr2094 { // dr2094: 5
   static_assert(__is_trivially_assignable(B, const B&), "");
 }
 
-// dr2096: dup 2598
+// cwg2096: dup 2598
diff --git a/clang/test/CXX/drs/dr21xx.cpp b/clang/test/CXX/drs/dr21xx.cpp
index a7e50df3f374..4fab10c279aa 100644
--- a/clang/test/CXX/drs/dr21xx.cpp
+++ b/clang/test/CXX/drs/dr21xx.cpp
@@ -11,7 +11,7 @@
 // cxx98-error@-1 {{variadic macros are a C99 feature}}
 #endif
 
-namespace dr2100 { // dr2100: 12
+namespace cwg2100 { // cwg2100: 12
   template<const int *P, bool = true> struct X {};
   template<typename T> struct A {
     static const int n = 1;
@@ -19,10 +19,10 @@ namespace dr2100 { // dr2100: 12
       return X<&n>::n; // ok, value-dependent
     }
     int g() {
-      static const int n = 2; // #dr2100-n
+      static const int n = 2; // #cwg2100-n
       return X<&n>::n; // ok, value-dependent
       // cxx98-14-error@-1 {{non-type template argument refers to object 'n' that does not have linkage}}
-      //   cxx98-14-note@#dr2100-n {{non-type template argument refers to object here}}
+      //   cxx98-14-note@#cwg2100-n {{non-type template argument refers to object here}}
     }
   };
   template<const int *P> struct X<P> {
@@ -41,12 +41,12 @@ namespace dr2100 { // dr2100: 12
     static const int n = 1;
     int f() {
       return Y<n>::declared_later;
-      // expected-error@-1 {{no member named 'declared_later' in 'dr2100::Y<1>'}}
+      // expected-error@-1 {{no member named 'declared_later' in 'cwg2100::Y<1>'}}
     }
     int g() {
       static const int n = 2;
       return Y<n>::declared_later;
-      // expected-error@-1 {{no member named 'declared_later' in 'dr2100::Y<2>'}}
+      // expected-error@-1 {{no member named 'declared_later' in 'cwg2100::Y<2>'}}
     }
   };
   template<int N> struct Y<N> {
@@ -54,22 +54,22 @@ namespace dr2100 { // dr2100: 12
   };
 }
 
-namespace dr2103 { // dr2103: yes
+namespace cwg2103 { // cwg2103: yes
   void f() {
     int a;
-    int &r = a; // #dr2103-r
+    int &r = a; // #cwg2103-r
     struct Inner {
       void f() {
         int &s = r;
-        // expected-error@-1 {{reference to local variable 'r' declared in enclosing function 'dr2103::f'}}
-        //   expected-note@#dr2103-r {{'r' declared here}}
+        // expected-error@-1 {{reference to local variable 'r' declared in enclosing function 'cwg2103::f'}}
+        //   expected-note@#cwg2103-r {{'r' declared here}}
         (void)s;
       }
     };
   }
 }
 
-namespace dr2120 { // dr2120: 7
+namespace cwg2120 { // cwg2120: 7
   struct A {};
   struct B : A {};
   struct C { A a; };
@@ -80,7 +80,7 @@ namespace dr2120 { // dr2120: 7
   static_assert(!__is_standard_layout(E), "");
 }
 
-namespace dr2126 { // dr2126: 12
+namespace cwg2126 { // cwg2126: 12
 #if __cplusplus >= 201103L
   struct A { int n; };
 
@@ -88,16 +88,16 @@ namespace dr2126 { // dr2126: 12
   A &b = (A &)(const A &)A{1};   // const temporary
   A &&c = (A &&)(const A &)A{1}; // const temporary
 
-  A &&d = {1};                   // non-const temporary #dr21260-d
-  const A &e = (A &)(A &&) A{1}; // non-const temporary #dr21260-e
-  A &&f = (A &&)(A &&) A{1};     // non-const temporary #dr21260-f
+  A &&d = {1};                   // non-const temporary #cwg21260-d
+  const A &e = (A &)(A &&) A{1}; // non-const temporary #cwg21260-e
+  A &&f = (A &&)(A &&) A{1};     // non-const temporary #cwg21260-f
 
   constexpr const A &g = {1};    // const temporary
-  constexpr A &&h = {1};         // non-const temporary #dr21260-h
+  constexpr A &&h = {1};         // non-const temporary #cwg21260-h
 
   struct B { const A &a; };
-  B i = {{1}};           // extending decl not usable in constant expr #dr21260-i
-  const B j = {{1}};     // extending decl not usable in constant expr #dr21260-j
+  B i = {{1}};           // extending decl not usable in constant expr #cwg21260-i
+  const B j = {{1}};     // extending decl not usable in constant expr #cwg21260-j
   constexpr B k = {{1}}; // extending decl usable in constant expr
 
   static_assert(a.n == 1, "");
@@ -106,33 +106,33 @@ namespace dr2126 { // dr2126: 12
   static_assert(d.n == 1, "");
   // since-cxx11-error@-1 {{static assertion expression is not an integral constant expression}}
   //   since-cxx11-note@-2 {{read of temporary is not allowed in a constant expression outside the expression that created the temporary}}
-  //   since-cxx11-note@#dr21260-d {{temporary created here}}
+  //   since-cxx11-note@#cwg21260-d {{temporary created here}}
   static_assert(e.n == 1, "");
   // since-cxx11-error@-1 {{static assertion expression is not an integral constant expression}}
   //   since-cxx11-note@-2 {{read of temporary is not allowed in a constant expression outside the expression that created the temporary}}
-  //   since-cxx11-note@#dr21260-e {{temporary created here}}
+  //   since-cxx11-note@#cwg21260-e {{temporary created here}}
   static_assert(f.n == 1, "");
   // since-cxx11-error@-1 {{static assertion expression is not an integral constant expression}}
   //   since-cxx11-note@-2 {{read of temporary is not allowed in a constant expression outside the expression that created the temporary}}
-  //   since-cxx11-note@#dr21260-f {{temporary created here}}
+  //   since-cxx11-note@#cwg21260-f {{temporary created here}}
   static_assert(g.n == 1, "");
   static_assert(h.n == 1, "");
   // since-cxx11-error@-1 {{static assertion expression is not an integral constant expression}}
   //   since-cxx11-note@-2 {{read of temporary is not allowed in a constant expression outside the expression that created the temporary}}
-  //   since-cxx11-note@#dr21260-h {{temporary created here}}
+  //   since-cxx11-note@#cwg21260-h {{temporary created here}}
   static_assert(i.a.n == 1, "");
   // since-cxx11-error@-1 {{static assertion expression is not an integral constant expression}}
   //   since-cxx11-note@-2 {{read of non-constexpr variable 'i' is not allowed in a constant expression}}
-  //   since-cxx11-note@#dr21260-i {{declared here}}
+  //   since-cxx11-note@#cwg21260-i {{declared here}}
   static_assert(j.a.n == 1, "");
   // since-cxx11-error@-1 {{static assertion expression is not an integral constant expression}}
   //   since-cxx11-note@-2 {{read of temporary is not allowed in a constant expression outside the expression that created the temporary}}
-  //   since-cxx11-note@#dr21260-j {{temporary created here}}
+  //   since-cxx11-note@#cwg21260-j {{temporary created here}}
   static_assert(k.a.n == 1, "");
 #endif
 }
 
-namespace dr2140 { // dr2140: 9
+namespace cwg2140 { // cwg2140: 9
 #if __cplusplus >= 201103L
   union U { int a; decltype(nullptr) b; };
   constexpr int *test(U u) {
@@ -142,7 +142,7 @@ namespace dr2140 { // dr2140: 9
 #endif
 }
 
-namespace dr2141 { // dr2141: 17
+namespace cwg2141 { // cwg2141: 17
 struct A{};
 
 template <typename T>
@@ -175,34 +175,34 @@ void foo() {
 }
 }
 
-namespace dr2157 { // dr2157: 11
+namespace cwg2157 { // cwg2157: 11
 #if __cplusplus >= 201103L
   enum E : int;
   struct X {
-    enum dr2157::E : int();
+    enum cwg2157::E : int();
     // since-cxx11-error@-1 {{ISO C++ only allows ':' in member enumeration declaration to introduce a fixed underlying type, not an anonymous bit-field}}
   };
 #endif
 }
 
-// dr2165: na
+// cwg2165: na
 
-namespace dr2170 { // dr2170: 9
+namespace cwg2170 { // cwg2170: 9
 #if __cplusplus >= 201103L
   void f() {
-    constexpr int arr[3] = {1, 2, 3}; // #dr2170-arr
+    constexpr int arr[3] = {1, 2, 3}; // #cwg2170-arr
     struct S {
       int get(int n) { return arr[n]; }
       const int &get_ref(int n) { return arr[n]; }
       // since-cxx11-warning@-1 {{reference to stack memory associated with local variable 'arr' returned}} FIXME
-      // since-cxx11-error@-2 {{reference to local variable 'arr' declared in enclosing function 'dr2170::f'}}
-      //   since-cxx11-note@#dr2170-arr {{'arr' declared here}}
+      // since-cxx11-error@-2 {{reference to local variable 'arr' declared in enclosing function 'cwg2170::f'}}
+      //   since-cxx11-note@#cwg2170-arr {{'arr' declared here}}
     };
   }
 #endif
 }
 
-namespace dr2171 { // dr2171: 15
+namespace cwg2171 { // cwg2171: 15
 #if __cplusplus >= 201103L
 
 struct NonConstCopy {
@@ -227,40 +227,40 @@ static_assert(!__is_trivially_assignable(NonConstCopy &&, NonConstCopy), "");
 static_assert(!__is_trivially_assignable(NonConstCopy &&, NonConstCopy &&), "");
 
 #endif
-} // namespace dr2171
+} // namespace cwg2171
 
-namespace dr2180 { // dr2180: yes
+namespace cwg2180 { // cwg2180: yes
   class A {
-    A &operator=(const A &); // #dr2180-A-copy
-    A &operator=(A &&); // #dr2180-A-move
+    A &operator=(const A &); // #cwg2180-A-copy
+    A &operator=(A &&); // #cwg2180-A-move
     // cxx98-error@-1 {{rvalue references are a C++11 extension}}
   };
 
-  struct B : virtual A { // #dr2180-B
+  struct B : virtual A { // #cwg2180-B
     B &operator=(const B &);
     B &operator=(B &&);
     // cxx98-error@-1 {{rvalue references are a C++11 extension}}
     virtual void foo() = 0;
   };
-  B &B::operator=(const B&) = default; // #dr2180-B-copy
+  B &B::operator=(const B&) = default; // #cwg2180-B-copy
   // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}}
-  // cxx98-error@-2 {{'operator=' is a private member of 'dr2180::A'}}
-  //   cxx98-note@-3 {{in defaulted copy assignment operator for 'dr2180::B' first required here}}
-  //   cxx98-note@#dr2180-A-copy {{implicitly declared private here}}
-  // since-cxx11-error@#dr2180-B-copy {{defaulting this copy assignment operator would delete it after its first declaration}}
-  //   since-cxx11-note@#dr2180-B {{copy assignment operator of 'B' is implicitly deleted because base class 'A' has an inaccessible copy assignment operator}}
-  B &B::operator=(B&&) = default; // #dr2180-B-move
+  // cxx98-error@-2 {{'operator=' is a private member of 'cwg2180::A'}}
+  //   cxx98-note@-3 {{in defaulted copy assignment operator for 'cwg2180::B' first required here}}
+  //   cxx98-note@#cwg2180-A-copy {{implicitly declared private here}}
+  // since-cxx11-error@#cwg2180-B-copy {{defaulting this copy assignment operator would delete it after its first declaration}}
+  //   since-cxx11-note@#cwg2180-B {{copy assignment operator of 'B' is implicitly deleted because base class 'A' has an inaccessible copy assignment operator}}
+  B &B::operator=(B&&) = default; // #cwg2180-B-move
   // cxx98-error@-1 {{rvalue references are a C++11 extension}}
   // cxx98-error@-2 {{defaulted function definitions are a C++11 extension}}
-  // cxx98-error@-3 {{'operator=' is a private member of 'dr2180::A'}}
-  //   cxx98-note@-4 {{in defaulted move assignment operator for 'dr2180::B' first required here}}
-  //   cxx98-note@#dr2180-A-move {{implicitly declared private here}}
-  // since-cxx11-error@#dr2180-B-move {{defaulting this move assignment operator would delete it after its first declaration}}
-  //   since-cxx11-note@#dr2180-B {{move assignment operator of 'B' is implicitly deleted because base class 'A' has an inaccessible move assignment operator}}
+  // cxx98-error@-3 {{'operator=' is a private member of 'cwg2180::A'}}
+  //   cxx98-note@-4 {{in defaulted move assignment operator for 'cwg2180::B' first required here}}
+  //   cxx98-note@#cwg2180-A-move {{implicitly declared private here}}
+  // since-cxx11-error@#cwg2180-B-move {{defaulting this move assignment operator would delete it after its first declaration}}
+  //   since-cxx11-note@#cwg2180-B {{move assignment operator of 'B' is implicitly deleted because base class 'A' has an inaccessible move assignment operator}}
 }
 
-namespace dr2199 { // dr2199: 3.8
-                   // NB: reusing part of dr407 test
+namespace cwg2199 { // cwg2199: 3.8
+                   // NB: reusing part of cwg407 test
 namespace A {
   struct S {};
 }
diff --git a/clang/test/CXX/drs/dr22xx.cpp b/clang/test/CXX/drs/dr22xx.cpp
index 3a13cb0471a7..797c3ed8546e 100644
--- a/clang/test/CXX/drs/dr22xx.cpp
+++ b/clang/test/CXX/drs/dr22xx.cpp
@@ -8,7 +8,7 @@
 
 
 #if __cplusplus >= 201103L
-namespace dr2211 { // dr2211: 8
+namespace cwg2211 { // cwg2211: 8
 void f() {
   int a;
   auto f = [a](int a) { (void)a; };
@@ -19,15 +19,15 @@ void f() {
 }
 #endif
 
-namespace dr2213 { // dr2213: yes
+namespace cwg2213 { // cwg2213: yes
 template <typename T, typename U>
 struct A;
 
 template <typename U>
 struct A<int, U>;
-} // namespace dr2213
+} // namespace cwg2213
 
-namespace dr2229 { // dr2229: 7
+namespace cwg2229 { // cwg2229: 7
 struct AnonBitfieldQualifiers {
   const unsigned : 1;
   // expected-error@-1 {{anonymous bit-field cannot have qualifiers}}
@@ -43,7 +43,7 @@ struct AnonBitfieldQualifiers {
 };
 }
 
-namespace dr2233 { // dr2233: 11
+namespace cwg2233 { // cwg2233: 11
 #if __cplusplus >= 201103L
 template <typename... T>
 void f(int i = 0, T... args) {}
@@ -107,7 +107,7 @@ namespace MultilevelSpecialization {
   };
   template<> template<int a, int b>
     void B<int, int>::f(int i, int (&arr1)[a], int (&arr2)[b]) {}
-    // since-cxx11-error@-1 {{out-of-line definition of 'f' does not match any declaration in 'dr2233::MultilevelSpecialization::B<int, int>'}}
+    // since-cxx11-error@-1 {{out-of-line definition of 'f' does not match any declaration in 'cwg2233::MultilevelSpecialization::B<int, int>'}}
   template<> template<>
     void B<int, int>::f<1, 1>(int i, int (&arr1a)[1], int (&arr2a)[1]) {}
 }
@@ -130,12 +130,12 @@ namespace CheckAfterMerging2 {
   void h() { f<int>(); }
 }
 #endif
-} // namespace dr2233
+} // namespace cwg2233
 
-namespace dr2267 { // dr2267: no
+namespace cwg2267 { // cwg2267: no
 #if __cplusplus >= 201103L
 struct A {} a;
-struct B { explicit B(const A&); }; // #dr2267-struct-B
+struct B { explicit B(const A&); }; // #cwg2267-struct-B
 
 struct D { D(); };
 struct C { explicit operator D(); } c;
@@ -144,9 +144,9 @@ B b1(a);
 const B &b2{a}; // FIXME ill-formed
 const B &b3(a);
 // since-cxx11-error@-1 {{no viable conversion from 'struct A' to 'const B'}}
-//   since-cxx11-note@#dr2267-struct-B {{candidate constructor (the implicit copy constructor) not viable: no known conversion from 'struct A' to 'const B &' for 1st argument}}
-//   since-cxx11-note@#dr2267-struct-B {{candidate constructor (the implicit move constructor) not viable: no known conversion from 'struct A' to 'B &&' for 1st argument}}
-//   since-cxx11-note@#dr2267-struct-B {{explicit constructor is not a candidate}}
+//   since-cxx11-note@#cwg2267-struct-B {{candidate constructor (the implicit copy constructor) not viable: no known conversion from 'struct A' to 'const B &' for 1st argument}}
+//   since-cxx11-note@#cwg2267-struct-B {{candidate constructor (the implicit move constructor) not viable: no known conversion from 'struct A' to 'B &&' for 1st argument}}
+//   since-cxx11-note@#cwg2267-struct-B {{explicit constructor is not a candidate}}
 
 D d1(c);
 const D &d2{c}; // FIXME ill-formed
@@ -154,34 +154,34 @@ const D &d3(c); // FIXME ill-formed
 #endif
 }
 
-namespace dr2273 { // dr2273: 3.3
+namespace cwg2273 { // cwg2273: 3.3
 #if __cplusplus >= 201103L
 struct A {
-  A(int = 0) = delete; // #dr2273-A
+  A(int = 0) = delete; // #cwg2273-A
 };
 
-struct B : A { // #dr2273-B
+struct B : A { // #cwg2273-B
   using A::A;
 };
 
 B b;
 // since-cxx11-error@-1 {{call to implicitly-deleted default constructor of 'B'}}
-//   since-cxx11-note@#dr2273-B {{default constructor of 'B' is implicitly deleted because base class 'A' has a deleted default constructor}}
-//   since-cxx11-note@#dr2273-A {{'A' has been explicitly marked deleted here}}
+//   since-cxx11-note@#cwg2273-B {{default constructor of 'B' is implicitly deleted because base class 'A' has a deleted default constructor}}
+//   since-cxx11-note@#cwg2273-A {{'A' has been explicitly marked deleted here}}
 #endif
 }
 
-namespace dr2277 { // dr2277: partial
+namespace cwg2277 { // cwg2277: partial
 #if __cplusplus >= 201103L
 struct A {
   A(int, int = 0);
-  void f(int, int = 0); // #dr2277-A-f
+  void f(int, int = 0); // #cwg2277-A-f
 };
 struct B : A {
   B(int);
   using A::A;
 
-  void f(int); // #dr2277-B-f
+  void f(int); // #cwg2277-B-f
   using A::f;
 };
 
@@ -189,13 +189,13 @@ void g() {
   B b{0};
   b.f(0); // FIXME: this is well-formed for the same reason as initialization of 'b' above
   // since-cxx11-error@-1 {{call to member function 'f' is ambiguous}}
-  //   since-cxx11-note@#dr2277-A-f {{candidate function}}
-  //   since-cxx11-note@#dr2277-B-f {{candidate function}}
+  //   since-cxx11-note@#cwg2277-A-f {{candidate function}}
+  //   since-cxx11-note@#cwg2277-B-f {{candidate function}}
 }
 #endif
 }
 
-namespace dr2292 { // dr2292: 9
+namespace cwg2292 { // cwg2292: 9
 #if __cplusplus >= 201103L
   template<typename T> using id = T;
   void test(int *p) {
diff --git a/clang/test/CXX/drs/dr23xx.cpp b/clang/test/CXX/drs/dr23xx.cpp
index 812d95a39b63..db5b7c3cd3c9 100644
--- a/clang/test/CXX/drs/dr23xx.cpp
+++ b/clang/test/CXX/drs/dr23xx.cpp
@@ -7,7 +7,7 @@
 // RUN: %clang_cc1 -std=c++2c %s -verify=expected,since-cxx11,since-cxx14,since-cxx17,since-cxx20 -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s
 
 #if __cplusplus >= 201103L
-namespace dr2303 { // dr2303: 12
+namespace cwg2303 { // cwg2303: 12
 template <typename... T>
 struct A;
 template <>
@@ -17,12 +17,12 @@ struct A<T, Ts...> : A<Ts...> {};
 struct B : A<int, int> {};
 struct C : A<int, int>, A<int> {};
 /* since-cxx11-warning@-1 {{direct base 'A<int>' is inaccessible due to ambiguity:
-    struct dr2303::C -> A<int, int> -> A<int>
-    struct dr2303::C -> A<int>}} */
+    struct cwg2303::C -> A<int, int> -> A<int>
+    struct cwg2303::C -> A<int>}} */
 struct D : A<int>, A<int, int> {};
 /* since-cxx11-warning@-1 {{direct base 'A<int>' is inaccessible due to ambiguity:
-    struct dr2303::D -> A<int>
-    struct dr2303::D -> A<int, int> -> A<int>}} */
+    struct cwg2303::D -> A<int>
+    struct cwg2303::D -> A<int, int> -> A<int>}} */
 struct E : A<int, int> {};
 struct F : B, E {};
 
@@ -41,17 +41,17 @@ void g() {
   f(D{});
   f(F{});
   /* since-cxx11-error@-1 {{ambiguous conversion from derived class 'const F' to base class 'const A<int, int>':
-    struct dr2303::F -> B -> A<int, int>
-    struct dr2303::F -> E -> A<int, int>}} */
+    struct cwg2303::F -> B -> A<int, int>
+    struct cwg2303::F -> E -> A<int, int>}} */
 }
-} // namespace dr2303
+} // namespace cwg2303
 #endif
 
-// dr2331: na
-// dr2335 is in dr2335.cxx
+// cwg2331: na
+// cwg2335 is in cwg2335.cxx
 
 #if __cplusplus >= 201103L
-namespace dr2338 { // dr2338: 12
+namespace cwg2338 { // cwg2338: 12
 namespace B {
 enum E : bool { Zero, One };
 static_assert((int)(E)2 == 1, "");
@@ -60,17 +60,17 @@ namespace D {
 enum class E : bool { Zero, One };
 static_assert((int)(E)2 == 1, "");
 } // namespace D
-} // namespace dr2338
+} // namespace cwg2338
 #endif
 
-namespace dr2346 { // dr2346: 11
+namespace cwg2346 { // cwg2346: 11
   void test() {
     const int i2 = 0;
     extern void h2b(int x = i2 + 0); // ok, not odr-use
   }
 }
 
-namespace dr2352 { // dr2352: 10
+namespace cwg2352 { // cwg2352: 10
   int **p;
   const int *const *const &f1() { return p; }
   int *const *const &f2() { return p; }
@@ -106,7 +106,7 @@ namespace dr2352 { // dr2352: 10
 #endif
 }
 
-namespace dr2353 { // dr2353: 9
+namespace cwg2353 { // cwg2353: 9
   struct X {
     static const int n = 0;
   };
@@ -138,16 +138,16 @@ namespace dr2353 { // dr2353: 9
 #pragma clang __debug dump not_use_2
 }
 
-namespace dr2354 { // dr2354: 15
+namespace cwg2354 { // cwg2354: 15
 #if __cplusplus >= 201103L
 enum alignas(64) A {};
 // since-cxx11-error@-1 {{'alignas' attribute cannot be applied to an enumeration}}
 enum struct alignas(64) B {};
 // since-cxx11-error@-1 {{'alignas' attribute cannot be applied to an enumeration}}
 #endif
-} // namespace dr2354
+} // namespace cwg2354
 
-namespace dr2356 { // dr2356: 4
+namespace cwg2356 { // cwg2356: 4
 #if __cplusplus >= 201103L
 struct A {
   A();
@@ -159,11 +159,11 @@ struct B : A {
   B(const B &);                   // #3
   B(B &&) = default;              // #4, implicitly deleted
   // since-cxx11-warning@-1 {{explicitly defaulted move constructor is implicitly deleted}}
-  //   since-cxx11-note@#dr2356-X {{move constructor of 'B' is implicitly deleted because field 'x' has a deleted move constructor}}
-  //   since-cxx11-note@#dr2356-X {{'X' has been explicitly marked deleted here}}
+  //   since-cxx11-note@#cwg2356-X {{move constructor of 'B' is implicitly deleted because field 'x' has a deleted move constructor}}
+  //   since-cxx11-note@#cwg2356-X {{'X' has been explicitly marked deleted here}}
   //   since-cxx11-note@-4 {{replace 'default' with 'delete'}}
 
-  struct X { X(X &&) = delete; } x; // #dr2356-X
+  struct X { X(X &&) = delete; } x; // #cwg2356-X
 };
 extern B b1;
 B b2 = static_cast<B&&>(b1);      // calls #3: #1, #2, and #4 are not viable
@@ -173,7 +173,7 @@ B b3 = C();                       // calls #3
 }
 
 #if __cplusplus >= 201402L
-namespace dr2358 { // dr2358: 16
+namespace cwg2358 { // cwg2358: 16
   void f2() {
     int i = 1;
     void g1(int = [xxx=1] { return xxx; }());  // OK
@@ -185,7 +185,7 @@ namespace dr2358 { // dr2358: 16
 
 // CWG2363 was closed as NAD, but its resolution does affirm that
 // a friend declaration cannot have an opaque-enumm-specifier.
-namespace dr2363 { // dr2363: yes
+namespace cwg2363 { // cwg2363: yes
 #if __cplusplus >= 201103L
 enum class E0;
 enum E1 : int;
@@ -215,9 +215,9 @@ struct A {
   // expected-note@-3 {{remove 'enum class' to befriend an enum}}
 };
 #endif
-} // namespace dr2363
+} // namespace cwg2363
 
-namespace dr2370 { // dr2370: no
+namespace cwg2370 { // cwg2370: no
 namespace N {
 typedef int type;
 void g(type);
@@ -229,33 +229,33 @@ class C {
   // friend void N::g(type);
   friend void N::h(N_type);
 };
-} // namespace dr2370
+} // namespace cwg2370
 
 #if __cplusplus >= 201702L
 // Otherwise, if the qualified-id std::tuple_size<E> names a complete class
 // type **with a member value**, the expression std::tuple_size<E>::value shall
 // be a well-formed integral constant expression
-namespace dr2386 { // dr2386: 9
+namespace cwg2386 { // cwg2386: 9
 struct Bad1 { int a, b; };
 struct Bad2 { int a, b; };
-} // namespace dr2386
+} // namespace cwg2386
 namespace std {
 template <typename T> struct tuple_size;
-template <> struct tuple_size<dr2386::Bad1> {};
-template <> struct tuple_size<dr2386::Bad2> {
+template <> struct tuple_size<cwg2386::Bad1> {};
+template <> struct tuple_size<cwg2386::Bad2> {
   static const int value = 42;
 };
 } // namespace std
-namespace dr2386 {
+namespace cwg2386 {
 void no_value() { auto [x, y] = Bad1(); }
 void wrong_value() { auto [x, y] = Bad2(); }
 // since-cxx17-error@-1 {{type 'Bad2' decomposes into 42 elements, but only 2 names were provided}}
-} // namespace dr2386
+} // namespace cwg2386
 #endif
 
-// dr2385: na
+// cwg2385: na
 
-namespace dr2387 { // dr2387: 9
+namespace cwg2387 { // cwg2387: 9
 #if __cplusplus >= 201402L
   template<int> int a = 0;
   extern template int a<0>; // ok
@@ -273,9 +273,9 @@ namespace dr2387 { // dr2387: 9
 #endif
 }
 
-// dr2390 is in dr2390.cpp
+// cwg2390 is in cwg2390.cpp
 
-namespace dr2394 { // dr2394: 15
+namespace cwg2394 { // cwg2394: 15
 
 struct A {};
 const A a;
@@ -286,7 +286,7 @@ B b;
 
 }
 
-namespace dr2396 { // dr2396: no
+namespace cwg2396 { // cwg2396: no
   struct A {
     struct B;
     operator B B::*();
@@ -295,7 +295,7 @@ namespace dr2396 { // dr2396: no
 
   // FIXME: per P1787 "Calling a conversion function" example, all of the
   // examples below are well-formed, with B resolving to A::B, but currently
-  // it's been resolved to dr2396::B.
+  // it's been resolved to cwg2396::B.
 
   // void f(A a) { a.operator B B::*(); }
   // void g(A a) { a.operator decltype(B()) B::*(); }
@@ -303,13 +303,13 @@ namespace dr2396 { // dr2396: no
 }
 
 #if __cplusplus >= 201103L
-namespace dr2397 { // dr2397: 17
+namespace cwg2397 { // cwg2397: 17
   void foo() {
     int a[5];
 
     auto (&b)[5] = a;
     auto (*c)[5] = &a;
   }
-} // namespace dr2397
+} // namespace cwg2397
 
 #endif
diff --git a/clang/test/CXX/drs/dr24xx.cpp b/clang/test/CXX/drs/dr24xx.cpp
index 4534ed26e56d..5ffaebda68c1 100644
--- a/clang/test/CXX/drs/dr24xx.cpp
+++ b/clang/test/CXX/drs/dr24xx.cpp
@@ -10,7 +10,7 @@
 // expected-no-diagnostics
 #endif
 
-namespace dr2406 { // dr2406: 5
+namespace cwg2406 { // cwg2406: 5
 #if __cplusplus >= 201703L
 void fallthrough(int n) {
   void g(), h(), i();
@@ -45,7 +45,7 @@ void fallthrough(int n) {
 #endif
 }
 
-namespace dr2450 { // dr2450: 18 review P2308R1
+namespace cwg2450 { // cwg2450: 18 review P2308R1
 #if __cplusplus >= 202302L
 struct S {int a;};
 template <S s>
@@ -59,7 +59,7 @@ f<{.a= 0}>();
 #endif
 }
 
-namespace dr2459 { // dr2459: 18 drafting P2308R1
+namespace cwg2459 { // cwg2459: 18 drafting P2308R1
 #if __cplusplus >= 202302L
 struct A {
   constexpr A(float) {}
@@ -69,7 +69,7 @@ X<1> x;
 #endif
 }
 
-namespace dr2445 { // dr2445: 19
+namespace cwg2445 { // cwg2445: 19
 #if __cplusplus >= 202002L
   template <typename> constexpr bool F = false;
   template <typename T> struct A { };
diff --git a/clang/test/CXX/drs/dr25xx.cpp b/clang/test/CXX/drs/dr25xx.cpp
index 46532486e50e..62b2a0a088cc 100644
--- a/clang/test/CXX/drs/dr25xx.cpp
+++ b/clang/test/CXX/drs/dr25xx.cpp
@@ -10,9 +10,9 @@
 // expected-no-diagnostics
 #endif
 
-// dr2504 is in dr2504.cpp
+// cwg2504 is in cwg2504.cpp
 
-namespace dr2516 { // dr2516: 3.0
+namespace cwg2516 { // cwg2516: 3.0
                    // NB: reusing 1482 test
 #if __cplusplus >= 201103L
 template <typename T> struct S {
@@ -21,9 +21,9 @@ template <typename T> struct S {
 enum E2 : S<E2>::I { e };
 // since-cxx11-error@-1 {{use of undeclared identifier 'E2'}}
 #endif
-} // namespace dr2516
+} // namespace cwg2516
 
-namespace dr2518 { // dr2518: 17
+namespace cwg2518 { // cwg2518: 17
 
 #if __cplusplus >= 201103L
 template <class T>
@@ -32,13 +32,13 @@ void f(T t) {
   // cxx11-14-error@-1 {{constexpr if is a C++17 extension}}
     static_assert(false, "must be int-sized");
     // since-cxx11-error@-1 {{static assertion failed: must be int-sized}}
-    //   since-cxx11-note@#dr2518-f-c {{in instantiation of function template specialization 'dr2518::f<char>' requested here}}
+    //   since-cxx11-note@#cwg2518-f-c {{in instantiation of function template specialization 'cwg2518::f<char>' requested here}}
   }
 }
 
 void g(char c) {
   f(0);
-  f(c); // #dr2518-f-c
+  f(c); // #cwg2518-f-c
 }
 
 template <typename Ty>
@@ -46,7 +46,7 @@ struct S {
   static_assert(false);
   // cxx11-14-error@-1 {{'static_assert' with no message is a C++17 extension}}
   // since-cxx11-error@-2 {{static assertion failed}}
-  //   since-cxx11-note@#dr2518-S-double {{in instantiation of template class 'dr2518::S<double>' requested here}}
+  //   since-cxx11-note@#cwg2518-S-double {{in instantiation of template class 'cwg2518::S<double>' requested here}}
 };
 
 template <>
@@ -58,13 +58,13 @@ struct S<float> {};
 int test_specialization() {
   S<int> s1;
   S<float> s2;
-  S<double> s3; // #dr2518-S-double
+  S<double> s3; // #cwg2518-S-double
 }
 #endif
 
 }
 
-namespace dr2521 { // dr2521: 17
+namespace cwg2521 { // cwg2521: 17
 #if __cplusplus >= 201103L
 #pragma clang diagnostic push
 #pragma clang diagnostic warning "-Wdeprecated-literal-operator"
@@ -76,61 +76,61 @@ template <char... Chars> decltype(sizeof 0)
 operator""  _div();
 // since-cxx11-warning@-1 {{identifier '_div' preceded by whitespace in a literal operator declaration is deprecated}}
 
-using ::dr2521::operator"" _\u03C0___;
-using ::dr2521::operator""_div;
+using ::cwg2521::operator"" _\u03C0___;
+using ::cwg2521::operator""_div;
 // since-cxx11-warning@-2 {{identifier '_π___' preceded by whitespace in a literal operator declaration is deprecated}}
 #pragma clang diagnostic pop
 #endif
-} // namespace dr2521
+} // namespace cwg2521
 
 
 #if __cplusplus >= 202302L
-namespace dr2553 { // dr2553: 18 review 2023-07-14
+namespace cwg2553 { // cwg2553: 18 review 2023-07-14
 struct B {
   virtual void f(this B&); 
   // since-cxx23-error@-1 {{an explicit object parameter cannot appear in a virtual function}}
   static void f(this B&);
   // since-cxx23-error@-1 {{an explicit object parameter cannot appear in a static function}}
-  virtual void g(); // #dr2553-g
+  virtual void g(); // #cwg2553-g
 };
 struct D : B {
   void g(this D&);
   // since-cxx23-error@-1 {{an explicit object parameter cannot appear in a virtual function}}
-  //   since-cxx23-note@#dr2553-g {{overridden virtual function is here}}
+  //   since-cxx23-note@#cwg2553-g {{overridden virtual function is here}}
 };
 
 }
 #endif
 
 #if __cplusplus >= 202302L
-namespace dr2554 { // dr2554: 18 review 2021-12-10
+namespace cwg2554 { // cwg2554: 18 review 2021-12-10
 struct B {
-  virtual void f(); // #dr2554-g
+  virtual void f(); // #cwg2554-g
 };
 
 struct D : B {
   void f(this D&);
   // since-cxx23-error@-1 {{an explicit object parameter cannot appear in a virtual function}}
-  //   since-cxx23-note@#dr2554-g {{overridden virtual function is here}}
+  //   since-cxx23-note@#cwg2554-g {{overridden virtual function is here}}
 };
 
 struct D2 : B {
   void f(this B&);
   // since-cxx23-error@-1 {{an explicit object parameter cannot appear in a virtual function}}
-  //   since-cxx23-note@#dr2554-g {{overridden virtual function is here}}
+  //   since-cxx23-note@#cwg2554-g {{overridden virtual function is here}}
 };
 struct T {};
 struct D3 : B {
   void f(this T&);
   // since-cxx23-error@-1 {{an explicit object parameter cannot appear in a virtual function}}
-  //   since-cxx23-note@#dr2554-g {{overridden virtual function is here}}
+  //   since-cxx23-note@#cwg2554-g {{overridden virtual function is here}}
 };
 
 }
 #endif
 
 #if __cplusplus >= 202302L
-namespace dr2561 { // dr2561: 18 review 2023-11-09
+namespace cwg2561 { // cwg2561: 18 review 2023-11-09
 struct C {
     constexpr C(auto) { }
 };
@@ -145,7 +145,7 @@ void foo() {
 #endif
 
 
-namespace dr2565 { // dr2565: 16 open 2023-06-07
+namespace cwg2565 { // cwg2565: 16 open 2023-06-07
 #if __cplusplus >= 202002L
   template<typename T>
     concept C = requires (typename T::type x) {
@@ -165,28 +165,28 @@ namespace dr2565 { // dr2565: 16 open 2023-06-07
   static_assert(is_referenceable<int>::value);
 
   template<typename T, typename U>
-  concept TwoParams = requires (T *a, U b){ true;}; // #dr2565-TPC
+  concept TwoParams = requires (T *a, U b){ true;}; // #cwg2565-TPC
 
   template<typename T, typename U>
-    requires TwoParams<T, U> // #dr2565-TPSREQ
+    requires TwoParams<T, U> // #cwg2565-TPSREQ
   struct TwoParamsStruct{};
 
   using TPSU = TwoParamsStruct<void, void>;
   // since-cxx20-error@-1 {{constraints not satisfied for class template 'TwoParamsStruct'}}
-  //   since-cxx20-note@#dr2565-TPSREQ {{because 'TwoParams<void, void>' evaluated to false}}
-  //   since-cxx20-note@#dr2565-TPC {{because 'b' would be invalid: argument may not have 'void' type}}
+  //   since-cxx20-note@#cwg2565-TPSREQ {{because 'TwoParams<void, void>' evaluated to false}}
+  //   since-cxx20-note@#cwg2565-TPC {{because 'b' would be invalid: argument may not have 'void' type}}
 
   template<typename T, typename ...U>
-  concept Variadic = requires (U* ... a, T b){ true;}; // #dr2565-VC
+  concept Variadic = requires (U* ... a, T b){ true;}; // #cwg2565-VC
 
   template<typename T, typename ...U>
-    requires Variadic<T, U...> // #dr2565-VSREQ
+    requires Variadic<T, U...> // #cwg2565-VSREQ
   struct VariadicStruct{};
 
   using VSU = VariadicStruct<void, int, char, double>;
   // since-cxx20-error@-1 {{constraints not satisfied for class template 'VariadicStruct'}}
-  //   since-cxx20-note@#dr2565-VSREQ {{because 'Variadic<void, int, char, double>' evaluated to false}}
-  //   since-cxx20-note@#dr2565-VC {{because 'b' would be invalid: argument may not have 'void' type}}
+  //   since-cxx20-note@#cwg2565-VSREQ {{because 'Variadic<void, int, char, double>' evaluated to false}}
+  //   since-cxx20-note@#cwg2565-VC {{because 'b' would be invalid: argument may not have 'void' type}}
 
   template<typename T>
   concept ErrorRequires = requires (ErrorRequires auto x) {
@@ -211,7 +211,7 @@ namespace dr2565 { // dr2565: 16 open 2023-06-07
 #endif
 }
 
-namespace dr2583 { // dr2583: 19
+namespace cwg2583 { // cwg2583: 19
 #if __cplusplus >= 201103L
 struct A {
   int i;
@@ -236,9 +236,9 @@ union V {
 static_assert(!__is_layout_compatible(A, B), "");
 static_assert(__is_layout_compatible(U, V), "");
 #endif
-} // namespace dr2583
+} // namespace cwg2583
 
-namespace dr2598 { // dr2598: 18
+namespace cwg2598 { // cwg2598: 18
 #if __cplusplus >= 201103L
 struct NonLiteral {
     NonLiteral();
diff --git a/clang/test/CXX/drs/dr26xx.cpp b/clang/test/CXX/drs/dr26xx.cpp
index f151c9eea051..f7a05b9827a2 100644
--- a/clang/test/CXX/drs/dr26xx.cpp
+++ b/clang/test/CXX/drs/dr26xx.cpp
@@ -7,7 +7,7 @@
 // RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx20,since-cxx23
 
 
-namespace dr2621 { // dr2621: 16
+namespace cwg2621 { // cwg2621: 16
 #if __cplusplus >= 202002L
 enum class E { a };
 namespace One {
@@ -16,7 +16,7 @@ using enum E_t; // typedef ok
 auto v = a;
 }
 namespace Two {
-using dr2621::E;
+using cwg2621::E;
 int E; // we see this
 using enum E;
 // since-cxx20-error@-1 {{unknown type name E}}
@@ -24,32 +24,32 @@ using enum E;
 #endif
 }
 
-namespace dr2628 { // dr2628: no
+namespace cwg2628 { // cwg2628: no
                    // this was reverted for the 16.x release
                    // due to regressions, see the issue for more details:
                    // https://github.com/llvm/llvm-project/issues/60777
 #if __cplusplus >= 202002L
 template <bool A = false, bool B = false>
 struct foo {
-  // The expected notes below should be removed when dr2628 is fully implemented again
-  constexpr foo() requires (!A && !B) = delete; // #dr2628-ctor-1
-  constexpr foo() requires (A || B) = delete; //  #dr2628-ctor-2
+  // The expected notes below should be removed when cwg2628 is fully implemented again
+  constexpr foo() requires (!A && !B) = delete; // #cwg2628-ctor-1
+  constexpr foo() requires (A || B) = delete; //  #cwg2628-ctor-2
 };
 
 void f() {
-  // The FIXME's below should be the expected errors when dr2628 is
+  // The FIXME's below should be the expected errors when cwg2628 is
   // fully implemented again.
-  foo fooable; // #dr2628-fooable
+  foo fooable; // #cwg2628-fooable
   // since-cxx20-error@-1 {{ambiguous deduction for template arguments of 'foo'}}
-  //   since-cxx20-note@#dr2628-ctor-1 {{candidate function [with A = false, B = false]}}
-  //   since-cxx20-note@#dr2628-ctor-2 {{candidate function [with A = false, B = false]}}
-  // FIXME-since-cxx20-error@#dr2628-fooable {{call to deleted}} 
-  //   FIXME-since-cxx20-note@#dr2628-ctor {{marked deleted here}} 
+  //   since-cxx20-note@#cwg2628-ctor-1 {{candidate function [with A = false, B = false]}}
+  //   since-cxx20-note@#cwg2628-ctor-2 {{candidate function [with A = false, B = false]}}
+  // FIXME-since-cxx20-error@#cwg2628-fooable {{call to deleted}} 
+  //   FIXME-since-cxx20-note@#cwg2628-ctor {{marked deleted here}} 
 }
 #endif
 }
 
-namespace dr2631 { // dr2631: 16
+namespace cwg2631 { // cwg2631: 16
 #if __cplusplus >= 202002L
   constexpr int g();
   consteval int f() {
@@ -67,7 +67,7 @@ namespace dr2631 { // dr2631: 16
 #endif
 }
 
-namespace dr2635 { // dr2635: 16
+namespace cwg2635 { // cwg2635: 16
 #if __cplusplus >= 202002L
 template<typename T>
 concept UnaryC = true;
@@ -97,9 +97,9 @@ void TemplUse() {
 #endif
 }
 
-// dr2636: na
+// cwg2636: na
 
-namespace dr2640 { // dr2640: 16
+namespace cwg2640 { // cwg2640: 16
 
 int \N{Λ} = 0;
 // expected-error@-1 {{'Λ' is not a valid Unicode character name}}
@@ -113,18 +113,18 @@ const char* emoji = "\N{🤡}";
 //   expected-note@-6 {{did you mean BAT ('🦇' U+1F987)?}}
 
 #define z(x) 0
-#define dr2640_a z(
-int x = dr2640_a\N{abc});
+#define cwg2640_a z(
+int x = cwg2640_a\N{abc});
 // expected-error@-1 {{'abc' is not a valid Unicode character name}}
-int y = dr2640_a\N{LOTUS});
+int y = cwg2640_a\N{LOTUS});
 // expected-error@-1 {{character <U+1FAB7> not allowed in an identifier}}
-// expected-error@-2 {{use of undeclared identifier 'dr2640_a🪷'}}
+// expected-error@-2 {{use of undeclared identifier 'cwg2640_a🪷'}}
 // expected-error@-3 {{extraneous ')' before ';'}}
 }
 
-// dr2642: na
+// cwg2642: na
 
-namespace dr2644 { // dr2644: 8
+namespace cwg2644 { // cwg2644: 8
 #if __cplusplus >= 201103L
 auto z = [a = 42](int a) {
 // cxx11-warning@-1 {{initialized lambda captures are a C++14 extension}}
@@ -136,20 +136,20 @@ auto z = [a = 42](int a) {
 }
 
 #if __cplusplus >= 202302L
-namespace dr2650 { // dr2650: 17
+namespace cwg2650 { // cwg2650: 17
 template <class T, T> struct S {};
-template <class T> int f(S<T, T{}>*); // #dr2650-f
+template <class T> int f(S<T, T{}>*); // #cwg2650-f
 class X {
   int m;
 };
 int i0 = f<X>(0);
 // since-cxx23-error@-1 {{no matching function for call to 'f'}}
-//   since-cxx23-note@#dr2650-f {{type 'X' of non-type template parameter is not a structural type}}
+//   since-cxx23-note@#cwg2650-f {{type 'X' of non-type template parameter is not a structural type}}
 }
 #endif
 
 #if __cplusplus >= 202302L
-namespace dr2653 { // dr2653: 18
+namespace cwg2653 { // cwg2653: 18
   struct Test { void f(this const auto& = Test{}); };
   // since-cxx23-error@-1 {{the explicit object parameter cannot have a default argument}}
   auto L = [](this const auto& = Test{}){};
@@ -157,7 +157,7 @@ namespace dr2653 { // dr2653: 18
 }
 #endif
 
-namespace dr2654 { // dr2654: 16
+namespace cwg2654 { // cwg2654: 16
 void f() {
     int neck, tail;
     volatile int brachiosaur;
@@ -167,7 +167,7 @@ void f() {
 }
 }
 
-namespace dr2681 { // dr2681: 17
+namespace cwg2681 { // cwg2681: 17
 #if __cplusplus >= 202002L
 using size_t = decltype(sizeof(int));
 
@@ -180,7 +180,7 @@ struct I {
   volatile T array[N];
 };
 template<size_t N>
-struct J { // #dr2681-J
+struct J { // #cwg2681-J
   unsigned char array[N];
 };
 
@@ -191,13 +191,13 @@ static_assert(__is_same(decltype(i), I<char, 4>));
 
 J j = { "ghi" };
 // since-cxx20-error@-1 {{no viable constructor or deduction guide}}
-//   since-cxx20-note@#dr2681-J {{candidate template ignored: could not match 'J<N>' against 'const char *'}}
-//   since-cxx20-note@#dr2681-J {{candidate template ignored: could not match 'const unsigned char' against 'const char'}}
-//   since-cxx20-note@#dr2681-J {{candidate function template not viable: requires 0 arguments, but 1 was provided}}
+//   since-cxx20-note@#cwg2681-J {{candidate template ignored: could not match 'J<N>' against 'const char *'}}
+//   since-cxx20-note@#cwg2681-J {{candidate template ignored: could not match 'const unsigned char' against 'const char'}}
+//   since-cxx20-note@#cwg2681-J {{candidate function template not viable: requires 0 arguments, but 1 was provided}}
 #endif
 }
 
-namespace dr2672 { // dr2672: 18
+namespace cwg2672 { // cwg2672: 18
 #if __cplusplus >= 202002L
 template <class T>
 void f(T) requires requires { []() { T::invalid; } (); };
@@ -205,8 +205,8 @@ void f(T) requires requires { []() { T::invalid; } (); };
 //   since-cxx20-note@-2 {{while substituting into a lambda expression here}}
 //   since-cxx20-note@-3 {{in instantiation of requirement here}}
 //   since-cxx20-note@-4 {{while substituting template arguments into constraint expression here}}
-//   since-cxx20-note@#dr2672-f-0 {{while checking constraint satisfaction for template 'f<int>' required here}}
-//   since-cxx20-note@#dr2672-f-0 {{in instantiation of function template specialization 'dr2672::f<int>' requested here}}
+//   since-cxx20-note@#cwg2672-f-0 {{while checking constraint satisfaction for template 'f<int>' required here}}
+//   since-cxx20-note@#cwg2672-f-0 {{in instantiation of function template specialization 'cwg2672::f<int>' requested here}}
 void f(...);
 
 template <class T>
@@ -216,14 +216,14 @@ void bar(T) requires requires {
 void bar(...);
 
 void m() {
-  f(0); // #dr2672-f-0
+  f(0); // #cwg2672-f-0
   bar(0);
 }
 #endif
 }
 
 #if __cplusplus >= 202302L
-namespace dr2687 { // dr2687: 18
+namespace cwg2687 { // cwg2687: 18
 struct S{
     void f(int);
     static void g(int);
@@ -232,7 +232,7 @@ struct S{
 
 void test() {
     (&S::f)(1);
-    // since-cxx23-error@-1 {{called object type 'void (dr2687::S::*)(int)' is not a function or function pointer}}
+    // since-cxx23-error@-1 {{called object type 'void (cwg2687::S::*)(int)' is not a function or function pointer}}
     (&S::g)(1);
     (&S::h)(S(), 1);
 }
diff --git a/clang/test/CXX/drs/dr27xx.cpp b/clang/test/CXX/drs/dr27xx.cpp
index c956c4355abd..0434427d6c92 100644
--- a/clang/test/CXX/drs/dr27xx.cpp
+++ b/clang/test/CXX/drs/dr27xx.cpp
@@ -10,7 +10,7 @@
 // expected-no-diagnostics
 #endif
 
-namespace dr2759 { // dr2759: 19
+namespace cwg2759 { // cwg2759: 19
 #if __cplusplus >= 201103L
 
 struct CStruct {
@@ -91,18 +91,18 @@ static_assert(!__is_layout_compatible(UnionLayout, UnionLayout3), "");
 static_assert(!__is_layout_compatible(StructWithAnonUnion, StructWithAnonUnion2), "");
 static_assert(!__is_layout_compatible(StructWithAnonUnion, StructWithAnonUnion3), "");
 #endif
-} // namespace dr2759
+} // namespace cwg2759
 
-namespace dr2789 { // dr2789: 18
+namespace cwg2789 { // cwg2789: 18
 #if __cplusplus >= 202302L
 template <typename T = int>
 struct Base {
-    constexpr void g(); // #dr2789-g1
+    constexpr void g(); // #cwg2789-g1
 };
 
 template <typename T = int>
 struct Base2 {
-    constexpr void g() requires true;  // #dr2789-g2
+    constexpr void g() requires true;  // #cwg2789-g2
 };
 
 template <typename T = int>
@@ -119,13 +119,13 @@ void test() {
     s.f();
     s.g();
     // since-cxx23-error@-1 {{call to member function 'g' is ambiguous}}
-    //   since-cxx23-note@#dr2789-g1 {{candidate function}}
-    //   since-cxx23-note@#dr2789-g2 {{candidate function}}
+    //   since-cxx23-note@#cwg2789-g1 {{candidate function}}
+    //   since-cxx23-note@#cwg2789-g2 {{candidate function}}
 }
 #endif
 }
 
-namespace dr2798 { // dr2798: 17
+namespace cwg2798 { // cwg2798: 17
 #if __cpp_static_assert >= 202306
 struct string {
   constexpr string() {
@@ -147,5 +147,5 @@ consteval X f() { return {}; }
 static_assert(false, f().s);
 // since-cxx26-error@-1 {{static assertion failed: Hello}}
 #endif
-} // namespace dr2798
+} // namespace cwg2798
 
diff --git a/clang/test/CXX/drs/dr28xx.cpp b/clang/test/CXX/drs/dr28xx.cpp
index 9b21d3410a04..4d9b0c76758d 100644
--- a/clang/test/CXX/drs/dr28xx.cpp
+++ b/clang/test/CXX/drs/dr28xx.cpp
@@ -10,7 +10,7 @@
 // expected-no-diagnostics
 #endif
 
-namespace dr2847 { // dr2847: 19
+namespace cwg2847 { // cwg2847: 19
 
 #if __cplusplus >= 202002L
 
@@ -57,9 +57,9 @@ void B<int>::g() requires true;
 
 #endif
 
-} // namespace dr2847
+} // namespace cwg2847
 
-namespace dr2858 { // dr2858: 19
+namespace cwg2858 { // cwg2858: 19
 
 #if __cplusplus > 202302L
 
@@ -80,4 +80,4 @@ struct A {
 
 #endif
 
-} // namespace dr2858
+} // namespace cwg2858
diff --git a/clang/test/CXX/drs/dr2xx.cpp b/clang/test/CXX/drs/dr2xx.cpp
index cbb8734e10c6..e655e7226d51 100644
--- a/clang/test/CXX/drs/dr2xx.cpp
+++ b/clang/test/CXX/drs/dr2xx.cpp
@@ -16,7 +16,7 @@ typedef __SIZE_TYPE__ size_t;
 #define fold
 #endif
 
-namespace dr200 { // dr200: dup 214
+namespace cwg200 { // cwg200: dup 214
   template <class T> T f(int);
   template <class T, class U> T f(U) = delete;
   // cxx98-error@-1 {{deleted function definitions are a C++11 extension}}
@@ -26,9 +26,9 @@ namespace dr200 { // dr200: dup 214
   }
 }
 
-// dr201 is in dr201.cpp
+// cwg201 is in cwg201.cpp
 
-namespace dr202 { // dr202: 3.1
+namespace cwg202 { // cwg202: 3.1
   template<typename T> T f();
   template<int (*g)()> struct X {
     int arr[fold(g == &f<int>) ? 1 : -1];
@@ -36,19 +36,19 @@ namespace dr202 { // dr202: 3.1
   template struct X<f>;
 }
 
-// FIXME (export) dr204: no
+// FIXME (export) cwg204: no
 
-namespace dr206 { // dr206: yes
-  struct S; // #dr206-S
+namespace cwg206 { // cwg206: yes
+  struct S; // #cwg206-S
   template<typename T> struct Q { S s; };
   // expected-error@-1 {{field has incomplete type 'S'}}
-  //   expected-note@#dr206-S {{forward declaration of 'dr206::S'}}
+  //   expected-note@#cwg206-S {{forward declaration of 'cwg206::S'}}
   template<typename T> void f() { S s; }
   // expected-error@-1 {{variable has incomplete type 'S'}}
-  //   expected-note@#dr206-S {{forward declaration of 'dr206::S'}}
+  //   expected-note@#cwg206-S {{forward declaration of 'cwg206::S'}}
 }
 
-namespace dr207 { // dr207: yes
+namespace cwg207 { // cwg207: yes
   class A {
   protected:
     static void f() {}
@@ -63,22 +63,22 @@ namespace dr207 { // dr207: yes
   };
 }
 
-// dr208 FIXME: write codegen test
+// cwg208 FIXME: write codegen test
 
-namespace dr209 { // dr209: 3.2
+namespace cwg209 { // cwg209: 3.2
   class A {
-    void f(); // #dr209-A-f
+    void f(); // #cwg209-A-f
   };
   class B {
     friend void A::f();
-    // expected-error@-1 {{friend function 'f' is a private member of 'dr209::A'}}
-    //   expected-note@#dr209-A-f {{implicitly declared private here}}
+    // expected-error@-1 {{friend function 'f' is a private member of 'cwg209::A'}}
+    //   expected-note@#cwg209-A-f {{implicitly declared private here}}
   };
 }
 
-// dr210 is in dr210.cpp
+// cwg210 is in cwg210.cpp
 
-namespace dr211 { // dr211: yes
+namespace cwg211 { // cwg211: yes
   struct A {
     A() try {
       throw 0;
@@ -89,26 +89,26 @@ namespace dr211 { // dr211: yes
   };
 }
 
-namespace dr213 { // dr213: yes
+namespace cwg213 { // cwg213: yes
   template <class T> struct A : T {
     void h(T t) {
       char &r1 = f(t);
       int &r2 = g(t);
       // expected-error@-1 {{explicit qualification required to use member 'g' from dependent base class}}
-      //   expected-note@#dr213-instantiation {{in instantiation of member function 'dr213::A<dr213::B>::h' requested here}}
-      //   expected-note@#dr213-B-g {{member is declared here}}
+      //   expected-note@#cwg213-instantiation {{in instantiation of member function 'cwg213::A<cwg213::B>::h' requested here}}
+      //   expected-note@#cwg213-B-g {{member is declared here}}
     }
   };
   struct B {
     int &f(B);
-    int &g(B); // #dr213-B-g
+    int &g(B); // #cwg213-B-g
   };
   char &f(B);
 
-  template void A<B>::h(B); // #dr213-instantiation
+  template void A<B>::h(B); // #cwg213-instantiation
 }
 
-namespace dr214 { // dr214: yes
+namespace cwg214 { // cwg214: yes
   template<typename T, typename U> T checked_cast(U from) { U::error; }
   template<typename T, typename U> T checked_cast(U *from);
   class C {};
@@ -121,7 +121,7 @@ namespace dr214 { // dr214: yes
   }
 }
 
-namespace dr215 { // dr215: 2.9
+namespace cwg215 { // cwg215: 2.9
   template<typename T> class X {
     friend void T::foo();
     int n;
@@ -131,7 +131,7 @@ namespace dr215 { // dr215: 2.9
   };
 }
 
-namespace dr216 { // dr216: no
+namespace cwg216 { // cwg216: no
   // FIXME: Should reject this: 'f' has linkage but its type does not,
   // and 'f' is odr-used but not defined in this TU.
   typedef enum { e } *E;
@@ -147,7 +147,7 @@ namespace dr216 { // dr216: no
   void g(S s, S::E e) { s.f(e); }
 }
 
-namespace dr217 { // dr217: yes
+namespace cwg217 { // cwg217: yes
   template<typename T> struct S {
     void f(int);
   };
@@ -155,7 +155,7 @@ namespace dr217 { // dr217: yes
   // expected-error@-1 {{default arguments cannot be added to an out-of-line definition of a member of a class template}}
 }
 
-namespace dr218 { // dr218: yes
+namespace cwg218 { // cwg218: yes
                   // NB: also dup 405
   namespace A {
     struct S {};
@@ -219,12 +219,12 @@ namespace dr218 { // dr218: yes
   void testG(G::C<G::X::A, G::Y::B> gc) { f(gc); }
 }
 
-// dr219: na
-// dr220: na
+// cwg219: na
+// cwg220: na
 
-namespace dr221 { // dr221: 3.6
-  struct A { // #dr221-S
-    A &operator=(int&); // #dr221-S-copy-assign
+namespace cwg221 { // cwg221: 3.6
+  struct A { // #cwg221-S
+    A &operator=(int&); // #cwg221-S-copy-assign
     A &operator+=(int&);
     static A &operator=(A&, double&);
     // expected-error@-1 {{overloaded 'operator=' cannot be a static member function}}
@@ -243,20 +243,20 @@ namespace dr221 { // dr221: 3.6
     a += n;
     a = c;
     // expected-error@-1 {{no viable overloaded '='}}
-    //   expected-note@#dr221-S-copy-assign {{candidate function not viable: no known conversion from 'char' to 'int &' for 1st argument}}
-    //   since-cxx11-note@#dr221-S {{candidate function (the implicit move assignment operator) not viable: no known conversion from 'char' to 'A' for 1st argument}}
-    //   expected-note@#dr221-S {{candidate function (the implicit copy assignment operator) not viable: no known conversion from 'char' to 'const A' for 1st argument}}
+    //   expected-note@#cwg221-S-copy-assign {{candidate function not viable: no known conversion from 'char' to 'int &' for 1st argument}}
+    //   since-cxx11-note@#cwg221-S {{candidate function (the implicit move assignment operator) not viable: no known conversion from 'char' to 'A' for 1st argument}}
+    //   expected-note@#cwg221-S {{candidate function (the implicit copy assignment operator) not viable: no known conversion from 'char' to 'const A' for 1st argument}}
     a += c;
     a = f;
     // expected-error@-1 {{no viable overloaded '='}}
-    //   expected-note@#dr221-S-copy-assign {{candidate function not viable: no known conversion from 'float' to 'int &' for 1st argument}}
-    //   since-cxx11-note@#dr221-S {{candidate function (the implicit move assignment operator) not viable: no known conversion from 'float' to 'A' for 1st argument}}
-    //   expected-note@#dr221-S {{candidate function (the implicit copy assignment operator) not viable: no known conversion from 'float' to 'const A' for 1st argument}}
+    //   expected-note@#cwg221-S-copy-assign {{candidate function not viable: no known conversion from 'float' to 'int &' for 1st argument}}
+    //   since-cxx11-note@#cwg221-S {{candidate function (the implicit move assignment operator) not viable: no known conversion from 'float' to 'A' for 1st argument}}
+    //   expected-note@#cwg221-S {{candidate function (the implicit copy assignment operator) not viable: no known conversion from 'float' to 'const A' for 1st argument}}
     a += f;
   }
 }
 
-namespace dr222 { // dr222: dup 637
+namespace cwg222 { // cwg222: dup 637
   void f(int a, int b, int c, int *x) {
 #pragma clang diagnostic push
 #pragma clang diagnostic warning "-Wunsequenced"
@@ -276,9 +276,9 @@ namespace dr222 { // dr222: dup 637
   }
 }
 
-// dr223: na
+// cwg223: na
 
-namespace dr224 { // dr224: 16
+namespace cwg224 { // cwg224: 16
   namespace example1 {
     template <class T> class A {
       typedef int type;
@@ -286,7 +286,7 @@ namespace dr224 { // dr224: 16
       A<T>::type b;
       A<T*>::type c;
       // cxx98-17-error@-1 {{missing 'typename' prior to dependent type name A<T *>::type; implicit 'typename' is a C++20 extension}}
-      ::dr224::example1::A<T>::type d;
+      ::cwg224::example1::A<T>::type d;
 
       class B {
         typedef int type;
@@ -295,7 +295,7 @@ namespace dr224 { // dr224: 16
         A<T>::type b;
         A<T*>::type c;
         // cxx98-17-error@-1 {{missing 'typename' prior to dependent type name A<T *>::type; implicit 'typename' is a C++20 extension}}
-        ::dr224::example1::A<T>::type d;
+        ::cwg224::example1::A<T>::type d;
 
         B::type e;
         A<T>::B::type f;
@@ -348,15 +348,15 @@ namespace dr224 { // dr224: 16
   }
 }
 
-// dr225: yes
-template<typename T> void dr225_f(T t) { dr225_g(t); }
-// expected-error@-1 {{call to function 'dr225_g' that is neither visible in the template definition nor found by argument-dependent lookup}}
-//   expected-note@#dr225-f {{in instantiation of function template specialization 'dr225_f<int>' requested here}}
-//   expected-note@#dr225-g {{'dr225_g' should be declared prior to the call site}}
-void dr225_g(int); // #dr225-g
-template void dr225_f(int); // #dr225-f
+// cwg225: yes
+template<typename T> void cwg225_f(T t) { cwg225_g(t); }
+// expected-error@-1 {{call to function 'cwg225_g' that is neither visible in the template definition nor found by argument-dependent lookup}}
+//   expected-note@#cwg225-f {{in instantiation of function template specialization 'cwg225_f<int>' requested here}}
+//   expected-note@#cwg225-g {{'cwg225_g' should be declared prior to the call site}}
+void cwg225_g(int); // #cwg225-g
+template void cwg225_f(int); // #cwg225-f
 
-namespace dr226 { // dr226: no
+namespace cwg226 { // cwg226: no
   // FIXME: This appears to be wrong: default arguments for function templates
   // are listed as a defect (in c++98) not an extension. EDG accepts them in
   // strict c++98 mode.
@@ -406,14 +406,14 @@ namespace dr226 { // dr226: no
   int x = foo(0, 0);
 }
 
-void dr227(bool b) { // dr227: yes
+void cwg227(bool b) { // cwg227: yes
   if (b)
     int n;
   else
     int n;
 }
 
-namespace dr228 { // dr228: yes
+namespace cwg228 { // cwg228: yes
   template <class T> struct X {
     void f();
   };
@@ -422,50 +422,50 @@ namespace dr228 { // dr228: yes
   };
 }
 
-namespace dr229 { // dr229: 2.9
+namespace cwg229 { // cwg229: 2.9
   template<typename T> void f();
   template<typename T> void f<T*>() {}
   // expected-error@-1 {{function template partial specialization is not allowed}}
   template<> void f<int>() {}
 }
 
-namespace dr230 { // dr230: 3.0
+namespace cwg230 { // cwg230: 3.0
   struct S {
     S() { f(); }
-    // expected-warning@-1 {{call to pure virtual member function 'f' has undefined behavior; overrides of 'f' in subclasses are not available in the constructor of 'dr230::S'}}
-    //   expected-note@#dr230-f {{'f' declared here}}
-    virtual void f() = 0; // #dr230-f
+    // expected-warning@-1 {{call to pure virtual member function 'f' has undefined behavior; overrides of 'f' in subclasses are not available in the constructor of 'cwg230::S'}}
+    //   expected-note@#cwg230-f {{'f' declared here}}
+    virtual void f() = 0; // #cwg230-f
   };
 }
 
-namespace dr231 { // dr231: yes
+namespace cwg231 { // cwg231: yes
   namespace outer {
     namespace inner {
-      int i; // #dr231-i
+      int i; // #cwg231-i
     }
     void f() { using namespace inner; }
     int j = i;
     // expected-error@-1 {{use of undeclared identifier 'i'; did you mean 'inner::i'?}}
-    //   expected-note@#dr231-i {{'inner::i' declared here}}
+    //   expected-note@#cwg231-i {{'inner::i' declared here}}
   }
 }
 
-// dr234: na
-// dr235: na
+// cwg234: na
+// cwg235: na
 
-namespace dr236 { // dr236: 3.2
+namespace cwg236 { // cwg236: 3.2
   void *p = int();
   // cxx98-warning@-1 {{expression which evaluates to zero treated as a null pointer constant of type 'void *'}}
   // since-cxx11-error@-2 {{cannot initialize a variable of type 'void *' with an rvalue of type 'int'}}
 }
 
-namespace dr237 { // dr237: dup 470
+namespace cwg237 { // cwg237: dup 470
   template<typename T> struct A { void f() { T::error; } };
   template<typename T> struct B : A<T> {};
   template struct B<int>; // ok
 }
 
-namespace dr239 { // dr239: yes
+namespace cwg239 { // cwg239: yes
   namespace NS {
     class T {};
     void f(T);
@@ -481,64 +481,64 @@ namespace dr239 { // dr239: yes
   }
 }
 
-// dr240: dup 616
+// cwg240: dup 616
 
-namespace dr241 { // dr241: yes
+namespace cwg241 { // cwg241: yes
   namespace A {
     struct B {};
-    template <int X> void f(); // #dr241-A-f
+    template <int X> void f(); // #cwg241-A-f
     template <int X> void g(B);
   }
   namespace C {
-    template <class T> void f(T t); // #dr241-C-f
-    template <class T> void g(T t); // #dr241-C-g
+    template <class T> void f(T t); // #cwg241-C-f
+    template <class T> void g(T t); // #cwg241-C-g
   }
   void h(A::B b) {
     f<3>(b);
     // expected-error@-1 {{no matching function for call to 'f'}}
-    //   expected-note@#dr241-A-f {{candidate function template not viable: requires 0 arguments, but 1 was provided}}
+    //   expected-note@#cwg241-A-f {{candidate function template not viable: requires 0 arguments, but 1 was provided}}
     // cxx98-17-error@-3 {{use of function template name with no prior declaration in function call with explicit template arguments is a C++20 extension}}
     g<3>(b);
     // cxx98-17-error@-1 {{use of function template name with no prior declaration in function call with explicit template arguments is a C++20 extension}}
     A::f<3>(b);
     // expected-error@-1 {{no matching function for call to 'f'}}
-    //   expected-note@#dr241-A-f {{candidate function template not viable: requires 0 arguments, but 1 was provided}}
+    //   expected-note@#cwg241-A-f {{candidate function template not viable: requires 0 arguments, but 1 was provided}}
     A::g<3>(b);
     C::f<3>(b);
     // expected-error@-1 {{no matching function for call to 'f'}}
-    //   expected-note@#dr241-C-f {{candidate template ignored: invalid explicitly-specified argument for template parameter 'T'}}
+    //   expected-note@#cwg241-C-f {{candidate template ignored: invalid explicitly-specified argument for template parameter 'T'}}
     C::g<3>(b);
     // expected-error@-1 {{no matching function for call to 'g'}}
-    //   expected-note@#dr241-C-g {{candidate template ignored: invalid explicitly-specified argument for template parameter 'T'}}
+    //   expected-note@#cwg241-C-g {{candidate template ignored: invalid explicitly-specified argument for template parameter 'T'}}
     using C::f;
     using C::g;
     f<3>(b);
     // expected-error@-1 {{no matching function for call to 'f'}}
-    //   expected-note@#dr241-C-f {{candidate template ignored: invalid explicitly-specified argument for template parameter 'T'}}
-    //   expected-note@#dr241-A-f {{candidate function template not viable: requires 0 arguments, but 1 was provided}}
+    //   expected-note@#cwg241-C-f {{candidate template ignored: invalid explicitly-specified argument for template parameter 'T'}}
+    //   expected-note@#cwg241-A-f {{candidate function template not viable: requires 0 arguments, but 1 was provided}}
     g<3>(b);
   }
 }
 
-namespace dr243 { // dr243: yes
+namespace cwg243 { // cwg243: yes
   struct B;
   struct A {
-    A(B); // #dr243-A
+    A(B); // #cwg243-A
   };
   struct B {
-    operator A() = delete; // #dr243-B
+    operator A() = delete; // #cwg243-B
     // cxx98-error@-1 {{deleted function definitions are a C++11 extension}}
   } b;
   A a1(b);
   A a2 = b;
   // expected-error@-1 {{conversion from 'struct B' to 'A' is ambiguous}}
-  //   expected-note@#dr243-A {{candidate constructor}}
-  //   expected-note@#dr243-B {{candidate function has been explicitly deleted}}
+  //   expected-note@#cwg243-A {{candidate constructor}}
+  //   expected-note@#cwg243-B {{candidate function has been explicitly deleted}}
 }
 
-namespace dr244 { // dr244: 11
-                  // NB: this test is reused by dr399
-  struct B {}; // #dr244-B
+namespace cwg244 { // cwg244: 11
+                  // NB: this test is reused by cwg399
+  struct B {}; // #cwg244-B
   struct D : B {};
 
   D D_object;
@@ -547,18 +547,18 @@ namespace dr244 { // dr244: 11
 
   void f() {
     D_object.~B();
-    // expected-error@-1 {{destructor type 'dr244::B' in object destruction expression does not match the type 'D' of the object being destroyed}}
-    //   expected-note@#dr244-B {{type 'dr244::B' found by destructor name lookup}}
+    // expected-error@-1 {{destructor type 'cwg244::B' in object destruction expression does not match the type 'D' of the object being destroyed}}
+    //   expected-note@#cwg244-B {{type 'cwg244::B' found by destructor name lookup}}
     D_object.B::~B();
     D_object.D::~B(); // FIXME: Missing diagnostic for this.
     B_ptr->~B();
     B_ptr->~B_alias();
     B_ptr->B_alias::~B();
     B_ptr->B_alias::~B_alias();
-    B_ptr->dr244::~B();
-    // expected-error@-1 {{qualified member access refers to a member in namespace 'dr244'}}
-    B_ptr->dr244::~B_alias();
-    // expected-error@-1 {{qualified member access refers to a member in namespace 'dr244'}}
+    B_ptr->cwg244::~B();
+    // expected-error@-1 {{qualified member access refers to a member in namespace 'cwg244'}}
+    B_ptr->cwg244::~B_alias();
+    // expected-error@-1 {{qualified member access refers to a member in namespace 'cwg244'}}
   }
 
   template<typename T, typename U>
@@ -570,10 +570,10 @@ namespace dr244 { // dr244: 11
     B_ptr->~B_alias();
     B_ptr->B_alias::~B();
     B_ptr->B_alias::~B_alias();
-    B_ptr->dr244::~B();
-    // expected-error@-1 {{'dr244' does not refer to a type name in pseudo-destructor expression; expected the name of type 'T'}}
-    B_ptr->dr244::~B_alias();
-    // expected-error@-1 {{'dr244' does not refer to a type name in pseudo-destructor expression; expected the name of type 'T'}}
+    B_ptr->cwg244::~B();
+    // expected-error@-1 {{'cwg244' does not refer to a type name in pseudo-destructor expression; expected the name of type 'T'}}
+    B_ptr->cwg244::~B_alias();
+    // expected-error@-1 {{'cwg244' does not refer to a type name in pseudo-destructor expression; expected the name of type 'T'}}
   }
   template void f<B, D>(B*, D);
 
@@ -582,7 +582,7 @@ namespace dr244 { // dr244: 11
     typedef E<int> F;
   }
   void g(N::F f) {
-    typedef N::F G; // #dr244-G
+    typedef N::F G; // #cwg244-G
     f.~G();
     f.G::~E();
     // expected-error@-1 {{ISO C++ requires the name after '::~' to be found in the same scope as the name before '::~'}}
@@ -601,7 +601,7 @@ namespace dr244 { // dr244: 11
     // Rejecting this seems correct, but most compilers accept, so we do also.
     f.N::F::~G();
     // expected-error@-1 {{qualified destructor name only found in lexical scope; omit the qualifier to find this type name by unqualified lookup}}
-    //   expected-note@#dr244-G {{type 'G' (aka 'E<int>') found by destructor name lookup}}
+    //   expected-note@#cwg244-G {{type 'G' (aka 'E<int>') found by destructor name lookup}}
   }
 
   // Bizarrely, compilers perform lookup in the scope for qualified destructor
@@ -613,46 +613,46 @@ namespace dr244 { // dr244: 11
     }
     template <typename U> void f(typename N::S<U>::Inner *p) {
       typedef typename N::S<U>::Inner T;
-      p->::dr244::QualifiedLookupInScope::N::S<U>::Inner::~T();
-      // expected-error@-1 {{no type named 'T' in 'dr244::QualifiedLookupInScope::N::S<int>'}}
-      //   expected-note@#dr244-f {{in instantiation of function template specialization 'dr244::QualifiedLookupInScope::f<int>' requested here}}
+      p->::cwg244::QualifiedLookupInScope::N::S<U>::Inner::~T();
+      // expected-error@-1 {{no type named 'T' in 'cwg244::QualifiedLookupInScope::N::S<int>'}}
+      //   expected-note@#cwg244-f {{in instantiation of function template specialization 'cwg244::QualifiedLookupInScope::f<int>' requested here}}
     }
-    template void f<int>(N::S<int>::Inner *); // #dr244-f
+    template void f<int>(N::S<int>::Inner *); // #cwg244-f
 
     template <typename U> void g(U *p) {
       typedef U T;
       p->T::~T();
       p->U::~T();
-      p->::dr244::QualifiedLookupInScope::N::S<int>::Inner::~T();
+      p->::cwg244::QualifiedLookupInScope::N::S<int>::Inner::~T();
       // expected-error@-1 {{'T' does not refer to a type name in pseudo-destructor expression; expected the name of type 'U'}}
     }
     template void g(N::S<int>::Inner *);
   }
 }
 
-namespace dr245 { // dr245: yes
+namespace cwg245 { // cwg245: yes
   struct S {
-    enum E {}; // #dr245-E
+    enum E {}; // #cwg245-E
     class E *p;
     // expected-error@-1 {{use of 'E' with tag type that does not match previous declaration}}
-    //   expected-note@#dr245-E {{previous use is here}}
+    //   expected-note@#cwg245-E {{previous use is here}}
   };
 }
 
-namespace dr246 { // dr246: 3.2
+namespace cwg246 { // cwg246: 3.2
   struct S {
-    S() try { // #dr246-try
+    S() try { // #cwg246-try
       throw 0;
 X: ;
     } catch (int) {
       goto X;
       // expected-error@-1 {{cannot jump from this goto statement to its label}}
-      //   expected-note@#dr246-try {{jump bypasses initialization of try block}}
+      //   expected-note@#cwg246-try {{jump bypasses initialization of try block}}
     }
   };
 }
 
-namespace dr247 { // dr247: yes
+namespace cwg247 { // cwg247: yes
   struct A {};
   struct B : A {
     void f();
@@ -678,16 +678,16 @@ namespace dr247 { // dr247: yes
   void (F::*i)() = &F::f;
 }
 
-namespace dr248 { // dr248: sup P1949
+namespace cwg248 { // cwg248: sup P1949
   int \u040d\u040e = 0;
 }
 
-namespace dr249 { // dr249: yes
+namespace cwg249 { // cwg249: yes
   template<typename T> struct X { void f(); };
   template<typename T> void X<T>::f() {}
 }
 
-namespace dr250 { // dr250: yes
+namespace cwg250 { // cwg250: yes
   typedef void (*FPtr)(double x[]);
 
   template<int I> void f(double x[]);
@@ -698,49 +698,49 @@ namespace dr250 { // dr250: yes
   FPtr gp = &g<>;
 }
 
-namespace dr252 { // dr252: 3.1
+namespace cwg252 { // cwg252: 3.1
   struct A {
-    void operator delete(void*); // #dr252-A
+    void operator delete(void*); // #cwg252-A
   };
   struct B {
-    void operator delete(void*); // #dr252-B
+    void operator delete(void*); // #cwg252-B
   };
   struct C : A, B {
     virtual ~C();
   };
   C::~C() {}
   // expected-error@-1 {{member 'operator delete' found in multiple base classes of different types}}
-  //   expected-note@#dr252-A {{member found by ambiguous name lookup}}
-  //   expected-note@#dr252-B {{member found by ambiguous name lookup}}
+  //   expected-note@#cwg252-A {{member found by ambiguous name lookup}}
+  //   expected-note@#cwg252-B {{member found by ambiguous name lookup}}
 
   struct D {
-    void operator delete(void*, int); // #dr252-D
+    void operator delete(void*, int); // #cwg252-D
     virtual ~D();
   };
   D::~D() {}
   // expected-error@-1 {{no suitable member 'operator delete' in 'D'}}
-  //   expected-note@#dr252-D {{member 'operator delete' declared here}}
+  //   expected-note@#cwg252-D {{member 'operator delete' declared here}}
 
   struct E {
     void operator delete(void*, int);
-    void operator delete(void*) = delete; // #dr252-E
+    void operator delete(void*) = delete; // #cwg252-E
     // cxx98-error@-1 {{deleted function definitions are a C++11 extension}} 
     virtual ~E();
   };
   E::~E() {}
   // expected-error@-1 {{attempt to use a deleted function}}
-  //   expected-note@#dr252-E {{'operator delete' has been explicitly marked deleted here}}
+  //   expected-note@#cwg252-E {{'operator delete' has been explicitly marked deleted here}}
 
   struct F {
     // If both functions are available, the first one is a placement delete.
     void operator delete(void*, size_t);
-    void operator delete(void*) = delete; // #dr252-F
+    void operator delete(void*) = delete; // #cwg252-F
     // cxx98-error@-1 {{deleted function definitions are a C++11 extension}}
     virtual ~F();
   };
   F::~F() {}
   // expected-error@-1 {{attempt to use a deleted function}}
-  //   expected-note@#dr252-F {{'operator delete' has been explicitly marked deleted here}}
+  //   expected-note@#cwg252-F {{'operator delete' has been explicitly marked deleted here}}
 
   struct G {
     void operator delete(void*, size_t);
@@ -749,33 +749,33 @@ namespace dr252 { // dr252: 3.1
   G::~G() {}
 }
 
-namespace dr254 { // dr254: 2.9
+namespace cwg254 { // cwg254: 2.9
   template<typename T> struct A {
     typedef typename T::type type; // ok even if this is a typedef-name, because
                                    // it's not an elaborated-type-specifier
     typedef struct T::type foo;
     // expected-error@-1 {{typedef 'type' cannot be referenced with a struct specifier}}
-    //   expected-note@#dr254-instantiation {{in instantiation of template class 'dr254::A<dr254::C>' requested here}}
-    //   expected-note@#dr254-C {{declared here}}
+    //   expected-note@#cwg254-instantiation {{in instantiation of template class 'cwg254::A<cwg254::C>' requested here}}
+    //   expected-note@#cwg254-C {{declared here}}
   };
   struct B { struct type {}; };
-  struct C { typedef struct {} type; }; // #dr254-C
+  struct C { typedef struct {} type; }; // #cwg254-C
   A<B>::type n;
-  A<C>::type n; // #dr254-instantiation
+  A<C>::type n; // #cwg254-instantiation
 }
 
-namespace dr255 { // dr255: yes
+namespace cwg255 { // cwg255: yes
 struct S {
   void operator delete(void *){};
   void operator delete(void *, int){};
 };
 void f(S *p) { delete p; }
-} // namespace dr255
+} // namespace cwg255
 
-// dr256: dup 624
+// cwg256: dup 624
 
-namespace dr257 { // dr257: 3.4
-  struct A { A(int); }; // #dr257-A
+namespace cwg257 { // cwg257: 3.4
+  struct A { A(int); }; // #cwg257-A
   struct B : virtual A {
     B() {}
     virtual void f() = 0;
@@ -785,13 +785,13 @@ namespace dr257 { // dr257: 3.4
   };
   struct D : B {
     D() {}
-    // expected-error@-1 {{constructor for 'dr257::D' must explicitly initialize the base class 'A' which does not have a default constructor}}
-    //   expected-note@#dr257-A {{'dr257::A' declared here}}
+    // expected-error@-1 {{constructor for 'cwg257::D' must explicitly initialize the base class 'A' which does not have a default constructor}}
+    //   expected-note@#cwg257-A {{'cwg257::A' declared here}}
     void f();
   };
 }
 
-namespace dr258 { // dr258: 2.8
+namespace cwg258 { // cwg258: 2.8
   struct A {
     void f(const int);
     template<typename> void g(int);
@@ -802,13 +802,13 @@ namespace dr258 { // dr258: 2.8
     using A::g;
     using A::h;
     int &f(int);
-    template<int> int &g(int); // #dr258-B-g
+    template<int> int &g(int); // #cwg258-B-g
     int &h();
   } b;
   int &w = b.f(0);
   int &x = b.g<int>(0);
   // expected-error@-1 {{no matching member function for call to 'g'}}
-  //   expected-note@#dr258-B-g {{candidate template ignored: invalid explicitly-specified argument for 1st template parameter}}
+  //   expected-note@#cwg258-B-g {{candidate template ignored: invalid explicitly-specified argument for 1st template parameter}}
   int &y = b.h();
   float &z = const_cast<const B&>(b).h();
 
@@ -820,53 +820,53 @@ namespace dr258 { // dr258: 2.8
   } d;
 
   struct E {
-    virtual void f() = 0; // #dr258-E-f
+    virtual void f() = 0; // #cwg258-E-f
   };
   struct F : E {
     void f() const {}
   } f;
   // expected-error@-1 {{variable type 'struct F' is an abstract class}}
-  //   expected-note@#dr258-E-f {{unimplemented pure virtual method 'f' in 'F'}}
+  //   expected-note@#cwg258-E-f {{unimplemented pure virtual method 'f' in 'F'}}
 }
 
-namespace dr259 { // dr259: 4
+namespace cwg259 { // cwg259: 4
   template<typename T> struct A {};
-  template struct A<int>; // #dr259-A-int 
+  template struct A<int>; // #cwg259-A-int 
   template struct A<int>;
   // expected-error@-1 {{duplicate explicit instantiation of 'A<int>'}}
-  //   expected-note@#dr259-A-int {{previous explicit instantiation is here}}
+  //   expected-note@#cwg259-A-int {{previous explicit instantiation is here}}
 
-  template<> struct A<float>; // #dr259-A-float
+  template<> struct A<float>; // #cwg259-A-float
   template struct A<float>;
   // expected-warning@-1 {{explicit instantiation of 'A<float>' that occurs after an explicit specialization has no effect}}
-  //   expected-note@#dr259-A-float {{previous template specialization is here}}
+  //   expected-note@#cwg259-A-float {{previous template specialization is here}}
 
-  template struct A<char>; // #dr259-A-char
+  template struct A<char>; // #cwg259-A-char
   template<> struct A<char>;
-  // expected-error@-1 {{explicit specialization of 'dr259::A<char>' after instantiation}}
-  //   expected-note@#dr259-A-char {{explicit instantiation first required here}}
+  // expected-error@-1 {{explicit specialization of 'cwg259::A<char>' after instantiation}}
+  //   expected-note@#cwg259-A-char {{explicit instantiation first required here}}
 
   template<> struct A<double>;
   template<> struct A<double>;
-  template<> struct A<double> {}; // #dr259-A-double
+  template<> struct A<double> {}; // #cwg259-A-double
   template<> struct A<double> {};
   // expected-error@-1 {{redefinition of 'A<double>'}}
-  //   expected-note@#dr259-A-double {{previous definition is here}}
+  //   expected-note@#cwg259-A-double {{previous definition is here}}
 
-  template<typename T> struct B; // #dr259-B
+  template<typename T> struct B; // #cwg259-B
   template struct B<int>;
-  // expected-error@-1 {{explicit instantiation of undefined template 'dr259::B<int>'}}
-  //   expected-note@#dr259-B {{template is declared here}}
+  // expected-error@-1 {{explicit instantiation of undefined template 'cwg259::B<int>'}}
+  //   expected-note@#cwg259-B {{template is declared here}}
 
-  template<> struct B<float>; // #dr259-B-float
+  template<> struct B<float>; // #cwg259-B-float
   template struct B<float>;
   // expected-warning@-1 {{explicit instantiation of 'B<float>' that occurs after an explicit specialization has no effect}}
-  //   expected-note@#dr259-B-float {{previous template specialization is here}}
+  //   expected-note@#cwg259-B-float {{previous template specialization is here}}
 }
 
-// FIXME: When dr260 is resolved, also add tests for DR507.
+// FIXME: When cwg260 is resolved, also add tests for CWG507.
 
-namespace dr261 { // dr261: no
+namespace cwg261 { // cwg261: no
 #pragma clang diagnostic push
 #pragma clang diagnostic warning "-Wused-but-marked-unused"
 
@@ -901,14 +901,14 @@ namespace dr261 { // dr261: no
 #pragma clang diagnostic pop
 }
 
-namespace dr262 { // dr262: yes
+namespace cwg262 { // cwg262: yes
   int f(int = 0, ...);
   int k = f();
   int l = f(0);
   int m = f(0, 0);
 }
 
-namespace dr263 { // dr263: 3.3
+namespace cwg263 { // cwg263: 3.3
   struct X {};
   struct Y {
 #if __cplusplus < 201103L
@@ -928,12 +928,12 @@ namespace dr263 { // dr263: 3.3
   };
 }
 
-// dr265: dup 353
-// dr266: na
-// dr269: na
-// dr270: na
+// cwg265: dup 353
+// cwg266: na
+// cwg269: na
+// cwg270: na
 
-namespace dr272 { // dr272: yes
+namespace cwg272 { // cwg272: yes
   struct X {
     void f() {
       this->~X();
@@ -946,7 +946,7 @@ namespace dr272 { // dr272: yes
 
 #include <stdarg.h>
 #include <stddef.h>
-namespace dr273 { // dr273: yes
+namespace cwg273 { // cwg273: yes
   struct A {
     int n;
   };
@@ -959,12 +959,12 @@ namespace dr273 { // dr273: yes
   }
 }
 
-// dr274: na
+// cwg274: na
 
-namespace dr275 { // dr275: no
+namespace cwg275 { // cwg275: no
   namespace N {
-    template <class T> void f(T) {} // #dr275-N-f
-    template <class T> void g(T) {} // #dr275-N-g
+    template <class T> void f(T) {} // #cwg275-N-f
+    template <class T> void g(T) {} // #cwg275-N-g
     template <> void f(int);
     template <> void f(char);
     template <> void f(double);
@@ -980,19 +980,19 @@ namespace dr275 { // dr275: no
     template <> void g(char) {}
     // FIXME: this should be rejected in c++98 too
     template void f(long);
-    // since-cxx11-error@-1 {{explicit instantiation of 'dr275::N::f' must occur in namespace 'N'}}
-    //   since-cxx11-note@#dr275-N-f {{explicit instantiation refers here}}
+    // since-cxx11-error@-1 {{explicit instantiation of 'cwg275::N::f' must occur in namespace 'N'}}
+    //   since-cxx11-note@#cwg275-N-f {{explicit instantiation refers here}}
     // FIXME: this should be rejected in c++98 too
     template void N::f(unsigned long);
     // since-cxx11-error@-1 {{explicit instantiation of 'f' not in a namespace enclosing 'N'}}
-    //   since-cxx11-note@#dr275-N-f {{explicit instantiation refers here}}
+    //   since-cxx11-note@#cwg275-N-f {{explicit instantiation refers here}}
     template void h(long);
     // expected-error@-1 {{explicit instantiation of 'h' does not refer to a function template, variable template, member function, member class, or static data member}}
     template <> void f(double) {}
     // expected-error@-1 {{no function template matches function template specialization 'f'}}
   }
 
-  template <class T> void g(T) {} // #dr275-g 
+  template <class T> void g(T) {} // #cwg275-g 
 
   template <> void N::f(char) {}
   template <> void f(int) {}
@@ -1000,87 +1000,87 @@ namespace dr275 { // dr275: no
 
   // FIXME: this should be rejected in c++98 too
   template void f(short);
-  // since-cxx11-error@-1 {{explicit instantiation of 'dr275::N::f' must occur in namespace 'N'}}
-  //   since-cxx11-note@#dr275-N-f {{explicit instantiation refers here}}
+  // since-cxx11-error@-1 {{explicit instantiation of 'cwg275::N::f' must occur in namespace 'N'}}
+  //   since-cxx11-note@#cwg275-N-f {{explicit instantiation refers here}}
   template void N::f(unsigned short);
 
   // FIXME: this should probably be valid. the wording from the issue
   // doesn't clarify this, but it follows from the usual rules.
   template void g(int);
   // expected-error@-1 {{partial ordering for explicit instantiation of 'g' is ambiguous}}
-  //   expected-note@#dr275-g {{explicit instantiation candidate function 'dr275::g<int>' template here [with T = int]}}
-  //   expected-note@#dr275-N-g {{explicit instantiation candidate function 'dr275::N::g<int>' template here [with T = int]}}
+  //   expected-note@#cwg275-g {{explicit instantiation candidate function 'cwg275::g<int>' template here [with T = int]}}
+  //   expected-note@#cwg275-N-g {{explicit instantiation candidate function 'cwg275::N::g<int>' template here [with T = int]}}
 
   // FIXME: likewise, this should also be valid.
-  template<typename T> void f(T) {} // #dr275-f
+  template<typename T> void f(T) {} // #cwg275-f
   template void f(short);
   // expected-error@-1 {{partial ordering for explicit instantiation of 'f' is ambiguous}}
-  //   expected-note@#dr275-f {{explicit instantiation candidate function 'dr275::f<short>' template here [with T = short]}}
-  //   expected-note@#dr275-N-f {{explicit instantiation candidate function 'dr275::N::f<short>' template here [with T = short]}}
+  //   expected-note@#cwg275-f {{explicit instantiation candidate function 'cwg275::f<short>' template here [with T = short]}}
+  //   expected-note@#cwg275-N-f {{explicit instantiation candidate function 'cwg275::N::f<short>' template here [with T = short]}}
 }
 
-// dr276: na
+// cwg276: na
 
-namespace dr277 { // dr277: 3.1
+namespace cwg277 { // cwg277: 3.1
   typedef int *intp;
   int *p = intp();
   int a[fold(intp() ? -1 : 1)];
 }
 
-namespace dr280 { // dr280: 2.9
+namespace cwg280 { // cwg280: 2.9
   typedef void f0();
   typedef void f1(int);
   typedef void f2(int, int);
   typedef void f3(int, int, int);
   struct A {
-    operator f1*(); // #dr280-A-f1
+    operator f1*(); // #cwg280-A-f1
     operator f2*();
   };
   struct B {
-    operator f0*(); // #dr280-B-f0
+    operator f0*(); // #cwg280-B-f0
   private:
-    operator f3*(); // #dr280-B-f3
+    operator f3*(); // #cwg280-B-f3
   };
   struct C {
-    operator f0*(); // #dr280-C-f0
-    operator f1*(); // #dr280-C-f1
-    operator f2*(); // #dr280-C-f2
-    operator f3*(); // #dr280-C-f3
+    operator f0*(); // #cwg280-C-f0
+    operator f1*(); // #cwg280-C-f1
+    operator f2*(); // #cwg280-C-f2
+    operator f3*(); // #cwg280-C-f3
   };
-  struct D : private A, B { // #dr280-D
-    operator f2*(); // #dr280-D-f2
+  struct D : private A, B { // #cwg280-D
+    operator f2*(); // #cwg280-D-f2
   } d;
   struct E : C, D {} e;
   void g() {
     d(); // ok, public
     d(0);
-    // expected-error@-1 {{'operator void (*)(int)' is a private member of 'dr280::A'}}
-    //   expected-note@#dr280-D {{constrained by private inheritance here}}
-    //   expected-note@#dr280-A-f1 {{member is declared here}}
+    // expected-error@-1 {{'operator void (*)(int)' is a private member of 'cwg280::A'}}
+    //   expected-note@#cwg280-D {{constrained by private inheritance here}}
+    //   expected-note@#cwg280-A-f1 {{member is declared here}}
     d(0, 0); // ok, suppressed by member in D
     d(0, 0, 0);
-    // expected-error@-1 {{'operator void (*)(int, int, int)' is a private member of 'dr280::B'}}
-    //   expected-note@#dr280-B-f3 {{declared private here}}
+    // expected-error@-1 {{'operator void (*)(int, int, int)' is a private member of 'cwg280::B'}}
+    //   expected-note@#cwg280-B-f3 {{declared private here}}
     e();
     // expected-error@-1 {{call to object of type 'struct E' is ambiguous}}
-    //   expected-note@#dr280-B-f0 {{conversion candidate of type 'void (*)()'}}
-    //   expected-note@#dr280-C-f0 {{conversion candidate of type 'void (*)()'}}
+    //   expected-note@#cwg280-B-f0 {{conversion candidate of type 'void (*)()'}}
+    //   expected-note@#cwg280-C-f0 {{conversion candidate of type 'void (*)()'}}
     e(0);
     // expected-error@-1 {{call to object of type 'struct E' is ambiguous}}
-    //   expected-note@#dr280-A-f1 {{conversion candidate of type 'void (*)(int)'}}
-    //   expected-note@#dr280-C-f1 {{conversion candidate of type 'void (*)(int)'}}
+    //   expected-note@#cwg280-A-f1 {{conversion candidate of type 'void (*)(int)'}}
+    //   expected-note@#cwg280-C-f1 {{conversion candidate of type 'void (*)(int)'}}
     e(0, 0);
     // expected-error@-1 {{call to object of type 'struct E' is ambiguous}}
-    //   expected-note@#dr280-C-f2 {{conversion candidate of type 'void (*)(int, int)'}}
-    //   expected-note@#dr280-D-f2 {{conversion candidate of type 'void (*)(int, int)'}}
+    //   expected-note@#cwg280-C-f2 {{conversion candidate of type 'void (*)(int, int)'}}
+    //   expected-note@#cwg280-D-f2 {{conversion candidate of type 'void (*)(int, int)'}}
     e(0, 0, 0);
     // expected-error@-1 {{call to object of type 'struct E' is ambiguous}}
-    //   expected-note@#dr280-B-f3 {{conversion candidate of type 'void (*)(int, int, int)'}}
-    //   expected-note@#dr280-C-f3 {{conversion candidate of type 'void (*)(int, int, int)'}}
+    //   expected-note@#cwg280-B-f3 {{conversion candidate of type 'void (*)(int, int, int)'}}
+    //   expected-note@#cwg280-C-f3 {{conversion candidate of type 'void (*)(int, int, int)'}}
   }
 }
 
-namespace dr281 { // dr281: no
+namespace cwg281 { // cwg281: no
   void a();
   inline void b();
 
@@ -1097,19 +1097,19 @@ namespace dr281 { // dr281: no
   };
 }
 
-namespace dr283 { // dr283: yes
-  template<typename T> // #dr283-template
+namespace cwg283 { // cwg283: yes
+  template<typename T> // #cwg283-template
   struct S {
     friend class T;
     // expected-error@-1 {{declaration of 'T' shadows template parameter}}
-    //   expected-note@#dr283-template {{template parameter is declared here}}
+    //   expected-note@#cwg283-template {{template parameter is declared here}}
     class T;
     // expected-error@-1 {{declaration of 'T' shadows template parameter}}
-    //   expected-note@#dr283-template {{template parameter is declared here}}
+    //   expected-note@#cwg283-template {{template parameter is declared here}}
   };
 }
 
-namespace dr284 { // dr284: no
+namespace cwg284 { // cwg284: no
   namespace A {
     struct X;
     enum Y {};
@@ -1122,11 +1122,11 @@ namespace dr284 { // dr284: no
     using A::Z;
   }
   struct B::V {};
-  // expected-error@-1 {{no struct named 'V' in namespace 'dr284::B'}}
+  // expected-error@-1 {{no struct named 'V' in namespace 'cwg284::B'}}
   struct B::W {};
   struct B::X {}; // FIXME: ill-formed
-  enum B::Y e; // ok per dr417
-  class B::Z z; // ok per dr417
+  enum B::Y e; // ok per cwg417
+  class B::Z z; // ok per cwg417
 
   struct C {
     struct X;
@@ -1140,26 +1140,26 @@ namespace dr284 { // dr284: no
     using C::Z;
   };
   struct D::V {};
-  // expected-error@-1 {{no struct named 'V' in 'dr284::D'}}
+  // expected-error@-1 {{no struct named 'V' in 'cwg284::D'}}
   struct D::W {};
   struct D::X {}; // FIXME: ill-formed
-  enum D::Y e2; // ok per dr417
-  class D::Z z2; // ok per dr417
+  enum D::Y e2; // ok per cwg417
+  class D::Z z2; // ok per cwg417
 }
 
-namespace dr285 { // dr285: yes
-  template<typename T> void f(T, int); // #dr285-f-T-int
-  template<typename T> void f(int, T); // #dr285-f-int-T
+namespace cwg285 { // cwg285: yes
+  template<typename T> void f(T, int); // #cwg285-f-T-int
+  template<typename T> void f(int, T); // #cwg285-f-int-T
   template<> void f<int>(int, int) {}
   // expected-error@-1 {{function template specialization 'f' ambiguously refers to more than one function template; explicitly specify additional template arguments to identify a particular function template}}
-  //   expected-note@#dr285-f-int-T {{function template 'dr285::f<int>' matches specialization [with T = int]}}
-  //   expected-note@#dr285-f-T-int {{function template 'dr285::f<int>' matches specialization [with T = int]}}
+  //   expected-note@#cwg285-f-int-T {{function template 'cwg285::f<int>' matches specialization [with T = int]}}
+  //   expected-note@#cwg285-f-T-int {{function template 'cwg285::f<int>' matches specialization [with T = int]}}
 }
 
-namespace dr286 { // dr286: 2.8
+namespace cwg286 { // cwg286: 2.8
   template<class T> struct A {
     class C {
-      template<class T2> struct B {}; // #dr286-B 
+      template<class T2> struct B {}; // #cwg286-B 
     };
   };
 
@@ -1168,29 +1168,29 @@ namespace dr286 { // dr286: 2.8
   struct A<T>::C::B<T2*> { };
 
   A<short>::C::B<int*> absip;
-  // expected-error@-1 {{'B' is a private member of 'dr286::A<short>::C'}}
-  //   expected-note@#dr286-B {{implicitly declared private here}}
+  // expected-error@-1 {{'B' is a private member of 'cwg286::A<short>::C'}}
+  //   expected-note@#cwg286-B {{implicitly declared private here}}
 }
 
-// dr288: na
+// cwg288: na
 
-namespace dr289 { // dr289: yes
-  struct A; // #dr289-A
+namespace cwg289 { // cwg289: yes
+  struct A; // #cwg289-A
   struct B : A {};
   // expected-error@-1 {{base class has incomplete type}}
-  //   expected-note@#dr289-A {{forward declaration of 'dr289::A'}}
+  //   expected-note@#cwg289-A {{forward declaration of 'cwg289::A'}}
 
   template<typename T> struct C { typename T::error error; };
   // expected-error@-1 {{type 'int' cannot be used prior to '::' because it has no members}}
-  //   expected-note@#dr289-C-int {{in instantiation of template class 'dr289::C<int>' requested here}}
-  struct D : C<int> {}; // #dr289-C-int
+  //   expected-note@#cwg289-C-int {{in instantiation of template class 'cwg289::C<int>' requested here}}
+  struct D : C<int> {}; // #cwg289-C-int
 }
 
-// dr290: na
-// dr291: dup 391
-// dr292 is in dr292.cpp
+// cwg290: na
+// cwg291: dup 391
+// cwg292 is in cwg292.cpp
 
-namespace dr294 { // dr294: no
+namespace cwg294 { // cwg294: no
   void f() throw(int);
   // since-cxx17-error@-1 {{ISO C++17 does not allow dynamic exception specifications}}
   //   since-cxx17-note@-2 {{use 'noexcept(false)' instead}}
@@ -1218,7 +1218,7 @@ namespace dr294 { // dr294: no
   }
 }
 
-namespace dr295 { // dr295: 3.7
+namespace cwg295 { // cwg295: 3.7
   typedef int f();
   const f g;
   // expected-warning@-1 {{'const' qualifier on function type 'f' (aka 'int ()') has no effect}}
@@ -1237,21 +1237,21 @@ namespace dr295 { // dr295: 3.7
   // expected-warning@-1 {{'volatile' qualifier on function type 'U' (aka 'int ()') has no effect}}
 }
 
-namespace dr296 { // dr296: yes
+namespace cwg296 { // cwg296: yes
   struct A {
     static operator int() { return 0; }
     // expected-error@-1 {{conversion function must be a non-static member function}}
   };
 }
 
-namespace dr298 { // dr298: 3.1
+namespace cwg298 { // cwg298: 3.1
   struct A {
     typedef int type;
     A();
     ~A();
   };
-  typedef A B; // #dr298-B
-  typedef const A C; // #dr298-C
+  typedef A B; // #cwg298-B
+  typedef const A C; // #cwg298-C
 
   A::type i1;
   B::type i2;
@@ -1260,21 +1260,21 @@ namespace dr298 { // dr298: 3.1
   struct A a;
   struct B b;
   // expected-error@-1 {{typedef 'B' cannot be referenced with a struct specifier}}
-  //   expected-note@#dr298-B {{declared here}}
+  //   expected-note@#cwg298-B {{declared here}}
   struct C c;
   // expected-error@-1 {{typedef 'C' cannot be referenced with a struct specifier}}
-  //   expected-note@#dr298-C {{declared here}}
+  //   expected-note@#cwg298-C {{declared here}}
 
   B::B() {}
   // expected-error@-1 {{a type specifier is required for all declarations}}
   B::A() {} // ok
   C::~C() {}
-  // expected-error@-1 {{destructor cannot be declared using a typedef 'C' (aka 'const dr298::A') of the class name}}
+  // expected-error@-1 {{destructor cannot be declared using a typedef 'C' (aka 'const cwg298::A') of the class name}}
 
-  typedef struct D E; // #dr298-E
+  typedef struct D E; // #cwg298-E
   struct E {};
   // expected-error@-1 {{definition of type 'E' conflicts with typedef of the same name}}
-  //   expected-note@#dr298-E {{'E' declared here}}
+  //   expected-note@#cwg298-E {{'E' declared here}}
 
   struct F {
     ~F();
@@ -1283,22 +1283,22 @@ namespace dr298 { // dr298: 3.1
   G::~F() {} // ok
 }
 
-namespace dr299 { // dr299: 2.8 c++11
+namespace cwg299 { // cwg299: 2.8 c++11
   struct S {
     operator int();
   };
   struct T {
-    operator int(); // #dr299-int
-    operator unsigned short(); // #dr299-ushort
+    operator int(); // #cwg299-int
+    operator unsigned short(); // #cwg299-ushort
   };
   // FIXME: should this apply to c++98 mode?
   int *p = new int[S()];
   // cxx98-error@-1 {{implicit conversion from array size expression of type 'S' to integral type 'int' is a C++11 extension}}
-  int *q = new int[T()]; // #dr299-q
-  // cxx98-11-error@#dr299-q {{ambiguous conversion of array size expression of type 'T' to an integral or enumeration type}}
-  //  cxx98-11-note@#dr299-int {{conversion to integral type 'int' declared here}}
-  //  cxx98-11-note@#dr299-ushort {{conversion to integral type 'unsigned short' declared here}}
-  // since-cxx14-error-re@#dr299-q {{{{conversion from 'T' to 'unsigned (long long|long|int)' is ambiguous}}}}
-  //  since-cxx14-note@#dr299-int {{candidate function}}
-  //  since-cxx14-note@#dr299-ushort {{candidate function}}
+  int *q = new int[T()]; // #cwg299-q
+  // cxx98-11-error@#cwg299-q {{ambiguous conversion of array size expression of type 'T' to an integral or enumeration type}}
+  //  cxx98-11-note@#cwg299-int {{conversion to integral type 'int' declared here}}
+  //  cxx98-11-note@#cwg299-ushort {{conversion to integral type 'unsigned short' declared here}}
+  // since-cxx14-error-re@#cwg299-q {{{{conversion from 'T' to 'unsigned (long long|long|int)' is ambiguous}}}}
+  //  since-cxx14-note@#cwg299-int {{candidate function}}
+  //  since-cxx14-note@#cwg299-ushort {{candidate function}}
 }
diff --git a/clang/test/CXX/drs/dr3xx.cpp b/clang/test/CXX/drs/dr3xx.cpp
index d5cfebccf813..6d1c6958ac8e 100644
--- a/clang/test/CXX/drs/dr3xx.cpp
+++ b/clang/test/CXX/drs/dr3xx.cpp
@@ -5,14 +5,14 @@
 // RUN: %clang_cc1 -std=c++11 -verify=expected,cxx98-14,cxx98-17,cxx98-20,cxx11-14,since-cxx11 -triple %itanium_abi_triple %s -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++98 -verify=expected,cxx98-14,cxx98-17,cxx98-20,cxx98 -triple %itanium_abi_triple %s -fexceptions -fcxx-exceptions -pedantic-errors
 
-namespace dr300 { // dr300: yes
+namespace cwg300 { // cwg300: yes
   template<typename R, typename A> void f(R (&)(A)) {}
   int g(int);
   void h() { f(g); }
 }
 
-namespace dr301 { // dr301: 3.5
-  // see also dr38
+namespace cwg301 { // cwg301: 3.5
+  // see also cwg38
   struct S;
   template<typename T> void operator+(T, T);
   void operator-(S, S);
@@ -66,55 +66,55 @@ namespace dr301 { // dr301: 3.5
   // expected-error@-3 {{declaration does not declare anything}}
 }
 
-namespace dr302 { // dr302: 3.0
+namespace cwg302 { // cwg302: 3.0
   struct A { A(); ~A(); };
 #if __cplusplus < 201103L
   struct B {
-  // expected-error@-1 {{implicit default constructor for 'dr302::B' must explicitly initialize the const member 'n'}}
-  //   expected-note@#dr302-b {{in implicit default constructor for 'dr302::B' first required here}}
-  //   expected-note@#dr302-B-n {{declared here}}
-    const int n; // #dr302-B-n
+  // expected-error@-1 {{implicit default constructor for 'cwg302::B' must explicitly initialize the const member 'n'}}
+  //   expected-note@#cwg302-b {{in implicit default constructor for 'cwg302::B' first required here}}
+  //   expected-note@#cwg302-B-n {{declared here}}
+    const int n; // #cwg302-B-n
     A a;
-  } b = B(); // #dr302-b
+  } b = B(); // #cwg302-b
   // Trivial default constructor C::C() is not called here.
   struct C {
     const int n;
   } c = C();
 #else
   struct B {
-    const int n; // #dr302-B-n
+    const int n; // #cwg302-B-n
     A a;
   } b = B();
   // expected-error@-1 {{call to implicitly-deleted default constructor of 'B'}}
-  //   expected-note@#dr302-B-n {{default constructor of 'B' is implicitly deleted because field 'n' of const-qualified type 'const int' would not be initialized}}
+  //   expected-note@#cwg302-B-n {{default constructor of 'B' is implicitly deleted because field 'n' of const-qualified type 'const int' would not be initialized}}
   // C::C() is called here, because even though it's trivial, it's deleted.
   struct C {
-    const int n; // #dr302-C-n
+    const int n; // #cwg302-C-n
   } c = C();
   // expected-error@-1 {{call to implicitly-deleted default constructor of 'C'}}
-  //   expected-note@#dr302-C-n {{default constructor of 'C' is implicitly deleted because field 'n' of const-qualified type 'const int' would not be initialized}}
+  //   expected-note@#cwg302-C-n {{default constructor of 'C' is implicitly deleted because field 'n' of const-qualified type 'const int' would not be initialized}}
   struct D {
     const int n = 0;
   } d = D();
 #endif
 }
 
-// dr303: na
+// cwg303: na
 
-namespace dr304 { // dr304: 2.9
+namespace cwg304 { // cwg304: 2.9
   typedef int &a;
   int n = a();
   // expected-error@-1 {{reference to type 'int' requires an initializer}}
 
-  struct S { int &b; }; // #dr304-S
+  struct S { int &b; }; // #cwg304-S
   // cxx98-error@-1 {{reference to type 'int' requires an initializer}}
-  //   cxx98-note@#dr304-m {{in value-initialization of type 'S' here}}
-  int m = S().b; // #dr304-m
+  //   cxx98-note@#cwg304-m {{in value-initialization of type 'S' here}}
+  int m = S().b; // #cwg304-m
   // since-cxx11-error@-1 {{call to implicitly-deleted default constructor of 'S'}}
-  //   since-cxx11-note@#dr304-S {{default constructor of 'S' is implicitly deleted because field 'b' of reference type 'int &' would not be initialized}}
+  //   since-cxx11-note@#cwg304-S {{default constructor of 'S' is implicitly deleted because field 'b' of reference type 'int &' would not be initialized}}
 }
 
-namespace dr305 { // dr305: no
+namespace cwg305 { // cwg305: no
   struct A {
     typedef A C;
   };
@@ -129,10 +129,10 @@ namespace dr305 { // dr305: no
     b->~C();
   }
   void h(B *b) {
-    struct B {}; // #dr305-h-B
+    struct B {}; // #cwg305-h-B
     b->~B();
-    // expected-error@-1 {{destructor type 'B' in object destruction expression does not match the type 'B' (aka 'dr305::A') of the object being destroyed}}
-    //   expected-note@#dr305-h-B {{type 'B' found by destructor name lookup}}
+    // expected-error@-1 {{destructor type 'B' in object destruction expression does not match the type 'B' (aka 'cwg305::A') of the object being destroyed}}
+    //   expected-note@#cwg305-h-B {{type 'B' found by destructor name lookup}}
   }
 
   template<typename T> struct X {};
@@ -141,7 +141,7 @@ namespace dr305 { // dr305: no
     x->~X<int>();
     x->~X();
     x->~X<char>();
-    // expected-error@-1 {{no member named '~X' in 'dr305::X<int>'}}
+    // expected-error@-1 {{no member named '~X' in 'cwg305::X<int>'}}
   }
 
 #if __cplusplus >= 201103L
@@ -158,9 +158,9 @@ namespace dr305 { // dr305: no
   };
   void k(Z *z) {
     z->~T1<int>();
-    // expected-error@-1 {{no member named 'T1' in 'dr305::Z'}}
+    // expected-error@-1 {{no member named 'T1' in 'cwg305::Z'}}
     z->~T2<int>();
-    // expected-error@-1 {{no member named '~int' in 'dr305::Z'}}
+    // expected-error@-1 {{no member named '~int' in 'cwg305::Z'}}
     z->~T2<Z>();
   }
 
@@ -170,29 +170,29 @@ namespace dr305 { // dr305: no
   }
   template<typename A> using R = Q::R<int>;
   void qr(Q::R<int> x) { x.~R<char>(); }
-  // expected-error@-1 {{no member named '~R' in 'dr305::Q::R<int>'}}
+  // expected-error@-1 {{no member named '~R' in 'cwg305::Q::R<int>'}}
 #endif
 }
 
-namespace dr306 { // dr306: dup 39
+namespace cwg306 { // cwg306: dup 39
   struct A { struct B {}; };
   struct C { typedef A::B B; };
   struct D : A, A::B, C {};
   D::B b;
 
-  struct X {}; // #dr306-X
-  template<typename T> struct Y { typedef T X; }; // #dr306-typedef-X
+  struct X {}; // #cwg306-X
+  template<typename T> struct Y { typedef T X; }; // #cwg306-typedef-X
   template<typename T> struct Z : X, Y<T> {};
   Z<X>::X zx;
   Z<const X>::X zcx;
   // expected-error@-1 {{member 'X' found in multiple base classes of different types}}
-  //   expected-note@#dr306-X {{member type 'dr306::X' found}}
-  //   expected-note@#dr306-typedef-X {{member type 'const dr306::X' found}}
+  //   expected-note@#cwg306-X {{member type 'cwg306::X' found}}
+  //   expected-note@#cwg306-typedef-X {{member type 'const cwg306::X' found}}
 }
 
-// dr307: na
+// cwg307: na
 
-namespace dr308 { // dr308: 3.7
+namespace cwg308 { // cwg308: 3.7
   // This is mostly an ABI library issue.
   struct A {};
   struct B : A {};
@@ -210,19 +210,19 @@ namespace dr308 { // dr308: 3.7
     // positive rate should be acceptably low.
     try {
       throw D();
-    } catch (const A&) { // #dr308-catch-A
+    } catch (const A&) { // #cwg308-catch-A
       // unreachable
     } catch (const B&) {
       // expected-warning@-1 {{exception of type 'const B &' will be caught by earlier handler}}
-      //   expected-note@#dr308-catch-A {{for type 'const A &'}}
+      //   expected-note@#cwg308-catch-A {{for type 'const A &'}}
       // get here instead
     }
   }
 }
 
-// dr309: dup 485
+// cwg309: dup 485
 
-namespace dr311 { // dr311: 3.0
+namespace cwg311 { // cwg311: 3.0
   namespace X { namespace Y {} }
   namespace X::Y {}
   // cxx98-14-error@-1 {{nested namespace definition is a C++17 extension; define each namespace separately}}
@@ -231,23 +231,23 @@ namespace dr311 { // dr311: 3.0
     // cxx98-14-error@-1 {{nested namespace definition is a C++17 extension; define each namespace separately}}
   }
   // FIXME: The diagnostics here are not very good.
-  namespace ::dr311::X {}
+  namespace ::cwg311::X {}
   // expected-error@-1 {{expected identifier or '{'}}
   // expected-warning@-2 {{extra qualification on member 'X'}}
   // expected-error@-3 {{a type specifier is required for all declarations}}
   // expected-error@-4 {{expected ';' after top level declarator}}
 }
 
-// dr312: dup 616
+// cwg312: dup 616
 
-namespace dr313 { // dr313: dup 299 c++11
+namespace cwg313 { // cwg313: dup 299 c++11
   struct A { operator int() const; };
   // FIXME: should this be available in c++98 mode?
   int *p = new int[A()];
   // cxx98-error@-1 {{implicit conversion from array size expression of type 'A' to integral type 'int' is a C++11 extension}}
 }
 
-namespace dr314 { // dr314: no
+namespace cwg314 { // cwg314: no
                   // NB: dup 1710
 template <typename T> struct A {
   template <typename U> struct B {};
@@ -260,16 +260,16 @@ template <typename T> struct C2 : public A<T>::B<T> {
   C2() : A<T>::B<T>() {}
   // expected-error@-1 {{use 'template' keyword to treat 'B' as a dependent template name}}
 };
-} // namespace dr314
+} // namespace cwg314
 
-// dr315: na
-// dr316: sup 1004
+// cwg315: na
+// cwg316: sup 1004
 
-namespace dr317 { // dr317: 3.5
-  void f() {} // #dr317-f
+namespace cwg317 { // cwg317: 3.5
+  void f() {} // #cwg317-f
   inline void f();
   // expected-error@-1 {{inline declaration of 'f' follows non-inline definition}}
-  //   expected-note@#dr317-f {{previous definition is here}}
+  //   expected-note@#cwg317-f {{previous definition is here}}
 
   int g();
   int n = g();
@@ -277,19 +277,19 @@ namespace dr317 { // dr317: 3.5
 
   int h();
   int m = h();
-  int h() { return 0; } // #dr317-h
+  int h() { return 0; } // #cwg317-h
   inline int h();
   // expected-error@-1 {{inline declaration of 'h' follows non-inline definition}}
-  //   expected-note@#dr317-h {{previous definition is here}}
+  //   expected-note@#cwg317-h {{previous definition is here}}
 }
 
-namespace dr318 { // dr318: sup 1310
+namespace cwg318 { // cwg318: sup 1310
   struct A {};
   struct A::A a;
 }
 
-namespace dr319 { // dr319: no
-  // FIXME: dup dr389
+namespace cwg319 { // cwg319: no
+  // FIXME: dup cwg389
   // FIXME: We don't have a diagnostic for a name with linkage
   //        having a type without linkage.
   typedef struct {
@@ -325,7 +325,7 @@ namespace dr319 { // dr319: no
   }
 }
 
-namespace dr320 { // dr320: yes
+namespace cwg320 { // cwg320: yes
 #if __cplusplus >= 201103L
   struct X {
     constexpr X() {}
@@ -338,7 +338,7 @@ namespace dr320 { // dr320: yes
 #endif
 }
 
-namespace dr321 { // dr321: dup 557
+namespace cwg321 { // cwg321: dup 557
   namespace N {
     template<int> struct A {
       template<int> struct B;
@@ -358,7 +358,7 @@ namespace dr321 { // dr321: dup 557
   bool x = i == j;
 }
 
-namespace dr322 { // dr322: 2.8
+namespace cwg322 { // cwg322: 2.8
   struct A {
     template<typename T> operator T&();
   } a;
@@ -366,18 +366,18 @@ namespace dr322 { // dr322: 2.8
   int &s = a;
 }
 
-// dr323: no
+// cwg323: no
 
-namespace dr324 { // dr324: 3.6
-  struct S { int n : 1; } s; // #dr324-n
+namespace cwg324 { // cwg324: 3.6
+  struct S { int n : 1; } s; // #cwg324-n
   int &a = s.n;
   // expected-error@-1 {{non-const reference cannot bind to bit-field 'n'}}
-  //   expected-note@#dr324-n {{bit-field is declared here}}
+  //   expected-note@#cwg324-n {{bit-field is declared here}}
   int *b = &s.n;
   // expected-error@-1 {{address of bit-field requested}}
   int &c = (s.n = 0);
   // expected-error@-1 {{non-const reference cannot bind to bit-field 'n'}}
-  //   expected-note@#dr324-n {{bit-field is declared here}}
+  //   expected-note@#cwg324-n {{bit-field is declared here}}
   int *d = &(s.n = 0);
   // expected-error@-1 {{address of bit-field requested}}
   // FIXME: why don't we emit a note here, as for the rest of this type of diagnostic in this test?
@@ -387,19 +387,19 @@ namespace dr324 { // dr324: 3.6
   // expected-error@-1 {{address of bit-field requested}}
   int &g = (void(), s.n);
   // expected-error@-1 {{non-const reference cannot bind to bit-field 'n'}}
-  //   expected-note@#dr324-n {{bit-field is declared here}}
+  //   expected-note@#cwg324-n {{bit-field is declared here}}
   int *h = &(void(), s.n);
   // expected-error@-1 {{address of bit-field requested}}
   int *i = &++s.n;
   // expected-error@-1 {{address of bit-field requested}}
 }
 
-namespace dr326 { // dr326: 3.1
+namespace cwg326 { // cwg326: 3.1
   struct S {};
   int test[__is_trivially_constructible(S, const S&) ? 1 : -1];
 }
 
-namespace dr327 { // dr327: dup 538
+namespace cwg327 { // cwg327: dup 538
   struct A;
   class A {};
 
@@ -407,40 +407,40 @@ namespace dr327 { // dr327: dup 538
   struct B {};
 }
 
-namespace dr328 { // dr328: yes
-  struct A; // #dr328-A
+namespace cwg328 { // cwg328: yes
+  struct A; // #cwg328-A
   struct B { A a; };
   // expected-error@-1 {{field has incomplete type 'A'}}
-  //   expected-note@#dr328-A {{forward declaration of 'dr328::A'}}
+  //   expected-note@#cwg328-A {{forward declaration of 'cwg328::A'}}
   template<typename> struct C { A a; };
   // expected-error@-1 {{field has incomplete type 'A'}}
-  //   expected-note@#dr328-A {{forward declaration of 'dr328::A'}}
+  //   expected-note@#cwg328-A {{forward declaration of 'cwg328::A'}}
   A *p = new A[0];
   // expected-error@-1 {{allocation of incomplete type 'A'}}
-  //   expected-note@#dr328-A {{forward declaration of 'dr328::A'}}
+  //   expected-note@#cwg328-A {{forward declaration of 'cwg328::A'}}
 }
 
-namespace dr329 { // dr329: 3.5
+namespace cwg329 { // cwg329: 3.5
   struct B {};
   template<typename T> struct A : B {
     friend void f(A a) { g(a); }
     friend void h(A a) { g(a); }
     // expected-error@-1 {{use of undeclared identifier 'g'}}
-    //   expected-note@#dr329-h-call {{in instantiation of member function 'dr329::h' requested here}}
-    friend void i(B b) {} // #dr329-i
+    //   expected-note@#cwg329-h-call {{in instantiation of member function 'cwg329::h' requested here}}
+    friend void i(B b) {} // #cwg329-i
     // expected-error@-1 {{redefinition of 'i'}}
-    //   expected-note@#dr329-b {{in instantiation of template class 'dr329::A<char>' requested here}}
-    //   expected-note@#dr329-i {{previous definition is here}}
+    //   expected-note@#cwg329-b {{in instantiation of template class 'cwg329::A<char>' requested here}}
+    //   expected-note@#cwg329-i {{previous definition is here}}
   };
   A<int> a; 
-  A<char> b; // #dr329-b
+  A<char> b; // #cwg329-b
 
   void test() {
-    h(a); // #dr329-h-call
+    h(a); // #cwg329-h-call
   }
 }
 
-namespace dr330 { // dr330: 7
+namespace cwg330 { // cwg330: 7
   // Conversions between P and Q will be allowed by P0388.
   typedef int *(*P)[3];
   typedef const int *const (*Q)[3];
@@ -526,27 +526,27 @@ namespace dr330 { // dr330: 7
     void f(A *a) {
       (void) reinterpret_cast<B1*>(a);
       (void) reinterpret_cast<B2*>(a);
-      // expected-error@-1 {{ISO C++ does not allow reinterpret_cast from 'A *' (aka 'const volatile int (*)[1][2][3]') to 'B2 *' (aka 'int *const dr330::swift_17882::X::***') because it casts away qualifiers, even though the source and destination types are unrelated}}
+      // expected-error@-1 {{ISO C++ does not allow reinterpret_cast from 'A *' (aka 'const volatile int (*)[1][2][3]') to 'B2 *' (aka 'int *const cwg330::swift_17882::X::***') because it casts away qualifiers, even though the source and destination types are unrelated}}
       (void) reinterpret_cast<B3*>(a);
-      // expected-error@-1 {{ISO C++ does not allow reinterpret_cast from 'A *' (aka 'const volatile int (*)[1][2][3]') to 'B3 *' (aka 'int *dr330::swift_17882::X::*volatile **') because it casts away qualifiers, even though the source and destination types are unrelated}}
+      // expected-error@-1 {{ISO C++ does not allow reinterpret_cast from 'A *' (aka 'const volatile int (*)[1][2][3]') to 'B3 *' (aka 'int *cwg330::swift_17882::X::*volatile **') because it casts away qualifiers, even though the source and destination types are unrelated}}
       (void) reinterpret_cast<B4*>(a);
     }
   }
 }
 
-namespace dr331 { // dr331: 11
+namespace cwg331 { // cwg331: 11
   struct A {
-    A(volatile A&); // #dr331-A-ctor
+    A(volatile A&); // #cwg331-A-ctor
   };
   const A a;
   // expected-error@-1 {{no matching constructor for initialization of 'const A'}}
-  //   expected-note@#dr331-A-ctor {{candidate constructor not viable: requires 1 argument, but 0 were provided}}
+  //   expected-note@#cwg331-A-ctor {{candidate constructor not viable: requires 1 argument, but 0 were provided}}
   const A b(a);
   // expected-error@-1 {{no matching constructor for initialization of 'const A'}}
-  //   expected-note@#dr331-A-ctor {{candidate constructor not viable: 1st argument ('const A') would lose const qualifier}}
+  //   expected-note@#cwg331-A-ctor {{candidate constructor not viable: 1st argument ('const A') would lose const qualifier}}
 }
 
-namespace dr332 { // dr332: dup 577
+namespace cwg332 { // cwg332: dup 577
   void f(volatile void);
   // expected-error@-1 {{'void' as parameter must not have type qualifiers}}
   // cxx20-23-warning@-2 {{volatile-qualified parameter type 'volatile void' is deprecated}}
@@ -557,14 +557,14 @@ namespace dr332 { // dr332: dup 577
   // cxx20-23-warning@-2 {{volatile-qualified parameter type 'volatile void' is deprecated}}
 }
 
-namespace dr333 { // dr333: yes
+namespace cwg333 { // cwg333: yes
   int n = 0;
   int f(int(n));
   int g((int(n)));
   int h = f(g);
 }
 
-namespace dr334 { // dr334: yes
+namespace cwg334 { // cwg334: yes
   template<typename T> void f() {
     T x;
     f((x, 123));
@@ -576,9 +576,9 @@ namespace dr334 { // dr334: yes
   template void f<S>();
 }
 
-// dr335: no
+// cwg335: no
 
-namespace dr336 { // dr336: yes
+namespace cwg336 { // cwg336: yes
   namespace Pre {
     template<class T1> class A {
       template<class T2> class B {
@@ -588,7 +588,7 @@ namespace dr336 { // dr336: yes
     };
     template<> template<class X> class A<int>::B {};
     template<> template<> template<class T> void A<int>::B<double>::mf1(T t) {}
-    // expected-error@-1 {{out-of-line definition of 'mf1' does not match any declaration in 'dr336::Pre::A<int>::B<double>'}}
+    // expected-error@-1 {{out-of-line definition of 'mf1' does not match any declaration in 'cwg336::Pre::A<int>::B<double>'}}
     template<class Y> template<> void A<Y>::B<double>::mf2() {}
     // expected-error@-1 {{nested name specifier 'A<Y>::B<double>::' for declaration does not refer into a class, class template or class template partial specialization}}
   }
@@ -609,7 +609,7 @@ namespace dr336 { // dr336: yes
   }
 }
 
-namespace dr337 { // dr337: yes
+namespace cwg337 { // cwg337: yes
   template<typename T> void f(T (*)[1]);
   template<typename T> int &f(...);
 
@@ -617,7 +617,7 @@ namespace dr337 { // dr337: yes
   int &r = f<A>(0);
 
   // FIXME: The language rules here are completely broken. We cannot determine
-  // whether an incomplete type is abstract. See DR1640, which will probably
+  // whether an incomplete type is abstract. See CWG1640, which will probably
   // supersede this one and remove this rule.
   struct B;
   int &s = f<B>(0);
@@ -625,20 +625,20 @@ namespace dr337 { // dr337: yes
   struct B { virtual ~B() = 0; };
 }
 
-namespace dr339 { // dr339: 2.8
+namespace cwg339 { // cwg339: 2.8
   template <int I> struct A { static const int value = I; };
 
   char xxx(int);
   char (&xxx(float))[2];
 
-  template<class T> A<sizeof(xxx((T)0))> f(T) {} // #dr339-f 
+  template<class T> A<sizeof(xxx((T)0))> f(T) {} // #cwg339-f 
 
   void test() {
     A<1> a = f(0);
     A<2> b = f(0.0f);
     A<3> c = f("foo");
     // expected-error@-1 {{no matching function}}
-    //   expected-note@#dr339-f {{candidate}}
+    //   expected-note@#cwg339-f {{candidate}}
   }
 
 
@@ -658,50 +658,50 @@ namespace dr339 { // dr339: 2.8
   A<1> c = make_A<char>();
 }
 
-namespace dr340 { // dr340: yes
+namespace cwg340 { // cwg340: yes
   struct A { A(int); };
   struct B { B(A, A, int); };
   int x, y;
   B b(A(x), A(y), 3);
 }
 
-namespace dr341 { // dr341: sup 1708
+namespace cwg341 { // cwg341: sup 1708
   namespace A {
     int n;
-    extern "C" int &dr341_a = n; // #dr341_a
+    extern "C" int &cwg341_a = n; // #cwg341_a
   }
   namespace B {
-    extern "C" int &dr341_a = dr341_a;
-    // expected-error@-1 {{redefinition of 'dr341_a'}}
-    //   expected-note@#dr341_a {{previous definition is here}} 
+    extern "C" int &cwg341_a = cwg341_a;
+    // expected-error@-1 {{redefinition of 'cwg341_a'}}
+    //   expected-note@#cwg341_a {{previous definition is here}} 
   }
-  extern "C" void dr341_b(); // #dr341_b 
+  extern "C" void cwg341_b(); // #cwg341_b 
 }
-int dr341_a;
-// expected-error@-1 {{declaration of 'dr341_a' in global scope conflicts with declaration with C language linkage}}
-//   expected-note@#dr341_a {{declared with C language linkage here}}
-int dr341_b;
-// expected-error@-1 {{declaration of 'dr341_b' in global scope conflicts with declaration with C language linkage}}
-//   expected-note@#dr341_b {{declared with C language linkage here}}
-int dr341_c; // #dr341_c
-int dr341_d; // #dr341_d
-namespace dr341 {
-  extern "C" int dr341_c;
-  // expected-error@-1 {{declaration of 'dr341_c' with C language linkage conflicts with declaration in global scope}}
-  //   expected-note@#dr341_c {{declared in global scope here}}
-  extern "C" void dr341_d();
-  // expected-error@-1 {{declaration of 'dr341_d' with C language linkage conflicts with declaration in global scope}}
-  //   expected-note@#dr341_d {{declared in global scope here}}
-
-  namespace A { extern "C" int dr341_e; } // #dr341_e 
-  namespace B { extern "C" void dr341_e(); }
-  // expected-error@-1 {{redefinition of 'dr341_e' as different kind of symbol}}
-  //   expected-note@#dr341_e {{previous definition is here}}
+int cwg341_a;
+// expected-error@-1 {{declaration of 'cwg341_a' in global scope conflicts with declaration with C language linkage}}
+//   expected-note@#cwg341_a {{declared with C language linkage here}}
+int cwg341_b;
+// expected-error@-1 {{declaration of 'cwg341_b' in global scope conflicts with declaration with C language linkage}}
+//   expected-note@#cwg341_b {{declared with C language linkage here}}
+int cwg341_c; // #cwg341_c
+int cwg341_d; // #cwg341_d
+namespace cwg341 {
+  extern "C" int cwg341_c;
+  // expected-error@-1 {{declaration of 'cwg341_c' with C language linkage conflicts with declaration in global scope}}
+  //   expected-note@#cwg341_c {{declared in global scope here}}
+  extern "C" void cwg341_d();
+  // expected-error@-1 {{declaration of 'cwg341_d' with C language linkage conflicts with declaration in global scope}}
+  //   expected-note@#cwg341_d {{declared in global scope here}}
+
+  namespace A { extern "C" int cwg341_e; } // #cwg341_e 
+  namespace B { extern "C" void cwg341_e(); }
+  // expected-error@-1 {{redefinition of 'cwg341_e' as different kind of symbol}}
+  //   expected-note@#cwg341_e {{previous definition is here}}
 }
 
-// dr342: na
+// cwg342: na
 
-namespace dr343 { // dr343: no
+namespace cwg343 { // cwg343: no
   // FIXME: dup 1710
   template<typename T> struct A {
     template<typename U> struct B {};
@@ -714,32 +714,32 @@ namespace dr343 { // dr343: no
   };
 }
 
-namespace dr344 { // dr344: dup 1435
+namespace cwg344 { // cwg344: dup 1435
   struct A { inline virtual ~A(); };
   struct B { friend A::~A(); };
 }
 
-namespace dr345 { // dr345: yes
+namespace cwg345 { // cwg345: yes
   struct A {
     struct X {};
-    int X; // #dr345-int-X
+    int X; // #cwg345-int-X
   };
   struct B {
     struct X {};
   };
   template <class T> void f(T t) { typename T::X x; }
-  // expected-error@-1 {{typename specifier refers to non-type member 'X' in 'dr345::A'}}
-  //   expected-note@#dr345-f-a {{in instantiation of function template specialization 'dr345::f<dr345::A>' requested here}}
-  //   expected-note@#dr345-int-X {{referenced member 'X' is declared here}}
+  // expected-error@-1 {{typename specifier refers to non-type member 'X' in 'cwg345::A'}}
+  //   expected-note@#cwg345-f-a {{in instantiation of function template specialization 'cwg345::f<cwg345::A>' requested here}}
+  //   expected-note@#cwg345-int-X {{referenced member 'X' is declared here}}
   void f(A a, B b) {
     f(b);
-    f(a); // #dr345-f-a
+    f(a); // #cwg345-f-a
   }
 }
 
-// dr346: na
+// cwg346: na
 
-namespace dr347 { // dr347: yes
+namespace cwg347 { // cwg347: yes
   struct base {
     struct nested;
     static int n;
@@ -750,31 +750,31 @@ namespace dr347 { // dr347: yes
   struct derived : base {};
 
   struct derived::nested {};
-  // expected-error@-1 {{no struct named 'nested' in 'dr347::derived'}}
+  // expected-error@-1 {{no struct named 'nested' in 'cwg347::derived'}}
   int derived::n;
-  // expected-error@-1 {{no member named 'n' in 'dr347::derived'}}
+  // expected-error@-1 {{no member named 'n' in 'cwg347::derived'}}
   void derived::f() {}
-  // expected-error@-1 {{out-of-line definition of 'f' does not match any declaration in 'dr347::derived'}}
+  // expected-error@-1 {{out-of-line definition of 'f' does not match any declaration in 'cwg347::derived'}}
   void derived::g() {}
-  // expected-error@-1 {{out-of-line definition of 'g' does not match any declaration in 'dr347::derived'}}
+  // expected-error@-1 {{out-of-line definition of 'g' does not match any declaration in 'cwg347::derived'}}
 }
 
-// dr348: na
+// cwg348: na
 
-namespace dr349 { // dr349: no
+namespace cwg349 { // cwg349: no
   struct A {
     template <class T> operator T ***() {
       int ***p = 0;
       return p;
       // cxx98-20-error@-1 {{cannot initialize return object of type 'const int ***' with an lvalue of type 'int ***'}}
       // since-cxx23-error@-2 {{cannot initialize return object of type 'const int ***' with an rvalue of type 'int ***'}}
-      //   expected-note@#dr349-p1 {{in instantiation of function template specialization 'dr349::A::operator const int ***<const int>' requested here}}
+      //   expected-note@#cwg349-p1 {{in instantiation of function template specialization 'cwg349::A::operator const int ***<const int>' requested here}}
     }
   };
 
   // FIXME: This is valid.
   A a;
-  const int *const *const *p1 = a; // #dr349-p1
+  const int *const *const *p1 = a; // #cwg349-p1
 
   struct B {
     template <class T> operator T ***() {
@@ -788,28 +788,28 @@ namespace dr349 { // dr349: no
   const int *const *const *p2 = b;
 }
 
-// dr351: na
+// cwg351: na
 
-namespace dr352 { // dr352: 2.8
+namespace cwg352 { // cwg352: 2.8
   namespace example1 {
     namespace A {
       enum E {};
-      template<typename R, typename A> void foo(E, R (*)(A)); // #dr352-foo
+      template<typename R, typename A> void foo(E, R (*)(A)); // #cwg352-foo
     }
 
     template<typename T> void arg(T);
-    template<typename T> int arg(T) = delete; // #dr352-deleted
+    template<typename T> int arg(T) = delete; // #cwg352-deleted
     // cxx98-error@-1 {{deleted function definitions are a C++11 extension}}
 
     void f(A::E e) {
       foo(e, &arg);
       // expected-error@-1 {{no matching function for call to 'foo'}}
-      //   expected-note@#dr352-foo {{candidate template ignored: couldn't infer template argument 'R'}}
+      //   expected-note@#cwg352-foo {{candidate template ignored: couldn't infer template argument 'R'}}
 
       using A::foo;
       foo<int, int>(e, &arg);
       // expected-error@-1 {{attempt to use a deleted function}}
-      //   expected-note@#dr352-deleted {{'arg<int>' has been explicitly marked deleted here}}
+      //   expected-note@#cwg352-deleted {{'arg<int>' has been explicitly marked deleted here}}
     }
 
     int arg(int);
@@ -817,7 +817,7 @@ namespace dr352 { // dr352: 2.8
     void g(A::E e) {
       foo(e, &arg);
       // expected-error@-1 {{no matching function for call to 'foo'}}
-      //   expected-note@#dr352-foo {{candidate template ignored: couldn't infer template argument 'R'}} 
+      //   expected-note@#cwg352-foo {{candidate template ignored: couldn't infer template argument 'R'}} 
 
       using A::foo;
       foo<int, int>(e, &arg); // ok, uses non-template
@@ -826,7 +826,7 @@ namespace dr352 { // dr352: 2.8
 
   namespace contexts {
     template<int I> void f1(int (&)[I]);
-    template<int I> void f2(int (&)[I+1]); // #dr352-f2
+    template<int I> void f2(int (&)[I+1]); // #cwg352-f2
     template<int I> void f3(int (&)[I+1], int (&)[I]);
     void f() {
       int a[4];
@@ -834,13 +834,13 @@ namespace dr352 { // dr352: 2.8
       f1(a);
       f2(a);
       // expected-error@-1 {{no matching function for call to 'f2'}}
-      //   expected-note@#dr352-f2 {{candidate template ignored: couldn't infer template argument 'I'}}
+      //   expected-note@#cwg352-f2 {{candidate template ignored: couldn't infer template argument 'I'}}
       f3(a, b);
     }
 
     template<int I> struct S {};
     template<int I> void g1(S<I>);
-    template<int I> void g2(S<I+1>); // #dr352-g2
+    template<int I> void g2(S<I+1>); // #cwg352-g2
     template<int I> void g3(S<I+1>, S<I>);
     void g() {
       S<4> a;
@@ -848,24 +848,24 @@ namespace dr352 { // dr352: 2.8
       g1(a);
       g2(a);
       // expected-error@-1 {{no matching function for call to 'g2'}}
-      //   expected-note@#dr352-g2 {{candidate template ignored: couldn't infer template argument 'I'}}
+      //   expected-note@#cwg352-g2 {{candidate template ignored: couldn't infer template argument 'I'}}
       g3(a, b);
     }
 
-    template<typename T> void h1(T = 0); // #dr352-h1
+    template<typename T> void h1(T = 0); // #cwg352-h1
     template<typename T> void h2(T, T = 0);
     void h() {
       h1();
       // expected-error@-1 {{no matching function for call to 'h1'}}
-      //   expected-note@#dr352-h1 {{candidate template ignored: couldn't infer template argument 'T'}}
+      //   expected-note@#cwg352-h1 {{candidate template ignored: couldn't infer template argument 'T'}}
       h1(0);
       h1<int>();
       h2(0);
     }
 
     template<typename T> int tmpl(T);
-    template<typename R, typename A> void i1(R (*)(A)); // #dr352-i1
-    template<typename R, typename A> void i2(R, A, R (*)(A)); // #dr352-i2
+    template<typename R, typename A> void i1(R (*)(A)); // #cwg352-i1
+    template<typename R, typename A> void i2(R, A, R (*)(A)); // #cwg352-i2
     void i() {
       extern int single(int);
       i1(single);
@@ -874,20 +874,20 @@ namespace dr352 { // dr352: 2.8
       extern int ambig(float), ambig(int);
       i1(ambig);
       // expected-error@-1 {{no matching function for call to 'i1'}}
-      //   expected-note@#dr352-i1 {{candidate template ignored: couldn't infer template argument 'R'}}
+      //   expected-note@#cwg352-i1 {{candidate template ignored: couldn't infer template argument 'R'}}
       i2(0, 0, ambig);
 
       extern void no_match(float), no_match(int);
       i1(no_match);
       // expected-error@-1 {{no matching function for call to 'i1'}}
-      //   expected-note@#dr352-i1 {{candidate template ignored: couldn't infer template argument 'R'}}
+      //   expected-note@#cwg352-i1 {{candidate template ignored: couldn't infer template argument 'R'}}
       i2(0, 0, no_match);
       // expected-error@-1 {{no matching function for call to 'i2'}}
-      //   expected-note@#dr352-i2 {{candidate function [with R = int, A = int] not viable: no overload of 'no_match' matching 'int (*)(int)' for 3rd argument}}
+      //   expected-note@#cwg352-i2 {{candidate function [with R = int, A = int] not viable: no overload of 'no_match' matching 'int (*)(int)' for 3rd argument}}
 
       i1(tmpl);
       // expected-error@-1 {{no matching function for call to 'i1'}}
-      //   expected-note@#dr352-i1 {{candidate template ignored: couldn't infer template argument 'R'}}
+      //   expected-note@#cwg352-i1 {{candidate template ignored: couldn't infer template argument 'R'}}
       i2(0, 0, tmpl);
     }
   }
@@ -918,109 +918,109 @@ namespace dr352 { // dr352: 2.8
 
   namespace example5 {
     template<int I> class A {};
-    template<int I> void g(A<I+1>); // #dr352-g 
+    template<int I> void g(A<I+1>); // #cwg352-g 
     template<int I> void f(A<I>, A<I+1>);
     void h(A<1> a1, A<2> a2) {
       g(a1);
       // expected-error@-1 {{no matching function for call to 'g'}}
-      //   expected-note@#dr352-g {{candidate template ignored: couldn't infer template argument 'I'}}
+      //   expected-note@#cwg352-g {{candidate template ignored: couldn't infer template argument 'I'}}
       g<0>(a1);
       f(a1, a2);
     }
   }
 }
 
-// dr353 needs an IRGen test.
+// cwg353 needs an IRGen test.
 
-namespace dr354 { // dr354: yes c++11
+namespace cwg354 { // cwg354: yes c++11
   // FIXME: Should we allow this in C++98 too?
   struct S {};
 
-  template<int*> struct ptr {}; // #dr354-ptr
-  ptr<0> p0; // #dr354-p0
-  // cxx98-error@#dr354-p0 {{non-type template argument does not refer to any declaration}}
-  //   cxx98-note@#dr354-ptr {{template parameter is declared here}}
-  // cxx11-14-error@#dr354-p0 {{null non-type template argument must be cast to template parameter type 'int *'}}
-  //   cxx11-14-note@#dr354-ptr {{template parameter is declared here}}
-  // since-cxx17-error@#dr354-p0 {{conversion from 'int' to 'int *' is not allowed in a converted constant expression}}
+  template<int*> struct ptr {}; // #cwg354-ptr
+  ptr<0> p0; // #cwg354-p0
+  // cxx98-error@#cwg354-p0 {{non-type template argument does not refer to any declaration}}
+  //   cxx98-note@#cwg354-ptr {{template parameter is declared here}}
+  // cxx11-14-error@#cwg354-p0 {{null non-type template argument must be cast to template parameter type 'int *'}}
+  //   cxx11-14-note@#cwg354-ptr {{template parameter is declared here}}
+  // since-cxx17-error@#cwg354-p0 {{conversion from 'int' to 'int *' is not allowed in a converted constant expression}}
   ptr<(int*)0> p1;
   // cxx98-error@-1 {{non-type template argument does not refer to any declaration}}
-  //   cxx98-note@#dr354-ptr {{template parameter is declared here}}
-  ptr<(float*)0> p2; // #dr354-p2
-  // cxx98-error@#dr354-p2 {{non-type template argument does not refer to any declaration}}
-  //   cxx98-note@#dr354-ptr {{template parameter is declared here}}
-  // cxx11-14-error@#dr354-p2 {{null non-type template argument of type 'float *' does not match template parameter of type 'int *'}}
-  //   cxx11-14-note@#dr354-ptr {{template parameter is declared here}}
-  // since-cxx17-error@#dr354-p2 {{value of type 'float *' is not implicitly convertible to 'int *'}}
-  ptr<(int S::*)0> p3; // #dr354-p3
-  // cxx98-error@#dr354-p3 {{non-type template argument does not refer to any declaration}}
-  //   cxx98-note@#dr354-ptr {{template parameter is declared here}}
-  // cxx11-14-error@#dr354-p3 {{null non-type template argument of type 'int dr354::S::*' does not match template parameter of type 'int *'}}
-  //   cxx11-14-note@#dr354-ptr {{template parameter is declared here}}
-  // since-cxx17-error@#dr354-p3 {{value of type 'int dr354::S::*' is not implicitly convertible to 'int *'}}
-
-  template<int*> int both(); // #dr354-both-int-ptr
-  template<int> int both(); // #dr354-both-int
+  //   cxx98-note@#cwg354-ptr {{template parameter is declared here}}
+  ptr<(float*)0> p2; // #cwg354-p2
+  // cxx98-error@#cwg354-p2 {{non-type template argument does not refer to any declaration}}
+  //   cxx98-note@#cwg354-ptr {{template parameter is declared here}}
+  // cxx11-14-error@#cwg354-p2 {{null non-type template argument of type 'float *' does not match template parameter of type 'int *'}}
+  //   cxx11-14-note@#cwg354-ptr {{template parameter is declared here}}
+  // since-cxx17-error@#cwg354-p2 {{value of type 'float *' is not implicitly convertible to 'int *'}}
+  ptr<(int S::*)0> p3; // #cwg354-p3
+  // cxx98-error@#cwg354-p3 {{non-type template argument does not refer to any declaration}}
+  //   cxx98-note@#cwg354-ptr {{template parameter is declared here}}
+  // cxx11-14-error@#cwg354-p3 {{null non-type template argument of type 'int cwg354::S::*' does not match template parameter of type 'int *'}}
+  //   cxx11-14-note@#cwg354-ptr {{template parameter is declared here}}
+  // since-cxx17-error@#cwg354-p3 {{value of type 'int cwg354::S::*' is not implicitly convertible to 'int *'}}
+
+  template<int*> int both(); // #cwg354-both-int-ptr
+  template<int> int both(); // #cwg354-both-int
   int b0 = both<0>();
   int b1 = both<(int*)0>();
   // cxx98-error@-1 {{no matching function for call to 'both'}}
-  //   cxx98-note@#dr354-both-int-ptr {{candidate template ignored: invalid explicitly-specified argument for 1st template parameter}}
-  //   cxx98-note@#dr354-both-int {{candidate template ignored: invalid explicitly-specified argument for 1st template parameter}}
-
-  template<int S::*> struct ptr_mem {}; // #dr354-ptr_mem
-  ptr_mem<0> m0; // #dr354-m0
-  // cxx98-error@#dr354-m0 {{non-type template argument of type 'int' cannot be converted to a value of type 'int dr354::S::*'}}
-  //   cxx98-note@#dr354-ptr_mem {{template parameter is declared here}}
-  // cxx11-14-error@#dr354-m0 {{null non-type template argument must be cast to template parameter type 'int dr354::S::*'}}
-  //   cxx11-14-note@#dr354-ptr_mem {{template parameter is declared here}}
-  // since-cxx17-error@#dr354-m0 {{conversion from 'int' to 'int dr354::S::*' is not allowed in a converted constant expression}}
+  //   cxx98-note@#cwg354-both-int-ptr {{candidate template ignored: invalid explicitly-specified argument for 1st template parameter}}
+  //   cxx98-note@#cwg354-both-int {{candidate template ignored: invalid explicitly-specified argument for 1st template parameter}}
+
+  template<int S::*> struct ptr_mem {}; // #cwg354-ptr_mem
+  ptr_mem<0> m0; // #cwg354-m0
+  // cxx98-error@#cwg354-m0 {{non-type template argument of type 'int' cannot be converted to a value of type 'int cwg354::S::*'}}
+  //   cxx98-note@#cwg354-ptr_mem {{template parameter is declared here}}
+  // cxx11-14-error@#cwg354-m0 {{null non-type template argument must be cast to template parameter type 'int cwg354::S::*'}}
+  //   cxx11-14-note@#cwg354-ptr_mem {{template parameter is declared here}}
+  // since-cxx17-error@#cwg354-m0 {{conversion from 'int' to 'int cwg354::S::*' is not allowed in a converted constant expression}}
   ptr_mem<(int S::*)0> m1;
   // cxx98-error@-1 {{non-type template argument is not a pointer to member constant}}
-  ptr_mem<(float S::*)0> m2; // #dr354-m2
-  // cxx98-error@#dr354-m2 {{non-type template argument of type 'float dr354::S::*' cannot be converted to a value of type 'int dr354::S::*'}}
-  //   cxx98-note@#dr354-ptr_mem {{template parameter is declared here}}
-  // cxx11-14-error@#dr354-m2 {{null non-type template argument of type 'float dr354::S::*' does not match template parameter of type 'int dr354::S::*'}}
-  //   cxx11-14-note@#dr354-ptr_mem {{template parameter is declared here}}
-  // since-cxx17-error@#dr354-m2 {{value of type 'float dr354::S::*' is not implicitly convertible to 'int dr354::S::*'}}
-  ptr_mem<(int *)0> m3; // #dr354-m3
-  // cxx98-error@#dr354-m3 {{non-type template argument of type 'int *' cannot be converted to a value of type 'int dr354::S::*'}}
-  //   cxx98-note@#dr354-ptr_mem {{template parameter is declared here}}
-  // cxx11-14-error@#dr354-m3 {{null non-type template argument of type 'int *' does not match template parameter of type 'int dr354::S::*'}}
-  //   cxx11-14-note@#dr354-ptr_mem {{template parameter is declared here}}
-  // since-cxx17-error@#dr354-m3 {{value of type 'int *' is not implicitly convertible to 'int dr354::S::*'}}
+  ptr_mem<(float S::*)0> m2; // #cwg354-m2
+  // cxx98-error@#cwg354-m2 {{non-type template argument of type 'float cwg354::S::*' cannot be converted to a value of type 'int cwg354::S::*'}}
+  //   cxx98-note@#cwg354-ptr_mem {{template parameter is declared here}}
+  // cxx11-14-error@#cwg354-m2 {{null non-type template argument of type 'float cwg354::S::*' does not match template parameter of type 'int cwg354::S::*'}}
+  //   cxx11-14-note@#cwg354-ptr_mem {{template parameter is declared here}}
+  // since-cxx17-error@#cwg354-m2 {{value of type 'float cwg354::S::*' is not implicitly convertible to 'int cwg354::S::*'}}
+  ptr_mem<(int *)0> m3; // #cwg354-m3
+  // cxx98-error@#cwg354-m3 {{non-type template argument of type 'int *' cannot be converted to a value of type 'int cwg354::S::*'}}
+  //   cxx98-note@#cwg354-ptr_mem {{template parameter is declared here}}
+  // cxx11-14-error@#cwg354-m3 {{null non-type template argument of type 'int *' does not match template parameter of type 'int cwg354::S::*'}}
+  //   cxx11-14-note@#cwg354-ptr_mem {{template parameter is declared here}}
+  // since-cxx17-error@#cwg354-m3 {{value of type 'int *' is not implicitly convertible to 'int cwg354::S::*'}}
 }
 
-struct dr355_S; // dr355: yes
-struct ::dr355_S {};
-// expected-warning@-1 {{extra qualification on member 'dr355_S'}}
-namespace dr355 { struct ::dr355_S s; }
+struct cwg355_S; // cwg355: yes
+struct ::cwg355_S {};
+// expected-warning@-1 {{extra qualification on member 'cwg355_S'}}
+namespace cwg355 { struct ::cwg355_S s; }
 
-// dr356: na
+// cwg356: na
 
-namespace dr357 { // dr357: yes
+namespace cwg357 { // cwg357: yes
   template<typename T> struct A {
-    void f() const; // #dr357-f
+    void f() const; // #cwg357-f
   };
   template<typename T> void A<T>::f() {}
   // expected-error@-1 {{out-of-line definition of 'f' does not match any declaration in 'A<T>'}}
-  //   expected-note@#dr357-f {{member declaration does not match because it is const qualified}}
+  //   expected-note@#cwg357-f {{member declaration does not match because it is const qualified}}
 
   struct B {
     template<typename T> void f();
   };
   template<typename T> void B::f() const {}
-  // expected-error@-1 {{out-of-line definition of 'f' does not match any declaration in 'dr357::B'}}
+  // expected-error@-1 {{out-of-line definition of 'f' does not match any declaration in 'cwg357::B'}}
 }
 
-namespace dr358 { // dr358: yes
-  extern "C" void dr358_f();
+namespace cwg358 { // cwg358: yes
+  extern "C" void cwg358_f();
   namespace N {
     int var;
-    extern "C" void dr358_f() { var = 10; }
+    extern "C" void cwg358_f() { var = 10; }
   }
 }
 
-namespace dr359 { // dr359: yes
+namespace cwg359 { // cwg359: yes
   // Note, the example in the DR is wrong; it doesn't contain an anonymous
   // union.
   struct E {
@@ -1049,7 +1049,7 @@ namespace dr359 { // dr359: yes
   };
 }
 
-namespace dr360 { // dr360: yes
+namespace cwg360 { // cwg360: yes
 struct A {
   int foo();
   int bar();
@@ -1060,28 +1060,28 @@ protected:
 
 struct B : A {
 private:
-  using A::foo; // #dr360-using-foo
+  using A::foo; // #cwg360-using-foo
 protected:
-  using A::bar; // #dr360-using-bar
+  using A::bar; // #cwg360-using-bar
 public:
   using A::baz;
 };
 
 int main() {
   int foo = B().foo();
-  // expected-error@-1 {{'foo' is a private member of 'dr360::B'}}
-  //   expected-note@#dr360-using-foo {{declared private here}}
+  // expected-error@-1 {{'foo' is a private member of 'cwg360::B'}}
+  //   expected-note@#cwg360-using-foo {{declared private here}}
   int bar = B().bar();
-  // expected-error@-1 {{'bar' is a protected member of 'dr360::B'}}
-  //   expected-note@#dr360-using-bar {{declared protected here}}
+  // expected-error@-1 {{'bar' is a protected member of 'cwg360::B'}}
+  //   expected-note@#cwg360-using-bar {{declared protected here}}
   int baz = B().baz();
 }
-} // namespace dr360
+} // namespace cwg360
 
-// dr362: na
-// dr363: na
+// cwg362: na
+// cwg363: na
 
-namespace dr364 { // dr364: yes
+namespace cwg364 { // cwg364: yes
   struct S {
     static void f(int);
     void f(char);
@@ -1094,11 +1094,11 @@ namespace dr364 { // dr364: yes
   }
 }
 
-// dr366: yes
+// cwg366: yes
 #if "foo" // expected-error {{invalid token at start of a preprocessor expression}} 
 #endif
 
-namespace dr367 { // dr367: yes
+namespace cwg367 { // cwg367: yes
   // FIXME: These diagnostics are terrible. Don't diagnose an ill-formed global
   // array as being a VLA!
   int a[true ? throw 0 : 4];
@@ -1116,30 +1116,30 @@ namespace dr367 { // dr367: yes
   // cxx98-error@-2 {{variable length array folded to constant array as an extension}}
 }
 
-namespace dr368 { // dr368: 3.6
-  template<typename T, T> struct S {}; // #dr368-S
+namespace cwg368 { // cwg368: 3.6
+  template<typename T, T> struct S {}; // #cwg368-S
   template<typename T> int f(S<T, T()> *);
   // expected-error@-1 {{template argument for non-type template parameter is treated as function type 'T ()'}}
-  //   expected-note@#dr368-S {{template parameter is declared here}}
-  template<typename T> int g(S<T, (T())> *); // #dr368-g
-  template<typename T> int g(S<T, true ? T() : T()> *); // #dr368-g-2
+  //   expected-note@#cwg368-S {{template parameter is declared here}}
+  template<typename T> int g(S<T, (T())> *); // #cwg368-g
+  template<typename T> int g(S<T, true ? T() : T()> *); // #cwg368-g-2
   struct X {};
-  int n = g<X>(0); // #dr368-g-call
-  // cxx98-17-error@#dr368-g-call {{no matching function for call to 'g'}}
-  //   cxx98-17-note@#dr368-g {{candidate template ignored: substitution failure [with T = X]: a non-type template parameter cannot have type 'X' before C++20}}
-  //   cxx98-17-note@#dr368-g-2 {{candidate template ignored: substitution failure [with T = X]: a non-type template parameter cannot have type 'X' before C++20}}
-  // cxx20-23-error@#dr368-g-call {{call to 'g' is ambiguous}}
-  //   cxx20-23-note@#dr368-g {{candidate function [with T = dr368::X]}}
-  //   cxx20-23-note@#dr368-g-2 {{candidate function [with T = dr368::X]}}
+  int n = g<X>(0); // #cwg368-g-call
+  // cxx98-17-error@#cwg368-g-call {{no matching function for call to 'g'}}
+  //   cxx98-17-note@#cwg368-g {{candidate template ignored: substitution failure [with T = X]: a non-type template parameter cannot have type 'X' before C++20}}
+  //   cxx98-17-note@#cwg368-g-2 {{candidate template ignored: substitution failure [with T = X]: a non-type template parameter cannot have type 'X' before C++20}}
+  // cxx20-23-error@#cwg368-g-call {{call to 'g' is ambiguous}}
+  //   cxx20-23-note@#cwg368-g {{candidate function [with T = cwg368::X]}}
+  //   cxx20-23-note@#cwg368-g-2 {{candidate function [with T = cwg368::X]}}
 }
 
-// dr370: na
+// cwg370: na
 
-namespace dr372 { // dr372: no
+namespace cwg372 { // cwg372: no
   namespace example1 {
     template<typename T> struct X {
     protected:
-      typedef T Type; // #dr372-ex1-Type
+      typedef T Type; // #cwg372-ex1-Type
     };
     template<typename T> struct Y {};
 
@@ -1150,55 +1150,55 @@ namespace dr372 { // dr372: no
              template<typename> class T2> struct Z1 :
       T1<T>,
       T2<typename T1<T>::Type> {};
-      // expected-error@-1 {{'Type' is a protected member of 'dr372::example1::X<int>'}}
-      //   expected-note@#dr372-z1 {{in instantiation of template class 'dr372::example1::Z1<int, dr372::example1::X, dr372::example1::Y>' requested here}}
-      //   expected-note@#dr372-ex1-Type {{declared protected here}}
+      // expected-error@-1 {{'Type' is a protected member of 'cwg372::example1::X<int>'}}
+      //   expected-note@#cwg372-z1 {{in instantiation of template class 'cwg372::example1::Z1<int, cwg372::example1::X, cwg372::example1::Y>' requested here}}
+      //   expected-note@#cwg372-ex1-Type {{declared protected here}}
 
     template<typename T,
              template<typename> class T1,
              template<typename> class T2> struct Z2 :
       T2<typename T1<T>::Type>,
-      // expected-error@-1 {{'Type' is a protected member of 'dr372::example1::X<int>'}}
-      //   expected-note@#dr372-z2 {{in instantiation of template class 'dr372::example1::Z2<int, dr372::example1::X, dr372::example1::Y>' requested here}}
-      //   expected-note@#dr372-ex1-Type {{declared protected here}}
+      // expected-error@-1 {{'Type' is a protected member of 'cwg372::example1::X<int>'}}
+      //   expected-note@#cwg372-z2 {{in instantiation of template class 'cwg372::example1::Z2<int, cwg372::example1::X, cwg372::example1::Y>' requested here}}
+      //   expected-note@#cwg372-ex1-Type {{declared protected here}}
       T1<T> {};
 
-    Z1<int, X, Y> z1; // #dr372-z1
-    Z2<int, X, Y> z2; // #dr372-z2
+    Z1<int, X, Y> z1; // #cwg372-z1
+    Z2<int, X, Y> z2; // #cwg372-z2
   }
 
   namespace example2 {
     struct X {
     private:
-      typedef int Type; // #dr372-ex2-Type
+      typedef int Type; // #cwg372-ex2-Type
     };
     template<typename T> struct A {
       typename T::Type t;
-      // expected-error@-1 {{'Type' is a private member of 'dr372::example2::X'}}
-      //   expected-note@#dr372-ax {{in instantiation of template class 'dr372::example2::A<dr372::example2::X>' requested here}}
-      //   expected-note@#dr372-ex2-Type {{declared private here}}
+      // expected-error@-1 {{'Type' is a private member of 'cwg372::example2::X'}}
+      //   expected-note@#cwg372-ax {{in instantiation of template class 'cwg372::example2::A<cwg372::example2::X>' requested here}}
+      //   expected-note@#cwg372-ex2-Type {{declared private here}}
     };
-    A<X> ax; // #dr372-ax
+    A<X> ax; // #cwg372-ax
   }
 
   namespace example3 {
     struct A {
     protected:
-      typedef int N; // #dr372-N
+      typedef int N; // #cwg372-N
     };
 
     template<typename T> struct B {};
     template<typename U> struct C : U, B<typename U::N> {};
-    // expected-error@-1 {{'N' is a protected member of 'dr372::example3::A'}}
-    //   expected-note@#dr372-x {{in instantiation of template class 'dr372::example3::C<dr372::example3::A>' requested here}}
-    //   expected-note@#dr372-N {{declared protected here}}
+    // expected-error@-1 {{'N' is a protected member of 'cwg372::example3::A'}}
+    //   expected-note@#cwg372-x {{in instantiation of template class 'cwg372::example3::C<cwg372::example3::A>' requested here}}
+    //   expected-note@#cwg372-N {{declared protected here}}
     template<typename U> struct D : B<typename U::N>, U {};
-    // expected-error@-1 {{'N' is a protected member of 'dr372::example3::A'}}
-    //   expected-note@#dr372-y {{in instantiation of template class 'dr372::example3::D<dr372::example3::A>' requested here}}
-    //   expected-note@#dr372-N {{declared protected here}}
+    // expected-error@-1 {{'N' is a protected member of 'cwg372::example3::A'}}
+    //   expected-note@#cwg372-y {{in instantiation of template class 'cwg372::example3::D<cwg372::example3::A>' requested here}}
+    //   expected-note@#cwg372-N {{declared protected here}}
 
-    C<A> x; // #dr372-x
-    D<A> y; // #dr372-y
+    C<A> x; // #cwg372-x
+    D<A> y; // #cwg372-y
   }
 
   namespace example4 {
@@ -1219,49 +1219,49 @@ namespace dr372 { // dr372: no
   namespace std_example {
     class A {
     protected:
-      struct B {}; // #dr372-B-std
+      struct B {}; // #cwg372-B-std
     };
     struct D : A::B, A {};
-    // expected-error@-1 {{'B' is a protected member of 'dr372::std_example::A'}}
-    //   expected-note@#dr372-B-std {{declared protected here}}
+    // expected-error@-1 {{'B' is a protected member of 'cwg372::std_example::A'}}
+    //   expected-note@#cwg372-B-std {{declared protected here}}
   }
 
   // FIXME: This is valid: deriving from A::B gives access to A::B!
   namespace badwolf {
     class A {
     protected:
-      struct B; // #dr372-B
+      struct B; // #cwg372-B
     };
     struct A::B : A {};
     struct C : A::B {};
-    // expected-error@-1 {{'B' is a protected member of 'dr372::badwolf::A'}}
-    //   expected-note@#dr372-B {{declared protected here}}
+    // expected-error@-1 {{'B' is a protected member of 'cwg372::badwolf::A'}}
+    //   expected-note@#cwg372-B {{declared protected here}}
   }
 }
 
-namespace dr373 { // dr373: 5
-  namespace X { int dr373; }
-  struct dr373 { // #dr373-struct
+namespace cwg373 { // cwg373: 5
+  namespace X { int cwg373; }
+  struct cwg373 { // #cwg373-struct
     void f() {
-      using namespace dr373::X;
-      int k = dr373;
-      // expected-error@-1 {{'dr373' does not refer to a value}}
-      //   expected-note@#dr373-struct {{declared here}}
-      namespace Y = dr373::X;
-      k = Y::dr373;
+      using namespace cwg373::X;
+      int k = cwg373;
+      // expected-error@-1 {{'cwg373' does not refer to a value}}
+      //   expected-note@#cwg373-struct {{declared here}}
+      namespace Y = cwg373::X;
+      k = Y::cwg373;
     }
   };
 
-  struct A { struct B {}; }; // #dr373-A 
+  struct A { struct B {}; }; // #cwg373-A 
   namespace X = A::B;
   // expected-error@-1 {{expected namespace name}}
-  //   expected-note@#dr373-A {{'A' declared here}}
+  //   expected-note@#cwg373-A {{'A' declared here}}
   using namespace A::B;
   // expected-error@-1 {{expected namespace name}}
-  //   expected-note@#dr373-A {{'A' declared here}}
+  //   expected-note@#cwg373-A {{'A' declared here}}
 }
 
-namespace dr374 { // dr374: 7
+namespace cwg374 { // cwg374: 7
                   // NB 2.9 c++11
   namespace N {
     template<typename T> void f();
@@ -1272,10 +1272,10 @@ namespace dr374 { // dr374: 7
   template<> struct N::A<int> {};
 }
 
-// dr375: dup 345
-// dr376: na
+// cwg375: dup 345
+// cwg376: na
 
-namespace dr377 { // dr377: yes
+namespace cwg377 { // cwg377: yes
   enum E {
   // expected-error@-1 {{enumeration values exceed range of largest integer}}
     a = -__LONG_LONG_MAX__ - 1,
@@ -1286,10 +1286,10 @@ namespace dr377 { // dr377: yes
   };
 }
 
-// dr378: dup 276
-// dr379: na
+// cwg378: dup 276
+// cwg379: na
 
-namespace dr381 { // dr381: yes
+namespace cwg381 { // cwg381: yes
   struct A {
     int a;
   };
@@ -1301,26 +1301,26 @@ namespace dr381 { // dr381: yes
   void f() {
     E e;
     e.B::a = 0;
-    /* expected-error@-1 {{ambiguous conversion from derived class 'E' to base class 'dr381::B':
-    struct dr381::E -> C -> B
-    struct dr381::E -> D -> B}} */
+    /* expected-error@-1 {{ambiguous conversion from derived class 'E' to base class 'cwg381::B':
+    struct cwg381::E -> C -> B
+    struct cwg381::E -> D -> B}} */
     F f;
     f.A::a = 1;
   }
 }
 
-namespace dr382 { // dr382: yes c++11
+namespace cwg382 { // cwg382: yes c++11
   // FIXME: Should we allow this in C++98 mode?
   struct A { typedef int T; };
   typename A::T t;
   // cxx98-error@-1 {{'typename' occurs outside of a template}}
-  typename dr382::A a;
+  typename cwg382::A a;
   // cxx98-error@-1 {{'typename' occurs outside of a template}}
   typename A b;
   // expected-error@-1 {{expected a qualified name after 'typename'}}
 }
 
-namespace dr383 { // dr383: yes
+namespace cwg383 { // cwg383: yes
   struct A { A &operator=(const A&); };
   struct B { ~B(); };
   union C { C &operator=(const C&); };
@@ -1328,7 +1328,7 @@ namespace dr383 { // dr383: yes
   int check[(__is_pod(A) || __is_pod(B) || __is_pod(C) || __is_pod(D)) ? -1 : 1];
 }
 
-namespace dr384 { // dr384: yes
+namespace cwg384 { // cwg384: yes
   namespace N1 {
     template<typename T> struct Base {};
     template<typename T> struct X {
@@ -1350,35 +1350,35 @@ namespace dr384 { // dr384: yes
   }
 }
 
-namespace dr385 { // dr385: 2.8
+namespace cwg385 { // cwg385: 2.8
   struct A { protected: void f(); };
   struct B : A { using A::f; };
   struct C : A { void g(B b) { b.f(); } };
   void h(B b) { b.f(); }
 
-  struct D { int n; }; // #dr385-n
-  struct E : protected D {}; // #dr385-E
+  struct D { int n; }; // #cwg385-n
+  struct E : protected D {}; // #cwg385-E
   struct F : E { friend int i(E); };
   int i(E e) { return e.n; }
-  // expected-error@-1 {{'n' is a protected member of 'dr385::D'}}
-  //   expected-note@#dr385-E {{constrained by protected inheritance here}}
-  //   expected-note@#dr385-n {{member is declared here}}
+  // expected-error@-1 {{'n' is a protected member of 'cwg385::D'}}
+  //   expected-note@#cwg385-E {{constrained by protected inheritance here}}
+  //   expected-note@#cwg385-n {{member is declared here}}
 }
 
-namespace dr387 { // dr387: 2.8
+namespace cwg387 { // cwg387: 2.8
   namespace old {
     template<typename T> class number {
-      number(int); // #dr387-number-ctor
+      number(int); // #cwg387-number-ctor
       friend number gcd(number &x, number &y) {}
     };
 
     void g() {
       number<double> a(3);
-      // expected-error@-1 {{calling a private constructor of class 'dr387::old::number<double>'}}
-      //   expected-note@#dr387-number-ctor {{implicitly declared private here}}
+      // expected-error@-1 {{calling a private constructor of class 'cwg387::old::number<double>'}}
+      //   expected-note@#cwg387-number-ctor {{implicitly declared private here}}
       number<double> b(4);
-      // expected-error@-1 {{calling a private constructor of class 'dr387::old::number<double>'}}
-      //   expected-note@#dr387-number-ctor {{implicitly declared private here}}
+      // expected-error@-1 {{calling a private constructor of class 'cwg387::old::number<double>'}}
+      //   expected-note@#cwg387-number-ctor {{implicitly declared private here}}
       a = gcd(a, b);
       b = gcd(3, 4);
       // expected-error@-1 {{use of undeclared identifier 'gcd'}}
@@ -1401,14 +1401,14 @@ namespace dr387 { // dr387: 2.8
   }
 }
 
-// FIXME: dr388 needs libc++abi test
+// FIXME: cwg388 needs libc++abi test
 
-namespace dr389 { // dr389: no
+namespace cwg389 { // cwg389: no
   struct S {
     typedef struct {} A;
     typedef enum {} B;
-    typedef struct {} const C; // #dr389-C
-    typedef enum {} const D; // #dr389-D
+    typedef struct {} const C; // #cwg389-C
+    typedef enum {} const D; // #cwg389-D
   };
   template<typename> struct T {};
 
@@ -1422,8 +1422,8 @@ namespace dr389 { // dr389: no
   typedef void (*WithLinkage8)(WithLinkage2 WithLinkage1::*, WithLinkage5 *);
   typedef T<WithLinkage5> WithLinkage9;
 
-  typedef struct {} *WithoutLinkage1; // #dr389-no-link-1
-  typedef enum {} const WithoutLinkage2; // #dr389-no-link-2
+  typedef struct {} *WithoutLinkage1; // #cwg389-no-link-1
+  typedef enum {} const WithoutLinkage2; // #cwg389-no-link-2
   // These two types don't have linkage even though they are externally visible
   // and the ODR requires them to be merged across TUs.
   typedef S::C WithoutLinkage3;
@@ -1451,19 +1451,19 @@ namespace dr389 { // dr389: no
 
   typedef T<WithoutLinkage1> BadArg1;
   // expected-error@-1 {{template argument uses unnamed type}}
-  //   expected-note@#dr389-no-link-1 {{unnamed type used in template argument was declared here}}
+  //   expected-note@#cwg389-no-link-1 {{unnamed type used in template argument was declared here}}
   typedef T<WithoutLinkage2> BadArg2;
   // expected-error@-1 {{template argument uses unnamed type}}
-  //   expected-note@#dr389-no-link-2 {{unnamed type used in template argument was declared here}}
+  //   expected-note@#cwg389-no-link-2 {{unnamed type used in template argument was declared here}}
   typedef T<WithoutLinkage3> BadArg3;
   // expected-error@-1 {{template argument uses unnamed type}}
-  //   expected-note@#dr389-C {{unnamed type used in template argument was declared here}}
+  //   expected-note@#cwg389-C {{unnamed type used in template argument was declared here}}
   typedef T<WithoutLinkage4> BadArg4;
   // expected-error@-1 {{template argument uses unnamed type}}
-  //   expected-note@#dr389-D {{unnamed type used in template argument was declared here}}
+  //   expected-note@#cwg389-D {{unnamed type used in template argument was declared here}}
   typedef T<WithoutLinkage5> BadArg5;
   // expected-error@-1 {{template argument uses unnamed type}}
-  //   expected-note@#dr389-C {{unnamed type used in template argument was declared here}}
+  //   expected-note@#cwg389-C {{unnamed type used in template argument was declared here}}
 #endif
 
   extern WithLinkage1 withLinkage1;
@@ -1488,11 +1488,11 @@ namespace dr389 { // dr389: no
 
   // OK, extern "C".
   extern "C" {
-    extern WithoutLinkage1 dr389_withoutLinkage1;
-    extern WithoutLinkage2 dr389_withoutLinkage2;
-    extern WithoutLinkage3 dr389_withoutLinkage3;
-    extern WithoutLinkage4 dr389_withoutLinkage4;
-    extern WithoutLinkage5 dr389_withoutLinkage5;
+    extern WithoutLinkage1 cwg389_withoutLinkage1;
+    extern WithoutLinkage2 cwg389_withoutLinkage2;
+    extern WithoutLinkage3 cwg389_withoutLinkage3;
+    extern WithoutLinkage4 cwg389_withoutLinkage4;
+    extern WithoutLinkage5 cwg389_withoutLinkage5;
   }
 
   // OK, defined.
@@ -1511,9 +1511,9 @@ namespace dr389 { // dr389: no
     use(&withoutLinkage1); use(&withoutLinkage2); use(&withoutLinkage3);
     use(&withoutLinkage4); use(&withoutLinkage5);
 
-    use(&dr389_withoutLinkage1); use(&dr389_withoutLinkage2);
-    use(&dr389_withoutLinkage3); use(&dr389_withoutLinkage4);
-    use(&dr389_withoutLinkage5);
+    use(&cwg389_withoutLinkage1); use(&cwg389_withoutLinkage2);
+    use(&cwg389_withoutLinkage3); use(&cwg389_withoutLinkage4);
+    use(&cwg389_withoutLinkage5);
 
     use(&withoutLinkageDef1); use(&withoutLinkageDef2); use(&withoutLinkageDef3);
     use(&withoutLinkageDef4); use(&withoutLinkageDef5);
@@ -1525,39 +1525,39 @@ namespace dr389 { // dr389: no
   }
 }
 
-namespace dr390 { // dr390: 3.3
+namespace cwg390 { // cwg390: 3.3
   template<typename T>
   struct A {
     A() { f(); }
-    // expected-warning@-1 {{call to pure virtual member function 'f' has undefined behavior; overrides of 'f' in subclasses are not available in the constructor of 'dr390::A<int>'}}
-    //   expected-note@#dr390-A-int {{in instantiation of member function 'dr390::A<int>::A' requested here}}
-    //   expected-note@#dr390-f {{'f' declared here}}
-    virtual void f() = 0; // #dr390-f
+    // expected-warning@-1 {{call to pure virtual member function 'f' has undefined behavior; overrides of 'f' in subclasses are not available in the constructor of 'cwg390::A<int>'}}
+    //   expected-note@#cwg390-A-int {{in instantiation of member function 'cwg390::A<int>::A' requested here}}
+    //   expected-note@#cwg390-f {{'f' declared here}}
+    virtual void f() = 0; // #cwg390-f
     virtual ~A() = 0;
   };
   template<typename T> A<T>::~A() { T::error; }
   // expected-error@-1 {{type 'int' cannot be used prior to '::' because it has no members}}
-  //   expected-note@#dr390-A-int {{in instantiation of member function 'dr390::A<int>::~A' requested here}}
+  //   expected-note@#cwg390-A-int {{in instantiation of member function 'cwg390::A<int>::~A' requested here}}
   template<typename T> void A<T>::f() { T::error; } // ok, not odr-used
-  struct B : A<int> { // #dr390-A-int
+  struct B : A<int> { // #cwg390-A-int
     void f() {}
   } b;
 }
 
-namespace dr391 { // dr391: 2.8 c++11
+namespace cwg391 { // cwg391: 2.8 c++11
   // FIXME: Should this apply to C++98 too?
-  class A { A(const A&); }; // #dr391-A
+  class A { A(const A&); }; // #cwg391-A
   A fa();
   const A &a = fa();
-  // cxx98-error@-1 {{C++98 requires an accessible copy constructor for class 'dr391::A' when binding a reference to a temporary; was private}}
-  //   cxx98-note@#dr391-A {{implicitly declared private here}}
+  // cxx98-error@-1 {{C++98 requires an accessible copy constructor for class 'cwg391::A' when binding a reference to a temporary; was private}}
+  //   cxx98-note@#cwg391-A {{implicitly declared private here}}
 
-  struct B { B(const B&) = delete; }; // #dr391-B
+  struct B { B(const B&) = delete; }; // #cwg391-B
   // cxx98-error@-1 {{deleted function definitions are a C++11 extension}}
   B fb();
   const B &b = fb();
   // cxx98-error@-1 {{copying variable of type 'B' invokes deleted constructor}}
-  //   cxx98-note@#dr391-B {{'B' has been explicitly marked deleted here}}
+  //   cxx98-note@#cwg391-B {{'B' has been explicitly marked deleted here}}
 
   template<typename T>
   struct C {
@@ -1567,9 +1567,9 @@ namespace dr391 { // dr391: 2.8 c++11
   const C<int> &c = fc();
 }
 
-// dr392 is in dr392.cpp
+// cwg392 is in cwg392.cpp
 
-namespace dr393 { // dr393: 2.7
+namespace cwg393 { // cwg393: 2.7
 template <typename T>
 struct S {};
 
@@ -1585,11 +1585,11 @@ void h() {
   T (*p) [] = (U(*)[])0;
   sp_assert_convertible<T[]>( (U(*)[])0 );
 }
-} // namespace dr393
+} // namespace cwg393
 
-// dr394: na
+// cwg394: na
 
-namespace dr395 { // dr395: 3.0
+namespace cwg395 { // cwg395: 3.0
   struct S {
     template <typename T, int N>(&operator T())[N];
     // expected-error@-1 {{cannot specify any part of a return type in the declaration of a conversion function}}
@@ -1613,13 +1613,13 @@ namespace dr395 { // dr395: 3.0
     };
 
     template <class T, class U>
-    operator typename ptr_mem_fun_t<T, U>::type() const { // #dr395-conv-func
+    operator typename ptr_mem_fun_t<T, U>::type() const { // #cwg395-conv-func
       return 0;
     }
   } null1;
   int (S::*p)() = null1;
-  // expected-error@-1 {{no viable conversion from 'struct null1_t' to 'int (dr395::S::*)()'}}
-  //   expected-note@#dr395-conv-func {{candidate template ignored: couldn't infer template argument 'T'}}
+  // expected-error@-1 {{no viable conversion from 'struct null1_t' to 'int (cwg395::S::*)()'}}
+  //   expected-note@#cwg395-conv-func {{candidate template ignored: couldn't infer template argument 'T'}}
 
   template <typename T> using id = T;
   // cxx98-error@-1 {{alias declarations are a C++11 extension}}
@@ -1637,22 +1637,22 @@ namespace dr395 { // dr395: 3.0
   int (S::*q)() = null2;
 }
 
-namespace dr396 { // dr396: yes
+namespace cwg396 { // cwg396: yes
   void f() {
     auto int a();
     // since-cxx11-error@-1 {{'auto' storage class specifier is not permitted in C++11, and will not be supported in future releases}}
     // expected-error@-2 {{illegal storage class on function}}
-    int (i); // #dr396-i
+    int (i); // #cwg396-i
     auto int (i);
     // since-cxx11-error@-1 {{'auto' storage class specifier is not permitted in C++11, and will not be supported in future releases}}
     // expected-error@-2 {{redefinition of 'i'}}
-    //   expected-note@#dr396-i {{previous definition is here}}
+    //   expected-note@#cwg396-i {{previous definition is here}}
   }
 }
 
-// dr397: sup 1823
+// cwg397: sup 1823
 
-namespace dr398 { // dr398: yes
+namespace cwg398 { // cwg398: yes
   namespace example1 {
     struct S {
       static int const I = 42;
@@ -1666,9 +1666,9 @@ namespace dr398 { // dr398: yes
   namespace example2 {
     template <int I> struct X {};
     template <template <class T> class> struct Z {};
-    template <class T> void f(typename T::Y *) {} // #dr398-f
-    template <class T> void g(X<T::N> *) {} // #dr398-g
-    template <class T> void h(Z<T::template TT> *) {} // #dr398-h
+    template <class T> void f(typename T::Y *) {} // #cwg398-f
+    template <class T> void g(X<T::N> *) {} // #cwg398-g
+    template <class T> void h(Z<T::template TT> *) {} // #cwg398-h
     struct A {};
     struct B {
       int Y;
@@ -1683,23 +1683,23 @@ namespace dr398 { // dr398: yes
     void test() {
       f<A>(0);
       // expected-error@-1 {{no matching function for call to 'f'}}
-      //   expected-note@#dr398-f {{candidate template ignored: substitution failure [with T = A]: no type named 'Y' in 'dr398::example2::A'}}
+      //   expected-note@#cwg398-f {{candidate template ignored: substitution failure [with T = A]: no type named 'Y' in 'cwg398::example2::A'}}
       f<B>(0);
       // expected-error@-1 {{no matching function for call to 'f'}}
-      //   expected-note@#dr398-f {{candidate template ignored: substitution failure [with T = B]: typename specifier refers to non-type member 'Y' in 'dr398::example2::B'}}
+      //   expected-note@#cwg398-f {{candidate template ignored: substitution failure [with T = B]: typename specifier refers to non-type member 'Y' in 'cwg398::example2::B'}}
       g<C>(0);
       // expected-error@-1 {{no matching function for call to 'g'}}
-      //   expected-note@#dr398-g {{candidate template ignored: substitution failure [with T = C]: missing 'typename' prior to dependent type name 'C::N'}}
+      //   expected-note@#cwg398-g {{candidate template ignored: substitution failure [with T = C]: missing 'typename' prior to dependent type name 'C::N'}}
       h<D>(0);
       // expected-error@-1 {{no matching function for call to 'h'}}
-      //   expected-note@#dr398-h {{candidate template ignored: substitution failure [with T = D]: 'TT' following the 'template' keyword does not refer to a template}}
+      //   expected-note@#cwg398-h {{candidate template ignored: substitution failure [with T = D]: 'TT' following the 'template' keyword does not refer to a template}}
     }
   }
 }
 
-namespace dr399 { // dr399: 11
-                  // NB: reuse dr244 test
-  struct B {}; // #dr399-B
+namespace cwg399 { // cwg399: 11
+                  // NB: reuse cwg244 test
+  struct B {}; // #cwg399-B
   struct D : B {};
 
   D D_object;
@@ -1708,18 +1708,18 @@ namespace dr399 { // dr399: 11
 
   void f() {
     D_object.~B();
-    // expected-error@-1 {{destructor type 'dr399::B' in object destruction expression does not match the type 'D' of the object being destroyed}}
-    //   expected-note@#dr399-B {{type 'dr399::B' found by destructor name lookup}}
+    // expected-error@-1 {{destructor type 'cwg399::B' in object destruction expression does not match the type 'D' of the object being destroyed}}
+    //   expected-note@#cwg399-B {{type 'cwg399::B' found by destructor name lookup}}
     D_object.B::~B();
     D_object.D::~B(); // FIXME: Missing diagnostic for this.
     B_ptr->~B();
     B_ptr->~B_alias();
     B_ptr->B_alias::~B();
     B_ptr->B_alias::~B_alias();
-    B_ptr->dr399::~B();
-    // expected-error@-1 {{qualified member access refers to a member in namespace 'dr399'}}
-    B_ptr->dr399::~B_alias();
-    // expected-error@-1 {{qualified member access refers to a member in namespace 'dr399'}}
+    B_ptr->cwg399::~B();
+    // expected-error@-1 {{qualified member access refers to a member in namespace 'cwg399'}}
+    B_ptr->cwg399::~B_alias();
+    // expected-error@-1 {{qualified member access refers to a member in namespace 'cwg399'}}
   }
 
   template<typename T, typename U>
@@ -1731,10 +1731,10 @@ namespace dr399 { // dr399: 11
     B_ptr->~B_alias();
     B_ptr->B_alias::~B();
     B_ptr->B_alias::~B_alias();
-    B_ptr->dr399::~B();
-    // expected-error@-1 {{'dr399' does not refer to a type name in pseudo-destructor expression; expected the name of type 'T'}}
-    B_ptr->dr399::~B_alias();
-    // expected-error@-1 {{'dr399' does not refer to a type name in pseudo-destructor expression; expected the name of type 'T'}}
+    B_ptr->cwg399::~B();
+    // expected-error@-1 {{'cwg399' does not refer to a type name in pseudo-destructor expression; expected the name of type 'T'}}
+    B_ptr->cwg399::~B_alias();
+    // expected-error@-1 {{'cwg399' does not refer to a type name in pseudo-destructor expression; expected the name of type 'T'}}
   }
   template void f<B, D>(B*, D);
 
@@ -1743,7 +1743,7 @@ namespace dr399 { // dr399: 11
     typedef E<int> F;
   }
   void g(N::F f) {
-    typedef N::F G; // #dr399-G
+    typedef N::F G; // #cwg399-G
     f.~G();
     f.G::~E();
     // expected-error@-1 {{ISO C++ requires the name after '::~' to be found in the same scope as the name before '::~'}}
@@ -1762,7 +1762,7 @@ namespace dr399 { // dr399: 11
     // Rejecting this seems correct, but most compilers accept, so we do also.
     f.N::F::~G();
     // expected-error@-1 {{qualified destructor name only found in lexical scope; omit the qualifier to find this type name by unqualified lookup}}
-    //   expected-note@#dr399-G {{type 'G' (aka 'E<int>') found by destructor name lookup}}
+    //   expected-note@#cwg399-G {{type 'G' (aka 'E<int>') found by destructor name lookup}}
   }
 
   // Bizarrely, compilers perform lookup in the scope for qualified destructor
@@ -1774,17 +1774,17 @@ namespace dr399 { // dr399: 11
     }
     template <typename U> void f(typename N::S<U>::Inner *p) {
       typedef typename N::S<U>::Inner T;
-      p->::dr399::QualifiedLookupInScope::N::S<U>::Inner::~T();
-      // expected-error@-1 {{no type named 'T' in 'dr399::QualifiedLookupInScope::N::S<int>'}}
-      //   expected-note@#dr399-f {{in instantiation of function template specialization 'dr399::QualifiedLookupInScope::f<int>' requested here}}
+      p->::cwg399::QualifiedLookupInScope::N::S<U>::Inner::~T();
+      // expected-error@-1 {{no type named 'T' in 'cwg399::QualifiedLookupInScope::N::S<int>'}}
+      //   expected-note@#cwg399-f {{in instantiation of function template specialization 'cwg399::QualifiedLookupInScope::f<int>' requested here}}
     }
-    template void f<int>(N::S<int>::Inner *); // #dr399-f
+    template void f<int>(N::S<int>::Inner *); // #cwg399-f
 
     template <typename U> void g(U *p) {
       typedef U T;
       p->T::~T();
       p->U::~T();
-      p->::dr399::QualifiedLookupInScope::N::S<int>::Inner::~T();
+      p->::cwg399::QualifiedLookupInScope::N::S<int>::Inner::~T();
       // expected-error@-1 {{'T' does not refer to a type name in pseudo-destructor expression; expected the name of type 'U'}}
     }
     template void g(N::S<int>::Inner *);
diff --git a/clang/test/CXX/drs/dr4xx.cpp b/clang/test/CXX/drs/dr4xx.cpp
index 9b7a9777202f..611b79147078 100644
--- a/clang/test/CXX/drs/dr4xx.cpp
+++ b/clang/test/CXX/drs/dr4xx.cpp
@@ -11,56 +11,56 @@ __extension__ typedef __SIZE_TYPE__ size_t;
 
 namespace std { struct type_info; }
 
-namespace dr400 { // dr400: yes
-  struct A { int a; struct a {}; }; // #dr400-A
-  struct B { int a; struct a {}; }; // #dr400-B
+namespace cwg400 { // cwg400: yes
+  struct A { int a; struct a {}; }; // #cwg400-A
+  struct B { int a; struct a {}; }; // #cwg400-B
   struct C : A, B { using A::a; struct a b; };
   struct D : A, B { 
     using A::a;
     // FIXME: we should issue a single diagnostic
-    using B::a; // #dr400-using-B-a
-    // expected-error@#dr400-using-B-a {{target of using declaration conflicts with declaration already in scope}}
-    //   expected-note@#dr400-B {{target of using declaration}}
-    //   expected-note@#dr400-A {{conflicting declaration}}
-    // expected-error@#dr400-using-B-a {{target of using declaration conflicts with declaration already in scope}}
-    //   expected-note@#dr400-B {{target of using declaration}}
-    //   expected-note@#dr400-A {{conflicting declaration}}
+    using B::a; // #cwg400-using-B-a
+    // expected-error@#cwg400-using-B-a {{target of using declaration conflicts with declaration already in scope}}
+    //   expected-note@#cwg400-B {{target of using declaration}}
+    //   expected-note@#cwg400-A {{conflicting declaration}}
+    // expected-error@#cwg400-using-B-a {{target of using declaration conflicts with declaration already in scope}}
+    //   expected-note@#cwg400-B {{target of using declaration}}
+    //   expected-note@#cwg400-A {{conflicting declaration}}
     struct a b;
   };
   struct E : A, B { struct a b; };
   // expected-error@-1 {{member 'a' found in multiple base classes of different types}}
-  //   expected-note@#dr400-A {{member type 'dr400::A::a' found by ambiguous name lookup}}
-  //   expected-note@#dr400-B {{member type 'dr400::B::a' found by ambiguous name lookup}}
-}
-
-namespace dr401 { // dr401: 2.8
-  template<class T, class U = typename T::type> class A : public T {}; // #dr401-A
-  // expected-error@#dr401-A {{'type' is a private member of 'dr401::C'}}
-  //   expected-note@#dr402-friend-A-C {{in instantiation of default argument for 'A<C>' required here}}
-  //   expected-note@#dr402-C-type {{implicitly declared private here}}
-  // expected-error@#dr401-A {{'type' is a protected member of 'dr401::B'}}
-  //   expected-note@#dr402-b {{in instantiation of default argument for 'A<B>' required here}}
-  //   expected-note@#dr402-B-type {{declared protected here}}
-  // expected-error@#dr401-A {{'type' is a private member of 'dr401::D'}}
-  //   expected-note@#dr402-d {{in instantiation of default argument for 'A<D>' required here}}
-  //   expected-note@#dr402-D-type {{implicitly declared private here}}
+  //   expected-note@#cwg400-A {{member type 'cwg400::A::a' found by ambiguous name lookup}}
+  //   expected-note@#cwg400-B {{member type 'cwg400::B::a' found by ambiguous name lookup}}
+}
+
+namespace cwg401 { // cwg401: 2.8
+  template<class T, class U = typename T::type> class A : public T {}; // #cwg401-A
+  // expected-error@#cwg401-A {{'type' is a private member of 'cwg401::C'}}
+  //   expected-note@#cwg402-friend-A-C {{in instantiation of default argument for 'A<C>' required here}}
+  //   expected-note@#cwg402-C-type {{implicitly declared private here}}
+  // expected-error@#cwg401-A {{'type' is a protected member of 'cwg401::B'}}
+  //   expected-note@#cwg402-b {{in instantiation of default argument for 'A<B>' required here}}
+  //   expected-note@#cwg402-B-type {{declared protected here}}
+  // expected-error@#cwg401-A {{'type' is a private member of 'cwg401::D'}}
+  //   expected-note@#cwg402-d {{in instantiation of default argument for 'A<D>' required here}}
+  //   expected-note@#cwg402-D-type {{implicitly declared private here}}
   class B {
   protected:
-    typedef int type; // #dr402-B-type
+    typedef int type; // #cwg402-B-type
   };
 
   class C {
-    typedef int type; // #dr402-C-type
-    friend class A<C>; // #dr402-friend-A-C
+    typedef int type; // #cwg402-C-type
+    friend class A<C>; // #cwg402-friend-A-C
   };
 
   class D {
-    typedef int type; // #dr402-D-type
+    typedef int type; // #cwg402-D-type
     friend class A<D, int>;
   };
 
-  A<B> *b; // #dr402-b
-  A<D> *d; // #dr402-d
+  A<B> *b; // #cwg402-b
+  A<D> *d; // #cwg402-d
 
   struct E {
     template<class T, class U = typename T::type> class A : public T {};
@@ -73,18 +73,18 @@ namespace dr401 { // dr401: 2.8
 
   // FIXME: Why do we get different diagnostics in C++11 onwards here? We seem
   // to not treat the default template argument as a SFINAE context in C++98.
-  template<class T, class U = typename T::type> void f(T) {} // #dr402-f
+  template<class T, class U = typename T::type> void f(T) {} // #cwg402-f
   // cxx98-error@-1 {{default template arguments for a function template are a C++11 extension}}
-  // cxx98-error@-2 {{'type' is a protected member of 'dr401::B'}}
+  // cxx98-error@-2 {{'type' is a protected member of 'cwg401::B'}}
   //   cxx98-note@-3 {{in instantiation of default argument for 'f<B>' required here}}
-  //   cxx98-note@#dr402-f-b {{while substituting deduced template arguments into function template 'f' [with T = B, U = (no value)]}}
-  //   cxx98-note@#dr402-B-type {{declared protected here}}
-  void g(B b) { f(b); } // #dr402-f-b
+  //   cxx98-note@#cwg402-f-b {{while substituting deduced template arguments into function template 'f' [with T = B, U = (no value)]}}
+  //   cxx98-note@#cwg402-B-type {{declared protected here}}
+  void g(B b) { f(b); } // #cwg402-f-b
   // since-cxx11-error@-1 {{no matching function for call to 'f'}}
-  //   since-cxx11-note@#dr402-f {{candidate template ignored: substitution failure [with T = B, U = typename B::type]: 'type' is a protected member of 'dr401::B'}}
+  //   since-cxx11-note@#cwg402-f {{candidate template ignored: substitution failure [with T = B, U = typename B::type]: 'type' is a protected member of 'cwg401::B'}}
 }
 
-namespace dr403 { // dr403: yes
+namespace cwg403 { // cwg403: yes
   namespace A {
     struct S {};
     int f(void*);
@@ -98,10 +98,10 @@ namespace dr403 { // dr403: yes
                 // template-id as we can make it.
 }
 
-// dr404: na
+// cwg404: na
 // (NB: also sup 594)
 
-namespace dr405 { // dr405: yes
+namespace cwg405 { // cwg405: yes
                   // NB: also dup 218
   namespace A {
     struct S {};
@@ -145,7 +145,7 @@ namespace dr405 { // dr405: yes
   // expected-error@-1 {{use of undeclared identifier 'f'}}
 }
 
-namespace dr406 { // dr406: 2.9
+namespace cwg406 { // cwg406: 2.9
   typedef struct {
     static int n;
     // expected-error@-1 {{static data member 'n' not allowed in anonymous struct}}
@@ -156,35 +156,35 @@ namespace dr406 { // dr406: 2.9
   } B;
 }
 
-namespace dr407 { // dr407: 3.8
-                  // NB: reused by dr1894 and dr2199
+namespace cwg407 { // cwg407: 3.8
+                  // NB: reused by cwg1894 and cwg2199
   struct S;
   typedef struct S S;
   void f() {
     struct S *p;
     {
-      typedef struct S S; // #dr407-typedef-S
+      typedef struct S S; // #cwg407-typedef-S
       struct S *p;
       // expected-error@-1 {{typedef 'S' cannot be referenced with a struct specifier}}
-      //   expected-note@#dr407-typedef-S {{declared here}}
+      //   expected-note@#cwg407-typedef-S {{declared here}}
     }
   }
   struct S {};
 
   namespace UsingDir {
     namespace A {
-      struct S {}; // #dr407-A-S
+      struct S {}; // #cwg407-A-S
     }
     namespace B {
-      typedef int S; // #dr407-B-S
+      typedef int S; // #cwg407-B-S
     }
     namespace C {
       using namespace A;
       using namespace B;
       struct S s;
       // expected-error@-1 {{ambiguous}}
-      //   expected-note@#dr407-A-S {{candidate found by name lookup is 'dr407::UsingDir::A::S'}}
-      //   expected-note@#dr407-B-S {{candidate found by name lookup is 'dr407::UsingDir::B::S'}}
+      //   expected-note@#cwg407-A-S {{candidate found by name lookup is 'cwg407::UsingDir::A::S'}}
+      //   expected-note@#cwg407-B-S {{candidate found by name lookup is 'cwg407::UsingDir::B::S'}}
     }
     namespace D {
       using A::S;
@@ -193,7 +193,7 @@ namespace dr407 { // dr407: 3.8
     }
     namespace E {
       // The standard doesn't say whether this is valid. We interpret
-      // DR407 as meaning "if lookup finds both a tag and a typedef with the
+      // CWG407 as meaning "if lookup finds both a tag and a typedef with the
       // same type, then it's OK in an elaborated-type-specifier".
       typedef A::S S;
       using A::S;
@@ -216,7 +216,7 @@ namespace dr407 { // dr407: 3.8
   }
 }
 
-namespace dr408 { // dr408: 3.4
+namespace cwg408 { // cwg408: 3.4
   template<int N> void g() { int arr[N != 1 ? 1 : -1]; }
   template<> void g<2>() { }
 
@@ -245,7 +245,7 @@ namespace dr408 { // dr408: 3.4
   template void R<int>::f();
 }
 
-namespace dr409 { // dr409: yes
+namespace cwg409 { // cwg409: yes
   template<typename T> struct A {
     typedef int B;
     B b1;
@@ -256,7 +256,7 @@ namespace dr409 { // dr409: yes
   };
 }
 
-namespace dr410 { // dr410: no
+namespace cwg410 { // cwg410: no
   template<class T> void f(T);
   void g(int);
   namespace M {
@@ -269,7 +269,7 @@ namespace dr410 { // dr410: no
       template<class T> void i(T);
       friend void i<>(int);
     private:
-      static void z(); // #dr410-z
+      static void z(); // #cwg410-z
     };
 
     template<> void h(int) { A::z(); }
@@ -279,13 +279,13 @@ namespace dr410 { // dr410: no
   }
   template<> void f(int) { M::A::z(); }
   void g(int) { M::A::z(); }
-  // expected-error@-1 {{'z' is a private member of 'dr410::M::A'}}
-  //   expected-note@#dr410-z {{declared private here}}
+  // expected-error@-1 {{'z' is a private member of 'cwg410::M::A'}}
+  //   expected-note@#cwg410-z {{declared private here}}
 }
 
-// dr412 is in dr412.cpp
+// cwg412 is in cwg412.cpp
 
-namespace dr413 { // dr413: yes
+namespace cwg413 { // cwg413: yes
   struct S {
     int a;
     int : 17;
@@ -295,7 +295,7 @@ namespace dr413 { // dr413: yes
   // expected-error@-1 {{excess elements in struct initializer}}
 
   struct E {};
-  struct T { // #dr413-T
+  struct T { // #cwg413-T
     int a;
     E e;
     int b;
@@ -303,10 +303,10 @@ namespace dr413 { // dr413: yes
   T t1 = { 1, {}, 2 };
   T t2 = { 1, 2 };
   // expected-error@-1 {{initializer for aggregate with no elements requires explicit braces}}
-  //   expected-note@#dr413-T {{'dr413::T' declared here}}
+  //   expected-note@#cwg413-T {{'cwg413::T' declared here}}
 }
 
-namespace dr414 { // dr414: dup 305
+namespace cwg414 { // cwg414: dup 305
   struct X {};
   void f() {
     X x;
@@ -315,13 +315,13 @@ namespace dr414 { // dr414: dup 305
   }
 }
 
-namespace dr415 { // dr415: yes
+namespace cwg415 { // cwg415: yes
   template<typename T> void f(T, ...) { T::error; }
   void f(int, int);
   void g() { f(0, 0); } // ok
 }
 
-namespace dr416 { // dr416: yes
+namespace cwg416 { // cwg416: yes
   extern struct A a;
   int &operator+(const A&, const A&);
   int &k = a + a;
@@ -329,14 +329,14 @@ namespace dr416 { // dr416: yes
   float &f = a + a;
 }
 
-namespace dr417 { // dr417: no
+namespace cwg417 { // cwg417: no
   struct A;
-  struct dr417::A {};
+  struct cwg417::A {};
   // expected-warning@-1 {{extra qualification on member 'A'}}
   struct B { struct X; };
   struct C : B {};
   struct C::X {};
-  // expected-error@-1 {{no struct named 'X' in 'dr417::C'}}
+  // expected-error@-1 {{no struct named 'X' in 'cwg417::C'}}
   struct B::X { struct Y; };
   struct C::X::Y {}; // ok!
   namespace N {
@@ -347,24 +347,24 @@ namespace dr417 { // dr417: no
   }
   // FIXME: This is ill-formed.
   using N::D;
-  struct dr417::D {};
+  struct cwg417::D {};
   // expected-warning@-1 {{extra qualification on member 'D'}}
   using namespace N;
-  struct dr417::E {};
-  // expected-error@-1 {{no struct named 'E' in namespace 'dr417'}}
+  struct cwg417::E {};
+  // expected-error@-1 {{no struct named 'E' in namespace 'cwg417'}}
   // expected-warning@-2 {{extra qualification on member 'E'}}
   struct N::F {};
   struct G;
   using N::H;
   namespace M {
-    struct dr417::G {};
-    // expected-error@-1 {{cannot define or redeclare 'G' here because namespace 'M' does not enclose namespace 'dr417'}}
-    struct dr417::H {};
-    // expected-error@-1 {{cannot define or redeclare 'H' here because namespace 'M' does not enclose namespace 'dr417'}}
+    struct cwg417::G {};
+    // expected-error@-1 {{cannot define or redeclare 'G' here because namespace 'M' does not enclose namespace 'cwg417'}}
+    struct cwg417::H {};
+    // expected-error@-1 {{cannot define or redeclare 'H' here because namespace 'M' does not enclose namespace 'cwg417'}}
   }
 }
 
-namespace dr418 { // dr418: no
+namespace cwg418 { // cwg418: no
 namespace example1 {
 void f1(int, int = 0);
 void f1(int = 0, int);
@@ -374,7 +374,7 @@ void g() { f1(); }
 
 namespace example2 {
 namespace A {
-void f2(int); // #dr418-f2
+void f2(int); // #cwg418-f2
 }
 namespace B {
 using A::f2;
@@ -386,7 +386,7 @@ void g2() {
   using B::f2;
   f2();
   // expected-error@-1 {{no matching function for call to 'f2'}}
-  //   expected-note@#dr418-f2 {{candidate function not viable: requires 1 argument, but 0 were provided}}
+  //   expected-note@#cwg418-f2 {{candidate function not viable: requires 1 argument, but 0 were provided}}
 }
 } // namespace example2
 
@@ -407,9 +407,9 @@ void use() {
   f(); // FIXME: this should fail
 }
 } // namespace example3
-} // namespace dr418
+} // namespace cwg418
 
-namespace dr420 { // dr420: 9
+namespace cwg420 { // cwg420: 9
   template<typename T> struct ptr {
     T *operator->() const;
     T &operator*() const;
@@ -453,15 +453,15 @@ namespace dr420 { // dr420: 9
     q->~id<int>();
     p->id<int>::~id<int>();
     q->id<int>::~id<int>();
-    p->template id<int>::~id<int>(); // OK since dr2292
-    q->template id<int>::~id<int>(); // OK since dr2292
+    p->template id<int>::~id<int>(); // OK since cwg2292
+    q->template id<int>::~id<int>(); // OK since cwg2292
     p->A::template id<int>::~id<int>();
     q->A::template id<int>::~id<int>();
   }
 #endif
 }
 
-namespace dr421 { // dr421: yes
+namespace cwg421 { // cwg421: yes
   struct X { X(); int n; int &r; };
   int *p = &X().n;
   // cxx98-error@-1 {{taking the address of a temporary object of type 'int'}}
@@ -469,32 +469,32 @@ namespace dr421 { // dr421: yes
   int *q = &X().r;
 }
 
-namespace dr422 { // dr422: yes
+namespace cwg422 { // cwg422: yes
   template<typename T, typename U> void f() {
-    typedef T type; // #dr422-typedef-T
+    typedef T type; // #cwg422-typedef-T
     typedef U type;
     // expected-error@-1 {{typedef redefinition with different types ('char' vs 'int')}}
-    //   expected-note@#dr422-f-int-char {{in instantiation of function template specialization 'dr422::f<int, char>' requested here}}
-    //   expected-note@#dr422-typedef-T {{previous definition is here}}
+    //   expected-note@#cwg422-f-int-char {{in instantiation of function template specialization 'cwg422::f<int, char>' requested here}}
+    //   expected-note@#cwg422-typedef-T {{previous definition is here}}
   }
   template void f<int, int>();
-  template void f<int, char>(); // #dr422-f-int-char
+  template void f<int, char>(); // #cwg422-f-int-char
 }
 
-namespace dr423 { // dr423: yes
+namespace cwg423 { // cwg423: yes
   template<typename T> struct X { operator T&(); };
   void f(X<int> x) { x += 1; }
 }
 
-namespace dr424 { // dr424: yes
+namespace cwg424 { // cwg424: yes
   struct A {
-    typedef int N; // #dr424-N
+    typedef int N; // #cwg424-N
     typedef int N;
     // expected-error@-1 {{redefinition of 'N'}}
-    //   expected-note@#dr424-N {{previous definition is here}}
+    //   expected-note@#cwg424-N {{previous definition is here}}
 
     struct X;
-    typedef X X; // #dr424-X
+    typedef X X; // #cwg424-X
     struct X {};
 
     struct X *p;
@@ -503,20 +503,20 @@ namespace dr424 { // dr424: yes
 
     typedef X X;
     // expected-error@-1 {{redefinition of 'X'}}
-    //   expected-note@#dr424-X {{previous definition is here}}
+    //   expected-note@#cwg424-X {{previous definition is here}}
   };
   struct B {
     typedef int M;
   };
   struct C : B {
-    typedef int M; // #dr424-M
+    typedef int M; // #cwg424-M
     typedef int M;
     // expected-error@-1 {{redefinition of 'M'}}
-    //   expected-note@#dr424-M {{previous definition is here}}
+    //   expected-note@#cwg424-M {{previous definition is here}}
   };
 }
 
-namespace dr425 { // dr425: yes
+namespace cwg425 { // cwg425: yes
   struct A { template<typename T> operator T() const; } a;
   float f = 1.0f * a;
   // expected-error@-1 {{use of overloaded operator '*' is ambiguous (with operand types 'float' and 'struct A')}}
@@ -532,10 +532,10 @@ namespace dr425 { // dr425: yes
   float g = 1.0f * b; // ok
 }
 
-namespace dr427 { // dr427: yes
+namespace cwg427 { // cwg427: yes
   struct B {};
   struct D : public B {
-    D(B &) = delete; // #dr427-D
+    D(B &) = delete; // #cwg427-D
     // cxx98-error@-1 {{deleted function definitions are a C++11 extension}}
   };
 
@@ -545,12 +545,12 @@ namespace dr427 { // dr427: yes
   const D &d3 = (const D&)b;
   const D &d4(b);
   // expected-error@-1 {{conversion function from 'B' to 'const D' invokes a deleted function}}
-  //   expected-note@#dr427-D {{'D' has been explicitly marked deleted here}}
+  //   expected-note@#cwg427-D {{'D' has been explicitly marked deleted here}}
 }
 
-namespace dr428 { // dr428: yes
+namespace cwg428 { // cwg428: yes
   template<typename T> T make();
-  extern struct X x; // #dr428-X
+  extern struct X x; // #cwg428-X
   void f() {
     throw void();
     // expected-error@-1 {{cannot throw}}
@@ -558,30 +558,30 @@ namespace dr428 { // dr428: yes
     throw make<const volatile void*>();
     throw x;
     // expected-error@-1 {{cannot throw}}
-    //   expected-note@#dr428-X {{forward declaration of 'dr428::X'}}
+    //   expected-note@#cwg428-X {{forward declaration of 'cwg428::X'}}
     throw make<X&>();
     // expected-error@-1 {{cannot throw}}
-    //   expected-note@#dr428-X {{forward declaration of 'dr428::X'}}
+    //   expected-note@#cwg428-X {{forward declaration of 'cwg428::X'}}
     throw make<X*>();
     // expected-error@-1 {{cannot throw}}
-    //   expected-note@#dr428-X {{forward declaration of 'dr428::X'}}
+    //   expected-note@#cwg428-X {{forward declaration of 'cwg428::X'}}
     throw make<const volatile X&>();
     // expected-error@-1 {{cannot throw}}
-    //   expected-note@#dr428-X {{forward declaration of 'dr428::X'}}
+    //   expected-note@#cwg428-X {{forward declaration of 'cwg428::X'}}
     throw make<const volatile X*>();
     // expected-error@-1 {{cannot throw}}
-    //   expected-note@#dr428-X {{forward declaration of 'dr428::X'}}
+    //   expected-note@#cwg428-X {{forward declaration of 'cwg428::X'}}
   }
 }
 
-namespace dr429 { // dr429: 2.8 c++11
+namespace cwg429 { // cwg429: 2.8 c++11
   // FIXME: This rule is obviously intended to apply to C++98 as well.
   struct A {
     static void *operator new(size_t, size_t);
-    static void operator delete(void*, size_t); // #dr429-delete
+    static void operator delete(void*, size_t); // #cwg429-delete
   } *a = new (0) A;
   // since-cxx11-error@-1 {{'new' expression with placement arguments refers to non-placement 'operator delete'}}
-  //   since-cxx11-note@#dr429-delete {{here}}
+  //   since-cxx11-note@#cwg429-delete {{here}}
   struct B {
     static void *operator new(size_t, size_t);
     static void operator delete(void*);
@@ -589,7 +589,7 @@ namespace dr429 { // dr429: 2.8 c++11
   } *b = new (0) B; // ok, second delete is not a non-placement deallocation function
 }
 
-namespace dr430 { // dr430: yes c++11
+namespace cwg430 { // cwg430: yes c++11
   // resolved by n2239
   // FIXME: This should apply in C++98 too.
   void f(int n) {
@@ -598,7 +598,7 @@ namespace dr430 { // dr430: yes c++11
   }
 }
 
-namespace dr431 { // dr431: yes
+namespace cwg431 { // cwg431: yes
   struct A {
     template<typename T> T *get();
     template<typename T> struct B {
@@ -630,7 +630,7 @@ namespace dr431 { // dr431: yes
   }
 }
 
-namespace dr432 { // dr432: 3.0
+namespace cwg432 { // cwg432: 3.0
   template<typename T> struct A {};
   template<typename T> struct B : A<B> {};
   // expected-error@-1 {{use of class template 'B' requires template arguments}}
@@ -643,7 +643,7 @@ namespace dr432 { // dr432: 3.0
 #endif
 }
 
-namespace dr433 { // dr433: yes
+namespace cwg433 { // cwg433: yes
   template<class T> struct S {
     void f(union U*);
   };
@@ -653,7 +653,7 @@ namespace dr433 { // dr433: yes
   S<int> s;
 }
 
-namespace dr434 { // dr434: sup 2352
+namespace cwg434 { // cwg434: sup 2352
   void f() {
     const int ci = 0;
     int *pi = 0;
@@ -671,16 +671,16 @@ namespace dr434 { // dr434: sup 2352
 #endif
 }
 
-// dr435: na
+// cwg435: na
 
-namespace dr436 { // dr436: yes
-  enum E { f }; // #dr436-f
+namespace cwg436 { // cwg436: yes
+  enum E { f }; // #cwg436-f
   void f();
   // expected-error@-1 {{redefinition of 'f' as different kind of symbol}}
-  //   expected-note@#dr436-f {{previous definition is here}}
+  //   expected-note@#cwg436-f {{previous definition is here}}
 }
 
-namespace dr437 { // dr437: sup 1308
+namespace cwg437 { // cwg437: sup 1308
   // This is superseded by 1308, which is in turn superseded by 1330,
   // which restores this rule.
   template<typename U> struct T : U {};
@@ -699,46 +699,46 @@ namespace dr437 { // dr437: sup 1308
   };
 }
 
-// dr438 is in dr438.cpp
-// dr439 is in dr439.cpp
-// dr441 is in dr441.cpp
-// dr442: sup 348
-// dr443: na
+// cwg438 is in cwg438.cpp
+// cwg439 is in cwg439.cpp
+// cwg441 is in cwg441.cpp
+// cwg442: sup 348
+// cwg443: na
 
-namespace dr444 { // dr444: yes
+namespace cwg444 { // cwg444: yes
   struct D;
-  struct B { // #dr444-B
-    D &operator=(D &) = delete; // #dr444-deleted
+  struct B { // #cwg444-B
+    D &operator=(D &) = delete; // #cwg444-deleted
     // cxx98-error@-1 {{deleted function definitions are a C++11 extension}}
   };
-  struct D : B { // #dr444-D
+  struct D : B { // #cwg444-D
     using B::operator=;
   } extern d;
   void f() {
     d = d;
     // expected-error@-1 {{overload resolution selected deleted operator '='}}
-    //   expected-note@#dr444-deleted {{candidate function has been explicitly deleted}}
-    //   expected-note@#dr444-D {{candidate function (the implicit copy assignment operator)}}
-    //   expected-note@#dr444-B {{candidate function (the implicit copy assignment operator)}}
-    //   since-cxx11-note@#dr444-B {{candidate function (the implicit move assignment operator) not viable: expects an rvalue for 1st argument}}
-    //   since-cxx11-note@#dr444-D {{candidate function (the implicit move assignment operator) not viable: expects an rvalue for 1st argument}}
+    //   expected-note@#cwg444-deleted {{candidate function has been explicitly deleted}}
+    //   expected-note@#cwg444-D {{candidate function (the implicit copy assignment operator)}}
+    //   expected-note@#cwg444-B {{candidate function (the implicit copy assignment operator)}}
+    //   since-cxx11-note@#cwg444-B {{candidate function (the implicit move assignment operator) not viable: expects an rvalue for 1st argument}}
+    //   since-cxx11-note@#cwg444-D {{candidate function (the implicit move assignment operator) not viable: expects an rvalue for 1st argument}}
   }
 }
 
-namespace dr445 { // dr445: 3.2
-  class A { void f(); }; // #dr445-f
+namespace cwg445 { // cwg445: 3.2
+  class A { void f(); }; // #cwg445-f
   struct B {
     friend void A::f();
-    // expected-error@-1 {{friend function 'f' is a private member of 'dr445::A'}}
-    //   expected-note@#dr445-f {{implicitly declared private here}}
+    // expected-error@-1 {{friend function 'f' is a private member of 'cwg445::A'}}
+    //   expected-note@#cwg445-f {{implicitly declared private here}}
   };
 }
 
-namespace dr446 { // dr446: 2.8
+namespace cwg446 { // cwg446: 2.8
   struct C;
   struct A {
     A();
-    A(const A&) = delete; // #dr446-deleted
+    A(const A&) = delete; // #cwg446-deleted
     // cxx98-error@-1 {{deleted function definitions are a C++11 extension}}
     A(const C&);
   };
@@ -747,28 +747,28 @@ namespace dr446 { // dr446: 2.8
     void(b ? a : a);
     b ? A() : a;
     // expected-error@-1 {{call to deleted constructor of 'A'}}
-    //   expected-note@#dr446-deleted {{'A' has been explicitly marked deleted here}}
+    //   expected-note@#cwg446-deleted {{'A' has been explicitly marked deleted here}}
     b ? a : A();
     // expected-error@-1 {{call to deleted constructor of 'A'}}
-    //   expected-note@#dr446-deleted {{'A' has been explicitly marked deleted here}}
+    //   expected-note@#cwg446-deleted {{'A' has been explicitly marked deleted here}}
     b ? A() : A();
     // cxx98-14-error@-1 {{call to deleted constructor of 'A'}}
-    //   expected-note@#dr446-deleted {{'A' has been explicitly marked deleted here}}
+    //   expected-note@#cwg446-deleted {{'A' has been explicitly marked deleted here}}
 
     void(b ? a : c);
     b ? a : C();
     // expected-error@-1 {{call to deleted constructor of 'A'}}
-    //   cxx98-14-note@#dr446-deleted {{'A' has been explicitly marked deleted here}}
+    //   cxx98-14-note@#cwg446-deleted {{'A' has been explicitly marked deleted here}}
     b ? c : A();
     // cxx98-14-error@-1 {{call to deleted constructor of 'A'}}
-    //   cxx98-14-note@#dr446-deleted {{'A' has been explicitly marked deleted here}}
+    //   cxx98-14-note@#cwg446-deleted {{'A' has been explicitly marked deleted here}}
     b ? A() : C();
     // cxx98-14-error@-1 {{call to deleted constructor of 'A'}}
-    //   cxx98-14-note@#dr446-deleted {{'A' has been explicitly marked deleted here}}
+    //   cxx98-14-note@#cwg446-deleted {{'A' has been explicitly marked deleted here}}
   }
 }
 
-namespace dr447 { // dr447: yes
+namespace cwg447 { // cwg447: yes
   struct A { int n; int a[4]; };
   template<int> struct U {
     typedef int type;
@@ -796,60 +796,60 @@ namespace dr447 { // dr447: yes
   }
 }
 
-namespace dr448 { // dr448: 2.8
-  template<typename T = int> void f(int); // #dr448-f-int
+namespace cwg448 { // cwg448: 2.8
+  template<typename T = int> void f(int); // #cwg448-f-int
   // cxx98-error@-1 {{default template arguments for a function template are a C++11 extension}}
   template<typename T> void g(T t) {
     f<T>(t);
     // expected-error@-1 {{call to function 'f' that is neither visible in the template definition nor found by argument-dependent lookup}}
-    //   expected-note@#dr448-g {{in instantiation of function template specialization 'dr448::g<dr448::HideFromADL::X>' requested here}}
-    //   expected-note@#dr448-f-T {{'f' should be declared prior to the call site or in namespace 'dr448::HideFromADL'}}
-    dr448::f(t);
+    //   expected-note@#cwg448-g {{in instantiation of function template specialization 'cwg448::g<cwg448::HideFromADL::X>' requested here}}
+    //   expected-note@#cwg448-f-T {{'f' should be declared prior to the call site or in namespace 'cwg448::HideFromADL'}}
+    cwg448::f(t);
     // expected-error@-1 {{no matching function for call to 'f'}}
-    //   expected-note@#dr448-f-int {{candidate function template not viable: no known conversion from 'dr448::HideFromADL::X' to 'int' for 1st argument}}
+    //   expected-note@#cwg448-f-int {{candidate function template not viable: no known conversion from 'cwg448::HideFromADL::X' to 'int' for 1st argument}}
   }
-  template<typename T> void f(T); // #dr448-f-T
+  template<typename T> void f(T); // #cwg448-f-T
   namespace HideFromADL { struct X {}; }
   template void g(int); // ok
-  template void g(HideFromADL::X); // #dr448-g
+  template void g(HideFromADL::X); // #cwg448-g
 }
 
-// dr449: na
+// cwg449: na
 
-namespace dr450 { // dr450: yes
+namespace cwg450 { // cwg450: yes
   typedef int A[3];
   void f1(const A &);
-  void f2(A &); // #dr450-f2
+  void f2(A &); // #cwg450-f2
   struct S { A n; };
   void g() {
     f1(S().n);
     f2(S().n);
     // expected-error@-1 {{no matching function for call to 'f2'}}}
-    //   expected-note@#dr450-f2 {{candidate function not viable: expects an lvalue for 1st argument}}
+    //   expected-note@#cwg450-f2 {{candidate function not viable: expects an lvalue for 1st argument}}
   }
 #if __cplusplus >= 201103L
   void h() {
     f1(A{});
     f2(A{});
     // expected-error@-1 {{no matching function for call to 'f2'}}}
-    //   expected-note@#dr450-f2 {{candidate function not viable: expects an lvalue for 1st argument}}
+    //   expected-note@#cwg450-f2 {{candidate function not viable: expects an lvalue for 1st argument}}
   }
 #endif
 }
 
-namespace dr451 { // dr451: yes
+namespace cwg451 { // cwg451: yes
   const int a = 1 / 0;
   // expected-warning@-1 {{division by zero is undefined}}
-  const int b = 1 / 0; // #dr451-b
+  const int b = 1 / 0; // #cwg451-b
   // expected-warning@-1 {{division by zero is undefined}}
-  int arr[b]; // #dr451-arr
+  int arr[b]; // #cwg451-arr
   // expected-error@-1 {{variable length arrays in C++ are a Clang extension}}
   //   expected-note@-2 {{initializer of 'b' is not a constant expression}}
-  //   expected-note@#dr451-b {{declared here}}
-  // expected-error@#dr451-arr {{variable length array declaration not allowed at file scope}}
+  //   expected-note@#cwg451-b {{declared here}}
+  // expected-error@#cwg451-arr {{variable length array declaration not allowed at file scope}}
 }
 
-namespace dr452 { // dr452: yes
+namespace cwg452 { // cwg452: yes
   struct A {
     int a, b, c;
     A *p;
@@ -858,9 +858,9 @@ namespace dr452 { // dr452: yes
   };
 }
 
-// dr454 FIXME write a codegen test
+// cwg454 FIXME write a codegen test
 
-namespace dr456 { // dr456: yes
+namespace cwg456 { // cwg456: yes
   // sup 903 c++11
   const int null = 0;
   void *p = null;
@@ -873,7 +873,7 @@ namespace dr456 { // dr456: yes
   // since-cxx11-error@-2 {{cannot initialize a variable of type 'void *' with an lvalue of type 'const bool'}}
 }
 
-namespace dr457 { // dr457: yes
+namespace cwg457 { // cwg457: yes
   const int a = 1;
   const volatile int b = 1;
   int ax[a];
@@ -890,7 +890,7 @@ namespace dr457 { // dr457: yes
   };
 }
 
-namespace dr458 { // dr458: 11
+namespace cwg458 { // cwg458: 11
   struct A {
     int T;
     int f();
@@ -906,11 +906,11 @@ namespace dr458 { // dr458: 11
   int A::f() {
     return T;
   }
-  template<typename T> // #dr458-g-T
+  template<typename T> // #cwg458-g-T
   int A::g() {
     return T;
     // expected-error@-1 {{'T' does not refer to a value}}
-    //   expected-note@#dr458-g-T {{declared here}}
+    //   expected-note@#cwg458-g-T {{declared here}}
   }
 
   template<typename T>
@@ -923,33 +923,33 @@ namespace dr458 { // dr458: 11
     return T;
   }
   template<typename U>
-  template<typename T> // #dr458-h-T
+  template<typename T> // #cwg458-h-T
   int B<U>::h() {
     return T;
     // expected-error@-1 {{'T' does not refer to a value}}
-    //   expected-note@#dr458-h-T {{declared here}}
+    //   expected-note@#cwg458-h-T {{declared here}}
   }
 }
 
-namespace dr460 { // dr460: yes
+namespace cwg460 { // cwg460: yes
   namespace X { namespace Q { int n; } }
   namespace Y {
     using X;
     // expected-error@-1 {{using declaration requires a qualified name}}
-    using dr460::X;
+    using cwg460::X;
     // expected-error@-1 {{using declaration cannot refer to a namespace}}
     using X::Q;
     // expected-error@-1 {{using declaration cannot refer to a namespace}}
   }
 }
 
-// dr461: na
-// dr462 is in dr462.cpp
-// dr463: na
-// dr464: na
-// dr465: na
+// cwg461: na
+// cwg462 is in cwg462.cpp
+// cwg463: na
+// cwg464: na
+// cwg465: na
 
-namespace dr466 { // dr466: 2.8
+namespace cwg466 { // cwg466: 2.8
 typedef int I;
 typedef const int CI;
 typedef volatile int VI;
@@ -980,7 +980,7 @@ void g(int a, CI b, VI c) {
 }
 }
 
-namespace dr467 { // dr467: yes
+namespace cwg467 { // cwg467: yes
   int stuff();
 
   int f() {
@@ -995,51 +995,51 @@ namespace dr467 { // dr467: yes
   int g() {
     goto later;
     // expected-error@-1 {{cannot jump from this goto statement to its label}}
-    //   expected-note@#dr467-k {{jump bypasses variable initialization}}
-    int k = stuff(); // #dr467-k
+    //   expected-note@#cwg467-k {{jump bypasses variable initialization}}
+    int k = stuff(); // #cwg467-k
   later:
     return k;
   }
 }
 
-namespace dr468 { // dr468: yes c++11
+namespace cwg468 { // cwg468: yes c++11
   // FIXME: Should we allow this in C++98 too?
   template<typename> struct A {
     template<typename> struct B {
       static int C;
     };
   };
-  int k = dr468::template A<int>::template B<char>::C;
+  int k = cwg468::template A<int>::template B<char>::C;
   // cxx98-error@-1 {{'template' keyword outside of a template}}
   // cxx98-error@-2 {{'template' keyword outside of a template}}
 }
 
-namespace dr469 { // dr469: no
-  template<typename T> struct X; // #dr469-X
+namespace cwg469 { // cwg469: no
+  template<typename T> struct X; // #cwg469-X
   template<typename T> struct X<const T> {};
   X<int&> x;
-  // expected-error@-1 {{implicit instantiation of undefined template 'dr469::X<int &>'}}
-  //   expected-note@#dr469-X {{template is declared here}}
+  // expected-error@-1 {{implicit instantiation of undefined template 'cwg469::X<int &>'}}
+  //   expected-note@#cwg469-X {{template is declared here}}
 }
 
-namespace dr470 { // dr470: yes
+namespace cwg470 { // cwg470: yes
   template<typename T> struct A {
     struct B {};
   };
   template<typename T> struct C {
   };
 
-  template struct A<int>; // #dr470-A-int
+  template struct A<int>; // #cwg470-A-int
   template struct A<int>::B;
   // expected-error@-1 {{duplicate explicit instantiation of 'B'}}
-  //   expected-note@#dr470-A-int {{previous explicit instantiation is here}}
+  //   expected-note@#cwg470-A-int {{previous explicit instantiation is here}}
 
   // ok, instantiating C<char> doesn't instantiate base class members.
   template struct A<char>;
   template struct C<char>;
 }
 
-namespace dr471 { // dr471: 2.8
+namespace cwg471 { // cwg471: 2.8
   struct A { int n; };
   struct B : private virtual A {};
   struct C : protected virtual A {};
@@ -1050,50 +1050,50 @@ namespace dr471 { // dr471: 2.8
   struct F : E, B { int f() { return n; } };
   struct G : virtual A {
   private:
-    using A::n; // #dr471-G-using
+    using A::n; // #cwg471-G-using
   };
   struct H : B, G { int f() { return n; } };
-  // expected-error@-1 {{'n' is a private member of 'dr471::G'}}
-  //   expected-note@#dr471-G-using {{declared private here}}
+  // expected-error@-1 {{'n' is a private member of 'cwg471::G'}}
+  //   expected-note@#cwg471-G-using {{declared private here}}
 }
 
-namespace dr472 { // dr472: no drafting 2011-04
+namespace cwg472 { // cwg472: no drafting 2011-04
 struct B {
-  int i; // #dr472-i
+  int i; // #cwg472-i
 };
-struct I : protected B {}; // #dr472-struct-I
+struct I : protected B {}; // #cwg472-struct-I
 struct D : public I {
   void f(I *ip) {
     ip->i = 0;
-    // expected-error@-1 {{'i' is a protected member of 'dr472::B'}}
-    //   expected-note@#dr472-struct-I {{constrained by protected inheritance here}}
-    //   expected-note@#dr472-i {{member is declared here}}
+    // expected-error@-1 {{'i' is a protected member of 'cwg472::B'}}
+    //   expected-note@#cwg472-struct-I {{constrained by protected inheritance here}}
+    //   expected-note@#cwg472-i {{member is declared here}}
     B *bp = ip;
     bp->i = 5;
   }
 };
 }
 
-namespace dr474 { // dr474: 3.4
+namespace cwg474 { // cwg474: 3.4
   namespace N {
     struct S {
       void f();
     };
   }
   void N::S::f() {
-    void g(); // #dr474-g
+    void g(); // #cwg474-g
   }
   int g();
   namespace N {
     int g();
     // expected-error@-1 {{functions that differ only in their return type cannot be overloaded}}
-    //   expected-note@#dr474-g {{previous declaration is here}}
+    //   expected-note@#cwg474-g {{previous declaration is here}}
   }
 }
 
-// dr475 FIXME write a libc++abi test
+// cwg475 FIXME write a libc++abi test
 
-namespace dr477 { // dr477: 3.5
+namespace cwg477 { // cwg477: 3.5
   struct A {
     explicit A();
     virtual void f();
@@ -1110,56 +1110,56 @@ namespace dr477 { // dr477: 3.5
   // expected-error@-1 {{can only be specified inside the class definition}}
 }
 
-namespace dr478 { // dr478: yes
-  struct A { virtual void f() = 0; }; // #dr478-f
+namespace cwg478 { // cwg478: yes
+  struct A { virtual void f() = 0; }; // #cwg478-f
   void f(A *a);
   void f(A a[10]);
   // expected-error@-1 {{array of abstract class type 'A'}}
-  //   expected-note@#dr478-f {{unimplemented pure virtual method 'f' in 'A'}}
+  //   expected-note@#cwg478-f {{unimplemented pure virtual method 'f' in 'A'}}
 }
 
-namespace dr479 { // dr479: 2.8
+namespace cwg479 { // cwg479: 2.8
   struct S {
     S();
   private:
-    S(const S&); // #dr479-S-copy-ctor
-    ~S(); // #dr479-S-dtor
+    S(const S&); // #cwg479-S-copy-ctor
+    ~S(); // #cwg479-S-dtor
   };
   void f() {
     throw S();
     // expected-error@-1 {{temporary of type 'S' has private destructor}}
-    //   expected-note@#dr479-S-dtor {{declared private here}}
+    //   expected-note@#cwg479-S-dtor {{declared private here}}
     // expected-error@-3 {{exception object of type 'S' has private destructor}}
-    //   expected-note@#dr479-S-dtor {{declared private here}}
-    // cxx98-error@-5 {{C++98 requires an accessible copy constructor for class 'dr479::S' when binding a reference to a temporary; was private}}
-    //   cxx98-note@#dr479-S-copy-ctor {{declared private here}}
-    // cxx98-14-error@-7 {{calling a private constructor of class 'dr479::S'}}
-    //   cxx98-14-note@#dr479-S-copy-ctor {{declared private here}}
+    //   expected-note@#cwg479-S-dtor {{declared private here}}
+    // cxx98-error@-5 {{C++98 requires an accessible copy constructor for class 'cwg479::S' when binding a reference to a temporary; was private}}
+    //   cxx98-note@#cwg479-S-copy-ctor {{declared private here}}
+    // cxx98-14-error@-7 {{calling a private constructor of class 'cwg479::S'}}
+    //   cxx98-14-note@#cwg479-S-copy-ctor {{declared private here}}
   }
   void g() {
     S s;
     // expected-error@-1 {{variable of type 'S' has private destructor}}
-    //   expected-note@#dr479-S-dtor {{declared private here}}
+    //   expected-note@#cwg479-S-dtor {{declared private here}}
     throw s;
     // expected-error@-1 {{exception object of type 'S' has private destructor}}
-    //   expected-note@#dr479-S-dtor {{declared private here}}
-    // expected-error@-3 {{calling a private constructor of class 'dr479::S'}}
-    //   expected-note@#dr479-S-copy-ctor {{declared private here}}
+    //   expected-note@#cwg479-S-dtor {{declared private here}}
+    // expected-error@-3 {{calling a private constructor of class 'cwg479::S'}}
+    //   expected-note@#cwg479-S-copy-ctor {{declared private here}}
   }
   void h() {
     try {
       f();
       g();
     } catch (S s) {
-      // expected-error@-1 {{calling a private constructor of class 'dr479::S'}}
-      //   expected-note@#dr479-S-copy-ctor {{declared private here}}
+      // expected-error@-1 {{calling a private constructor of class 'cwg479::S'}}
+      //   expected-note@#cwg479-S-copy-ctor {{declared private here}}
       // expected-error@-3 {{variable of type 'S' has private destructor}}
-      //   expected-note@#dr479-S-dtor {{declared private here}}
+      //   expected-note@#cwg479-S-dtor {{declared private here}}
     }
   }
 }
 
-namespace dr480 { // dr480: yes
+namespace cwg480 { // cwg480: yes
   struct A { int n; };
   struct B : A {};
   struct C : virtual B {};
@@ -1167,24 +1167,24 @@ namespace dr480 { // dr480: yes
 
   int A::*a = &A::n;
   int D::*b = a;
-  // expected-error@-1 {{conversion from pointer to member of class 'dr480::A' to pointer to member of class 'dr480::D' via virtual base 'dr480::B' is not allowed}}
+  // expected-error@-1 {{conversion from pointer to member of class 'cwg480::A' to pointer to member of class 'cwg480::D' via virtual base 'cwg480::B' is not allowed}}
 
   extern int D::*c;
   int A::*d = static_cast<int A::*>(c);
-  // expected-error@-1 {{conversion from pointer to member of class 'dr480::D' to pointer to member of class 'dr480::A' via virtual base 'dr480::B' is not allowed}}
+  // expected-error@-1 {{conversion from pointer to member of class 'cwg480::D' to pointer to member of class 'cwg480::A' via virtual base 'cwg480::B' is not allowed}}
 
   D *e;
   A *f = e;
   D *g = static_cast<D*>(f);
-  // expected-error@-1 {{cannot cast 'dr480::A *' to 'D *' via virtual base 'dr480::B'}}
+  // expected-error@-1 {{cannot cast 'cwg480::A *' to 'D *' via virtual base 'cwg480::B'}}
 
   extern D &i;
   A &j = i;
   D &k = static_cast<D&>(j);
-  // expected-error@-1 {{cannot cast 'A' to 'D &' via virtual base 'dr480::B'}}
+  // expected-error@-1 {{cannot cast 'A' to 'D &' via virtual base 'cwg480::B'}}
 }
 
-namespace dr481 { // dr481: 2.8
+namespace cwg481 { // cwg481: 2.8
   template<class T, T U> class A { T *x; };
   T *x;
   // expected-error@-1 {{unknown type name 'T'}}
@@ -1199,15 +1199,15 @@ namespace dr481 { // dr481: 2.8
 
   template<typename A = C, typename C = A> struct E {
     void f() {
-      typedef ::dr481::C c; // #dr481-c
+      typedef ::cwg481::C c; // #cwg481-c
       typedef C c;
-      // expected-error@-1 {{typedef redefinition with different types ('int' vs '::dr481::C')}}
-      //   expected-note@#dr481-E-int {{in instantiation of member function 'dr481::E<int>::f' requested here}}
-      //   expected-note@#dr481-c {{previous definition is here}}
+      // expected-error@-1 {{typedef redefinition with different types ('int' vs '::cwg481::C')}}
+      //   expected-note@#cwg481-E-int {{in instantiation of member function 'cwg481::E<int>::f' requested here}}
+      //   expected-note@#cwg481-c {{previous definition is here}}
     }
   };
   template struct E<>; // ok
-  template struct E<int>; // #dr481-E-int
+  template struct E<int>; // #cwg481-E-int
 
   template<template<typename U_no_typo_correction> class A,
            A<int> *B,
@@ -1229,13 +1229,13 @@ namespace dr481 { // dr481: 2.8
   I<123, char*, J> *j;
 }
 
-namespace dr482 { // dr482: 3.5
+namespace cwg482 { // cwg482: 3.5
   extern int a;
   void f();
 
-  int dr482::a = 0;
+  int cwg482::a = 0;
   // expected-warning@-1 {{extra qualification on member 'a'}}
-  void dr482::f() {}
+  void cwg482::f() {}
   // expected-warning@-1 {{extra qualification on member 'f'}}
 
   inline namespace X {
@@ -1244,19 +1244,19 @@ namespace dr482 { // dr482: 3.5
     void g();
     struct S;
   }
-  int dr482::b = 0;
+  int cwg482::b = 0;
   // expected-warning@-1 {{extra qualification on member 'b'}}
-  void dr482::g() {}
+  void cwg482::g() {}
   // expected-warning@-1 {{extra qualification on member 'g'}}
-  struct dr482::S {};
+  struct cwg482::S {};
   // expected-warning@-1 {{extra qualification on member 'S'}}
 
-  void dr482::f();
+  void cwg482::f();
   // expected-warning@-1 {{extra qualification on member 'f'}}
-  void dr482::g();
+  void cwg482::g();
   // expected-warning@-1 {{extra qualification on member 'g'}}
 
-  // FIXME: The following are valid in DR482's wording, but these are bugs in
+  // FIXME: The following are valid in CWG482's wording, but these are bugs in
   // the wording which we deliberately don't implement.
   namespace N { typedef int type; }
   typedef int N::type;
@@ -1274,7 +1274,7 @@ namespace dr482 { // dr482: 3.5
   };
 }
 
-namespace dr483 { // dr483: yes
+namespace cwg483 { // cwg483: yes
   namespace climits {
     int check1[__SCHAR_MAX__ >= 127 ? 1 : -1];
     int check2[__SHRT_MAX__ >= 32767 ? 1 : -1];
@@ -1293,7 +1293,7 @@ namespace dr483 { // dr483: yes
   }
 }
 
-namespace dr484 { // dr484: yes
+namespace cwg484 { // cwg484: yes
   struct A {
     A();
     void f();
@@ -1312,18 +1312,18 @@ namespace dr484 { // dr484: yes
   };
 
   struct C;
-  typedef C CT; // #dr484-typedef-CT
+  typedef C CT; // #cwg484-typedef-CT
   struct CT {};
   // expected-error@-1 {{definition of type 'CT' conflicts with typedef of the same name}}
-  //   expected-note@#dr484-typedef-CT {{'CT' declared here}}
+  //   expected-note@#cwg484-typedef-CT {{'CT' declared here}}
 
   namespace N {
     struct D;
-    typedef D DT; // #dr484-typedef-DT
+    typedef D DT; // #cwg484-typedef-DT
   }
   struct N::DT {};
   // expected-error@-1 {{definition of type 'DT' conflicts with typedef of the same name}}
-  //   expected-note@#dr484-typedef-DT {{'DT' declared here}}
+  //   expected-note@#cwg484-typedef-DT {{'DT' declared here}}
 
   typedef struct {
     S();
@@ -1331,7 +1331,7 @@ namespace dr484 { // dr484: yes
   } S;
 }
 
-namespace dr485 { // dr485: yes
+namespace cwg485 { // cwg485: yes
   namespace N {
     struct S {};
     int operator+(S, S);
@@ -1344,8 +1344,8 @@ namespace dr485 { // dr485: yes
   int b = f<int>(s);
 }
 
-namespace dr486 { // dr486: yes
-  template<typename T> T f(T *); // #dr486-f
+namespace cwg486 { // cwg486: yes
+  template<typename T> T f(T *); // #cwg486-f
   int &f(...);
 
   void g();
@@ -1356,24 +1356,24 @@ namespace dr486 { // dr486: yes
     int &b = f(&n);
     f<void()>(&g);
     // expected-error@-1 {{no matching function for call to 'f'}}
-    //   expected-note@#dr486-f {{candidate template ignored: substitution failure [with T = void ()]: function cannot return function type 'void ()'}}
+    //   expected-note@#cwg486-f {{candidate template ignored: substitution failure [with T = void ()]: function cannot return function type 'void ()'}}
     f<int[10]>(&n);
     // expected-error@-1 {{no matching function for call to 'f'}}
-    //   expected-note@#dr486-f {{candidate template ignored: substitution failure [with T = int[10]]: function cannot return array type 'int[10]'}}
+    //   expected-note@#cwg486-f {{candidate template ignored: substitution failure [with T = int[10]]: function cannot return array type 'int[10]'}}
   }
 }
 
-namespace dr487 { // dr487: yes
+namespace cwg487 { // cwg487: yes
   enum E { e };
-  int operator+(int, E); // #dr487-operator-plus
-  int i[4 + e]; // #dr487-i
+  int operator+(int, E); // #cwg487-operator-plus
+  int i[4 + e]; // #cwg487-i
   // expected-error@-1 {{variable length arrays in C++ are a Clang extension}}
   //   since-cxx11-note@-2 {{non-constexpr function 'operator+' cannot be used in a constant expression}}
-  //   since-cxx11-note@#dr487-operator-plus {{declared here}}
-  // expected-error@#dr487-i {{variable length array declaration not allowed at file scope}}
+  //   since-cxx11-note@#cwg487-operator-plus {{declared here}}
+  // expected-error@#cwg487-i {{variable length array declaration not allowed at file scope}}
 }
 
-namespace dr488 { // dr488: yes c++11
+namespace cwg488 { // cwg488: yes c++11
   template <typename T> void f(T);
   void f(int);
   void g() {
@@ -1387,19 +1387,19 @@ namespace dr488 { // dr488: yes c++11
   }
 }
 
-// dr489: na
+// cwg489: na
 
-namespace dr490 { // dr490: 2.8
+namespace cwg490 { // cwg490: 2.8
   template<typename T> struct X {};
 
   struct A {
     typedef int T;
-    struct K {}; // #dr490-k
+    struct K {}; // #cwg490-k
 
     int f(T);
     int g(T);
     int h(X<T>);
-    int X<T>::*i(); // #dr490-i
+    int X<T>::*i(); // #cwg490-i
     int K::*j();
 
     template<typename T> T k();
@@ -1417,11 +1417,11 @@ namespace dr490 { // dr490: 2.8
     // FIXME: Per this DR, these two are valid! That is another defect
     // (no number yet...) which will eventually supersede this one.
     friend int X<T>::*A::i();
-    // expected-error@-1 {{return type of out-of-line definition of 'dr490::A::i' differs from that in the declaration}}
-    //   expected-note@#dr490-i {{previous declaration is here}}
+    // expected-error@-1 {{return type of out-of-line definition of 'cwg490::A::i' differs from that in the declaration}}
+    //   expected-note@#cwg490-i {{previous declaration is here}}
     friend int K::*A::j();
     // expected-error@-1 {{use of undeclared identifier 'K'; did you mean 'A::K'?}}
-    //   expected-note@#dr490-k {{'A::K' declared here}}
+    //   expected-note@#cwg490-k {{'A::K' declared here}}
 
     // ok, lookup finds B::T, not A::T, so return type matches
     friend char A::k<T>();
@@ -1433,15 +1433,15 @@ namespace dr490 { // dr490: 2.8
   };
 }
 
-namespace dr491 { // dr491: dup 413
+namespace cwg491 { // cwg491: dup 413
   struct A {} a, b[3] = { a, {} };
   A c[2] = { a, {}, b[1] };
   // expected-error@-1 {{excess elements in array initializer}}
 }
 
-// dr492 is in dr492.cpp
+// cwg492 is in cwg492.cpp
 
-namespace dr493 { // dr493: dup 976
+namespace cwg493 { // cwg493: dup 976
   struct X {
     template <class T> operator const T &() const;
   };
@@ -1451,7 +1451,7 @@ namespace dr493 { // dr493: dup 976
   }
 }
 
-namespace dr494 { // dr494: dup 372
+namespace cwg494 { // cwg494: dup 372
   class A {
     class B {};
     friend class C;
@@ -1464,7 +1464,7 @@ namespace dr494 { // dr494: dup 372
   };
 }
 
-namespace dr495 { // dr495: 3.5
+namespace cwg495 { // cwg495: 3.5
   template<typename T>
   struct S {
     operator int() { return T::error; }
@@ -1482,11 +1482,11 @@ namespace dr495 { // dr495: 3.5
   long n2 = s2;
 }
 
-namespace dr496 { // dr496: sup 2094
+namespace cwg496 { // cwg496: sup 2094
   struct A { int n; };
   struct B { volatile int n; };
   int check1[ __is_trivially_copyable(const int) ? 1 : -1];
-  // This checks the dr2094 behavior, not dr496
+  // This checks the cwg2094 behavior, not cwg496
   int check2[ __is_trivially_copyable(volatile int) ? 1 : -1];
   int check3[ __is_trivially_constructible(A, const A&) ? 1 : -1];
   int check4[ __is_trivially_constructible(B, const B&) ? 1 : -1];
@@ -1494,7 +1494,7 @@ namespace dr496 { // dr496: sup 2094
   int check6[ __is_trivially_assignable(B, const B&) ? 1 : -1];
 }
 
-namespace dr497 { // dr497: sup 253
+namespace cwg497 { // cwg497: sup 253
   void before() {
     struct S {
       mutable int i;
@@ -1517,7 +1517,7 @@ namespace dr497 { // dr497: sup 253
   }
 }
 
-namespace dr499 { // dr499: yes
+namespace cwg499 { // cwg499: yes
   extern char str[];
   void f() { throw str; }
 }
diff --git a/clang/test/CXX/drs/dr5xx.cpp b/clang/test/CXX/drs/dr5xx.cpp
index d473f4827455..0fe64102d70b 100644
--- a/clang/test/CXX/drs/dr5xx.cpp
+++ b/clang/test/CXX/drs/dr5xx.cpp
@@ -9,20 +9,20 @@
 // pointing at the implicit operator new. We can't match such a diagnostic
 // with -verify.
 __extension__ typedef __SIZE_TYPE__ size_t;
-void *operator new(size_t); // #dr5xx-global-operator-new
+void *operator new(size_t); // #cwg5xx-global-operator-new
 // cxx98-error@-1 {{'operator new' is missing exception specification 'throw(std::bad_alloc)'}}
 #if __cplusplus > 201402L
 namespace std {
   enum class align_val_t : size_t {};
 }
-void *operator new(size_t, std::align_val_t); // #dr5xx-global-operator-new-aligned
+void *operator new(size_t, std::align_val_t); // #cwg5xx-global-operator-new-aligned
 #endif
 
 namespace std {
   struct type_info;
 }
 
-namespace dr500 { // dr500: dup 372
+namespace cwg500 { // cwg500: dup 372
   class D;
   class A {
     class B;
@@ -34,7 +34,7 @@ namespace dr500 { // dr500: dup 372
   class D : public A::B {};
 }
 
-namespace dr501 { // dr501: yes
+namespace cwg501 { // cwg501: yes
   struct A {
     friend void f() {}
     void g() {
@@ -44,7 +44,7 @@ namespace dr501 { // dr501: yes
   };
 }
 
-namespace dr502 { // dr502: yes
+namespace cwg502 { // cwg502: yes
   struct Q {};
   template<typename T> struct A {
     enum E { e = 1 };
@@ -59,7 +59,7 @@ namespace dr502 { // dr502: yes
   template struct A<int>;
 }
 
-namespace dr505 { // dr505: yes
+namespace cwg505 { // cwg505: yes
   const char *exts = "\e\(\{\[\%";
   // expected-error@-1 {{use of non-standard escape character '\e'}}
   // expected-error@-2 {{use of non-standard escape character '\('}}
@@ -70,7 +70,7 @@ namespace dr505 { // dr505: yes
   // expected-error@-1 {{unknown escape sequence '\Q'}}
 }
 
-namespace dr506 { // dr506: yes
+namespace cwg506 { // cwg506: yes
   struct NonPod { ~NonPod(); };
   void f(...);
   void g(NonPod np) { f(np); }
@@ -78,33 +78,33 @@ namespace dr506 { // dr506: yes
   // since-cxx11-error@-2 {{cannot pass object of non-trivial type 'NonPod' through variadic function; call will abort at runtime}}
 }
 
-// FIXME: Add tests here once DR260 is resolved.
-// dr507: dup 260
+// FIXME: Add tests here once CWG260 is resolved.
+// cwg507: dup 260
 
-// dr508: na
-// dr509: na
-// dr510: na
+// cwg508: na
+// cwg509: na
+// cwg510: na
 
-namespace dr512 { // dr512: yes
-  struct A { // #dr512-A
-    A(int); // #dr512-A-ctor
+namespace cwg512 { // cwg512: yes
+  struct A { // #cwg512-A
+    A(int); // #cwg512-A-ctor
   };
   union U { A a; };
   // cxx98-error@-1 {{union member 'a' has a non-trivial default constructor}}
-  //   cxx98-note@#dr512-A {{because type 'dr512::A' has no default constructor}}
-  //   cxx98-note@#dr512-A-ctor {{implicit default constructor suppressed by user-declared constructor}}
+  //   cxx98-note@#cwg512-A {{because type 'cwg512::A' has no default constructor}}
+  //   cxx98-note@#cwg512-A-ctor {{implicit default constructor suppressed by user-declared constructor}}
 }
 
-// dr513: na
+// cwg513: na
 
-namespace dr514 { // dr514: yes
+namespace cwg514 { // cwg514: yes
   namespace A { extern int x, y; }
   int A::x = y;
 }
 
-namespace dr515 { // dr515: sup 1017
-  // FIXME: dr1017 reverses the wording of dr515, but the current draft has
-  // dr515's wording, with a different fix for dr1017.
+namespace cwg515 { // cwg515: sup 1017
+  // FIXME: cwg1017 reverses the wording of cwg515, but the current draft has
+  // cwg515's wording, with a different fix for cwg1017.
 
   struct X { int n; };
   template<typename T> struct Y : T {
@@ -117,9 +117,9 @@ namespace dr515 { // dr515: sup 1017
   // cxx98-error@-1 {{invalid use of non-static data member 'a'}}
 }
 
-// dr516: na
+// cwg516: na
 
-namespace dr517 { // dr517: no
+namespace cwg517 { // cwg517: no
   // This is NDR, but we should diagnose it anyway.
   template<typename T> struct S {};
   template<typename T> int v = 0;
@@ -140,25 +140,25 @@ namespace dr517 { // dr517: no
   template<typename T> int v<T&> = 0;
 }
 
-namespace dr518 { // dr518: yes c++11
+namespace cwg518 { // cwg518: yes c++11
   enum E { e, };
   // cxx98-error@-1 {{commas at the end of enumerator lists are a C++11 extension}}
 }
 
-// dr519 is in dr519.cpp
-// dr520: na
+// cwg519 is in cwg519.cpp
+// cwg520: na
 
-// dr521: no
+// cwg521: no
 // FIXME: The wording here is broken. It's not reasonable to expect a
 // diagnostic here. Once the relevant DR gets a number, mark this as a dup.
 
-namespace dr522 { // dr522: yes
+namespace cwg522 { // cwg522: yes
   struct S {};
   template<typename T> void b1(volatile T &);
   template<typename T> void b2(volatile T * const *);
   template<typename T> void b2(volatile T * const S::*);
   template<typename T> void b2(volatile T * const S::* const *);
-  template<typename T> void b2a(volatile T *S::* const *); // #dr522-b2a
+  template<typename T> void b2a(volatile T *S::* const *); // #cwg522-b2a
 
   template<typename T> struct Base {};
   struct Derived : Base<int> {};
@@ -178,28 +178,28 @@ namespace dr522 { // dr522: yes
     b2(am);
     b2a(am);
     // expected-error@-1 {{no matching function for call to 'b2a'}}
-    //   expected-note@#dr522-b2a {{candidate template ignored: deduced type 'volatile int *dr522::S::*const *' of 1st parameter does not match adjusted type 'int *dr522::S::**' of argument}}
+    //   expected-note@#cwg522-b2a {{candidate template ignored: deduced type 'volatile int *cwg522::S::*const *' of 1st parameter does not match adjusted type 'int *cwg522::S::**' of argument}}
     b3(d);
     b3(cd);
   }
 }
 
-namespace dr524 { // dr524: yes
+namespace cwg524 { // cwg524: yes
   template<typename T> void f(T a, T b) { operator+(a, b); }
   // expected-error@-1 {{call to function 'operator+' that is neither visible in the template definition nor found by argument-dependent lookup}}
-  //   expected-note@#dr524-f-N-S {{in instantiation of function template specialization 'dr524::f<dr524::N::S>' requested here}}
-  //   expected-note@#dr524-operator-plus {{'operator+' should be declared prior to the call site or in namespace 'dr524::N'}}
+  //   expected-note@#cwg524-f-N-S {{in instantiation of function template specialization 'cwg524::f<cwg524::N::S>' requested here}}
+  //   expected-note@#cwg524-operator-plus {{'operator+' should be declared prior to the call site or in namespace 'cwg524::N'}}
 
   struct S {};
   void operator+(S, S);
   template void f(S, S);
 
   namespace N { struct S {}; }
-  void operator+(N::S, N::S); // #dr524-operator-plus
-  template void f(N::S, N::S); // #dr524-f-N-S
+  void operator+(N::S, N::S); // #cwg524-operator-plus
+  template void f(N::S, N::S); // #cwg524-f-N-S
 }
 
-namespace dr525 { // dr525: yes
+namespace cwg525 { // cwg525: yes
   namespace before {
     // Note, the example was correct prior to the change; instantiation is
     // required for cases like this:
@@ -211,37 +211,37 @@ namespace dr525 { // dr525: yes
   namespace after {
     template <class T> struct D { typename T::error e; };
     // expected-error@-1 {{type 'double' cannot be used prior to '::' because it has no members}}
-    //   expected-note@#dr525-ppp {{in instantiation of template class 'dr525::after::D<double>' requested here}}
+    //   expected-note@#cwg525-ppp {{in instantiation of template class 'cwg525::after::D<double>' requested here}}
     void g(D<double> *ppp) {
-      delete ppp; // #dr525-ppp
+      delete ppp; // #cwg525-ppp
     }
   }
 }
 
-namespace dr526 { // dr526: yes
+namespace cwg526 { // cwg526: yes
   template<int> struct S {};
   template<int N> void f1(S<N> s);
-  template<int N> void f2(S<(N)> s); // #dr526-f2
-  template<int N> void f3(S<+N> s); // #dr526-f3
+  template<int N> void f2(S<(N)> s); // #cwg526-f2
+  template<int N> void f3(S<+N> s); // #cwg526-f3
   template<int N> void g1(int (&)[N]);
-  template<int N> void g2(int (&)[(N)]); // #dr526-g2
-  template<int N> void g3(int (&)[+N]); // #dr526-g3
+  template<int N> void g2(int (&)[(N)]); // #cwg526-g2
+  template<int N> void g3(int (&)[+N]); // #cwg526-g3
 
   void test(int (&a)[3], S<3> s) {
     f1(s);
     f2(s);
     // expected-error@-1 {{no matching function for call to 'f2'}}
-    //   expected-note@#dr526-f2 {{candidate template ignored: couldn't infer template argument 'N'}}
+    //   expected-note@#cwg526-f2 {{candidate template ignored: couldn't infer template argument 'N'}}
     f3(s);
     // expected-error@-1 {{no matching function for call to 'f3'}}
-    //   expected-note@#dr526-f3 {{candidate template ignored: couldn't infer template argument 'N'}}
+    //   expected-note@#cwg526-f3 {{candidate template ignored: couldn't infer template argument 'N'}}
     g1(a);
     g2(a);
     // expected-error@-1 {{no matching function for call to 'g2'}}
-    //   expected-note@#dr526-g2 {{candidate template ignored: couldn't infer template argument 'N'}}
+    //   expected-note@#cwg526-g2 {{candidate template ignored: couldn't infer template argument 'N'}}
     g3(a);
     // expected-error@-1 {{no matching function for call to 'g3'}}
-    //   expected-note@#dr526-g3 {{candidate template ignored: couldn't infer template argument 'N'}}
+    //   expected-note@#cwg526-g3 {{candidate template ignored: couldn't infer template argument 'N'}}
   }
 
   template<int N> struct X {
@@ -254,34 +254,34 @@ namespace dr526 { // dr526: yes
   };
 }
 
-namespace dr527 { // dr527: na
+namespace cwg527 { // cwg527: na
   // This DR is meaningless. It removes a required diagnostic from the case
   // where a not-externally-visible object is odr-used but not defined, which
   // requires a diagnostic for a different reason.
-  extern struct { int x; } a; // FIXME: We should reject this, per dr389.
+  extern struct { int x; } a; // FIXME: We should reject this, per cwg389.
   static struct { int x; } b;
   extern "C" struct { int x; } c;
   namespace { extern struct { int x; } d; }
   typedef struct { int x; } *P;
-  struct E { static P e; }; // FIXME: We should reject this, per dr389.
+  struct E { static P e; }; // FIXME: We should reject this, per cwg389.
   namespace { struct F { static P f; }; }
 
   int ax = a.x, bx = b.x, cx = c.x, dx = d.x, ex = E::e->x, fx = F::f->x;
 }
 
-namespace dr528 { // dr528: 2.7
+namespace cwg528 { // cwg528: 2.7
 
-struct S; // #dr528-S
+struct S; // #cwg528-S
 
 void f() {
   typeid(S);
   // expected-error@-1 {{'typeid' of incomplete type 'S'}}
-  //   expected-note@#dr528-S {{forward declaration of 'dr528::S'}}
+  //   expected-note@#cwg528-S {{forward declaration of 'cwg528::S'}}
 }
 
-} // namespace dr528
+} // namespace cwg528
 
-namespace dr530 { // dr530: yes
+namespace cwg530 { // cwg530: yes
   template<int*> struct S { enum { N = 1 }; };
   template<void(*)()> struct T { enum { N = 1 }; };
   int n;
@@ -290,7 +290,7 @@ namespace dr530 { // dr530: yes
   int b[T<&f>::N];
 }
 
-namespace dr531 { // dr531: partial
+namespace cwg531 { // cwg531: partial
   namespace good {
     template<typename T> struct A {
       void f(T) { T::error; }
@@ -334,7 +334,7 @@ namespace dr531 { // dr531: partial
       void f(T) { T::error; }
       template<typename U> void g(T, U) { T::error; }
       struct B { typename T::error error; };
-      template<typename U> struct C { typename T::error error; }; // #dr531-C
+      template<typename U> struct C { typename T::error error; }; // #cwg531-C
       static T n;
     };
     template<typename T> T A<T>::n = T::error;
@@ -342,20 +342,20 @@ namespace dr531 { // dr531: partial
     void A<int>::f(int) {}
     // expected-error@-1 {{template specialization requires 'template<>'}}
     template<typename U> void A<int>::g(int, U) {}
-    // expected-error@-1 {{template parameter list matching the non-templated nested type 'dr531::bad::A<int>' should be empty}}
+    // expected-error@-1 {{template parameter list matching the non-templated nested type 'cwg531::bad::A<int>' should be empty}}
     struct A<int>::B {};
     // expected-error@-1 {{template specialization requires 'template<>'}}
     template<typename U> struct A<int>::C {};
-    // expected-error@-1 {{template parameter list matching the non-templated nested type 'dr531::bad::A<int>' should be empty}}
+    // expected-error@-1 {{template parameter list matching the non-templated nested type 'cwg531::bad::A<int>' should be empty}}
     // expected-error@-2 {{redefinition of 'C' as different kind of symbol}}
-    //   expected-note@#dr531-C {{previous definition is here}}
+    //   expected-note@#cwg531-C {{previous definition is here}}
     int A<int>::n = 0;
     // expected-error@-1 {{template specialization requires 'template<>'}}
 
-    template<> struct A<char> { // #dr531-A-char
+    template<> struct A<char> { // #cwg531-A-char
       void f(char);
       template<typename U> void g(char, U);
-      struct B; // #dr531-B
+      struct B; // #cwg531-B
       template<typename U> struct C;
       static char n;
     };
@@ -364,14 +364,14 @@ namespace dr531 { // dr531: partial
     // expected-error@-1 {{no function template matches function template specialization 'f'}}
     template<> template<typename U> void A<char>::g(char, U) {}
     // expected-error@-1 {{extraneous template parameter list in template specialization}}
-    //   expected-note@#dr531-A-char {{'template<>' header not required for explicitly-specialized class 'dr531::bad::A<char>' declared here}}
+    //   expected-note@#cwg531-A-char {{'template<>' header not required for explicitly-specialized class 'cwg531::bad::A<char>' declared here}}
     template<> struct A<char>::B {};
     // expected-error@-1 {{extraneous 'template<>' in declaration of struct 'B'}}
-    // expected-error@-2 {{specialization of member 'dr531::bad::A<char>::B' does not specialize an instantiated member}}
-    //  expected-note@#dr531-B {{attempt to specialize declaration here}}
+    // expected-error@-2 {{specialization of member 'cwg531::bad::A<char>::B' does not specialize an instantiated member}}
+    //  expected-note@#cwg531-B {{attempt to specialize declaration here}}
     template<> template<typename U> struct A<char>::C {};
     // expected-error@-1 {{extraneous template parameter list in template specialization}}
-    //   expected-note@#dr531-A-char {{'template<>' header not required for explicitly-specialized class 'dr531::bad::A<char>' declared here}}
+    //   expected-note@#cwg531-A-char {{'template<>' header not required for explicitly-specialized class 'cwg531::bad::A<char>' declared here}}
     template<> char A<char>::n = 0;
     // expected-error@-1 {{extraneous 'template<>' in declaration of variable 'n'}}
   }
@@ -388,11 +388,11 @@ namespace dr531 { // dr531: partial
     };
     template<> template<typename U> void A<int>::B<U>::f() {}
     template<typename U> void A<int>::B<U>::g() {}
-    // expected-error@-1 {{template parameter list matching the non-templated nested type 'dr531::nested::A<int>' should be empty ('template<>')}}
+    // expected-error@-1 {{template parameter list matching the non-templated nested type 'cwg531::nested::A<int>' should be empty ('template<>')}}
 
     template<> template<typename U> template<typename V> void A<int>::B<U>::h() {}
     template<typename U> template<typename V> void A<int>::B<U>::i() {}
-    // expected-error@-1 {{template parameter list matching the non-templated nested type 'dr531::nested::A<int>' should be empty ('template<>')}}
+    // expected-error@-1 {{template parameter list matching the non-templated nested type 'cwg531::nested::A<int>' should be empty ('template<>')}}
 
 #if __cplusplus <= 201703L
     // FIXME: All of those declarations shouldn't crash in C++20 mode.
@@ -403,13 +403,13 @@ namespace dr531 { // dr531: partial
     template<> void A<int>::B<char>::f() {}
     // cxx98-17-error@-1 {{template specialization requires 'template<>'}}
     template<> template<typename V> void A<int>::B<char>::h() {}
-    // cxx98-17-error@-1 {{template parameter list matching the non-templated nested type 'dr531::nested::A<int>::B<char>' should be empty ('template<>')}}
+    // cxx98-17-error@-1 {{template parameter list matching the non-templated nested type 'cwg531::nested::A<int>::B<char>' should be empty ('template<>')}}
 #endif
   }
 }
 
 // PR8130
-namespace dr532 { // dr532: 3.5
+namespace cwg532 { // cwg532: 3.5
   struct A { };
 
   template<class T> struct B {
@@ -424,16 +424,16 @@ namespace dr532 { // dr532: 3.5
   }
 }
 
-// dr533: na
+// cwg533: na
 
-namespace dr534 { // dr534: 2.9
+namespace cwg534 { // cwg534: 2.9
   struct S {};
   template<typename T> void operator+(S, T);
   template<typename T> void operator+<T*>(S, T*) {}
   // expected-error@-1 {{function template partial specialization is not allowed}}
 }
 
-namespace dr535 { // dr535: yes
+namespace cwg535 { // cwg535: yes
   class X { private: X(const X&); };
   struct A {
     X x;
@@ -464,12 +464,12 @@ namespace dr535 { // dr535: yes
 #endif
 }
 
-// dr536: na
-// dr537: na
-// dr538: na
+// cwg536: na
+// cwg537: na
+// cwg538: na
 
-// dr539: yes
-const dr539(
+// cwg539: yes
+const cwg539(
 // expected-error@-1 {{a type specifier is required for all declarations}}
     const a) {
     // expected-error@-1 {{unknown type name 'a'}}
@@ -521,21 +521,21 @@ const dr539(
 #endif
 }
 
-namespace dr540 { // dr540: yes
+namespace cwg540 { // cwg540: yes
   typedef int &a;
   typedef const a &a;
   // expected-warning@-1 {{'const' qualifier on reference type 'a' (aka 'int &') has no effect}}
   typedef const int &b;
   typedef b &b;
-  typedef const a &c; // #dr540-typedef-a-c
+  typedef const a &c; // #cwg540-typedef-a-c
   // expected-warning@-1 {{'const' qualifier on reference type 'a' (aka 'int &') has no effect}}
-  typedef const b &c; // #dr540-typedef-b-c
-  // expected-error@#dr540-typedef-b-c {{typedef redefinition with different types ('const int &' vs 'int &')}}
-  //   expected-note@#dr540-typedef-a-c {{previous definition is here}}
-  // expected-warning@#dr540-typedef-b-c {{'const' qualifier on reference type 'b' (aka 'const int &') has no effect}}
+  typedef const b &c; // #cwg540-typedef-b-c
+  // expected-error@#cwg540-typedef-b-c {{typedef redefinition with different types ('const int &' vs 'int &')}}
+  //   expected-note@#cwg540-typedef-a-c {{previous definition is here}}
+  // expected-warning@#cwg540-typedef-b-c {{'const' qualifier on reference type 'b' (aka 'const int &') has no effect}}
 }
 
-namespace dr541 { // dr541: yes
+namespace cwg541 { // cwg541: yes
   template<int> struct X { typedef int type; };
   template<typename T> struct S {
     int f(T);
@@ -565,45 +565,45 @@ namespace dr541 { // dr541: yes
   };
 }
 
-namespace dr542 { // dr542: yes
+namespace cwg542 { // cwg542: yes
 #if __cplusplus >= 201103L
   // In C++20 A and B are no longer aggregates and thus the constructor is
   // called, which fails.
-  struct A { A() = delete; int n; }; // #dr542-A
+  struct A { A() = delete; int n; }; // #cwg542-A
   // ok, constructor not called
-  A a[32] = {}; // #dr542-a
+  A a[32] = {}; // #cwg542-a
   // since-cxx20-error@-1 {{call to deleted constructor of 'A'}}
-  //   since-cxx20-note@#dr542-A {{'A' has been explicitly marked deleted here}}
-  //   since-cxx20-note@#dr542-a {{in implicit initialization of array element 0 with omitted initializer}}
+  //   since-cxx20-note@#cwg542-A {{'A' has been explicitly marked deleted here}}
+  //   since-cxx20-note@#cwg542-a {{in implicit initialization of array element 0 with omitted initializer}}
 
   struct B {
     int n;
   private:
-    B() = default; // #dr542-B-ctor
+    B() = default; // #cwg542-B-ctor
   };
   B b[32] = {}; // ok, constructor not called
-  // since-cxx20-error@-1 {{calling a private constructor of class 'dr542::B'}}
-  //   since-cxx20-note@#dr542-B-ctor {{declared private here}}
+  // since-cxx20-error@-1 {{calling a private constructor of class 'cwg542::B'}}
+  //   since-cxx20-note@#cwg542-B-ctor {{declared private here}}
 #endif
 }
 
-namespace dr543 { // dr543: 3.0
-  // In C++98+DR543, this is valid because value-initialization doesn't call a
+namespace cwg543 { // cwg543: 3.0
+  // In C++98+CWG543, this is valid because value-initialization doesn't call a
   // trivial default constructor, so we never notice that defining the
   // constructor would be ill-formed.
   //
-  // In C++11+DR543, this is ill-formed, because the default constructor is
+  // In C++11+CWG543, this is ill-formed, because the default constructor is
   // deleted, and value-initialization *does* call a deleted default
   // constructor, even if it is trivial.
   struct A {
-    const int n; // #dr543-A-n
+    const int n; // #cwg543-A-n
   };
   A a = A();
   // since-cxx11-error@-1 {{call to implicitly-deleted default constructor of 'A'}}
-  //   since-cxx11-note@#dr543-A-n {{default constructor of 'A' is implicitly deleted because field 'n' of const-qualified type 'const int' would not be initialized}}
+  //   since-cxx11-note@#cwg543-A-n {{default constructor of 'A' is implicitly deleted because field 'n' of const-qualified type 'const int' would not be initialized}}
 }
 
-namespace dr544 { // dr544: yes
+namespace cwg544 { // cwg544: yes
   int *n;
 
   template<class T> struct A { int n; };
@@ -612,13 +612,13 @@ namespace dr544 { // dr544: yes
   int k = B<int>().get();
 }
 
-namespace dr546 { // dr546: yes
+namespace cwg546 { // cwg546: yes
   template<typename T> struct A { void f(); };
   template struct A<int>;
   template<typename T> void A<T>::f() { T::error; }
 }
 
-namespace dr547 { // dr547: 3.2
+namespace cwg547 { // cwg547: 3.2
   template<typename T> struct X;
   template<typename T> struct X<T() const> {};
   template<typename T, typename C> X<T> f(T C::*) { return X<T>(); }
@@ -627,16 +627,16 @@ namespace dr547 { // dr547: 3.2
   X<void() const> x = f(&S::f);
 }
 
-namespace dr548 { // dr548: dup 482
+namespace cwg548 { // cwg548: dup 482
   template<typename T> struct S {};
   template<typename T> void f() {}
-  template struct dr548::S<int>;
-  template void dr548::f<int>();
+  template struct cwg548::S<int>;
+  template void cwg548::f<int>();
 }
 
-// dr550: dup 393
+// cwg550: dup 393
 
-namespace dr551 { // dr551: yes c++11
+namespace cwg551 { // cwg551: yes c++11
   // FIXME: This obviously should apply in C++98 mode too.
   template<typename T> void f() {}
   template inline void f<int>();
@@ -653,25 +653,25 @@ namespace dr551 { // dr551: yes c++11
   // since-cxx11-error@-1 {{explicit instantiation cannot be 'inline'}}
 }
 
-namespace dr552 { // dr552: yes
+namespace cwg552 { // cwg552: yes
   template<typename T, typename T::U> struct X {};
   struct Y { typedef int U; };
   X<Y, 0> x;
 }
 
-// dr553: 2.7
-struct dr553_class {
-  friend void *operator new(size_t, dr553_class);
+// cwg553: 2.7
+struct cwg553_class {
+  friend void *operator new(size_t, cwg553_class);
 };
-namespace dr553 {
-  dr553_class c;
+namespace cwg553 {
+  cwg553_class c;
   // Contrary to the apparent intention of the DR, operator new is not actually
   // looked up with a lookup mechanism that performs ADL; the standard says it
   // "is looked up in global scope", where it is not visible.
   void *p = new (c) int;
   // expected-error@-1 {{no matching function for call to 'operator new'}}
-  //   since-cxx17-note@#dr5xx-global-operator-new-aligned {{candidate function not viable: no known conversion from 'dr553_class' to 'std::align_val_t' for 2nd argument}}
-  //   expected-note@#dr5xx-global-operator-new {{candidate function not viable: requires 1 argument, but 2 were provided}}
+  //   since-cxx17-note@#cwg5xx-global-operator-new-aligned {{candidate function not viable: no known conversion from 'cwg553_class' to 'std::align_val_t' for 2nd argument}}
+  //   expected-note@#cwg5xx-global-operator-new {{candidate function not viable: requires 1 argument, but 2 were provided}}
 
   struct namespace_scope {
     friend void *operator new(size_t, namespace_scope);
@@ -679,9 +679,9 @@ namespace dr553 {
   };
 }
 
-// dr554: na
+// cwg554: na
 
-namespace dr555 { // dr555: 2.8
+namespace cwg555 { // cwg555: 2.8
 typedef int I;
 typedef const int CI;
 typedef volatile int VI;
@@ -734,11 +734,11 @@ void g(int &a, CI &b, VI &c) {
   c.CI::~CI();
   c.VI::~VI();
 }
-} // namespace dr555
+} // namespace cwg555
 
-// dr556: na
+// cwg556: na
 
-namespace dr557 { // dr557: 3.1
+namespace cwg557 { // cwg557: 3.1
   template<typename T> struct S {
     friend void f(S<T> *);
     friend void g(S<S<T> > *);
@@ -749,7 +749,7 @@ namespace dr557 { // dr557: 3.1
   }
 }
 
-namespace dr558 { // dr558: 2.9
+namespace cwg558 { // cwg558: 2.9
   wchar_t a = L'\uD7FF';
   wchar_t b = L'\xD7FF';
   wchar_t c = L'\uD800';
@@ -762,9 +762,9 @@ namespace dr558 { // dr558: 2.9
   wchar_t h = L'\xE000';
 }
 
-template<typename> struct dr559 { typedef int T; dr559::T u; }; // dr559: yes
+template<typename> struct cwg559 { typedef int T; cwg559::T u; }; // cwg559: yes
 
-namespace dr560 { // dr560: 16
+namespace cwg560 { // cwg560: 16
 
 template <class T>
 struct Outer {
@@ -776,9 +776,9 @@ template <class T>
 Outer<T>::Inner* Outer<T>::Inner::self() { return this; }
 // cxx98-17-error@-1 {{missing 'typename' prior to dependent type name Outer<T>::Inner; implicit 'typename' is a C++20 extension}}
 
-} // namespace dr560
+} // namespace cwg560
 
-namespace dr561 { // dr561: yes
+namespace cwg561 { // cwg561: yes
   template<typename T> void f(int);
   template<typename T> void g(T t) {
     f<T>(t);
@@ -792,20 +792,20 @@ namespace dr561 { // dr561: yes
   }
 }
 
-// dr562: na
+// cwg562: na
 
-namespace dr564 { // dr564: yes
+namespace cwg564 { // cwg564: yes
   extern "C++" void f(int);
   void f(int); // ok
   extern "C++" { extern int n; }
   int n; // ok
 }
 
-namespace dr565 { // dr565: yes
+namespace cwg565 { // cwg565: yes
   namespace N {
-    template<typename T> int f(T); // #dr565-f
+    template<typename T> int f(T); // #cwg565-f
   }
-  using N::f; // #dr565-using
+  using N::f; // #cwg565-using
   template<typename T> int f(T*);
   template<typename T> void f(T);
   template<typename T, int = 0> int f(T);
@@ -813,19 +813,19 @@ namespace dr565 { // dr565: yes
   template<typename T> int f(T, int = 0);
   template<typename T> int f(T);
   // expected-error@-1 {{declaration conflicts with target of using declaration already in scope}}
-  //   expected-note@#dr565-f {{target of using declaration}}
-  //   expected-note@#dr565-using {{using declaration}}
+  //   expected-note@#cwg565-f {{target of using declaration}}
+  //   expected-note@#cwg565-using {{using declaration}}
 }
 
-namespace dr566 { // dr566: yes
+namespace cwg566 { // cwg566: yes
 #if __cplusplus >= 201103L
   int check[int(-3.99) == -3 ? 1 : -1];
 #endif
 }
 
-// dr567: na
+// cwg567: na
 
-namespace dr568 { // dr568: 3.0 c++11
+namespace cwg568 { // cwg568: 3.0 c++11
   // FIXME: This is a DR issue against C++98, so should probably apply there
   // too.
   struct x { int y; };
@@ -860,35 +860,35 @@ namespace dr568 { // dr568: 3.0 c++11
   void jump() {
     goto x;
     // cxx98-error@-1 {{cannot jump from this goto statement to its label}}
-    //   cxx98-note@#dr568-t {{jump bypasses initialization of non-POD variable}}
-    trivial t; // #dr568-t
+    //   cxx98-note@#cwg568-t {{jump bypasses initialization of non-POD variable}}
+    trivial t; // #cwg568-t
   x: ;
   }
 }
 
-namespace dr569 { // dr569: yes c++11
+namespace cwg569 { // cwg569: yes c++11
   // FIXME: This is a DR issue against C++98, so should probably apply there
   // too.
   ;;;;;
   // cxx98-error@-1 {{C++11 extension}}
 }
 
-namespace dr570 { // dr570: dup 633
+namespace cwg570 { // cwg570: dup 633
   int n;
-  int &r = n; // #dr570-r
+  int &r = n; // #cwg570-r
   int &r = n;
   // expected-error@-1 {{redefinition of 'r'}}
-  //   expected-note@#dr570-r {{previous definition is here}}
+  //   expected-note@#cwg570-r {{previous definition is here}}
 }
 
-// dr571 is in dr571.cpp
+// cwg571 is in cwg571.cpp
 
-namespace dr572 { // dr572: yes
+namespace cwg572 { // cwg572: yes
   enum E { a = 1, b = 2 };
   int check[a + b == 3 ? 1 : -1];
 }
 
-namespace dr573 { // dr573: no
+namespace cwg573 { // cwg573: no
   void *a;
   int *b = reinterpret_cast<int*>(a);
   void (*c)() = reinterpret_cast<void(*)()>(a);
@@ -904,62 +904,62 @@ namespace dr573 { // dr573: no
   template<int*> struct T;
 }
 
-namespace dr574 { // dr574: 3.0
+namespace cwg574 { // cwg574: 3.0
   struct A {
-    A &operator=(const A&) const; // #dr574-A-copy-assign
+    A &operator=(const A&) const; // #cwg574-A-copy-assign
   };
   struct B {
-    B &operator=(const B&) volatile; // #dr574-B-copy-assign
+    B &operator=(const B&) volatile; // #cwg574-B-copy-assign
   };
 #if __cplusplus >= 201103L
   struct C {
-    C &operator=(const C&) &; // #dr574-C-copy-assign
+    C &operator=(const C&) &; // #cwg574-C-copy-assign
   };
   struct D {
-    D &operator=(const D&) &&; // #dr574-D-copy-assign
+    D &operator=(const D&) &&; // #cwg574-D-copy-assign
   };
   void test(C c, D d) {
     c = c;
     C() = c;
     // since-cxx11-error@-1 {{no viable overloaded '='}}
-    //   since-cxx11-note@#dr574-C-copy-assign {{candidate function not viable: expects an lvalue for object argument}}
+    //   since-cxx11-note@#cwg574-C-copy-assign {{candidate function not viable: expects an lvalue for object argument}}
     d = d;
     // since-cxx11-error@-1 {{no viable overloaded '='}}
-    //   since-cxx11-note@#dr574-D-copy-assign {{candidate function not viable: expects an rvalue for object argument}}
+    //   since-cxx11-note@#cwg574-D-copy-assign {{candidate function not viable: expects an rvalue for object argument}}
     D() = d;
   }
 #endif
   struct Test {
     friend A &A::operator=(const A&);
-    // expected-error@-1 {{friend declaration of 'operator=' does not match any declaration in 'dr574::A'}}
-    //   expected-note@#dr574-A-copy-assign {{candidate function has different qualifiers (expected unqualified but found 'const')}}
+    // expected-error@-1 {{friend declaration of 'operator=' does not match any declaration in 'cwg574::A'}}
+    //   expected-note@#cwg574-A-copy-assign {{candidate function has different qualifiers (expected unqualified but found 'const')}}
     friend B &B::operator=(const B&);
-    // expected-error@-1 {{friend declaration of 'operator=' does not match any declaration in 'dr574::B'}}
-    //   expected-note@#dr574-B-copy-assign {{candidate function has different qualifiers (expected unqualified but found 'volatile')}}
+    // expected-error@-1 {{friend declaration of 'operator=' does not match any declaration in 'cwg574::B'}}
+    //   expected-note@#cwg574-B-copy-assign {{candidate function has different qualifiers (expected unqualified but found 'volatile')}}
 #if __cplusplus >= 202302L
     friend C &C::operator=(const C&);
     // since-cxx23-error@-1 {{conflicting types for 'operator='}}
-    //   since-cxx23-note@#dr574-C-copy-assign {{previous declaration is here}}
+    //   since-cxx23-note@#cwg574-C-copy-assign {{previous declaration is here}}
     friend D &D::operator=(const D&);
     // since-cxx23-error@-1 {{conflicting types for 'operator='}}
-    //   since-cxx23-note@#dr574-D-copy-assign {{previous declaration is here}}
+    //   since-cxx23-note@#cwg574-D-copy-assign {{previous declaration is here}}
 #elif __cplusplus >= 201103L
     // FIXME: We shouldn't produce the 'cannot overload' diagnostics here.
-    friend C &C::operator=(const C&); // #dr574-test-C
-    // since-cxx11-error@#dr574-test-C {{cannot overload}}
-    //   since-cxx11-note@#dr574-C-copy-assign {{previous declaration is here}}
-    // since-cxx11-error@#dr574-test-C {{friend declaration of 'operator=' does not match any declaration in 'dr574::C'}}
-    //   since-cxx11-note@#dr574-C-copy-assign {{candidate function}}
-    friend D &D::operator=(const D&); // #dr574-test-D
-    // since-cxx11-error@#dr574-test-D {{cannot overload a member function without a ref-qualifier with a member function with ref-qualifier '&&'}}
-    //   since-cxx11-note@#dr574-D-copy-assign {{previous declaration is here}}
-    // since-cxx11-error@#dr574-test-D {{friend declaration of 'operator=' does not match any declaration in 'dr574::D'}}
-    //   since-cxx11-note@#dr574-D-copy-assign {{candidate function}}
+    friend C &C::operator=(const C&); // #cwg574-test-C
+    // since-cxx11-error@#cwg574-test-C {{cannot overload}}
+    //   since-cxx11-note@#cwg574-C-copy-assign {{previous declaration is here}}
+    // since-cxx11-error@#cwg574-test-C {{friend declaration of 'operator=' does not match any declaration in 'cwg574::C'}}
+    //   since-cxx11-note@#cwg574-C-copy-assign {{candidate function}}
+    friend D &D::operator=(const D&); // #cwg574-test-D
+    // since-cxx11-error@#cwg574-test-D {{cannot overload a member function without a ref-qualifier with a member function with ref-qualifier '&&'}}
+    //   since-cxx11-note@#cwg574-D-copy-assign {{previous declaration is here}}
+    // since-cxx11-error@#cwg574-test-D {{friend declaration of 'operator=' does not match any declaration in 'cwg574::D'}}
+    //   since-cxx11-note@#cwg574-D-copy-assign {{candidate function}}
 #endif
   };
 }
 
-namespace dr575 { // dr575: yes
+namespace cwg575 { // cwg575: yes
   template<typename T, typename U = typename T::type> void a(T); void a(...);
   // cxx98-error@-1 {{default template arguments for a function template are a C++11 extension}}
   template<typename T, typename T::type U = 0> void b(T); void b(...);
@@ -968,12 +968,12 @@ namespace dr575 { // dr575: yes
   // cxx98-error@-1 {{default template arguments for a function template are a C++11 extension}}
   template<typename T> void d(T, int = T::value); void d(...);
   // expected-error@-1 {{type 'int' cannot be used prior to '::' because it has no members}}
-  //   expected-note@#dr575-d {{in instantiation of default function argument expression for 'd<int>' required here}}
+  //   expected-note@#cwg575-d {{in instantiation of default function argument expression for 'd<int>' required here}}
   void x() {
     a(0);
     b(0);
     c(0);
-    d(0); // #dr575-d
+    d(0); // #cwg575-d
   }
 
   template<typename T = int&> void f(T* = 0);
@@ -987,7 +987,7 @@ namespace dr575 { // dr575: yes
   void *p = h((void*)0);
 }
 
-namespace dr576 { // dr576: 3.5
+namespace cwg576 { // cwg576: 3.5
   typedef void f() {}
   // expected-error@-1 {{function definition declared 'typedef'}}
   void f(typedef int n);
@@ -995,7 +995,7 @@ namespace dr576 { // dr576: 3.5
   void f(char c) { typedef int n; }
 }
 
-namespace dr577 { // dr577: 3.5
+namespace cwg577 { // cwg577: 3.5
   typedef void V;
   typedef const void CV;
   void a(void);
@@ -1010,35 +1010,35 @@ namespace dr577 { // dr577: 3.5
   void (*g)(V) = a;
   void (*h)(CV);
   // expected-error@-1 {{'void' as parameter must not have type qualifiers}}
-  template<typename T> void i(T); // #dr577-i
-  template<typename T> void j(void (*)(T)); // #dr577-j
+  template<typename T> void i(T); // #cwg577-i
+  template<typename T> void j(void (*)(T)); // #cwg577-j
   void k() {
     a();
     c();
     i<void>();
     // expected-error@-1 {{no matching function for call to 'i'}}
-    //   expected-note@#dr577-i {{candidate function template not viable: requires 1 argument, but 0 were provided}}
+    //   expected-note@#cwg577-i {{candidate function template not viable: requires 1 argument, but 0 were provided}}
     i<const void>();
     // expected-error@-1 {{no matching function for call to 'i'}}
-    //   expected-note@#dr577-i {{candidate function template not viable: requires 1 argument, but 0 were provided}}
+    //   expected-note@#cwg577-i {{candidate function template not viable: requires 1 argument, but 0 were provided}}
     j<void>(0);
     // expected-error@-1 {{no matching function for call to 'j'}}
-    //   expected-note@#dr577-j {{candidate template ignored: substitution failure [with T = void]: argument may not have 'void' type}}
+    //   expected-note@#cwg577-j {{candidate template ignored: substitution failure [with T = void]: argument may not have 'void' type}}
     j<const void>(0);
     // expected-error@-1 {{no matching function for call to 'j'}}
-    //   expected-note@#dr577-j {{candidate template ignored: substitution failure [with T = const void]: argument may not have 'void' type}}
+    //   expected-note@#cwg577-j {{candidate template ignored: substitution failure [with T = const void]: argument may not have 'void' type}}
   }
 }
 
-namespace dr580 { // dr580: partial
+namespace cwg580 { // cwg580: partial
   class C;
   struct A { static C c; };
   struct B { static C c; };
   class C {
-    C(); // #dr580-C-ctor
-    ~C(); // #dr580-C-dtor
+    C(); // #cwg580-C-ctor
+    ~C(); // #cwg580-C-dtor
 
-    typedef int I; // #dr580-I
+    typedef int I; // #cwg580-I
     template<int> struct X;
     template<int> friend struct Y;
     template<int> void f();
@@ -1049,8 +1049,8 @@ namespace dr580 { // dr580: partial
   template<C::I> struct C::X {};
   template<C::I> struct Y {};
   template<C::I> struct Z {};
-  // expected-error@-1 {{'I' is a private member of 'dr580::C'}}
-  //   expected-note@#dr580-I {{implicitly declared private here}}
+  // expected-error@-1 {{'I' is a private member of 'cwg580::C'}}
+  //   expected-note@#cwg580-I {{implicitly declared private here}}
 
   struct C2 {
     class X {
@@ -1069,20 +1069,20 @@ namespace dr580 { // dr580: partial
   template<C::I> void C::f() {}
   template<C::I> void g() {}
   template<C::I> void h() {}
-  // expected-error@-1 {{'I' is a private member of 'dr580::C'}}
-  //   expected-note@#dr580-I {{implicitly declared private here}}
+  // expected-error@-1 {{'I' is a private member of 'cwg580::C'}}
+  //   expected-note@#cwg580-I {{implicitly declared private here}}
 
   C A::c;
-  C B::c; // #dr580-c
-  // expected-error@#dr580-c {{calling a private constructor of class 'dr580::C'}}
-  //   expected-note@#dr580-C-ctor {{implicitly declared private here}}
-  // expected-error@#dr580-c {{variable of type 'C' has private destructor}}
-  //   expected-note@#dr580-C-dtor {{implicitly declared private here}}
+  C B::c; // #cwg580-c
+  // expected-error@#cwg580-c {{calling a private constructor of class 'cwg580::C'}}
+  //   expected-note@#cwg580-C-ctor {{implicitly declared private here}}
+  // expected-error@#cwg580-c {{variable of type 'C' has private destructor}}
+  //   expected-note@#cwg580-C-dtor {{implicitly declared private here}}
 }
 
-// dr582: na
+// cwg582: na
 
-namespace dr583 { // dr583: 4
+namespace cwg583 { // cwg583: 4
   // see n3624
   int *p;
   bool b1 = p < 0;
@@ -1095,35 +1095,35 @@ namespace dr583 { // dr583: 4
   // expected-error@-1 {{ordered comparison between pointer and zero ('int *' and 'int')}}
 }
 
-// dr584: na
+// cwg584: na
 
-namespace dr585 { // dr585: 3.0
-  template<typename> struct T; // #dr585-struct-T
+namespace cwg585 { // cwg585: 3.0
+  template<typename> struct T; // #cwg585-struct-T
   struct A {
     friend T;
     // cxx98-14-error@-1 {{a type specifier is required for all declarations}}
     // cxx98-14-error@-2 {{friends can only be classes or functions}}
     // since-cxx17-error@-3 {{use of class template 'T' requires template arguments; argument deduction not allowed in friend declaration}}
-    //   since-cxx17-note@#dr585-struct-T {{template is declared here}}
+    //   since-cxx17-note@#cwg585-struct-T {{template is declared here}}
     // FIXME: It's not clear whether the standard allows this or what it means,
-    // but the DR585 writeup suggests it as an alternative.
+    // but the CWG585 writeup suggests it as an alternative.
     template<typename U> friend T<U>;
     // expected-error@-1 {{friend type templates must use an elaborated type}}
   };
-  template<template<typename> class T> struct B { // #dr585-template-T
+  template<template<typename> class T> struct B { // #cwg585-template-T
     friend T;
     // cxx98-14-error@-1 {{a type specifier is required for all declarations}}
     // cxx98-14-error@-2 {{friends can only be classes or functions}}
     // since-cxx17-error@-3 {{use of template template parameter 'T' requires template arguments; argument deduction not allowed in friend declaration}}
-    //   since-cxx17-note@#dr585-template-T {{template is declared here}}
+    //   since-cxx17-note@#cwg585-template-T {{template is declared here}}
     template<typename U> friend T<U>;
     // expected-error@-1 {{friend type templates must use an elaborated type}}
   };
 }
 
-// dr586: na
+// cwg586: na
 
-namespace dr587 { // dr587: 3.2
+namespace cwg587 { // cwg587: 3.2
   template<typename T> void f(bool b, const T x, T y) {
     const T *p = &(b ? x : y);
   }
@@ -1132,22 +1132,22 @@ namespace dr587 { // dr587: 3.2
   template void f(bool, const S, S);
 }
 
-namespace dr588 { // dr588: yes
-  struct A { int n; }; // #dr588-A
+namespace cwg588 { // cwg588: yes
+  struct A { int n; }; // #cwg588-A
   template<typename T> int f() {
     struct S : A, T { int f() { return n; } } s;
     int a = s.f();
     int b = s.n;
     // expected-error@-1 {{member 'n' found in multiple base classes of different types}}
-    //   expected-note@#dr588-k {{in instantiation of function template specialization 'dr588::f<dr588::B>' requested here}}
-    //   expected-note@#dr588-A {{member found by ambiguous name lookup}}
-    //   expected-note@#dr588-B {{member found by ambiguous name lookup}}
+    //   expected-note@#cwg588-k {{in instantiation of function template specialization 'cwg588::f<cwg588::B>' requested here}}
+    //   expected-note@#cwg588-A {{member found by ambiguous name lookup}}
+    //   expected-note@#cwg588-B {{member found by ambiguous name lookup}}
   }
-  struct B { int n; }; // #dr588-B
-  int k = f<B>(); // #dr588-k
+  struct B { int n; }; // #cwg588-B
+  int k = f<B>(); // #cwg588-k
 }
 
-namespace dr589 { // dr589: yes
+namespace cwg589 { // cwg589: yes
   struct B { };
   struct D : B { };
   D f();
@@ -1159,7 +1159,7 @@ namespace dr589 { // dr589: yes
   // expected-error@-1 {{taking the address of a temporary object of type 'const B'}}
 }
 
-namespace dr590 { // dr590: yes
+namespace cwg590 { // cwg590: yes
   template<typename T> struct A {
     struct B {
       struct C {
@@ -1170,7 +1170,7 @@ namespace dr590 { // dr590: yes
   template<typename T> typename A<T>::B::C A<T>::B::C::f(A<T>::B::C) {}
 }
 
-namespace dr591 { // dr591: no
+namespace cwg591 { // cwg591: no
   template<typename T> struct A {
     typedef int M;
     struct B {
@@ -1186,11 +1186,11 @@ namespace dr591 { // dr591: no
   };
 }
 
-// dr592: na
-// dr593 is in dr593.cpp
-// dr594: na
+// cwg592: na
+// cwg593 is in cwg593.cpp
+// cwg594: na
 
-namespace dr595 { // dr595: dup 1330
+namespace cwg595 { // cwg595: dup 1330
   template<class T> struct X {
     void f() throw(T) {}
     // since-cxx17-error@-1 {{ISO C++17 does not allow dynamic exception specifications}}
@@ -1201,9 +1201,9 @@ namespace dr595 { // dr595: dup 1330
   };
 }
 
-// dr597: na
+// cwg597: na
 
-namespace dr598 { // dr598: yes
+namespace cwg598 { // cwg598: yes
   namespace N {
     void f(int);
     void f(char);
@@ -1225,11 +1225,11 @@ namespace dr598 { // dr598: yes
   int &t = h(N::i);
 }
 
-namespace dr599 { // dr599: partial
+namespace cwg599 { // cwg599: partial
   typedef int Fn();
   struct S { operator void*(); };
   struct T { operator Fn*(); };
-  struct U { operator int*(); operator void*(); }; // #dr599-U
+  struct U { operator int*(); operator void*(); }; // #cwg599-U
   struct V { operator int*(); operator Fn*(); };
   void f(void *p, void (*q)(), S s, T t, U u, V v) {
     delete p;
@@ -1244,8 +1244,8 @@ namespace dr599 { // dr599: partial
     // extension allowing deletion of pointers to void.
     delete u;
     // expected-error@-1 {{ambiguous conversion of delete expression of type 'U' to a pointer}}
-    //   expected-note@#dr599-U {{conversion to pointer type 'int *'}}
-    //   expected-note@#dr599-U {{conversion to pointer type 'void *'}}
+    //   expected-note@#cwg599-U {{conversion to pointer type 'int *'}}
+    //   expected-note@#cwg599-U {{conversion to pointer type 'void *'}}
     delete v;
   }
 }
diff --git a/clang/test/CXX/drs/dr6xx.cpp b/clang/test/CXX/drs/dr6xx.cpp
index eeb41eee9c30..d004d5aa4c79 100644
--- a/clang/test/CXX/drs/dr6xx.cpp
+++ b/clang/test/CXX/drs/dr6xx.cpp
@@ -5,29 +5,29 @@
 // RUN: %clang_cc1 -std=c++20 %s -verify=expected,cxx11-20,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -fno-spell-checking
 // RUN: %clang_cc1 -std=c++23 %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -fno-spell-checking
 
-namespace dr600 { // dr600: 2.8
+namespace cwg600 { // cwg600: 2.8
 struct S {
   void f(int);
 
 private:
-  void f(double); // #dr600-f-double
+  void f(double); // #cwg600-f-double
 };
 
 void g(S *sp) {
   sp->f(2);
   // access control is applied after overload resolution
   sp->f(2.2);
-  // expected-error@-1 {{'f' is a private member of 'dr600::S'}}
-  //   expected-note@#dr600-f-double {{declared private here}}
+  // expected-error@-1 {{'f' is a private member of 'cwg600::S'}}
+  //   expected-note@#cwg600-f-double {{declared private here}}
 }
-} // namespace dr600
+} // namespace cwg600
 
 namespace std {
   struct type_info {};
   __extension__ typedef __SIZE_TYPE__ size_t;
 } // namespace std
 
-namespace dr601 { // dr601: yes
+namespace cwg601 { // cwg601: yes
 #if __cplusplus >= 201103L
 #define MAX __LLONG_MAX__
 #else
@@ -58,7 +58,7 @@ static_assert(0x8000000000000000 < -1, "0x8000000000000000 should be unsigned");
 #undef MAX
 }
 
-namespace dr602 { // dr602: yes
+namespace cwg602 { // cwg602: yes
   template<class T> struct A {
     template<class U> friend struct A;
   };
@@ -73,31 +73,31 @@ namespace dr602 { // dr602: yes
   B<int> b;
 }
 
-namespace dr603 { // dr603: yes
+namespace cwg603 { // cwg603: yes
   template<unsigned char> struct S {};
   typedef S<'\001'> S1;
   typedef S<(1ul << __CHAR_BIT__) + 1> S1;
   // since-cxx11-error@-1 {{cannot be narrowed}}
 }
 
-// dr604: na
-// dr605 is in dr605.cpp
+// cwg604: na
+// cwg605 is in cwg605.cpp
 
-namespace dr606 { // dr606: 3.0
+namespace cwg606 { // cwg606: 3.0
 #if __cplusplus >= 201103L
   template<typename T> struct S {};
-  template<typename T> void f(S<T> &&); // #dr606-f
+  template<typename T> void f(S<T> &&); // #cwg606-f
   template<typename T> void g(T &&);
-  template<typename T> void h(const T &&); // #dr606-h
+  template<typename T> void h(const T &&); // #cwg606-h
 
   void test(S<int> s) {
     f(s);
     // since-cxx11-error@-1 {{no matching function for call to 'f'}}
-    //   since-cxx11-note@#dr606-f {{candidate function [with T = int] not viable: expects an rvalue for 1st argument}}
+    //   since-cxx11-note@#cwg606-f {{candidate function [with T = int] not viable: expects an rvalue for 1st argument}}
     g(s);
     h(s);
     // since-cxx11-error@-1 {{no matching function for call to 'h'}}
-    //   since-cxx11-note@#dr606-h {{candidate function [with T = dr606::S<int>] not viable: expects an rvalue for 1st argument}}
+    //   since-cxx11-note@#cwg606-h {{candidate function [with T = cwg606::S<int>] not viable: expects an rvalue for 1st argument}}
 
     g(test);
     h(test); // ok, an rvalue reference can bind to a function lvalue
@@ -105,7 +105,7 @@ namespace dr606 { // dr606: 3.0
 #endif
 }
 
-namespace dr607 { // dr607: yes
+namespace cwg607 { // cwg607: yes
 namespace example1 {
 struct Y {};
 
@@ -131,25 +131,25 @@ struct D : B {
 
 N::D::D() : typedef_B(0) {}
 } // namespace example2
-} // namespace dr607
+} // namespace cwg607
 
-namespace dr608 { // dr608: yes
+namespace cwg608 { // cwg608: yes
   struct A { virtual void f(); };
   struct B : A {};
   struct C : A { void f(); };
   struct D : B, C {};
 }
 
-int dr610[-0u == 0u ? 1 : -1]; // dr610: yes
+int cwg610[-0u == 0u ? 1 : -1]; // cwg610: yes
 
-namespace dr611 { // dr611: yes
+namespace cwg611 { // cwg611: yes
   int k;
   struct S { int &r; } s = { k ? k : k };
 }
 
-// dr612: na
+// cwg612: na
 
-namespace dr613 { // dr613: yes c++11
+namespace cwg613 { // cwg613: yes c++11
   // see also n2253
   struct A { int n; static void f(); };
   int f(int);
@@ -158,7 +158,7 @@ namespace dr613 { // dr613: yes c++11
 
   int an1 = sizeof(A::n);
   // cxx98-error@-1 {{invalid use of non-static data member 'n'}}
-  int an2 = sizeof(A::n + 1); // valid per dr850
+  int an2 = sizeof(A::n + 1); // valid per cwg850
   // cxx98-error@-1 {{invalid use of non-static data member 'n'}}
   int an3 = sizeof A::n;
   // cxx98-error@-1 {{invalid use of non-static data member 'n'}}
@@ -186,15 +186,15 @@ namespace dr613 { // dr613: yes c++11
   }
 }
 
-int dr614_a[(-1) / 2 == 0 ? 1 : -1]; // dr614: yes
-int dr614_b[(-1) % 2 == -1 ? 1 : -1];
+int cwg614_a[(-1) / 2 == 0 ? 1 : -1]; // cwg614: yes
+int cwg614_b[(-1) % 2 == -1 ? 1 : -1];
 
-namespace dr615 { // dr615: yes
+namespace cwg615 { // cwg615: yes
   int f();
   static int n = f();
 }
 
-namespace dr616 { // dr616: 4
+namespace cwg616 { // cwg616: 4
 #if __cplusplus >= 201103L
   struct S { int n; } s;
   S f();
@@ -212,13 +212,13 @@ namespace dr616 { // dr616: 4
 #endif
 }
 
-namespace dr618 { // dr618: yes
+namespace cwg618 { // cwg618: yes
 #if (unsigned)-1 > 0
 #error wrong
 #endif
 }
 
-namespace dr619 { // dr619: yes
+namespace cwg619 { // cwg619: yes
   extern int x[10];
   struct S { static int x[10]; };
 
@@ -240,22 +240,22 @@ namespace dr619 { // dr619: yes
   }
 }
 
-// dr620: dup 568
+// cwg620: dup 568
 
-namespace dr621 { // dr621: yes
+namespace cwg621 { // cwg621: yes
   template<typename T> T f();
-  template<> int f() {} // #dr621-f
+  template<> int f() {} // #cwg621-f
   template<> int f<int>() {}
   // expected-error@-1 {{redefinition of 'f<int>'}}
-  //   expected-note@#dr621-f {{previous definition is here}}
+  //   expected-note@#cwg621-f {{previous definition is here}}
 }
 
-// dr623: na
+// cwg623: na
 // FIXME: Add documentation saying we allow invalid pointer values.
 
-// dr624 needs a libc++abi test.
+// cwg624 needs a libc++abi test.
 
-namespace dr625 { // dr625: yes
+namespace cwg625 { // cwg625: yes
   template<typename T> struct A {};
   A<auto> x = A<int>();
   // cxx98-error@-1 {{'auto' type specifier is a C++11 extension}}
@@ -266,14 +266,14 @@ namespace dr625 { // dr625: yes
   // expected-error@-2 {{'auto' not allowed in function prototype}}
 }
 
-namespace dr626 { // dr626: yes
+namespace cwg626 { // cwg626: yes
 #define STR(x) #x
   char c[2] = STR(c); // ok, type matches
   wchar_t w[2] = STR(w);
   // expected-error@-1 {{initializing wide char array with non-wide string literal}}
 }
 
-namespace dr627 { // dr627: yes
+namespace cwg627 { // cwg627: yes
   void f() {
     // FIXME: emitted diagnostic have a room for improvement
     true a = 0;
@@ -283,22 +283,22 @@ namespace dr627 { // dr627: yes
   }
 }
 
-// dr628: na
+// cwg628: na
 
-namespace dr629 { // dr629: 2.9
+namespace cwg629 { // cwg629: 2.9
   typedef int T;
   int n = 1;
   void f() {
-    auto T = 2; // #dr629-T
+    auto T = 2; // #cwg629-T
     // cxx98-error@-1 {{expected unqualified-id}}
 
     auto T(n);
     // since-cxx11-error@-1 {{redefinition of 'T'}}
-    //   since-cxx11-note@#dr629-T {{previous definition is here}}
+    //   since-cxx11-note@#cwg629-T {{previous definition is here}}
   }
 }
 
-namespace dr630 { // dr630: yes
+namespace cwg630 { // cwg630: yes
 const bool MB_EQ_WC =
     ' ' == L' ' && '\t' == L'\t' && '\v' == L'\v' && '\r' == L'\r' &&
     '\n' == L'\n' && //
@@ -334,17 +334,17 @@ _Static_assert(MB_EQ_WC, "!__STDC_MB_MIGHT_NEQ_WC__ but some character differs")
 #endif
 }
 
-// dr631: na
+// cwg631: na
 
-namespace dr632 { // dr632: yes
+namespace cwg632 { // cwg632: yes
   struct S { int n; } s = {{5}};
   // expected-warning@-1 {{braces around scalar initializer}}
 }
 
-// dr633: na
+// cwg633: na
 // see also n2993
 
-namespace dr634 { // dr634: yes
+namespace cwg634 { // cwg634: yes
   struct S { S(); S(const S&); virtual void f(); ~S(); };
   int f(...);
   char f(int);
@@ -356,7 +356,7 @@ namespace dr634 { // dr634: yes
   // since-cxx11-error@-2 {{cannot pass object of non-trivial type 'S' through variadic function; call will abort at runtime}}
 }
 
-namespace dr635 { // dr635: yes
+namespace cwg635 { // cwg635: yes
   template<typename T> struct A { A(); ~A(); };
   template<typename T> A<T>::A<T>() {}
   // expected-error@-1 {{out-of-line constructor for 'A' cannot have template arguments}}
@@ -375,15 +375,15 @@ namespace dr635 { // dr635: yes
   /*FIXME: needed for error recovery:*/;
 
   template<typename T> struct D { template<typename U> D(); D(); };
-  template<typename T> D<T>::D() {} // #dr635-D
+  template<typename T> D<T>::D() {} // #cwg635-D
   template<typename T> template<typename U> D<T>::D() {}
-  template<typename T> D<T>::D<T>() {} // #dr635-D-T
-  // expected-error@#dr635-D-T {{out-of-line constructor for 'D' cannot have template arguments}}
-  // expected-error@#dr635-D-T {{redefinition of 'D<T>'}}
-  //   expected-note@#dr635-D {{previous definition is here}}
+  template<typename T> D<T>::D<T>() {} // #cwg635-D-T
+  // expected-error@#cwg635-D-T {{out-of-line constructor for 'D' cannot have template arguments}}
+  // expected-error@#cwg635-D-T {{redefinition of 'D<T>'}}
+  //   expected-note@#cwg635-D {{previous definition is here}}
 }
 
-namespace dr637 { // dr637: yes
+namespace cwg637 { // cwg637: yes
   void f(int i) {
     i = ++i + 1;
     i = i++ + 1;
@@ -391,7 +391,7 @@ namespace dr637 { // dr637: yes
   }
 }
 
-namespace dr638 { // dr638: no
+namespace cwg638 { // cwg638: no
   template<typename T> struct A {
     struct B;
     void f();
@@ -426,23 +426,23 @@ namespace dr638 { // dr638: no
   };
 }
 
-namespace dr639 { // dr639: 3.3
+namespace cwg639 { // cwg639: 3.3
   void f(int i) {
     void((i = 0) + (i = 0));
     // expected-warning@-1 {{multiple unsequenced modifications to 'i'}}
   }
 }
 
-namespace dr641 { // dr641: yes
+namespace cwg641 { // cwg641: yes
   namespace std_example {
     struct abc;
 
     struct xyz {
-      xyz(); // #dr641-xyz-ctor
-      xyz(xyz &); // #dr641-xyz-copy-ctor
+      xyz(); // #cwg641-xyz-ctor
+      xyz(xyz &); // #cwg641-xyz-copy-ctor
 
       operator xyz &() = delete;
-      // expected-warning@-1 {{conversion function converting 'dr641::std_example::xyz' to itself will never be used}}
+      // expected-warning@-1 {{conversion function converting 'cwg641::std_example::xyz' to itself will never be used}}
       // cxx98-error@-2 {{deleted function definitions are a C++11 extension}}
       operator abc &() = delete;
       // cxx98-error@-1 {{deleted function definitions are a C++11 extension}}
@@ -451,15 +451,15 @@ namespace dr641 { // dr641: yes
     struct abc : xyz {};
 
     template<typename T>
-    void use(T &); // #dr641-use
+    void use(T &); // #cwg641-use
     void test() {
       use<xyz>(xyz());
       // expected-error@-1 {{no matching function for call to 'use'}}
-      //   expected-note@#dr641-use {{candidate function template not viable: expects an lvalue for 1st argument}}
+      //   expected-note@#cwg641-use {{candidate function template not viable: expects an lvalue for 1st argument}}
       use<const xyz>(xyz());
       // cxx98-error@-1 {{no viable constructor copying parameter of type 'xyz'; C++98 requires a copy constructor when binding a reference to a temporary}}
-      //   cxx98-note@#dr641-xyz-copy-ctor {{candidate constructor not viable: expects an lvalue for 1st argument}}
-      //   cxx98-note@#dr641-xyz-ctor {{candidate constructor not viable: requires 0 arguments, but 1 was provided}}
+      //   cxx98-note@#cwg641-xyz-copy-ctor {{candidate constructor not viable: expects an lvalue for 1st argument}}
+      //   cxx98-note@#cwg641-xyz-ctor {{candidate constructor not viable: requires 0 arguments, but 1 was provided}}
     }
   }
 
@@ -470,20 +470,20 @@ namespace dr641 { // dr641: yes
     // cxx98-error@-1 {{default template arguments for a function template are a C++11 extension}}
   };
   A a;
-  void f(A&); // #dr641-f
+  void f(A&); // #cwg641-f
   void g(const A ca) {
     f(A());
     // expected-error@-1 {{no matching function for call to 'f'}}
-    //   expected-note@#dr641-f {{candidate function not viable: expects an lvalue for 1st argument}}
+    //   expected-note@#cwg641-f {{candidate function not viable: expects an lvalue for 1st argument}}
     f(ca);
     // expected-error@-1 {{no matching function for call to 'f'}}
-    //   expected-note@#dr641-f {{candidate function not viable: 1st argument ('const A') would lose const qualifier}}
+    //   expected-note@#cwg641-f {{candidate function not viable: 1st argument ('const A') would lose const qualifier}}
     (void)A();
     (void)ca;
   }
 }
 
-namespace dr642 { // dr642: yes
+namespace cwg642 { // cwg642: yes
   void f() {
     const int i = 2;
     {
@@ -501,16 +501,16 @@ namespace dr642 { // dr642: yes
 }
 
 #if __cplusplus >= 201103L
-namespace dr643 { // dr643: 3.2
+namespace cwg643 { // cwg643: 3.2
   struct A {
     int x;
     auto f() -> decltype(this->x);
     auto f(A &a) -> decltype(a.x);
     auto g() -> decltype(x);
     auto h() -> decltype(this->y);
-    // since-cxx11-error@-1 {{no member named 'y' in 'dr643::A'}}
+    // since-cxx11-error@-1 {{no member named 'y' in 'cwg643::A'}}
     auto h(A &a) -> decltype(a.y);
-    // since-cxx11-error@-1 {{no member named 'y' in 'dr643::A'}}
+    // since-cxx11-error@-1 {{no member named 'y' in 'cwg643::A'}}
     auto i() -> decltype(y);
     // since-cxx11-error@-1 {{use of undeclared identifier 'y'}}
     int y;
@@ -519,7 +519,7 @@ namespace dr643 { // dr643: 3.2
 #endif
 
 #if __cplusplus >= 201103L
-namespace dr644 { // dr644: partial
+namespace cwg644 { // cwg644: partial
   struct A {
     A() = default;
     int x, y;
@@ -535,7 +535,7 @@ namespace dr644 { // dr644: partial
   struct D { C c; };
   static_assert(!__is_literal_type(D), "");
 
-  // FIXME: According to DR644, E<C> is a literal type despite having virtual
+  // FIXME: According to CWG644, E<C> is a literal type despite having virtual
   // base classes. This appears to be a wording defect.
   template<typename T>
   struct E : T {
@@ -545,12 +545,12 @@ namespace dr644 { // dr644: partial
 }
 #endif
 
-// dr645 increases permission to optimize; it's not clear that it's possible to
+// cwg645 increases permission to optimize; it's not clear that it's possible to
 // test for this.
-// dr645: na
+// cwg645: na
 
 #if __cplusplus >= 201103L
-namespace dr646 { // dr646: sup 981
+namespace cwg646 { // cwg646: sup 981
   struct A {
     constexpr A(const A&) = default; // ok
   };
@@ -564,28 +564,28 @@ namespace dr646 { // dr646: sup 981
 #endif
 
 #if __cplusplus >= 201103L
-namespace dr647 { // dr647: 3.1
-  // This is partially superseded by dr1358.
+namespace cwg647 { // cwg647: 3.1
+  // This is partially superseded by cwg1358.
   struct A {
     constexpr virtual void f() const;
     constexpr virtual void g() const {}
     // cxx11-17-error@-1 {{virtual function cannot be constexpr}}
   };
 
-  struct X { virtual void f() const; }; // #dr647-f
+  struct X { virtual void f() const; }; // #cwg647-f
   struct B : X {
     constexpr void f() const {}
     // cxx11-17-error@-1 {{virtual function cannot be constexpr}}
-    //   cxx11-17-note@#dr647-f {{overridden virtual function is here}}
+    //   cxx11-17-note@#cwg647-f {{overridden virtual function is here}}
   };
 
-  struct NonLiteral { NonLiteral() {} }; // #dr647-NonLiteral
+  struct NonLiteral { NonLiteral() {} }; // #cwg647-NonLiteral
 
   struct C {
     constexpr C(NonLiteral);
     constexpr C(NonLiteral, int) {}
     // cxx11-20-error@-1 {{constexpr constructor's 1st parameter type 'NonLiteral' is not a literal type}}
-    //   cxx11-20-note@#dr647-NonLiteral {{'NonLiteral' is not literal because it is not an aggregate and has no constexpr constructors other than copy or move constructors}}
+    //   cxx11-20-note@#cwg647-NonLiteral {{'NonLiteral' is not literal because it is not an aggregate and has no constexpr constructors other than copy or move constructors}}
     constexpr C() try {} catch (...) {}
     // cxx11-17-error@-1 {{function try block in constexpr constructor is a C++20 extension}}
     // cxx11-error@-2 {{use of this statement in a constexpr constructor is a C++14 extension}}
@@ -594,7 +594,7 @@ namespace dr647 { // dr647: 3.1
   struct D {
     operator int() const;
     constexpr D(int) {}
-    D(float); // #dr647-D-float-ctor
+    D(float); // #cwg647-D-float-ctor
   };
   constexpr int get();
   struct E {
@@ -602,7 +602,7 @@ namespace dr647 { // dr647: 3.1
     D d;
 
     // FIXME: We should diagnose this, as the conversion function is not
-    // constexpr. However, that part of this issue is supreseded by dr1364 and
+    // constexpr. However, that part of this issue is supreseded by cwg1364 and
     // others; no diagnostic is required for this any more.
     constexpr E()
         : n(D(0)),
@@ -610,22 +610,22 @@ namespace dr647 { // dr647: 3.1
 
     constexpr E(int)
     // cxx11-20-error@-1 {{constexpr constructor never produces a constant expression}}
-    //   cxx11-20-note@#dr647-int-d {{non-constexpr constructor 'D' cannot be used in a constant expression}}
-    //   cxx11-20-note@#dr647-D-float-ctor {{declared here}}
+    //   cxx11-20-note@#cwg647-int-d {{non-constexpr constructor 'D' cannot be used in a constant expression}}
+    //   cxx11-20-note@#cwg647-D-float-ctor {{declared here}}
         : n(0),
-          d(0.0f) {} // #dr647-int-d
+          d(0.0f) {} // #cwg647-int-d
     constexpr E(float f)
     // cxx11-20-error@-1 {{never produces a constant expression}}
-    //   cxx11-20-note@#dr647-float-d {{non-constexpr constructor}}
-    //   cxx11-20-note@#dr647-D-float-ctor {{declared here}}
+    //   cxx11-20-note@#cwg647-float-d {{non-constexpr constructor}}
+    //   cxx11-20-note@#cwg647-D-float-ctor {{declared here}}
         : n(get()),
-          d(D(0) + f) {} // #dr647-float-d
+          d(D(0) + f) {} // #cwg647-float-d
   };
 }
 #endif
 
 #if __cplusplus >= 201103L
-namespace dr648 { // dr648: yes
+namespace cwg648 { // cwg648: yes
   int f();
   constexpr int a = (true ? 1 : f());
   constexpr int b = false && f();
@@ -634,7 +634,7 @@ namespace dr648 { // dr648: yes
 #endif
 
 #if __cplusplus >= 201103L
-namespace dr649 { // dr649: 3.5
+namespace cwg649 { // cwg649: 3.5
 // Maximum alignment is 8192 bytes for Windows, and 4 GB for Linux
 alignas(0x200000000) int n;
 // since-cxx11-error-re@-1 {{{{requested alignment must be (8192|4294967296) bytes or smaller}}}}
@@ -645,15 +645,15 @@ struct Y {
   // since-cxx11-error-re@-1 {{{{requested alignment must be (8192|4294967296) bytes or smaller}}}}
 };
   struct alignas(256) Z {};
-  // This part is superseded by dr2130 and eventually by aligned allocation support.
+  // This part is superseded by cwg2130 and eventually by aligned allocation support.
   auto *p = new Z;
 }
 #endif
 
-// dr650 is in dr650.cpp
+// cwg650 is in cwg650.cpp
 
 #if __cplusplus >= 201103L
-namespace dr651 { // dr651: yes
+namespace cwg651 { // cwg651: yes
   struct X {
     virtual X &f();
   };
@@ -666,16 +666,16 @@ namespace dr651 { // dr651: yes
 #endif
 
 #if __cplusplus >= 201103L
-namespace dr652 { // dr652: yes
+namespace cwg652 { // cwg652: yes
   constexpr int n = 1.2 * 3.4;
   static_assert(n == 4, "");
 }
 #endif
 
-// dr653 is in dr653.cpp
+// cwg653 is in cwg653.cpp
 
 #if __cplusplus >= 201103L
-namespace dr654 { // dr654: sup 1423
+namespace cwg654 { // cwg654: sup 1423
   void f() {
     if (nullptr) {}
     // since-cxx11-warning@-1 {{implicit conversion of nullptr constant to 'bool'}}
@@ -701,31 +701,31 @@ namespace dr654 { // dr654: sup 1423
 }
 #endif
 
-namespace dr655 { // dr655: yes
-  struct A { A(int); }; // #dr655-A
+namespace cwg655 { // cwg655: yes
+  struct A { A(int); }; // #cwg655-A
   struct B : A {
-    A a; // #dr655-a
+    A a; // #cwg655-a
     B();
     B(int) : B() {}
     // cxx98-error@-1 {{delegating constructors are permitted only in C++11}}
-    B(int*) : A() {} // #dr655-delegating-to-A
+    B(int*) : A() {} // #cwg655-delegating-to-A
     // expected-error@-1 {{no matching constructor for initialization of 'A'}}
-    //   expected-note@#dr655-A {{candidate constructor not viable: requires 1 argument, but 0 were provided}}
-    //   expected-note@#dr655-A {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 0 were provided}}
-    //   since-cxx11-note@#dr655-A {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 0 were provided}}
-    // expected-error@#dr655-delegating-to-A {{constructor for 'dr655::B' must explicitly initialize the member 'a' which does not have a default constructor}}
-    //   expected-note@#dr655-a {{member is declared here}}
-    //   expected-note@#dr655-A {{'dr655::A' declared here}}
+    //   expected-note@#cwg655-A {{candidate constructor not viable: requires 1 argument, but 0 were provided}}
+    //   expected-note@#cwg655-A {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 0 were provided}}
+    //   since-cxx11-note@#cwg655-A {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 0 were provided}}
+    // expected-error@#cwg655-delegating-to-A {{constructor for 'cwg655::B' must explicitly initialize the member 'a' which does not have a default constructor}}
+    //   expected-note@#cwg655-a {{member is declared here}}
+    //   expected-note@#cwg655-A {{'cwg655::A' declared here}}
   };
 }
 
-namespace dr656 { // dr656: yes
+namespace cwg656 { // cwg656: yes
   struct A { A(const A&) = delete; };
   // cxx98-error@-1 {{deleted function definitions are a C++11 extension}}
   struct B : A {};
   struct X { operator B(); } x;
   const A &r = x;
-  struct Y : private A { // #dr656-Y
+  struct Y : private A { // #cwg656-Y
     operator B() volatile;
   };
   extern Y y;
@@ -733,7 +733,7 @@ namespace dr656 { // dr656: yes
   // Conversion not considered due to reference-related types.
   const A &s = y;
   // expected-error@-1 {{cannot cast 'const Y' to its private base class 'const A'}}
-  //   expected-note@#dr656-Y {{declared private here}}
+  //   expected-note@#cwg656-Y {{declared private here}}
   const A &t = vy;
   // expected-error@-1 {{binding reference of type 'const A' to value of type 'volatile Y' drops 'volatile' qualifier}}
 
@@ -741,27 +741,27 @@ namespace dr656 { // dr656: yes
   struct D : C {};
   const D &d = c; // ok, D not reference-related to C
 
-  template<typename T> void accept(T); // #dr656-accept-T
-  template<typename T> void accept(...) = delete; // #dr656-accept-var
+  template<typename T> void accept(T); // #cwg656-accept-T
+  template<typename T> void accept(...) = delete; // #cwg656-accept-var
   // cxx98-error@-1 {{deleted function definitions are a C++11 extension}}
   void f() {
     accept<const A&>(x);
     accept<const A&>(y);
-    // expected-error@-1 {{cannot cast 'const Y' to its private base class 'const dr656::A'}}
-    //   expected-note@#dr656-Y {{declared private here}}
-    accept<const A&>(vy); // #dr656-vy
+    // expected-error@-1 {{cannot cast 'const Y' to its private base class 'const cwg656::A'}}
+    //   expected-note@#cwg656-Y {{declared private here}}
+    accept<const A&>(vy); // #cwg656-vy
     // expected-error@-1 {{call to deleted function 'accept'}}
-    //   expected-note@#dr656-accept-var {{candidate function [with T = const dr656::A &] has been explicitly deleted}}
-    //   expected-note@#dr656-accept-T {{candidate function template not viable: no known conversion from 'volatile Y' to 'const A &' for 1st argument}}
-    // expected-error@#dr656-vy {{no matching constructor for initialization of 'volatile Y'}}
-    //   expected-note@#dr656-Y {{candidate constructor (the implicit copy constructor) not viable: 1st argument ('volatile Y') would lose volatile qualifier}}
-    //   expected-note@#dr656-Y {{candidate constructor (the implicit default constructor) not viable: requires 0 arguments, but 1 was provided}}
+    //   expected-note@#cwg656-accept-var {{candidate function [with T = const cwg656::A &] has been explicitly deleted}}
+    //   expected-note@#cwg656-accept-T {{candidate function template not viable: no known conversion from 'volatile Y' to 'const A &' for 1st argument}}
+    // expected-error@#cwg656-vy {{no matching constructor for initialization of 'volatile Y'}}
+    //   expected-note@#cwg656-Y {{candidate constructor (the implicit copy constructor) not viable: 1st argument ('volatile Y') would lose volatile qualifier}}
+    //   expected-note@#cwg656-Y {{candidate constructor (the implicit default constructor) not viable: requires 0 arguments, but 1 was provided}}
     accept<const D&>(c);
   }
 }
 
-namespace dr657 { // dr657: partial
-  struct Abs { virtual void x() = 0; }; // #dr657-Abs
+namespace cwg657 { // cwg657: partial
+  struct Abs { virtual void x() = 0; }; // #cwg657-Abs
   struct Der : public Abs { virtual void x(); };
 
   struct Cnvt { template<typename F> Cnvt(F); };
@@ -779,7 +779,7 @@ namespace dr657 { // dr657: partial
 
   struct C { C(Abs) {} };
   // expected-error@-1 {{parameter type 'Abs' is an abstract class}}
-  //   expected-note@#dr657-Abs {{unimplemented pure virtual method 'x' in 'Abs'}}
+  //   expected-note@#cwg657-Abs {{unimplemented pure virtual method 'x' in 'Abs'}}
   struct Q { operator Abs() { __builtin_unreachable(); } } q;
   // expected-error@-1 {{return type 'Abs' is an abstract class}}
 #if __cplusplus >= 201703L
@@ -798,23 +798,23 @@ namespace dr657 { // dr657: partial
   Cnvt2<Abs>::type err;
 }
 
-// dr658 is in dr658.cpp
+// cwg658 is in cwg658.cpp
 
 #if __cplusplus >= 201103L
-namespace dr659 { // dr659: 3.0
+namespace cwg659 { // cwg659: 3.0
   static_assert(alignof(char) == alignof(char&), "");
   static_assert(alignof(int) == alignof(int&), "");
   int n = alignof(int(&)());
   // since-cxx11-error@-1 {{invalid application of 'alignof' to a function type}}
-  struct A; // #dr659-A
+  struct A; // #cwg659-A
   int m = alignof(A&);
   // since-cxx11-error@-1 {{invalid application of 'alignof' to an incomplete type 'A'}}
-  //   since-cxx11-note@#dr659-A {{forward declaration of 'dr659::A'}}
+  //   since-cxx11-note@#cwg659-A {{forward declaration of 'cwg659::A'}}
 }
 #endif
 
 #if __cplusplus >= 201103L
-namespace dr660 { // dr660: 3.0
+namespace cwg660 { // cwg660: 3.0
   enum : int { a };
   enum class { b };
   // since-cxx11-error@-1 {{scoped enumeration requires a name}}
@@ -829,27 +829,27 @@ namespace dr660 { // dr660: 3.0
 }
 #endif
 
-// dr661 is in dr661.cpp
+// cwg661 is in cwg661.cpp
 
-namespace dr662 { // dr662: yes
+namespace cwg662 { // cwg662: yes
   template <typename T> void f(T t) {
     T &tr = t;
     T *tp = &t;
     // expected-error@-1 {{'tp' declared as a pointer to a reference of type 'int &'}}
-    //   expected-note@#dr662-f-call {{in instantiation of function template specialization 'dr662::f<int &>' requested here}}
+    //   expected-note@#cwg662-f-call {{in instantiation of function template specialization 'cwg662::f<int &>' requested here}}
 #if __cplusplus >= 201103L
     auto *ap = &t;
 #endif
   }
-  void g(int n) { f<int&>(n); } // #dr662-f-call
+  void g(int n) { f<int&>(n); } // #cwg662-f-call
 }
 
-namespace dr663 { // dr663: sup P1949
+namespace cwg663 { // cwg663: sup P1949
   int ЍЎ = 123;
 }
 
 #if __cplusplus >= 201103L
-namespace dr664 { // dr664: yes
+namespace cwg664 { // cwg664: yes
   struct A { A(const A&) = delete; };
   A &&f(A &&a, int n) {
     if (n)
@@ -859,41 +859,41 @@ namespace dr664 { // dr664: yes
 }
 #endif
 
-namespace dr665 { // dr665: 2.8
+namespace cwg665 { // cwg665: 2.8
   struct A { virtual ~A(); };
   struct B : A {} *b;
-  struct C : private A {} *c; // #dr665-C
+  struct C : private A {} *c; // #cwg665-C
   struct D : B, C {} *d;
 
   struct VB : virtual A {} *vb;
-  struct VC : private virtual A {} *vc; // #dr665-VC
+  struct VC : private virtual A {} *vc; // #cwg665-VC
   struct VD : VB, VC {} *vd;
 
   void f() {
     (void)dynamic_cast<A*>(b);
     (void)dynamic_cast<A*>(c);
-    // expected-error@-1 {{cannot cast 'dr665::C' to its private base class 'dr665::A'}}
-    //   expected-note@#dr665-C {{declared private here}}
+    // expected-error@-1 {{cannot cast 'cwg665::C' to its private base class 'cwg665::A'}}
+    //   expected-note@#cwg665-C {{declared private here}}
     (void)dynamic_cast<A*>(d);
-    /* expected-error@-1 {{ambiguous conversion from derived class 'dr665::D' to base class 'dr665::A':
-    struct dr665::D -> B -> A
-    struct dr665::D -> C -> A}} */
+    /* expected-error@-1 {{ambiguous conversion from derived class 'cwg665::D' to base class 'cwg665::A':
+    struct cwg665::D -> B -> A
+    struct cwg665::D -> C -> A}} */
     (void)dynamic_cast<A*>(vb);
     (void)dynamic_cast<A*>(vc); // emitting diagnostic, even though it could be valid at runtime
-    // expected-error@-1 {{cannot cast 'dr665::VC' to its private base class 'dr665::A'}}
-    //   expected-note@#dr665-VC {{declared private here}}
+    // expected-error@-1 {{cannot cast 'cwg665::VC' to its private base class 'cwg665::A'}}
+    //   expected-note@#cwg665-VC {{declared private here}}
     (void)dynamic_cast<A*>(vd);
   }
 }
 
-namespace dr666 { // dr666: 2.8
+namespace cwg666 { // cwg666: 2.8
   struct P { friend P operator*(P, P); P(int); } p(0);
 
   template<int> int f();
   template<typename T> int f() {
     T::type *p = 0;
     // expected-error@-1 {{missing 'typename' prior to dependent type name 'Y::type'}}
-    //   expected-note@#dr666-f-Y {{in instantiation of function template specialization 'dr666::f<dr666::Y>' requested here}}
+    //   expected-note@#cwg666-f-Y {{in instantiation of function template specialization 'cwg666::f<cwg666::Y>' requested here}}
     int a(T::type);
     // expected-error@-1 {{missing 'typename' prior to dependent type name 'Y::type'}}
     return f<T::type>();
@@ -902,18 +902,18 @@ namespace dr666 { // dr666: 2.8
   struct X { static const int type = 0; };
   struct Y { typedef int type; };
   int a = f<X>();
-  int b = f<Y>(); // #dr666-f-Y
+  int b = f<Y>(); // #cwg666-f-Y
 }
 
 // Triviality is entirely different in C++98.
 #if __cplusplus >= 201103L
-namespace dr667 { // dr667: 8
+namespace cwg667 { // cwg667: 8
   struct A {
-    A() = default; // #dr667-A-ctor
+    A() = default; // #cwg667-A-ctor
     // since-cxx11-warning@-1 {{explicitly defaulted default constructor is implicitly deleted}}
-    //   since-cxx11-note@#dr667-r {{default constructor of 'A' is implicitly deleted because field 'r' of reference type 'int &' would not be initialized}}
-    //   since-cxx11-note@#dr667-A-ctor {{replace 'default' with 'delete'}}
-    int &r; // #dr667-r
+    //   since-cxx11-note@#cwg667-r {{default constructor of 'A' is implicitly deleted because field 'r' of reference type 'int &' would not be initialized}}
+    //   since-cxx11-note@#cwg667-A-ctor {{replace 'default' with 'delete'}}
+    int &r; // #cwg667-r
   };
   static_assert(!__is_trivially_constructible(A), "");
 
@@ -931,10 +931,10 @@ namespace dr667 { // dr667: 8
 }
 #endif
 
-// dr668 needs an libc++abi test
+// cwg668 needs an libc++abi test
 
 #if __cplusplus >= 201103L
-namespace dr669 { // dr669: yes
+namespace cwg669 { // cwg669: yes
   void f() {
     int n;
     using T = decltype(n);
@@ -962,7 +962,7 @@ namespace dr669 { // dr669: yes
 }
 #endif
 
-namespace dr671 { // dr671: 2.9
+namespace cwg671 { // cwg671: 2.9
   enum class E { e };
   // cxx98-error@-1 {{scoped enumerations are a C++11 extension}}
   E e = static_cast<E>(0);
@@ -971,9 +971,9 @@ namespace dr671 { // dr671: 2.9
   int m = static_cast<int>(e);
 }
 
-// dr672 is in dr672.cpp
+// cwg672 is in cwg672.cpp
 
-namespace dr673 { // dr673: yes
+namespace cwg673 { // cwg673: yes
   template<typename> struct X { static const int n = 0; };
 
   class A {
@@ -991,7 +991,7 @@ namespace dr673 { // dr673: yes
   // expected-error@-1 {{unknown type name 'F'}}
 }
 
-namespace dr674 { // dr674: 8
+namespace cwg674 { // cwg674: 8
   template<typename T> int f(T);
 
   int g(int);
@@ -1001,25 +1001,25 @@ namespace dr674 { // dr674: 8
   template<typename T> int h(T);
 
   class X {
-    friend int dr674::f(int);
-    friend int dr674::g(int);
-    friend int dr674::h<>(int);
-    int n; // #dr674-X-n
+    friend int cwg674::f(int);
+    friend int cwg674::g(int);
+    friend int cwg674::h<>(int);
+    int n; // #cwg674-X-n
   };
 
   template<typename T> int f(T) { return X().n; }
   int g(int) { return X().n; }
   template<typename T> int g(T) { return X().n; }
-  // expected-error@-1 {{'n' is a private member of 'dr674::X'}}
-  //   expected-note@#dr674-g-int {{in instantiation of function template specialization 'dr674::g<int>' requested here}}
-  //   expected-note@#dr674-X-n {{implicitly declared private here}}
+  // expected-error@-1 {{'n' is a private member of 'cwg674::X'}}
+  //   expected-note@#cwg674-g-int {{in instantiation of function template specialization 'cwg674::g<int>' requested here}}
+  //   expected-note@#cwg674-X-n {{implicitly declared private here}}
   int h(int) { return X().n; }
-  // expected-error@-1 {{'n' is a private member of 'dr674::X'}}
-  //   expected-note@#dr674-X-n {{implicitly declared private here}}
+  // expected-error@-1 {{'n' is a private member of 'cwg674::X'}}
+  //   expected-note@#cwg674-X-n {{implicitly declared private here}}
   template<typename T> int h(T) { return X().n; }
 
   template int f(int);
-  template int g(int); // #dr674-g-int
+  template int g(int); // #cwg674-g-int
   template int h(int);
 
 
@@ -1037,27 +1037,27 @@ namespace dr674 { // dr674: 8
     friend int Y::f(int);
     friend int Y::g(int);
     friend int Y::h<>(int);
-    int n; // #dr674-Z-n
+    int n; // #cwg674-Z-n
   };
 
   template<typename T> int Y::f(T) { return Z().n; }
   int Y::g(int) { return Z().n; }
   template<typename T> int Y::g(T) { return Z().n; }
-  // expected-error@-1 {{'n' is a private member of 'dr674::Z'}}
-  //   expected-note@#dr674-Y-g-int {{in instantiation of function template specialization 'dr674::Y::g<int>' requested here}}
-  //   expected-note@#dr674-Z-n {{implicitly declared private here}}
+  // expected-error@-1 {{'n' is a private member of 'cwg674::Z'}}
+  //   expected-note@#cwg674-Y-g-int {{in instantiation of function template specialization 'cwg674::Y::g<int>' requested here}}
+  //   expected-note@#cwg674-Z-n {{implicitly declared private here}}
   int Y::h(int) { return Z().n; }
-  // expected-error@-1 {{'n' is a private member of 'dr674::Z'}}
-  //   expected-note@#dr674-Z-n {{implicitly declared private here}}
+  // expected-error@-1 {{'n' is a private member of 'cwg674::Z'}}
+  //   expected-note@#cwg674-Z-n {{implicitly declared private here}}
   template<typename T> int Y::h(T) { return Z().n; }
 
   // FIXME: Should the <> be required here?
   template int Y::f<>(int);
-  template int Y::g<>(int); // #dr674-Y-g-int
+  template int Y::g<>(int); // #cwg674-Y-g-int
   template int Y::h<>(int);
 }
 
-namespace dr675 { // dr675: dup 739
+namespace cwg675 { // cwg675: dup 739
   template<typename T> struct A { T n : 1; };
 #if __cplusplus >= 201103L
   static_assert(A<char>{1}.n < 0, "");
@@ -1069,47 +1069,47 @@ namespace dr675 { // dr675: dup 739
 #endif
 }
 
-// dr676: na
+// cwg676: na
 
-namespace dr677 { // dr677: no
+namespace cwg677 { // cwg677: no
   struct A {
     void *operator new(std::size_t);
-    void operator delete(void*) = delete; // #dr677-A-delete
+    void operator delete(void*) = delete; // #cwg677-A-delete
     // cxx98-error@-1 {{deleted function definitions are a C++11 extension}} 
   };
   struct B {
     void *operator new(std::size_t);
-    void operator delete(void*) = delete; // #dr677-B-delete
+    void operator delete(void*) = delete; // #cwg677-B-delete
     // cxx98-error@-1 {{deleted function definitions are a C++11 extension}}
     virtual ~B();
   };
   void f(A *p) { delete p; }
   // expected-error@-1 {{attempt to use a deleted function}}
-  //   expected-note@#dr677-A-delete {{'operator delete' has been explicitly marked deleted here}}
+  //   expected-note@#cwg677-A-delete {{'operator delete' has been explicitly marked deleted here}}
   // FIXME: This appears to be valid; we shouldn't even be looking up the 'operator delete' here.
   void f(B *p) { delete p; }
   // expected-error@-1 {{attempt to use a deleted function}}
-  //   expected-note@#dr677-B-delete {{'operator delete' has been explicitly marked deleted here}}
+  //   expected-note@#cwg677-B-delete {{'operator delete' has been explicitly marked deleted here}}
   B::~B() {}
   // expected-error@-1 {{attempt to use a deleted function}}
-  //   expected-note@#dr677-B-delete {{'operator delete' has been explicitly marked deleted here}}
+  //   expected-note@#cwg677-B-delete {{'operator delete' has been explicitly marked deleted here}}
 }
 
-// dr678 FIXME: check that the modules ODR check catches this
+// cwg678 FIXME: check that the modules ODR check catches this
 
-namespace dr679 { // dr679: yes
+namespace cwg679 { // cwg679: yes
   struct X {};
   template<int> void operator+(X, X);
-  template<> void operator+<0>(X, X) {} // #dr679-def
+  template<> void operator+<0>(X, X) {} // #cwg679-def
   template<> void operator+<0>(X, X) {}
   // expected-error@-1 {{redefinition of 'operator+<0>'}}
-  //   expected-note@#dr679-def {{previous definition is here}}
+  //   expected-note@#cwg679-def {{previous definition is here}}
 }
 
-// dr680: na
+// cwg680: na
 
 #if __cplusplus >= 201103L
-namespace dr681 { // dr681: partial
+namespace cwg681 { // cwg681: partial
   auto *a() -> int;
   // since-cxx11-error@-1 {{function with trailing return type must specify return type 'auto', not 'auto *'}}
   auto (*b)() -> int;
@@ -1132,7 +1132,7 @@ namespace dr681 { // dr681: partial
 #endif
 
 #if __cplusplus >= 201103L
-namespace dr683 { // dr683: yes
+namespace cwg683 { // cwg683: yes
   struct A {
     A() = default;
     A(const A&) = default;
@@ -1150,18 +1150,18 @@ namespace dr683 { // dr683: yes
 #endif
 
 #if __cplusplus >= 201103L
-namespace dr684 { // dr684: sup 1454
+namespace cwg684 { // cwg684: sup 1454
   void f() {
-    int a;  // #dr684-a
+    int a;  // #cwg684-a
     constexpr int *p = &a;
     // expected-error@-1 {{constexpr variable 'p' must be initialized by a constant expression}}
     //   expected-note@-2 {{pointer to 'a' is not a constant expression}}
-    //   expected-note@#dr684-a {{here}}
+    //   expected-note@#cwg684-a {{here}}
   }
 }
 #endif
 
-namespace dr685 { // dr685: yes
+namespace cwg685 { // cwg685: yes
   enum E : long { e };
   // cxx98-error@-1 {{enumeration types with a fixed underlying type are a C++11 extension}}
   void f(int);
@@ -1178,20 +1178,20 @@ namespace dr685 { // dr685: yes
   void i(long);
   int c = i(g);
 
-  int j(unsigned int); // #dr685-j-uint
-  void j(long); // #dr685-j-long
+  int j(unsigned int); // #cwg685-j-uint
+  void j(long); // #cwg685-j-long
   int d = j(g);
   // expected-error@-1 {{call to 'j' is ambiguous}}
-  //   expected-note@#dr685-j-uint {{candidate function}}
-  //   expected-note@#dr685-j-long {{candidate function}}
+  //   expected-note@#cwg685-j-uint {{candidate function}}
+  //   expected-note@#cwg685-j-long {{candidate function}}
 
-  // Valid per dr1601
+  // Valid per cwg1601
   int k(short);
   void k(int);
   int x = k(g);
 }
 
-namespace dr686 { // dr686: 3.0
+namespace cwg686 { // cwg686: 3.0
   void f() {
     (void)dynamic_cast<struct A*>(0);
     // expected-error@-1 {{'A' is an incomplete type}}
@@ -1214,11 +1214,11 @@ namespace dr686 { // dr686: 3.0
     (void)sizeof(struct F*);
     (void)sizeof(struct F{}*);
     // expected-error@-1 {{'F' cannot be defined in a type specifier}}
-    (void)new struct G*; // #dr686-G
-    (void)new struct G{}*; // #dr686-G-def
+    (void)new struct G*; // #cwg686-G
+    (void)new struct G{}*; // #cwg686-G-def
     // expected-error@-1 {{allocation of incomplete type 'struct G'}}
-    //   expected-note@#dr686-G {{forward declaration of 'G'}}
-    // since-cxx11-error@#dr686-G-def {{expected expression}}
+    //   expected-note@#cwg686-G {{forward declaration of 'G'}}
+    // since-cxx11-error@#cwg686-G-def {{expected expression}}
 #if __cplusplus >= 201103L
     (void)alignof(struct H*);
     (void)alignof(struct H{}*);
@@ -1256,10 +1256,10 @@ namespace dr686 { // dr686: 3.0
   }
   template<struct R *> struct X;
   template<struct R {} *> struct Y;
-  // expected-error@-1 {{'dr686::R' cannot be defined in a type specifier}}
+  // expected-error@-1 {{'cwg686::R' cannot be defined in a type specifier}}
 }
 
-namespace dr687 { // dr687 (9 c++20, but the issue is still considered open)
+namespace cwg687 { // cwg687 (9 c++20, but the issue is still considered open)
   template<typename T> void f(T a) {
     // This is valid in C++20.
     g<int>(a);
@@ -1271,8 +1271,8 @@ namespace dr687 { // dr687 (9 c++20, but the issue is still considered open)
   }
 }
 
-namespace dr692 { // dr692: 16
-  // Also see dr1395.
+namespace cwg692 { // cwg692: 16
+  // Also see cwg1395.
 
   namespace temp_func_order_example2 {
     template <typename... T> struct A1 {};
@@ -1287,11 +1287,11 @@ namespace dr692 { // dr692: 16
     // cxx98-error@-1 {{variadic templates are a C++11 extension}}
     // cxx98-error@-2 {{deleted function definitions are a C++11 extension}}
     template <typename T1> void e2(A2<T1>);
-    template <typename T, typename U> void f(U, A1<U, T> *p = 0) = delete; // #dr692-f-deleted
+    template <typename T, typename U> void f(U, A1<U, T> *p = 0) = delete; // #cwg692-f-deleted
     // cxx98-error@-1 {{deleted function definitions are a C++11 extension}}
-    template <typename U> int &f(U, A1<U, U> *p = 0); // #dr692-f
-    template <typename T> void g(T, T = T()); // #dr692-g
-    template <typename T, typename... U> void g(T, U...); // #dr692-g-variadic
+    template <typename U> int &f(U, A1<U, U> *p = 0); // #cwg692-f
+    template <typename T> void g(T, T = T()); // #cwg692-g
+    template <typename T, typename... U> void g(T, U...); // #cwg692-g-variadic
     // cxx98-error@-1 {{variadic templates are a C++11 extension}}
     void h() {
       A1<int, int> a;
@@ -1302,12 +1302,12 @@ namespace dr692 { // dr692: 16
       e2(b2);
       f<int>(42);
       // expected-error@-1 {{call to 'f' is ambiguous}}
-      //   expected-note@#dr692-f-deleted {{candidate function [with T = int, U = int] has been explicitly deleted}}
-      //   expected-note@#dr692-f {{candidate function [with U = int]}}
+      //   expected-note@#cwg692-f-deleted {{candidate function [with T = int, U = int] has been explicitly deleted}}
+      //   expected-note@#cwg692-f {{candidate function [with U = int]}}
       g(42);
       // expected-error@-1 {{ambiguous}}
-      //   expected-note@#dr692-g {{candidate function [with T = int]}}
-      //   expected-note@#dr692-g-variadic {{candidate function [with T = int, U = <>]}}
+      //   expected-note@#cwg692-g {{candidate function [with T = int]}}
+      //   expected-note@#cwg692-g-variadic {{candidate function [with T = int, U = <>]}}
     }
   }
 
@@ -1319,7 +1319,7 @@ namespace dr692 { // dr692: 16
     // cxx98-error@-1 {{variadic templates are a C++11 extension}}
     template <typename T> void g(T);
     void h(int i) {
-      // This is made ambiguous by dr692, but made valid again by dr1395.
+      // This is made ambiguous by cwg692, but made valid again by cwg1395.
       f(&i);
       int &r = g(&i);
     }
@@ -1362,23 +1362,23 @@ namespace dr692 { // dr692: 16
   }
 }
 
-namespace dr696 { // dr696: 3.1
+namespace cwg696 { // cwg696: 3.1
   void f(const int*);
   void g() {
-    const int N = 10; // #dr696-N
+    const int N = 10; // #cwg696-N
     struct A {
       void h() {
         int arr[N]; (void)arr;
         f(&N);
-        // expected-error@-1 {{reference to local variable 'N' declared in enclosing function 'dr696::g'}}
-        //   expected-note@#dr696-N {{'N' declared here}}
+        // expected-error@-1 {{reference to local variable 'N' declared in enclosing function 'cwg696::g'}}
+        //   expected-note@#cwg696-N {{'N' declared here}}
       }
     };
 #if __cplusplus >= 201103L
     (void) [] { int arr[N]; (void)arr; };
     (void)[] { f(&N); };
     // since-cxx11-error@-1 {{variable 'N' cannot be implicitly captured in a lambda with no capture-default specified}}
-    //   since-cxx11-note@#dr696-N {{'N' declared here}}
+    //   since-cxx11-note@#cwg696-N {{'N' declared here}}
     //   since-cxx11-note@-3 {{lambda expression begins here}}
     //   since-cxx11-note@-4 {{capture 'N' by value}}
     //   since-cxx11-note@-5 {{capture 'N' by reference}}
diff --git a/clang/test/CXX/drs/dr7xx.cpp b/clang/test/CXX/drs/dr7xx.cpp
index 2cbdc218ab7b..69ee6d6d4e6a 100644
--- a/clang/test/CXX/drs/dr7xx.cpp
+++ b/clang/test/CXX/drs/dr7xx.cpp
@@ -4,10 +4,10 @@
 // RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++17 %s -verify=expected,since-cxx14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++2a %s -verify=expected,since-cxx14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors
 
-namespace dr705 { // dr705: yes
+namespace cwg705 { // cwg705: yes
   namespace N {
     struct S {};
-    void f(S); // #dr705-f
+    void f(S); // #cwg705-f
   }
 
   void g() {
@@ -15,14 +15,14 @@ namespace dr705 { // dr705: yes
     f(s);      // ok
     (f)(s);
     // expected-error@-1 {{use of undeclared identifier 'f'}}
-    //   expected-note@#dr705-f {{'N::f' declared here}}
+    //   expected-note@#cwg705-f {{'N::f' declared here}}
   }
 }
 
-namespace dr712 { // dr712: partial
+namespace cwg712 { // cwg712: partial
   void use(int);
   void f() {
-    const int a = 0; // #dr712-f-a
+    const int a = 0; // #cwg712-f-a
     struct X {
       void g(bool cond) {
         use(a);
@@ -34,20 +34,20 @@ namespace dr712 { // dr712: partial
         // expected-warning@-2 {{left operand of comma operator has no effect}}
 
         (void)a;
-        // expected-error@-1 {{reference to local variable 'a' declared in enclosing function 'dr712::f'}} FIXME
-        //   expected-note@#dr712-f-a {{'a' declared here}}
+        // expected-error@-1 {{reference to local variable 'a' declared in enclosing function 'cwg712::f'}} FIXME
+        //   expected-note@#cwg712-f-a {{'a' declared here}}
         (void)(a);
-        // expected-error@-1 {{reference to local variable 'a' declared in enclosing function 'dr712::f'}} FIXME
-        //   expected-note@#dr712-f-a {{'a' declared here}}
-        (void)(cond ? a : a); // #dr712-ternary
-        // expected-error@#dr712-ternary {{reference to local variable 'a' declared in enclosing function 'dr712::f'}} FIXME
-        //   expected-note@#dr712-f-a {{'a' declared here}}
-        // expected-error@#dr712-ternary {{reference to local variable 'a' declared in enclosing function 'dr712::f'}} FIXME
-        //   expected-note@#dr712-f-a {{'a' declared here}}
-        (void)(cond, a); // #dr712-comma
-        // expected-error@-1 {{reference to local variable 'a' declared in enclosing function 'dr712::f'}} FIXME
-        //   expected-note@#dr712-f-a {{'a' declared here}}
-        // expected-warning@#dr712-comma {{left operand of comma operator has no effect}}
+        // expected-error@-1 {{reference to local variable 'a' declared in enclosing function 'cwg712::f'}} FIXME
+        //   expected-note@#cwg712-f-a {{'a' declared here}}
+        (void)(cond ? a : a); // #cwg712-ternary
+        // expected-error@#cwg712-ternary {{reference to local variable 'a' declared in enclosing function 'cwg712::f'}} FIXME
+        //   expected-note@#cwg712-f-a {{'a' declared here}}
+        // expected-error@#cwg712-ternary {{reference to local variable 'a' declared in enclosing function 'cwg712::f'}} FIXME
+        //   expected-note@#cwg712-f-a {{'a' declared here}}
+        (void)(cond, a); // #cwg712-comma
+        // expected-error@-1 {{reference to local variable 'a' declared in enclosing function 'cwg712::f'}} FIXME
+        //   expected-note@#cwg712-f-a {{'a' declared here}}
+        // expected-warning@#cwg712-comma {{left operand of comma operator has no effect}}
       }
     };
   }
@@ -55,29 +55,29 @@ namespace dr712 { // dr712: partial
 #if __cplusplus >= 201103L
   void g() {
     struct A { int n; };
-    constexpr A a = {0};  // #dr712-g-a
+    constexpr A a = {0};  // #cwg712-g-a
     struct X {
       void g(bool cond) {
         use(a.n);
         use(a.*&A::n);
 
         (void)a.n;
-        // since-cxx11-error@-1 {{reference to local variable 'a' declared in enclosing function 'dr712::g'}} FIXME
-        //   since-cxx11-note@#dr712-g-a {{'a' declared here}}
+        // since-cxx11-error@-1 {{reference to local variable 'a' declared in enclosing function 'cwg712::g'}} FIXME
+        //   since-cxx11-note@#cwg712-g-a {{'a' declared here}}
         (void)(a.*&A::n);
-        // since-cxx11-error@-1 {{reference to local variable 'a' declared in enclosing function 'dr712::g'}} FIXME
-        //   since-cxx11-note@#dr712-g-a {{'a' declared here}}
+        // since-cxx11-error@-1 {{reference to local variable 'a' declared in enclosing function 'cwg712::g'}} FIXME
+        //   since-cxx11-note@#cwg712-g-a {{'a' declared here}}
       }
     };
   }
 #endif
 }
 
-namespace dr727 { // dr727: partial
+namespace cwg727 { // cwg727: partial
   struct A {
-    template<typename T> struct C; // #dr727-C
-    template<typename T> void f(); // #dr727-f
-    template<typename T> static int N; // #dr727-N
+    template<typename T> struct C; // #cwg727-C
+    template<typename T> void f(); // #cwg727-f
+    template<typename T> static int N; // #cwg727-N
     // cxx98-11-error@-1 {{variable templates are a C++14 extension}}
 
     template<> struct C<int>;
@@ -90,40 +90,40 @@ namespace dr727 { // dr727: partial
     struct B {
       template<> struct C<float>;
       // expected-error@-1 {{class template specialization of 'C' not in class 'A' or an enclosing namespace}}
-      //   expected-note@#dr727-C {{explicitly specialized declaration is here}}
+      //   expected-note@#cwg727-C {{explicitly specialized declaration is here}}
       template<> void f<float>();
       // expected-error@-1 {{no function template matches function template specialization 'f'}}
       template<> static int N<float>;
       // expected-error@-1 {{variable template specialization of 'N' not in class 'A' or an enclosing namespace}}
-      //   expected-note@#dr727-N {{explicitly specialized declaration is here}}
+      //   expected-note@#cwg727-N {{explicitly specialized declaration is here}}
 
       template<typename T> struct C<T**>;
       // expected-error@-1 {{class template partial specialization of 'C' not in class 'A' or an enclosing namespace}}
-      //   expected-note@#dr727-C {{explicitly specialized declaration is here}}
+      //   expected-note@#cwg727-C {{explicitly specialized declaration is here}}
       template<typename T> static int N<T**>;
       // expected-error@-1 {{variable template partial specialization of 'N' not in class 'A' or an enclosing namespace}}
-      //   expected-note@#dr727-N {{explicitly specialized declaration is here}}
+      //   expected-note@#cwg727-N {{explicitly specialized declaration is here}}
 
       template<> struct A::C<double>;
       // expected-error@-1 {{non-friend class member 'C' cannot have a qualified name}}
       // expected-error@-2 {{class template specialization of 'C' not in class 'A' or an enclosing namespace}}
-      //   expected-note@#dr727-C {{explicitly specialized declaration is here}}
+      //   expected-note@#cwg727-C {{explicitly specialized declaration is here}}
       template<> void A::f<double>();
       // expected-error@-1 {{o function template matches function template specialization 'f'}}
       // expected-error@-2 {{non-friend class member 'f' cannot have a qualified name}}
       template<> static int A::N<double>;
       // expected-error@-1 {{non-friend class member 'N' cannot have a qualified name}}
       // expected-error@-2 {{variable template specialization of 'N' not in class 'A' or an enclosing namespace}}
-      //   expected-note@#dr727-N {{explicitly specialized declaration is here}}
+      //   expected-note@#cwg727-N {{explicitly specialized declaration is here}}
 
       template<typename T> struct A::C<T***>;
       // expected-error@-1 {{non-friend class member 'C' cannot have a qualified name}}
       // expected-error@-2 {{class template partial specialization of 'C' not in class 'A' or an enclosing namespace}}
-      //   expected-note@#dr727-C {{explicitly specialized declaration is here}}
+      //   expected-note@#cwg727-C {{explicitly specialized declaration is here}}
       template<typename T> static int A::N<T***>;
       // expected-error@-1 {{non-friend class member 'N' cannot have a qualified name}}
       // expected-error@-2 {{variable template partial specialization of 'N' not in class 'A' or an enclosing namespace}}
-      //   expected-note@#dr727-N {{explicitly specialized declaration is here}}
+      //   expected-note@#cwg727-N {{explicitly specialized declaration is here}}
     };
   };
 
@@ -137,34 +137,34 @@ namespace dr727 { // dr727: partial
   namespace C {
     template<> struct A::C<long>;
     // expected-error@-1 {{class template specialization of 'C' not in class 'A' or an enclosing namespace}}
-    //   expected-note@#dr727-C {{explicitly specialized declaration is here}}
+    //   expected-note@#cwg727-C {{explicitly specialized declaration is here}}
     template<> void A::f<long>();
     // expected-error@-1 {{function template specialization of 'f' not in class 'A' or an enclosing namespace}}
-    //   expected-note@#dr727-f {{explicitly specialized declaration is here}}
+    //   expected-note@#cwg727-f {{explicitly specialized declaration is here}}
     template<> int A::N<long>;
     // expected-error@-1 {{variable template specialization of 'N' not in class 'A' or an enclosing namespace}}
-    //   expected-note@#dr727-N {{explicitly specialized declaration is here}}
+    //   expected-note@#cwg727-N {{explicitly specialized declaration is here}}
 
     template<typename T> struct A::C<T*****>;
     // expected-error@-1 {{class template partial specialization of 'C' not in class 'A' or an enclosing namespace}}
-    //   expected-note@#dr727-C {{explicitly specialized declaration is here}}
+    //   expected-note@#cwg727-C {{explicitly specialized declaration is here}}
     template<typename T> int A::N<T*****>;
     // expected-error@-1 {{variable template partial specialization of 'N' not in class 'A' or an enclosing namespace}}
-    //   expected-note@#dr727-N {{explicitly specialized declaration is here}}
+    //   expected-note@#cwg727-N {{explicitly specialized declaration is here}}
   }
 
   template<typename>
   struct D {
     template<typename T> struct C { typename T::error e; };
     // expected-error@-1 {{type 'float' cannot be used prior to '::' because it has no members}}
-    //   expected-note@#dr727-C-float {{in instantiation of template class 'dr727::D<int>::C<float>' requested here}}
+    //   expected-note@#cwg727-C-float {{in instantiation of template class 'cwg727::D<int>::C<float>' requested here}}
     template<typename T> void f() { T::error; }
     // expected-error@-1 {{type 'float' cannot be used prior to '::' because it has no members}}
-    //   expected-note@#dr727-f-float {{in instantiation of function template specialization 'dr727::D<int>::f<float>' requested here}}
+    //   expected-note@#cwg727-f-float {{in instantiation of function template specialization 'cwg727::D<int>::f<float>' requested here}}
     template<typename T> static const int N = T::error;
     // cxx98-11-error@-1 {{variable templates are a C++14 extension}}
     // expected-error@-2 {{type 'float' cannot be used prior to '::' because it has no members}}
-    //   expected-note@#dr727-N-float {{in instantiation of static data member 'dr727::D<int>::N<float>' requested here}}
+    //   expected-note@#cwg727-N-float {{in instantiation of static data member 'cwg727::D<int>::N<float>' requested here}}
 
     template<> struct C<int> {};
     template<> void f<int>() {}
@@ -188,9 +188,9 @@ namespace dr727 { // dr727: partial
     D<int>::C<int*>();
     int b = D<int>::N<int*>;
 
-    D<int>::C<float>(); // #dr727-C-float
-    di.f<float>(); // #dr727-f-float
-    int c = D<int>::N<float>; // #dr727-N-float
+    D<int>::C<float>(); // #cwg727-C-float
+    di.f<float>(); // #cwg727-f-float
+    int c = D<int>::N<float>; // #cwg727-N-float
   }
 
   namespace mixed_inner_outer_specialization {
@@ -210,14 +210,14 @@ namespace dr727 { // dr727: partial
 #if __cplusplus >= 201402L
     template<int> struct B {
       template<int> static const int u = 1;
-      template<> static const int u<0> = 2; // #dr727-u0
+      template<> static const int u<0> = 2; // #cwg727-u0
 
       // Note that in C++17 onwards, these are implicitly inline, and so the
       // initializer of v<0> is not instantiated with the declaration. In
       // C++14, v<0> is a non-defining declaration and its initializer is
       // instantiated with the class.
       template<int> static constexpr int v = 1;
-      template<> static constexpr int v<0> = 2; // #dr727-v0
+      template<> static constexpr int v<0> = 2; // #cwg727-v0
 
       template<int> static const inline int w = 1;
       // cxx14-error@-1 {{inline variables are a C++17 extension}}
@@ -228,12 +228,12 @@ namespace dr727 { // dr727: partial
     template<> template<int> constexpr int B<0>::u = 3;
     template<> template<> constexpr int B<0>::u<0> = 4;
     // since-cxx14-error@-1 {{static data member 'u' already has an initializer}}
-    //   since-cxx14-note@#dr727-u0 {{previous initialization is here}}
+    //   since-cxx14-note@#cwg727-u0 {{previous initialization is here}}
 
     template<> template<int> constexpr int B<0>::v = 3;
     template<> template<> constexpr int B<0>::v<0> = 4;
     // cxx14-error@-1 {{static data member 'v' already has an initializer}}
-    //   cxx14-note@#dr727-v0 {{previous initialization is here}}
+    //   cxx14-note@#cwg727-v0 {{previous initialization is here}}
 
     template<> template<int> constexpr int B<0>::w = 3;
     template<> template<> constexpr int B<0>::w<0> = 4;
@@ -246,7 +246,7 @@ namespace dr727 { // dr727: partial
     static_assert(B<1>().v<0> == 2, "");
     static_assert(B<0>().v<1> == 3, "");
     static_assert(B<0>().v<0> == 4, "");
-    // cxx14-error@-1 {{static assertion failed due to requirement 'dr727::mixed_inner_outer_specialization::B<0>().v<0> == 4'}}
+    // cxx14-error@-1 {{static assertion failed due to requirement 'cwg727::mixed_inner_outer_specialization::B<0>().v<0> == 4'}}
     //   cxx14-note@-2 {{expression evaluates to '2 == 4'}}
 
     static_assert(B<1>().w<1> == 1, "");
@@ -269,21 +269,21 @@ namespace dr727 { // dr727: partial
 
     template<typename> static int v1;
     // cxx98-11-error@-1 {{variable templates are a C++14 extension}}
-    template<> static int v1<T>; // #dr727-v1-T
+    template<> static int v1<T>; // #cwg727-v1-T
     template<> static int v1<U>;
     // expected-error@-1 {{duplicate member 'v1'}}
-    //   expected-note@#dr727-Collision-int-int {{in instantiation of template class 'dr727::Collision<int, int>' requested here}}
-    //   expected-note@#dr727-v1-T {{previous}}
+    //   expected-note@#cwg727-Collision-int-int {{in instantiation of template class 'cwg727::Collision<int, int>' requested here}}
+    //   expected-note@#cwg727-v1-T {{previous}}
 
     template<typename> static inline int v2;
     // cxx98-11-error@-1 {{variable templates are a C++14 extension}}
     // cxx98-14-error@-2 {{inline variables are a C++17 extension}}
-    template<> static inline int v2<T>; // #dr727-v2-T
+    template<> static inline int v2<T>; // #cwg727-v2-T
     // cxx98-14-error@-1 {{inline variables are a C++17 extension}} 
     template<> static inline int v2<U>;
     // cxx98-14-error@-1 {{inline variables are a C++17 extension}}
     // expected-error@-2 {{duplicate member 'v2'}}
-    //   expected-note@#dr727-v2-T {{previous declaration is here}}
+    //   expected-note@#cwg727-v2-T {{previous declaration is here}}
 
     // FIXME: Missing diagnostic for duplicate class explicit specialization.
     template<typename> struct S1;
@@ -291,15 +291,15 @@ namespace dr727 { // dr727: partial
     template<> struct S1<U>;
 
     template<typename> struct S2;
-    template<> struct S2<T> {}; // #dr727-S2-T
+    template<> struct S2<T> {}; // #cwg727-S2-T
     template<> struct S2<U> {};
     // expected-error@-1 {{redefinition of 'S2<int>'}}
-    //   expected-note@#dr727-S2-T {{previous}}
+    //   expected-note@#cwg727-S2-T {{previous}}
   };
-  Collision<int, int> c; // #dr727-Collision-int-int
+  Collision<int, int> c; // #cwg727-Collision-int-int
 }
 
-namespace dr777 { // dr777: 3.7
+namespace cwg777 { // cwg777: 3.7
 #if __cplusplus >= 201103L
 template <typename... T>
 void f(int i = 0, T ...args) {}
diff --git a/clang/test/CXX/drs/dr8xx.cpp b/clang/test/CXX/drs/dr8xx.cpp
index b031e65095cd..eba601300584 100644
--- a/clang/test/CXX/drs/dr8xx.cpp
+++ b/clang/test/CXX/drs/dr8xx.cpp
@@ -10,18 +10,18 @@
 // expected-no-diagnostics
 #endif
 
-namespace dr873 { // dr873: 3.0
+namespace cwg873 { // cwg873: 3.0
 #if __cplusplus >= 201103L
 template <typename T> void f(T &&);
-template <> void f(int &) = delete;  // #dr873-lvalue-ref
-template <> void f(int &&) = delete; // #dr873-rvalue-ref
+template <> void f(int &) = delete;  // #cwg873-lvalue-ref
+template <> void f(int &&) = delete; // #cwg873-rvalue-ref
 void g(int i) {
   f(i); // calls f<int&>(int&)
   // since-cxx11-error@-1 {{call to deleted function 'f'}}
-  //   since-cxx11-note@#dr873-lvalue-ref {{candidate function [with T = int &] has been implicitly deleted}}
+  //   since-cxx11-note@#cwg873-lvalue-ref {{candidate function [with T = int &] has been implicitly deleted}}
   f(0); // calls f<int>(int&&)
   // since-cxx11-error@-1 {{call to deleted function 'f'}}
-  //   since-cxx11-note@#dr873-rvalue-ref {{candidate function [with T = int] has been implicitly deleted}}
+  //   since-cxx11-note@#cwg873-rvalue-ref {{candidate function [with T = int] has been implicitly deleted}}
 }
 #endif
-} // namespace dr873
+} // namespace cwg873
diff --git a/clang/test/CXX/drs/dr9xx.cpp b/clang/test/CXX/drs/dr9xx.cpp
index 7ea295a684b7..8ecb149c355f 100644
--- a/clang/test/CXX/drs/dr9xx.cpp
+++ b/clang/test/CXX/drs/dr9xx.cpp
@@ -14,7 +14,7 @@ namespace std {
   };
 }
 
-namespace dr948 { // dr948: 3.7
+namespace cwg948 { // cwg948: 3.7
 #if __cplusplus >= 201103L
   class A {
   public:
@@ -43,34 +43,34 @@ namespace dr948 { // dr948: 3.7
 #endif
 }
 
-namespace dr952 { // dr952: 2.8
+namespace cwg952 { // cwg952: 2.8
 namespace example1 {
 struct A {
-  typedef int I; // #dr952-I
+  typedef int I; // #cwg952-I
 };
-struct B : private A { // #dr952-B
+struct B : private A { // #cwg952-B
 };
 struct C : B {
   void f() {
     I i1;
-    // expected-error@-1 {{'I' is a private member of 'dr952::example1::A'}}
-    //   expected-note@#dr952-B {{constrained by private inheritance here}}
-    //   expected-note@#dr952-I {{member is declared here}}
+    // expected-error@-1 {{'I' is a private member of 'cwg952::example1::A'}}
+    //   expected-note@#cwg952-B {{constrained by private inheritance here}}
+    //   expected-note@#cwg952-I {{member is declared here}}
   }
   I i2;
-  // expected-error@-1 {{'I' is a private member of 'dr952::example1::A'}}
-  //   expected-note@#dr952-B {{constrained by private inheritance here}}
-  //   expected-note@#dr952-I {{member is declared here}}
+  // expected-error@-1 {{'I' is a private member of 'cwg952::example1::A'}}
+  //   expected-note@#cwg952-B {{constrained by private inheritance here}}
+  //   expected-note@#cwg952-I {{member is declared here}}
   struct D {
     I i3;
-    // expected-error@-1 {{'I' is a private member of 'dr952::example1::A'}}
-    //   expected-note@#dr952-B {{constrained by private inheritance here}}
-    //   expected-note@#dr952-I {{member is declared here}}
+    // expected-error@-1 {{'I' is a private member of 'cwg952::example1::A'}}
+    //   expected-note@#cwg952-B {{constrained by private inheritance here}}
+    //   expected-note@#cwg952-I {{member is declared here}}
     void g() {
       I i4;
-      // expected-error@-1 {{'I' is a private member of 'dr952::example1::A'}}
-      //   expected-note@#dr952-B {{constrained by private inheritance here}}
-      //   expected-note@#dr952-I {{member is declared here}}
+      // expected-error@-1 {{'I' is a private member of 'cwg952::example1::A'}}
+      //   expected-note@#cwg952-B {{constrained by private inheritance here}}
+      //   expected-note@#cwg952-I {{member is declared here}}
     }
   };
 };
@@ -84,9 +84,9 @@ struct B : A {
   friend int get(B) { return x; }
 };
 } // namespace example2
-} // namespace dr952
+} // namespace cwg952
 
-namespace dr974 { // dr974: yes
+namespace cwg974 { // cwg974: yes
 #if __cplusplus >= 201103L
   void test() {
     auto lam = [](int x = 42) { return x; };
@@ -94,23 +94,23 @@ namespace dr974 { // dr974: yes
 #endif
 }
 
-namespace dr977 { // dr977: yes
-enum E { e = E() }; // #dr977-E
+namespace cwg977 { // cwg977: yes
+enum E { e = E() }; // #cwg977-E
 #if !defined(_WIN32) || defined(__MINGW32__)
-// expected-error@#dr977-E {{invalid use of incomplete type 'E'}}
-//   expected-note@#dr977-E {{definition of 'dr977::E' is not complete until the closing '}'}}
+// expected-error@#cwg977-E {{invalid use of incomplete type 'E'}}
+//   expected-note@#cwg977-E {{definition of 'cwg977::E' is not complete until the closing '}'}}
 #endif
 #if __cplusplus >= 201103L
 enum E2 : int { e2 = E2() };
 enum struct E3 { e = static_cast<int>(E3()) };
 enum struct E4 : int { e = static_cast<int>(E4()) };
 #endif
-} // namespace dr977
+} // namespace cwg977
 
-namespace dr990 { // dr990: 3.5
+namespace cwg990 { // cwg990: 3.5
 #if __cplusplus >= 201103L
-  struct A { // #dr990-A
-    A(std::initializer_list<int>); // #dr990-A-init-list
+  struct A { // #cwg990-A
+    A(std::initializer_list<int>); // #cwg990-A-init-list
   };
   struct B {
     A a;
@@ -118,20 +118,20 @@ namespace dr990 { // dr990: 3.5
   B b1 { };
   B b2 { 1 };
   // since-cxx11-error@-1 {{no viable conversion from 'int' to 'A'}}
-  //   since-cxx11-note@#dr990-A {{candidate constructor (the implicit copy constructor) not viable: no known conversion from 'int' to 'const A &' for 1st argument}}
-  //   since-cxx11-note@#dr990-A {{candidate constructor (the implicit move constructor) not viable: no known conversion from 'int' to 'A &&' for 1st argument}}
-  //   since-cxx11-note@#dr990-A-init-list {{candidate constructor not viable: no known conversion from 'int' to 'std::initializer_list<int>' for 1st argument}}
+  //   since-cxx11-note@#cwg990-A {{candidate constructor (the implicit copy constructor) not viable: no known conversion from 'int' to 'const A &' for 1st argument}}
+  //   since-cxx11-note@#cwg990-A {{candidate constructor (the implicit move constructor) not viable: no known conversion from 'int' to 'A &&' for 1st argument}}
+  //   since-cxx11-note@#cwg990-A-init-list {{candidate constructor not viable: no known conversion from 'int' to 'std::initializer_list<int>' for 1st argument}}
   B b3 { { 1 } };
 
   struct C {
     C();
     C(int);
-    C(std::initializer_list<int>) = delete; // #dr990-deleted
+    C(std::initializer_list<int>) = delete; // #cwg990-deleted
   };
   C c1[3] { 1 }; // ok
   C c2[3] { 1, {2} };
   // since-cxx11-error@-1 {{call to deleted constructor of 'C'}}
-  //   since-cxx11-note@#dr990-deleted {{'C' has been explicitly marked deleted here}}
+  //   since-cxx11-note@#cwg990-deleted {{'C' has been explicitly marked deleted here}}
 
   struct D {
     D();
diff --git a/clang/www/make_cxx_dr_status b/clang/www/make_cxx_dr_status
index 7183e1a6d2be..7c0cf77a1524 100755
--- a/clang/www/make_cxx_dr_status
+++ b/clang/www/make_cxx_dr_status
@@ -32,7 +32,7 @@ def parse(dr):
   return DR(section, issue, url, status, title)
 
 def collect_tests():
-  status_re = re.compile(r'\bdr([0-9]+): (.*)')
+  status_re = re.compile(r'\bcwg([0-9]+): (.*)')
   status_map = {}
   for test_cpp in os.listdir(dr_test_dir):
     if not test_cpp.endswith('.cpp'):
@@ -42,12 +42,12 @@ def collect_tests():
     for match in re.finditer(status_re, open(test_cpp, 'r').read()):
       dr_number = int(match.group(1))
       if dr_number in status_map:
-        print("error: Comment for dr{} encountered more than once. Duplicate found in {}".format(dr_number, test_cpp))
+        print("error: Comment for cwg{} encountered more than once. Duplicate found in {}".format(dr_number, test_cpp))
         sys.exit(1)
       status_map[dr_number] = match.group(2)
       found_any = True
     if not found_any:
-      print("warning:%s: no '// dr123: foo' comments in this file" % test_cpp, file=sys.stderr)
+      print("warning:%s: no '// cwg123: foo' comments in this file" % test_cpp, file=sys.stderr)
   return status_map
 
 def get_issues(path):
-- 
GitLab


From 4138367444a2e3431cdfc7ab6e16c9cab5c76ad8 Mon Sep 17 00:00:00 2001
From: Ian Anderson <iana@apple.com>
Date: Fri, 12 Apr 2024 22:36:43 -0700
Subject: [PATCH 136/464] [modules] allow use of ptrauth module from
 no_undeclared_includes system modules (#88432)

Co-authored-by: Alex Lorenz <arphaman@gmail.com>
---
 clang/lib/Basic/Module.cpp                                | 4 ++++
 .../Inputs/ptrauth-include-from-darwin/module.modulemap   | 8 ++++++++
 .../Modules/Inputs/ptrauth-include-from-darwin/ptrauth.h  | 1 +
 .../Modules/Inputs/ptrauth-include-from-darwin/stddef.h   | 1 +
 clang/test/Modules/ptrauth-include-from-darwin.m          | 6 ++++++
 5 files changed, 20 insertions(+)
 create mode 100644 clang/test/Modules/Inputs/ptrauth-include-from-darwin/module.modulemap
 create mode 100644 clang/test/Modules/Inputs/ptrauth-include-from-darwin/ptrauth.h
 create mode 100644 clang/test/Modules/Inputs/ptrauth-include-from-darwin/stddef.h
 create mode 100644 clang/test/Modules/ptrauth-include-from-darwin.m

diff --git a/clang/lib/Basic/Module.cpp b/clang/lib/Basic/Module.cpp
index 256365d66bb9..bb212cde8788 100644
--- a/clang/lib/Basic/Module.cpp
+++ b/clang/lib/Basic/Module.cpp
@@ -305,6 +305,10 @@ bool Module::directlyUses(const Module *Requested) {
   if (Requested->fullModuleNameIs({"_Builtin_stddef", "max_align_t"}) ||
       Requested->fullModuleNameIs({"_Builtin_stddef_wint_t"}))
     return true;
+  // Darwin is allowed is to use our builtin 'ptrauth.h' and its accompanying
+  // module.
+  if (!Requested->Parent && Requested->Name == "ptrauth")
+    return true;
 
   if (NoUndeclaredIncludes)
     UndeclaredUses.insert(Requested);
diff --git a/clang/test/Modules/Inputs/ptrauth-include-from-darwin/module.modulemap b/clang/test/Modules/Inputs/ptrauth-include-from-darwin/module.modulemap
new file mode 100644
index 000000000000..741b9bb1efc5
--- /dev/null
+++ b/clang/test/Modules/Inputs/ptrauth-include-from-darwin/module.modulemap
@@ -0,0 +1,8 @@
+module libc [no_undeclared_includes] {
+  module stddef { header "stddef.h" export * }
+}
+
+module ptrauth {
+  header "ptrauth.h"
+  export *
+}
diff --git a/clang/test/Modules/Inputs/ptrauth-include-from-darwin/ptrauth.h b/clang/test/Modules/Inputs/ptrauth-include-from-darwin/ptrauth.h
new file mode 100644
index 000000000000..c8620b64b2ce
--- /dev/null
+++ b/clang/test/Modules/Inputs/ptrauth-include-from-darwin/ptrauth.h
@@ -0,0 +1 @@
+void foo();
diff --git a/clang/test/Modules/Inputs/ptrauth-include-from-darwin/stddef.h b/clang/test/Modules/Inputs/ptrauth-include-from-darwin/stddef.h
new file mode 100644
index 000000000000..777a524fc671
--- /dev/null
+++ b/clang/test/Modules/Inputs/ptrauth-include-from-darwin/stddef.h
@@ -0,0 +1 @@
+@import ptrauth;
diff --git a/clang/test/Modules/ptrauth-include-from-darwin.m b/clang/test/Modules/ptrauth-include-from-darwin.m
new file mode 100644
index 000000000000..72b0c36e7cb7
--- /dev/null
+++ b/clang/test/Modules/ptrauth-include-from-darwin.m
@@ -0,0 +1,6 @@
+// RUN: rm -rf %t
+// RUN: %clang_cc1 -fmodules-cache-path=%t -fmodules -fimplicit-module-maps -I %S/Inputs/ptrauth-include-from-darwin %s -verify
+// expected-no-diagnostics
+
+@import libc;
+void bar() { foo(); }
-- 
GitLab


From 5b36bf549c8c900d0f01451e02abd821f38d4538 Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Sat, 13 Apr 2024 08:38:07 +0300
Subject: [PATCH 137/464] [clang][NFC] Tidy up extension warnings about
 `_Static_assert`

Some tests in `dr6xx.cpp` were using C11 `_Static_assert`, and were expecting extension warnings in C++98 mode because of that. This is noise, and we can do better than that.
---
 clang/test/CXX/drs/dr6xx.cpp | 26 ++++++++++++--------------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/clang/test/CXX/drs/dr6xx.cpp b/clang/test/CXX/drs/dr6xx.cpp
index d004d5aa4c79..9d3613ae8589 100644
--- a/clang/test/CXX/drs/dr6xx.cpp
+++ b/clang/test/CXX/drs/dr6xx.cpp
@@ -5,6 +5,11 @@
 // RUN: %clang_cc1 -std=c++20 %s -verify=expected,cxx11-20,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -fno-spell-checking
 // RUN: %clang_cc1 -std=c++23 %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors -fno-spell-checking
 
+#if __cplusplus == 199711L
+#define static_assert(...) __extension__ _Static_assert(__VA_ARGS__)
+// cxx98-error@-1 {{variadic macros are a C99 feature}}
+#endif
+
 namespace cwg600 { // cwg600: 2.8
 struct S {
   void f(int);
@@ -43,8 +48,7 @@ namespace cwg601 { // cwg601: yes
 #endif
 
 #if __INT_MAX__ == 0x7FFFFFFF
-_Static_assert(0x80000000 < -1, "0x80000000 should be unsigned");
-// expected-error@-1 {{'_Static_assert' is a C11 extension}}
+static_assert(0x80000000 < -1, "0x80000000 should be unsigned");
 #endif
 
 #if MAX > 0xFFFFFFFFFFFFFFFF && 0x8000000000000000 < -1
@@ -223,15 +227,12 @@ namespace cwg619 { // cwg619: yes
   struct S { static int x[10]; };
 
   int x[];
-  _Static_assert(sizeof(x) == sizeof(int) * 10, "");
-  // expected-error@-1 {{'_Static_assert' is a C11 extension}}
+  static_assert(sizeof(x) == sizeof(int) * 10, "");
   extern int x[];
-  _Static_assert(sizeof(x) == sizeof(int) * 10, "");
-  // expected-error@-1 {{'_Static_assert' is a C11 extension}}
+  static_assert(sizeof(x) == sizeof(int) * 10, "");
 
   int S::x[];
-  _Static_assert(sizeof(S::x) == sizeof(int) * 10, "");
-  // expected-error@-1 {{'_Static_assert' is a C11 extension}}
+  static_assert(sizeof(S::x) == sizeof(int) * 10, "");
 
   void f() {
     extern int x[];
@@ -325,12 +326,10 @@ const bool MB_EQ_WC =
     ',' == L',' && '\\' == L'\\' && '"' == L'"' && '\'' == L'\'';
 #if __STDC_MB_MIGHT_NEQ_WC__
 #ifndef __FreeBSD__ // PR22208, FreeBSD expects us to give a bad (but conforming) answer here.
-_Static_assert(!MB_EQ_WC, "__STDC_MB_MIGHT_NEQ_WC__ but all basic source characters have same representation");
-// expected-error@-1 {{'_Static_assert' is a C11 extension}}
+static_assert(!MB_EQ_WC, "__STDC_MB_MIGHT_NEQ_WC__ but all basic source characters have same representation");
 #endif
 #else
-_Static_assert(MB_EQ_WC, "!__STDC_MB_MIGHT_NEQ_WC__ but some character differs");
-// expected-error@-1 {{'_Static_assert' is a C11 extension}}
+static_assert(MB_EQ_WC, "!__STDC_MB_MIGHT_NEQ_WC__ but some character differs");
 #endif
 }
 
@@ -488,8 +487,7 @@ namespace cwg642 { // cwg642: yes
     const int i = 2;
     {
       char i[i];
-      _Static_assert(sizeof(i) == 2, "");
-      // expected-error@-1 {{'_Static_assert' is a C11 extension}}
+      static_assert(sizeof(i) == 2, "");
     }
   }
 
-- 
GitLab


From 8d8502a1b6e4e2b775d16be456f6b5ee544dcc55 Mon Sep 17 00:00:00 2001
From: Hsiangkai Wang <hsiangkai.wang@arm.com>
Date: Sat, 13 Apr 2024 07:00:46 +0100
Subject: [PATCH 138/464] [mlir][spirv] Add op decoration NoContraction
 (#88578)

---
 mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp |  1 +
 mlir/lib/Target/SPIRV/Serialization/Serializer.cpp     |  1 +
 mlir/test/Target/SPIRV/decorations.mlir                | 10 ++++++++++
 3 files changed, 12 insertions(+)

diff --git a/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp b/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
index 57e8d9a77296..faaa42023a80 100644
--- a/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
+++ b/mlir/lib/Target/SPIRV/Deserialization/Deserializer.cpp
@@ -310,6 +310,7 @@ LogicalResult spirv::Deserializer::processDecoration(ArrayRef<uint32_t> words) {
   case spirv::Decoration::RelaxedPrecision:
   case spirv::Decoration::Restrict:
   case spirv::Decoration::RestrictPointer:
+  case spirv::Decoration::NoContraction:
     if (words.size() != 2) {
       return emitError(unknownLoc, "OpDecoration with ")
              << decorationName << "needs a single target <id>";
diff --git a/mlir/lib/Target/SPIRV/Serialization/Serializer.cpp b/mlir/lib/Target/SPIRV/Serialization/Serializer.cpp
index 9a74ac115e95..200abdf993ce 100644
--- a/mlir/lib/Target/SPIRV/Serialization/Serializer.cpp
+++ b/mlir/lib/Target/SPIRV/Serialization/Serializer.cpp
@@ -276,6 +276,7 @@ LogicalResult Serializer::processDecorationAttr(Location loc, uint32_t resultID,
   case spirv::Decoration::RelaxedPrecision:
   case spirv::Decoration::Restrict:
   case spirv::Decoration::RestrictPointer:
+  case spirv::Decoration::NoContraction:
     // For unit attributes and decoration attributes, the args list
     // has no values so we do nothing.
     if (isa<UnitAttr, DecorationAttr>(attr))
diff --git a/mlir/test/Target/SPIRV/decorations.mlir b/mlir/test/Target/SPIRV/decorations.mlir
index e8f75a1edbd0..195773735431 100644
--- a/mlir/test/Target/SPIRV/decorations.mlir
+++ b/mlir/test/Target/SPIRV/decorations.mlir
@@ -87,3 +87,13 @@ spirv.func @fadd_decorations(%arg: f32) -> f32 "None" {
   spirv.ReturnValue %0 : f32
 }
 }
+
+// -----
+
+spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Kernel], []> {
+spirv.func @fmul_decorations(%arg: f32) -> f32 "None" {
+  // CHECK: spirv.FMul %{{.*}}, %{{.*}} {no_contraction}
+  %0 = spirv.FMul %arg, %arg {no_contraction} : f32
+  spirv.ReturnValue %0 : f32
+}
+}
-- 
GitLab


From 37ebf2ae748f8cbfab756321e04fc7e8c86996a4 Mon Sep 17 00:00:00 2001
From: Lang Hames <lhames@gmail.com>
Date: Fri, 12 Apr 2024 21:28:55 -0600
Subject: [PATCH 139/464] [ORC] Replace more KV loop variables with structured
 bindings.

Coding my way home: 4.42841S, 102.96190W
---
 llvm/lib/ExecutionEngine/Orc/Core.cpp | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/ExecutionEngine/Orc/Core.cpp b/llvm/lib/ExecutionEngine/Orc/Core.cpp
index 4841a2d8c4fd..f03dd434b704 100644
--- a/llvm/lib/ExecutionEngine/Orc/Core.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/Core.cpp
@@ -187,8 +187,8 @@ AsynchronousSymbolQuery::AsynchronousSymbolQuery(
 
   OutstandingSymbolsCount = Symbols.size();
 
-  for (auto &KV : Symbols)
-    ResolvedSymbols[KV.first] = ExecutorSymbolDef();
+  for (auto &[Name, Flags] : Symbols)
+    ResolvedSymbols[Name] = ExecutorSymbolDef();
 }
 
 void AsynchronousSymbolQuery::notifySymbolMetRequiredState(
@@ -271,8 +271,8 @@ void AsynchronousSymbolQuery::dropSymbol(const SymbolStringPtr &Name) {
 void AsynchronousSymbolQuery::detach() {
   ResolvedSymbols.clear();
   OutstandingSymbolsCount = 0;
-  for (auto &KV : QueryRegistrations)
-    KV.first->detachQueryHelper(*this, KV.second);
+  for (auto &[JD, Syms] : QueryRegistrations)
+    JD->detachQueryHelper(*this, Syms);
   QueryRegistrations.clear();
 }
 
@@ -312,8 +312,8 @@ void AbsoluteSymbolsMaterializationUnit::discard(const JITDylib &JD,
 MaterializationUnit::Interface
 AbsoluteSymbolsMaterializationUnit::extractFlags(const SymbolMap &Symbols) {
   SymbolFlagsMap Flags;
-  for (const auto &KV : Symbols)
-    Flags[KV.first] = KV.second.getFlags();
+  for (const auto &[Name, Def] : Symbols)
+    Flags[Name] = Def.getFlags();
   return MaterializationUnit::Interface(std::move(Flags), nullptr);
 }
 
-- 
GitLab


From 3cf8535dbf0bf5fafa99ea1f300e2384a7254fba Mon Sep 17 00:00:00 2001
From: Phoebe Wang <phoebe.wang@intel.com>
Date: Sat, 13 Apr 2024 14:51:29 +0800
Subject: [PATCH 140/464] [X86][BF16] Improve vectorization of BF16 (#88486)

1. Move expansion to combineFP_EXTEND to help with small vectors;
2. Combine FP_ROUND to reduce fptrunc then fpextend after promotion;
---
 llvm/lib/Target/X86/X86ISelLowering.cpp      |  53 +++--
 llvm/test/CodeGen/X86/bfloat.ll              | 234 +++----------------
 llvm/test/CodeGen/X86/concat-fpext-v2bf16.ll |   8 +-
 3 files changed, 66 insertions(+), 229 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index b7cb4b7dafeb..edfa05a66288 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -21433,25 +21433,9 @@ SDValue X86TargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
     return Res;
   }
 
-  if (!SVT.isVector())
+  if (!SVT.isVector() || SVT.getVectorElementType() == MVT::bf16)
     return Op;
 
-  if (SVT.getVectorElementType() == MVT::bf16) {
-    // FIXME: Do we need to support strict FP?
-    assert(!IsStrict && "Strict FP doesn't support BF16");
-    if (VT.getVectorElementType() == MVT::f64) {
-      MVT TmpVT = VT.changeVectorElementType(MVT::f32);
-      return DAG.getNode(ISD::FP_EXTEND, DL, VT,
-                         DAG.getNode(ISD::FP_EXTEND, DL, TmpVT, In));
-    }
-    assert(VT.getVectorElementType() == MVT::f32 && "Unexpected fpext");
-    MVT NVT = SVT.changeVectorElementType(MVT::i32);
-    In = DAG.getBitcast(SVT.changeTypeToInteger(), In);
-    In = DAG.getNode(ISD::ZERO_EXTEND, DL, NVT, In);
-    In = DAG.getNode(ISD::SHL, DL, NVT, In, DAG.getConstant(16, DL, NVT));
-    return DAG.getBitcast(VT, In);
-  }
-
   if (SVT.getVectorElementType() == MVT::f16) {
     if (Subtarget.hasFP16() && isTypeLegal(SVT))
       return Op;
@@ -56517,17 +56501,40 @@ static SDValue combineFP16_TO_FP(SDNode *N, SelectionDAG &DAG,
 
 static SDValue combineFP_EXTEND(SDNode *N, SelectionDAG &DAG,
                                 const X86Subtarget &Subtarget) {
+  EVT VT = N->getValueType(0);
+  bool IsStrict = N->isStrictFPOpcode();
+  SDValue Src = N->getOperand(IsStrict ? 1 : 0);
+  EVT SrcVT = Src.getValueType();
+
+  SDLoc dl(N);
+  if (SrcVT.getScalarType() == MVT::bf16) {
+    if (!IsStrict && Src.getOpcode() == ISD::FP_ROUND &&
+        Src.getOperand(0).getValueType() == VT)
+      return Src.getOperand(0);
+
+    if (!SrcVT.isVector())
+      return SDValue();
+
+    assert(!IsStrict && "Strict FP doesn't support BF16");
+    if (VT.getVectorElementType() == MVT::f64) {
+      MVT TmpVT = VT.getSimpleVT().changeVectorElementType(MVT::f32);
+      return DAG.getNode(ISD::FP_EXTEND, dl, VT,
+                         DAG.getNode(ISD::FP_EXTEND, dl, TmpVT, Src));
+    }
+    assert(VT.getVectorElementType() == MVT::f32 && "Unexpected fpext");
+    MVT NVT = SrcVT.getSimpleVT().changeVectorElementType(MVT::i32);
+    Src = DAG.getBitcast(SrcVT.changeTypeToInteger(), Src);
+    Src = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Src);
+    Src = DAG.getNode(ISD::SHL, dl, NVT, Src, DAG.getConstant(16, dl, NVT));
+    return DAG.getBitcast(VT, Src);
+  }
+
   if (!Subtarget.hasF16C() || Subtarget.useSoftFloat())
     return SDValue();
 
   if (Subtarget.hasFP16())
     return SDValue();
 
-  bool IsStrict = N->isStrictFPOpcode();
-  EVT VT = N->getValueType(0);
-  SDValue Src = N->getOperand(IsStrict ? 1 : 0);
-  EVT SrcVT = Src.getValueType();
-
   if (!SrcVT.isVector() || SrcVT.getVectorElementType() != MVT::f16)
     return SDValue();
 
@@ -56539,8 +56546,6 @@ static SDValue combineFP_EXTEND(SDNode *N, SelectionDAG &DAG,
   if (NumElts == 1 || !isPowerOf2_32(NumElts))
     return SDValue();
 
-  SDLoc dl(N);
-
   // Convert the input to vXi16.
   EVT IntVT = SrcVT.changeVectorElementTypeToInteger();
   Src = DAG.getBitcast(IntVT, Src);
diff --git a/llvm/test/CodeGen/X86/bfloat.ll b/llvm/test/CodeGen/X86/bfloat.ll
index 8a2109a1c78d..39d8e2d50c91 100644
--- a/llvm/test/CodeGen/X86/bfloat.ll
+++ b/llvm/test/CodeGen/X86/bfloat.ll
@@ -1629,22 +1629,8 @@ define <4 x float> @pr64460_1(<4 x bfloat> %a) {
 ;
 ; SSE2-LABEL: pr64460_1:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pextrw $1, %xmm0, %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm2
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm1
-; SSE2-NEXT:    pextrw $3, %xmm0, %eax
-; SSE2-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm2
-; SSE2-NEXT:    movd %xmm0, %eax
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm0
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
@@ -1666,41 +1652,11 @@ define <8 x float> @pr64460_2(<8 x bfloat> %a) {
 ;
 ; SSE2-LABEL: pr64460_2:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movq %xmm0, %rdx
-; SSE2-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1,1]
-; SSE2-NEXT:    movq %xmm0, %rcx
-; SSE2-NEXT:    movq %rcx, %rax
-; SSE2-NEXT:    shrq $32, %rax
-; SSE2-NEXT:    movq %rdx, %rsi
-; SSE2-NEXT:    shrq $32, %rsi
-; SSE2-NEXT:    movl %edx, %edi
-; SSE2-NEXT:    andl $-65536, %edi # imm = 0xFFFF0000
-; SSE2-NEXT:    movd %edi, %xmm1
-; SSE2-NEXT:    movl %edx, %edi
-; SSE2-NEXT:    shll $16, %edi
-; SSE2-NEXT:    movd %edi, %xmm0
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT:    shrq $48, %rdx
-; SSE2-NEXT:    shll $16, %edx
-; SSE2-NEXT:    movd %edx, %xmm1
-; SSE2-NEXT:    shll $16, %esi
-; SSE2-NEXT:    movd %esi, %xmm2
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; SSE2-NEXT:    movl %ecx, %edx
-; SSE2-NEXT:    andl $-65536, %edx # imm = 0xFFFF0000
-; SSE2-NEXT:    movd %edx, %xmm2
-; SSE2-NEXT:    movl %ecx, %edx
-; SSE2-NEXT:    shll $16, %edx
-; SSE2-NEXT:    movd %edx, %xmm1
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; SSE2-NEXT:    shrq $48, %rcx
-; SSE2-NEXT:    shll $16, %ecx
-; SSE2-NEXT:    movd %ecx, %xmm2
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: pr64460_2:
@@ -1721,76 +1677,16 @@ define <16 x float> @pr64460_3(<16 x bfloat> %a) {
 ;
 ; SSE2-LABEL: pr64460_3:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movq %xmm1, %rdi
-; SSE2-NEXT:    punpckhqdq {{.*#+}} xmm1 = xmm1[1,1]
-; SSE2-NEXT:    movq %xmm1, %rcx
-; SSE2-NEXT:    movq %rcx, %rax
-; SSE2-NEXT:    shrq $32, %rax
-; SSE2-NEXT:    movq %xmm0, %r9
-; SSE2-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1,1]
-; SSE2-NEXT:    movq %xmm0, %rsi
-; SSE2-NEXT:    movq %rsi, %rdx
-; SSE2-NEXT:    shrq $32, %rdx
-; SSE2-NEXT:    movq %rdi, %r8
-; SSE2-NEXT:    shrq $32, %r8
-; SSE2-NEXT:    movq %r9, %r10
-; SSE2-NEXT:    shrq $32, %r10
-; SSE2-NEXT:    movl %r9d, %r11d
-; SSE2-NEXT:    andl $-65536, %r11d # imm = 0xFFFF0000
-; SSE2-NEXT:    movd %r11d, %xmm1
-; SSE2-NEXT:    movl %r9d, %r11d
-; SSE2-NEXT:    shll $16, %r11d
-; SSE2-NEXT:    movd %r11d, %xmm0
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT:    shrq $48, %r9
-; SSE2-NEXT:    shll $16, %r9d
-; SSE2-NEXT:    movd %r9d, %xmm1
-; SSE2-NEXT:    shll $16, %r10d
-; SSE2-NEXT:    movd %r10d, %xmm2
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; SSE2-NEXT:    movl %edi, %r9d
-; SSE2-NEXT:    andl $-65536, %r9d # imm = 0xFFFF0000
-; SSE2-NEXT:    movd %r9d, %xmm1
-; SSE2-NEXT:    movl %edi, %r9d
-; SSE2-NEXT:    shll $16, %r9d
-; SSE2-NEXT:    movd %r9d, %xmm2
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSE2-NEXT:    shrq $48, %rdi
-; SSE2-NEXT:    shll $16, %edi
-; SSE2-NEXT:    movd %edi, %xmm1
-; SSE2-NEXT:    shll $16, %r8d
-; SSE2-NEXT:    movd %r8d, %xmm3
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT:    movl %esi, %edi
-; SSE2-NEXT:    andl $-65536, %edi # imm = 0xFFFF0000
-; SSE2-NEXT:    movd %edi, %xmm3
-; SSE2-NEXT:    movl %esi, %edi
-; SSE2-NEXT:    shll $16, %edi
-; SSE2-NEXT:    movd %edi, %xmm1
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; SSE2-NEXT:    shrq $48, %rsi
-; SSE2-NEXT:    shll $16, %esi
-; SSE2-NEXT:    movd %esi, %xmm3
-; SSE2-NEXT:    shll $16, %edx
-; SSE2-NEXT:    movd %edx, %xmm4
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm4[0]
-; SSE2-NEXT:    movl %ecx, %edx
-; SSE2-NEXT:    andl $-65536, %edx # imm = 0xFFFF0000
-; SSE2-NEXT:    movd %edx, %xmm4
-; SSE2-NEXT:    movl %ecx, %edx
-; SSE2-NEXT:    shll $16, %edx
-; SSE2-NEXT:    movd %edx, %xmm3
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; SSE2-NEXT:    shrq $48, %rcx
-; SSE2-NEXT:    shll $16, %ecx
-; SSE2-NEXT:    movd %ecx, %xmm4
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm5
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm5[0]
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3]
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; SSE2-NEXT:    movdqa %xmm5, %xmm0
+; SSE2-NEXT:    movdqa %xmm4, %xmm1
 ; SSE2-NEXT:    retq
 ;
 ; F16-LABEL: pr64460_3:
@@ -1822,47 +1718,17 @@ define <8 x double> @pr64460_4(<8 x bfloat> %a) {
 ;
 ; SSE2-LABEL: pr64460_4:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movq %xmm0, %rsi
-; SSE2-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1,1]
-; SSE2-NEXT:    movq %xmm0, %rdx
-; SSE2-NEXT:    movq %rdx, %rax
-; SSE2-NEXT:    shrq $32, %rax
-; SSE2-NEXT:    movq %rdx, %rcx
-; SSE2-NEXT:    shrq $48, %rcx
-; SSE2-NEXT:    movq %rsi, %rdi
-; SSE2-NEXT:    shrq $32, %rdi
-; SSE2-NEXT:    movq %rsi, %r8
-; SSE2-NEXT:    shrq $48, %r8
-; SSE2-NEXT:    movl %esi, %r9d
-; SSE2-NEXT:    andl $-65536, %r9d # imm = 0xFFFF0000
-; SSE2-NEXT:    movd %r9d, %xmm0
-; SSE2-NEXT:    cvtss2sd %xmm0, %xmm1
-; SSE2-NEXT:    shll $16, %esi
-; SSE2-NEXT:    movd %esi, %xmm0
-; SSE2-NEXT:    cvtss2sd %xmm0, %xmm0
-; SSE2-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE2-NEXT:    shll $16, %r8d
-; SSE2-NEXT:    movd %r8d, %xmm1
-; SSE2-NEXT:    cvtss2sd %xmm1, %xmm2
-; SSE2-NEXT:    shll $16, %edi
-; SSE2-NEXT:    movd %edi, %xmm1
-; SSE2-NEXT:    cvtss2sd %xmm1, %xmm1
-; SSE2-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; SSE2-NEXT:    movl %edx, %esi
-; SSE2-NEXT:    andl $-65536, %esi # imm = 0xFFFF0000
-; SSE2-NEXT:    movd %esi, %xmm2
-; SSE2-NEXT:    cvtss2sd %xmm2, %xmm3
-; SSE2-NEXT:    shll $16, %edx
-; SSE2-NEXT:    movd %edx, %xmm2
-; SSE2-NEXT:    cvtss2sd %xmm2, %xmm2
-; SSE2-NEXT:    movlhps {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT:    shll $16, %ecx
-; SSE2-NEXT:    movd %ecx, %xmm3
-; SSE2-NEXT:    cvtss2sd %xmm3, %xmm4
-; SSE2-NEXT:    shll $16, %eax
-; SSE2-NEXT:    movd %eax, %xmm3
-; SSE2-NEXT:    cvtss2sd %xmm3, %xmm3
-; SSE2-NEXT:    movlhps {{.*#+}} xmm3 = xmm3[0],xmm4[0]
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-NEXT:    cvtps2pd %xmm1, %xmm4
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:    cvtps2pd %xmm3, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE2-NEXT:    cvtps2pd %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
+; SSE2-NEXT:    cvtps2pd %xmm0, %xmm3
+; SSE2-NEXT:    movaps %xmm4, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; F16-LABEL: pr64460_4:
@@ -1874,45 +1740,11 @@ define <8 x double> @pr64460_4(<8 x bfloat> %a) {
 ;
 ; AVXNC-LABEL: pr64460_4:
 ; AVXNC:       # %bb.0:
-; AVXNC-NEXT:    vpextrw $3, %xmm0, %eax
-; AVXNC-NEXT:    shll $16, %eax
-; AVXNC-NEXT:    vmovd %eax, %xmm1
-; AVXNC-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
-; AVXNC-NEXT:    vpextrw $2, %xmm0, %eax
-; AVXNC-NEXT:    shll $16, %eax
-; AVXNC-NEXT:    vmovd %eax, %xmm2
-; AVXNC-NEXT:    vcvtss2sd %xmm2, %xmm2, %xmm2
-; AVXNC-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVXNC-NEXT:    vpextrw $1, %xmm0, %eax
-; AVXNC-NEXT:    shll $16, %eax
-; AVXNC-NEXT:    vmovd %eax, %xmm2
-; AVXNC-NEXT:    vcvtss2sd %xmm2, %xmm2, %xmm2
-; AVXNC-NEXT:    vmovd %xmm0, %eax
-; AVXNC-NEXT:    shll $16, %eax
-; AVXNC-NEXT:    vmovd %eax, %xmm3
-; AVXNC-NEXT:    vcvtss2sd %xmm3, %xmm3, %xmm3
-; AVXNC-NEXT:    vmovlhps {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; AVXNC-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm2
-; AVXNC-NEXT:    vpextrw $7, %xmm0, %eax
-; AVXNC-NEXT:    shll $16, %eax
-; AVXNC-NEXT:    vmovd %eax, %xmm1
-; AVXNC-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1
-; AVXNC-NEXT:    vpextrw $6, %xmm0, %eax
-; AVXNC-NEXT:    shll $16, %eax
-; AVXNC-NEXT:    vmovd %eax, %xmm3
-; AVXNC-NEXT:    vcvtss2sd %xmm3, %xmm3, %xmm3
-; AVXNC-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm3[0],xmm1[0]
-; AVXNC-NEXT:    vpextrw $5, %xmm0, %eax
-; AVXNC-NEXT:    shll $16, %eax
-; AVXNC-NEXT:    vmovd %eax, %xmm3
-; AVXNC-NEXT:    vcvtss2sd %xmm3, %xmm3, %xmm3
-; AVXNC-NEXT:    vpextrw $4, %xmm0, %eax
-; AVXNC-NEXT:    shll $16, %eax
-; AVXNC-NEXT:    vmovd %eax, %xmm0
-; AVXNC-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0
-; AVXNC-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm3[0]
-; AVXNC-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
-; AVXNC-NEXT:    vmovaps %ymm2, %ymm0
+; AVXNC-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVXNC-NEXT:    vpslld $16, %ymm0, %ymm1
+; AVXNC-NEXT:    vcvtps2pd %xmm1, %ymm0
+; AVXNC-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVXNC-NEXT:    vcvtps2pd %xmm1, %ymm1
 ; AVXNC-NEXT:    retq
   %b = fpext <8 x bfloat> %a to <8 x double>
   ret <8 x double> %b
diff --git a/llvm/test/CodeGen/X86/concat-fpext-v2bf16.ll b/llvm/test/CodeGen/X86/concat-fpext-v2bf16.ll
index eff1937b5934..c079a44bc5ef 100644
--- a/llvm/test/CodeGen/X86/concat-fpext-v2bf16.ll
+++ b/llvm/test/CodeGen/X86/concat-fpext-v2bf16.ll
@@ -10,11 +10,11 @@ define void @test(<2 x ptr> %ptr) {
 ; CHECK-NEXT:  # %bb.2: # %loop.127.preheader
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:  .LBB0_1: # %ifmerge.89
-; CHECK-NEXT:    movzwl (%rax), %eax
-; CHECK-NEXT:    shll $16, %eax
-; CHECK-NEXT:    vmovd %eax, %xmm0
-; CHECK-NEXT:    vmulss %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    vbroadcastss %xmm0, %xmm0
+; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vpbroadcastw (%rax), %xmm2
+; CHECK-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; CHECK-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vmovlps %xmm0, (%rax)
 entry:
   br label %then.13
-- 
GitLab


From df9c00beea5be08fc47aaea58eae333fc6cbef33 Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333@gmail.com>
Date: Sat, 13 Apr 2024 15:07:55 +0800
Subject: [PATCH 141/464]  [CVP] Use at-use info in `processBinOp` (#88523)

This patch uses `getConstantRangeAtUse` to infer nsw/nuw flags with
at-use info. It will enables more optimizations in InstCombine.

Compile-time impact:
http://llvm-compile-time-tracker.com/compare.php?from=a5ed14bc8e122fa5ac0aa81f8d8390931bd6b4e4&to=a83d3402b663439b91cb37a046fb7ac0220ba5c7&stat=instructions%3Au

Related issue: #87854
---
 .../Scalar/CorrelatedValuePropagation.cpp     | 11 +++---
 .../CorrelatedValuePropagation/cond-at-use.ll | 34 +++++++++++++++++++
 .../phi-common-val.ll                         |  3 +-
 3 files changed, 40 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
index fb334e8292eb..715cdaff9727 100644
--- a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
+++ b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
@@ -1105,13 +1105,10 @@ static bool processBinOp(BinaryOperator *BinOp, LazyValueInfo *LVI) {
     return false;
 
   Instruction::BinaryOps Opcode = BinOp->getOpcode();
-  Value *LHS = BinOp->getOperand(0);
-  Value *RHS = BinOp->getOperand(1);
-
-  ConstantRange LRange =
-      LVI->getConstantRange(LHS, BinOp, /*UndefAllowed*/ false);
-  ConstantRange RRange =
-      LVI->getConstantRange(RHS, BinOp, /*UndefAllowed*/ false);
+  ConstantRange LRange = LVI->getConstantRangeAtUse(BinOp->getOperandUse(0),
+                                                    /*UndefAllowed=*/false);
+  ConstantRange RRange = LVI->getConstantRangeAtUse(BinOp->getOperandUse(1),
+                                                    /*UndefAllowed=*/false);
 
   bool Changed = false;
   bool NewNUW = false, NewNSW = false;
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/cond-at-use.ll b/llvm/test/Transforms/CorrelatedValuePropagation/cond-at-use.ll
index 0b95139f3dcb..3af4c70a5621 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/cond-at-use.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/cond-at-use.ll
@@ -596,3 +596,37 @@ define i16 @and_elide_poison_flags_missing_noundef(i16 %a) {
   %sel = select i1 %cmp, i16 %and, i16 24
   ret i16 %sel
 }
+
+define i32 @pr87854(i32 noundef %x.1, i32 noundef %i) {
+; CHECK-LABEL: @pr87854(
+; CHECK-NEXT:    [[COND:%.*]] = icmp sgt i32 [[X_1:%.*]], -1
+; CHECK-NEXT:    tail call void @llvm.assume(i1 [[COND]])
+; CHECK-NEXT:    [[INBOUNDS:%.*]] = icmp ult i32 [[I:%.*]], [[X_1]]
+; CHECK-NEXT:    [[NEXT:%.*]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[INBOUNDS]], i32 [[NEXT]], i32 -1
+; CHECK-NEXT:    ret i32 [[SPEC_SELECT]]
+;
+  %cond = icmp sgt i32 %x.1, -1
+  tail call void @llvm.assume(i1 %cond)
+  %inbounds = icmp ult i32 %i, %x.1
+  %next = add i32 %i, 1
+  %spec.select = select i1 %inbounds, i32 %next, i32 -1
+  ret i32 %spec.select
+}
+
+define i64 @test_shl_nsw_at_use(i64 noundef %x) {
+; CHECK-LABEL: @test_shl_nsw_at_use(
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[X:%.*]], 2147483648
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i64 [[ADD]], 4294967296
+; CHECK-NEXT:    [[SHL:%.*]] = shl nsw i64 [[X]], 32
+; CHECK-NEXT:    [[SHR:%.*]] = ashr exact i64 [[SHL]], 32
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i64 [[SHR]], i64 0
+; CHECK-NEXT:    ret i64 [[RES]]
+;
+  %add = add i64 %x, 2147483648
+  %cmp = icmp ult i64 %add, 4294967296
+  %shl = shl i64 %x, 32
+  %shr = ashr exact i64 %shl, 32
+  %res = select i1 %cmp, i64 %shr, i64 0
+  ret i64 %res
+}
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/phi-common-val.ll b/llvm/test/Transforms/CorrelatedValuePropagation/phi-common-val.ll
index 7f102bc1e4fe..7e712947e37a 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/phi-common-val.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/phi-common-val.ll
@@ -156,12 +156,13 @@ bb3:
 define i32 @PR43802_without_nowrap(i32 %arg) {
 ; CHECK-LABEL: @PR43802_without_nowrap(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SUB:%.*]] = sub i32 0, [[ARG:%.*]]
+; CHECK-NEXT:    [[SUB1:%.*]] = sub nsw i32 0, [[ARG:%.*]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[ARG]], -2147483648
 ; CHECK-NEXT:    br i1 [[CMP]], label [[BB2:%.*]], label [[BB3:%.*]]
 ; CHECK:       bb2:
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
+; CHECK-NEXT:    [[SUB:%.*]] = phi i32 [ -2147483648, [[BB2]] ], [ [[SUB1]], [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    ret i32 [[SUB]]
 ;
 entry:
-- 
GitLab


From 6d66db3890a18e3926a49cbfeb28e99c464cfcd5 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Sat, 13 Apr 2024 08:14:40 +0100
Subject: [PATCH 142/464] [SLP] Initial vectorization of non-power-of-2 ops.
 (#77790)

This patch enables vectorization for non-power-of-2 VFs. Initially only
VFs where adding 1 makes the VF a power-of-2, i.e. we can still make
relatively effective use of the vectors.

It relies on the existing target cost-models to return accurate costs
for
non-power-of-2 vectors. I checked mostly AArch64 and X86 and
there the costs seem reasonable for the costs I checked, although
I expect there will be a need to refine both the cost-models and
lowering
to make most effective use of non-power-of-2 SLP vectorization.

Note that re-ordering and shuffling is not implemented for nodes
requiring padding yet to keep the initial implementation simpler.

The feature is guarded by a new flag, off by defaul for now.

PR: https://github.com/llvm/llvm-project/pull/77790
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    |  83 ++++-
 .../SLPVectorizer/AArch64/vec15-base.ll       |  70 ++--
 .../SLPVectorizer/AArch64/vec3-base.ll        | 247 +++++++++-----
 .../SLPVectorizer/AArch64/vec3-calls.ll       |   3 +-
 .../AArch64/vec3-reorder-reshuffle.ll         | 266 ++++++++++-----
 .../Transforms/SLPVectorizer/X86/odd_store.ll |  84 +++--
 .../Transforms/SLPVectorizer/X86/vec3-base.ll |  35 +-
 .../SLPVectorizer/X86/vec3-calls.ll           |  33 +-
 .../X86/vec3-gather-some-loads.ll             |  74 +++--
 .../X86/vec3-reorder-reshuffle.ll             | 248 ++++++++------
 .../X86/vect_copyable_in_binops.ll            | 303 ++++++++++--------
 11 files changed, 947 insertions(+), 499 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 7b5b0efca8d3..09db50132c89 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -190,6 +190,10 @@ static cl::opt<bool>
     ViewSLPTree("view-slp-tree", cl::Hidden,
                 cl::desc("Display the SLP trees with Graphviz"));
 
+static cl::opt<bool> VectorizeNonPowerOf2(
+    "slp-vectorize-non-power-of-2", cl::init(false), cl::Hidden,
+    cl::desc("Try to vectorize with non-power-of-2 number of elements."));
+
 // Limit the number of alias checks. The limit is chosen so that
 // it has no negative effect on the llvm benchmarks.
 static const unsigned AliasedCheckLimit = 10;
@@ -2829,6 +2833,14 @@ private:
                           SmallVectorImpl<Value *> *OpScalars = nullptr,
                           SmallVectorImpl<Value *> *AltScalars = nullptr) const;
 
+    /// Return true if this is a non-power-of-2 node.
+    bool isNonPowOf2Vec() const {
+      bool IsNonPowerOf2 = !isPowerOf2_32(Scalars.size());
+      assert((!IsNonPowerOf2 || ReuseShuffleIndices.empty()) &&
+             "Reshuffling not supported with non-power-of-2 vectors yet.");
+      return IsNonPowerOf2;
+    }
+
 #ifndef NDEBUG
     /// Debug printer.
     LLVM_DUMP_METHOD void dump() const {
@@ -2994,9 +3006,11 @@ private:
       MustGather.insert(VL.begin(), VL.end());
     }
 
-    if (UserTreeIdx.UserTE)
+    if (UserTreeIdx.UserTE) {
       Last->UserTreeIndices.push_back(UserTreeIdx);
-
+      assert((!Last->isNonPowOf2Vec() || Last->ReorderIndices.empty()) &&
+             "Reordering isn't implemented for non-power-of-2 nodes yet");
+    }
     return Last;
   }
 
@@ -4256,6 +4270,13 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
   auto *VecTy = FixedVectorType::get(ScalarTy, Sz);
   // Check the order of pointer operands or that all pointers are the same.
   bool IsSorted = sortPtrAccesses(PointerOps, ScalarTy, *DL, *SE, Order);
+  // FIXME: Reordering isn't implemented for non-power-of-2 nodes yet.
+  if (!Order.empty() && !isPowerOf2_32(VL.size())) {
+    assert(VectorizeNonPowerOf2 && "non-power-of-2 number of loads only "
+                                   "supported with VectorizeNonPowerOf2");
+    return LoadsState::Gather;
+  }
+
   Align CommonAlignment = computeCommonAlignment<LoadInst>(VL);
   if (!IsSorted && Sz > MinProfitableStridedLoads && TTI->isTypeLegal(VecTy) &&
       TTI->isLegalStridedLoadStore(VecTy, CommonAlignment) &&
@@ -4575,6 +4596,10 @@ static bool areTwoInsertFromSameBuildVector(
 
 std::optional<BoUpSLP::OrdersType>
 BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom) {
+  // FIXME: Vectorizing is not supported yet for non-power-of-2 ops.
+  if (TE.isNonPowOf2Vec())
+    return std::nullopt;
+
   // No need to reorder if need to shuffle reuses, still need to shuffle the
   // node.
   if (!TE.ReuseShuffleIndices.empty()) {
@@ -5145,6 +5170,10 @@ bool BoUpSLP::canReorderOperands(
     TreeEntry *UserTE, SmallVectorImpl<std::pair<unsigned, TreeEntry *>> &Edges,
     ArrayRef<TreeEntry *> ReorderableGathers,
     SmallVectorImpl<TreeEntry *> &GatherOps) {
+  // FIXME: Reordering isn't implemented for non-power-of-2 nodes yet.
+  if (UserTE->isNonPowOf2Vec())
+    return false;
+
   for (unsigned I = 0, E = UserTE->getNumOperands(); I < E; ++I) {
     if (any_of(Edges, [I](const std::pair<unsigned, TreeEntry *> &OpData) {
           return OpData.first == I &&
@@ -5318,6 +5347,9 @@ void BoUpSLP::reorderBottomToTop(bool IgnoreReorder) {
         }
         auto Res = OrdersUses.insert(std::make_pair(OrdersType(), 0));
         const auto AllowsReordering = [&](const TreeEntry *TE) {
+          // FIXME: Reordering isn't implemented for non-power-of-2 nodes yet.
+          if (TE->isNonPowOf2Vec())
+            return false;
           if (!TE->ReorderIndices.empty() || !TE->ReuseShuffleIndices.empty() ||
               (TE->State == TreeEntry::Vectorize && TE->isAltShuffle()) ||
               (IgnoreReorder && TE->Idx == 0))
@@ -5944,6 +5976,9 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
   case Instruction::ExtractValue:
   case Instruction::ExtractElement: {
     bool Reuse = canReuseExtract(VL, VL0, CurrentOrder);
+    // FIXME: Vectorizing is not supported yet for non-power-of-2 ops.
+    if (!isPowerOf2_32(VL.size()))
+      return TreeEntry::NeedToGather;
     if (Reuse || !CurrentOrder.empty())
       return TreeEntry::Vectorize;
     LLVM_DEBUG(dbgs() << "SLP: Gather extract sequence.\n");
@@ -6258,6 +6293,13 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
     if (NumUniqueScalarValues == VL.size()) {
       ReuseShuffleIndicies.clear();
     } else {
+      // FIXME: Reshuffing scalars is not supported yet for non-power-of-2 ops.
+      if (UserTreeIdx.UserTE && UserTreeIdx.UserTE->isNonPowOf2Vec()) {
+        LLVM_DEBUG(dbgs() << "SLP: Reshuffling scalars not yet supported "
+                             "for nodes with padding.\n");
+        newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx);
+        return false;
+      }
       LLVM_DEBUG(dbgs() << "SLP: Shuffle for reused scalars.\n");
       if (NumUniqueScalarValues <= 1 ||
           (UniquePositions.size() == 1 && all_of(UniqueValues,
@@ -7868,7 +7910,8 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis {
         for (unsigned I = 0, End = VL.size(); I < End; I += VF) {
           if (VectorizedLoads.contains(VL[I]))
             continue;
-          GatherCost += getBuildVectorCost(VL.slice(I, VF), Root);
+          GatherCost +=
+              getBuildVectorCost(VL.slice(I, std::min(End - I, VF)), Root);
         }
         // Exclude potentially vectorized loads from list of gathered
         // scalars.
@@ -10678,6 +10721,9 @@ BoUpSLP::isGatherShuffledEntry(
   // No need to check for the topmost gather node.
   if (TE == VectorizableTree.front().get())
     return {};
+  // FIXME: Gathering for non-power-of-2 nodes not implemented yet.
+  if (TE->isNonPowOf2Vec())
+    return {};
   Mask.assign(VL.size(), PoisonMaskElem);
   assert(TE->UserTreeIndices.size() == 1 &&
          "Expected only single user of the gather node.");
@@ -14995,8 +15041,13 @@ bool SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
   const unsigned Sz = R.getVectorElementSize(Chain[0]);
   unsigned VF = Chain.size();
 
-  if (!isPowerOf2_32(Sz) || !isPowerOf2_32(VF) || VF < 2 || VF < MinVF)
-    return false;
+  if (!isPowerOf2_32(Sz) || !isPowerOf2_32(VF) || VF < 2 || VF < MinVF) {
+    // Check if vectorizing with a non-power-of-2 VF should be considered. At
+    // the moment, only consider cases where VF + 1 is a power-of-2, i.e. almost
+    // all vector lanes are used.
+    if (!VectorizeNonPowerOf2 || (VF < MinVF && VF + 1 != MinVF))
+      return false;
+  }
 
   LLVM_DEBUG(dbgs() << "SLP: Analyzing " << VF << " stores at offset " << Idx
                     << "\n");
@@ -15095,14 +15146,22 @@ bool SLPVectorizerPass::vectorizeStores(ArrayRef<StoreInst *> Stores,
         continue;
       }
 
+      unsigned NonPowerOf2VF = 0;
+      if (VectorizeNonPowerOf2) {
+        // First try vectorizing with a non-power-of-2 VF. At the moment, only
+        // consider cases where VF + 1 is a power-of-2, i.e. almost all vector
+        // lanes are used.
+        unsigned CandVF = Operands.size();
+        if (isPowerOf2_32(CandVF + 1) && CandVF <= MaxVF)
+          NonPowerOf2VF = CandVF;
+      }
+
       unsigned Sz = 1 + Log2_32(MaxVF) - Log2_32(MinVF);
-      SmallVector<unsigned> CandidateVFs(Sz);
-      // FIXME: Is division-by-2 the correct step? Should we assert that the
-      // register size is a power-of-2?
-      unsigned Size = MaxVF;
-      for_each(CandidateVFs, [&](unsigned &VF) {
-        VF = Size;
-        Size /= 2;
+      SmallVector<unsigned> CandidateVFs(Sz + (NonPowerOf2VF > 0 ? 1 : 0));
+      unsigned Size = MinVF;
+      for_each(reverse(CandidateVFs), [&](unsigned &VF) {
+        VF = Size > MaxVF ? NonPowerOf2VF : Size;
+        Size *= 2;
       });
       unsigned StartIdx = 0;
       for (unsigned Size : CandidateVFs) {
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/vec15-base.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/vec15-base.ll
index b9e959d50bef..7b27489782fc 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/vec15-base.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/vec15-base.ll
@@ -1,35 +1,45 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt -passes=slp-vectorizer -mtriple=arm64-apple-ios -S %s | FileCheck %s
+; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2 -mtriple=arm64-apple-ios -S %s | FileCheck --check-prefixes=NON-POW2 %s
+; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2=false -mtriple=arm64-apple-ios -S %s | FileCheck --check-prefixes=POW2-ONLY %s
 
 define void @v15_load_i8_mul_by_constant_store(ptr %src, ptr noalias %dst) {
-; CHECK-LABEL: define void @v15_load_i8_mul_by_constant_store(
-; CHECK-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]]) {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[GEP_SRC_0:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i8 0
-; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr [[GEP_SRC_0]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nsw <8 x i8> [[TMP0]], <i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10>
-; CHECK-NEXT:    store <8 x i8> [[TMP1]], ptr [[DST]], align 1
-; CHECK-NEXT:    [[GEP_SRC_8:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i8 8
-; CHECK-NEXT:    [[DST_8:%.*]] = getelementptr i8, ptr [[DST]], i8 8
-; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i8>, ptr [[GEP_SRC_8]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nsw <4 x i8> [[TMP2]], <i8 10, i8 10, i8 10, i8 10>
-; CHECK-NEXT:    store <4 x i8> [[TMP3]], ptr [[DST_8]], align 1
-; CHECK-NEXT:    [[GEP_SRC_12:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i8 12
-; CHECK-NEXT:    [[L_SRC_12:%.*]] = load i8, ptr [[GEP_SRC_12]], align 4
-; CHECK-NEXT:    [[MUL_12:%.*]] = mul nsw i8 [[L_SRC_12]], 10
-; CHECK-NEXT:    [[DST_12:%.*]] = getelementptr i8, ptr [[DST]], i8 12
-; CHECK-NEXT:    store i8 [[MUL_12]], ptr [[DST_12]], align 1
-; CHECK-NEXT:    [[GEP_SRC_13:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i8 13
-; CHECK-NEXT:    [[L_SRC_13:%.*]] = load i8, ptr [[GEP_SRC_13]], align 4
-; CHECK-NEXT:    [[MUL_13:%.*]] = mul nsw i8 [[L_SRC_13]], 10
-; CHECK-NEXT:    [[DST_13:%.*]] = getelementptr i8, ptr [[DST]], i8 13
-; CHECK-NEXT:    store i8 [[MUL_13]], ptr [[DST_13]], align 1
-; CHECK-NEXT:    [[GEP_SRC_14:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i8 14
-; CHECK-NEXT:    [[L_SRC_14:%.*]] = load i8, ptr [[GEP_SRC_14]], align 4
-; CHECK-NEXT:    [[MUL_14:%.*]] = mul nsw i8 [[L_SRC_14]], 10
-; CHECK-NEXT:    [[DST_14:%.*]] = getelementptr i8, ptr [[DST]], i8 14
-; CHECK-NEXT:    store i8 [[MUL_14]], ptr [[DST_14]], align 1
-; CHECK-NEXT:    ret void
+; NON-POW2-LABEL: define void @v15_load_i8_mul_by_constant_store(
+; NON-POW2-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]]) {
+; NON-POW2-NEXT:  entry:
+; NON-POW2-NEXT:    [[GEP_SRC_0:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i8 0
+; NON-POW2-NEXT:    [[TMP0:%.*]] = load <15 x i8>, ptr [[GEP_SRC_0]], align 4
+; NON-POW2-NEXT:    [[TMP1:%.*]] = mul nsw <15 x i8> [[TMP0]], <i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10>
+; NON-POW2-NEXT:    store <15 x i8> [[TMP1]], ptr [[DST]], align 1
+; NON-POW2-NEXT:    ret void
+;
+; POW2-ONLY-LABEL: define void @v15_load_i8_mul_by_constant_store(
+; POW2-ONLY-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]]) {
+; POW2-ONLY-NEXT:  entry:
+; POW2-ONLY-NEXT:    [[GEP_SRC_0:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i8 0
+; POW2-ONLY-NEXT:    [[TMP0:%.*]] = load <8 x i8>, ptr [[GEP_SRC_0]], align 4
+; POW2-ONLY-NEXT:    [[TMP1:%.*]] = mul nsw <8 x i8> [[TMP0]], <i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10, i8 10>
+; POW2-ONLY-NEXT:    store <8 x i8> [[TMP1]], ptr [[DST]], align 1
+; POW2-ONLY-NEXT:    [[GEP_SRC_8:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i8 8
+; POW2-ONLY-NEXT:    [[DST_8:%.*]] = getelementptr i8, ptr [[DST]], i8 8
+; POW2-ONLY-NEXT:    [[TMP2:%.*]] = load <4 x i8>, ptr [[GEP_SRC_8]], align 4
+; POW2-ONLY-NEXT:    [[TMP3:%.*]] = mul nsw <4 x i8> [[TMP2]], <i8 10, i8 10, i8 10, i8 10>
+; POW2-ONLY-NEXT:    store <4 x i8> [[TMP3]], ptr [[DST_8]], align 1
+; POW2-ONLY-NEXT:    [[GEP_SRC_12:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i8 12
+; POW2-ONLY-NEXT:    [[L_SRC_12:%.*]] = load i8, ptr [[GEP_SRC_12]], align 4
+; POW2-ONLY-NEXT:    [[MUL_12:%.*]] = mul nsw i8 [[L_SRC_12]], 10
+; POW2-ONLY-NEXT:    [[DST_12:%.*]] = getelementptr i8, ptr [[DST]], i8 12
+; POW2-ONLY-NEXT:    store i8 [[MUL_12]], ptr [[DST_12]], align 1
+; POW2-ONLY-NEXT:    [[GEP_SRC_13:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i8 13
+; POW2-ONLY-NEXT:    [[L_SRC_13:%.*]] = load i8, ptr [[GEP_SRC_13]], align 4
+; POW2-ONLY-NEXT:    [[MUL_13:%.*]] = mul nsw i8 [[L_SRC_13]], 10
+; POW2-ONLY-NEXT:    [[DST_13:%.*]] = getelementptr i8, ptr [[DST]], i8 13
+; POW2-ONLY-NEXT:    store i8 [[MUL_13]], ptr [[DST_13]], align 1
+; POW2-ONLY-NEXT:    [[GEP_SRC_14:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i8 14
+; POW2-ONLY-NEXT:    [[L_SRC_14:%.*]] = load i8, ptr [[GEP_SRC_14]], align 4
+; POW2-ONLY-NEXT:    [[MUL_14:%.*]] = mul nsw i8 [[L_SRC_14]], 10
+; POW2-ONLY-NEXT:    [[DST_14:%.*]] = getelementptr i8, ptr [[DST]], i8 14
+; POW2-ONLY-NEXT:    store i8 [[MUL_14]], ptr [[DST_14]], align 1
+; POW2-ONLY-NEXT:    ret void
 ;
 entry:
   %gep.src.0 = getelementptr inbounds i8, ptr %src, i8 0
@@ -123,5 +133,3 @@ entry:
 
   ret void
 }
-
-
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-base.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-base.ll
index 59ffbf7ef9b2..c18811a35c1e 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-base.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-base.ll
@@ -1,16 +1,69 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -passes=slp-vectorizer -mtriple=arm64-apple-ios -S %s | FileCheck %s
+; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2 -mtriple=arm64-apple-ios -S %s | FileCheck --check-prefixes=CHECK,NON-POW2 %s
+; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2=false -mtriple=arm64-apple-ios -S %s | FileCheck --check-prefixes=CHECK,POW2-ONLY %s
 
 define void @v3_load_i32_mul_by_constant_store(ptr %src, ptr %dst) {
-; CHECK-LABEL: @v3_load_i32_mul_by_constant_store(
+; NON-POW2-LABEL: @v3_load_i32_mul_by_constant_store(
+; NON-POW2-NEXT:  entry:
+; NON-POW2-NEXT:    [[GEP_SRC_0:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i32 0
+; NON-POW2-NEXT:    [[TMP0:%.*]] = load <3 x i32>, ptr [[GEP_SRC_0]], align 4
+; NON-POW2-NEXT:    [[TMP1:%.*]] = mul nsw <3 x i32> [[TMP0]], <i32 10, i32 10, i32 10>
+; NON-POW2-NEXT:    store <3 x i32> [[TMP1]], ptr [[DST:%.*]], align 4
+; NON-POW2-NEXT:    ret void
+;
+; POW2-ONLY-LABEL: @v3_load_i32_mul_by_constant_store(
+; POW2-ONLY-NEXT:  entry:
+; POW2-ONLY-NEXT:    [[GEP_SRC_0:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i32 0
+; POW2-ONLY-NEXT:    [[GEP_SRC_2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 2
+; POW2-ONLY-NEXT:    [[L_SRC_2:%.*]] = load i32, ptr [[GEP_SRC_2]], align 4
+; POW2-ONLY-NEXT:    [[MUL_2:%.*]] = mul nsw i32 [[L_SRC_2]], 10
+; POW2-ONLY-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr [[GEP_SRC_0]], align 4
+; POW2-ONLY-NEXT:    [[TMP1:%.*]] = mul nsw <2 x i32> [[TMP0]], <i32 10, i32 10>
+; POW2-ONLY-NEXT:    store <2 x i32> [[TMP1]], ptr [[DST:%.*]], align 4
+; POW2-ONLY-NEXT:    [[DST_2:%.*]] = getelementptr i32, ptr [[DST]], i32 2
+; POW2-ONLY-NEXT:    store i32 [[MUL_2]], ptr [[DST_2]], align 4
+; POW2-ONLY-NEXT:    ret void
+;
+entry:
+  %gep.src.0 = getelementptr inbounds i32, ptr %src, i32 0
+  %l.src.0 = load i32, ptr %gep.src.0, align 4
+  %mul.0 = mul nsw i32 %l.src.0, 10
+
+  %gep.src.1 = getelementptr inbounds i32, ptr %src, i32 1
+  %l.src.1 = load i32, ptr %gep.src.1, align 4
+  %mul.1 = mul nsw i32 %l.src.1, 10
+
+  %gep.src.2 = getelementptr inbounds i32, ptr %src, i32 2
+  %l.src.2 = load i32, ptr %gep.src.2, align 4
+  %mul.2 = mul nsw i32 %l.src.2, 10
+
+  store i32 %mul.0, ptr %dst
+
+  %dst.1 = getelementptr i32, ptr %dst, i32 1
+  store i32 %mul.1, ptr %dst.1
+
+  %dst.2 = getelementptr i32, ptr %dst, i32 2
+  store i32 %mul.2, ptr %dst.2
+
+  ret void
+}
+
+; Should no be vectorized with a undef/poison element as padding, as division by undef/poison may cause UB.
+define void @v3_load_i32_udiv_by_constant_store(ptr %src, ptr %dst) {
+; CHECK-LABEL: @v3_load_i32_udiv_by_constant_store(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[GEP_SRC_0:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i32 0
+; CHECK-NEXT:    [[L_SRC_0:%.*]] = load i32, ptr [[GEP_SRC_0]], align 4
+; CHECK-NEXT:    [[MUL_0:%.*]] = udiv i32 10, [[L_SRC_0]]
+; CHECK-NEXT:    [[GEP_SRC_1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 1
+; CHECK-NEXT:    [[L_SRC_1:%.*]] = load i32, ptr [[GEP_SRC_1]], align 4
+; CHECK-NEXT:    [[MUL_1:%.*]] = udiv i32 10, [[L_SRC_1]]
 ; CHECK-NEXT:    [[GEP_SRC_2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 2
 ; CHECK-NEXT:    [[L_SRC_2:%.*]] = load i32, ptr [[GEP_SRC_2]], align 4
-; CHECK-NEXT:    [[MUL_2:%.*]] = mul nsw i32 [[L_SRC_2]], 10
-; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr [[GEP_SRC_0]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = mul nsw <2 x i32> [[TMP0]], <i32 10, i32 10>
-; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[DST:%.*]], align 4
+; CHECK-NEXT:    [[MUL_2:%.*]] = udiv i32 10, [[L_SRC_2]]
+; CHECK-NEXT:    store i32 [[MUL_0]], ptr [[DST:%.*]], align 4
+; CHECK-NEXT:    [[DST_1:%.*]] = getelementptr i32, ptr [[DST]], i32 1
+; CHECK-NEXT:    store i32 [[MUL_1]], ptr [[DST_1]], align 4
 ; CHECK-NEXT:    [[DST_2:%.*]] = getelementptr i32, ptr [[DST]], i32 2
 ; CHECK-NEXT:    store i32 [[MUL_2]], ptr [[DST_2]], align 4
 ; CHECK-NEXT:    ret void
@@ -18,15 +71,15 @@ define void @v3_load_i32_mul_by_constant_store(ptr %src, ptr %dst) {
 entry:
   %gep.src.0 = getelementptr inbounds i32, ptr %src, i32 0
   %l.src.0 = load i32, ptr %gep.src.0, align 4
-  %mul.0 = mul nsw i32 %l.src.0, 10
+  %mul.0 = udiv i32 10, %l.src.0
 
   %gep.src.1 = getelementptr inbounds i32, ptr %src, i32 1
   %l.src.1 = load i32, ptr %gep.src.1, align 4
-  %mul.1 = mul nsw i32 %l.src.1, 10
+  %mul.1 = udiv i32 10, %l.src.1
 
   %gep.src.2 = getelementptr inbounds i32, ptr %src, i32 2
   %l.src.2 = load i32, ptr %gep.src.2, align 4
-  %mul.2 = mul nsw i32 %l.src.2, 10
+  %mul.2 = udiv i32 10, %l.src.2
 
   store i32 %mul.0, ptr %dst
 
@@ -39,23 +92,35 @@ entry:
   ret void
 }
 
+
+
 define void @v3_load_i32_mul_store(ptr %src.1, ptr %src.2, ptr %dst) {
-; CHECK-LABEL: @v3_load_i32_mul_store(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[GEP_SRC_1_0:%.*]] = getelementptr inbounds i32, ptr [[SRC_1:%.*]], i32 0
-; CHECK-NEXT:    [[GEP_SRC_2_0:%.*]] = getelementptr inbounds i32, ptr [[SRC_2:%.*]], i32 0
-; CHECK-NEXT:    [[GEP_SRC_1_2:%.*]] = getelementptr inbounds i32, ptr [[SRC_1]], i32 2
-; CHECK-NEXT:    [[L_SRC_1_2:%.*]] = load i32, ptr [[GEP_SRC_1_2]], align 4
-; CHECK-NEXT:    [[GEP_SRC_2_2:%.*]] = getelementptr inbounds i32, ptr [[SRC_2]], i32 2
-; CHECK-NEXT:    [[L_SRC_2_2:%.*]] = load i32, ptr [[GEP_SRC_2_2]], align 4
-; CHECK-NEXT:    [[MUL_2:%.*]] = mul nsw i32 [[L_SRC_1_2]], [[L_SRC_2_2]]
-; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr [[GEP_SRC_1_0]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr [[GEP_SRC_2_0]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = mul nsw <2 x i32> [[TMP0]], [[TMP1]]
-; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[DST:%.*]], align 4
-; CHECK-NEXT:    [[DST_2:%.*]] = getelementptr i32, ptr [[DST]], i32 2
-; CHECK-NEXT:    store i32 [[MUL_2]], ptr [[DST_2]], align 4
-; CHECK-NEXT:    ret void
+; NON-POW2-LABEL: @v3_load_i32_mul_store(
+; NON-POW2-NEXT:  entry:
+; NON-POW2-NEXT:    [[GEP_SRC_1_0:%.*]] = getelementptr inbounds i32, ptr [[SRC_1:%.*]], i32 0
+; NON-POW2-NEXT:    [[GEP_SRC_2_0:%.*]] = getelementptr inbounds i32, ptr [[SRC_2:%.*]], i32 0
+; NON-POW2-NEXT:    [[TMP0:%.*]] = load <3 x i32>, ptr [[GEP_SRC_1_0]], align 4
+; NON-POW2-NEXT:    [[TMP1:%.*]] = load <3 x i32>, ptr [[GEP_SRC_2_0]], align 4
+; NON-POW2-NEXT:    [[TMP2:%.*]] = mul nsw <3 x i32> [[TMP0]], [[TMP1]]
+; NON-POW2-NEXT:    store <3 x i32> [[TMP2]], ptr [[DST:%.*]], align 4
+; NON-POW2-NEXT:    ret void
+;
+; POW2-ONLY-LABEL: @v3_load_i32_mul_store(
+; POW2-ONLY-NEXT:  entry:
+; POW2-ONLY-NEXT:    [[GEP_SRC_1_0:%.*]] = getelementptr inbounds i32, ptr [[SRC_1:%.*]], i32 0
+; POW2-ONLY-NEXT:    [[GEP_SRC_2_0:%.*]] = getelementptr inbounds i32, ptr [[SRC_2:%.*]], i32 0
+; POW2-ONLY-NEXT:    [[GEP_SRC_1_2:%.*]] = getelementptr inbounds i32, ptr [[SRC_1]], i32 2
+; POW2-ONLY-NEXT:    [[L_SRC_1_2:%.*]] = load i32, ptr [[GEP_SRC_1_2]], align 4
+; POW2-ONLY-NEXT:    [[GEP_SRC_2_2:%.*]] = getelementptr inbounds i32, ptr [[SRC_2]], i32 2
+; POW2-ONLY-NEXT:    [[L_SRC_2_2:%.*]] = load i32, ptr [[GEP_SRC_2_2]], align 4
+; POW2-ONLY-NEXT:    [[MUL_2:%.*]] = mul nsw i32 [[L_SRC_1_2]], [[L_SRC_2_2]]
+; POW2-ONLY-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr [[GEP_SRC_1_0]], align 4
+; POW2-ONLY-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr [[GEP_SRC_2_0]], align 4
+; POW2-ONLY-NEXT:    [[TMP2:%.*]] = mul nsw <2 x i32> [[TMP0]], [[TMP1]]
+; POW2-ONLY-NEXT:    store <2 x i32> [[TMP2]], ptr [[DST:%.*]], align 4
+; POW2-ONLY-NEXT:    [[DST_2:%.*]] = getelementptr i32, ptr [[DST]], i32 2
+; POW2-ONLY-NEXT:    store i32 [[MUL_2]], ptr [[DST_2]], align 4
+; POW2-ONLY-NEXT:    ret void
 ;
 entry:
   %gep.src.1.0 = getelementptr inbounds i32, ptr %src.1, i32 0
@@ -88,24 +153,35 @@ entry:
 }
 
 define void @v3_load_i32_mul_add_const_store(ptr %src.1, ptr %src.2, ptr %dst) {
-; CHECK-LABEL: @v3_load_i32_mul_add_const_store(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[GEP_SRC_1_0:%.*]] = getelementptr inbounds i32, ptr [[SRC_1:%.*]], i32 0
-; CHECK-NEXT:    [[GEP_SRC_2_0:%.*]] = getelementptr inbounds i32, ptr [[SRC_2:%.*]], i32 0
-; CHECK-NEXT:    [[GEP_SRC_1_2:%.*]] = getelementptr inbounds i32, ptr [[SRC_1]], i32 2
-; CHECK-NEXT:    [[L_SRC_1_2:%.*]] = load i32, ptr [[GEP_SRC_1_2]], align 4
-; CHECK-NEXT:    [[GEP_SRC_2_2:%.*]] = getelementptr inbounds i32, ptr [[SRC_2]], i32 2
-; CHECK-NEXT:    [[L_SRC_2_2:%.*]] = load i32, ptr [[GEP_SRC_2_2]], align 4
-; CHECK-NEXT:    [[MUL_2:%.*]] = mul nsw i32 [[L_SRC_1_2]], [[L_SRC_2_2]]
-; CHECK-NEXT:    [[ADD_2:%.*]] = add i32 [[MUL_2]], 9
-; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr [[GEP_SRC_1_0]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr [[GEP_SRC_2_0]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = mul nsw <2 x i32> [[TMP0]], [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = add <2 x i32> [[TMP2]], <i32 9, i32 9>
-; CHECK-NEXT:    store <2 x i32> [[TMP3]], ptr [[DST:%.*]], align 4
-; CHECK-NEXT:    [[DST_2:%.*]] = getelementptr i32, ptr [[DST]], i32 2
-; CHECK-NEXT:    store i32 [[ADD_2]], ptr [[DST_2]], align 4
-; CHECK-NEXT:    ret void
+; NON-POW2-LABEL: @v3_load_i32_mul_add_const_store(
+; NON-POW2-NEXT:  entry:
+; NON-POW2-NEXT:    [[GEP_SRC_1_0:%.*]] = getelementptr inbounds i32, ptr [[SRC_1:%.*]], i32 0
+; NON-POW2-NEXT:    [[GEP_SRC_2_0:%.*]] = getelementptr inbounds i32, ptr [[SRC_2:%.*]], i32 0
+; NON-POW2-NEXT:    [[TMP0:%.*]] = load <3 x i32>, ptr [[GEP_SRC_1_0]], align 4
+; NON-POW2-NEXT:    [[TMP1:%.*]] = load <3 x i32>, ptr [[GEP_SRC_2_0]], align 4
+; NON-POW2-NEXT:    [[TMP2:%.*]] = mul nsw <3 x i32> [[TMP0]], [[TMP1]]
+; NON-POW2-NEXT:    [[TMP3:%.*]] = add <3 x i32> [[TMP2]], <i32 9, i32 9, i32 9>
+; NON-POW2-NEXT:    store <3 x i32> [[TMP3]], ptr [[DST:%.*]], align 4
+; NON-POW2-NEXT:    ret void
+;
+; POW2-ONLY-LABEL: @v3_load_i32_mul_add_const_store(
+; POW2-ONLY-NEXT:  entry:
+; POW2-ONLY-NEXT:    [[GEP_SRC_1_0:%.*]] = getelementptr inbounds i32, ptr [[SRC_1:%.*]], i32 0
+; POW2-ONLY-NEXT:    [[GEP_SRC_2_0:%.*]] = getelementptr inbounds i32, ptr [[SRC_2:%.*]], i32 0
+; POW2-ONLY-NEXT:    [[GEP_SRC_1_2:%.*]] = getelementptr inbounds i32, ptr [[SRC_1]], i32 2
+; POW2-ONLY-NEXT:    [[L_SRC_1_2:%.*]] = load i32, ptr [[GEP_SRC_1_2]], align 4
+; POW2-ONLY-NEXT:    [[GEP_SRC_2_2:%.*]] = getelementptr inbounds i32, ptr [[SRC_2]], i32 2
+; POW2-ONLY-NEXT:    [[L_SRC_2_2:%.*]] = load i32, ptr [[GEP_SRC_2_2]], align 4
+; POW2-ONLY-NEXT:    [[MUL_2:%.*]] = mul nsw i32 [[L_SRC_1_2]], [[L_SRC_2_2]]
+; POW2-ONLY-NEXT:    [[ADD_2:%.*]] = add i32 [[MUL_2]], 9
+; POW2-ONLY-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr [[GEP_SRC_1_0]], align 4
+; POW2-ONLY-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr [[GEP_SRC_2_0]], align 4
+; POW2-ONLY-NEXT:    [[TMP2:%.*]] = mul nsw <2 x i32> [[TMP0]], [[TMP1]]
+; POW2-ONLY-NEXT:    [[TMP3:%.*]] = add <2 x i32> [[TMP2]], <i32 9, i32 9>
+; POW2-ONLY-NEXT:    store <2 x i32> [[TMP3]], ptr [[DST:%.*]], align 4
+; POW2-ONLY-NEXT:    [[DST_2:%.*]] = getelementptr i32, ptr [[DST]], i32 2
+; POW2-ONLY-NEXT:    store i32 [[ADD_2]], ptr [[DST_2]], align 4
+; POW2-ONLY-NEXT:    ret void
 ;
 entry:
   %gep.src.1.0 = getelementptr inbounds i32, ptr %src.1, i32 0
@@ -141,18 +217,26 @@ entry:
 }
 
 define void @v3_load_f32_fadd_fadd_by_constant_store(ptr %src, ptr %dst) {
-; CHECK-LABEL: @v3_load_f32_fadd_fadd_by_constant_store(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[GEP_SRC_0:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i32 0
-; CHECK-NEXT:    [[GEP_SRC_2:%.*]] = getelementptr inbounds float, ptr [[SRC]], i32 2
-; CHECK-NEXT:    [[L_SRC_2:%.*]] = load float, ptr [[GEP_SRC_2]], align 4
-; CHECK-NEXT:    [[FADD_2:%.*]] = fadd float [[L_SRC_2]], 1.000000e+01
-; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x float>, ptr [[GEP_SRC_0]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = fadd <2 x float> [[TMP0]], <float 1.000000e+01, float 1.000000e+01>
-; CHECK-NEXT:    store <2 x float> [[TMP1]], ptr [[DST:%.*]], align 4
-; CHECK-NEXT:    [[DST_2:%.*]] = getelementptr float, ptr [[DST]], i32 2
-; CHECK-NEXT:    store float [[FADD_2]], ptr [[DST_2]], align 4
-; CHECK-NEXT:    ret void
+; NON-POW2-LABEL: @v3_load_f32_fadd_fadd_by_constant_store(
+; NON-POW2-NEXT:  entry:
+; NON-POW2-NEXT:    [[GEP_SRC_0:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i32 0
+; NON-POW2-NEXT:    [[TMP0:%.*]] = load <3 x float>, ptr [[GEP_SRC_0]], align 4
+; NON-POW2-NEXT:    [[TMP1:%.*]] = fadd <3 x float> [[TMP0]], <float 1.000000e+01, float 1.000000e+01, float 1.000000e+01>
+; NON-POW2-NEXT:    store <3 x float> [[TMP1]], ptr [[DST:%.*]], align 4
+; NON-POW2-NEXT:    ret void
+;
+; POW2-ONLY-LABEL: @v3_load_f32_fadd_fadd_by_constant_store(
+; POW2-ONLY-NEXT:  entry:
+; POW2-ONLY-NEXT:    [[GEP_SRC_0:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i32 0
+; POW2-ONLY-NEXT:    [[GEP_SRC_2:%.*]] = getelementptr inbounds float, ptr [[SRC]], i32 2
+; POW2-ONLY-NEXT:    [[L_SRC_2:%.*]] = load float, ptr [[GEP_SRC_2]], align 4
+; POW2-ONLY-NEXT:    [[FADD_2:%.*]] = fadd float [[L_SRC_2]], 1.000000e+01
+; POW2-ONLY-NEXT:    [[TMP0:%.*]] = load <2 x float>, ptr [[GEP_SRC_0]], align 4
+; POW2-ONLY-NEXT:    [[TMP1:%.*]] = fadd <2 x float> [[TMP0]], <float 1.000000e+01, float 1.000000e+01>
+; POW2-ONLY-NEXT:    store <2 x float> [[TMP1]], ptr [[DST:%.*]], align 4
+; POW2-ONLY-NEXT:    [[DST_2:%.*]] = getelementptr float, ptr [[DST]], i32 2
+; POW2-ONLY-NEXT:    store float [[FADD_2]], ptr [[DST_2]], align 4
+; POW2-ONLY-NEXT:    ret void
 ;
 entry:
   %gep.src.0 = getelementptr inbounds float, ptr %src, i32 0
@@ -179,18 +263,28 @@ entry:
 }
 
 define void @phi_store3(ptr %dst) {
-; CHECK-LABEL: @phi_store3(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br label [[EXIT:%.*]]
-; CHECK:       invoke.cont8.loopexit:
-; CHECK-NEXT:    br label [[EXIT]]
-; CHECK:       exit:
-; CHECK-NEXT:    [[P_2:%.*]] = phi i32 [ 3, [[ENTRY:%.*]] ], [ 0, [[INVOKE_CONT8_LOOPEXIT:%.*]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = phi <2 x i32> [ <i32 1, i32 2>, [[ENTRY]] ], [ poison, [[INVOKE_CONT8_LOOPEXIT]] ]
-; CHECK-NEXT:    [[DST_2:%.*]] = getelementptr i32, ptr [[DST:%.*]], i32 2
-; CHECK-NEXT:    store <2 x i32> [[TMP0]], ptr [[DST]], align 4
-; CHECK-NEXT:    store i32 [[P_2]], ptr [[DST_2]], align 4
-; CHECK-NEXT:    ret void
+; NON-POW2-LABEL: @phi_store3(
+; NON-POW2-NEXT:  entry:
+; NON-POW2-NEXT:    br label [[EXIT:%.*]]
+; NON-POW2:       invoke.cont8.loopexit:
+; NON-POW2-NEXT:    br label [[EXIT]]
+; NON-POW2:       exit:
+; NON-POW2-NEXT:    [[TMP0:%.*]] = phi <3 x i32> [ <i32 1, i32 2, i32 3>, [[ENTRY:%.*]] ], [ poison, [[INVOKE_CONT8_LOOPEXIT:%.*]] ]
+; NON-POW2-NEXT:    store <3 x i32> [[TMP0]], ptr [[DST:%.*]], align 4
+; NON-POW2-NEXT:    ret void
+;
+; POW2-ONLY-LABEL: @phi_store3(
+; POW2-ONLY-NEXT:  entry:
+; POW2-ONLY-NEXT:    br label [[EXIT:%.*]]
+; POW2-ONLY:       invoke.cont8.loopexit:
+; POW2-ONLY-NEXT:    br label [[EXIT]]
+; POW2-ONLY:       exit:
+; POW2-ONLY-NEXT:    [[P_2:%.*]] = phi i32 [ 3, [[ENTRY:%.*]] ], [ 0, [[INVOKE_CONT8_LOOPEXIT:%.*]] ]
+; POW2-ONLY-NEXT:    [[TMP0:%.*]] = phi <2 x i32> [ <i32 1, i32 2>, [[ENTRY]] ], [ poison, [[INVOKE_CONT8_LOOPEXIT]] ]
+; POW2-ONLY-NEXT:    [[DST_2:%.*]] = getelementptr i32, ptr [[DST:%.*]], i32 2
+; POW2-ONLY-NEXT:    store <2 x i32> [[TMP0]], ptr [[DST]], align 4
+; POW2-ONLY-NEXT:    store i32 [[P_2]], ptr [[DST_2]], align 4
+; POW2-ONLY-NEXT:    ret void
 ;
 entry:
   br label %exit
@@ -213,13 +307,18 @@ exit:
 }
 
 define void @store_try_reorder(ptr %dst) {
-; CHECK-LABEL: @store_try_reorder(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[ADD:%.*]] = add i32 0, 0
-; CHECK-NEXT:    store i32 [[ADD]], ptr [[DST:%.*]], align 4
-; CHECK-NEXT:    [[ARRAYIDX_I1887:%.*]] = getelementptr i32, ptr [[DST]], i64 1
-; CHECK-NEXT:    store <2 x i32> zeroinitializer, ptr [[ARRAYIDX_I1887]], align 4
-; CHECK-NEXT:    ret void
+; NON-POW2-LABEL: @store_try_reorder(
+; NON-POW2-NEXT:  entry:
+; NON-POW2-NEXT:    store <3 x i32> zeroinitializer, ptr [[DST:%.*]], align 4
+; NON-POW2-NEXT:    ret void
+;
+; POW2-ONLY-LABEL: @store_try_reorder(
+; POW2-ONLY-NEXT:  entry:
+; POW2-ONLY-NEXT:    [[ADD:%.*]] = add i32 0, 0
+; POW2-ONLY-NEXT:    store i32 [[ADD]], ptr [[DST:%.*]], align 4
+; POW2-ONLY-NEXT:    [[ARRAYIDX_I1887:%.*]] = getelementptr i32, ptr [[DST]], i64 1
+; POW2-ONLY-NEXT:    store <2 x i32> zeroinitializer, ptr [[ARRAYIDX_I1887]], align 4
+; POW2-ONLY-NEXT:    ret void
 ;
 entry:
   %add = add i32 0, 0
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-calls.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-calls.ll
index 2cb84eeb7fc8..67746f2cbf5d 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-calls.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-calls.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -passes=slp-vectorizer -mtriple=arm64-apple-ios -S %s | FileCheck %s
+; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2 -mtriple=arm64-apple-ios -S %s | FileCheck --check-prefixes=CHECK %s
+; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2=false -mtriple=arm64-apple-ios -S %s | FileCheck --check-prefixes=CHECK %s
 
 define void @vec3_vectorize_call(ptr %Colour, float %0) {
 ; CHECK-LABEL: @vec3_vectorize_call(
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-reorder-reshuffle.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-reorder-reshuffle.ll
index e4925961f284..47d918eabdfe 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-reorder-reshuffle.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-reorder-reshuffle.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt -passes=slp-vectorizer -mtriple=arm64-apple-ios -S %s | FileCheck %s
+; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2 -mtriple=arm64-apple-ios -S %s | FileCheck --check-prefixes=CHECK,NON-POW2 %s
+; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2=false -mtriple=arm64-apple-ios -S %s | FileCheck --check-prefixes=CHECK,POW2-ONLY %s
 
 %struct.zot = type { i32, i32, i32 }
 
@@ -139,21 +140,34 @@ if.end668:                                        ; preds = %if.then665, %entry
 }
 
 define void @gather_2(ptr %mat1, float %0, float %1) {
-; CHECK-LABEL: define void @gather_2(
-; CHECK-SAME: ptr [[MAT1:%.*]], float [[TMP0:%.*]], float [[TMP1:%.*]]) {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i32 0
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x float> <float 0.000000e+00, float poison>, float [[TMP1]], i32 1
-; CHECK-NEXT:    [[TMP5:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP3]], <2 x float> [[TMP6]], <2 x float> zeroinitializer)
-; CHECK-NEXT:    [[TMP4:%.*]] = call float @llvm.fmuladd.f32(float [[TMP0]], float [[TMP1]], float 0.000000e+00)
-; CHECK-NEXT:    [[TMP7:%.*]] = fmul float [[TMP4]], 0.000000e+00
-; CHECK-NEXT:    [[ARRAYIDX163:%.*]] = getelementptr [4 x [4 x float]], ptr [[MAT1]], i64 0, i64 1
-; CHECK-NEXT:    [[ARRAYIDX5_I_I_I280:%.*]] = getelementptr [4 x [4 x float]], ptr [[MAT1]], i64 0, i64 1, i64 2
-; CHECK-NEXT:    [[TMP8:%.*]] = fmul <2 x float> [[TMP5]], zeroinitializer
-; CHECK-NEXT:    store <2 x float> [[TMP8]], ptr [[ARRAYIDX163]], align 4
-; CHECK-NEXT:    store float [[TMP7]], ptr [[ARRAYIDX5_I_I_I280]], align 4
-; CHECK-NEXT:    ret void
+; NON-POW2-LABEL: define void @gather_2(
+; NON-POW2-SAME: ptr [[MAT1:%.*]], float [[TMP0:%.*]], float [[TMP1:%.*]]) {
+; NON-POW2-NEXT:  entry:
+; NON-POW2-NEXT:    [[TMP2:%.*]] = insertelement <3 x float> poison, float [[TMP0]], i32 0
+; NON-POW2-NEXT:    [[TMP3:%.*]] = shufflevector <3 x float> [[TMP2]], <3 x float> poison, <3 x i32> zeroinitializer
+; NON-POW2-NEXT:    [[TMP4:%.*]] = insertelement <3 x float> <float 0.000000e+00, float poison, float poison>, float [[TMP1]], i32 1
+; NON-POW2-NEXT:    [[TMP5:%.*]] = shufflevector <3 x float> [[TMP4]], <3 x float> poison, <3 x i32> <i32 0, i32 1, i32 1>
+; NON-POW2-NEXT:    [[TMP6:%.*]] = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> [[TMP3]], <3 x float> [[TMP5]], <3 x float> zeroinitializer)
+; NON-POW2-NEXT:    [[ARRAYIDX163:%.*]] = getelementptr [4 x [4 x float]], ptr [[MAT1]], i64 0, i64 1
+; NON-POW2-NEXT:    [[TMP7:%.*]] = fmul <3 x float> [[TMP6]], zeroinitializer
+; NON-POW2-NEXT:    store <3 x float> [[TMP7]], ptr [[ARRAYIDX163]], align 4
+; NON-POW2-NEXT:    ret void
+;
+; POW2-ONLY-LABEL: define void @gather_2(
+; POW2-ONLY-SAME: ptr [[MAT1:%.*]], float [[TMP0:%.*]], float [[TMP1:%.*]]) {
+; POW2-ONLY-NEXT:  entry:
+; POW2-ONLY-NEXT:    [[TMP2:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i32 0
+; POW2-ONLY-NEXT:    [[TMP3:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <2 x i32> zeroinitializer
+; POW2-ONLY-NEXT:    [[TMP4:%.*]] = insertelement <2 x float> <float 0.000000e+00, float poison>, float [[TMP1]], i32 1
+; POW2-ONLY-NEXT:    [[TMP5:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP3]], <2 x float> [[TMP4]], <2 x float> zeroinitializer)
+; POW2-ONLY-NEXT:    [[TMP6:%.*]] = call float @llvm.fmuladd.f32(float [[TMP0]], float [[TMP1]], float 0.000000e+00)
+; POW2-ONLY-NEXT:    [[TMP7:%.*]] = fmul float [[TMP6]], 0.000000e+00
+; POW2-ONLY-NEXT:    [[ARRAYIDX163:%.*]] = getelementptr [4 x [4 x float]], ptr [[MAT1]], i64 0, i64 1
+; POW2-ONLY-NEXT:    [[ARRAYIDX5_I_I_I280:%.*]] = getelementptr [4 x [4 x float]], ptr [[MAT1]], i64 0, i64 1, i64 2
+; POW2-ONLY-NEXT:    [[TMP8:%.*]] = fmul <2 x float> [[TMP5]], zeroinitializer
+; POW2-ONLY-NEXT:    store <2 x float> [[TMP8]], ptr [[ARRAYIDX163]], align 4
+; POW2-ONLY-NEXT:    store float [[TMP7]], ptr [[ARRAYIDX5_I_I_I280]], align 4
+; POW2-ONLY-NEXT:    ret void
 ;
 entry:
   %2 = call float @llvm.fmuladd.f32(float %0, float 0.000000e+00, float 0.000000e+00)
@@ -172,31 +186,47 @@ entry:
 }
 
 define i32 @reorder_indices_1(float %0) {
-; CHECK-LABEL: define i32 @reorder_indices_1(
-; CHECK-SAME: float [[TMP0:%.*]]) {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[NOR1:%.*]] = alloca [0 x [3 x float]], i32 0, align 4
-; CHECK-NEXT:    [[ARRAYIDX2_I265:%.*]] = getelementptr float, ptr [[NOR1]], i64 2
-; CHECK-NEXT:    [[TMP1:%.*]] = load float, ptr [[ARRAYIDX2_I265]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x float>, ptr [[NOR1]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x float> [[TMP2]], i32 0
-; CHECK-NEXT:    [[TMP4:%.*]] = fneg float [[TMP3]]
-; CHECK-NEXT:    [[NEG11_I:%.*]] = fmul float [[TMP4]], [[TMP0]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call float @llvm.fmuladd.f32(float [[TMP1]], float 0.000000e+00, float [[NEG11_I]])
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <2 x i32> <i32 1, i32 poison>
-; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x float> [[TMP6]], float [[TMP1]], i32 1
-; CHECK-NEXT:    [[TMP8:%.*]] = fneg <2 x float> [[TMP7]]
-; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i32 0
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP11:%.*]] = fmul <2 x float> [[TMP8]], [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP2]], <2 x float> zeroinitializer, <2 x float> [[TMP11]])
-; CHECK-NEXT:    [[TMP13:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP10]], <2 x float> [[TMP12]], <2 x float> zeroinitializer)
-; CHECK-NEXT:    [[TMP14:%.*]] = call float @llvm.fmuladd.f32(float [[TMP0]], float [[TMP5]], float 0.000000e+00)
-; CHECK-NEXT:    [[TMP15:%.*]] = fmul <2 x float> [[TMP13]], zeroinitializer
-; CHECK-NEXT:    [[MUL6_I_I_I:%.*]] = fmul float [[TMP14]], 0.000000e+00
-; CHECK-NEXT:    store <2 x float> [[TMP15]], ptr [[NOR1]], align 4
-; CHECK-NEXT:    store float [[MUL6_I_I_I]], ptr [[ARRAYIDX2_I265]], align 4
-; CHECK-NEXT:    ret i32 0
+; NON-POW2-LABEL: define i32 @reorder_indices_1(
+; NON-POW2-SAME: float [[TMP0:%.*]]) {
+; NON-POW2-NEXT:  entry:
+; NON-POW2-NEXT:    [[NOR1:%.*]] = alloca [0 x [3 x float]], i32 0, align 4
+; NON-POW2-NEXT:    [[TMP1:%.*]] = load <3 x float>, ptr [[NOR1]], align 4
+; NON-POW2-NEXT:    [[TMP2:%.*]] = shufflevector <3 x float> [[TMP1]], <3 x float> poison, <3 x i32> <i32 1, i32 2, i32 0>
+; NON-POW2-NEXT:    [[TMP3:%.*]] = fneg <3 x float> [[TMP2]]
+; NON-POW2-NEXT:    [[TMP4:%.*]] = insertelement <3 x float> poison, float [[TMP0]], i32 0
+; NON-POW2-NEXT:    [[TMP5:%.*]] = shufflevector <3 x float> [[TMP4]], <3 x float> poison, <3 x i32> zeroinitializer
+; NON-POW2-NEXT:    [[TMP6:%.*]] = fmul <3 x float> [[TMP3]], [[TMP5]]
+; NON-POW2-NEXT:    [[TMP7:%.*]] = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> [[TMP1]], <3 x float> zeroinitializer, <3 x float> [[TMP6]])
+; NON-POW2-NEXT:    [[TMP8:%.*]] = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> [[TMP5]], <3 x float> [[TMP7]], <3 x float> zeroinitializer)
+; NON-POW2-NEXT:    [[TMP9:%.*]] = fmul <3 x float> [[TMP8]], zeroinitializer
+; NON-POW2-NEXT:    store <3 x float> [[TMP9]], ptr [[NOR1]], align 4
+; NON-POW2-NEXT:    ret i32 0
+;
+; POW2-ONLY-LABEL: define i32 @reorder_indices_1(
+; POW2-ONLY-SAME: float [[TMP0:%.*]]) {
+; POW2-ONLY-NEXT:  entry:
+; POW2-ONLY-NEXT:    [[NOR1:%.*]] = alloca [0 x [3 x float]], i32 0, align 4
+; POW2-ONLY-NEXT:    [[ARRAYIDX2_I265:%.*]] = getelementptr float, ptr [[NOR1]], i64 2
+; POW2-ONLY-NEXT:    [[TMP1:%.*]] = load float, ptr [[ARRAYIDX2_I265]], align 4
+; POW2-ONLY-NEXT:    [[TMP2:%.*]] = load <2 x float>, ptr [[NOR1]], align 4
+; POW2-ONLY-NEXT:    [[TMP3:%.*]] = extractelement <2 x float> [[TMP2]], i32 0
+; POW2-ONLY-NEXT:    [[TMP4:%.*]] = fneg float [[TMP3]]
+; POW2-ONLY-NEXT:    [[NEG11_I:%.*]] = fmul float [[TMP4]], [[TMP0]]
+; POW2-ONLY-NEXT:    [[TMP5:%.*]] = call float @llvm.fmuladd.f32(float [[TMP1]], float 0.000000e+00, float [[NEG11_I]])
+; POW2-ONLY-NEXT:    [[TMP6:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <2 x i32> <i32 1, i32 poison>
+; POW2-ONLY-NEXT:    [[TMP7:%.*]] = insertelement <2 x float> [[TMP6]], float [[TMP1]], i32 1
+; POW2-ONLY-NEXT:    [[TMP8:%.*]] = fneg <2 x float> [[TMP7]]
+; POW2-ONLY-NEXT:    [[TMP9:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i32 0
+; POW2-ONLY-NEXT:    [[TMP10:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> poison, <2 x i32> zeroinitializer
+; POW2-ONLY-NEXT:    [[TMP11:%.*]] = fmul <2 x float> [[TMP8]], [[TMP10]]
+; POW2-ONLY-NEXT:    [[TMP12:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP2]], <2 x float> zeroinitializer, <2 x float> [[TMP11]])
+; POW2-ONLY-NEXT:    [[TMP13:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP10]], <2 x float> [[TMP12]], <2 x float> zeroinitializer)
+; POW2-ONLY-NEXT:    [[TMP14:%.*]] = call float @llvm.fmuladd.f32(float [[TMP0]], float [[TMP5]], float 0.000000e+00)
+; POW2-ONLY-NEXT:    [[TMP15:%.*]] = fmul <2 x float> [[TMP13]], zeroinitializer
+; POW2-ONLY-NEXT:    [[MUL6_I_I_I:%.*]] = fmul float [[TMP14]], 0.000000e+00
+; POW2-ONLY-NEXT:    store <2 x float> [[TMP15]], ptr [[NOR1]], align 4
+; POW2-ONLY-NEXT:    store float [[MUL6_I_I_I]], ptr [[ARRAYIDX2_I265]], align 4
+; POW2-ONLY-NEXT:    ret i32 0
 ;
 entry:
   %nor1 = alloca [0 x [3 x float]], i32 0, align 4
@@ -227,19 +257,28 @@ entry:
 }
 
 define void @reorder_indices_2(ptr %spoint) {
-; CHECK-LABEL: define void @reorder_indices_2(
-; CHECK-SAME: ptr [[SPOINT:%.*]]) {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <3 x float> zeroinitializer, i64 0
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call float @llvm.fmuladd.f32(float [[TMP0]], float 0.000000e+00, float 0.000000e+00)
-; CHECK-NEXT:    [[MUL4_I461:%.*]] = fmul float [[TMP1]], 0.000000e+00
-; CHECK-NEXT:    [[DSCO:%.*]] = getelementptr float, ptr [[SPOINT]], i64 0
-; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> zeroinitializer, <2 x float> zeroinitializer, <2 x float> zeroinitializer)
-; CHECK-NEXT:    [[TMP3:%.*]] = fmul <2 x float> [[TMP2]], zeroinitializer
-; CHECK-NEXT:    store <2 x float> [[TMP3]], ptr [[DSCO]], align 4
-; CHECK-NEXT:    [[ARRAYIDX5_I476:%.*]] = getelementptr float, ptr [[SPOINT]], i64 2
-; CHECK-NEXT:    store float [[MUL4_I461]], ptr [[ARRAYIDX5_I476]], align 4
-; CHECK-NEXT:    ret void
+; NON-POW2-LABEL: define void @reorder_indices_2(
+; NON-POW2-SAME: ptr [[SPOINT:%.*]]) {
+; NON-POW2-NEXT:  entry:
+; NON-POW2-NEXT:    [[DSCO:%.*]] = getelementptr float, ptr [[SPOINT]], i64 0
+; NON-POW2-NEXT:    [[TMP0:%.*]] = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> zeroinitializer, <3 x float> zeroinitializer, <3 x float> zeroinitializer)
+; NON-POW2-NEXT:    [[TMP1:%.*]] = fmul <3 x float> [[TMP0]], zeroinitializer
+; NON-POW2-NEXT:    store <3 x float> [[TMP1]], ptr [[DSCO]], align 4
+; NON-POW2-NEXT:    ret void
+;
+; POW2-ONLY-LABEL: define void @reorder_indices_2(
+; POW2-ONLY-SAME: ptr [[SPOINT:%.*]]) {
+; POW2-ONLY-NEXT:  entry:
+; POW2-ONLY-NEXT:    [[TMP0:%.*]] = extractelement <3 x float> zeroinitializer, i64 0
+; POW2-ONLY-NEXT:    [[TMP1:%.*]] = tail call float @llvm.fmuladd.f32(float [[TMP0]], float 0.000000e+00, float 0.000000e+00)
+; POW2-ONLY-NEXT:    [[MUL4_I461:%.*]] = fmul float [[TMP1]], 0.000000e+00
+; POW2-ONLY-NEXT:    [[DSCO:%.*]] = getelementptr float, ptr [[SPOINT]], i64 0
+; POW2-ONLY-NEXT:    [[TMP2:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> zeroinitializer, <2 x float> zeroinitializer, <2 x float> zeroinitializer)
+; POW2-ONLY-NEXT:    [[TMP3:%.*]] = fmul <2 x float> [[TMP2]], zeroinitializer
+; POW2-ONLY-NEXT:    store <2 x float> [[TMP3]], ptr [[DSCO]], align 4
+; POW2-ONLY-NEXT:    [[ARRAYIDX5_I476:%.*]] = getelementptr float, ptr [[SPOINT]], i64 2
+; POW2-ONLY-NEXT:    store float [[MUL4_I461]], ptr [[ARRAYIDX5_I476]], align 4
+; POW2-ONLY-NEXT:    ret void
 ;
 entry:
   %0 = extractelement <3 x float> zeroinitializer, i64 1
@@ -291,19 +330,30 @@ entry:
 }
 
 define void @reuse_shuffle_indidces_1(ptr %col, float %0, float %1) {
-; CHECK-LABEL: define void @reuse_shuffle_indidces_1(
-; CHECK-SAME: ptr [[COL:%.*]], float [[TMP0:%.*]], float [[TMP1:%.*]]) {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x float> poison, float [[TMP1]], i32 0
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> [[TMP2]], float [[TMP0]], i32 1
-; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x float> [[TMP3]], zeroinitializer
-; CHECK-NEXT:    [[TMP5:%.*]] = fadd <2 x float> [[TMP4]], zeroinitializer
-; CHECK-NEXT:    store <2 x float> [[TMP5]], ptr [[COL]], align 4
-; CHECK-NEXT:    [[ARRAYIDX33:%.*]] = getelementptr float, ptr [[COL]], i64 2
-; CHECK-NEXT:    [[MUL38:%.*]] = fmul float [[TMP0]], 0.000000e+00
-; CHECK-NEXT:    [[TMP6:%.*]] = fadd float [[MUL38]], 0.000000e+00
-; CHECK-NEXT:    store float [[TMP6]], ptr [[ARRAYIDX33]], align 4
-; CHECK-NEXT:    ret void
+; NON-POW2-LABEL: define void @reuse_shuffle_indidces_1(
+; NON-POW2-SAME: ptr [[COL:%.*]], float [[TMP0:%.*]], float [[TMP1:%.*]]) {
+; NON-POW2-NEXT:  entry:
+; NON-POW2-NEXT:    [[TMP2:%.*]] = insertelement <3 x float> poison, float [[TMP1]], i32 0
+; NON-POW2-NEXT:    [[TMP3:%.*]] = insertelement <3 x float> [[TMP2]], float [[TMP0]], i32 1
+; NON-POW2-NEXT:    [[TMP4:%.*]] = shufflevector <3 x float> [[TMP3]], <3 x float> poison, <3 x i32> <i32 0, i32 1, i32 1>
+; NON-POW2-NEXT:    [[TMP5:%.*]] = fmul <3 x float> [[TMP4]], zeroinitializer
+; NON-POW2-NEXT:    [[TMP6:%.*]] = fadd <3 x float> [[TMP5]], zeroinitializer
+; NON-POW2-NEXT:    store <3 x float> [[TMP6]], ptr [[COL]], align 4
+; NON-POW2-NEXT:    ret void
+;
+; POW2-ONLY-LABEL: define void @reuse_shuffle_indidces_1(
+; POW2-ONLY-SAME: ptr [[COL:%.*]], float [[TMP0:%.*]], float [[TMP1:%.*]]) {
+; POW2-ONLY-NEXT:  entry:
+; POW2-ONLY-NEXT:    [[TMP2:%.*]] = insertelement <2 x float> poison, float [[TMP1]], i32 0
+; POW2-ONLY-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> [[TMP2]], float [[TMP0]], i32 1
+; POW2-ONLY-NEXT:    [[TMP4:%.*]] = fmul <2 x float> [[TMP3]], zeroinitializer
+; POW2-ONLY-NEXT:    [[TMP5:%.*]] = fadd <2 x float> [[TMP4]], zeroinitializer
+; POW2-ONLY-NEXT:    store <2 x float> [[TMP5]], ptr [[COL]], align 4
+; POW2-ONLY-NEXT:    [[ARRAYIDX33:%.*]] = getelementptr float, ptr [[COL]], i64 2
+; POW2-ONLY-NEXT:    [[MUL38:%.*]] = fmul float [[TMP0]], 0.000000e+00
+; POW2-ONLY-NEXT:    [[TMP6:%.*]] = fadd float [[MUL38]], 0.000000e+00
+; POW2-ONLY-NEXT:    store float [[TMP6]], ptr [[ARRAYIDX33]], align 4
+; POW2-ONLY-NEXT:    ret void
 ;
 entry:
   %mul24 = fmul float %1, 0.000000e+00
@@ -487,18 +537,24 @@ entry:
 }
 
 define void @vec3_extract(<3 x i16> %pixel.sroa.0.4.vec.insert606, ptr %call3.i536) {
-; CHECK-LABEL: define void @vec3_extract(
-; CHECK-SAME: <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606:%.*]], ptr [[CALL3_I536:%.*]]) {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[PIXEL_SROA_0_4_VEC_EXTRACT:%.*]] = extractelement <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606]], i64 2
-; CHECK-NEXT:    [[RED668:%.*]] = getelementptr i16, ptr [[CALL3_I536]], i64 2
-; CHECK-NEXT:    store i16 [[PIXEL_SROA_0_4_VEC_EXTRACT]], ptr [[RED668]], align 2
-; CHECK-NEXT:    [[PIXEL_SROA_0_2_VEC_EXTRACT:%.*]] = extractelement <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606]], i64 1
-; CHECK-NEXT:    [[GREEN670:%.*]] = getelementptr i16, ptr [[CALL3_I536]], i64 1
-; CHECK-NEXT:    store i16 [[PIXEL_SROA_0_2_VEC_EXTRACT]], ptr [[GREEN670]], align 2
-; CHECK-NEXT:    [[PIXEL_SROA_0_0_VEC_EXTRACT:%.*]] = extractelement <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606]], i64 0
-; CHECK-NEXT:    store i16 [[PIXEL_SROA_0_0_VEC_EXTRACT]], ptr [[CALL3_I536]], align 2
-; CHECK-NEXT:    ret void
+; NON-POW2-LABEL: define void @vec3_extract(
+; NON-POW2-SAME: <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606:%.*]], ptr [[CALL3_I536:%.*]]) {
+; NON-POW2-NEXT:  entry:
+; NON-POW2-NEXT:    store <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606]], ptr [[CALL3_I536]], align 2
+; NON-POW2-NEXT:    ret void
+;
+; POW2-ONLY-LABEL: define void @vec3_extract(
+; POW2-ONLY-SAME: <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606:%.*]], ptr [[CALL3_I536:%.*]]) {
+; POW2-ONLY-NEXT:  entry:
+; POW2-ONLY-NEXT:    [[PIXEL_SROA_0_4_VEC_EXTRACT:%.*]] = extractelement <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606]], i64 2
+; POW2-ONLY-NEXT:    [[RED668:%.*]] = getelementptr i16, ptr [[CALL3_I536]], i64 2
+; POW2-ONLY-NEXT:    store i16 [[PIXEL_SROA_0_4_VEC_EXTRACT]], ptr [[RED668]], align 2
+; POW2-ONLY-NEXT:    [[PIXEL_SROA_0_2_VEC_EXTRACT:%.*]] = extractelement <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606]], i64 1
+; POW2-ONLY-NEXT:    [[GREEN670:%.*]] = getelementptr i16, ptr [[CALL3_I536]], i64 1
+; POW2-ONLY-NEXT:    store i16 [[PIXEL_SROA_0_2_VEC_EXTRACT]], ptr [[GREEN670]], align 2
+; POW2-ONLY-NEXT:    [[PIXEL_SROA_0_0_VEC_EXTRACT:%.*]] = extractelement <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606]], i64 0
+; POW2-ONLY-NEXT:    store i16 [[PIXEL_SROA_0_0_VEC_EXTRACT]], ptr [[CALL3_I536]], align 2
+; POW2-ONLY-NEXT:    ret void
 ;
 entry:
   %pixel.sroa.0.4.vec.extract = extractelement <3 x i16> %pixel.sroa.0.4.vec.insert606, i64 2
@@ -512,4 +568,56 @@ entry:
   ret void
 }
 
+define void @can_reorder_vec3_op_with_padding(ptr %A, <3 x float> %in) {
+; NON-POW2-LABEL: define void @can_reorder_vec3_op_with_padding(
+; NON-POW2-SAME: ptr [[A:%.*]], <3 x float> [[IN:%.*]]) {
+; NON-POW2-NEXT:  entry:
+; NON-POW2-NEXT:    [[TMP0:%.*]] = shufflevector <3 x float> [[IN]], <3 x float> poison, <3 x i32> <i32 1, i32 2, i32 0>
+; NON-POW2-NEXT:    [[TMP1:%.*]] = fsub <3 x float> [[TMP0]], [[TMP0]]
+; NON-POW2-NEXT:    [[TMP2:%.*]] = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> [[TMP1]], <3 x float> <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>, <3 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>)
+; NON-POW2-NEXT:    [[TMP3:%.*]] = fmul <3 x float> [[TMP2]], <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>
+; NON-POW2-NEXT:    store <3 x float> [[TMP3]], ptr [[A]], align 4
+; NON-POW2-NEXT:    ret void
+;
+; POW2-ONLY-LABEL: define void @can_reorder_vec3_op_with_padding(
+; POW2-ONLY-SAME: ptr [[A:%.*]], <3 x float> [[IN:%.*]]) {
+; POW2-ONLY-NEXT:  entry:
+; POW2-ONLY-NEXT:    [[ARRAYIDX42_I:%.*]] = getelementptr float, ptr [[A]], i64 2
+; POW2-ONLY-NEXT:    [[TMP0:%.*]] = extractelement <3 x float> [[IN]], i64 0
+; POW2-ONLY-NEXT:    [[SUB_I362:%.*]] = fsub float [[TMP0]], [[TMP0]]
+; POW2-ONLY-NEXT:    [[TMP1:%.*]] = call float @llvm.fmuladd.f32(float [[SUB_I362]], float 2.000000e+00, float 3.000000e+00)
+; POW2-ONLY-NEXT:    [[MUL6_I_I_I_I:%.*]] = fmul float [[TMP1]], 3.000000e+00
+; POW2-ONLY-NEXT:    [[TMP2:%.*]] = shufflevector <3 x float> [[IN]], <3 x float> poison, <2 x i32> <i32 1, i32 2>
+; POW2-ONLY-NEXT:    [[TMP3:%.*]] = fsub <2 x float> [[TMP2]], [[TMP2]]
+; POW2-ONLY-NEXT:    [[TMP4:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP3]], <2 x float> <float 2.000000e+00, float 2.000000e+00>, <2 x float> <float 3.000000e+00, float 3.000000e+00>)
+; POW2-ONLY-NEXT:    [[TMP5:%.*]] = fmul <2 x float> [[TMP4]], <float 3.000000e+00, float 3.000000e+00>
+; POW2-ONLY-NEXT:    store <2 x float> [[TMP5]], ptr [[A]], align 4
+; POW2-ONLY-NEXT:    store float [[MUL6_I_I_I_I]], ptr [[ARRAYIDX42_I]], align 4
+; POW2-ONLY-NEXT:    ret void
+;
+entry:
+  %arrayidx42.i = getelementptr float, ptr %A, i64 2
+  %arrayidx35.i = getelementptr float, ptr %A, i64 1
+  %0 = extractelement <3 x float> %in, i64 0
+  %1 = extractelement <3 x float> %in, i64 0
+  %sub.i362 = fsub float %0, %1
+  %2 = extractelement <3 x float> %in, i64 1
+  %3 = extractelement <3 x float> %in, i64 1
+  %sub5.i = fsub float %2, %3
+  %4 = extractelement <3 x float> %in, i64 2
+  %5 = extractelement <3 x float> %in, i64 2
+  %sub9.i = fsub float %4, %5
+  %6 = call float @llvm.fmuladd.f32(float %sub5.i, float 2.000000e+00, float 3.000000e+00)
+  %7 = call float @llvm.fmuladd.f32(float %sub9.i, float 2.000000e+00, float 3.000000e+00)
+  %8 = call float @llvm.fmuladd.f32(float %sub.i362, float 2.000000e+00, float 3.000000e+00)
+  %mul.i.i.i.i373 = fmul float %6, 3.000000e+00
+  %mul3.i.i.i.i = fmul float %7, 3.000000e+00
+  %mul6.i.i.i.i = fmul float %8, 3.000000e+00
+  store float %mul.i.i.i.i373, ptr %A, align 4
+  store float %mul3.i.i.i.i, ptr %arrayidx35.i, align 4
+  store float %mul6.i.i.i.i, ptr %arrayidx42.i, align 4
+  ret void
+}
+
 declare float @llvm.fmuladd.f32(float, float, float)
+declare double @llvm.fmuladd.f64(double, double, double)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/odd_store.ll b/llvm/test/Transforms/SLPVectorizer/X86/odd_store.ll
index 4795ac655920..5f2c42d5c2de 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/odd_store.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/odd_store.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -passes=slp-vectorizer,dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer,dce -slp-vectorize-non-power-of-2 -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck --check-prefixes=CHECK,NON-POW2 %s
+; RUN: opt < %s -passes=slp-vectorizer,dce -slp-vectorize-non-power-of-2=false -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck --check-prefixes=CHECK,POW2-ONLY %s
 
 ;int foo(char * restrict A, ptr restrict B, float T) {
 ;  A[0] = (T * B[10] + 4.0);
@@ -8,31 +9,43 @@
 ;}
 
 define i32 @foo(ptr noalias nocapture %A, ptr noalias nocapture %B, float %T) {
-; CHECK-LABEL: @foo(
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 10
-; CHECK-NEXT:    [[TMP2:%.*]] = load float, ptr [[TMP1]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = fmul float [[TMP2]], [[T:%.*]]
-; CHECK-NEXT:    [[TMP4:%.*]] = fpext float [[TMP3]] to double
-; CHECK-NEXT:    [[TMP5:%.*]] = fadd double [[TMP4]], 4.000000e+00
-; CHECK-NEXT:    [[TMP6:%.*]] = fptosi double [[TMP5]] to i8
-; CHECK-NEXT:    store i8 [[TMP6]], ptr [[A:%.*]], align 1
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[B]], i64 11
-; CHECK-NEXT:    [[TMP8:%.*]] = load float, ptr [[TMP7]], align 4
-; CHECK-NEXT:    [[TMP9:%.*]] = fmul float [[TMP8]], [[T]]
-; CHECK-NEXT:    [[TMP10:%.*]] = fpext float [[TMP9]] to double
-; CHECK-NEXT:    [[TMP11:%.*]] = fadd double [[TMP10]], 5.000000e+00
-; CHECK-NEXT:    [[TMP12:%.*]] = fptosi double [[TMP11]] to i8
-; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 1
-; CHECK-NEXT:    store i8 [[TMP12]], ptr [[TMP13]], align 1
-; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, ptr [[B]], i64 12
-; CHECK-NEXT:    [[TMP15:%.*]] = load float, ptr [[TMP14]], align 4
-; CHECK-NEXT:    [[TMP16:%.*]] = fmul float [[TMP15]], [[T]]
-; CHECK-NEXT:    [[TMP17:%.*]] = fpext float [[TMP16]] to double
-; CHECK-NEXT:    [[TMP18:%.*]] = fadd double [[TMP17]], 6.000000e+00
-; CHECK-NEXT:    [[TMP19:%.*]] = fptosi double [[TMP18]] to i8
-; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 2
-; CHECK-NEXT:    store i8 [[TMP19]], ptr [[TMP20]], align 1
-; CHECK-NEXT:    ret i32 undef
+; NON-POW2-LABEL: @foo(
+; NON-POW2-NEXT:    [[TMP1:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 10
+; NON-POW2-NEXT:    [[TMP2:%.*]] = load <3 x float>, ptr [[TMP1]], align 4
+; NON-POW2-NEXT:    [[TMP3:%.*]] = insertelement <3 x float> poison, float [[T:%.*]], i32 0
+; NON-POW2-NEXT:    [[TMP4:%.*]] = shufflevector <3 x float> [[TMP3]], <3 x float> poison, <3 x i32> zeroinitializer
+; NON-POW2-NEXT:    [[TMP5:%.*]] = fmul <3 x float> [[TMP2]], [[TMP4]]
+; NON-POW2-NEXT:    [[TMP6:%.*]] = fpext <3 x float> [[TMP5]] to <3 x double>
+; NON-POW2-NEXT:    [[TMP7:%.*]] = fadd <3 x double> [[TMP6]], <double 4.000000e+00, double 5.000000e+00, double 6.000000e+00>
+; NON-POW2-NEXT:    [[TMP8:%.*]] = fptosi <3 x double> [[TMP7]] to <3 x i8>
+; NON-POW2-NEXT:    store <3 x i8> [[TMP8]], ptr [[A:%.*]], align 1
+; NON-POW2-NEXT:    ret i32 undef
+;
+; POW2-ONLY-LABEL: @foo(
+; POW2-ONLY-NEXT:    [[TMP1:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 10
+; POW2-ONLY-NEXT:    [[TMP2:%.*]] = load float, ptr [[TMP1]], align 4
+; POW2-ONLY-NEXT:    [[TMP3:%.*]] = fmul float [[TMP2]], [[T:%.*]]
+; POW2-ONLY-NEXT:    [[TMP4:%.*]] = fpext float [[TMP3]] to double
+; POW2-ONLY-NEXT:    [[TMP5:%.*]] = fadd double [[TMP4]], 4.000000e+00
+; POW2-ONLY-NEXT:    [[TMP6:%.*]] = fptosi double [[TMP5]] to i8
+; POW2-ONLY-NEXT:    store i8 [[TMP6]], ptr [[A:%.*]], align 1
+; POW2-ONLY-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[B]], i64 11
+; POW2-ONLY-NEXT:    [[TMP8:%.*]] = load float, ptr [[TMP7]], align 4
+; POW2-ONLY-NEXT:    [[TMP9:%.*]] = fmul float [[TMP8]], [[T]]
+; POW2-ONLY-NEXT:    [[TMP10:%.*]] = fpext float [[TMP9]] to double
+; POW2-ONLY-NEXT:    [[TMP11:%.*]] = fadd double [[TMP10]], 5.000000e+00
+; POW2-ONLY-NEXT:    [[TMP12:%.*]] = fptosi double [[TMP11]] to i8
+; POW2-ONLY-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 1
+; POW2-ONLY-NEXT:    store i8 [[TMP12]], ptr [[TMP13]], align 1
+; POW2-ONLY-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, ptr [[B]], i64 12
+; POW2-ONLY-NEXT:    [[TMP15:%.*]] = load float, ptr [[TMP14]], align 4
+; POW2-ONLY-NEXT:    [[TMP16:%.*]] = fmul float [[TMP15]], [[T]]
+; POW2-ONLY-NEXT:    [[TMP17:%.*]] = fpext float [[TMP16]] to double
+; POW2-ONLY-NEXT:    [[TMP18:%.*]] = fadd double [[TMP17]], 6.000000e+00
+; POW2-ONLY-NEXT:    [[TMP19:%.*]] = fptosi double [[TMP18]] to i8
+; POW2-ONLY-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 2
+; POW2-ONLY-NEXT:    store i8 [[TMP19]], ptr [[TMP20]], align 1
+; POW2-ONLY-NEXT:    ret i32 undef
 ;
   %1 = getelementptr inbounds float, ptr %B, i64 10
   %2 = load float, ptr %1, align 4
@@ -91,13 +104,18 @@ define void @test_v4f32_v2f32_splat_store(<4 x float> %f, ptr %p){
 }
 
 define void @test_v4f32_v3f32_store(<4 x float> %f, ptr %p){
-; CHECK-LABEL: @test_v4f32_v3f32_store(
-; CHECK-NEXT:    [[X2:%.*]] = extractelement <4 x float> [[F:%.*]], i64 2
-; CHECK-NEXT:    [[P2:%.*]] = getelementptr inbounds float, ptr [[P:%.*]], i64 2
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[F]], <4 x float> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT:    store <2 x float> [[TMP1]], ptr [[P]], align 4
-; CHECK-NEXT:    store float [[X2]], ptr [[P2]], align 4
-; CHECK-NEXT:    ret void
+; NON-POW2-LABEL: @test_v4f32_v3f32_store(
+; NON-POW2-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[F:%.*]], <4 x float> poison, <3 x i32> <i32 0, i32 1, i32 2>
+; NON-POW2-NEXT:    store <3 x float> [[TMP1]], ptr [[P:%.*]], align 4
+; NON-POW2-NEXT:    ret void
+;
+; POW2-ONLY-LABEL: @test_v4f32_v3f32_store(
+; POW2-ONLY-NEXT:    [[X2:%.*]] = extractelement <4 x float> [[F:%.*]], i64 2
+; POW2-ONLY-NEXT:    [[P2:%.*]] = getelementptr inbounds float, ptr [[P:%.*]], i64 2
+; POW2-ONLY-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[F]], <4 x float> poison, <2 x i32> <i32 0, i32 1>
+; POW2-ONLY-NEXT:    store <2 x float> [[TMP1]], ptr [[P]], align 4
+; POW2-ONLY-NEXT:    store float [[X2]], ptr [[P2]], align 4
+; POW2-ONLY-NEXT:    ret void
 ;
   %x0 = extractelement <4 x float> %f, i64 0
   %x1 = extractelement <4 x float> %f, i64 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vec3-base.ll b/llvm/test/Transforms/SLPVectorizer/X86/vec3-base.ll
index 6560fc6a1452..96d4b84e0369 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/vec3-base.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/vec3-base.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -passes=slp-vectorizer -mtriple=x86_64-apple-macosx -S %s | FileCheck %s
+; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2 -mtriple=x86_64-apple-macosx -S %s | FileCheck --check-prefixes=CHECK,NON-POW2 %s
+; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2=false -mtriple=x86_64-apple-macosx -S %s | FileCheck --check-prefixes=CHECK,POW2-ONLY %s
 
 define void @v3_load_i32_mul_by_constant_store(ptr %src, ptr %dst) {
 ; CHECK-LABEL: @v3_load_i32_mul_by_constant_store(
@@ -161,18 +162,26 @@ entry:
 }
 
 define void @v3_load_f32_fadd_fadd_by_constant_store(ptr %src, ptr %dst) {
-; CHECK-LABEL: @v3_load_f32_fadd_fadd_by_constant_store(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[GEP_SRC_0:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i32 0
-; CHECK-NEXT:    [[GEP_SRC_2:%.*]] = getelementptr inbounds float, ptr [[SRC]], i32 2
-; CHECK-NEXT:    [[L_SRC_2:%.*]] = load float, ptr [[GEP_SRC_2]], align 4
-; CHECK-NEXT:    [[FADD_2:%.*]] = fadd float [[L_SRC_2]], 1.000000e+01
-; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x float>, ptr [[GEP_SRC_0]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = fadd <2 x float> [[TMP0]], <float 1.000000e+01, float 1.000000e+01>
-; CHECK-NEXT:    store <2 x float> [[TMP1]], ptr [[DST:%.*]], align 4
-; CHECK-NEXT:    [[DST_2:%.*]] = getelementptr float, ptr [[DST]], i32 2
-; CHECK-NEXT:    store float [[FADD_2]], ptr [[DST_2]], align 4
-; CHECK-NEXT:    ret void
+; NON-POW2-LABEL: @v3_load_f32_fadd_fadd_by_constant_store(
+; NON-POW2-NEXT:  entry:
+; NON-POW2-NEXT:    [[GEP_SRC_0:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i32 0
+; NON-POW2-NEXT:    [[TMP0:%.*]] = load <3 x float>, ptr [[GEP_SRC_0]], align 4
+; NON-POW2-NEXT:    [[TMP1:%.*]] = fadd <3 x float> [[TMP0]], <float 1.000000e+01, float 1.000000e+01, float 1.000000e+01>
+; NON-POW2-NEXT:    store <3 x float> [[TMP1]], ptr [[DST:%.*]], align 4
+; NON-POW2-NEXT:    ret void
+;
+; POW2-ONLY-LABEL: @v3_load_f32_fadd_fadd_by_constant_store(
+; POW2-ONLY-NEXT:  entry:
+; POW2-ONLY-NEXT:    [[GEP_SRC_0:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i32 0
+; POW2-ONLY-NEXT:    [[GEP_SRC_2:%.*]] = getelementptr inbounds float, ptr [[SRC]], i32 2
+; POW2-ONLY-NEXT:    [[L_SRC_2:%.*]] = load float, ptr [[GEP_SRC_2]], align 4
+; POW2-ONLY-NEXT:    [[FADD_2:%.*]] = fadd float [[L_SRC_2]], 1.000000e+01
+; POW2-ONLY-NEXT:    [[TMP0:%.*]] = load <2 x float>, ptr [[GEP_SRC_0]], align 4
+; POW2-ONLY-NEXT:    [[TMP1:%.*]] = fadd <2 x float> [[TMP0]], <float 1.000000e+01, float 1.000000e+01>
+; POW2-ONLY-NEXT:    store <2 x float> [[TMP1]], ptr [[DST:%.*]], align 4
+; POW2-ONLY-NEXT:    [[DST_2:%.*]] = getelementptr float, ptr [[DST]], i32 2
+; POW2-ONLY-NEXT:    store float [[FADD_2]], ptr [[DST_2]], align 4
+; POW2-ONLY-NEXT:    ret void
 ;
 entry:
   %gep.src.0 = getelementptr inbounds float, ptr %src, i32 0
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vec3-calls.ll b/llvm/test/Transforms/SLPVectorizer/X86/vec3-calls.ll
index 71b9315839ec..243087c6d8d9 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/vec3-calls.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/vec3-calls.ll
@@ -1,16 +1,29 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -passes=slp-vectorizer -mtriple=x86_64-apple-macosx -S %s | FileCheck %s
+; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2 -mtriple=x86_64-apple-macosx -S %s | FileCheck --check-prefixes=CHECK,NON-POW2 %s
+; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2=false -mtriple=x86_64-apple-macosx -S %s | FileCheck --check-prefixes=CHECK,POW2-ONLY %s
 
 define void @vec3_vectorize_call(ptr %Colour, float %0) {
-; CHECK-LABEL: @vec3_vectorize_call(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x float>, ptr [[COLOUR:%.*]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP1]], <2 x float> zeroinitializer, <2 x float> zeroinitializer)
-; CHECK-NEXT:    store <2 x float> [[TMP2]], ptr [[COLOUR]], align 4
-; CHECK-NEXT:    [[ARRAYIDX99_I1:%.*]] = getelementptr float, ptr [[COLOUR]], i64 2
-; CHECK-NEXT:    [[TMP3:%.*]] = call float @llvm.fmuladd.f32(float [[TMP0:%.*]], float 0.000000e+00, float 0.000000e+00)
-; CHECK-NEXT:    store float [[TMP3]], ptr [[ARRAYIDX99_I1]], align 4
-; CHECK-NEXT:    ret void
+; NON-POW2-LABEL: @vec3_vectorize_call(
+; NON-POW2-NEXT:  entry:
+; NON-POW2-NEXT:    [[TMP1:%.*]] = load float, ptr [[COLOUR:%.*]], align 4
+; NON-POW2-NEXT:    [[ARRAYIDX91_I:%.*]] = getelementptr float, ptr [[COLOUR]], i64 1
+; NON-POW2-NEXT:    [[TMP2:%.*]] = load float, ptr [[ARRAYIDX91_I]], align 4
+; NON-POW2-NEXT:    [[TMP3:%.*]] = insertelement <3 x float> poison, float [[TMP0:%.*]], i32 2
+; NON-POW2-NEXT:    [[TMP4:%.*]] = insertelement <3 x float> [[TMP3]], float [[TMP1]], i32 0
+; NON-POW2-NEXT:    [[TMP5:%.*]] = insertelement <3 x float> [[TMP4]], float [[TMP2]], i32 1
+; NON-POW2-NEXT:    [[TMP6:%.*]] = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> [[TMP5]], <3 x float> zeroinitializer, <3 x float> zeroinitializer)
+; NON-POW2-NEXT:    store <3 x float> [[TMP6]], ptr [[COLOUR]], align 4
+; NON-POW2-NEXT:    ret void
+;
+; POW2-ONLY-LABEL: @vec3_vectorize_call(
+; POW2-ONLY-NEXT:  entry:
+; POW2-ONLY-NEXT:    [[TMP1:%.*]] = load <2 x float>, ptr [[COLOUR:%.*]], align 4
+; POW2-ONLY-NEXT:    [[TMP2:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP1]], <2 x float> zeroinitializer, <2 x float> zeroinitializer)
+; POW2-ONLY-NEXT:    store <2 x float> [[TMP2]], ptr [[COLOUR]], align 4
+; POW2-ONLY-NEXT:    [[ARRAYIDX99_I1:%.*]] = getelementptr float, ptr [[COLOUR]], i64 2
+; POW2-ONLY-NEXT:    [[TMP3:%.*]] = call float @llvm.fmuladd.f32(float [[TMP0:%.*]], float 0.000000e+00, float 0.000000e+00)
+; POW2-ONLY-NEXT:    store float [[TMP3]], ptr [[ARRAYIDX99_I1]], align 4
+; POW2-ONLY-NEXT:    ret void
 ;
 entry:
   %1 = load float, ptr %Colour, align 4
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vec3-gather-some-loads.ll b/llvm/test/Transforms/SLPVectorizer/X86/vec3-gather-some-loads.ll
index 1411f9416f69..e8adda0bdc70 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/vec3-gather-some-loads.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/vec3-gather-some-loads.ll
@@ -1,35 +1,55 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt -passes=slp-vectorizer -mtriple=x86_64-apple-macosx -S %s | FileCheck %s
+; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2 -mtriple=x86_64-apple-macosx -S %s | FileCheck --check-prefixes=NON-POW2 %s
+; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2=false -mtriple=x86_64-apple-macosx -S %s | FileCheck --check-prefixes=POW2-ONLY %s
 
 target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
 
 define void @test_insert_loads(ptr %A, ptr noalias %B, float %0) #0 {
-; CHECK-LABEL: define void @test_insert_loads(
-; CHECK-SAME: ptr [[A:%.*]], ptr noalias [[B:%.*]], float [[TMP0:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[MULADD_0:%.*]] = tail call float @llvm.fmuladd.f32(float [[TMP0]], float 1.000000e+00, float 1.000000e+00)
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP2]], <2 x float> <float 3.000000e+00, float 2.000000e+00>, <2 x float> <float 3.000000e+00, float 2.000000e+00>)
-; CHECK-NEXT:    [[A_28:%.*]] = getelementptr i8, ptr [[A]], i64 28
-; CHECK-NEXT:    [[L_A_28:%.*]] = load float, ptr [[A_28]], align 4
-; CHECK-NEXT:    [[A_12:%.*]] = getelementptr i8, ptr [[A]], i64 12
-; CHECK-NEXT:    [[L_A_12:%.*]] = load float, ptr [[A_12]], align 4
-; CHECK-NEXT:    [[GEP_4:%.*]] = getelementptr i8, ptr [[B]], i64 4
-; CHECK-NEXT:    [[L_B_0:%.*]] = load float, ptr [[B]], align 4
-; CHECK-NEXT:    [[GEP_28:%.*]] = getelementptr i8, ptr [[B]], i64 28
-; CHECK-NEXT:    [[GEP_20:%.*]] = getelementptr i8, ptr [[B]], i64 20
-; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x float> <float poison, float poison, float poison, float 4.000000e+00>, float [[L_A_12]], i32 0
-; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x float> [[TMP6]], float [[L_A_28]], i32 1
-; CHECK-NEXT:    [[TMP8:%.*]] = shufflevector <4 x float> [[TMP7]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 3>
-; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x float> <float poison, float 0.000000e+00, float 0.000000e+00, float 4.000000e+00>, float [[L_B_0]], i32 0
-; CHECK-NEXT:    [[TMP10:%.*]] = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> [[TMP5]], <4 x float> [[TMP8]], <4 x float> [[TMP9]])
-; CHECK-NEXT:    store <4 x float> [[TMP10]], ptr [[GEP_4]], align 4
-; CHECK-NEXT:    store <2 x float> [[TMP3]], ptr [[GEP_20]], align 4
-; CHECK-NEXT:    store float [[MULADD_0]], ptr [[GEP_28]], align 4
-; CHECK-NEXT:    ret void
+; NON-POW2-LABEL: define void @test_insert_loads(
+; NON-POW2-SAME: ptr [[A:%.*]], ptr noalias [[B:%.*]], float [[TMP0:%.*]]) #[[ATTR0:[0-9]+]] {
+; NON-POW2-NEXT:  entry:
+; NON-POW2-NEXT:    [[A_28:%.*]] = getelementptr i8, ptr [[A]], i64 28
+; NON-POW2-NEXT:    [[L_A_28:%.*]] = load float, ptr [[A_28]], align 4
+; NON-POW2-NEXT:    [[A_12:%.*]] = getelementptr i8, ptr [[A]], i64 12
+; NON-POW2-NEXT:    [[L_A_12:%.*]] = load float, ptr [[A_12]], align 4
+; NON-POW2-NEXT:    [[GEP_4:%.*]] = getelementptr i8, ptr [[B]], i64 4
+; NON-POW2-NEXT:    [[L_B_0:%.*]] = load float, ptr [[B]], align 4
+; NON-POW2-NEXT:    [[TMP1:%.*]] = insertelement <7 x float> poison, float [[TMP0]], i32 0
+; NON-POW2-NEXT:    [[TMP2:%.*]] = shufflevector <7 x float> [[TMP1]], <7 x float> poison, <7 x i32> zeroinitializer
+; NON-POW2-NEXT:    [[TMP3:%.*]] = insertelement <7 x float> <float poison, float poison, float poison, float 4.000000e+00, float 3.000000e+00, float 2.000000e+00, float 1.000000e+00>, float [[L_A_12]], i32 0
+; NON-POW2-NEXT:    [[TMP4:%.*]] = insertelement <7 x float> [[TMP3]], float [[L_A_28]], i32 1
+; NON-POW2-NEXT:    [[TMP5:%.*]] = shufflevector <7 x float> [[TMP4]], <7 x float> poison, <7 x i32> <i32 0, i32 1, i32 1, i32 3, i32 4, i32 5, i32 6>
+; NON-POW2-NEXT:    [[TMP6:%.*]] = insertelement <7 x float> <float poison, float 0.000000e+00, float 0.000000e+00, float 4.000000e+00, float 3.000000e+00, float 2.000000e+00, float 1.000000e+00>, float [[L_B_0]], i32 0
+; NON-POW2-NEXT:    [[TMP7:%.*]] = call <7 x float> @llvm.fmuladd.v7f32(<7 x float> [[TMP2]], <7 x float> [[TMP5]], <7 x float> [[TMP6]])
+; NON-POW2-NEXT:    store <7 x float> [[TMP7]], ptr [[GEP_4]], align 4
+; NON-POW2-NEXT:    ret void
+;
+; POW2-ONLY-LABEL: define void @test_insert_loads(
+; POW2-ONLY-SAME: ptr [[A:%.*]], ptr noalias [[B:%.*]], float [[TMP0:%.*]]) #[[ATTR0:[0-9]+]] {
+; POW2-ONLY-NEXT:  entry:
+; POW2-ONLY-NEXT:    [[MULADD_0:%.*]] = tail call float @llvm.fmuladd.f32(float [[TMP0]], float 1.000000e+00, float 1.000000e+00)
+; POW2-ONLY-NEXT:    [[TMP1:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i32 0
+; POW2-ONLY-NEXT:    [[TMP2:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <2 x i32> zeroinitializer
+; POW2-ONLY-NEXT:    [[TMP3:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP2]], <2 x float> <float 3.000000e+00, float 2.000000e+00>, <2 x float> <float 3.000000e+00, float 2.000000e+00>)
+; POW2-ONLY-NEXT:    [[A_28:%.*]] = getelementptr i8, ptr [[A]], i64 28
+; POW2-ONLY-NEXT:    [[L_A_28:%.*]] = load float, ptr [[A_28]], align 4
+; POW2-ONLY-NEXT:    [[A_12:%.*]] = getelementptr i8, ptr [[A]], i64 12
+; POW2-ONLY-NEXT:    [[L_A_12:%.*]] = load float, ptr [[A_12]], align 4
+; POW2-ONLY-NEXT:    [[GEP_4:%.*]] = getelementptr i8, ptr [[B]], i64 4
+; POW2-ONLY-NEXT:    [[L_B_0:%.*]] = load float, ptr [[B]], align 4
+; POW2-ONLY-NEXT:    [[GEP_28:%.*]] = getelementptr i8, ptr [[B]], i64 28
+; POW2-ONLY-NEXT:    [[GEP_20:%.*]] = getelementptr i8, ptr [[B]], i64 20
+; POW2-ONLY-NEXT:    [[TMP4:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i32 0
+; POW2-ONLY-NEXT:    [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> poison, <4 x i32> zeroinitializer
+; POW2-ONLY-NEXT:    [[TMP6:%.*]] = insertelement <4 x float> <float poison, float poison, float poison, float 4.000000e+00>, float [[L_A_12]], i32 0
+; POW2-ONLY-NEXT:    [[TMP7:%.*]] = insertelement <4 x float> [[TMP6]], float [[L_A_28]], i32 1
+; POW2-ONLY-NEXT:    [[TMP8:%.*]] = shufflevector <4 x float> [[TMP7]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 3>
+; POW2-ONLY-NEXT:    [[TMP9:%.*]] = insertelement <4 x float> <float poison, float 0.000000e+00, float 0.000000e+00, float 4.000000e+00>, float [[L_B_0]], i32 0
+; POW2-ONLY-NEXT:    [[TMP10:%.*]] = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> [[TMP5]], <4 x float> [[TMP8]], <4 x float> [[TMP9]])
+; POW2-ONLY-NEXT:    store <4 x float> [[TMP10]], ptr [[GEP_4]], align 4
+; POW2-ONLY-NEXT:    store <2 x float> [[TMP3]], ptr [[GEP_20]], align 4
+; POW2-ONLY-NEXT:    store float [[MULADD_0]], ptr [[GEP_28]], align 4
+; POW2-ONLY-NEXT:    ret void
 ;
 entry:
   %muladd.0 = tail call float @llvm.fmuladd.f32(float %0, float 1.000000e+00, float 1.000000e+00)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vec3-reorder-reshuffle.ll b/llvm/test/Transforms/SLPVectorizer/X86/vec3-reorder-reshuffle.ll
index 1faeea716b17..1399b4c35c78 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/vec3-reorder-reshuffle.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/vec3-reorder-reshuffle.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt -passes=slp-vectorizer -mtriple=x86_64-apple-macosx -S %s | FileCheck %s
+; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2 -mtriple=x86_64-apple-macosx -S %s | FileCheck --check-prefixes=CHECK,NON-POW2 %s
+; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2=false -mtriple=x86_64-apple-macosx -S %s | FileCheck --check-prefixes=CHECK,POW2-ONLY %s
 
 %struct.zot = type { i32, i32, i32 }
 
@@ -138,21 +139,34 @@ if.end668:                                        ; preds = %if.then665, %entry
 }
 
 define void @gather_2(ptr %mat1, float %0, float %1) {
-; CHECK-LABEL: define void @gather_2(
-; CHECK-SAME: ptr [[MAT1:%.*]], float [[TMP0:%.*]], float [[TMP1:%.*]]) {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i32 0
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x float> <float 0.000000e+00, float poison>, float [[TMP1]], i32 1
-; CHECK-NEXT:    [[TMP5:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP3]], <2 x float> [[TMP4]], <2 x float> zeroinitializer)
-; CHECK-NEXT:    [[TMP6:%.*]] = call float @llvm.fmuladd.f32(float [[TMP0]], float [[TMP1]], float 0.000000e+00)
-; CHECK-NEXT:    [[TMP7:%.*]] = fmul float [[TMP6]], 0.000000e+00
-; CHECK-NEXT:    [[ARRAYIDX163:%.*]] = getelementptr [4 x [4 x float]], ptr [[MAT1]], i64 0, i64 1
-; CHECK-NEXT:    [[ARRAYIDX5_I_I_I280:%.*]] = getelementptr [4 x [4 x float]], ptr [[MAT1]], i64 0, i64 1, i64 2
-; CHECK-NEXT:    [[TMP8:%.*]] = fmul <2 x float> [[TMP5]], zeroinitializer
-; CHECK-NEXT:    store <2 x float> [[TMP8]], ptr [[ARRAYIDX163]], align 4
-; CHECK-NEXT:    store float [[TMP7]], ptr [[ARRAYIDX5_I_I_I280]], align 4
-; CHECK-NEXT:    ret void
+; NON-POW2-LABEL: define void @gather_2(
+; NON-POW2-SAME: ptr [[MAT1:%.*]], float [[TMP0:%.*]], float [[TMP1:%.*]]) {
+; NON-POW2-NEXT:  entry:
+; NON-POW2-NEXT:    [[TMP2:%.*]] = insertelement <3 x float> poison, float [[TMP0]], i32 0
+; NON-POW2-NEXT:    [[TMP4:%.*]] = shufflevector <3 x float> [[TMP2]], <3 x float> poison, <3 x i32> zeroinitializer
+; NON-POW2-NEXT:    [[TMP5:%.*]] = insertelement <3 x float> <float 0.000000e+00, float poison, float poison>, float [[TMP1]], i32 1
+; NON-POW2-NEXT:    [[TMP6:%.*]] = shufflevector <3 x float> [[TMP5]], <3 x float> poison, <3 x i32> <i32 0, i32 1, i32 1>
+; NON-POW2-NEXT:    [[TMP7:%.*]] = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> [[TMP4]], <3 x float> [[TMP6]], <3 x float> zeroinitializer)
+; NON-POW2-NEXT:    [[ARRAYIDX163:%.*]] = getelementptr [4 x [4 x float]], ptr [[MAT1]], i64 0, i64 1
+; NON-POW2-NEXT:    [[TMP8:%.*]] = fmul <3 x float> [[TMP7]], zeroinitializer
+; NON-POW2-NEXT:    store <3 x float> [[TMP8]], ptr [[ARRAYIDX163]], align 4
+; NON-POW2-NEXT:    ret void
+;
+; POW2-ONLY-LABEL: define void @gather_2(
+; POW2-ONLY-SAME: ptr [[MAT1:%.*]], float [[TMP0:%.*]], float [[TMP1:%.*]]) {
+; POW2-ONLY-NEXT:  entry:
+; POW2-ONLY-NEXT:    [[TMP2:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i32 0
+; POW2-ONLY-NEXT:    [[TMP3:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <2 x i32> zeroinitializer
+; POW2-ONLY-NEXT:    [[TMP4:%.*]] = insertelement <2 x float> <float 0.000000e+00, float poison>, float [[TMP1]], i32 1
+; POW2-ONLY-NEXT:    [[TMP5:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP3]], <2 x float> [[TMP4]], <2 x float> zeroinitializer)
+; POW2-ONLY-NEXT:    [[TMP6:%.*]] = call float @llvm.fmuladd.f32(float [[TMP0]], float [[TMP1]], float 0.000000e+00)
+; POW2-ONLY-NEXT:    [[TMP7:%.*]] = fmul float [[TMP6]], 0.000000e+00
+; POW2-ONLY-NEXT:    [[ARRAYIDX163:%.*]] = getelementptr [4 x [4 x float]], ptr [[MAT1]], i64 0, i64 1
+; POW2-ONLY-NEXT:    [[ARRAYIDX5_I_I_I280:%.*]] = getelementptr [4 x [4 x float]], ptr [[MAT1]], i64 0, i64 1, i64 2
+; POW2-ONLY-NEXT:    [[TMP8:%.*]] = fmul <2 x float> [[TMP5]], zeroinitializer
+; POW2-ONLY-NEXT:    store <2 x float> [[TMP8]], ptr [[ARRAYIDX163]], align 4
+; POW2-ONLY-NEXT:    store float [[TMP7]], ptr [[ARRAYIDX5_I_I_I280]], align 4
+; POW2-ONLY-NEXT:    ret void
 ;
 entry:
   %2 = call float @llvm.fmuladd.f32(float %0, float 0.000000e+00, float 0.000000e+00)
@@ -171,31 +185,47 @@ entry:
 }
 
 define i32 @reorder_indices_1(float %0) {
-; CHECK-LABEL: define i32 @reorder_indices_1(
-; CHECK-SAME: float [[TMP0:%.*]]) {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[NOR1:%.*]] = alloca [0 x [3 x float]], i32 0, align 4
-; CHECK-NEXT:    [[ARRAYIDX2_I265:%.*]] = getelementptr float, ptr [[NOR1]], i64 2
-; CHECK-NEXT:    [[TMP1:%.*]] = load float, ptr [[ARRAYIDX2_I265]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x float>, ptr [[NOR1]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x float> [[TMP2]], i32 0
-; CHECK-NEXT:    [[TMP4:%.*]] = fneg float [[TMP3]]
-; CHECK-NEXT:    [[NEG11_I:%.*]] = fmul float [[TMP4]], [[TMP0]]
-; CHECK-NEXT:    [[TMP5:%.*]] = call float @llvm.fmuladd.f32(float [[TMP1]], float 0.000000e+00, float [[NEG11_I]])
-; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <2 x i32> <i32 1, i32 poison>
-; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x float> [[TMP6]], float [[TMP1]], i32 1
-; CHECK-NEXT:    [[TMP8:%.*]] = fneg <2 x float> [[TMP7]]
-; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i32 0
-; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP11:%.*]] = fmul <2 x float> [[TMP8]], [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP2]], <2 x float> zeroinitializer, <2 x float> [[TMP11]])
-; CHECK-NEXT:    [[TMP13:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP10]], <2 x float> [[TMP12]], <2 x float> zeroinitializer)
-; CHECK-NEXT:    [[TMP14:%.*]] = call float @llvm.fmuladd.f32(float [[TMP0]], float [[TMP5]], float 0.000000e+00)
-; CHECK-NEXT:    [[TMP15:%.*]] = fmul <2 x float> [[TMP13]], zeroinitializer
-; CHECK-NEXT:    [[MUL6_I_I_I:%.*]] = fmul float [[TMP14]], 0.000000e+00
-; CHECK-NEXT:    store <2 x float> [[TMP15]], ptr [[NOR1]], align 4
-; CHECK-NEXT:    store float [[MUL6_I_I_I]], ptr [[ARRAYIDX2_I265]], align 4
-; CHECK-NEXT:    ret i32 0
+; NON-POW2-LABEL: define i32 @reorder_indices_1(
+; NON-POW2-SAME: float [[TMP0:%.*]]) {
+; NON-POW2-NEXT:  entry:
+; NON-POW2-NEXT:    [[NOR1:%.*]] = alloca [0 x [3 x float]], i32 0, align 4
+; NON-POW2-NEXT:    [[TMP1:%.*]] = load <3 x float>, ptr [[NOR1]], align 4
+; NON-POW2-NEXT:    [[TMP2:%.*]] = shufflevector <3 x float> [[TMP1]], <3 x float> poison, <3 x i32> <i32 1, i32 2, i32 0>
+; NON-POW2-NEXT:    [[TMP3:%.*]] = fneg <3 x float> [[TMP2]]
+; NON-POW2-NEXT:    [[TMP4:%.*]] = insertelement <3 x float> poison, float [[TMP0]], i32 0
+; NON-POW2-NEXT:    [[TMP5:%.*]] = shufflevector <3 x float> [[TMP4]], <3 x float> poison, <3 x i32> zeroinitializer
+; NON-POW2-NEXT:    [[TMP6:%.*]] = fmul <3 x float> [[TMP3]], [[TMP5]]
+; NON-POW2-NEXT:    [[TMP7:%.*]] = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> [[TMP1]], <3 x float> zeroinitializer, <3 x float> [[TMP6]])
+; NON-POW2-NEXT:    [[TMP8:%.*]] = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> [[TMP5]], <3 x float> [[TMP7]], <3 x float> zeroinitializer)
+; NON-POW2-NEXT:    [[TMP9:%.*]] = fmul <3 x float> [[TMP8]], zeroinitializer
+; NON-POW2-NEXT:    store <3 x float> [[TMP9]], ptr [[NOR1]], align 4
+; NON-POW2-NEXT:    ret i32 0
+;
+; POW2-ONLY-LABEL: define i32 @reorder_indices_1(
+; POW2-ONLY-SAME: float [[TMP0:%.*]]) {
+; POW2-ONLY-NEXT:  entry:
+; POW2-ONLY-NEXT:    [[NOR1:%.*]] = alloca [0 x [3 x float]], i32 0, align 4
+; POW2-ONLY-NEXT:    [[ARRAYIDX2_I265:%.*]] = getelementptr float, ptr [[NOR1]], i64 2
+; POW2-ONLY-NEXT:    [[TMP1:%.*]] = load float, ptr [[ARRAYIDX2_I265]], align 4
+; POW2-ONLY-NEXT:    [[TMP2:%.*]] = load <2 x float>, ptr [[NOR1]], align 4
+; POW2-ONLY-NEXT:    [[TMP3:%.*]] = extractelement <2 x float> [[TMP2]], i32 0
+; POW2-ONLY-NEXT:    [[TMP4:%.*]] = fneg float [[TMP3]]
+; POW2-ONLY-NEXT:    [[NEG11_I:%.*]] = fmul float [[TMP4]], [[TMP0]]
+; POW2-ONLY-NEXT:    [[TMP5:%.*]] = call float @llvm.fmuladd.f32(float [[TMP1]], float 0.000000e+00, float [[NEG11_I]])
+; POW2-ONLY-NEXT:    [[TMP6:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <2 x i32> <i32 1, i32 poison>
+; POW2-ONLY-NEXT:    [[TMP7:%.*]] = insertelement <2 x float> [[TMP6]], float [[TMP1]], i32 1
+; POW2-ONLY-NEXT:    [[TMP8:%.*]] = fneg <2 x float> [[TMP7]]
+; POW2-ONLY-NEXT:    [[TMP9:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i32 0
+; POW2-ONLY-NEXT:    [[TMP10:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> poison, <2 x i32> zeroinitializer
+; POW2-ONLY-NEXT:    [[TMP11:%.*]] = fmul <2 x float> [[TMP8]], [[TMP10]]
+; POW2-ONLY-NEXT:    [[TMP12:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP2]], <2 x float> zeroinitializer, <2 x float> [[TMP11]])
+; POW2-ONLY-NEXT:    [[TMP13:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP10]], <2 x float> [[TMP12]], <2 x float> zeroinitializer)
+; POW2-ONLY-NEXT:    [[TMP14:%.*]] = call float @llvm.fmuladd.f32(float [[TMP0]], float [[TMP5]], float 0.000000e+00)
+; POW2-ONLY-NEXT:    [[TMP15:%.*]] = fmul <2 x float> [[TMP13]], zeroinitializer
+; POW2-ONLY-NEXT:    [[MUL6_I_I_I:%.*]] = fmul float [[TMP14]], 0.000000e+00
+; POW2-ONLY-NEXT:    store <2 x float> [[TMP15]], ptr [[NOR1]], align 4
+; POW2-ONLY-NEXT:    store float [[MUL6_I_I_I]], ptr [[ARRAYIDX2_I265]], align 4
+; POW2-ONLY-NEXT:    ret i32 0
 ;
 entry:
   %nor1 = alloca [0 x [3 x float]], i32 0, align 4
@@ -226,19 +256,28 @@ entry:
 }
 
 define void @reorder_indices_2(ptr %spoint) {
-; CHECK-LABEL: define void @reorder_indices_2(
-; CHECK-SAME: ptr [[SPOINT:%.*]]) {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <3 x float> zeroinitializer, i64 0
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call float @llvm.fmuladd.f32(float [[TMP0]], float 0.000000e+00, float 0.000000e+00)
-; CHECK-NEXT:    [[MUL4_I461:%.*]] = fmul float [[TMP1]], 0.000000e+00
-; CHECK-NEXT:    [[DSCO:%.*]] = getelementptr float, ptr [[SPOINT]], i64 0
-; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> zeroinitializer, <2 x float> zeroinitializer, <2 x float> zeroinitializer)
-; CHECK-NEXT:    [[TMP3:%.*]] = fmul <2 x float> [[TMP2]], zeroinitializer
-; CHECK-NEXT:    store <2 x float> [[TMP3]], ptr [[DSCO]], align 4
-; CHECK-NEXT:    [[ARRAYIDX5_I476:%.*]] = getelementptr float, ptr [[SPOINT]], i64 2
-; CHECK-NEXT:    store float [[MUL4_I461]], ptr [[ARRAYIDX5_I476]], align 4
-; CHECK-NEXT:    ret void
+; NON-POW2-LABEL: define void @reorder_indices_2(
+; NON-POW2-SAME: ptr [[SPOINT:%.*]]) {
+; NON-POW2-NEXT:  entry:
+; NON-POW2-NEXT:    [[DSCO:%.*]] = getelementptr float, ptr [[SPOINT]], i64 0
+; NON-POW2-NEXT:    [[TMP0:%.*]] = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> zeroinitializer, <3 x float> zeroinitializer, <3 x float> zeroinitializer)
+; NON-POW2-NEXT:    [[TMP1:%.*]] = fmul <3 x float> [[TMP0]], zeroinitializer
+; NON-POW2-NEXT:    store <3 x float> [[TMP1]], ptr [[DSCO]], align 4
+; NON-POW2-NEXT:    ret void
+;
+; POW2-ONLY-LABEL: define void @reorder_indices_2(
+; POW2-ONLY-SAME: ptr [[SPOINT:%.*]]) {
+; POW2-ONLY-NEXT:  entry:
+; POW2-ONLY-NEXT:    [[TMP0:%.*]] = extractelement <3 x float> zeroinitializer, i64 0
+; POW2-ONLY-NEXT:    [[TMP1:%.*]] = tail call float @llvm.fmuladd.f32(float [[TMP0]], float 0.000000e+00, float 0.000000e+00)
+; POW2-ONLY-NEXT:    [[MUL4_I461:%.*]] = fmul float [[TMP1]], 0.000000e+00
+; POW2-ONLY-NEXT:    [[DSCO:%.*]] = getelementptr float, ptr [[SPOINT]], i64 0
+; POW2-ONLY-NEXT:    [[TMP2:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> zeroinitializer, <2 x float> zeroinitializer, <2 x float> zeroinitializer)
+; POW2-ONLY-NEXT:    [[TMP3:%.*]] = fmul <2 x float> [[TMP2]], zeroinitializer
+; POW2-ONLY-NEXT:    store <2 x float> [[TMP3]], ptr [[DSCO]], align 4
+; POW2-ONLY-NEXT:    [[ARRAYIDX5_I476:%.*]] = getelementptr float, ptr [[SPOINT]], i64 2
+; POW2-ONLY-NEXT:    store float [[MUL4_I461]], ptr [[ARRAYIDX5_I476]], align 4
+; POW2-ONLY-NEXT:    ret void
 ;
 entry:
   %0 = extractelement <3 x float> zeroinitializer, i64 1
@@ -290,19 +329,30 @@ entry:
 }
 
 define void @reuse_shuffle_indidces_1(ptr %col, float %0, float %1) {
-; CHECK-LABEL: define void @reuse_shuffle_indidces_1(
-; CHECK-SAME: ptr [[COL:%.*]], float [[TMP0:%.*]], float [[TMP1:%.*]]) {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x float> poison, float [[TMP1]], i32 0
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> [[TMP2]], float [[TMP0]], i32 1
-; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x float> [[TMP3]], zeroinitializer
-; CHECK-NEXT:    [[TMP5:%.*]] = fadd <2 x float> [[TMP4]], zeroinitializer
-; CHECK-NEXT:    store <2 x float> [[TMP5]], ptr [[COL]], align 4
-; CHECK-NEXT:    [[ARRAYIDX33:%.*]] = getelementptr float, ptr [[COL]], i64 2
-; CHECK-NEXT:    [[MUL38:%.*]] = fmul float [[TMP0]], 0.000000e+00
-; CHECK-NEXT:    [[TMP6:%.*]] = fadd float [[MUL38]], 0.000000e+00
-; CHECK-NEXT:    store float [[TMP6]], ptr [[ARRAYIDX33]], align 4
-; CHECK-NEXT:    ret void
+; NON-POW2-LABEL: define void @reuse_shuffle_indidces_1(
+; NON-POW2-SAME: ptr [[COL:%.*]], float [[TMP0:%.*]], float [[TMP1:%.*]]) {
+; NON-POW2-NEXT:  entry:
+; NON-POW2-NEXT:    [[TMP2:%.*]] = insertelement <3 x float> poison, float [[TMP1]], i32 0
+; NON-POW2-NEXT:    [[TMP3:%.*]] = insertelement <3 x float> [[TMP2]], float [[TMP0]], i32 1
+; NON-POW2-NEXT:    [[TMP4:%.*]] = shufflevector <3 x float> [[TMP3]], <3 x float> poison, <3 x i32> <i32 0, i32 1, i32 1>
+; NON-POW2-NEXT:    [[TMP5:%.*]] = fmul <3 x float> [[TMP4]], zeroinitializer
+; NON-POW2-NEXT:    [[TMP6:%.*]] = fadd <3 x float> [[TMP5]], zeroinitializer
+; NON-POW2-NEXT:    store <3 x float> [[TMP6]], ptr [[COL]], align 4
+; NON-POW2-NEXT:    ret void
+;
+; POW2-ONLY-LABEL: define void @reuse_shuffle_indidces_1(
+; POW2-ONLY-SAME: ptr [[COL:%.*]], float [[TMP0:%.*]], float [[TMP1:%.*]]) {
+; POW2-ONLY-NEXT:  entry:
+; POW2-ONLY-NEXT:    [[TMP2:%.*]] = insertelement <2 x float> poison, float [[TMP1]], i32 0
+; POW2-ONLY-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> [[TMP2]], float [[TMP0]], i32 1
+; POW2-ONLY-NEXT:    [[TMP4:%.*]] = fmul <2 x float> [[TMP3]], zeroinitializer
+; POW2-ONLY-NEXT:    [[TMP5:%.*]] = fadd <2 x float> [[TMP4]], zeroinitializer
+; POW2-ONLY-NEXT:    store <2 x float> [[TMP5]], ptr [[COL]], align 4
+; POW2-ONLY-NEXT:    [[ARRAYIDX33:%.*]] = getelementptr float, ptr [[COL]], i64 2
+; POW2-ONLY-NEXT:    [[MUL38:%.*]] = fmul float [[TMP0]], 0.000000e+00
+; POW2-ONLY-NEXT:    [[TMP6:%.*]] = fadd float [[MUL38]], 0.000000e+00
+; POW2-ONLY-NEXT:    store float [[TMP6]], ptr [[ARRAYIDX33]], align 4
+; POW2-ONLY-NEXT:    ret void
 ;
 entry:
   %mul24 = fmul float %1, 0.000000e+00
@@ -350,20 +400,32 @@ entry:
 }
 
 define void @reuse_shuffle_indices_cost_crash_2(ptr %bezt, float %0) {
-; CHECK-LABEL: define void @reuse_shuffle_indices_cost_crash_2(
-; CHECK-SAME: ptr [[BEZT:%.*]], float [[TMP0:%.*]]) {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[FNEG:%.*]] = fmul float [[TMP0]], 0.000000e+00
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i32 0
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> poison, float [[FNEG]], i32 0
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP5:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP2]], <2 x float> [[TMP4]], <2 x float> zeroinitializer)
-; CHECK-NEXT:    store <2 x float> [[TMP5]], ptr [[BEZT]], align 4
-; CHECK-NEXT:    [[TMP6:%.*]] = tail call float @llvm.fmuladd.f32(float [[FNEG]], float 0.000000e+00, float 0.000000e+00)
-; CHECK-NEXT:    [[ARRAYIDX8_I831:%.*]] = getelementptr float, ptr [[BEZT]], i64 2
-; CHECK-NEXT:    store float [[TMP6]], ptr [[ARRAYIDX8_I831]], align 4
-; CHECK-NEXT:    ret void
+; NON-POW2-LABEL: define void @reuse_shuffle_indices_cost_crash_2(
+; NON-POW2-SAME: ptr [[BEZT:%.*]], float [[TMP0:%.*]]) {
+; NON-POW2-NEXT:  entry:
+; NON-POW2-NEXT:    [[FNEG:%.*]] = fmul float [[TMP0]], 0.000000e+00
+; NON-POW2-NEXT:    [[TMP1:%.*]] = insertelement <3 x float> poison, float [[FNEG]], i32 0
+; NON-POW2-NEXT:    [[TMP2:%.*]] = shufflevector <3 x float> [[TMP1]], <3 x float> poison, <3 x i32> zeroinitializer
+; NON-POW2-NEXT:    [[TMP3:%.*]] = insertelement <3 x float> <float poison, float poison, float 0.000000e+00>, float [[TMP0]], i32 0
+; NON-POW2-NEXT:    [[TMP4:%.*]] = shufflevector <3 x float> [[TMP3]], <3 x float> poison, <3 x i32> <i32 0, i32 0, i32 2>
+; NON-POW2-NEXT:    [[TMP5:%.*]] = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> [[TMP2]], <3 x float> [[TMP4]], <3 x float> zeroinitializer)
+; NON-POW2-NEXT:    store <3 x float> [[TMP5]], ptr [[BEZT]], align 4
+; NON-POW2-NEXT:    ret void
+;
+; POW2-ONLY-LABEL: define void @reuse_shuffle_indices_cost_crash_2(
+; POW2-ONLY-SAME: ptr [[BEZT:%.*]], float [[TMP0:%.*]]) {
+; POW2-ONLY-NEXT:  entry:
+; POW2-ONLY-NEXT:    [[FNEG:%.*]] = fmul float [[TMP0]], 0.000000e+00
+; POW2-ONLY-NEXT:    [[TMP1:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i32 0
+; POW2-ONLY-NEXT:    [[TMP2:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <2 x i32> zeroinitializer
+; POW2-ONLY-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> poison, float [[FNEG]], i32 0
+; POW2-ONLY-NEXT:    [[TMP4:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> poison, <2 x i32> zeroinitializer
+; POW2-ONLY-NEXT:    [[TMP5:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP2]], <2 x float> [[TMP4]], <2 x float> zeroinitializer)
+; POW2-ONLY-NEXT:    store <2 x float> [[TMP5]], ptr [[BEZT]], align 4
+; POW2-ONLY-NEXT:    [[TMP6:%.*]] = tail call float @llvm.fmuladd.f32(float [[FNEG]], float 0.000000e+00, float 0.000000e+00)
+; POW2-ONLY-NEXT:    [[ARRAYIDX8_I831:%.*]] = getelementptr float, ptr [[BEZT]], i64 2
+; POW2-ONLY-NEXT:    store float [[TMP6]], ptr [[ARRAYIDX8_I831]], align 4
+; POW2-ONLY-NEXT:    ret void
 ;
 entry:
   %fneg = fmul float %0, 0.000000e+00
@@ -487,15 +549,21 @@ entry:
 }
 
 define void @vec3_extract(<3 x i16> %pixel.sroa.0.4.vec.insert606, ptr %call3.i536) {
-; CHECK-LABEL: define void @vec3_extract(
-; CHECK-SAME: <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606:%.*]], ptr [[CALL3_I536:%.*]]) {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[PIXEL_SROA_0_4_VEC_EXTRACT:%.*]] = extractelement <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606]], i64 2
-; CHECK-NEXT:    [[RED668:%.*]] = getelementptr i16, ptr [[CALL3_I536]], i64 2
-; CHECK-NEXT:    store i16 [[PIXEL_SROA_0_4_VEC_EXTRACT]], ptr [[RED668]], align 2
-; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT:    store <2 x i16> [[TMP0]], ptr [[CALL3_I536]], align 2
-; CHECK-NEXT:    ret void
+; NON-POW2-LABEL: define void @vec3_extract(
+; NON-POW2-SAME: <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606:%.*]], ptr [[CALL3_I536:%.*]]) {
+; NON-POW2-NEXT:  entry:
+; NON-POW2-NEXT:    store <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606]], ptr [[CALL3_I536]], align 2
+; NON-POW2-NEXT:    ret void
+;
+; POW2-ONLY-LABEL: define void @vec3_extract(
+; POW2-ONLY-SAME: <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606:%.*]], ptr [[CALL3_I536:%.*]]) {
+; POW2-ONLY-NEXT:  entry:
+; POW2-ONLY-NEXT:    [[PIXEL_SROA_0_4_VEC_EXTRACT:%.*]] = extractelement <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606]], i64 2
+; POW2-ONLY-NEXT:    [[RED668:%.*]] = getelementptr i16, ptr [[CALL3_I536]], i64 2
+; POW2-ONLY-NEXT:    store i16 [[PIXEL_SROA_0_4_VEC_EXTRACT]], ptr [[RED668]], align 2
+; POW2-ONLY-NEXT:    [[TMP0:%.*]] = shufflevector <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606]], <3 x i16> poison, <2 x i32> <i32 0, i32 1>
+; POW2-ONLY-NEXT:    store <2 x i16> [[TMP0]], ptr [[CALL3_I536]], align 2
+; POW2-ONLY-NEXT:    ret void
 ;
 entry:
   %pixel.sroa.0.4.vec.extract = extractelement <3 x i16> %pixel.sroa.0.4.vec.insert606, i64 2
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vect_copyable_in_binops.ll b/llvm/test/Transforms/SLPVectorizer/X86/vect_copyable_in_binops.ll
index e8395fe69ea6..e30cb76d53d9 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/vect_copyable_in_binops.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/vect_copyable_in_binops.ll
@@ -1,12 +1,13 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mcpu=bdver2 < %s | FileCheck %s
+; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2 -S -mtriple=x86_64-unknown-linux-gnu -mcpu=bdver2 < %s | FileCheck --check-prefixes=CHECK,NON-POW2 %s
+; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2=false -S -mtriple=x86_64-unknown-linux-gnu -mcpu=bdver2 < %s | FileCheck --check-prefixes=CHECK,POW2-ONLY %s
 
 define void @add0(ptr noalias %dst, ptr noalias %src) {
 ; CHECK-LABEL: @add0(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[SRC:%.*]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 1, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    store <4 x i32> [[TMP2]], ptr [[DST:%.*]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr [[SRC:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[TMP0]], <i32 1, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    store <4 x i32> [[TMP1]], ptr [[DST:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -32,21 +33,32 @@ entry:
 }
 
 define void @add1(ptr noalias %dst, ptr noalias %src) {
-; CHECK-LABEL: @add1(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i64 1
-; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 1
-; CHECK-NEXT:    store i32 [[TMP0]], ptr [[DST]], align 4
-; CHECK-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 3
-; CHECK-NEXT:    [[INCDEC_PTR7:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 3
-; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr [[INCDEC_PTR]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <2 x i32> [[TMP2]], <i32 1, i32 2>
-; CHECK-NEXT:    store <2 x i32> [[TMP3]], ptr [[INCDEC_PTR1]], align 4
-; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[INCDEC_PTR5]], align 4
-; CHECK-NEXT:    [[ADD9:%.*]] = add nsw i32 [[TMP5]], 3
-; CHECK-NEXT:    store i32 [[ADD9]], ptr [[INCDEC_PTR7]], align 4
-; CHECK-NEXT:    ret void
+; NON-POW2-LABEL: @add1(
+; NON-POW2-NEXT:  entry:
+; NON-POW2-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i64 1
+; NON-POW2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[SRC]], align 4
+; NON-POW2-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 1
+; NON-POW2-NEXT:    store i32 [[TMP0]], ptr [[DST]], align 4
+; NON-POW2-NEXT:    [[TMP1:%.*]] = load <3 x i32>, ptr [[INCDEC_PTR]], align 4
+; NON-POW2-NEXT:    [[TMP2:%.*]] = add nsw <3 x i32> [[TMP1]], <i32 1, i32 2, i32 3>
+; NON-POW2-NEXT:    store <3 x i32> [[TMP2]], ptr [[INCDEC_PTR1]], align 4
+; NON-POW2-NEXT:    ret void
+;
+; POW2-ONLY-LABEL: @add1(
+; POW2-ONLY-NEXT:  entry:
+; POW2-ONLY-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i64 1
+; POW2-ONLY-NEXT:    [[TMP0:%.*]] = load i32, ptr [[SRC]], align 4
+; POW2-ONLY-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 1
+; POW2-ONLY-NEXT:    store i32 [[TMP0]], ptr [[DST]], align 4
+; POW2-ONLY-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 3
+; POW2-ONLY-NEXT:    [[INCDEC_PTR7:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 3
+; POW2-ONLY-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr [[INCDEC_PTR]], align 4
+; POW2-ONLY-NEXT:    [[TMP2:%.*]] = add nsw <2 x i32> [[TMP1]], <i32 1, i32 2>
+; POW2-ONLY-NEXT:    store <2 x i32> [[TMP2]], ptr [[INCDEC_PTR1]], align 4
+; POW2-ONLY-NEXT:    [[TMP3:%.*]] = load i32, ptr [[INCDEC_PTR5]], align 4
+; POW2-ONLY-NEXT:    [[ADD9:%.*]] = add nsw i32 [[TMP3]], 3
+; POW2-ONLY-NEXT:    store i32 [[ADD9]], ptr [[INCDEC_PTR7]], align 4
+; POW2-ONLY-NEXT:    ret void
 ;
 entry:
   %incdec.ptr = getelementptr inbounds i32, ptr %src, i64 1
@@ -81,9 +93,9 @@ define void @sub0(ptr noalias %dst, ptr noalias %src) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[INCDEC_PTR]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR3:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 2
 ; CHECK-NEXT:    store i32 [[TMP1]], ptr [[INCDEC_PTR1]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i32>, ptr [[INCDEC_PTR2]], align 4
-; CHECK-NEXT:    [[TMP4:%.*]] = add nsw <2 x i32> [[TMP3]], <i32 -2, i32 -3>
-; CHECK-NEXT:    store <2 x i32> [[TMP4]], ptr [[INCDEC_PTR3]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr [[INCDEC_PTR2]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <2 x i32> [[TMP2]], <i32 -2, i32 -3>
+; CHECK-NEXT:    store <2 x i32> [[TMP3]], ptr [[INCDEC_PTR3]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -110,9 +122,9 @@ entry:
 define void @sub1(ptr noalias %dst, ptr noalias %src) {
 ; CHECK-LABEL: @sub1(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[SRC:%.*]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 4, i32 -1, i32 -2, i32 -3>
-; CHECK-NEXT:    store <4 x i32> [[TMP2]], ptr [[DST:%.*]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr [[SRC:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[TMP0]], <i32 4, i32 -1, i32 -2, i32 -3>
+; CHECK-NEXT:    store <4 x i32> [[TMP1]], ptr [[DST:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -140,9 +152,9 @@ entry:
 define void @sub2(ptr noalias %dst, ptr noalias %src) {
 ; CHECK-LABEL: @sub2(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[SRC:%.*]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 -1, i32 -1, i32 -2, i32 -3>
-; CHECK-NEXT:    store <4 x i32> [[TMP2]], ptr [[DST:%.*]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr [[SRC:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw <4 x i32> [[TMP0]], <i32 -1, i32 -1, i32 -2, i32 -3>
+; CHECK-NEXT:    store <4 x i32> [[TMP1]], ptr [[DST:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -218,11 +230,11 @@ define void @addsub1(ptr noalias %dst, ptr noalias %src) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> <i32 0, i32 3>
 ; CHECK-NEXT:    store <2 x i32> [[TMP3]], ptr [[DST]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 3
-; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[INCDEC_PTR2]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[INCDEC_PTR2]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR6:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 3
-; CHECK-NEXT:    store i32 [[TMP6]], ptr [[INCDEC_PTR3]], align 4
-; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[INCDEC_PTR4]], align 4
-; CHECK-NEXT:    [[SUB8:%.*]] = sub nsw i32 [[TMP7]], -3
+; CHECK-NEXT:    store i32 [[TMP4]], ptr [[INCDEC_PTR3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[INCDEC_PTR4]], align 4
+; CHECK-NEXT:    [[SUB8:%.*]] = sub nsw i32 [[TMP5]], -3
 ; CHECK-NEXT:    store i32 [[SUB8]], ptr [[INCDEC_PTR6]], align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -252,15 +264,15 @@ define void @mul(ptr noalias %dst, ptr noalias %src) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i64 2
 ; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 2
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = mul nsw <2 x i32> [[TMP1]], <i32 257, i32 -3>
-; CHECK-NEXT:    store <2 x i32> [[TMP2]], ptr [[DST]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i32>, ptr [[SRC]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nsw <2 x i32> [[TMP0]], <i32 257, i32 -3>
+; CHECK-NEXT:    store <2 x i32> [[TMP1]], ptr [[DST]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 3
-; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[INCDEC_PTR2]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[INCDEC_PTR2]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR7:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 3
-; CHECK-NEXT:    store i32 [[TMP4]], ptr [[INCDEC_PTR4]], align 4
-; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[INCDEC_PTR5]], align 4
-; CHECK-NEXT:    [[MUL9:%.*]] = mul nsw i32 [[TMP5]], -9
+; CHECK-NEXT:    store i32 [[TMP2]], ptr [[INCDEC_PTR4]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[INCDEC_PTR5]], align 4
+; CHECK-NEXT:    [[MUL9:%.*]] = mul nsw i32 [[TMP3]], -9
 ; CHECK-NEXT:    store i32 [[MUL9]], ptr [[INCDEC_PTR7]], align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -286,21 +298,32 @@ entry:
 }
 
 define void @shl0(ptr noalias %dst, ptr noalias %src) {
-; CHECK-LABEL: @shl0(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i64 1
-; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 1
-; CHECK-NEXT:    store i32 [[TMP0]], ptr [[DST]], align 4
-; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 3
-; CHECK-NEXT:    [[INCDEC_PTR6:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 3
-; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, ptr [[INCDEC_PTR]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = shl <2 x i32> [[TMP2]], <i32 1, i32 2>
-; CHECK-NEXT:    store <2 x i32> [[TMP3]], ptr [[INCDEC_PTR1]], align 4
-; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[INCDEC_PTR4]], align 4
-; CHECK-NEXT:    [[SHL8:%.*]] = shl i32 [[TMP5]], 3
-; CHECK-NEXT:    store i32 [[SHL8]], ptr [[INCDEC_PTR6]], align 4
-; CHECK-NEXT:    ret void
+; NON-POW2-LABEL: @shl0(
+; NON-POW2-NEXT:  entry:
+; NON-POW2-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i64 1
+; NON-POW2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[SRC]], align 4
+; NON-POW2-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 1
+; NON-POW2-NEXT:    store i32 [[TMP0]], ptr [[DST]], align 4
+; NON-POW2-NEXT:    [[TMP1:%.*]] = load <3 x i32>, ptr [[INCDEC_PTR]], align 4
+; NON-POW2-NEXT:    [[TMP2:%.*]] = shl <3 x i32> [[TMP1]], <i32 1, i32 2, i32 3>
+; NON-POW2-NEXT:    store <3 x i32> [[TMP2]], ptr [[INCDEC_PTR1]], align 4
+; NON-POW2-NEXT:    ret void
+;
+; POW2-ONLY-LABEL: @shl0(
+; POW2-ONLY-NEXT:  entry:
+; POW2-ONLY-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i64 1
+; POW2-ONLY-NEXT:    [[TMP0:%.*]] = load i32, ptr [[SRC]], align 4
+; POW2-ONLY-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 1
+; POW2-ONLY-NEXT:    store i32 [[TMP0]], ptr [[DST]], align 4
+; POW2-ONLY-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 3
+; POW2-ONLY-NEXT:    [[INCDEC_PTR6:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 3
+; POW2-ONLY-NEXT:    [[TMP1:%.*]] = load <2 x i32>, ptr [[INCDEC_PTR]], align 4
+; POW2-ONLY-NEXT:    [[TMP2:%.*]] = shl <2 x i32> [[TMP1]], <i32 1, i32 2>
+; POW2-ONLY-NEXT:    store <2 x i32> [[TMP2]], ptr [[INCDEC_PTR1]], align 4
+; POW2-ONLY-NEXT:    [[TMP3:%.*]] = load i32, ptr [[INCDEC_PTR4]], align 4
+; POW2-ONLY-NEXT:    [[SHL8:%.*]] = shl i32 [[TMP3]], 3
+; POW2-ONLY-NEXT:    store i32 [[SHL8]], ptr [[INCDEC_PTR6]], align 4
+; POW2-ONLY-NEXT:    ret void
 ;
 entry:
   %incdec.ptr = getelementptr inbounds i32, ptr %src, i64 1
@@ -326,9 +349,9 @@ entry:
 define void @shl1(ptr noalias %dst, ptr noalias %src) {
 ; CHECK-LABEL: @shl1(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[SRC:%.*]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = shl <4 x i32> [[TMP1]], <i32 7, i32 1, i32 2, i32 3>
-; CHECK-NEXT:    store <4 x i32> [[TMP2]], ptr [[DST:%.*]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr [[SRC:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <4 x i32> [[TMP0]], <i32 7, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    store <4 x i32> [[TMP1]], ptr [[DST:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -356,9 +379,9 @@ entry:
 define void @add0f(ptr noalias %dst, ptr noalias %src) {
 ; CHECK-LABEL: @add0f(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[SRC:%.*]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = fadd fast <4 x float> [[TMP1]], <float 1.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>
-; CHECK-NEXT:    store <4 x float> [[TMP2]], ptr [[DST:%.*]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[SRC:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd fast <4 x float> [[TMP0]], <float 1.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>
+; CHECK-NEXT:    store <4 x float> [[TMP1]], ptr [[DST:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -384,21 +407,32 @@ entry:
 }
 
 define void @add1f(ptr noalias %dst, ptr noalias %src) {
-; CHECK-LABEL: @add1f(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i64 1
-; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, ptr [[DST:%.*]], i64 1
-; CHECK-NEXT:    store float [[TMP0]], ptr [[DST]], align 4
-; CHECK-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 3
-; CHECK-NEXT:    [[INCDEC_PTR7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 3
-; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x float>, ptr [[INCDEC_PTR]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = fadd fast <2 x float> [[TMP2]], <float 1.000000e+00, float 2.000000e+00>
-; CHECK-NEXT:    store <2 x float> [[TMP3]], ptr [[INCDEC_PTR1]], align 4
-; CHECK-NEXT:    [[TMP5:%.*]] = load float, ptr [[INCDEC_PTR5]], align 4
-; CHECK-NEXT:    [[ADD9:%.*]] = fadd fast float [[TMP5]], 3.000000e+00
-; CHECK-NEXT:    store float [[ADD9]], ptr [[INCDEC_PTR7]], align 4
-; CHECK-NEXT:    ret void
+; NON-POW2-LABEL: @add1f(
+; NON-POW2-NEXT:  entry:
+; NON-POW2-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i64 1
+; NON-POW2-NEXT:    [[TMP0:%.*]] = load float, ptr [[SRC]], align 4
+; NON-POW2-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, ptr [[DST:%.*]], i64 1
+; NON-POW2-NEXT:    store float [[TMP0]], ptr [[DST]], align 4
+; NON-POW2-NEXT:    [[TMP1:%.*]] = load <3 x float>, ptr [[INCDEC_PTR]], align 4
+; NON-POW2-NEXT:    [[TMP2:%.*]] = fadd fast <3 x float> [[TMP1]], <float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>
+; NON-POW2-NEXT:    store <3 x float> [[TMP2]], ptr [[INCDEC_PTR1]], align 4
+; NON-POW2-NEXT:    ret void
+;
+; POW2-ONLY-LABEL: @add1f(
+; POW2-ONLY-NEXT:  entry:
+; POW2-ONLY-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i64 1
+; POW2-ONLY-NEXT:    [[TMP0:%.*]] = load float, ptr [[SRC]], align 4
+; POW2-ONLY-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, ptr [[DST:%.*]], i64 1
+; POW2-ONLY-NEXT:    store float [[TMP0]], ptr [[DST]], align 4
+; POW2-ONLY-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 3
+; POW2-ONLY-NEXT:    [[INCDEC_PTR7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 3
+; POW2-ONLY-NEXT:    [[TMP1:%.*]] = load <2 x float>, ptr [[INCDEC_PTR]], align 4
+; POW2-ONLY-NEXT:    [[TMP2:%.*]] = fadd fast <2 x float> [[TMP1]], <float 1.000000e+00, float 2.000000e+00>
+; POW2-ONLY-NEXT:    store <2 x float> [[TMP2]], ptr [[INCDEC_PTR1]], align 4
+; POW2-ONLY-NEXT:    [[TMP3:%.*]] = load float, ptr [[INCDEC_PTR5]], align 4
+; POW2-ONLY-NEXT:    [[ADD9:%.*]] = fadd fast float [[TMP3]], 3.000000e+00
+; POW2-ONLY-NEXT:    store float [[ADD9]], ptr [[INCDEC_PTR7]], align 4
+; POW2-ONLY-NEXT:    ret void
 ;
 entry:
   %incdec.ptr = getelementptr inbounds float, ptr %src, i64 1
@@ -433,9 +467,9 @@ define void @sub0f(ptr noalias %dst, ptr noalias %src) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load float, ptr [[INCDEC_PTR]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 2
 ; CHECK-NEXT:    store float [[TMP1]], ptr [[INCDEC_PTR1]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x float>, ptr [[INCDEC_PTR2]], align 4
-; CHECK-NEXT:    [[TMP4:%.*]] = fadd fast <2 x float> [[TMP3]], <float -2.000000e+00, float -3.000000e+00>
-; CHECK-NEXT:    store <2 x float> [[TMP4]], ptr [[INCDEC_PTR4]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x float>, ptr [[INCDEC_PTR2]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd fast <2 x float> [[TMP2]], <float -2.000000e+00, float -3.000000e+00>
+; CHECK-NEXT:    store <2 x float> [[TMP3]], ptr [[INCDEC_PTR4]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -462,9 +496,9 @@ entry:
 define void @sub1f(ptr noalias %dst, ptr noalias %src) {
 ; CHECK-LABEL: @sub1f(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[SRC:%.*]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = fadd fast <4 x float> [[TMP1]], <float 4.000000e+00, float -1.000000e+00, float -2.000000e+00, float -3.000000e+00>
-; CHECK-NEXT:    store <4 x float> [[TMP2]], ptr [[DST:%.*]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[SRC:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd fast <4 x float> [[TMP0]], <float 4.000000e+00, float -1.000000e+00, float -2.000000e+00, float -3.000000e+00>
+; CHECK-NEXT:    store <4 x float> [[TMP1]], ptr [[DST:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -492,9 +526,9 @@ entry:
 define void @sub2f(ptr noalias %dst, ptr noalias %src) {
 ; CHECK-LABEL: @sub2f(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[SRC:%.*]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = fadd fast <4 x float> [[TMP1]], <float -1.000000e+00, float -1.000000e+00, float -2.000000e+00, float -3.000000e+00>
-; CHECK-NEXT:    store <4 x float> [[TMP2]], ptr [[DST:%.*]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[SRC:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd fast <4 x float> [[TMP0]], <float -1.000000e+00, float -1.000000e+00, float -2.000000e+00, float -3.000000e+00>
+; CHECK-NEXT:    store <4 x float> [[TMP1]], ptr [[DST:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -570,11 +604,11 @@ define void @addsub1f(ptr noalias %dst, ptr noalias %src) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> [[TMP2]], <2 x i32> <i32 0, i32 3>
 ; CHECK-NEXT:    store <2 x float> [[TMP3]], ptr [[DST]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 3
-; CHECK-NEXT:    [[TMP6:%.*]] = load float, ptr [[INCDEC_PTR2]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = load float, ptr [[INCDEC_PTR2]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR6:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 3
-; CHECK-NEXT:    store float [[TMP6]], ptr [[INCDEC_PTR3]], align 4
-; CHECK-NEXT:    [[TMP7:%.*]] = load float, ptr [[INCDEC_PTR4]], align 4
-; CHECK-NEXT:    [[SUB8:%.*]] = fsub fast float [[TMP7]], -3.000000e+00
+; CHECK-NEXT:    store float [[TMP4]], ptr [[INCDEC_PTR3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = load float, ptr [[INCDEC_PTR4]], align 4
+; CHECK-NEXT:    [[SUB8:%.*]] = fsub fast float [[TMP5]], -3.000000e+00
 ; CHECK-NEXT:    store float [[SUB8]], ptr [[INCDEC_PTR6]], align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -604,15 +638,15 @@ define void @mulf(ptr noalias %dst, ptr noalias %src) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i64 2
 ; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, ptr [[DST:%.*]], i64 2
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x float>, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast <2 x float> [[TMP1]], <float 2.570000e+02, float -3.000000e+00>
-; CHECK-NEXT:    store <2 x float> [[TMP2]], ptr [[DST]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x float>, ptr [[SRC]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast <2 x float> [[TMP0]], <float 2.570000e+02, float -3.000000e+00>
+; CHECK-NEXT:    store <2 x float> [[TMP1]], ptr [[DST]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 3
-; CHECK-NEXT:    [[TMP4:%.*]] = load float, ptr [[INCDEC_PTR2]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load float, ptr [[INCDEC_PTR2]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 3
-; CHECK-NEXT:    store float [[TMP4]], ptr [[INCDEC_PTR4]], align 4
-; CHECK-NEXT:    [[TMP5:%.*]] = load float, ptr [[INCDEC_PTR5]], align 4
-; CHECK-NEXT:    [[SUB9:%.*]] = fmul fast float [[TMP5]], -9.000000e+00
+; CHECK-NEXT:    store float [[TMP2]], ptr [[INCDEC_PTR4]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load float, ptr [[INCDEC_PTR5]], align 4
+; CHECK-NEXT:    [[SUB9:%.*]] = fmul fast float [[TMP3]], -9.000000e+00
 ; CHECK-NEXT:    store float [[SUB9]], ptr [[INCDEC_PTR7]], align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -640,9 +674,9 @@ entry:
 define void @add0fn(ptr noalias %dst, ptr noalias %src) {
 ; CHECK-LABEL: @add0fn(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[SRC:%.*]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = fadd <4 x float> [[TMP1]], <float 1.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>
-; CHECK-NEXT:    store <4 x float> [[TMP2]], ptr [[DST:%.*]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[SRC:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd <4 x float> [[TMP0]], <float 1.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>
+; CHECK-NEXT:    store <4 x float> [[TMP1]], ptr [[DST:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -668,21 +702,32 @@ entry:
 }
 
 define void @add1fn(ptr noalias %dst, ptr noalias %src) {
-; CHECK-LABEL: @add1fn(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i64 1
-; CHECK-NEXT:    [[TMP0:%.*]] = load float, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, ptr [[DST:%.*]], i64 1
-; CHECK-NEXT:    store float [[TMP0]], ptr [[DST]], align 4
-; CHECK-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 3
-; CHECK-NEXT:    [[INCDEC_PTR7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 3
-; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x float>, ptr [[INCDEC_PTR]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x float> [[TMP2]], <float 1.000000e+00, float 2.000000e+00>
-; CHECK-NEXT:    store <2 x float> [[TMP3]], ptr [[INCDEC_PTR1]], align 4
-; CHECK-NEXT:    [[TMP5:%.*]] = load float, ptr [[INCDEC_PTR5]], align 4
-; CHECK-NEXT:    [[ADD9:%.*]] = fadd float [[TMP5]], 3.000000e+00
-; CHECK-NEXT:    store float [[ADD9]], ptr [[INCDEC_PTR7]], align 4
-; CHECK-NEXT:    ret void
+; NON-POW2-LABEL: @add1fn(
+; NON-POW2-NEXT:  entry:
+; NON-POW2-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i64 1
+; NON-POW2-NEXT:    [[TMP0:%.*]] = load float, ptr [[SRC]], align 4
+; NON-POW2-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, ptr [[DST:%.*]], i64 1
+; NON-POW2-NEXT:    store float [[TMP0]], ptr [[DST]], align 4
+; NON-POW2-NEXT:    [[TMP1:%.*]] = load <3 x float>, ptr [[INCDEC_PTR]], align 4
+; NON-POW2-NEXT:    [[TMP2:%.*]] = fadd <3 x float> [[TMP1]], <float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>
+; NON-POW2-NEXT:    store <3 x float> [[TMP2]], ptr [[INCDEC_PTR1]], align 4
+; NON-POW2-NEXT:    ret void
+;
+; POW2-ONLY-LABEL: @add1fn(
+; POW2-ONLY-NEXT:  entry:
+; POW2-ONLY-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i64 1
+; POW2-ONLY-NEXT:    [[TMP0:%.*]] = load float, ptr [[SRC]], align 4
+; POW2-ONLY-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, ptr [[DST:%.*]], i64 1
+; POW2-ONLY-NEXT:    store float [[TMP0]], ptr [[DST]], align 4
+; POW2-ONLY-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 3
+; POW2-ONLY-NEXT:    [[INCDEC_PTR7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 3
+; POW2-ONLY-NEXT:    [[TMP1:%.*]] = load <2 x float>, ptr [[INCDEC_PTR]], align 4
+; POW2-ONLY-NEXT:    [[TMP2:%.*]] = fadd <2 x float> [[TMP1]], <float 1.000000e+00, float 2.000000e+00>
+; POW2-ONLY-NEXT:    store <2 x float> [[TMP2]], ptr [[INCDEC_PTR1]], align 4
+; POW2-ONLY-NEXT:    [[TMP3:%.*]] = load float, ptr [[INCDEC_PTR5]], align 4
+; POW2-ONLY-NEXT:    [[ADD9:%.*]] = fadd float [[TMP3]], 3.000000e+00
+; POW2-ONLY-NEXT:    store float [[ADD9]], ptr [[INCDEC_PTR7]], align 4
+; POW2-ONLY-NEXT:    ret void
 ;
 entry:
   %incdec.ptr = getelementptr inbounds float, ptr %src, i64 1
@@ -717,9 +762,9 @@ define void @sub0fn(ptr noalias %dst, ptr noalias %src) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load float, ptr [[INCDEC_PTR]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 2
 ; CHECK-NEXT:    store float [[TMP1]], ptr [[INCDEC_PTR1]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x float>, ptr [[INCDEC_PTR2]], align 4
-; CHECK-NEXT:    [[TMP4:%.*]] = fadd <2 x float> [[TMP3]], <float -2.000000e+00, float -3.000000e+00>
-; CHECK-NEXT:    store <2 x float> [[TMP4]], ptr [[INCDEC_PTR4]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x float>, ptr [[INCDEC_PTR2]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x float> [[TMP2]], <float -2.000000e+00, float -3.000000e+00>
+; CHECK-NEXT:    store <2 x float> [[TMP3]], ptr [[INCDEC_PTR4]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -746,9 +791,9 @@ entry:
 define void @sub1fn(ptr noalias %dst, ptr noalias %src) {
 ; CHECK-LABEL: @sub1fn(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[SRC:%.*]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = fadd <4 x float> [[TMP1]], <float 4.000000e+00, float -1.000000e+00, float -2.000000e+00, float -3.000000e+00>
-; CHECK-NEXT:    store <4 x float> [[TMP2]], ptr [[DST:%.*]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[SRC:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd <4 x float> [[TMP0]], <float 4.000000e+00, float -1.000000e+00, float -2.000000e+00, float -3.000000e+00>
+; CHECK-NEXT:    store <4 x float> [[TMP1]], ptr [[DST:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -776,9 +821,9 @@ entry:
 define void @sub2fn(ptr noalias %dst, ptr noalias %src) {
 ; CHECK-LABEL: @sub2fn(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[SRC:%.*]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = fadd <4 x float> [[TMP1]], <float -1.000000e+00, float -1.000000e+00, float -2.000000e+00, float -3.000000e+00>
-; CHECK-NEXT:    store <4 x float> [[TMP2]], ptr [[DST:%.*]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, ptr [[SRC:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd <4 x float> [[TMP0]], <float -1.000000e+00, float -1.000000e+00, float -2.000000e+00, float -3.000000e+00>
+; CHECK-NEXT:    store <4 x float> [[TMP1]], ptr [[DST:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -808,15 +853,15 @@ define void @mulfn(ptr noalias %dst, ptr noalias %src) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i64 2
 ; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, ptr [[DST:%.*]], i64 2
-; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x float>, ptr [[SRC]], align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = fmul <2 x float> [[TMP1]], <float 2.570000e+02, float -3.000000e+00>
-; CHECK-NEXT:    store <2 x float> [[TMP2]], ptr [[DST]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = load <2 x float>, ptr [[SRC]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <2 x float> [[TMP0]], <float 2.570000e+02, float -3.000000e+00>
+; CHECK-NEXT:    store <2 x float> [[TMP1]], ptr [[DST]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 3
-; CHECK-NEXT:    [[TMP4:%.*]] = load float, ptr [[INCDEC_PTR2]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load float, ptr [[INCDEC_PTR2]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 3
-; CHECK-NEXT:    store float [[TMP4]], ptr [[INCDEC_PTR4]], align 4
-; CHECK-NEXT:    [[TMP5:%.*]] = load float, ptr [[INCDEC_PTR5]], align 4
-; CHECK-NEXT:    [[SUB9:%.*]] = fmul fast float [[TMP5]], -9.000000e+00
+; CHECK-NEXT:    store float [[TMP2]], ptr [[INCDEC_PTR4]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load float, ptr [[INCDEC_PTR5]], align 4
+; CHECK-NEXT:    [[SUB9:%.*]] = fmul fast float [[TMP3]], -9.000000e+00
 ; CHECK-NEXT:    store float [[SUB9]], ptr [[INCDEC_PTR7]], align 4
 ; CHECK-NEXT:    ret void
 ;
-- 
GitLab


From 11f22f1a963ab3c8949cb723a63c07d7a409c8a8 Mon Sep 17 00:00:00 2001
From: Eric <eric@efcs.ca>
Date: Sat, 13 Apr 2024 03:16:11 -0400
Subject: [PATCH 143/464] [tzdb] Replace shared_mutex with mutex. (#87929)

The overhead of taking a std::mutex is much lower than taking a reader
lock on a shared mutex, even under heavy contention.

The benefit of shared_mutex only occurs as the amount of
time spent in the critical sections grows large enough.

In our case all we do is read a pointer and return the lock.
As a result, using a shared lock can be ~50%-100% slower

Here are the results for the provided benchmark on my machine:

```
2024-04-07T12:48:51-04:00
Running ./libcxx/benchmarks/shared_mutex_vs_mutex.libcxx.out
Run on (12 X 400 MHz CPU s)
CPU Caches:
  L1 Data 32 KiB (x6)
  L1 Instruction 32 KiB (x6)
  L2 Unified 1024 KiB (x6)
  L3 Unified 32768 KiB (x1)
Load Average: 2.70, 2.70, 1.63
---------------------------------------------------------------------
Benchmark                           Time             CPU   Iterations
---------------------------------------------------------------------
BM_shared_mutex/threads:1        13.9 ns         13.9 ns     50533700
BM_shared_mutex/threads:2        34.5 ns         68.9 ns      9957784
BM_shared_mutex/threads:4        38.4 ns          137 ns      4987772
BM_shared_mutex/threads:8        51.1 ns          358 ns      1974160
BM_shared_mutex/threads:32       57.1 ns          682 ns      1043648
BM_mutex/threads:1               5.54 ns         5.53 ns    125867422
BM_mutex/threads:2               15.5 ns         30.9 ns     21830116
BM_mutex/threads:4               15.4 ns         57.2 ns     12136920
BM_mutex/threads:8               19.3 ns          140 ns      4997080
BM_mutex/threads:32              20.8 ns          252 ns      2859808
```
---
 libcxx/benchmarks/CMakeLists.txt              |  1 +
 .../shared_mutex_vs_mutex.bench.cpp           | 41 +++++++++++++++++++
 libcxx/src/include/tzdb/tzdb_list_private.h   | 11 +++--
 3 files changed, 49 insertions(+), 4 deletions(-)
 create mode 100644 libcxx/benchmarks/shared_mutex_vs_mutex.bench.cpp

diff --git a/libcxx/benchmarks/CMakeLists.txt b/libcxx/benchmarks/CMakeLists.txt
index 928238c1ac69..527a2acf2d3b 100644
--- a/libcxx/benchmarks/CMakeLists.txt
+++ b/libcxx/benchmarks/CMakeLists.txt
@@ -221,6 +221,7 @@ set(BENCHMARK_TESTS
     map.bench.cpp
     monotonic_buffer.bench.cpp
     ordered_set.bench.cpp
+    shared_mutex_vs_mutex.bench.cpp
     stop_token.bench.cpp
     std_format_spec_string_unicode.bench.cpp
     string.bench.cpp
diff --git a/libcxx/benchmarks/shared_mutex_vs_mutex.bench.cpp b/libcxx/benchmarks/shared_mutex_vs_mutex.bench.cpp
new file mode 100644
index 000000000000..19d13b799cc1
--- /dev/null
+++ b/libcxx/benchmarks/shared_mutex_vs_mutex.bench.cpp
@@ -0,0 +1,41 @@
+//===----------------------------------------------------------------------===//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// This benchmark compares the performance of std::mutex and std::shared_mutex in contended scenarios.
+// it's meant to establish a baseline overhead for std::shared_mutex and std::mutex, and to help inform decisions about
+// which mutex to use when selecting a mutex type for a given use case.
+
+#include <atomic>
+#include <mutex>
+#include <numeric>
+#include <shared_mutex>
+#include <thread>
+
+#include "benchmark/benchmark.h"
+
+int global_value = 42;
+std::mutex m;
+std::shared_mutex sm;
+
+static void BM_shared_mutex(benchmark::State& state) {
+  for (auto _ : state) {
+    std::shared_lock<std::shared_mutex> lock(sm);
+    benchmark::DoNotOptimize(global_value);
+  }
+}
+
+static void BM_mutex(benchmark::State& state) {
+  for (auto _ : state) {
+    std::lock_guard<std::mutex> lock(m);
+    benchmark::DoNotOptimize(global_value);
+  }
+}
+
+BENCHMARK(BM_shared_mutex)->Threads(1)->Threads(2)->Threads(4)->Threads(8)->Threads(32);
+BENCHMARK(BM_mutex)->Threads(1)->Threads(2)->Threads(4)->Threads(8)->Threads(32);
+
+BENCHMARK_MAIN();
diff --git a/libcxx/src/include/tzdb/tzdb_list_private.h b/libcxx/src/include/tzdb/tzdb_list_private.h
index 969b2b9f8a9f..0be7b8e5a100 100644
--- a/libcxx/src/include/tzdb/tzdb_list_private.h
+++ b/libcxx/src/include/tzdb/tzdb_list_private.h
@@ -15,8 +15,11 @@
 #include <forward_list>
 
 // When threads are not available the locking is not required.
+// When threads are available, we use std::mutex over std::shared_mutex
+// due to the increased overhead of std::shared_mutex.
+// See shared_mutex_vs_mutex.bench.cpp
 #ifndef _LIBCPP_HAS_NO_THREADS
-#  include <shared_mutex>
+#  include <mutex>
 #endif
 
 #include "types_private.h"
@@ -56,7 +59,7 @@ public:
 
   const tzdb& __front() const noexcept {
 #ifndef _LIBCPP_HAS_NO_THREADS
-    shared_lock __lock{__mutex_};
+    unique_lock __lock{__mutex_};
 #endif
     return __tzdb_.front();
   }
@@ -72,7 +75,7 @@ public:
 
   const_iterator __begin() const noexcept {
 #ifndef _LIBCPP_HAS_NO_THREADS
-    shared_lock __lock{__mutex_};
+    unique_lock __lock{__mutex_};
 #endif
     return __tzdb_.begin();
   }
@@ -87,7 +90,7 @@ private:
   void __load_no_lock() { chrono::__init_tzdb(__tzdb_.emplace_front(), __rules_.emplace_front()); }
 
 #ifndef _LIBCPP_HAS_NO_THREADS
-  mutable shared_mutex __mutex_;
+  mutable mutex __mutex_;
 #endif
   forward_list<tzdb> __tzdb_;
 
-- 
GitLab


From 9931e7efc6505002e866ee553af5d812123c22f5 Mon Sep 17 00:00:00 2001
From: Hristo Hristov <hghristov.rmm@gmail.com>
Date: Sat, 13 Apr 2024 11:39:38 +0300
Subject: [PATCH 144/464] [libc++][spaceship][NFC] P0905R1: Symmetry for
 spaceship - mark as implemented (#87871)

Implemented: https://wg21.link/P0905R1 in
0913ca197855626c93fdae4184ae80ff5a26503a
- https://eel.is/c++draft/cmp.weakeq (Removed in
https://wg21.link/P1959R0)
- https://eel.is/c++draft/cmp.strongeq (Removed in
https://wg21.link/P1959R0)
- https://eel.is/c++draft/cmp.partialord
-
https://github.com/llvm/llvm-project/blob/4abb722ffa7fcf809faa4a479fdf2f78c685b351/libcxx/include/__compare/ordering.h#L101
- https://eel.is/c++draft/cmp.weakord
-
https://github.com/llvm/llvm-project/blob/4abb722ffa7fcf809faa4a479fdf2f78c685b351/libcxx/include/__compare/ordering.h#L173
- https://eel.is/c++draft/cmp.strongord
-
https://github.com/llvm/llvm-project/blob/4abb722ffa7fcf809faa4a479fdf2f78c685b351/libcxx/include/__compare/ordering.h#L250

References:
- https://wg21.link/P1959R0 - Remove `std::weak_equality` and
`std::strong_equality`
---
 libcxx/docs/Status/Cxx20Papers.csv     | 2 +-
 libcxx/docs/Status/SpaceshipPapers.csv | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/libcxx/docs/Status/Cxx20Papers.csv b/libcxx/docs/Status/Cxx20Papers.csv
index 77078b11a727..6e4678da2888 100644
--- a/libcxx/docs/Status/Cxx20Papers.csv
+++ b/libcxx/docs/Status/Cxx20Papers.csv
@@ -23,7 +23,7 @@
 "`P0754R2 <https://wg21.link/P0754R2>`__","LWG","<version>","Jacksonville","|Complete|","7.0"
 "`P0809R0 <https://wg21.link/P0809R0>`__","LWG","Comparing Unordered Containers","Jacksonville","|Nothing To Do|",""
 "`P0858R0 <https://wg21.link/P0858R0>`__","LWG","Constexpr iterator requirements","Jacksonville","|Complete|","12.0"
-"`P0905R1 <https://wg21.link/P0905R1>`__","CWG","Symmetry for spaceship","Jacksonville","","","|spaceship|"
+"`P0905R1 <https://wg21.link/P0905R1>`__","CWG","Symmetry for spaceship","Jacksonville","|Complete|","7.0","|spaceship|"
 "`P0966R1 <https://wg21.link/P0966R1>`__","LWG","``string::reserve``\  Should Not Shrink","Jacksonville","|Complete| [#note-P0966]_","12.0"
 "","","","","","",""
 "`P0019R8 <https://wg21.link/P0019R8>`__","LWG","Atomic Ref","Rapperswil","",""
diff --git a/libcxx/docs/Status/SpaceshipPapers.csv b/libcxx/docs/Status/SpaceshipPapers.csv
index 21b788ee4292..39e1f968c175 100644
--- a/libcxx/docs/Status/SpaceshipPapers.csv
+++ b/libcxx/docs/Status/SpaceshipPapers.csv
@@ -7,6 +7,6 @@
 `LWG3360 <https://wg21.link/LWG3360>`_,``three_way_comparable_with`` is inconsistent with similar concepts,|Nothing To Do|,
 `LWG3380 <https://wg21.link/LWG3380>`_,``common_type`` and comparison categories,|Nothing To Do|,
 `LWG3395 <https://wg21.link/LWG3395>`_,Definition for *three-way* comparison needs to be updated,|Nothing To Do|,
-`P0905R1 <https://wg21.link/P0905R1>`_,Symmetry for spaceship,,
+`P0905R1 <https://wg21.link/P0905R1>`_,Symmetry for spaceship,|Complete|,7.0
 `P1120R0 <https://wg21.link/P1120R0>`_,Consistency improvements for ``<=>`` and other comparison operators,,
 `LWG3431 <https://wg21.link/LWG3431>`_,``<=>`` for containers should require ``three_way_comparable<T>`` instead of ``<=>``,,
-- 
GitLab


From e481f565286e7fbf5dda9c4a0d00ca1daae8abeb Mon Sep 17 00:00:00 2001
From: Mark de Wever <koraq@xs4all.nl>
Date: Sat, 13 Apr 2024 11:50:02 +0200
Subject: [PATCH 145/464] [lldb][libc++] Adds local_t clock data formatters.
 (#88178)

---
 .../Language/CPlusPlus/CPlusPlusLanguage.cpp  |  25 ++++-
 .../Plugins/Language/CPlusPlus/LibCxx.cpp     |  38 ++++++-
 .../Plugins/Language/CPlusPlus/LibCxx.h       |   8 ++
 .../chrono/TestDataFormatterLibcxxChrono.py   | 104 ++++++++++++++++++
 .../data-formatter-stl/libcxx/chrono/main.cpp |  52 +++++++++
 5 files changed, 221 insertions(+), 6 deletions(-)

diff --git a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
index 4a536096a066..afb683f7d846 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
@@ -1050,7 +1050,7 @@ static void LoadLibCxxFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
                 "libc++ std::chrono::sys_seconds summary provider",
                 "^std::__[[:alnum:]]+::chrono::time_point<"
                 "std::__[[:alnum:]]+::chrono::system_clock, "
-                "std::__[[:alnum:]]+::chrono::duration<long long, "
+                "std::__[[:alnum:]]+::chrono::duration<.*, "
                 "std::__[[:alnum:]]+::ratio<1, 1> "
                 "> >$",
                 eTypeOptionHideChildren | eTypeOptionHideValue |
@@ -1068,6 +1068,29 @@ static void LoadLibCxxFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
                     eTypeOptionCascade,
                 true);
 
+  AddCXXSummary(
+      cpp_category_sp,
+      lldb_private::formatters::LibcxxChronoLocalSecondsSummaryProvider,
+      "libc++ std::chrono::local_seconds summary provider",
+      "^std::__[[:alnum:]]+::chrono::time_point<"
+      "std::__[[:alnum:]]+::chrono::local_t, "
+      "std::__[[:alnum:]]+::chrono::duration<.*, "
+      "std::__[[:alnum:]]+::ratio<1, 1> "
+      "> >$",
+      eTypeOptionHideChildren | eTypeOptionHideValue | eTypeOptionCascade,
+      true);
+  AddCXXSummary(cpp_category_sp,
+                lldb_private::formatters::LibcxxChronoLocalDaysSummaryProvider,
+                "libc++ std::chrono::local_seconds summary provider",
+                "^std::__[[:alnum:]]+::chrono::time_point<"
+                "std::__[[:alnum:]]+::chrono::local_t, "
+                "std::__[[:alnum:]]+::chrono::duration<int, "
+                "std::__[[:alnum:]]+::ratio<86400, 1> "
+                "> >$",
+                eTypeOptionHideChildren | eTypeOptionHideValue |
+                    eTypeOptionCascade,
+                true);
+
   // Chrono calendar types
 
   cpp_category_sp->AddTypeSummary(
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
index d2d50152c07c..e160fd076393 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.cpp
@@ -1087,8 +1087,10 @@ bool lldb_private::formatters::LibcxxWStringViewSummaryProvider(
                                         dataobj, size);
 }
 
-bool lldb_private::formatters::LibcxxChronoSysSecondsSummaryProvider(
-    ValueObject &valobj, Stream &stream, const TypeSummaryOptions &options) {
+static bool
+LibcxxChronoTimePointSecondsSummaryProvider(ValueObject &valobj, Stream &stream,
+                                            const TypeSummaryOptions &options,
+                                            const char *fmt) {
   ValueObjectSP ptr_sp = valobj.GetChildMemberWithName("__d_");
   if (!ptr_sp)
     return false;
@@ -1112,7 +1114,7 @@ bool lldb_private::formatters::LibcxxChronoSysSecondsSummaryProvider(
   else {
     std::array<char, 128> str;
     std::size_t size =
-        std::strftime(str.data(), str.size(), "%FT%H:%M:%SZ", gmtime(&seconds));
+        std::strftime(str.data(), str.size(), fmt, gmtime(&seconds));
     if (size == 0)
       return false;
 
@@ -1123,8 +1125,22 @@ bool lldb_private::formatters::LibcxxChronoSysSecondsSummaryProvider(
   return true;
 }
 
-bool lldb_private::formatters::LibcxxChronoSysDaysSummaryProvider(
+bool lldb_private::formatters::LibcxxChronoSysSecondsSummaryProvider(
     ValueObject &valobj, Stream &stream, const TypeSummaryOptions &options) {
+  return LibcxxChronoTimePointSecondsSummaryProvider(valobj, stream, options,
+                                                     "%FT%H:%M:%SZ");
+}
+
+bool lldb_private::formatters::LibcxxChronoLocalSecondsSummaryProvider(
+    ValueObject &valobj, Stream &stream, const TypeSummaryOptions &options) {
+  return LibcxxChronoTimePointSecondsSummaryProvider(valobj, stream, options,
+                                                     "%FT%H:%M:%S");
+}
+
+static bool
+LibcxxChronoTimepointDaysSummaryProvider(ValueObject &valobj, Stream &stream,
+                                         const TypeSummaryOptions &options,
+                                         const char *fmt) {
   ValueObjectSP ptr_sp = valobj.GetChildMemberWithName("__d_");
   if (!ptr_sp)
     return false;
@@ -1148,7 +1164,7 @@ bool lldb_private::formatters::LibcxxChronoSysDaysSummaryProvider(
 
     std::array<char, 128> str;
     std::size_t size =
-        std::strftime(str.data(), str.size(), "%FZ", gmtime(&seconds));
+        std::strftime(str.data(), str.size(), fmt, gmtime(&seconds));
     if (size == 0)
       return false;
 
@@ -1158,6 +1174,18 @@ bool lldb_private::formatters::LibcxxChronoSysDaysSummaryProvider(
   return true;
 }
 
+bool lldb_private::formatters::LibcxxChronoSysDaysSummaryProvider(
+    ValueObject &valobj, Stream &stream, const TypeSummaryOptions &options) {
+  return LibcxxChronoTimepointDaysSummaryProvider(valobj, stream, options,
+                                                  "%FZ");
+}
+
+bool lldb_private::formatters::LibcxxChronoLocalDaysSummaryProvider(
+    ValueObject &valobj, Stream &stream, const TypeSummaryOptions &options) {
+  return LibcxxChronoTimepointDaysSummaryProvider(valobj, stream, options,
+                                                  "%F");
+}
+
 bool lldb_private::formatters::LibcxxChronoMonthSummaryProvider(
     ValueObject &valobj, Stream &stream, const TypeSummaryOptions &options) {
   // FIXME: These are the names used in the C++20 ostream operator. Since LLVM
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
index d8b807d180e0..8e97174dc307 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
@@ -281,6 +281,14 @@ bool LibcxxChronoSysDaysSummaryProvider(
     ValueObject &valobj, Stream &stream,
     const TypeSummaryOptions &options); // libc++ std::chrono::sys_days
 
+bool LibcxxChronoLocalSecondsSummaryProvider(
+    ValueObject &valobj, Stream &stream,
+    const TypeSummaryOptions &options); // libc++ std::chrono::local_seconds
+
+bool LibcxxChronoLocalDaysSummaryProvider(
+    ValueObject &valobj, Stream &stream,
+    const TypeSummaryOptions &options); // libc++ std::chrono::local_days
+
 bool LibcxxChronoMonthSummaryProvider(
     ValueObject &valobj, Stream &stream,
     const TypeSummaryOptions &options); // libc++ std::chrono::month
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/chrono/TestDataFormatterLibcxxChrono.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/chrono/TestDataFormatterLibcxxChrono.py
index c30631589596..fb35481d5551 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/chrono/TestDataFormatterLibcxxChrono.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/chrono/TestDataFormatterLibcxxChrono.py
@@ -128,6 +128,110 @@ class LibcxxChronoDataFormatterTestCase(TestBase):
             substrs=["sd_max = timestamp=2147483647 days"],
         )
 
+        # local_seconds aliasses
+
+        self.expect(
+            "frame variable ls_tp",
+            substrs=["ls_tp = date/time=1970-01-01T00:00:00 timestamp=0 s"],
+        )
+        self.expect(
+            "frame variable ls_tp_d",
+            substrs=["ls_tp_d = date/time=1970-01-01T00:00:00 timestamp=0 s"],
+        )
+        self.expect(
+            "frame variable ls_tp_d_r",
+            substrs=["ls_tp_d_r = date/time=1970-01-01T00:00:00 timestamp=0 s"],
+        )
+        self.expect(
+            "frame variable ls_tp_d_r2",
+            substrs=["ls_tp_d_r2 = date/time=1970-01-01T00:00:00 timestamp=0 s"],
+        )
+
+        # local_seconds
+
+        self.expect(
+            "frame variable ls_0",
+            substrs=["ls_0 = date/time=1970-01-01T00:00:00 timestamp=0 s"],
+        )
+
+        self.expect(
+            "frame variable ls_neg_date_time",
+            substrs=[
+                "ls_neg_date_time = date/time=-32767-01-01T00:00:00 timestamp=-1096193779200 s"
+            ],
+        )
+        self.expect(
+            "frame variable ls_neg_seconds",
+            substrs=["ls_neg_seconds = timestamp=-1096193779201 s"],
+        )
+
+        self.expect(
+            "frame variable ls_pos_date_time",
+            substrs=[
+                "ls_pos_date_time = date/time=32767-12-31T23:59:59 timestamp=971890963199 s"
+            ],
+        )
+        self.expect(
+            "frame variable ls_pos_seconds",
+            substrs=["ls_pos_seconds = timestamp=971890963200 s"],
+        )
+
+        self.expect(
+            "frame variable ls_min",
+            substrs=["ls_min = timestamp=-9223372036854775808 s"],
+        )
+        self.expect(
+            "frame variable ls_max",
+            substrs=["ls_max = timestamp=9223372036854775807 s"],
+        )
+
+        # local_days aliasses
+
+        self.expect(
+            "frame variable ld_tp",
+            substrs=["ld_tp = date=1970-01-01 timestamp=0 days"],
+        )
+        self.expect(
+            "frame variable ld_tp_d_r",
+            substrs=["ld_tp_d_r = date=1970-01-01 timestamp=0 days"],
+        )
+        self.expect(
+            "frame variable ld_tp_d_r2",
+            substrs=["ld_tp_d_r2 = date=1970-01-01 timestamp=0 days"],
+        )
+
+        # local_days
+
+        self.expect(
+            "frame variable ld_0", substrs=["ld_0 = date=1970-01-01 timestamp=0 days"]
+        )
+        self.expect(
+            "frame variable ld_neg_date",
+            substrs=["ld_neg_date = date=-32767-01-01 timestamp=-12687428 days"],
+        )
+        self.expect(
+            "frame variable ld_neg_days",
+            substrs=["ld_neg_days = timestamp=-12687429 days"],
+        )
+
+        self.expect(
+            "frame variable ld_pos_date",
+            substrs=["ld_pos_date = date=32767-12-31 timestamp=11248737 days"],
+        )
+        self.expect(
+            "frame variable ld_pos_days",
+            substrs=["ld_pos_days = timestamp=11248738 days"],
+        )
+
+        self.expect(
+            "frame variable ld_min",
+            substrs=["ld_min = timestamp=-2147483648 days"],
+        )
+        self.expect(
+            "frame variable ld_max",
+            substrs=["ld_max = timestamp=2147483647 days"],
+        )
+
         self.expect("frame variable d_0", substrs=["d_0 = day=0"])
         self.expect("frame variable d_1", substrs=["d_1 = day=1"])
         self.expect("frame variable d_31", substrs=["d_31 = day=31"])
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/chrono/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/chrono/main.cpp
index 315c88a787d8..0ea42d1a107e 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/chrono/main.cpp
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/chrono/main.cpp
@@ -67,6 +67,58 @@ int main() {
   std::chrono::sys_days sd_max{
       std::chrono::days{std::numeric_limits<int>::max()}};
 
+  // local_seconds aliasses
+  std::chrono::time_point<std::chrono::local_t, std::chrono::seconds> ls_tp{
+      std::chrono::seconds{0}};
+  std::chrono::time_point<std::chrono::local_t,
+                          std::chrono::duration<long long>>
+      ls_tp_d{std::chrono::seconds{0}};
+  std::chrono::time_point<std::chrono::local_t,
+                          std::chrono::duration<long long, std::ratio<1>>>
+      ls_tp_d_r{std::chrono::seconds{0}};
+  std::chrono::time_point<std::chrono::local_t,
+                          std::chrono::duration<long long, std::ratio<1>>>
+      ls_tp_d_r2{std::chrono::seconds{0}};
+
+  // local_seconds
+  std::chrono::local_seconds ls_0{std::chrono::seconds{0}};
+  std::chrono::local_seconds ls_neg_date_time{
+      std::chrono::seconds{-1'096'193'779'200}};
+  std::chrono::local_seconds ls_neg_seconds{
+      std::chrono::seconds{-1'096'193'779'201}};
+  std::chrono::local_seconds ls_pos_date_time{
+      std::chrono::seconds{971'890'963'199}};
+  std::chrono::local_seconds ls_pos_seconds{
+      std::chrono::seconds{971'890'963'200}};
+  std::chrono::local_seconds ls_min{
+      std::chrono::seconds{std::numeric_limits<long long>::min()}};
+  std::chrono::local_seconds ls_max{
+      std::chrono::seconds{std::numeric_limits<long long>::max()}};
+
+  // local_days aliasses
+  std::chrono::time_point<std::chrono::local_t, std::chrono::days> ld_tp{
+      std::chrono::days{0}};
+  std::chrono::time_point<std::chrono::local_t,
+                          std::chrono::duration<int, std::ratio<86400>>>
+      ld_tp_d_r{std::chrono::days{0}};
+  std::chrono::time_point<std::chrono::local_t,
+                          std::chrono::duration<int, std::ratio<86400, 1>>>
+      ld_tp_d_r2{std::chrono::days{0}};
+
+  // local_days
+  std::chrono::local_days ld_0{std::chrono::days{0}};
+
+  std::chrono::local_days ld_neg_date{std::chrono::days{-12'687'428}};
+  std::chrono::local_days ld_neg_days{std::chrono::days{-12'687'429}};
+
+  std::chrono::local_days ld_pos_date{std::chrono::days{11'248'737}};
+  std::chrono::local_days ld_pos_days{std::chrono::days{11'248'738}};
+
+  std::chrono::local_days ld_min{
+      std::chrono::days{std::numeric_limits<int>::min()}};
+  std::chrono::local_days ld_max{
+      std::chrono::days{std::numeric_limits<int>::max()}};
+
   std::chrono::day d_0{0};
   std::chrono::day d_1{1};
   std::chrono::day d_31{31};
-- 
GitLab


From a33a754cff02fda7520fa63fbc6c85c274a44baa Mon Sep 17 00:00:00 2001
From: Mark de Wever <koraq@xs4all.nl>
Date: Sat, 13 Apr 2024 12:03:18 +0200
Subject: [PATCH 146/464] [lldb] Fixes comment typos.

---
 lldb/source/Plugins/Language/CPlusPlus/LibCxxSliceArray.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxSliceArray.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxSliceArray.cpp
index 32e67d2e38c5..c33d8e91af70 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxxSliceArray.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxSliceArray.cpp
@@ -73,9 +73,9 @@ private:
   size_t m_size = 0;
   /// slice_array.__stride_.
   size_t m_stride = 0;
-  /// The type of slize_array's template argument T.
+  /// The type of slice_array's template argument T.
   CompilerType m_element_type;
-  /// The sizeof slize_array's template argument T.
+  /// The sizeof slice_array's template argument T.
   uint32_t m_element_size = 0;
 };
 
-- 
GitLab


From 0412a8651aa6cbdd697e904a758e0f95e6635cee Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Sat, 13 Apr 2024 13:10:09 +0300
Subject: [PATCH 147/464] [clang] Add missing documentation entry for
 `__is_pointer_interconvertible_base_of()`

---
 clang/docs/LanguageExtensions.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst
index 7b23e4d1c2f3..96691b45d63a 100644
--- a/clang/docs/LanguageExtensions.rst
+++ b/clang/docs/LanguageExtensions.rst
@@ -1610,6 +1610,7 @@ The following type trait primitives are supported by Clang. Those traits marked
 * ``__is_pod`` (C++, GNU, Microsoft, Embarcadero):
   Note, the corresponding standard trait was deprecated in C++20.
 * ``__is_pointer`` (C++, Embarcadero)
+* ``__is_pointer_interconvertible_base_of`` (C++, GNU, Microsoft)
 * ``__is_polymorphic`` (C++, GNU, Microsoft, Embarcadero)
 * ``__is_reference`` (C++, Embarcadero)
 * ``__is_referenceable`` (C++, GNU, Microsoft, Embarcadero):
-- 
GitLab


From 6704faf6f8a26c21c71fd0efad6af6a8d6ee52b4 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Sat, 13 Apr 2024 11:12:33 +0100
Subject: [PATCH 148/464] [SLP] Use StoreTy to compute min VF.

This ensures that MinVF is a power-of-2, even if ValueTy's width is
not a power-of-2.

This should fix a number of buildbot failures with X86 bootstrapping.
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    |   2 +-
 .../trunc-store-value-ty-not-power-of-2.ll    | 109 ++++++++++++++++++
 2 files changed, 110 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/trunc-store-value-ty-not-power-of-2.ll

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 09db50132c89..b031b40a978f 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -15137,7 +15137,7 @@ bool SLPVectorizerPass::vectorizeStores(ArrayRef<StoreInst *> Stores,
       if (auto *Trunc = dyn_cast<TruncInst>(Store->getValueOperand()))
         ValueTy = Trunc->getSrcTy();
       unsigned MinVF = TTI->getStoreMinimumVF(
-          R.getMinVF(DL->getTypeSizeInBits(ValueTy)), StoreTy, ValueTy);
+          R.getMinVF(DL->getTypeSizeInBits(StoreTy)), StoreTy, ValueTy);
 
       if (MaxVF < MinVF) {
         LLVM_DEBUG(dbgs() << "SLP: Vectorization infeasible as MaxVF (" << MaxVF
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/trunc-store-value-ty-not-power-of-2.ll b/llvm/test/Transforms/SLPVectorizer/X86/trunc-store-value-ty-not-power-of-2.ll
new file mode 100644
index 000000000000..81b4ee40e7fd
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/trunc-store-value-ty-not-power-of-2.ll
@@ -0,0 +1,109 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -p slp-vectorizer -S %s | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @test_2_i24_stores(ptr %A) {
+; CHECK-LABEL: define void @test_2_i24_stores(
+; CHECK-SAME: ptr [[A:%.*]]) {
+; CHECK-NEXT:    [[L:%.*]] = load i24, ptr [[A]], align 4
+; CHECK-NEXT:    store i24 [[L]], ptr [[A]], align 1
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i24, ptr [[A]], i64 1
+; CHECK-NEXT:    store i24 0, ptr [[GEP]], align 1
+; CHECK-NEXT:    ret void
+;
+  %l = load i24, ptr %A
+  store i24 %l, ptr %A, align 1
+  %gep = getelementptr i24, ptr %A, i64 1
+  store i24 0, ptr %gep, align 1
+  ret void
+}
+
+define void @test_2_trunc_i24_to_i8(i24 %x, ptr %A) {
+; CHECK-LABEL: define void @test_2_trunc_i24_to_i8(
+; CHECK-SAME: i24 [[X:%.*]], ptr [[A:%.*]]) {
+; CHECK-NEXT:    [[T:%.*]] = trunc i24 [[X]] to i8
+; CHECK-NEXT:    store i8 [[T]], ptr [[A]], align 1
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i8, ptr [[A]], i64 1
+; CHECK-NEXT:    store i8 0, ptr [[GEP]], align 1
+; CHECK-NEXT:    ret void
+;
+  %t = trunc i24 %x to i8
+  store i8 %t, ptr %A, align 1
+  %gep = getelementptr i8, ptr %A, i64 1
+  store i8 0, ptr %gep, align 1
+  ret void
+}
+
+define void @test_4_trunc_i24_to_i8(i24 %x, ptr %A) {
+; CHECK-LABEL: define void @test_4_trunc_i24_to_i8(
+; CHECK-SAME: i24 [[X:%.*]], ptr [[A:%.*]]) {
+; CHECK-NEXT:    [[T:%.*]] = trunc i24 [[X]] to i8
+; CHECK-NEXT:    store i8 [[T]], ptr [[A]], align 1
+; CHECK-NEXT:    [[GEP_1:%.*]] = getelementptr i8, ptr [[A]], i64 1
+; CHECK-NEXT:    store i8 [[T]], ptr [[GEP_1]], align 1
+; CHECK-NEXT:    [[GEP_2:%.*]] = getelementptr i8, ptr [[A]], i64 2
+; CHECK-NEXT:    store i8 [[T]], ptr [[GEP_2]], align 1
+; CHECK-NEXT:    [[GEP_3:%.*]] = getelementptr i8, ptr [[A]], i64 3
+; CHECK-NEXT:    store i8 [[T]], ptr [[GEP_3]], align 1
+; CHECK-NEXT:    ret void
+;
+  %t = trunc i24 %x to i8
+  store i8 %t, ptr %A, align 1
+  %gep.1 = getelementptr i8, ptr %A, i64 1
+  store i8 %t, ptr %gep.1, align 1
+  %gep.2 = getelementptr i8, ptr %A, i64 2
+  store i8 %t, ptr %gep.2, align 1
+  %gep.3 = getelementptr i8, ptr %A, i64 3
+  store i8 %t, ptr %gep.3, align 1
+  ret void
+}
+
+define void @test_8_trunc_i24_to_i8(i24 %x, ptr %A) {
+; CHECK-LABEL: define void @test_8_trunc_i24_to_i8(
+; CHECK-SAME: i24 [[X:%.*]], ptr [[A:%.*]]) {
+; CHECK-NEXT:    [[T:%.*]] = trunc i24 [[X]] to i8
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <8 x i8> poison, i8 [[T]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <8 x i8> [[TMP1]], <8 x i8> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT:    store <8 x i8> [[TMP2]], ptr [[A]], align 1
+; CHECK-NEXT:    ret void
+;
+  %t = trunc i24 %x to i8
+  store i8 %t, ptr %A, align 1
+  %gep.1 = getelementptr i8, ptr %A, i64 1
+  store i8 %t, ptr %gep.1, align 1
+  %gep.2 = getelementptr i8, ptr %A, i64 2
+  store i8 %t, ptr %gep.2, align 1
+  %gep.3 = getelementptr i8, ptr %A, i64 3
+  store i8 %t, ptr %gep.3, align 1
+  %gep.4 = getelementptr i8, ptr %A, i64 4
+  store i8 %t, ptr %gep.4, align 1
+  %gep.5 = getelementptr i8, ptr %A, i64 5
+  store i8 %t, ptr %gep.5, align 1
+  %gep.6 = getelementptr i8, ptr %A, i64 6
+  store i8 %t, ptr %gep.6, align 1
+  %gep.7 = getelementptr i8, ptr %A, i64 7
+  store i8 %t, ptr %gep.7, align 1
+  ret void
+}
+
+define void @test_4_trunc_i24_to_i16(i24 %x, ptr %A) {
+; CHECK-LABEL: define void @test_4_trunc_i24_to_i16(
+; CHECK-SAME: i24 [[X:%.*]], ptr [[A:%.*]]) {
+; CHECK-NEXT:    [[T:%.*]] = trunc i24 [[X]] to i16
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i16> poison, i16 [[T]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i16> [[TMP1]], <4 x i16> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    store <4 x i16> [[TMP2]], ptr [[A]], align 1
+; CHECK-NEXT:    ret void
+;
+  %t = trunc i24 %x to i16
+  store i16 %t, ptr %A, align 1
+  %gep.1 = getelementptr i16, ptr %A, i64 1
+  store i16 %t, ptr %gep.1, align 1
+  %gep.2 = getelementptr i16, ptr %A, i64 2
+  store i16 %t, ptr %gep.2, align 1
+  %gep.3 = getelementptr i16, ptr %A, i64 3
+  store i16 %t, ptr %gep.3, align 1
+  ret void
+}
-- 
GitLab


From 6775285e7695f2d45cf455f5d31b2c9fa9362d3d Mon Sep 17 00:00:00 2001
From: Mark de Wever <koraq@xs4all.nl>
Date: Sat, 13 Apr 2024 12:32:46 +0200
Subject: [PATCH 149/464] [libc++][CMake] Removes LIBCXX_ENABLE_CLANG_TIDY.
 (#85794)

The clang-tidy selection in CMake was refactored in
https://github.com/llvm/llvm-project/pull/81362. During review it was
suggested to remove this CMake option.
---
 libcxx/CMakeLists.txt                         |  5 --
 libcxx/docs/ReleaseNotes/19.rst               |  3 +
 libcxx/test/tools/CMakeLists.txt              | 12 ++--
 .../tools/clang_tidy_checks/CMakeLists.txt    | 56 ++++++++++++++-----
 libcxx/utils/ci/buildkite-pipeline.yml        |  1 -
 libcxx/utils/ci/run-buildbot                  |  8 ---
 6 files changed, 48 insertions(+), 37 deletions(-)

diff --git a/libcxx/CMakeLists.txt b/libcxx/CMakeLists.txt
index e565c47c7668..043d5a8295c1 100644
--- a/libcxx/CMakeLists.txt
+++ b/libcxx/CMakeLists.txt
@@ -123,7 +123,6 @@ option(LIBCXX_ENABLE_VENDOR_AVAILABILITY_ANNOTATIONS
    to provide compile-time errors when using features unavailable on some version of
    the shared library they shipped should turn this on and see `include/__availability`
    for more details." OFF)
-option(LIBCXX_ENABLE_CLANG_TIDY "Whether to compile and run clang-tidy checks" OFF)
 
 if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
   set(LIBCXX_DEFAULT_TEST_CONFIG "llvm-libc++-shared-gcc.cfg.in")
@@ -863,10 +862,6 @@ add_subdirectory(modules)
 
 set(LIBCXX_TEST_DEPS "cxx_experimental")
 
-if (LIBCXX_ENABLE_CLANG_TIDY)
-  list(APPEND LIBCXX_TEST_DEPS cxx-tidy)
-endif()
-
 list(APPEND LIBCXX_TEST_DEPS generate-cxx-modules)
 
 if (LIBCXX_INCLUDE_BENCHMARKS)
diff --git a/libcxx/docs/ReleaseNotes/19.rst b/libcxx/docs/ReleaseNotes/19.rst
index 7bc0148c9ff0..390de789ddb6 100644
--- a/libcxx/docs/ReleaseNotes/19.rst
+++ b/libcxx/docs/ReleaseNotes/19.rst
@@ -132,3 +132,6 @@ Build System Changes
 
 - The ``LIBCXX_EXECUTOR`` and ``LIBCXXABI_EXECUTOR`` CMake variables have been removed. Please
   set ``LIBCXX_TEST_PARAMS`` to ``executor=<...>`` instead.
+
+- The Cmake variable ``LIBCXX_ENABLE_CLANG_TIDY`` has been removed. The build system has been changed
+  to automatically detect the presence of ``clang-tidy`` and the required ``Clang`` libraries.
diff --git a/libcxx/test/tools/CMakeLists.txt b/libcxx/test/tools/CMakeLists.txt
index e30ad6cdd820..6d99c53ad46d 100644
--- a/libcxx/test/tools/CMakeLists.txt
+++ b/libcxx/test/tools/CMakeLists.txt
@@ -1,12 +1,8 @@
 
 set(LIBCXX_TEST_TOOLS_PATH ${CMAKE_CURRENT_BINARY_DIR} PARENT_SCOPE)
 
-# TODO: Remove LIBCXX_ENABLE_CLANG_TIDY
-if(LIBCXX_ENABLE_CLANG_TIDY)
-  if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
-    message(STATUS "Clang-tidy can only be used when building libc++ with "
-                   "a clang compiler.")
-    return()
-  endif()
-  add_subdirectory(clang_tidy_checks)
+if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
+  message(STATUS "Clang-tidy tests are disabled due to non-clang based compiler.")
+  return()
 endif()
+add_subdirectory(clang_tidy_checks)
diff --git a/libcxx/test/tools/clang_tidy_checks/CMakeLists.txt b/libcxx/test/tools/clang_tidy_checks/CMakeLists.txt
index 74905a0c3ed1..28eed6144583 100644
--- a/libcxx/test/tools/clang_tidy_checks/CMakeLists.txt
+++ b/libcxx/test/tools/clang_tidy_checks/CMakeLists.txt
@@ -9,22 +9,14 @@ set(Clang_DIR_SAVE ${Clang_DIR})
 # versions must match. Otherwise there likely will be ODR-violations. This had
 # led to crashes and incorrect output of the clang-tidy based checks.
 find_package(Clang ${CMAKE_CXX_COMPILER_VERSION})
-
-set(SOURCES
-    abi_tag_on_virtual.cpp
-    header_exportable_declarations.cpp
-    hide_from_abi.cpp
-    proper_version_checks.cpp
-    qualify_declval.cpp
-    robust_against_adl.cpp
-    uglify_attributes.cpp
-
-    libcpp_module.cpp
-   )
-
 if(NOT Clang_FOUND)
-  message(STATUS "Could not find a suitable version of the Clang development package;
-                  custom libc++ clang-tidy checks will not be available.")
+  message(STATUS "Clang-tidy tests are disabled since the "
+                 "Clang development package is unavailable.")
+  return()
+endif()
+if(NOT TARGET clangTidy)
+  message(STATUS "Clang-tidy tests are disabled since the "
+                 "Clang development package has no clangTidy target.")
   return()
 endif()
 
@@ -54,6 +46,38 @@ if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
                         )
 endif()
 
+# In some cases even with the clangTidy target present the headers appear not to
+# be on the system. Run a short test to see whether the header is present.
+file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/test.cpp" "
+#if !__has_include(\"clang-tidy/ClangTidyCheck.h\")
+  # error No clang-tidy headers
+#endif
+int main(){}
+")
+try_compile(HAS_CLANG_TIDY_HEADERS
+  "${CMAKE_CURRENT_BINARY_DIR}"
+  "${CMAKE_CURRENT_BINARY_DIR}/test.cpp"
+   LINK_LIBRARIES clangTidy)
+
+if(NOT HAS_CLANG_TIDY_HEADERS)
+  message(STATUS "Clang-tidy tests are disabled since the "
+                 "clang-tidy headers are not present.")
+  return()
+endif()
+message(STATUS "Clang-tidy tests are enabled.")
+
+set(SOURCES
+    abi_tag_on_virtual.cpp
+    header_exportable_declarations.cpp
+    hide_from_abi.cpp
+    proper_version_checks.cpp
+    qualify_declval.cpp
+    robust_against_adl.cpp
+    uglify_attributes.cpp
+
+    libcpp_module.cpp
+   )
+
 add_library(cxx-tidy MODULE ${SOURCES})
 target_link_libraries(cxx-tidy clangTidy)
 
@@ -64,3 +88,5 @@ set_target_properties(cxx-tidy PROPERTIES
 
 set_target_properties(cxx-tidy PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
 set(CMAKE_SHARED_MODULE_SUFFIX_CXX .plugin) # Use a portable suffix to simplify how we can find it from Lit
+
+list(APPEND LIBCXX_TEST_DEPS cxx-tidy)
diff --git a/libcxx/utils/ci/buildkite-pipeline.yml b/libcxx/utils/ci/buildkite-pipeline.yml
index c43e41441872..4bacdec8f8d6 100644
--- a/libcxx/utils/ci/buildkite-pipeline.yml
+++ b/libcxx/utils/ci/buildkite-pipeline.yml
@@ -43,7 +43,6 @@ definitions:
 
 environment_definitions:
   _common_env: &common_env
-      ENABLE_CLANG_TIDY: "On"
       LLVM_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer-${LLVM_HEAD_VERSION}"
       CLANG_CRASH_DIAGNOSTICS_DIR: "crash_diagnostics"
       CC: clang-${LLVM_HEAD_VERSION}
diff --git a/libcxx/utils/ci/run-buildbot b/libcxx/utils/ci/run-buildbot
index a6f3eb174308..cc72f4639b1a 100755
--- a/libcxx/utils/ci/run-buildbot
+++ b/libcxx/utils/ci/run-buildbot
@@ -44,9 +44,6 @@ CMAKE               The CMake binary to use. This variable is optional.
 CLANG_FORMAT        The clang-format binary to use when generating the format
                     ignore list.
 
-ENABLE_CLANG_TIDY   Whether to compile and run clang-tidy checks. This variable
-                    is optional.
-
 EOF
 }
 
@@ -111,10 +108,6 @@ function clean() {
     rm -rf "${BUILD_DIR}"
 }
 
-if [ -z "${ENABLE_CLANG_TIDY}" ]; then
-    ENABLE_CLANG_TIDY=Off
-fi
-
 function generate-cmake-base() {
     echo "--- Generating CMake"
     ${CMAKE} \
@@ -126,7 +119,6 @@ function generate-cmake-base() {
           -DLIBCXX_ENABLE_WERROR=YES \
           -DLIBCXXABI_ENABLE_WERROR=YES \
           -DLIBUNWIND_ENABLE_WERROR=YES \
-          -DLIBCXX_ENABLE_CLANG_TIDY=${ENABLE_CLANG_TIDY} \
           -DLLVM_LIT_ARGS="-sv --xunit-xml-output test-results.xml --timeout=1500 --time-tests" \
           "${@}"
 }
-- 
GitLab


From f6ca37bf1809b2fa8f6615d4a30eadf8f479c700 Mon Sep 17 00:00:00 2001
From: paperchalice <liujunchang97@outlook.com>
Date: Sat, 13 Apr 2024 18:45:37 +0800
Subject: [PATCH 150/464] [NewPM] Set diagnostic handler in
 `MachineModuleAnalysis` (#88229)

`setDiagnosticHandler` is idempotent.
---
 llvm/lib/CodeGen/MachineModuleInfo.cpp     | 12 +++++-
 llvm/unittests/CodeGen/PassManagerTest.cpp | 50 +++++++++++++++++++++-
 2 files changed, 60 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/CodeGen/MachineModuleInfo.cpp b/llvm/lib/CodeGen/MachineModuleInfo.cpp
index f24288bb69de..1dba591d02b6 100644
--- a/llvm/lib/CodeGen/MachineModuleInfo.cpp
+++ b/llvm/lib/CodeGen/MachineModuleInfo.cpp
@@ -213,7 +213,6 @@ static unsigned getLocCookie(const SMDiagnostic &SMD, const SourceMgr &SrcMgr,
 bool MachineModuleInfoWrapperPass::doInitialization(Module &M) {
   MMI.initialize();
   MMI.TheModule = &M;
-  // FIXME: Do this for new pass manager.
   LLVMContext &Ctx = M.getContext();
   MMI.getContext().setDiagnosticHandler(
       [&Ctx, &M](const SMDiagnostic &SMD, bool IsInlineAsm,
@@ -240,6 +239,17 @@ AnalysisKey MachineModuleAnalysis::Key;
 MachineModuleAnalysis::Result
 MachineModuleAnalysis::run(Module &M, ModuleAnalysisManager &) {
   MMI.TheModule = &M;
+  LLVMContext &Ctx = M.getContext();
+  MMI.getContext().setDiagnosticHandler(
+      [&Ctx, &M](const SMDiagnostic &SMD, bool IsInlineAsm,
+                 const SourceMgr &SrcMgr,
+                 std::vector<const MDNode *> &LocInfos) {
+        unsigned LocCookie = 0;
+        if (IsInlineAsm)
+          LocCookie = getLocCookie(SMD, SrcMgr, LocInfos);
+        Ctx.diagnose(
+            DiagnosticInfoSrcMgr(SMD, M.getName(), IsInlineAsm, LocCookie));
+      });
   MMI.DbgInfoAvailable =
       !DisableDebugInfoPrinting && !M.debug_compile_units().empty();
   return Result(MMI);
diff --git a/llvm/unittests/CodeGen/PassManagerTest.cpp b/llvm/unittests/CodeGen/PassManagerTest.cpp
index 4283eb01a9c8..71f8832d4365 100644
--- a/llvm/unittests/CodeGen/PassManagerTest.cpp
+++ b/llvm/unittests/CodeGen/PassManagerTest.cpp
@@ -125,6 +125,15 @@ struct TestMachineModulePass : public PassInfoMixin<TestMachineModulePass> {
   std::vector<int> &Counts;
 };
 
+struct ReportWarningPass : public PassInfoMixin<ReportWarningPass> {
+  PreservedAnalyses run(MachineFunction &MF,
+                        MachineFunctionAnalysisManager &MFAM) {
+    auto &Ctx = MF.getContext();
+    Ctx.reportWarning(SMLoc(), "Test warning message.");
+    return PreservedAnalyses::all();
+  }
+};
+
 std::unique_ptr<Module> parseIR(LLVMContext &Context, const char *IR) {
   SMDiagnostic Err;
   return parseAssemblyString(IR, Err, Context);
@@ -176,10 +185,10 @@ TEST_F(PassManagerTest, Basic) {
   MachineModuleInfo MMI(LLVMTM);
 
   LoopAnalysisManager LAM;
+  MachineFunctionAnalysisManager MFAM;
   FunctionAnalysisManager FAM;
   CGSCCAnalysisManager CGAM;
   ModuleAnalysisManager MAM;
-  MachineFunctionAnalysisManager MFAM;
   PassBuilder PB(TM.get());
   PB.registerModuleAnalyses(MAM);
   PB.registerCGSCCAnalyses(CGAM);
@@ -204,10 +213,49 @@ TEST_F(PassManagerTest, Basic) {
   MFPM.addPass(TestMachineFunctionPass(Count, Counts));
   MPM.addPass(createModuleToMachineFunctionPassAdaptor(std::move(MFPM)));
 
+  testing::internal::CaptureStderr();
   MPM.run(*M, MAM);
+  std::string Output = testing::internal::GetCapturedStderr();
 
   EXPECT_EQ((std::vector<int>{10, 16, 18, 20, 30, 36, 38, 40}), Counts);
   EXPECT_EQ(40, Count);
 }
 
+TEST_F(PassManagerTest, DiagnosticHandler) {
+  if (!TM)
+    GTEST_SKIP();
+
+  LLVMTargetMachine *LLVMTM = static_cast<LLVMTargetMachine *>(TM.get());
+  M->setDataLayout(TM->createDataLayout());
+
+  MachineModuleInfo MMI(LLVMTM);
+
+  LoopAnalysisManager LAM;
+  MachineFunctionAnalysisManager MFAM;
+  FunctionAnalysisManager FAM;
+  CGSCCAnalysisManager CGAM;
+  ModuleAnalysisManager MAM;
+  PassBuilder PB(TM.get());
+  PB.registerModuleAnalyses(MAM);
+  PB.registerCGSCCAnalyses(CGAM);
+  PB.registerFunctionAnalyses(FAM);
+  PB.registerLoopAnalyses(LAM);
+  PB.registerMachineFunctionAnalyses(MFAM);
+  PB.crossRegisterProxies(LAM, FAM, CGAM, MAM, &MFAM);
+
+  MAM.registerPass([&] { return MachineModuleAnalysis(MMI); });
+
+  ModulePassManager MPM;
+  FunctionPassManager FPM;
+  MachineFunctionPassManager MFPM;
+  MFPM.addPass(ReportWarningPass());
+  MPM.addPass(createModuleToMachineFunctionPassAdaptor(std::move(MFPM)));
+  testing::internal::CaptureStderr();
+  MPM.run(*M, MAM);
+  std::string Output = testing::internal::GetCapturedStderr();
+
+  EXPECT_TRUE(Output.find("warning: <unknown>:0: Test warning message.") !=
+              std::string::npos);
+}
+
 } // namespace
-- 
GitLab


From 844b532713986999aa1ffed0883eff2d1339ec7a Mon Sep 17 00:00:00 2001
From: Andreas Jonson <andjo403@hotmail.com>
Date: Sat, 13 Apr 2024 13:01:36 +0200
Subject: [PATCH 151/464] [MergeFunc]  Handle ConstantRange attributes (#88584)

It is possible to update
[AttributeImpl::operator<](https://github.com/andjo403/llvm-project/blob/a9da350aadfb5c86d36ae18398471558b22c1309/llvm/lib/IR/Attributes.cpp#L744)
instead but feels strange to say that a range is less then an other
range.
---
 .../Transforms/Utils/FunctionComparator.cpp   |  12 ++
 .../call-and-invoke-with-ranges-attr.ll       | 111 ++++++++++++++++++
 2 files changed, 123 insertions(+)
 create mode 100644 llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges-attr.ll

diff --git a/llvm/lib/Transforms/Utils/FunctionComparator.cpp b/llvm/lib/Transforms/Utils/FunctionComparator.cpp
index 8cfb8ed07d24..67aeba7048f8 100644
--- a/llvm/lib/Transforms/Utils/FunctionComparator.cpp
+++ b/llvm/lib/Transforms/Utils/FunctionComparator.cpp
@@ -143,6 +143,18 @@ int FunctionComparator::cmpAttrs(const AttributeList L,
         if (int Res = cmpNumbers((uint64_t)TyL, (uint64_t)TyR))
           return Res;
         continue;
+      } else if (LA.isConstantRangeAttribute() &&
+                 RA.isConstantRangeAttribute()) {
+        if (LA.getKindAsEnum() != RA.getKindAsEnum())
+          return cmpNumbers(LA.getKindAsEnum(), RA.getKindAsEnum());
+
+        ConstantRange LCR = LA.getRange();
+        ConstantRange RCR = RA.getRange();
+        if (int Res = cmpAPInts(LCR.getLower(), RCR.getLower()))
+          return Res;
+        if (int Res = cmpAPInts(LCR.getUpper(), RCR.getUpper()))
+          return Res;
+        continue;
       }
       if (LA < RA)
         return -1;
diff --git a/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges-attr.ll b/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges-attr.ll
new file mode 100644
index 000000000000..e5d62319bf9d
--- /dev/null
+++ b/llvm/test/Transforms/MergeFunc/call-and-invoke-with-ranges-attr.ll
@@ -0,0 +1,111 @@
+; RUN: opt -passes=mergefunc -S < %s | FileCheck %s
+
+define i8 @call_with_range_attr(i8 range(i8 0, 2) %v) {
+  %out = call i8 @dummy2(i8 %v)
+  ret i8 %out
+}
+
+define i8 @call_no_range_attr(i8 %v) {
+; CHECK-LABEL: @call_no_range_attr
+; CHECK-NEXT: %out = call i8 @dummy2(i8 %v)
+; CHECK-NEXT: ret i8 %out
+  %out = call i8 @dummy2(i8 %v)
+  ret i8 %out
+}
+
+define i8 @call_different_range_attr(i8 range(i8 5, 7) %v) {
+; CHECK-LABEL: @call_different_range_attr
+; CHECK-NEXT: %out = call i8 @dummy2(i8 %v)
+; CHECK-NEXT: ret i8 %out
+  %out = call i8 @dummy2(i8 %v)
+  ret i8 %out
+}
+
+define i8 @call_with_range() {
+  %out = call range(i8 0, 2) i8 @dummy()
+  ret i8 %out
+}
+
+define i8 @call_no_range() {
+; CHECK-LABEL: @call_no_range
+; CHECK-NEXT: %out = call i8 @dummy()
+; CHECK-NEXT: ret i8 %out
+  %out = call i8 @dummy()
+  ret i8 %out
+}
+
+define i8 @call_different_range() {
+; CHECK-LABEL: @call_different_range
+; CHECK-NEXT: %out = call range(i8 5, 7) i8 @dummy()
+; CHECK-NEXT: ret i8 %out
+  %out = call range(i8 5, 7) i8 @dummy()
+  ret i8 %out
+}
+
+define i8 @invoke_with_range() personality ptr undef {
+  %out = invoke range(i8 0, 2) i8 @dummy() to label %next unwind label %lpad
+
+next:
+  ret i8 %out
+
+lpad:
+  %pad = landingpad { ptr, i32 } cleanup
+  resume { ptr, i32 } zeroinitializer
+}
+
+define i8 @invoke_no_range() personality ptr undef {
+; CHECK-LABEL: @invoke_no_range()
+; CHECK-NEXT: invoke i8 @dummy
+  %out = invoke i8 @dummy() to label %next unwind label %lpad
+
+next:
+  ret i8 %out
+
+lpad:
+  %pad = landingpad { ptr, i32 } cleanup
+  resume { ptr, i32 } zeroinitializer
+}
+
+define i8 @invoke_different_range() personality ptr undef {
+; CHECK-LABEL: @invoke_different_range()
+; CHECK-NEXT: invoke range(i8 5, 7) i8 @dummy
+  %out = invoke range(i8 5, 7) i8 @dummy() to label %next unwind label %lpad
+
+next:
+  ret i8 %out
+
+lpad:
+  %pad = landingpad { ptr, i32 } cleanup
+  resume { ptr, i32 } zeroinitializer
+}
+
+define i8 @invoke_with_same_range() personality ptr undef {
+; CHECK-LABEL: @invoke_with_same_range()
+; CHECK: tail call i8 @invoke_with_range()
+  %out = invoke range(i8 0, 2) i8 @dummy() to label %next unwind label %lpad
+
+next:
+  ret i8 %out
+
+lpad:
+  %pad = landingpad { ptr, i32 } cleanup
+  resume { ptr, i32 } zeroinitializer
+}
+
+define i8 @call_with_same_range() {
+; CHECK-LABEL: @call_with_same_range
+; CHECK: tail call i8 @call_with_range
+  %out = call range(i8 0, 2) i8 @dummy()
+  ret i8 %out
+}
+
+define i8 @call_with_same_range_attr(i8 range(i8 0, 2) %v) {
+; CHECK-LABEL: @call_with_same_range_attr
+; CHECK: tail call i8 @call_with_range_attr
+  %out = call i8 @dummy2(i8 %v)
+  ret i8 %out
+}
+
+declare i8 @dummy();
+declare i8 @dummy2(i8);
+declare i32 @__gxx_personality_v0(...)
-- 
GitLab


From 1a895bd95ea741dd4756121244f7c2b59e693f1c Mon Sep 17 00:00:00 2001
From: Mark de Wever <koraq@xs4all.nl>
Date: Sat, 13 Apr 2024 13:45:59 +0200
Subject: [PATCH 152/464] [libc++] Marks a variable const. (#88562)

This removes a TODO from the code base.
---
 libcxx/include/__algorithm/sort.h | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/libcxx/include/__algorithm/sort.h b/libcxx/include/__algorithm/sort.h
index 8a5e0211cdf4..07b5814639e9 100644
--- a/libcxx/include/__algorithm/sort.h
+++ b/libcxx/include/__algorithm/sort.h
@@ -696,9 +696,8 @@ __partition_with_equals_on_left(_RandomAccessIterator __first, _RandomAccessIter
   using _Ops = _IterOps<_AlgPolicy>;
   typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type;
   typedef typename std::iterator_traits<_RandomAccessIterator>::value_type value_type;
-  // TODO(LLVM18): Make __begin const, see https://reviews.llvm.org/D147089#4349748
-  _RandomAccessIterator __begin     = __first; // used for bounds checking, those are not moved around
-  const _RandomAccessIterator __end = __last;
+  const _RandomAccessIterator __begin = __first; // used for bounds checking, those are not moved around
+  const _RandomAccessIterator __end   = __last;
   (void)__end; //
   value_type __pivot(_Ops::__iter_move(__first));
   if (__comp(__pivot, *(__last - difference_type(1)))) {
-- 
GitLab


From 910ec6ff6dd9ed031e31800c70740fdd17cc1c2a Mon Sep 17 00:00:00 2001
From: Mark de Wever <koraq@xs4all.nl>
Date: Sat, 13 Apr 2024 13:46:34 +0200
Subject: [PATCH 153/464] [libc++] Undeprecate POSIX STREAM macros. (#88296)

LWG3869 Deprecate std::errc constants related to UNIX STREAMS

deprecates the POSIX macros ENODATA, ENOSR, ENOSTR, and ETIME. These
were deprecated in libc++ in
https://github.com/llvm/llvm-project/pull/80542. Based on the post
commit feedback the macro are no longer deprecated. Instead libc++
leaves the deprecation to the provider of errno.h.

---------

Co-authored-by: Hristo Hristov <zingam@outlook.com>
Co-authored-by: Louis Dionne <ldionne.2@gmail.com>
---
 libcxx/include/__system_error/errc.h          |  5 +++
 libcxx/include/cerrno                         | 18 +++------
 .../test/std/depr.cerro/cerrno.syn.verify.cpp | 37 -------------------
 3 files changed, 11 insertions(+), 49 deletions(-)
 delete mode 100644 libcxx/test/std/depr.cerro/cerrno.syn.verify.cpp

diff --git a/libcxx/include/__system_error/errc.h b/libcxx/include/__system_error/errc.h
index e9f3656b7b9c..0004c46e2279 100644
--- a/libcxx/include/__system_error/errc.h
+++ b/libcxx/include/__system_error/errc.h
@@ -112,6 +112,11 @@ enum class errc
 // macros. So GCC does not need the pushing and popping.
 //
 // TODO Remove this when the deprecated constants are removed.
+//
+// Note based on the post-review comments in
+// https://github.com/llvm/llvm-project/pull/80542 libc++ no longer deprecates
+// the macros. Since C libraries may start to deprecate these POSIX macros the
+// deprecation warning avoidance is kept.
 #if defined(_LIBCPP_COMPILER_CLANG_BASED)
 #  define _LIBCPP_SUPPRESS_DEPRECATED_ERRC_PUSH _LIBCPP_SUPPRESS_DEPRECATED_PUSH
 #  define _LIBCPP_SUPPRESS_DEPRECATED_ERRC_POP _LIBCPP_SUPPRESS_DEPRECATED_POP
diff --git a/libcxx/include/cerrno b/libcxx/include/cerrno
index 6171ae31f184..f1295680fed6 100644
--- a/libcxx/include/cerrno
+++ b/libcxx/include/cerrno
@@ -38,17 +38,11 @@ Macros:
 #  pragma GCC system_header
 #endif
 
-#ifdef ENODATA
-#  pragma clang deprecated(ENODATA, "ENODATA is deprecated in ISO C++")
-#endif
-#ifdef ENOSR
-#  pragma clang deprecated(ENOSR, "ENOSR is deprecated in ISO C++")
-#endif
-#ifdef ENOSTR
-#  pragma clang deprecated(ENOSTR, "ENOSTR is deprecated in ISO C++")
-#endif
-#ifdef ETIME
-#  pragma clang deprecated(ETIME, "ETIME is deprecated in ISO C++")
-#endif
+// LWG3869 Deprecate std::errc constants related to UNIX STREAMS
+//
+// This LWG issue deprecates the POSIX macros ENODATA, ENOSR, ENOSTR, and ETIME. These were
+// deprecated in libc++ in https://github.com/llvm/llvm-project/pull/80542.
+// Based on the post commit feedback the macro are no longer deprecated.
+// Instead libc++ leaves the deprecation to the provider of errno.h.
 
 #endif // _LIBCPP_CERRNO
diff --git a/libcxx/test/std/depr.cerro/cerrno.syn.verify.cpp b/libcxx/test/std/depr.cerro/cerrno.syn.verify.cpp
deleted file mode 100644
index 3a38605570da..000000000000
--- a/libcxx/test/std/depr.cerro/cerrno.syn.verify.cpp
+++ /dev/null
@@ -1,37 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: clang-modules-build
-// UNSUPPORTED: apple-clang && c++03
-
-// <cerrno>
-
-// tests LWG 3869 deprecated macros.
-//
-// Note the macros may not be defined. When they are not defined the
-// ifdef XXX does not trigger a deprecated message. So use them in the
-// ifdef and test for 2 deprecated messages.
-
-#include <cerrno>
-
-#ifdef ENODATA
-[[maybe_unused]] int nodata =
-    ENODATA; // expected-warning@cerrno.syn.verify.cpp:* 2 {{macro 'ENODATA' has been marked as deprecated}}
-#endif
-#ifdef ENOSR
-[[maybe_unused]] int nosr =
-    ENOSR; // expected-warning@cerrno.syn.verify.cpp:* 2 {{macro 'ENOSR' has been marked as deprecated}}
-#endif
-#ifdef ENOSTR
-[[maybe_unused]] int nostr =
-    ENOSTR; // expected-warning@cerrno.syn.verify.cpp:* 2 {{macro 'ENOSTR' has been marked as deprecated}}
-#endif
-#ifdef ETIME
-[[maybe_unused]] int timeout =
-    ETIME; // expected-warning@cerrno.syn.verify.cpp:* 2 {{macro 'ETIME' has been marked as deprecated}}
-#endif
-- 
GitLab


From cbfcfdf75e9939bc47ac7a7c11d2122a6ad426ed Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Sat, 13 Apr 2024 15:16:35 +0300
Subject: [PATCH 154/464] [clang][NFC] Add a test for CWG2254 to
 `is_pointer_interconvertible_base_of` tests

Resolution of that issue makes _any_ base class subobject interconvertible with the containing object, not just the first one.
---
 clang/test/SemaCXX/type-traits.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/clang/test/SemaCXX/type-traits.cpp b/clang/test/SemaCXX/type-traits.cpp
index d43701c3d976..dee4a29bd2bf 100644
--- a/clang/test/SemaCXX/type-traits.cpp
+++ b/clang/test/SemaCXX/type-traits.cpp
@@ -1864,6 +1864,7 @@ void is_pointer_interconvertible_base_of(int n)
   static_assert(!__is_pointer_interconvertible_base_of(Base2, Derived));
   static_assert(__is_pointer_interconvertible_base_of(Base, DerivedIndirect));
   static_assert(__is_pointer_interconvertible_base_of(Base, DerivedMultiple));
+  static_assert(__is_pointer_interconvertible_base_of(Base2, DerivedMultiple));
   static_assert(!__is_pointer_interconvertible_base_of(Base3, DerivedMultiple));
   static_assert(!__is_pointer_interconvertible_base_of(Base, DerivedAmbiguous));
   static_assert(__is_pointer_interconvertible_base_of(Base, DerivedPrivate));
-- 
GitLab


From 26852565a5f609e6b466f43c2f690ce3047d04c7 Mon Sep 17 00:00:00 2001
From: Mark de Wever <koraq@xs4all.nl>
Date: Sat, 13 Apr 2024 16:08:07 +0200
Subject: [PATCH 155/464] [libc++][TZDZ] Validates the database version.

It seems one of the tests requires a very recent timezone database. Skip
the test when the database is older.
---
 .../time.zone.members/get_info.sys_time.pass.cpp              | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/get_info.sys_time.pass.cpp b/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/get_info.sys_time.pass.cpp
index 2ad408968589..b710f3df2335 100644
--- a/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/get_info.sys_time.pass.cpp
+++ b/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/get_info.sys_time.pass.cpp
@@ -175,6 +175,10 @@ static void test_indian_kerguelen() {
   // Z Indian/Kerguelen 0 - -00 1950
   // 5 - +05
 
+  //  Make sure the entry is present in the database.
+  if (std::chrono::get_tzdb().version < "2024a")
+    return;
+
   const std::chrono::time_zone* tz = std::chrono::locate_zone("Indian/Kerguelen");
 
   std::chrono::sys_seconds transition =
-- 
GitLab


From 61f1f1300254e91334404f0d22e75e96f0d98a85 Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser@berlin.de>
Date: Sat, 13 Apr 2024 18:24:12 +0200
Subject: [PATCH 156/464] [libc++][NFC] Move basic ASan annotation functions
 into a utility header (#87220)

---
 libcxx/include/CMakeLists.txt              |   1 +
 libcxx/include/__config                    |   9 --
 libcxx/include/__debug_utils/sanitizers.h  | 104 +++++++++++++++++++++
 libcxx/include/__memory/allocator_traits.h |  12 ---
 libcxx/include/deque                       |  39 +-------
 libcxx/include/module.modulemap            |   1 +
 libcxx/include/string                      |   6 +-
 libcxx/include/vector                      |  11 +--
 8 files changed, 115 insertions(+), 68 deletions(-)
 create mode 100644 libcxx/include/__debug_utils/sanitizers.h

diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index d4e8c196a9a8..1b6c6de082da 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -345,6 +345,7 @@ set(files
   __coroutine/noop_coroutine_handle.h
   __coroutine/trivial_awaitables.h
   __debug_utils/randomize_range.h
+  __debug_utils/sanitizers.h
   __debug_utils/strict_weak_ordering_check.h
   __exception/exception.h
   __exception/exception_ptr.h
diff --git a/libcxx/include/__config b/libcxx/include/__config
index d98b54926bbe..82782b31c557 100644
--- a/libcxx/include/__config
+++ b/libcxx/include/__config
@@ -1061,15 +1061,6 @@ typedef __char32_t char32_t;
 #    define _LIBCPP_CONSTEXPR_SINCE_CXX23
 #  endif
 
-#  ifndef _LIBCPP_HAS_NO_ASAN
-extern "C" _LIBCPP_EXPORTED_FROM_ABI void
-__sanitizer_annotate_contiguous_container(const void*, const void*, const void*, const void*);
-extern "C" _LIBCPP_EXPORTED_FROM_ABI void __sanitizer_annotate_double_ended_contiguous_container(
-    const void*, const void*, const void*, const void*, const void*, const void*);
-extern "C" _LIBCPP_EXPORTED_FROM_ABI int
-__sanitizer_verify_double_ended_contiguous_container(const void*, const void*, const void*, const void*);
-#  endif
-
 // Try to find out if RTTI is disabled.
 #  if !defined(__cpp_rtti) || __cpp_rtti < 199711L
 #    define _LIBCPP_HAS_NO_RTTI
diff --git a/libcxx/include/__debug_utils/sanitizers.h b/libcxx/include/__debug_utils/sanitizers.h
new file mode 100644
index 000000000000..d8547e324933
--- /dev/null
+++ b/libcxx/include/__debug_utils/sanitizers.h
@@ -0,0 +1,104 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___LIBCXX_DEBUG_UTILS_SANITIZERS_H
+#define _LIBCPP___LIBCXX_DEBUG_UTILS_SANITIZERS_H
+
+#include <__config>
+#include <__type_traits/integral_constant.h>
+#include <__type_traits/is_constant_evaluated.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+#ifndef _LIBCPP_HAS_NO_ASAN
+
+extern "C" {
+_LIBCPP_EXPORTED_FROM_ABI void
+__sanitizer_annotate_contiguous_container(const void*, const void*, const void*, const void*);
+_LIBCPP_EXPORTED_FROM_ABI void __sanitizer_annotate_double_ended_contiguous_container(
+    const void*, const void*, const void*, const void*, const void*, const void*);
+_LIBCPP_EXPORTED_FROM_ABI int
+__sanitizer_verify_double_ended_contiguous_container(const void*, const void*, const void*, const void*);
+}
+
+#endif // _LIBCPP_HAS_NO_ASAN
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+// ASan choices
+#ifndef _LIBCPP_HAS_NO_ASAN
+#  define _LIBCPP_HAS_ASAN_CONTAINER_ANNOTATIONS_FOR_ALL_ALLOCATORS 1
+#endif
+
+#ifdef _LIBCPP_HAS_ASAN_CONTAINER_ANNOTATIONS_FOR_ALL_ALLOCATORS
+// __asan_annotate_container_with_allocator determines whether containers with custom allocators are annotated. This is
+// a public customization point to disable annotations if the custom allocator assumes that the memory isn't poisoned.
+// See the https://libcxx.llvm.org/UsingLibcxx.html#turning-off-asan-annotation-in-containers for more information.
+template <class _Alloc>
+struct __asan_annotate_container_with_allocator : true_type {};
+#endif
+
+// Annotate a double-ended contiguous range.
+// - [__first_storage, __last_storage) is the allocated memory region,
+// - [__first_old_contained, __last_old_contained) is the previously allowed (unpoisoned) range, and
+// - [__first_new_contained, __last_new_contained) is the new allowed (unpoisoned) range.
+template <class _Allocator>
+_LIBCPP_HIDE_FROM_ABI void __annotate_double_ended_contiguous_container(
+    const void* __first_storage,
+    const void* __last_storage,
+    const void* __first_old_contained,
+    const void* __last_old_contained,
+    const void* __first_new_contained,
+    const void* __last_new_contained) {
+#ifdef _LIBCPP_HAS_NO_ASAN
+  (void)__first_storage;
+  (void)__last_storage;
+  (void)__first_old_contained;
+  (void)__last_old_contained;
+  (void)__first_new_contained;
+  (void)__last_new_contained;
+#else
+  if (__asan_annotate_container_with_allocator<_Allocator>::value && __first_storage != nullptr)
+    __sanitizer_annotate_double_ended_contiguous_container(
+        __first_storage,
+        __last_storage,
+        __first_old_contained,
+        __last_old_contained,
+        __first_new_contained,
+        __last_new_contained);
+#endif
+}
+
+// Annotate a contiguous range.
+// [__first_storage, __last_storage) is the allocated memory region,
+// __old_last_contained is the previously last allowed (unpoisoned) element, and
+// __new_last_contained is the new last allowed (unpoisoned) element.
+template <class _Allocator>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 void __annotate_contiguous_container(
+    const void* __first_storage,
+    const void* __last_storage,
+    const void* __old_last_contained,
+    const void* __new_last_contained) {
+#ifdef _LIBCPP_HAS_NO_ASAN
+  (void)__first_storage;
+  (void)__last_storage;
+  (void)__old_last_contained;
+  (void)__new_last_contained;
+#else
+  if (!__libcpp_is_constant_evaluated() && __asan_annotate_container_with_allocator<_Allocator>::value &&
+      __first_storage != nullptr)
+    __sanitizer_annotate_contiguous_container(
+        __first_storage, __last_storage, __old_last_contained, __new_last_contained);
+#endif
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___LIBCXX_DEBUG_UTILS_SANITIZERS_H
diff --git a/libcxx/include/__memory/allocator_traits.h b/libcxx/include/__memory/allocator_traits.h
index efdaa5e4ed8a..7b3deb0f58e9 100644
--- a/libcxx/include/__memory/allocator_traits.h
+++ b/libcxx/include/__memory/allocator_traits.h
@@ -407,18 +407,6 @@ struct __is_cpp17_copy_insertable<
                    __has_construct<_Alloc, typename _Alloc::value_type*, const typename _Alloc::value_type&>::value > >
     : __is_cpp17_move_insertable<_Alloc> {};
 
-// ASan choices
-#ifndef _LIBCPP_HAS_NO_ASAN
-#  define _LIBCPP_HAS_ASAN_CONTAINER_ANNOTATIONS_FOR_ALL_ALLOCATORS 1
-#endif
-
-#ifdef _LIBCPP_HAS_ASAN_CONTAINER_ANNOTATIONS_FOR_ALL_ALLOCATORS
-template <class _Alloc>
-struct __asan_annotate_container_with_allocator : true_type {};
-template <class _Tp>
-struct __asan_annotate_container_with_allocator<allocator<_Tp> > : true_type {};
-#endif
-
 #undef _LIBCPP_ALLOCATOR_TRAITS_HAS_XXX
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/deque b/libcxx/include/deque
index a6472e46d426..d42669dd6dc0 100644
--- a/libcxx/include/deque
+++ b/libcxx/include/deque
@@ -191,6 +191,7 @@ template <class T, class Allocator, class Predicate>
 #include <__assert>
 #include <__availability>
 #include <__config>
+#include <__debug_utils/sanitizers.h>
 #include <__format/enable_insertable.h>
 #include <__fwd/deque.h>
 #include <__iterator/distance.h>
@@ -864,33 +865,6 @@ private:
     __asan_back_moved,
   };
 
-  // The following functions are no-ops outside of AddressSanitizer mode.
-  // We call annotations for every allocator, unless explicitly disabled.
-  //
-  // To disable annotations for a particular allocator, change value of
-  // __asan_annotate_container_with_allocator to false.
-  // For more details, see the "Using libc++" documentation page or
-  // the documentation for __sanitizer_annotate_contiguous_container.
-  _LIBCPP_HIDE_FROM_ABI void __annotate_double_ended_contiguous_container(
-      const void* __beg,
-      const void* __end,
-      const void* __old_con_beg,
-      const void* __old_con_end,
-      const void* __new_con_beg,
-      const void* __new_con_end) const {
-    (void)__beg;
-    (void)__end;
-    (void)__old_con_beg;
-    (void)__old_con_end;
-    (void)__new_con_beg;
-    (void)__new_con_end;
-#ifndef _LIBCPP_HAS_NO_ASAN
-    if (__beg != nullptr && __asan_annotate_container_with_allocator<_Allocator>::value)
-      __sanitizer_annotate_double_ended_contiguous_container(
-          __beg, __end, __old_con_beg, __old_con_end, __new_con_beg, __new_con_end);
-#endif
-  }
-
   _LIBCPP_HIDE_FROM_ABI void __annotate_from_to(
       size_type __beg,
       size_type __end,
@@ -990,7 +964,8 @@ private:
       const void* __new_beg = __front ? __new_edge : __old_edge;
       const void* __new_end = __front ? __old_edge : __new_edge;
 
-      __annotate_double_ended_contiguous_container(__mem_beg, __mem_end, __old_beg, __old_end, __new_beg, __new_end);
+      std::__annotate_double_ended_contiguous_container<_Allocator>(
+          __mem_beg, __mem_end, __old_beg, __old_end, __new_beg, __new_end);
     }
 #endif // !_LIBCPP_HAS_NO_ASAN
   }
@@ -1051,11 +1026,7 @@ private:
   }
 
   _LIBCPP_HIDE_FROM_ABI void __annotate_poison_block(const void* __beginning, const void* __end) const _NOEXCEPT {
-    (void)__beginning;
-    (void)__end;
-#ifndef _LIBCPP_HAS_NO_ASAN
-    __annotate_double_ended_contiguous_container(__beginning, __end, __beginning, __end, __end, __end);
-#endif
+    std::__annotate_double_ended_contiguous_container<_Allocator>(__beginning, __end, __beginning, __end, __end, __end);
   }
 
   _LIBCPP_HIDE_FROM_ABI void
@@ -1070,7 +1041,7 @@ private:
     if (__annotation_type == __asan_poison)
       __annotate_poison_block(__block_start, __block_end);
     else {
-      __annotate_double_ended_contiguous_container(
+      std::__annotate_double_ended_contiguous_container<_Allocator>(
           __block_start, __block_end, __block_start, __block_start, __block_start, __block_end);
     }
 #endif
diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap
index 9c0f0ddd20c9..372219e8f836 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap
@@ -1254,6 +1254,7 @@ module std_private_coroutine_noop_coroutine_handle [system] { header "__coroutin
 module std_private_coroutine_trivial_awaitables    [system] { header "__coroutine/trivial_awaitables.h" }
 
 module std_private_debug_utils_randomize_range            [system] { header "__debug_utils/randomize_range.h" }
+module std_private_debug_utils_sanitizers                 [system] { header "__debug_utils/sanitizers.h" }
 module std_private_debug_utils_strict_weak_ordering_check [system] {
   header "__debug_utils/strict_weak_ordering_check.h"
   export std_private_type_traits_is_constant_evaluated
diff --git a/libcxx/include/string b/libcxx/include/string
index a456f8cb80ee..4ea6df00bb7e 100644
--- a/libcxx/include/string
+++ b/libcxx/include/string
@@ -574,6 +574,7 @@ basic_string<char32_t> operator""s( const char32_t *str, size_t len );
 #include <__algorithm/remove_if.h>
 #include <__assert>
 #include <__config>
+#include <__debug_utils/sanitizers.h>
 #include <__format/enable_insertable.h>
 #include <__functional/hash.h>
 #include <__functional/unary_function.h>
@@ -1909,10 +1910,7 @@ private:
     (void)__old_mid;
     (void)__new_mid;
 #if !defined(_LIBCPP_HAS_NO_ASAN) && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
-    const void* __begin = data();
-    const void* __end   = data() + capacity() + 1;
-    if (__asan_annotate_container_with_allocator<allocator_type>::value && !__libcpp_is_constant_evaluated())
-      __sanitizer_annotate_contiguous_container(__begin, __end, __old_mid, __new_mid);
+    std::__annotate_contiguous_container<_Allocator>(data(), data() + capacity() + 1, __old_mid, __new_mid);
 #endif
   }
 
diff --git a/libcxx/include/vector b/libcxx/include/vector
index 1defc43a5247..7303d7ec0b24 100644
--- a/libcxx/include/vector
+++ b/libcxx/include/vector
@@ -320,6 +320,7 @@ template<class T, class charT> requires is-vector-bool-reference<T> // Since C++
 #include <__bit_reference>
 #include <__concepts/same_as.h>
 #include <__config>
+#include <__debug_utils/sanitizers.h>
 #include <__format/enable_insertable.h>
 #include <__format/formatter.h>
 #include <__format/formatter_bool.h>
@@ -834,15 +835,7 @@ private:
 
   _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
   __annotate_contiguous_container(const void* __old_mid, const void* __new_mid) const {
-    (void)__old_mid;
-    (void)__new_mid;
-#ifndef _LIBCPP_HAS_NO_ASAN
-    const void* __beg = data();
-    const void* __end = data() + capacity();
-    if (!__libcpp_is_constant_evaluated() && __beg != nullptr &&
-        __asan_annotate_container_with_allocator<_Allocator>::value)
-      __sanitizer_annotate_contiguous_container(__beg, __end, __old_mid, __new_mid);
-#endif
+    std::__annotate_contiguous_container<_Allocator>(data(), data() + capacity(), __old_mid, __new_mid);
   }
 
   _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __annotate_new(size_type __current_size) const _NOEXCEPT {
-- 
GitLab


From ecf44b4b719c33465d9ebfbc5910b1f761856e0b Mon Sep 17 00:00:00 2001
From: Mark de Wever <koraq@xs4all.nl>
Date: Sat, 13 Apr 2024 20:35:59 +0200
Subject: [PATCH 157/464] [libc++][TZDB] Removes test_indian_kerguelen test.

The work-around in 26852565a5f609e6b466f43c2f690ce3047d04c7 didn't fix
the CI. Since the entire local database is compared with the libc++
implementation in a separate this this change does not remove coverage.
---
 .../get_info.sys_time.pass.cpp                | 44 -------------------
 1 file changed, 44 deletions(-)

diff --git a/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/get_info.sys_time.pass.cpp b/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/get_info.sys_time.pass.cpp
index b710f3df2335..a751a2fb6347 100644
--- a/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/get_info.sys_time.pass.cpp
+++ b/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/get_info.sys_time.pass.cpp
@@ -169,49 +169,6 @@ static void test_durations() {
   assert_equal(expected, tz->get_info(std::chrono::sys_time<std::chrono::years>{}));
 }
 
-static void test_indian_kerguelen() {
-  // One change, no rules, no dst changes.
-
-  // Z Indian/Kerguelen 0 - -00 1950
-  // 5 - +05
-
-  //  Make sure the entry is present in the database.
-  if (std::chrono::get_tzdb().version < "2024a")
-    return;
-
-  const std::chrono::time_zone* tz = std::chrono::locate_zone("Indian/Kerguelen");
-
-  std::chrono::sys_seconds transition =
-      to_sys_seconds(std::chrono::year(1950), std::chrono::January, std::chrono::day(1));
-
-  assert_equal(
-      std::chrono::sys_info(
-          std::chrono::sys_seconds::min(), //
-          transition,                      //
-          std::chrono::seconds(0),         //
-          std::chrono::minutes(0),         //
-          "-00"),                          //
-      tz->get_info(std::chrono::sys_seconds::min()));
-
-  assert_equal(
-      std::chrono::sys_info(
-          std::chrono::sys_seconds::min(), //
-          transition,                      //
-          std::chrono::seconds(0),         //
-          std::chrono::minutes(0),         //
-          "-00"),                          //
-      tz->get_info(transition - std::chrono::seconds{1}));
-
-  assert_equal(
-      std::chrono::sys_info(
-          transition,                      //
-          std::chrono::sys_seconds::max(), //
-          std::chrono::hours(5),           //
-          std::chrono::minutes(0),         //
-          "+05"),                          //
-      tz->get_info(transition));
-}
-
 static void test_antarctica_syowa() {
   // One change, no rules, no dst changes
   // This change uses an ON field with a day number
@@ -1343,7 +1300,6 @@ int main(int, const char**) {
   // Basic tests
   test_gmt();
   test_durations();
-  test_indian_kerguelen();
   test_antarctica_syowa();
   test_asia_hong_kong();
   test_europe_berlin();
-- 
GitLab


From fad37526a3ea7d669af621342968029085862281 Mon Sep 17 00:00:00 2001
From: Edwin Vane <revane@google.com>
Date: Sat, 13 Apr 2024 15:01:06 -0400
Subject: [PATCH 158/464] [clang-tidy] Export fixes from check_clang_tidy.py
 (#88186)

Makes it possible to export fixes from running llvm-lit on a clang-tidy
test. To enable, modify the RUN invocation directly in the test with the
new -export flag. llvm-lit will report the test passed and fixes can be
found in the file specified to the -export flag.
---
 clang-tools-extra/docs/ReleaseNotes.rst       |  2 +
 .../test/clang-tidy/check_clang_tidy.py       | 74 ++++++++++++++-----
 2 files changed, 57 insertions(+), 19 deletions(-)

diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index b66be44e9f8a..1405fb0df1f8 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -100,6 +100,8 @@ Improvements to clang-tidy
 - Improved :program:`run-clang-tidy.py` script. Added argument `-source-filter`
   to filter source files from the compilation database, via a RegEx. In a
   similar fashion to what `-header-filter` does for header files.
+- Improved :program:`check_clang_tidy.py` script. Added argument `-export-fixes`
+  to aid in clang-tidy and test development.
 
 New checks
 ^^^^^^^^^^
diff --git a/clang-tools-extra/test/clang-tidy/check_clang_tidy.py b/clang-tools-extra/test/clang-tidy/check_clang_tidy.py
index 53ffca0bad8d..6d4b466afa69 100755
--- a/clang-tools-extra/test/clang-tidy/check_clang_tidy.py
+++ b/clang-tools-extra/test/clang-tidy/check_clang_tidy.py
@@ -8,25 +8,35 @@
 #
 # ===------------------------------------------------------------------------===#
 
-r"""
+"""
 ClangTidy Test Helper
 =====================
 
-This script runs clang-tidy in fix mode and verify fixes, messages or both.
+This script is used to simplify writing, running, and debugging tests compatible
+with llvm-lit. By default it runs clang-tidy in fix mode and uses FileCheck to
+verify messages and/or fixes.
+
+For debugging, with --export-fixes, the tool simply exports fixes to a provided
+file and does not run FileCheck.
 
-Usage:
-  check_clang_tidy.py [-resource-dir=<resource-dir>] \
-    [-assume-filename=<file-with-source-extension>] \
-    [-check-suffix=<comma-separated-file-check-suffixes>] \
-    [-check-suffixes=<comma-separated-file-check-suffixes>] \
-    [-std=c++(98|11|14|17|20)[-or-later]] \
-    <source-file> <check-name> <temp-file> \
-    -- [optional clang-tidy arguments]
+Extra arguments, those after the first -- if any, are passed to either
+clang-tidy or clang:
+* Arguments between the first -- and second -- are clang-tidy arguments.
+  * May be only whitespace if there are no clang-tidy arguments.
+  * clang-tidy's --config would go here.
+* Arguments after the second -- are clang arguments
+
+Examples
+--------
 
-Example:
   // RUN: %check_clang_tidy %s llvm-include-order %t -- -- -isystem %S/Inputs
 
-Notes:
+or
+
+  // RUN: %check_clang_tidy %s llvm-include-order --export-fixes=fixes.yaml %t -std=c++20
+
+Notes
+-----
   -std=c++(98|11|14|17|20)-or-later:
     This flag will cause multiple runs within the same check_clang_tidy
     execution. Make sure you don't have shared state across these runs.
@@ -34,6 +44,7 @@ Notes:
 
 import argparse
 import os
+import pathlib
 import re
 import subprocess
 import sys
@@ -88,6 +99,7 @@ class CheckRunner:
         self.has_check_fixes = False
         self.has_check_messages = False
         self.has_check_notes = False
+        self.export_fixes = args.export_fixes
         self.fixes = MessagePrefix("CHECK-FIXES")
         self.messages = MessagePrefix("CHECK-MESSAGES")
         self.notes = MessagePrefix("CHECK-NOTES")
@@ -181,7 +193,13 @@ class CheckRunner:
             [
                 "clang-tidy",
                 self.temp_file_name,
-                "-fix",
+            ]
+            + [
+                "-fix"
+                if self.export_fixes is None
+                else "--export-fixes=" + self.export_fixes
+            ]
+            + [
                 "--checks=-*," + self.check_name,
             ]
             + self.clang_tidy_extra_args
@@ -255,12 +273,14 @@ class CheckRunner:
 
     def run(self):
         self.read_input()
-        self.get_prefixes()
+        if self.export_fixes is None:
+            self.get_prefixes()
         self.prepare_test_inputs()
         clang_tidy_output = self.run_clang_tidy()
-        self.check_fixes()
-        self.check_messages(clang_tidy_output)
-        self.check_notes(clang_tidy_output)
+        if self.export_fixes is None:
+            self.check_fixes()
+            self.check_messages(clang_tidy_output)
+            self.check_notes(clang_tidy_output)
 
 
 def expand_std(std):
@@ -284,7 +304,11 @@ def csv(string):
 
 
 def parse_arguments():
-    parser = argparse.ArgumentParser()
+    parser = argparse.ArgumentParser(
+        prog=pathlib.Path(__file__).stem,
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
     parser.add_argument("-expect-clang-tidy-error", action="store_true")
     parser.add_argument("-resource-dir")
     parser.add_argument("-assume-filename")
@@ -298,7 +322,19 @@ def parse_arguments():
         type=csv,
         help="comma-separated list of FileCheck suffixes",
     )
-    parser.add_argument("-std", type=csv, default=["c++11-or-later"])
+    parser.add_argument(
+        "-export-fixes",
+        default=None,
+        type=str,
+        metavar="file",
+        help="A file to export fixes into instead of fixing.",
+    )
+    parser.add_argument(
+        "-std",
+        type=csv,
+        default=["c++11-or-later"],
+        help="Passed to clang. Special -or-later values are expanded.",
+    )
     return parser.parse_known_args()
 
 
-- 
GitLab


From ef9446bd2d362ec90cd681ae59463d16bf671fe8 Mon Sep 17 00:00:00 2001
From: Enna1 <xumingjie.enna1@bytedance.com>
Date: Sun, 14 Apr 2024 11:01:02 +0800
Subject: [PATCH 159/464] [GVN] lazily update dominator tree when merge
 unconditional branches in function (#88477)

With this change, the wall time for of GVN pass decreased from
873,745.492 ms to 367,375.304 ms in an our internal testcase.
---
 llvm/lib/Transforms/Scalar/GVN.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index 67fb2a5da3bb..86d5c9909f3d 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -2708,7 +2708,7 @@ bool GVNPass::runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT,
   bool Changed = false;
   bool ShouldContinue = true;
 
-  DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager);
+  DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
   // Merge unconditional branches, allowing PRE to catch more
   // optimization opportunities.
   for (BasicBlock &BB : llvm::make_early_inc_range(F)) {
@@ -2718,6 +2718,7 @@ bool GVNPass::runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT,
 
     Changed |= removedBlock;
   }
+  DTU.flush();
 
   unsigned Iteration = 0;
   while (ShouldContinue) {
-- 
GitLab


From 342aa3eb23992ad742d22ef0e0d8821a9b29de3e Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Sat, 13 Apr 2024 20:54:09 -0700
Subject: [PATCH 160/464] [LegalizeTypes][RISCV] Support scalable vectors in
 SplitVecRes_BITCAST/SplitVecOp_BITCAST. (#88590)

Instead of bitcasting to a scalar integer, try to split the other vector
type that is legal. We already handle the case where both types need to
be split.

Fixes #88576
---
 .../SelectionDAG/LegalizeVectorTypes.cpp      | 20 ++++++-
 llvm/test/CodeGen/RISCV/rvv/pr88576.ll        | 60 +++++++++++++++++++
 2 files changed, 78 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/CodeGen/RISCV/rvv/pr88576.ll

diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 1f6e0097f31a..8776d89f4c5b 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -1388,6 +1388,13 @@ void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
     report_fatal_error("Scalarization of scalable vectors is not supported.");
   }
 
+  if (LoVT.isScalableVector()) {
+    auto [InLo, InHi] = DAG.SplitVectorOperand(N, 0);
+    Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, InLo);
+    Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, InHi);
+    return;
+  }
+
   // In the general case, convert the input to an integer and split it by hand.
   EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
   EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
@@ -3259,16 +3266,25 @@ SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
   // For example, i64 = BITCAST v4i16 on alpha.  Typically the vector will
   // end up being split all the way down to individual components.  Convert the
   // split pieces into integers and reassemble.
+  EVT ResVT = N->getValueType(0);
   SDValue Lo, Hi;
   GetSplitVector(N->getOperand(0), Lo, Hi);
+  SDLoc dl(N);
+
+  if (ResVT.isScalableVector()) {
+    auto [LoVT, HiVT] = DAG.GetSplitDestVTs(ResVT);
+    Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
+    Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
+    return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
+  }
+
   Lo = BitConvertToInteger(Lo);
   Hi = BitConvertToInteger(Hi);
 
   if (DAG.getDataLayout().isBigEndian())
     std::swap(Lo, Hi);
 
-  return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0),
-                     JoinIntegers(Lo, Hi));
+  return DAG.getNode(ISD::BITCAST, dl, ResVT, JoinIntegers(Lo, Hi));
 }
 
 SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N,
diff --git a/llvm/test/CodeGen/RISCV/rvv/pr88576.ll b/llvm/test/CodeGen/RISCV/rvv/pr88576.ll
new file mode 100644
index 000000000000..b6e0d1e2ff4a
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/pr88576.ll
@@ -0,0 +1,60 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v | FileCheck %s
+
+define i1 @foo(<vscale x 16 x i8> %x, i64 %y) {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    slli a2, a1, 4
+; CHECK-NEXT:    addi a2, a2, -1
+; CHECK-NEXT:    bltu a0, a2, .LBB0_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    mv a0, a2
+; CHECK-NEXT:  .LBB0_2:
+; CHECK-NEXT:    addi sp, sp, -80
+; CHECK-NEXT:    .cfi_def_cfa_offset 80
+; CHECK-NEXT:    sd ra, 72(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    sd s0, 64(sp) # 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset ra, -8
+; CHECK-NEXT:    .cfi_offset s0, -16
+; CHECK-NEXT:    addi s0, sp, 80
+; CHECK-NEXT:    .cfi_def_cfa s0, 0
+; CHECK-NEXT:    csrr a2, vlenb
+; CHECK-NEXT:    slli a2, a2, 4
+; CHECK-NEXT:    sub sp, sp, a2
+; CHECK-NEXT:    andi sp, sp, -64
+; CHECK-NEXT:    addi a2, sp, 64
+; CHECK-NEXT:    add a0, a2, a0
+; CHECK-NEXT:    slli a1, a1, 3
+; CHECK-NEXT:    add a1, a2, a1
+; CHECK-NEXT:    vsetvli a3, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmv.v.i v16, 0
+; CHECK-NEXT:    vmv1r.v v0, v9
+; CHECK-NEXT:    vmerge.vim v24, v16, 1, v0
+; CHECK-NEXT:    vs8r.v v24, (a1)
+; CHECK-NEXT:    vmv1r.v v0, v8
+; CHECK-NEXT:    vmerge.vim v8, v16, 1, v0
+; CHECK-NEXT:    vs8r.v v8, (a2)
+; CHECK-NEXT:    lbu a0, 0(a0)
+; CHECK-NEXT:    addi sp, s0, -80
+; CHECK-NEXT:    ld ra, 72(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    ld s0, 64(sp) # 8-byte Folded Reload
+; CHECK-NEXT:    addi sp, sp, 80
+; CHECK-NEXT:    ret
+  %a = bitcast <vscale x 16 x i8> %x to <vscale x 128 x i1>
+  %b = extractelement <vscale x 128 x i1> %a, i64 %y
+  ret i1 %b
+}
+
+define i8 @bar(<vscale x 128 x i1> %x, i64 %y) {
+; CHECK-LABEL: bar:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmv1r.v v1, v8
+; CHECK-NEXT:    vsetivli zero, 1, e8, m2, ta, ma
+; CHECK-NEXT:    vslidedown.vx v8, v0, a0
+; CHECK-NEXT:    vmv.x.s a0, v8
+; CHECK-NEXT:    ret
+  %a = bitcast <vscale x 128 x i1> %x to <vscale x 16 x i8>
+  %b = extractelement <vscale x 16 x i8> %a, i64 %y
+  ret i8 %b
+}
-- 
GitLab


From 09327efdf0f02c4f865a4536db96cac539bb1c01 Mon Sep 17 00:00:00 2001
From: Chris Copeland <chris@chrisnc.net>
Date: Sat, 13 Apr 2024 23:37:58 -0700
Subject: [PATCH 161/464] [clang][docs] fix whitespace in AttrDocs.td (#88631)

Noticed this failure while working on a different change:
https://buildkite.com/llvm-project/clang-ci/builds/15553#018ed1bb-923a-48b5-9788-e8b6d3827aba
Seems to be introduced by #88448.
---
 clang/include/clang/Basic/AttrDocs.td | 32 +++++++++++++--------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index 8687c4f57d3f..a0bbe5861c57 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -1604,39 +1604,39 @@ specifies availability for the current target platform, the availability
 attributes are ignored. Supported platforms are:
 
 ``ios``
-  Apple's iOS operating system. The minimum deployment target is specified 
-  as part of the ``-target *arch*-apple-ios*version*`` command line argument. 
-  Alternatively, it can be specified by the ``-mtargetos=ios*version*`` 
+  Apple's iOS operating system. The minimum deployment target is specified
+  as part of the ``-target *arch*-apple-ios*version*`` command line argument.
+  Alternatively, it can be specified by the ``-mtargetos=ios*version*``
   command-line argument.
 
 ``macos``
-  Apple's macOS operating system. The minimum deployment target is specified 
-  as part of the ``-target *arch*-apple-macos*version*`` command line argument. 
-  Alternatively, it can be specified by the ``-mtargetos=macos*version*`` 
-  command-line argument. ``macosx`` is supported for 
+  Apple's macOS operating system. The minimum deployment target is specified
+  as part of the ``-target *arch*-apple-macos*version*`` command line argument.
+  Alternatively, it can be specified by the ``-mtargetos=macos*version*``
+  command-line argument. ``macosx`` is supported for
   backward-compatibility reasons, but it is deprecated.
 
 ``tvos``
-  Apple's tvOS operating system. The minimum deployment target is specified 
-  as part of the ``-target *arch*-apple-tvos*version*`` command line argument. 
-  Alternatively, it can be specified by the ``-mtargetos=tvos*version*`` 
+  Apple's tvOS operating system. The minimum deployment target is specified
+  as part of the ``-target *arch*-apple-tvos*version*`` command line argument.
+  Alternatively, it can be specified by the ``-mtargetos=tvos*version*``
   command-line argument.
 
 ``watchos``
   Apple's watchOS operating system. The minimum deployment target is specified
-  as part of the ``-target *arch*-apple-watchos*version*`` command line argument. 
-  Alternatively, it can be specified by the ``-mtargetos=watchos*version*`` 
+  as part of the ``-target *arch*-apple-watchos*version*`` command line argument.
+  Alternatively, it can be specified by the ``-mtargetos=watchos*version*``
   command-line argument.
 
 ``visionos``
   Apple's visionOS operating system. The minimum deployment target is specified
-  as part of the ``-target *arch*-apple-visionos*version*`` command line argument. 
-  Alternatively, it can be specified by the ``-mtargetos=visionos*version*`` 
+  as part of the ``-target *arch*-apple-visionos*version*`` command line argument.
+  Alternatively, it can be specified by the ``-mtargetos=visionos*version*``
   command-line argument.
 
 ``driverkit``
   Apple's DriverKit userspace kernel extensions. The minimum deployment target
-  is specified as part of the ``-target *arch*-apple-driverkit*version*`` 
+  is specified as part of the ``-target *arch*-apple-driverkit*version*``
   command line argument.
 
 A declaration can typically be used even when deploying back to a platform
@@ -7522,7 +7522,7 @@ means that it can e.g no longer be part of an initializer expression.
 
       /* This may print something else than "6 * 7 = 42",
          if there is a non-weak definition of "ANSWER" in
-	 an object linked in */
+         an object linked in */
       printf("6 * 7 = %d\n", ANSWER);
 
       return 0;
-- 
GitLab


From c6f9c84e498ee05a812511ae969773ff166fd25e Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser@berlin.de>
Date: Sun, 14 Apr 2024 12:08:30 +0200
Subject: [PATCH 162/464] [Clang] Reduce the size of Decl and classes derived
 from it (#87361)

Class | Old size (in bytes) | New size (in bytes)

----------------------------------|---------------------|--------------------
Decl                              | 40                  | 32
AccessSpecDecl                    | 40                  | 40
BlockDecl                         | 128                 | 120
CapturedDecl                      | 88                  | 80
EmptyDecl                         | 40                  | 32
ExportDecl                        | 80                  | 72
ExternCContextDecl                | 72                  | 64
FileScopeAsmDecl                  | 56                  | 48
FriendDecl                        | 64                  | 56
FriendTemplateDecl                | 64                  | 64
ImplicitConceptSpecializationDecl | 40                  | 40
ImportDecl                        | 56                  | 48
LifetimeExtendedTemporaryDecl     | 72                  | 64
LinkageSpecDecl                   | 80                  | 72
NamedDecl                         | 48                  | 40
ObjCPropertyImplDecl              | 96                  | 88
PragmaCommentDecl                 | 40                  | 40
PragmaDetectMismatchDecl          | 48                  | 40
RequiresExprBodyDecl              | 72                  | 64
StaticAssertDecl                  | 64                  | 56
TopLevelStmtDecl                  | 88                  | 80
TranslationUnitDecl               | 104                 | 96
BaseUsingDecl                     | 56                  | 48
UsingDecl                         | 88                  | 80
UsingEnumDecl                     | 72                  | 64
HLSLBufferDecl                    | 96                  | 88
LabelDecl                         | 80                  | 72
NamespaceAliasDecl                | 96                  | 88
NamespaceDecl                     | 112                 | 104
ObjCCompatibleAliasDecl           | 56                  | 48
ObjCContainerDecl                 | 88                  | 80
ObjCMethodDecl                    | 136                 | 128
ObjCPropertyDecl                  | 128                 | 120
TemplateDecl                      | 64                  | 56
BuiltinTemplateDecl               | 72                  | 64
TypeDecl                          | 64                  | 56
UnresolvedUsingIfExistsDecl       | 48                  | 40
UsingDirectiveDecl                | 88                  | 80
UsingPackDecl                     | 64                  | 56
UsingShadowDecl                   | 80                  | 72
ValueDecl                         | 56                  | 48

When parsing libc++'s `<string>` header the used memory is reduced from
42.8MB to 42.5MB.
---
 clang/include/clang/AST/DeclBase.h        | 72 +++++++++++++++--------
 clang/lib/AST/DeclBase.cpp                | 31 +++++++---
 clang/lib/Serialization/ASTReaderDecl.cpp |  2 +-
 3 files changed, 70 insertions(+), 35 deletions(-)

diff --git a/clang/include/clang/AST/DeclBase.h b/clang/include/clang/AST/DeclBase.h
index 858450926455..4bee18767dd1 100644
--- a/clang/include/clang/AST/DeclBase.h
+++ b/clang/include/clang/AST/DeclBase.h
@@ -268,17 +268,37 @@ private:
   ///   }
   ///   void A::f(); // SemanticDC == namespace 'A'
   ///                // LexicalDC == global namespace
-  llvm::PointerUnion<DeclContext*, MultipleDC*> DeclCtx;
 
-  bool isInSemaDC() const { return DeclCtx.is<DeclContext*>(); }
-  bool isOutOfSemaDC() const { return DeclCtx.is<MultipleDC*>(); }
+  // Compress the InvalidDecl and HasAttrs bits into DeclCtx to keep Decl below
+  // 32 bytes in size
+  llvm::PointerIntPair<
+      llvm::PointerIntPair<llvm::PointerUnion<DeclContext *, MultipleDC *>, 1,
+                           bool>,
+      1, bool>
+      DeclCtxWithInvalidDeclAndHasAttrs;
+
+  bool isInSemaDC() const {
+    return DeclCtxWithInvalidDeclAndHasAttrs.getPointer()
+        .getPointer()
+        .is<DeclContext *>();
+  }
+
+  bool isOutOfSemaDC() const {
+    return DeclCtxWithInvalidDeclAndHasAttrs.getPointer()
+        .getPointer()
+        .is<MultipleDC *>();
+  }
 
   MultipleDC *getMultipleDC() const {
-    return DeclCtx.get<MultipleDC*>();
+    return DeclCtxWithInvalidDeclAndHasAttrs.getPointer()
+        .getPointer()
+        .get<MultipleDC *>();
   }
 
   DeclContext *getSemanticDC() const {
-    return DeclCtx.get<DeclContext*>();
+    return DeclCtxWithInvalidDeclAndHasAttrs.getPointer()
+        .getPointer()
+        .get<DeclContext *>();
   }
 
   /// Loc - The location of this decl.
@@ -288,14 +308,6 @@ private:
   LLVM_PREFERRED_TYPE(Kind)
   unsigned DeclKind : 7;
 
-  /// InvalidDecl - This indicates a semantic error occurred.
-  LLVM_PREFERRED_TYPE(bool)
-  unsigned InvalidDecl :  1;
-
-  /// HasAttrs - This indicates whether the decl has attributes or not.
-  LLVM_PREFERRED_TYPE(bool)
-  unsigned HasAttrs : 1;
-
   /// Implicit - Whether this declaration was implicitly generated by
   /// the implementation rather than explicitly written by the user.
   LLVM_PREFERRED_TYPE(bool)
@@ -393,21 +405,22 @@ public:
 protected:
   Decl(Kind DK, DeclContext *DC, SourceLocation L)
       : NextInContextAndBits(nullptr, getModuleOwnershipKindForChildOf(DC)),
-        DeclCtx(DC), Loc(L), DeclKind(DK), InvalidDecl(false), HasAttrs(false),
-        Implicit(false), Used(false), Referenced(false),
+        DeclCtxWithInvalidDeclAndHasAttrs({DC, false}, false), Loc(L),
+        DeclKind(DK), Implicit(false), Used(false), Referenced(false),
         TopLevelDeclInObjCContainer(false), Access(AS_none), FromASTFile(0),
         IdentifierNamespace(getIdentifierNamespaceForKind(DK)),
         CacheValidAndLinkage(llvm::to_underlying(Linkage::Invalid)) {
-    if (StatisticsEnabled) add(DK);
+    if (StatisticsEnabled)
+      add(DK);
   }
 
   Decl(Kind DK, EmptyShell Empty)
-      : DeclKind(DK), InvalidDecl(false), HasAttrs(false), Implicit(false),
-        Used(false), Referenced(false), TopLevelDeclInObjCContainer(false),
-        Access(AS_none), FromASTFile(0),
+      : DeclKind(DK), Implicit(false), Used(false), Referenced(false),
+        TopLevelDeclInObjCContainer(false), Access(AS_none), FromASTFile(0),
         IdentifierNamespace(getIdentifierNamespaceForKind(DK)),
         CacheValidAndLinkage(llvm::to_underlying(Linkage::Invalid)) {
-    if (StatisticsEnabled) add(DK);
+    if (StatisticsEnabled)
+      add(DK);
   }
 
   virtual ~Decl();
@@ -520,7 +533,9 @@ public:
     return AccessSpecifier(Access);
   }
 
-  bool hasAttrs() const { return HasAttrs; }
+  bool hasAttrs() const {
+    return DeclCtxWithInvalidDeclAndHasAttrs.getPointer().getInt();
+  }
 
   void setAttrs(const AttrVec& Attrs) {
     return setAttrsImpl(Attrs, getASTContext());
@@ -549,13 +564,17 @@ public:
   }
 
   template <typename... Ts> void dropAttrs() {
-    if (!HasAttrs) return;
+    if (!hasAttrs())
+      return;
 
     AttrVec &Vec = getAttrs();
     llvm::erase_if(Vec, [](Attr *A) { return isa<Ts...>(A); });
 
-    if (Vec.empty())
-      HasAttrs = false;
+    if (Vec.empty()) {
+      auto InnerPtr = DeclCtxWithInvalidDeclAndHasAttrs.getPointer();
+      InnerPtr.setInt(false);
+      DeclCtxWithInvalidDeclAndHasAttrs.setPointer(InnerPtr);
+    }
   }
 
   template <typename T> void dropAttr() { dropAttrs<T>(); }
@@ -590,7 +609,10 @@ public:
   /// setInvalidDecl - Indicates the Decl had a semantic error. This
   /// allows for graceful error recovery.
   void setInvalidDecl(bool Invalid = true);
-  bool isInvalidDecl() const { return (bool) InvalidDecl; }
+
+  bool isInvalidDecl() const {
+    return DeclCtxWithInvalidDeclAndHasAttrs.getInt();
+  }
 
   /// isImplicit - Indicates whether the declaration was implicitly
   /// generated by the implementation. If false, this declaration
diff --git a/clang/lib/AST/DeclBase.cpp b/clang/lib/AST/DeclBase.cpp
index 66a727d9dd0c..8c83c711d882 100644
--- a/clang/lib/AST/DeclBase.cpp
+++ b/clang/lib/AST/DeclBase.cpp
@@ -52,6 +52,8 @@
 
 using namespace clang;
 
+static_assert(sizeof(Decl) <= 32, "Decl grew beyond 32 bytes!");
+
 //===----------------------------------------------------------------------===//
 //  Statistics
 //===----------------------------------------------------------------------===//
@@ -130,7 +132,7 @@ const char *Decl::getDeclKindName() const {
 }
 
 void Decl::setInvalidDecl(bool Invalid) {
-  InvalidDecl = Invalid;
+  DeclCtxWithInvalidDeclAndHasAttrs.setInt(Invalid);
   assert(!isa<TagDecl>(this) || !cast<TagDecl>(this)->isCompleteDefinition());
   if (!Invalid) {
     return;
@@ -334,7 +336,9 @@ void PrettyStackTraceDecl::print(raw_ostream &OS) const {
 Decl::~Decl() = default;
 
 void Decl::setDeclContext(DeclContext *DC) {
-  DeclCtx = DC;
+  auto InnerPtr = DeclCtxWithInvalidDeclAndHasAttrs.getPointer();
+  InnerPtr.setPointer(DC);
+  DeclCtxWithInvalidDeclAndHasAttrs.setPointer(InnerPtr);
 }
 
 void Decl::setLexicalDeclContext(DeclContext *DC) {
@@ -364,12 +368,16 @@ void Decl::setLexicalDeclContext(DeclContext *DC) {
 void Decl::setDeclContextsImpl(DeclContext *SemaDC, DeclContext *LexicalDC,
                                ASTContext &Ctx) {
   if (SemaDC == LexicalDC) {
-    DeclCtx = SemaDC;
+    auto InnerPtr = DeclCtxWithInvalidDeclAndHasAttrs.getPointer();
+    InnerPtr.setPointer(SemaDC);
+    DeclCtxWithInvalidDeclAndHasAttrs.setPointer(InnerPtr);
   } else {
     auto *MDC = new (Ctx) Decl::MultipleDC();
     MDC->SemanticDC = SemaDC;
     MDC->LexicalDC = LexicalDC;
-    DeclCtx = MDC;
+    auto InnerPtr = DeclCtxWithInvalidDeclAndHasAttrs.getPointer();
+    InnerPtr.setPointer(MDC);
+    DeclCtxWithInvalidDeclAndHasAttrs.setPointer(InnerPtr);
   }
 }
 
@@ -956,19 +964,24 @@ unsigned Decl::getIdentifierNamespaceForKind(Kind DeclKind) {
 }
 
 void Decl::setAttrsImpl(const AttrVec &attrs, ASTContext &Ctx) {
-  assert(!HasAttrs && "Decl already contains attrs.");
+  assert(!hasAttrs() && "Decl already contains attrs.");
 
   AttrVec &AttrBlank = Ctx.getDeclAttrs(this);
   assert(AttrBlank.empty() && "HasAttrs was wrong?");
 
   AttrBlank = attrs;
-  HasAttrs = true;
+  auto InnerPtr = DeclCtxWithInvalidDeclAndHasAttrs.getPointer();
+  InnerPtr.setInt(true);
+  DeclCtxWithInvalidDeclAndHasAttrs.setPointer(InnerPtr);
 }
 
 void Decl::dropAttrs() {
-  if (!HasAttrs) return;
+  if (!hasAttrs())
+    return;
 
-  HasAttrs = false;
+  auto InnerPtr = DeclCtxWithInvalidDeclAndHasAttrs.getPointer();
+  InnerPtr.setInt(false);
+  DeclCtxWithInvalidDeclAndHasAttrs.setPointer(InnerPtr);
   getASTContext().eraseDeclAttrs(this);
 }
 
@@ -996,7 +1009,7 @@ void Decl::addAttr(Attr *A) {
 }
 
 const AttrVec &Decl::getAttrs() const {
-  assert(HasAttrs && "No attrs to get!");
+  assert(hasAttrs() && "No attrs to get!");
   return getASTContext().getDeclAttrs(this);
 }
 
diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index ff72fc30c718..c5dd89e1c8a4 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -588,7 +588,7 @@ void ASTDeclReader::VisitDecl(Decl *D) {
   bool HasStandaloneLexicalDC = DeclBits.getNextBit();
   bool HasAttrs = DeclBits.getNextBit();
   D->setTopLevelDeclInObjCContainer(DeclBits.getNextBit());
-  D->InvalidDecl = DeclBits.getNextBit();
+  D->DeclCtxWithInvalidDeclAndHasAttrs.setInt(DeclBits.getNextBit());
   D->FromASTFile = true;
 
   if (D->isTemplateParameter() || D->isTemplateParameterPack() ||
-- 
GitLab


From d48d6ba9477aa380cd5a71f899d3cb6d629f175b Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Sun, 14 Apr 2024 14:14:37 +0400
Subject: [PATCH 163/464] [clang][NFC] Factor out VLA checks in type traits
 (#88646)

This is a follow-up to #88473 suggested by @cor3ntin in
https://github.com/llvm/llvm-project/pull/88473#discussion_r1562198117.
---
 clang/lib/Sema/SemaExprCXX.cpp | 59 ++++++++++++++++++----------------
 1 file changed, 31 insertions(+), 28 deletions(-)

diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index 1cfd3e56a583..d2f18d4aa9b9 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -5012,6 +5012,20 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType,
   return From;
 }
 
+/// Checks that type T is not a VLA.
+///
+/// @returns @c true if @p T is VLA and a diagnostic was emitted,
+/// @c false otherwise.
+static bool DiagnoseVLAInCXXTypeTrait(Sema &S, const TypeSourceInfo *T,
+                                      clang::tok::TokenKind TypeTraitID) {
+  if (!T->getType()->isVariableArrayType())
+    return false;
+
+  S.Diag(T->getTypeLoc().getBeginLoc(), diag::err_vla_unsupported)
+      << 1 << TypeTraitID;
+  return true;
+}
+
 /// Check the completeness of a type in a unary type trait.
 ///
 /// If the particular type trait requires a complete type, tries to complete
@@ -5188,7 +5202,9 @@ static bool HasNoThrowOperator(const RecordType *RT, OverloadedOperatorKind Op,
 }
 
 static bool EvaluateUnaryTypeTrait(Sema &Self, TypeTrait UTT,
-                                   SourceLocation KeyLoc, QualType T) {
+                                   SourceLocation KeyLoc,
+                                   TypeSourceInfo *TInfo) {
+  QualType T = TInfo->getType();
   assert(!T->isDependentType() && "Cannot evaluate traits of dependent type");
 
   ASTContext &C = Self.Context;
@@ -5205,21 +5221,13 @@ static bool EvaluateUnaryTypeTrait(Sema &Self, TypeTrait UTT,
   case UTT_IsArray:
     return T->isArrayType();
   case UTT_IsBoundedArray:
-    if (!T->isVariableArrayType()) {
-      return T->isArrayType() && !T->isIncompleteArrayType();
-    }
-
-    Self.Diag(KeyLoc, diag::err_vla_unsupported)
-        << 1 << tok::kw___is_bounded_array;
-    return false;
+    if (DiagnoseVLAInCXXTypeTrait(Self, TInfo, tok::kw___is_bounded_array))
+      return false;
+    return T->isArrayType() && !T->isIncompleteArrayType();
   case UTT_IsUnboundedArray:
-    if (!T->isVariableArrayType()) {
-      return T->isIncompleteArrayType();
-    }
-
-    Self.Diag(KeyLoc, diag::err_vla_unsupported)
-        << 1 << tok::kw___is_unbounded_array;
-    return false;
+    if (DiagnoseVLAInCXXTypeTrait(Self, TInfo, tok::kw___is_unbounded_array))
+      return false;
+    return T->isIncompleteArrayType();
   case UTT_IsPointer:
     return T->isAnyPointerType();
   case UTT_IsNullPointer:
@@ -5631,7 +5639,7 @@ static bool EvaluateBooleanTypeTrait(Sema &S, TypeTrait Kind,
     return false;
 
   if (Kind <= UTT_Last)
-    return EvaluateUnaryTypeTrait(S, Kind, KWLoc, Args[0]->getType());
+    return EvaluateUnaryTypeTrait(S, Kind, KWLoc, Args[0]);
 
   // Evaluate ReferenceBindsToTemporary and ReferenceConstructsFromTemporary
   // alongside the IsConstructible traits to avoid duplication.
@@ -6093,12 +6101,9 @@ static bool EvaluateBinaryTypeTrait(Sema &Self, TypeTrait BTT, const TypeSourceI
       Self.RequireCompleteType(Rhs->getTypeLoc().getBeginLoc(), RhsT,
                                diag::err_incomplete_type);
 
-    if (LhsT->isVariableArrayType())
-      Self.Diag(Lhs->getTypeLoc().getBeginLoc(), diag::err_vla_unsupported)
-          << 1 << tok::kw___is_layout_compatible;
-    if (RhsT->isVariableArrayType())
-      Self.Diag(Rhs->getTypeLoc().getBeginLoc(), diag::err_vla_unsupported)
-          << 1 << tok::kw___is_layout_compatible;
+    DiagnoseVLAInCXXTypeTrait(Self, Lhs, tok::kw___is_layout_compatible);
+    DiagnoseVLAInCXXTypeTrait(Self, Rhs, tok::kw___is_layout_compatible);
+
     return Self.IsLayoutCompatible(LhsT, RhsT);
   }
   case BTT_IsPointerInterconvertibleBaseOf: {
@@ -6108,12 +6113,10 @@ static bool EvaluateBinaryTypeTrait(Sema &Self, TypeTrait BTT, const TypeSourceI
                                diag::err_incomplete_type);
     }
 
-    if (LhsT->isVariableArrayType())
-      Self.Diag(Lhs->getTypeLoc().getBeginLoc(), diag::err_vla_unsupported)
-          << 1 << tok::kw___is_pointer_interconvertible_base_of;
-    if (RhsT->isVariableArrayType())
-      Self.Diag(Rhs->getTypeLoc().getBeginLoc(), diag::err_vla_unsupported)
-          << 1 << tok::kw___is_pointer_interconvertible_base_of;
+    DiagnoseVLAInCXXTypeTrait(Self, Lhs,
+                              tok::kw___is_pointer_interconvertible_base_of);
+    DiagnoseVLAInCXXTypeTrait(Self, Rhs,
+                              tok::kw___is_pointer_interconvertible_base_of);
 
     return Self.IsPointerInterconvertibleBaseOf(Lhs, Rhs);
   }
-- 
GitLab


From ed06b847d4e77d0b81fa6b095366bb070db57846 Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser@berlin.de>
Date: Sun, 14 Apr 2024 12:25:49 +0200
Subject: [PATCH 164/464] Revert "[Clang] Reduce the size of Decl and classes
 derived from it" (#88654)

Reverts llvm/llvm-project#87361

On 32 bit platforms there is only a single bit available in the
`DeclCtx`, resulting in an assertion failure.
---
 clang/include/clang/AST/DeclBase.h        | 72 ++++++++---------------
 clang/lib/AST/DeclBase.cpp                | 31 +++-------
 clang/lib/Serialization/ASTReaderDecl.cpp |  2 +-
 3 files changed, 35 insertions(+), 70 deletions(-)

diff --git a/clang/include/clang/AST/DeclBase.h b/clang/include/clang/AST/DeclBase.h
index 4bee18767dd1..858450926455 100644
--- a/clang/include/clang/AST/DeclBase.h
+++ b/clang/include/clang/AST/DeclBase.h
@@ -268,37 +268,17 @@ private:
   ///   }
   ///   void A::f(); // SemanticDC == namespace 'A'
   ///                // LexicalDC == global namespace
+  llvm::PointerUnion<DeclContext*, MultipleDC*> DeclCtx;
 
-  // Compress the InvalidDecl and HasAttrs bits into DeclCtx to keep Decl below
-  // 32 bytes in size
-  llvm::PointerIntPair<
-      llvm::PointerIntPair<llvm::PointerUnion<DeclContext *, MultipleDC *>, 1,
-                           bool>,
-      1, bool>
-      DeclCtxWithInvalidDeclAndHasAttrs;
-
-  bool isInSemaDC() const {
-    return DeclCtxWithInvalidDeclAndHasAttrs.getPointer()
-        .getPointer()
-        .is<DeclContext *>();
-  }
-
-  bool isOutOfSemaDC() const {
-    return DeclCtxWithInvalidDeclAndHasAttrs.getPointer()
-        .getPointer()
-        .is<MultipleDC *>();
-  }
+  bool isInSemaDC() const { return DeclCtx.is<DeclContext*>(); }
+  bool isOutOfSemaDC() const { return DeclCtx.is<MultipleDC*>(); }
 
   MultipleDC *getMultipleDC() const {
-    return DeclCtxWithInvalidDeclAndHasAttrs.getPointer()
-        .getPointer()
-        .get<MultipleDC *>();
+    return DeclCtx.get<MultipleDC*>();
   }
 
   DeclContext *getSemanticDC() const {
-    return DeclCtxWithInvalidDeclAndHasAttrs.getPointer()
-        .getPointer()
-        .get<DeclContext *>();
+    return DeclCtx.get<DeclContext*>();
   }
 
   /// Loc - The location of this decl.
@@ -308,6 +288,14 @@ private:
   LLVM_PREFERRED_TYPE(Kind)
   unsigned DeclKind : 7;
 
+  /// InvalidDecl - This indicates a semantic error occurred.
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned InvalidDecl :  1;
+
+  /// HasAttrs - This indicates whether the decl has attributes or not.
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned HasAttrs : 1;
+
   /// Implicit - Whether this declaration was implicitly generated by
   /// the implementation rather than explicitly written by the user.
   LLVM_PREFERRED_TYPE(bool)
@@ -405,22 +393,21 @@ public:
 protected:
   Decl(Kind DK, DeclContext *DC, SourceLocation L)
       : NextInContextAndBits(nullptr, getModuleOwnershipKindForChildOf(DC)),
-        DeclCtxWithInvalidDeclAndHasAttrs({DC, false}, false), Loc(L),
-        DeclKind(DK), Implicit(false), Used(false), Referenced(false),
+        DeclCtx(DC), Loc(L), DeclKind(DK), InvalidDecl(false), HasAttrs(false),
+        Implicit(false), Used(false), Referenced(false),
         TopLevelDeclInObjCContainer(false), Access(AS_none), FromASTFile(0),
         IdentifierNamespace(getIdentifierNamespaceForKind(DK)),
         CacheValidAndLinkage(llvm::to_underlying(Linkage::Invalid)) {
-    if (StatisticsEnabled)
-      add(DK);
+    if (StatisticsEnabled) add(DK);
   }
 
   Decl(Kind DK, EmptyShell Empty)
-      : DeclKind(DK), Implicit(false), Used(false), Referenced(false),
-        TopLevelDeclInObjCContainer(false), Access(AS_none), FromASTFile(0),
+      : DeclKind(DK), InvalidDecl(false), HasAttrs(false), Implicit(false),
+        Used(false), Referenced(false), TopLevelDeclInObjCContainer(false),
+        Access(AS_none), FromASTFile(0),
         IdentifierNamespace(getIdentifierNamespaceForKind(DK)),
         CacheValidAndLinkage(llvm::to_underlying(Linkage::Invalid)) {
-    if (StatisticsEnabled)
-      add(DK);
+    if (StatisticsEnabled) add(DK);
   }
 
   virtual ~Decl();
@@ -533,9 +520,7 @@ public:
     return AccessSpecifier(Access);
   }
 
-  bool hasAttrs() const {
-    return DeclCtxWithInvalidDeclAndHasAttrs.getPointer().getInt();
-  }
+  bool hasAttrs() const { return HasAttrs; }
 
   void setAttrs(const AttrVec& Attrs) {
     return setAttrsImpl(Attrs, getASTContext());
@@ -564,17 +549,13 @@ public:
   }
 
   template <typename... Ts> void dropAttrs() {
-    if (!hasAttrs())
-      return;
+    if (!HasAttrs) return;
 
     AttrVec &Vec = getAttrs();
     llvm::erase_if(Vec, [](Attr *A) { return isa<Ts...>(A); });
 
-    if (Vec.empty()) {
-      auto InnerPtr = DeclCtxWithInvalidDeclAndHasAttrs.getPointer();
-      InnerPtr.setInt(false);
-      DeclCtxWithInvalidDeclAndHasAttrs.setPointer(InnerPtr);
-    }
+    if (Vec.empty())
+      HasAttrs = false;
   }
 
   template <typename T> void dropAttr() { dropAttrs<T>(); }
@@ -609,10 +590,7 @@ public:
   /// setInvalidDecl - Indicates the Decl had a semantic error. This
   /// allows for graceful error recovery.
   void setInvalidDecl(bool Invalid = true);
-
-  bool isInvalidDecl() const {
-    return DeclCtxWithInvalidDeclAndHasAttrs.getInt();
-  }
+  bool isInvalidDecl() const { return (bool) InvalidDecl; }
 
   /// isImplicit - Indicates whether the declaration was implicitly
   /// generated by the implementation. If false, this declaration
diff --git a/clang/lib/AST/DeclBase.cpp b/clang/lib/AST/DeclBase.cpp
index 8c83c711d882..66a727d9dd0c 100644
--- a/clang/lib/AST/DeclBase.cpp
+++ b/clang/lib/AST/DeclBase.cpp
@@ -52,8 +52,6 @@
 
 using namespace clang;
 
-static_assert(sizeof(Decl) <= 32, "Decl grew beyond 32 bytes!");
-
 //===----------------------------------------------------------------------===//
 //  Statistics
 //===----------------------------------------------------------------------===//
@@ -132,7 +130,7 @@ const char *Decl::getDeclKindName() const {
 }
 
 void Decl::setInvalidDecl(bool Invalid) {
-  DeclCtxWithInvalidDeclAndHasAttrs.setInt(Invalid);
+  InvalidDecl = Invalid;
   assert(!isa<TagDecl>(this) || !cast<TagDecl>(this)->isCompleteDefinition());
   if (!Invalid) {
     return;
@@ -336,9 +334,7 @@ void PrettyStackTraceDecl::print(raw_ostream &OS) const {
 Decl::~Decl() = default;
 
 void Decl::setDeclContext(DeclContext *DC) {
-  auto InnerPtr = DeclCtxWithInvalidDeclAndHasAttrs.getPointer();
-  InnerPtr.setPointer(DC);
-  DeclCtxWithInvalidDeclAndHasAttrs.setPointer(InnerPtr);
+  DeclCtx = DC;
 }
 
 void Decl::setLexicalDeclContext(DeclContext *DC) {
@@ -368,16 +364,12 @@ void Decl::setLexicalDeclContext(DeclContext *DC) {
 void Decl::setDeclContextsImpl(DeclContext *SemaDC, DeclContext *LexicalDC,
                                ASTContext &Ctx) {
   if (SemaDC == LexicalDC) {
-    auto InnerPtr = DeclCtxWithInvalidDeclAndHasAttrs.getPointer();
-    InnerPtr.setPointer(SemaDC);
-    DeclCtxWithInvalidDeclAndHasAttrs.setPointer(InnerPtr);
+    DeclCtx = SemaDC;
   } else {
     auto *MDC = new (Ctx) Decl::MultipleDC();
     MDC->SemanticDC = SemaDC;
     MDC->LexicalDC = LexicalDC;
-    auto InnerPtr = DeclCtxWithInvalidDeclAndHasAttrs.getPointer();
-    InnerPtr.setPointer(MDC);
-    DeclCtxWithInvalidDeclAndHasAttrs.setPointer(InnerPtr);
+    DeclCtx = MDC;
   }
 }
 
@@ -964,24 +956,19 @@ unsigned Decl::getIdentifierNamespaceForKind(Kind DeclKind) {
 }
 
 void Decl::setAttrsImpl(const AttrVec &attrs, ASTContext &Ctx) {
-  assert(!hasAttrs() && "Decl already contains attrs.");
+  assert(!HasAttrs && "Decl already contains attrs.");
 
   AttrVec &AttrBlank = Ctx.getDeclAttrs(this);
   assert(AttrBlank.empty() && "HasAttrs was wrong?");
 
   AttrBlank = attrs;
-  auto InnerPtr = DeclCtxWithInvalidDeclAndHasAttrs.getPointer();
-  InnerPtr.setInt(true);
-  DeclCtxWithInvalidDeclAndHasAttrs.setPointer(InnerPtr);
+  HasAttrs = true;
 }
 
 void Decl::dropAttrs() {
-  if (!hasAttrs())
-    return;
+  if (!HasAttrs) return;
 
-  auto InnerPtr = DeclCtxWithInvalidDeclAndHasAttrs.getPointer();
-  InnerPtr.setInt(false);
-  DeclCtxWithInvalidDeclAndHasAttrs.setPointer(InnerPtr);
+  HasAttrs = false;
   getASTContext().eraseDeclAttrs(this);
 }
 
@@ -1009,7 +996,7 @@ void Decl::addAttr(Attr *A) {
 }
 
 const AttrVec &Decl::getAttrs() const {
-  assert(hasAttrs() && "No attrs to get!");
+  assert(HasAttrs && "No attrs to get!");
   return getASTContext().getDeclAttrs(this);
 }
 
diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index c5dd89e1c8a4..ff72fc30c718 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -588,7 +588,7 @@ void ASTDeclReader::VisitDecl(Decl *D) {
   bool HasStandaloneLexicalDC = DeclBits.getNextBit();
   bool HasAttrs = DeclBits.getNextBit();
   D->setTopLevelDeclInObjCContainer(DeclBits.getNextBit());
-  D->DeclCtxWithInvalidDeclAndHasAttrs.setInt(DeclBits.getNextBit());
+  D->InvalidDecl = DeclBits.getNextBit();
   D->FromASTFile = true;
 
   if (D->isTemplateParameter() || D->isTemplateParameterPack() ||
-- 
GitLab


From ef164cee90477e294ff692209b4cf97a0e1958ed Mon Sep 17 00:00:00 2001
From: Sirraide <aeternalmail@gmail.com>
Date: Sun, 14 Apr 2024 12:30:01 +0200
Subject: [PATCH 165/464] [Clang] [C++26] Implement P2573R2: `= delete("should
 have a reason");` (#86526)

This implements support for the `= delete("message")` syntax that was
only just added to C++26
([P2573R2](https://isocpp.org/files/papers/P2573R2.html#proposal-scope)).
---
 clang/docs/LanguageExtensions.rst             |   1 +
 clang/docs/ReleaseNotes.rst                   |   2 +
 clang/include/clang/AST/Decl.h                |  47 ++-
 clang/include/clang/AST/DeclBase.h            |   2 +-
 .../clang/Basic/DiagnosticParseKinds.td       |   6 +
 .../clang/Basic/DiagnosticSemaKinds.td        |  17 +-
 clang/include/clang/Parse/Parser.h            |   2 +
 clang/include/clang/Sema/Sema.h               |  17 +-
 clang/lib/AST/ASTImporter.cpp                 |  13 +
 clang/lib/AST/Decl.cpp                        |  69 +++--
 clang/lib/AST/DeclPrinter.cpp                 |   9 +-
 clang/lib/AST/JSONNodeDumper.cpp              |   3 +
 clang/lib/AST/ODRHash.cpp                     |   6 +
 clang/lib/AST/TextNodeDumper.cpp              |   3 +
 clang/lib/Frontend/InitPreprocessor.cpp       |   3 +
 clang/lib/Parse/ParseCXXInlineMethods.cpp     |  46 ++-
 clang/lib/Parse/ParseDecl.cpp                 |   1 +
 clang/lib/Parse/ParseDeclCXX.cpp              |   1 +
 clang/lib/Parse/Parser.cpp                    |   4 +-
 clang/lib/Sema/SemaCast.cpp                   |  20 +-
 clang/lib/Sema/SemaDecl.cpp                   |  12 +-
 clang/lib/Sema/SemaDeclCXX.cpp                |  18 +-
 clang/lib/Sema/SemaExpr.cpp                   |   7 +-
 clang/lib/Sema/SemaExprCXX.cpp                |  23 +-
 clang/lib/Sema/SemaInit.cpp                   |  19 +-
 clang/lib/Sema/SemaOverload.cpp               |  84 ++++--
 .../lib/Sema/SemaTemplateInstantiateDecl.cpp  |  11 +-
 clang/lib/Serialization/ASTReaderDecl.cpp     |  18 +-
 clang/lib/Serialization/ASTWriterDecl.cpp     |  11 +-
 .../ast-dump-cxx2c-delete-with-message.cpp    |  23 ++
 .../ast-print-cxx2c-delete-with-message.cpp   |  18 ++
 clang/test/Lexer/cxx-features.cpp             |   4 +
 clang/test/PCH/cxx2a-defaulted-comparison.cpp |   2 +-
 .../test/Parser/cxx2c-delete-with-message.cpp |  51 ++++
 .../SemaCXX/cxx2c-delete-with-message.cpp     | 273 ++++++++++++++++++
 clang/www/cxx_status.html                     |   2 +-
 36 files changed, 728 insertions(+), 120 deletions(-)
 create mode 100644 clang/test/AST/ast-dump-cxx2c-delete-with-message.cpp
 create mode 100644 clang/test/AST/ast-print-cxx2c-delete-with-message.cpp
 create mode 100644 clang/test/Parser/cxx2c-delete-with-message.cpp
 create mode 100644 clang/test/SemaCXX/cxx2c-delete-with-message.cpp

diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst
index 96691b45d63a..05c8f765b556 100644
--- a/clang/docs/LanguageExtensions.rst
+++ b/clang/docs/LanguageExtensions.rst
@@ -1493,6 +1493,7 @@ Conditional ``explicit``                     __cpp_conditional_explicit       C+
 ``if consteval``                             __cpp_if_consteval               C++23         C++20
 ``static operator()``                        __cpp_static_call_operator       C++23         C++03
 Attributes on Lambda-Expressions                                              C++23         C++11
+``= delete ("should have a reason");``       __cpp_deleted_function           C++26         C++03
 -------------------------------------------- -------------------------------- ------------- -------------
 Designated initializers (N494)                                                C99           C89
 Array & element qualification (N2607)                                         C23           C89
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 399d5ba796d1..ade8f4e93d5a 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -129,6 +129,8 @@ C++2c Feature Support
 
 - Implemented `P2662R3 Pack Indexing <https://wg21.link/P2662R3>`_.
 
+- Implemented `P2573R2: = delete("should have a reason"); <https://wg21.link/P2573R2>`_
+
 
 Resolutions to C++ Defect Reports
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/include/clang/AST/Decl.h b/clang/include/clang/AST/Decl.h
index ed6790acdfc7..01af50ca694f 100644
--- a/clang/include/clang/AST/Decl.h
+++ b/clang/include/clang/AST/Decl.h
@@ -1993,21 +1993,35 @@ public:
 
   };
 
-  /// Stashed information about a defaulted function definition whose body has
-  /// not yet been lazily generated.
-  class DefaultedFunctionInfo final
-      : llvm::TrailingObjects<DefaultedFunctionInfo, DeclAccessPair> {
+  /// Stashed information about a defaulted/deleted function body.
+  class DefaultedOrDeletedFunctionInfo final
+      : llvm::TrailingObjects<DefaultedOrDeletedFunctionInfo, DeclAccessPair,
+                              StringLiteral *> {
     friend TrailingObjects;
     unsigned NumLookups;
+    bool HasDeletedMessage;
+
+    size_t numTrailingObjects(OverloadToken<DeclAccessPair>) const {
+      return NumLookups;
+    }
 
   public:
-    static DefaultedFunctionInfo *Create(ASTContext &Context,
-                                         ArrayRef<DeclAccessPair> Lookups);
+    static DefaultedOrDeletedFunctionInfo *
+    Create(ASTContext &Context, ArrayRef<DeclAccessPair> Lookups,
+           StringLiteral *DeletedMessage = nullptr);
+
     /// Get the unqualified lookup results that should be used in this
     /// defaulted function definition.
     ArrayRef<DeclAccessPair> getUnqualifiedLookups() const {
       return {getTrailingObjects<DeclAccessPair>(), NumLookups};
     }
+
+    StringLiteral *getDeletedMessage() const {
+      return HasDeletedMessage ? *getTrailingObjects<StringLiteral *>()
+                               : nullptr;
+    }
+
+    void setDeletedMessage(StringLiteral *Message);
   };
 
 private:
@@ -2017,12 +2031,12 @@ private:
   ParmVarDecl **ParamInfo = nullptr;
 
   /// The active member of this union is determined by
-  /// FunctionDeclBits.HasDefaultedFunctionInfo.
+  /// FunctionDeclBits.HasDefaultedOrDeletedInfo.
   union {
     /// The body of the function.
     LazyDeclStmtPtr Body;
     /// Information about a future defaulted function definition.
-    DefaultedFunctionInfo *DefaultedInfo;
+    DefaultedOrDeletedFunctionInfo *DefaultedOrDeletedInfo;
   };
 
   unsigned ODRHash;
@@ -2280,18 +2294,18 @@ public:
 
   /// Returns whether this specific declaration of the function has a body.
   bool doesThisDeclarationHaveABody() const {
-    return (!FunctionDeclBits.HasDefaultedFunctionInfo && Body) ||
+    return (!FunctionDeclBits.HasDefaultedOrDeletedInfo && Body) ||
            isLateTemplateParsed();
   }
 
   void setBody(Stmt *B);
   void setLazyBody(uint64_t Offset) {
-    FunctionDeclBits.HasDefaultedFunctionInfo = false;
+    FunctionDeclBits.HasDefaultedOrDeletedInfo = false;
     Body = LazyDeclStmtPtr(Offset);
   }
 
-  void setDefaultedFunctionInfo(DefaultedFunctionInfo *Info);
-  DefaultedFunctionInfo *getDefaultedFunctionInfo() const;
+  void setDefaultedOrDeletedInfo(DefaultedOrDeletedFunctionInfo *Info);
+  DefaultedOrDeletedFunctionInfo *getDefalutedOrDeletedInfo() const;
 
   /// Whether this function is variadic.
   bool isVariadic() const;
@@ -2494,7 +2508,7 @@ public:
     return FunctionDeclBits.IsDeleted && !isDefaulted();
   }
 
-  void setDeletedAsWritten(bool D = true) { FunctionDeclBits.IsDeleted = D; }
+  void setDeletedAsWritten(bool D = true, StringLiteral *Message = nullptr);
 
   /// Determines whether this function is "main", which is the
   /// entry point into an executable program.
@@ -2650,6 +2664,13 @@ public:
       AC.push_back(TRC);
   }
 
+  /// Get the message that indicates why this function was deleted.
+  StringLiteral *getDeletedMessage() const {
+    return FunctionDeclBits.HasDefaultedOrDeletedInfo
+               ? DefaultedOrDeletedInfo->getDeletedMessage()
+               : nullptr;
+  }
+
   void setPreviousDeclaration(FunctionDecl * PrevDecl);
 
   FunctionDecl *getCanonicalDecl() override;
diff --git a/clang/include/clang/AST/DeclBase.h b/clang/include/clang/AST/DeclBase.h
index 858450926455..2194d268fa86 100644
--- a/clang/include/clang/AST/DeclBase.h
+++ b/clang/include/clang/AST/DeclBase.h
@@ -1739,7 +1739,7 @@ class DeclContext {
     LLVM_PREFERRED_TYPE(bool)
     uint64_t IsExplicitlyDefaulted : 1;
     LLVM_PREFERRED_TYPE(bool)
-    uint64_t HasDefaultedFunctionInfo : 1;
+    uint64_t HasDefaultedOrDeletedInfo : 1;
 
     /// For member functions of complete types, whether this is an ineligible
     /// special member function or an unselected destructor. See
diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td
index 46a44418a315..bb9ca2a50cc0 100644
--- a/clang/include/clang/Basic/DiagnosticParseKinds.td
+++ b/clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -941,6 +941,12 @@ def warn_cxx98_compat_defaulted_deleted_function : Warning<
   "%select{defaulted|deleted}0 function definitions are incompatible with C++98">,
   InGroup<CXX98Compat>, DefaultIgnore;
 
+def ext_delete_with_message : ExtWarn<
+  "'= delete' with a message is a C++2c extension">, InGroup<CXX26>;
+def warn_cxx23_delete_with_message : Warning<
+  "'= delete' with a message is incompatible with C++ standards before C++2c">,
+  DefaultIgnore, InGroup<CXXPre26Compat>;
+
 // C++11 default member initialization
 def ext_nonstatic_member_init : ExtWarn<
   "default member initializer for non-static data member is a C++11 "
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 774d2b53a382..5ec0218aedfe 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -4683,11 +4683,10 @@ def err_ovl_no_viable_member_function_in_call : Error<
   "no matching member function for call to %0">;
 def err_ovl_ambiguous_call : Error<
   "call to %0 is ambiguous">;
-def err_ovl_deleted_call : Error<"call to deleted function %0">;
+def err_ovl_deleted_call : Error<"call to deleted"
+  "%select{| member}0 function %1%select{|: %3}2">;
 def err_ovl_ambiguous_member_call : Error<
   "call to member function %0 is ambiguous">;
-def err_ovl_deleted_member_call : Error<
-  "call to deleted member function %0">;
 def note_ovl_too_many_candidates : Note<
     "remaining %0 candidate%s0 omitted; "
     "pass -fshow-overloads=all to show them">;
@@ -4915,12 +4914,12 @@ def err_ovl_ambiguous_conversion_in_cast : Error<
   "dynamic_cast|C-style cast|functional-style cast|}0 from %1 to %2">;
 def err_ovl_deleted_conversion_in_cast : Error<
   "%select{|static_cast|reinterpret_cast|dynamic_cast|C-style cast|"
-  "functional-style cast|}0 from %1 to %2 uses deleted function">;
+  "functional-style cast|}0 from %1 to %2 uses deleted function%select{|: %4}3">;
 def err_ovl_ambiguous_init : Error<"call to constructor of %0 is ambiguous">;
 def err_ref_init_ambiguous : Error<
   "reference initialization of type %0 with initializer of type %1 is ambiguous">;
 def err_ovl_deleted_init : Error<
-  "call to deleted constructor of %0">;
+  "call to deleted constructor of %0%select{|: %2}1">;
 def err_ovl_deleted_special_init : Error<
   "call to implicitly-deleted %select{default constructor|copy constructor|"
   "move constructor|copy assignment operator|move assignment operator|"
@@ -4946,7 +4945,7 @@ def note_ovl_ambiguous_oper_binary_reversed_candidate : Note<
 def err_ovl_no_viable_oper : Error<"no viable overloaded '%0'">;
 def note_assign_lhs_incomplete : Note<"type %0 is incomplete">;
 def err_ovl_deleted_oper : Error<
-  "overload resolution selected deleted operator '%0'">;
+  "overload resolution selected deleted operator '%0'%select{|: %2}1">;
 def err_ovl_deleted_special_oper : Error<
   "object of type %0 cannot be %select{constructed|copied|moved|assigned|"
   "assigned|destroyed}1 because its %sub{select_special_member_kind}1 is "
@@ -4983,7 +4982,7 @@ def err_ovl_ambiguous_object_call : Error<
 def err_ovl_ambiguous_subscript_call : Error<
   "call to subscript operator of type %0 is ambiguous">;
 def err_ovl_deleted_object_call : Error<
-  "call to deleted function call operator in type %0">;
+  "call to deleted function call operator in type %0%select{|: %2}1">;
 def note_ovl_surrogate_cand : Note<"conversion candidate of type %0">;
 def err_member_call_without_object : Error<
   "call to %select{non-static|explicit}0 member function without an object argument">;
@@ -8284,7 +8283,7 @@ def err_typecheck_nonviable_condition_incomplete : Error<
   "no viable conversion%diff{ from $ to incomplete type $|}0,1">;
 def err_typecheck_deleted_function : Error<
   "conversion function %diff{from $ to $|between types}0,1 "
-  "invokes a deleted function">;
+  "invokes a deleted function%select{|: %3}2">;
 
 def err_expected_class_or_namespace : Error<"%0 is not a class"
   "%select{ or namespace|, namespace, or enumeration}1">;
@@ -8884,7 +8883,7 @@ def err_nontemporal_builtin_must_be_pointer_intfltptr_or_vector : Error<
   "address argument to nontemporal builtin must be a pointer to integer, float, "
   "pointer, or a vector of such types (%0 invalid)">;
 
-def err_deleted_function_use : Error<"attempt to use a deleted function">;
+def err_deleted_function_use : Error<"attempt to use a deleted function%select{|: %1}0">;
 def err_deleted_inherited_ctor_use : Error<
   "constructor inherited by %0 from base class %1 is implicitly deleted">;
 
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index c719218731c3..5950dd74cfe8 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -1601,6 +1601,8 @@ private:
                                      const ParsedTemplateInfo &TemplateInfo,
                                      const VirtSpecifiers &VS,
                                      SourceLocation PureSpecLoc);
+  StringLiteral *ParseCXXDeletedFunctionMessage();
+  void SkipDeletedFunctionBody();
   void ParseCXXNonStaticMemberInitializer(Decl *VarD);
   void ParseLexedAttributes(ParsingClass &Class);
   void ParseLexedAttributeList(LateParsedAttrList &LAs, Decl *D,
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 3b252f94dcbe..c6e0332c3176 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -3045,14 +3045,18 @@ public:
   void ActOnDocumentableDecls(ArrayRef<Decl *> Group);
 
   enum class FnBodyKind {
-    /// C++ [dcl.fct.def.general]p1
+    /// C++26 [dcl.fct.def.general]p1
     /// function-body:
     ///   ctor-initializer[opt] compound-statement
     ///   function-try-block
     Other,
     ///   = default ;
     Default,
+    ///   deleted-function-body
+    ///
+    /// deleted-function-body:
     ///   = delete ;
+    ///   = delete ( unevaluated-string ) ;
     Delete
   };
 
@@ -4751,10 +4755,12 @@ public:
                                SourceLocation EqualLoc);
 
   void ActOnPureSpecifier(Decl *D, SourceLocation PureSpecLoc);
-  void SetDeclDeleted(Decl *dcl, SourceLocation DelLoc);
+  void SetDeclDeleted(Decl *dcl, SourceLocation DelLoc,
+                      StringLiteral *Message = nullptr);
   void SetDeclDefaulted(Decl *dcl, SourceLocation DefaultLoc);
 
-  void SetFunctionBodyKind(Decl *D, SourceLocation Loc, FnBodyKind BodyKind);
+  void SetFunctionBodyKind(Decl *D, SourceLocation Loc, FnBodyKind BodyKind,
+                           StringLiteral *DeletedMessage = nullptr);
   void ActOnStartTrailingRequiresClause(Scope *S, Declarator &D);
   ExprResult ActOnFinishTrailingRequiresClause(ExprResult ConstraintExpr);
   ExprResult ActOnRequiresClause(ExprResult ConstraintExpr);
@@ -8093,6 +8099,11 @@ public:
   bool IsFunctionConversion(QualType FromType, QualType ToType,
                             QualType &ResultTy);
   bool DiagnoseMultipleUserDefinedConversion(Expr *From, QualType ToType);
+  void DiagnoseUseOfDeletedFunction(SourceLocation Loc, SourceRange Range,
+                                    DeclarationName Name,
+                                    OverloadCandidateSet &CandidateSet,
+                                    FunctionDecl *Fn, MultiExprArg Args,
+                                    bool IsMember = false);
 
   ExprResult InitializeExplicitObjectArgument(Sema &S, Expr *Obj,
                                               FunctionDecl *Fun);
diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp
index a5e43fc63166..6aaa34c55ce3 100644
--- a/clang/lib/AST/ASTImporter.cpp
+++ b/clang/lib/AST/ASTImporter.cpp
@@ -3947,6 +3947,14 @@ ExpectedDecl ASTNodeImporter::VisitFunctionDecl(FunctionDecl *D) {
     // decl and its redeclarations may be required.
   }
 
+  StringLiteral *Msg = D->getDeletedMessage();
+  if (Msg) {
+    auto Imported = import(Msg);
+    if (!Imported)
+      return Imported.takeError();
+    Msg = *Imported;
+  }
+
   ToFunction->setQualifierInfo(ToQualifierLoc);
   ToFunction->setAccess(D->getAccess());
   ToFunction->setLexicalDeclContext(LexicalDC);
@@ -3961,6 +3969,11 @@ ExpectedDecl ASTNodeImporter::VisitFunctionDecl(FunctionDecl *D) {
   ToFunction->setRangeEnd(ToEndLoc);
   ToFunction->setDefaultLoc(ToDefaultLoc);
 
+  if (Msg)
+    ToFunction->setDefaultedOrDeletedInfo(
+        FunctionDecl::DefaultedOrDeletedFunctionInfo::Create(
+            Importer.getToContext(), {}, Msg));
+
   // Set the parameters.
   for (auto *Param : Parameters) {
     Param->setOwningFunction(ToFunction);
diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp
index 60e0a3aecf6c..2b2d5a2663a1 100644
--- a/clang/lib/AST/Decl.cpp
+++ b/clang/lib/AST/Decl.cpp
@@ -3058,7 +3058,7 @@ FunctionDecl::FunctionDecl(Kind DK, ASTContext &C, DeclContext *DC,
   FunctionDeclBits.IsTrivialForCall = false;
   FunctionDeclBits.IsDefaulted = false;
   FunctionDeclBits.IsExplicitlyDefaulted = false;
-  FunctionDeclBits.HasDefaultedFunctionInfo = false;
+  FunctionDeclBits.HasDefaultedOrDeletedInfo = false;
   FunctionDeclBits.IsIneligibleOrNotSelected = false;
   FunctionDeclBits.HasImplicitReturnZero = false;
   FunctionDeclBits.IsLateTemplateParsed = false;
@@ -3092,30 +3092,65 @@ bool FunctionDecl::isVariadic() const {
   return false;
 }
 
-FunctionDecl::DefaultedFunctionInfo *
-FunctionDecl::DefaultedFunctionInfo::Create(ASTContext &Context,
-                                            ArrayRef<DeclAccessPair> Lookups) {
-  DefaultedFunctionInfo *Info = new (Context.Allocate(
-      totalSizeToAlloc<DeclAccessPair>(Lookups.size()),
-      std::max(alignof(DefaultedFunctionInfo), alignof(DeclAccessPair))))
-      DefaultedFunctionInfo;
+FunctionDecl::DefaultedOrDeletedFunctionInfo *
+FunctionDecl::DefaultedOrDeletedFunctionInfo::Create(
+    ASTContext &Context, ArrayRef<DeclAccessPair> Lookups,
+    StringLiteral *DeletedMessage) {
+  static constexpr size_t Alignment =
+      std::max({alignof(DefaultedOrDeletedFunctionInfo),
+                alignof(DeclAccessPair), alignof(StringLiteral *)});
+  size_t Size = totalSizeToAlloc<DeclAccessPair, StringLiteral *>(
+      Lookups.size(), DeletedMessage != nullptr);
+
+  DefaultedOrDeletedFunctionInfo *Info =
+      new (Context.Allocate(Size, Alignment)) DefaultedOrDeletedFunctionInfo;
   Info->NumLookups = Lookups.size();
+  Info->HasDeletedMessage = DeletedMessage != nullptr;
+
   std::uninitialized_copy(Lookups.begin(), Lookups.end(),
                           Info->getTrailingObjects<DeclAccessPair>());
+  if (DeletedMessage)
+    *Info->getTrailingObjects<StringLiteral *>() = DeletedMessage;
   return Info;
 }
 
-void FunctionDecl::setDefaultedFunctionInfo(DefaultedFunctionInfo *Info) {
-  assert(!FunctionDeclBits.HasDefaultedFunctionInfo && "already have this");
+void FunctionDecl::setDefaultedOrDeletedInfo(
+    DefaultedOrDeletedFunctionInfo *Info) {
+  assert(!FunctionDeclBits.HasDefaultedOrDeletedInfo && "already have this");
   assert(!Body && "can't replace function body with defaulted function info");
 
-  FunctionDeclBits.HasDefaultedFunctionInfo = true;
-  DefaultedInfo = Info;
+  FunctionDeclBits.HasDefaultedOrDeletedInfo = true;
+  DefaultedOrDeletedInfo = Info;
+}
+
+void FunctionDecl::setDeletedAsWritten(bool D, StringLiteral *Message) {
+  FunctionDeclBits.IsDeleted = D;
+
+  if (Message) {
+    assert(isDeletedAsWritten() && "Function must be deleted");
+    if (FunctionDeclBits.HasDefaultedOrDeletedInfo)
+      DefaultedOrDeletedInfo->setDeletedMessage(Message);
+    else
+      setDefaultedOrDeletedInfo(DefaultedOrDeletedFunctionInfo::Create(
+          getASTContext(), /*Lookups=*/{}, Message));
+  }
+}
+
+void FunctionDecl::DefaultedOrDeletedFunctionInfo::setDeletedMessage(
+    StringLiteral *Message) {
+  // We should never get here with the DefaultedOrDeletedInfo populated, but
+  // no space allocated for the deleted message, since that would require
+  // recreating this, but setDefaultedOrDeletedInfo() disallows overwriting
+  // an already existing DefaultedOrDeletedFunctionInfo.
+  assert(HasDeletedMessage &&
+         "No space to store a delete message in this DefaultedOrDeletedInfo");
+  *getTrailingObjects<StringLiteral *>() = Message;
 }
 
-FunctionDecl::DefaultedFunctionInfo *
-FunctionDecl::getDefaultedFunctionInfo() const {
-  return FunctionDeclBits.HasDefaultedFunctionInfo ? DefaultedInfo : nullptr;
+FunctionDecl::DefaultedOrDeletedFunctionInfo *
+FunctionDecl::getDefalutedOrDeletedInfo() const {
+  return FunctionDeclBits.HasDefaultedOrDeletedInfo ? DefaultedOrDeletedInfo
+                                                    : nullptr;
 }
 
 bool FunctionDecl::hasBody(const FunctionDecl *&Definition) const {
@@ -3202,7 +3237,7 @@ Stmt *FunctionDecl::getBody(const FunctionDecl *&Definition) const {
   if (!hasBody(Definition))
     return nullptr;
 
-  assert(!Definition->FunctionDeclBits.HasDefaultedFunctionInfo &&
+  assert(!Definition->FunctionDeclBits.HasDefaultedOrDeletedInfo &&
          "definition should not have a body");
   if (Definition->Body)
     return Definition->Body.get(getASTContext().getExternalSource());
@@ -3211,7 +3246,7 @@ Stmt *FunctionDecl::getBody(const FunctionDecl *&Definition) const {
 }
 
 void FunctionDecl::setBody(Stmt *B) {
-  FunctionDeclBits.HasDefaultedFunctionInfo = false;
+  FunctionDeclBits.HasDefaultedOrDeletedInfo = false;
   Body = LazyDeclStmtPtr(B);
   if (B)
     EndRangeLoc = B->getEndLoc();
diff --git a/clang/lib/AST/DeclPrinter.cpp b/clang/lib/AST/DeclPrinter.cpp
index c66774dd1df1..93857adb990b 100644
--- a/clang/lib/AST/DeclPrinter.cpp
+++ b/clang/lib/AST/DeclPrinter.cpp
@@ -822,9 +822,14 @@ void DeclPrinter::VisitFunctionDecl(FunctionDecl *D) {
 
   if (D->isPureVirtual())
     Out << " = 0";
-  else if (D->isDeletedAsWritten())
+  else if (D->isDeletedAsWritten()) {
     Out << " = delete";
-  else if (D->isExplicitlyDefaulted())
+    if (const StringLiteral *M = D->getDeletedMessage()) {
+      Out << "(";
+      M->outputString(Out);
+      Out << ")";
+    }
+  } else if (D->isExplicitlyDefaulted())
     Out << " = default";
   else if (D->doesThisDeclarationHaveABody()) {
     if (!Policy.TerseOutput) {
diff --git a/clang/lib/AST/JSONNodeDumper.cpp b/clang/lib/AST/JSONNodeDumper.cpp
index 78115200f4f0..42608476b1c1 100644
--- a/clang/lib/AST/JSONNodeDumper.cpp
+++ b/clang/lib/AST/JSONNodeDumper.cpp
@@ -975,6 +975,9 @@ void JSONNodeDumper::VisitFunctionDecl(const FunctionDecl *FD) {
   if (FD->isDefaulted())
     JOS.attribute("explicitlyDefaulted",
                   FD->isDeleted() ? "deleted" : "default");
+
+  if (StringLiteral *Msg = FD->getDeletedMessage())
+    JOS.attribute("deletedMessage", Msg->getString());
 }
 
 void JSONNodeDumper::VisitEnumDecl(const EnumDecl *ED) {
diff --git a/clang/lib/AST/ODRHash.cpp b/clang/lib/AST/ODRHash.cpp
index e159a1b00be5..6f04739cf669 100644
--- a/clang/lib/AST/ODRHash.cpp
+++ b/clang/lib/AST/ODRHash.cpp
@@ -696,6 +696,12 @@ void ODRHash::AddFunctionDecl(const FunctionDecl *Function,
   AddBoolean(Function->isDeletedAsWritten());
   AddBoolean(Function->isExplicitlyDefaulted());
 
+  StringLiteral *DeletedMessage = Function->getDeletedMessage();
+  AddBoolean(DeletedMessage);
+
+  if (DeletedMessage)
+    ID.AddString(DeletedMessage->getBytes());
+
   AddDecl(Function);
 
   AddQualType(Function->getReturnType());
diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp
index 0973e63183bd..688daa64d619 100644
--- a/clang/lib/AST/TextNodeDumper.cpp
+++ b/clang/lib/AST/TextNodeDumper.cpp
@@ -1961,6 +1961,9 @@ void TextNodeDumper::VisitFunctionDecl(const FunctionDecl *D) {
   if (D->isTrivial())
     OS << " trivial";
 
+  if (const StringLiteral *M = D->getDeletedMessage())
+    AddChild("delete message", [=] { Visit(M); });
+
   if (D->isIneligibleOrNotSelected())
     OS << (isa<CXXDestructorDecl>(D) ? " not_selected" : " ineligible");
 
diff --git a/clang/lib/Frontend/InitPreprocessor.cpp b/clang/lib/Frontend/InitPreprocessor.cpp
index 84069e96f414..4f44c3b7b89d 100644
--- a/clang/lib/Frontend/InitPreprocessor.cpp
+++ b/clang/lib/Frontend/InitPreprocessor.cpp
@@ -747,6 +747,9 @@ static void InitializeCPlusPlusFeatureTestMacros(const LangOptions &LangOpts,
   Builder.defineMacro("__cpp_named_character_escapes", "202207L");
   Builder.defineMacro("__cpp_placeholder_variables", "202306L");
 
+  // C++26 features supported in earlier language modes.
+  Builder.defineMacro("__cpp_deleted_function", "202403L");
+
   if (LangOpts.Char8)
     Builder.defineMacro("__cpp_char8_t", "202207L");
   Builder.defineMacro("__cpp_impl_destroying_delete", "201806L");
diff --git a/clang/lib/Parse/ParseCXXInlineMethods.cpp b/clang/lib/Parse/ParseCXXInlineMethods.cpp
index d790060c17c0..d054eda279b8 100644
--- a/clang/lib/Parse/ParseCXXInlineMethods.cpp
+++ b/clang/lib/Parse/ParseCXXInlineMethods.cpp
@@ -20,6 +20,49 @@
 
 using namespace clang;
 
+/// Parse the optional ("message") part of a deleted-function-body.
+StringLiteral *Parser::ParseCXXDeletedFunctionMessage() {
+  if (!Tok.is(tok::l_paren))
+    return nullptr;
+  StringLiteral *Message = nullptr;
+  BalancedDelimiterTracker BT{*this, tok::l_paren};
+  BT.consumeOpen();
+
+  if (isTokenStringLiteral()) {
+    ExprResult Res = ParseUnevaluatedStringLiteralExpression();
+    if (Res.isUsable()) {
+      Message = Res.getAs<StringLiteral>();
+      Diag(Message->getBeginLoc(), getLangOpts().CPlusPlus26
+                                       ? diag::warn_cxx23_delete_with_message
+                                       : diag::ext_delete_with_message)
+          << Message->getSourceRange();
+    }
+  } else {
+    Diag(Tok.getLocation(), diag::err_expected_string_literal)
+        << /*Source='in'*/ 0 << "'delete'";
+    SkipUntil(tok::r_paren, StopAtSemi | StopBeforeMatch);
+  }
+
+  BT.consumeClose();
+  return Message;
+}
+
+/// If we've encountered '= delete' in a context where it is ill-formed, such
+/// as in the declaration of a non-function, also skip the ("message") part if
+/// it is present to avoid issuing further diagnostics.
+void Parser::SkipDeletedFunctionBody() {
+  if (!Tok.is(tok::l_paren))
+    return;
+
+  BalancedDelimiterTracker BT{*this, tok::l_paren};
+  BT.consumeOpen();
+
+  // Just skip to the end of the current declaration.
+  SkipUntil(tok::r_paren, tok::comma, StopAtSemi | StopBeforeMatch);
+  if (Tok.is(tok::r_paren))
+    BT.consumeClose();
+}
+
 /// ParseCXXInlineMethodDef - We parsed and verified that the specified
 /// Declarator is a well formed C++ inline method definition. Now lex its body
 /// and store its tokens for parsing after the C++ class is complete.
@@ -70,7 +113,8 @@ NamedDecl *Parser::ParseCXXInlineMethodDef(
                       ? diag::warn_cxx98_compat_defaulted_deleted_function
                       : diag::ext_defaulted_deleted_function)
         << 1 /* deleted */;
-      Actions.SetDeclDeleted(FnD, KWLoc);
+      StringLiteral *Message = ParseCXXDeletedFunctionMessage();
+      Actions.SetDeclDeleted(FnD, KWLoc, Message);
       Delete = true;
       if (auto *DeclAsFunction = dyn_cast<FunctionDecl>(FnD)) {
         DeclAsFunction->setRangeEnd(KWEndLoc);
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index 951e2210031a..a990ef398c82 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -2678,6 +2678,7 @@ Decl *Parser::ParseDeclarationAfterDeclaratorAndAttributes(
           << 1 /* delete */;
       else
         Diag(ConsumeToken(), diag::err_deleted_non_function);
+      SkipDeletedFunctionBody();
     } else if (Tok.is(tok::kw_default)) {
       if (D.isFunctionDeclarator())
         Diag(ConsumeToken(), diag::err_default_delete_in_multiple_declaration)
diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp
index 477d81cdc2c2..cd4803d51bc1 100644
--- a/clang/lib/Parse/ParseDeclCXX.cpp
+++ b/clang/lib/Parse/ParseDeclCXX.cpp
@@ -3397,6 +3397,7 @@ ExprResult Parser::ParseCXXMemberInitializer(Decl *D, bool IsFunction,
               << 1 /* delete */;
         else
           Diag(ConsumeToken(), diag::err_deleted_non_function);
+        SkipDeletedFunctionBody();
         return ExprError();
       }
     } else if (Tok.is(tok::kw_default)) {
diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp
index cc0e41ed221c..d6f2b9f448cd 100644
--- a/clang/lib/Parse/Parser.cpp
+++ b/clang/lib/Parse/Parser.cpp
@@ -1404,6 +1404,7 @@ Decl *Parser::ParseFunctionDefinition(ParsingDeclarator &D,
 
   // Parse function body eagerly if it is either '= delete;' or '= default;' as
   // ActOnStartOfFunctionDef needs to know whether the function is deleted.
+  StringLiteral *DeletedMessage = nullptr;
   Sema::FnBodyKind BodyKind = Sema::FnBodyKind::Other;
   SourceLocation KWLoc;
   if (TryConsumeToken(tok::equal)) {
@@ -1415,6 +1416,7 @@ Decl *Parser::ParseFunctionDefinition(ParsingDeclarator &D,
                       : diag::ext_defaulted_deleted_function)
           << 1 /* deleted */;
       BodyKind = Sema::FnBodyKind::Delete;
+      DeletedMessage = ParseCXXDeletedFunctionMessage();
     } else if (TryConsumeToken(tok::kw_default, KWLoc)) {
       Diag(KWLoc, getLangOpts().CPlusPlus11
                       ? diag::warn_cxx98_compat_defaulted_deleted_function
@@ -1473,7 +1475,7 @@ Decl *Parser::ParseFunctionDefinition(ParsingDeclarator &D,
   D.getMutableDeclSpec().abort();
 
   if (BodyKind != Sema::FnBodyKind::Other) {
-    Actions.SetFunctionBodyKind(Res, KWLoc, BodyKind);
+    Actions.SetFunctionBodyKind(Res, KWLoc, BodyKind, DeletedMessage);
     Stmt *GeneratedBody = Res ? Res->getBody() : nullptr;
     Actions.ActOnFinishFunctionBody(Res, GeneratedBody, false);
     return Res;
diff --git a/clang/lib/Sema/SemaCast.cpp b/clang/lib/Sema/SemaCast.cpp
index 9d85568d97b2..aac43e2cad60 100644
--- a/clang/lib/Sema/SemaCast.cpp
+++ b/clang/lib/Sema/SemaCast.cpp
@@ -498,10 +498,22 @@ static bool tryDiagnoseOverloadedCast(Sema &S, CastType CT,
     howManyCandidates = OCD_AmbiguousCandidates;
     break;
 
-  case OR_Deleted:
-    msg = diag::err_ovl_deleted_conversion_in_cast;
-    howManyCandidates = OCD_ViableCandidates;
-    break;
+  case OR_Deleted: {
+    OverloadCandidateSet::iterator Best;
+    OverloadingResult Res =
+        candidates.BestViableFunction(S, range.getBegin(), Best);
+    assert(Res == OR_Deleted && "Inconsistent overload resolution");
+
+    StringLiteral *Msg = Best->Function->getDeletedMessage();
+    candidates.NoteCandidates(
+        PartialDiagnosticAt(range.getBegin(),
+                            S.PDiag(diag::err_ovl_deleted_conversion_in_cast)
+                                << CT << srcType << destType << (Msg != nullptr)
+                                << (Msg ? Msg->getString() : StringRef())
+                                << range << src->getSourceRange()),
+        S, OCD_ViableCandidates, src);
+    return true;
+  }
   }
 
   candidates.NoteCandidates(
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 17032d137052..8b3b9d020db5 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -16083,7 +16083,17 @@ Decl *Sema::ActOnFinishFunctionBody(Decl *dcl, Stmt *Body,
     // This is meant to pop the context added in ActOnStartOfFunctionDef().
     ExitFunctionBodyRAII ExitRAII(*this, isLambdaCallOperator(FD));
     if (FD) {
-      FD->setBody(Body);
+      // If this is called by Parser::ParseFunctionDefinition() after marking
+      // the declaration as deleted, and if the deleted-function-body contains
+      // a message (C++26), then a DefaultedOrDeletedInfo will have already been
+      // added to store that message; do not overwrite it in that case.
+      //
+      // Since this would always set the body to 'nullptr' in that case anyway,
+      // which is already done when the function decl is initially created,
+      // always skipping this irrespective of whether there is a delete message
+      // should not be a problem.
+      if (!FD->isDeletedAsWritten())
+        FD->setBody(Body);
       FD->setWillHaveBody(false);
       CheckImmediateEscalatingFunctionDefinition(FD, FSI);
 
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index 1fe10375222c..7669171fea56 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -7982,7 +7982,7 @@ public:
   DefaultedComparisonVisitor(Sema &S, CXXRecordDecl *RD, FunctionDecl *FD,
                              DefaultedComparisonKind DCK)
       : S(S), RD(RD), FD(FD), DCK(DCK) {
-    if (auto *Info = FD->getDefaultedFunctionInfo()) {
+    if (auto *Info = FD->getDefalutedOrDeletedInfo()) {
       // FIXME: Change CreateOverloadedBinOp to take an ArrayRef instead of an
       // UnresolvedSet to avoid this copy.
       Fns.assign(Info->getUnqualifiedLookups().begin(),
@@ -8850,8 +8850,9 @@ bool Sema::CheckExplicitlyDefaultedComparison(Scope *S, FunctionDecl *FD,
     UnresolvedSet<32> Operators;
     lookupOperatorsForDefaultedComparison(*this, S, Operators,
                                           FD->getOverloadedOperator());
-    FD->setDefaultedFunctionInfo(FunctionDecl::DefaultedFunctionInfo::Create(
-        Context, Operators.pairs()));
+    FD->setDefaultedOrDeletedInfo(
+        FunctionDecl::DefaultedOrDeletedFunctionInfo::Create(
+            Context, Operators.pairs()));
   }
 
   // C++2a [class.compare.default]p1:
@@ -18159,7 +18160,8 @@ NamedDecl *Sema::ActOnFriendFunctionDecl(Scope *S, Declarator &D,
   return ND;
 }
 
-void Sema::SetDeclDeleted(Decl *Dcl, SourceLocation DelLoc) {
+void Sema::SetDeclDeleted(Decl *Dcl, SourceLocation DelLoc,
+                          StringLiteral *Message) {
   AdjustDeclIfTemplate(Dcl);
 
   FunctionDecl *Fn = dyn_cast_or_null<FunctionDecl>(Dcl);
@@ -18208,7 +18210,7 @@ void Sema::SetDeclDeleted(Decl *Dcl, SourceLocation DelLoc) {
   // C++11 [dcl.fct.def.delete]p4:
   //  A deleted function is implicitly inline.
   Fn->setImplicitlyInline();
-  Fn->setDeletedAsWritten();
+  Fn->setDeletedAsWritten(true, Message);
 }
 
 void Sema::SetDeclDefaulted(Decl *Dcl, SourceLocation DefaultLoc) {
@@ -18321,11 +18323,11 @@ void Sema::DiagnoseReturnInConstructorExceptionHandler(CXXTryStmt *TryBlock) {
   }
 }
 
-void Sema::SetFunctionBodyKind(Decl *D, SourceLocation Loc,
-                               FnBodyKind BodyKind) {
+void Sema::SetFunctionBodyKind(Decl *D, SourceLocation Loc, FnBodyKind BodyKind,
+                               StringLiteral *DeletedMessage) {
   switch (BodyKind) {
   case FnBodyKind::Delete:
-    SetDeclDeleted(D, Loc);
+    SetDeclDeleted(D, Loc, DeletedMessage);
     break;
   case FnBodyKind::Default:
     SetDeclDefaulted(D, Loc);
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 823bf36d88bc..505d068ac42e 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -273,8 +273,11 @@ bool Sema::DiagnoseUseOfDecl(NamedDecl *D, ArrayRef<SourceLocation> Locs,
         Diag(Loc, diag::err_deleted_inherited_ctor_use)
             << Ctor->getParent()
             << Ctor->getInheritedConstructor().getConstructor()->getParent();
-      else
-        Diag(Loc, diag::err_deleted_function_use);
+      else {
+        StringLiteral *Msg = FD->getDeletedMessage();
+        Diag(Loc, diag::err_deleted_function_use)
+            << (Msg != nullptr) << (Msg ? Msg->getString() : StringRef());
+      }
       NoteDeletedFunction(FD);
       return true;
     }
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index d2f18d4aa9b9..25f23a3abf17 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -2716,13 +2716,9 @@ static bool resolveAllocationOverload(
     return true;
 
   case OR_Deleted: {
-    if (Diagnose) {
-      Candidates.NoteCandidates(
-          PartialDiagnosticAt(R.getNameLoc(),
-                              S.PDiag(diag::err_ovl_deleted_call)
-                                  << R.getLookupName() << Range),
-          S, OCD_AllCandidates, Args);
-    }
+    if (Diagnose)
+      S.DiagnoseUseOfDeletedFunction(R.getNameLoc(), Range, R.getLookupName(),
+                                     Candidates, Best->Function, Args);
     return true;
   }
   }
@@ -3376,7 +3372,9 @@ bool Sema::FindDeallocationFunction(SourceLocation StartLoc, CXXRecordDecl *RD,
     // FIXME: DiagnoseUseOfDecl?
     if (Operator->isDeleted()) {
       if (Diagnose) {
-        Diag(StartLoc, diag::err_deleted_function_use);
+        StringLiteral *Msg = Operator->getDeletedMessage();
+        Diag(StartLoc, diag::err_deleted_function_use)
+            << (Msg != nullptr) << (Msg ? Msg->getString() : StringRef());
         NoteDeletedFunction(Operator);
       }
       return true;
@@ -3980,14 +3978,11 @@ static bool resolveBuiltinNewDeleteOverload(Sema &S, CallExpr *TheCall,
         S, OCD_AmbiguousCandidates, Args);
     return true;
 
-  case OR_Deleted: {
-    Candidates.NoteCandidates(
-        PartialDiagnosticAt(R.getNameLoc(), S.PDiag(diag::err_ovl_deleted_call)
-                                                << R.getLookupName() << Range),
-        S, OCD_AllCandidates, Args);
+  case OR_Deleted:
+    S.DiagnoseUseOfDeletedFunction(R.getNameLoc(), Range, R.getLookupName(),
+                                   Candidates, Best->Function, Args);
     return true;
   }
-  }
   llvm_unreachable("Unreachable, bad result from BestViableFunction");
 }
 
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index 6b8ce0f633a3..fb7a80ab0284 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -9763,12 +9763,15 @@ bool InitializationSequence::Diagnose(Sema &S,
       break;
     }
     case OR_Deleted: {
-      S.Diag(Kind.getLocation(), diag::err_typecheck_deleted_function)
-        << OnlyArg->getType() << DestType.getNonReferenceType()
-        << Args[0]->getSourceRange();
       OverloadCandidateSet::iterator Best;
       OverloadingResult Ovl
         = FailedCandidateSet.BestViableFunction(S, Kind.getLocation(), Best);
+
+      StringLiteral *Msg = Best->Function->getDeletedMessage();
+      S.Diag(Kind.getLocation(), diag::err_typecheck_deleted_function)
+          << OnlyArg->getType() << DestType.getNonReferenceType()
+          << (Msg != nullptr) << (Msg ? Msg->getString() : StringRef())
+          << Args[0]->getSourceRange();
       if (Ovl == OR_Deleted) {
         S.NoteDeletedFunction(Best->Function);
       } else {
@@ -10027,9 +10030,12 @@ bool InitializationSequence::Diagnose(Sema &S,
               << llvm::to_underlying(
                      S.getSpecialMember(cast<CXXMethodDecl>(Best->Function)))
               << DestType << ArgsRange;
-        else
+        else {
+          StringLiteral *Msg = Best->Function->getDeletedMessage();
           S.Diag(Kind.getLocation(), diag::err_ovl_deleted_init)
-              << DestType << ArgsRange;
+              << DestType << (Msg != nullptr)
+              << (Msg ? Msg->getString() : StringRef()) << ArgsRange;
+        }
 
         S.NoteDeletedFunction(Best->Function);
         break;
@@ -11061,6 +11067,9 @@ QualType Sema::DeduceTemplateSpecializationFromInitializer(
   }
 
   case OR_Deleted: {
+    // FIXME: There are no tests for this diagnostic, and it doesn't seem
+    // like we ever get here; attempts to trigger this seem to yield a
+    // generic c'all to deleted function' diagnostic instead.
     Diag(Kind.getLocation(), diag::err_deduced_class_template_deleted)
       << TemplateName;
     NoteDeletedFunction(Best->Function);
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index 397e7681828f..675eb6ec05e7 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -14171,15 +14171,13 @@ static ExprResult FinishOverloadedCallExpr(Sema &SemaRef, Scope *S, Expr *Fn,
     break;
 
   case OR_Deleted: {
-    CandidateSet->NoteCandidates(
-        PartialDiagnosticAt(Fn->getBeginLoc(),
-                            SemaRef.PDiag(diag::err_ovl_deleted_call)
-                                << ULE->getName() << Fn->getSourceRange()),
-        SemaRef, OCD_AllCandidates, Args);
+    FunctionDecl *FDecl = (*Best)->Function;
+    SemaRef.DiagnoseUseOfDeletedFunction(Fn->getBeginLoc(),
+                                         Fn->getSourceRange(), ULE->getName(),
+                                         *CandidateSet, FDecl, Args);
 
     // We emitted an error for the unavailable/deleted function call but keep
     // the call in the AST.
-    FunctionDecl *FDecl = (*Best)->Function;
     ExprResult Res =
         SemaRef.FixOverloadedFunctionReference(Fn, (*Best)->FoundDecl, FDecl);
     if (Res.isInvalid())
@@ -14531,20 +14529,24 @@ Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, UnaryOperatorKind Opc,
         UnaryOperator::getOpcodeStr(Opc), OpLoc);
     return ExprError();
 
-  case OR_Deleted:
+  case OR_Deleted: {
     // CreateOverloadedUnaryOp fills the first element of ArgsArray with the
     // object whose method was called. Later in NoteCandidates size of ArgsArray
     // is passed further and it eventually ends up compared to number of
     // function candidate parameters which never includes the object parameter,
     // so slice ArgsArray to make sure apples are compared to apples.
+    StringLiteral *Msg = Best->Function->getDeletedMessage();
     CandidateSet.NoteCandidates(
         PartialDiagnosticAt(OpLoc, PDiag(diag::err_ovl_deleted_oper)
                                        << UnaryOperator::getOpcodeStr(Opc)
+                                       << (Msg != nullptr)
+                                       << (Msg ? Msg->getString() : StringRef())
                                        << Input->getSourceRange()),
         *this, OCD_AllCandidates, ArgsArray.drop_front(),
         UnaryOperator::getOpcodeStr(Opc), OpLoc);
     return ExprError();
   }
+  }
 
   // Either we found no viable overloaded operator or we matched a
   // built-in operator. In either case, fall through to trying to
@@ -15061,7 +15063,7 @@ ExprResult Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
           OpLoc);
       return ExprError();
 
-    case OR_Deleted:
+    case OR_Deleted: {
       if (isImplicitlyDeleted(Best->Function)) {
         FunctionDecl *DeletedFD = Best->Function;
         DefaultedFunctionKind DFK = getDefaultedFunctionKind(DeletedFD);
@@ -15080,16 +15082,20 @@ ExprResult Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
         NoteDeletedFunction(DeletedFD);
         return ExprError();
       }
+
+      StringLiteral *Msg = Best->Function->getDeletedMessage();
       CandidateSet.NoteCandidates(
           PartialDiagnosticAt(
-              OpLoc, PDiag(diag::err_ovl_deleted_oper)
-                         << getOperatorSpelling(Best->Function->getDeclName()
-                                                    .getCXXOverloadedOperator())
-                         << Args[0]->getSourceRange()
-                         << Args[1]->getSourceRange()),
+              OpLoc,
+              PDiag(diag::err_ovl_deleted_oper)
+                  << getOperatorSpelling(Best->Function->getDeclName()
+                                             .getCXXOverloadedOperator())
+                  << (Msg != nullptr) << (Msg ? Msg->getString() : StringRef())
+                  << Args[0]->getSourceRange() << Args[1]->getSourceRange()),
           *this, OCD_AllCandidates, Args, BinaryOperator::getOpcodeStr(Opc),
           OpLoc);
       return ExprError();
+    }
   }
 
   // We matched a built-in operator; build it.
@@ -15401,14 +15407,18 @@ ExprResult Sema::CreateOverloadedArraySubscriptExpr(SourceLocation LLoc,
       }
       return ExprError();
 
-    case OR_Deleted:
+    case OR_Deleted: {
+      StringLiteral *Msg = Best->Function->getDeletedMessage();
       CandidateSet.NoteCandidates(
-          PartialDiagnosticAt(LLoc, PDiag(diag::err_ovl_deleted_oper)
-                                        << "[]" << Args[0]->getSourceRange()
-                                        << Range),
+          PartialDiagnosticAt(LLoc,
+                              PDiag(diag::err_ovl_deleted_oper)
+                                  << "[]" << (Msg != nullptr)
+                                  << (Msg ? Msg->getString() : StringRef())
+                                  << Args[0]->getSourceRange() << Range),
           *this, OCD_AllCandidates, Args, "[]", LLoc);
       return ExprError();
     }
+    }
 
   // We matched a built-in operator; build it.
   return CreateBuiltinArraySubscriptExpr(Args[0], LLoc, Args[1], RLoc);
@@ -15622,11 +15632,9 @@ ExprResult Sema::BuildCallToMemberFunction(Scope *S, Expr *MemExprE,
           *this, OCD_AmbiguousCandidates, Args);
       break;
     case OR_Deleted:
-      CandidateSet.NoteCandidates(
-          PartialDiagnosticAt(UnresExpr->getMemberLoc(),
-                              PDiag(diag::err_ovl_deleted_member_call)
-                                  << DeclName << MemExprE->getSourceRange()),
-          *this, OCD_AllCandidates, Args);
+      DiagnoseUseOfDeletedFunction(
+          UnresExpr->getMemberLoc(), MemExprE->getSourceRange(), DeclName,
+          CandidateSet, Best->Function, Args, /*IsMember=*/true);
       break;
     }
     // Overload resolution fails, try to recover.
@@ -15890,15 +15898,21 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Obj,
           *this, OCD_AmbiguousCandidates, Args);
     break;
 
-  case OR_Deleted:
+  case OR_Deleted: {
+    // FIXME: Is this diagnostic here really necessary? It seems that
+    //   1. we don't have any tests for this diagnostic, and
+    //   2. we already issue err_deleted_function_use for this later on anyway.
+    StringLiteral *Msg = Best->Function->getDeletedMessage();
     CandidateSet.NoteCandidates(
         PartialDiagnosticAt(Object.get()->getBeginLoc(),
                             PDiag(diag::err_ovl_deleted_object_call)
-                                << Object.get()->getType()
+                                << Object.get()->getType() << (Msg != nullptr)
+                                << (Msg ? Msg->getString() : StringRef())
                                 << Object.get()->getSourceRange()),
         *this, OCD_AllCandidates, Args);
     break;
   }
+  }
 
   if (Best == CandidateSet.end())
     return true;
@@ -16097,13 +16111,17 @@ Sema::BuildOverloadedArrowExpr(Scope *S, Expr *Base, SourceLocation OpLoc,
           *this, OCD_AmbiguousCandidates, Base);
     return ExprError();
 
-  case OR_Deleted:
+  case OR_Deleted: {
+    StringLiteral *Msg = Best->Function->getDeletedMessage();
     CandidateSet.NoteCandidates(
         PartialDiagnosticAt(OpLoc, PDiag(diag::err_ovl_deleted_oper)
-                                       << "->" << Base->getSourceRange()),
+                                       << "->" << (Msg != nullptr)
+                                       << (Msg ? Msg->getString() : StringRef())
+                                       << Base->getSourceRange()),
         *this, OCD_AllCandidates, Base);
     return ExprError();
   }
+  }
 
   CheckMemberOperatorAccess(OpLoc, Base, nullptr, Best->FoundDecl);
 
@@ -16517,3 +16535,17 @@ bool clang::shouldEnforceArgLimit(bool PartialOverloading,
         return false;
   return true;
 }
+
+void Sema::DiagnoseUseOfDeletedFunction(SourceLocation Loc, SourceRange Range,
+                                        DeclarationName Name,
+                                        OverloadCandidateSet &CandidateSet,
+                                        FunctionDecl *Fn, MultiExprArg Args,
+                                        bool IsMember) {
+  StringLiteral *Msg = Fn->getDeletedMessage();
+  CandidateSet.NoteCandidates(
+      PartialDiagnosticAt(Loc, PDiag(diag::err_ovl_deleted_call)
+                                   << IsMember << Name << (Msg != nullptr)
+                                   << (Msg ? Msg->getString() : StringRef())
+                                   << Range),
+      *this, OCD_AllCandidates, Args);
+}
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index c0469a47ab8b..21a139e434cb 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -2439,7 +2439,7 @@ Decl *TemplateDeclInstantiator::VisitFunctionDecl(
       return nullptr;
   }
   if (D->isDeleted())
-    SemaRef.SetDeclDeleted(Function, D->getLocation());
+    SemaRef.SetDeclDeleted(Function, D->getLocation(), D->getDeletedMessage());
 
   NamedDecl *PrincipalDecl =
       (TemplateParams ? cast<NamedDecl>(FunctionTemplate) : Function);
@@ -2815,7 +2815,8 @@ Decl *TemplateDeclInstantiator::VisitCXXMethodDecl(
       return nullptr;
   }
   if (D->isDeletedAsWritten())
-    SemaRef.SetDeclDeleted(Method, Method->getLocation());
+    SemaRef.SetDeclDeleted(Method, Method->getLocation(),
+                           D->getDeletedMessage());
 
   // If this is an explicit specialization, mark the implicitly-instantiated
   // template specialization as being an explicit specialization too.
@@ -4867,7 +4868,7 @@ TemplateDeclInstantiator::InitMethodInstantiation(CXXMethodDecl *New,
 bool TemplateDeclInstantiator::SubstDefaultedFunction(FunctionDecl *New,
                                                       FunctionDecl *Tmpl) {
   // Transfer across any unqualified lookups.
-  if (auto *DFI = Tmpl->getDefaultedFunctionInfo()) {
+  if (auto *DFI = Tmpl->getDefalutedOrDeletedInfo()) {
     SmallVector<DeclAccessPair, 32> Lookups;
     Lookups.reserve(DFI->getUnqualifiedLookups().size());
     bool AnyChanged = false;
@@ -4882,8 +4883,8 @@ bool TemplateDeclInstantiator::SubstDefaultedFunction(FunctionDecl *New,
 
     // It's unlikely that substitution will change any declarations. Don't
     // store an unnecessary copy in that case.
-    New->setDefaultedFunctionInfo(
-        AnyChanged ? FunctionDecl::DefaultedFunctionInfo::Create(
+    New->setDefaultedOrDeletedInfo(
+        AnyChanged ? FunctionDecl::DefaultedOrDeletedFunctionInfo::Create(
                          SemaRef.Context, Lookups)
                    : DFI);
   }
diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index ff72fc30c718..e4b6a75c118b 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -1103,16 +1103,26 @@ void ASTDeclReader::VisitFunctionDecl(FunctionDecl *FD) {
     FD->setHasODRHash(true);
   }
 
-  if (FD->isDefaulted()) {
-    if (unsigned NumLookups = Record.readInt()) {
+  if (FD->isDefaulted() || FD->isDeletedAsWritten()) {
+    // If 'Info' is nonzero, we need to read an DefaultedOrDeletedInfo; if,
+    // additionally, the second bit is also set, we also need to read
+    // a DeletedMessage for the DefaultedOrDeletedInfo.
+    if (auto Info = Record.readInt()) {
+      bool HasMessage = Info & 2;
+      StringLiteral *DeletedMessage =
+          HasMessage ? cast<StringLiteral>(Record.readExpr()) : nullptr;
+
+      unsigned NumLookups = Record.readInt();
       SmallVector<DeclAccessPair, 8> Lookups;
       for (unsigned I = 0; I != NumLookups; ++I) {
         NamedDecl *ND = Record.readDeclAs<NamedDecl>();
         AccessSpecifier AS = (AccessSpecifier)Record.readInt();
         Lookups.push_back(DeclAccessPair::make(ND, AS));
       }
-      FD->setDefaultedFunctionInfo(FunctionDecl::DefaultedFunctionInfo::Create(
-          Reader.getContext(), Lookups));
+
+      FD->setDefaultedOrDeletedInfo(
+          FunctionDecl::DefaultedOrDeletedFunctionInfo::Create(
+              Reader.getContext(), Lookups, DeletedMessage));
     }
   }
 
diff --git a/clang/lib/Serialization/ASTWriterDecl.cpp b/clang/lib/Serialization/ASTWriterDecl.cpp
index 4b417909751f..276b6257f1d8 100644
--- a/clang/lib/Serialization/ASTWriterDecl.cpp
+++ b/clang/lib/Serialization/ASTWriterDecl.cpp
@@ -749,8 +749,15 @@ void ASTDeclWriter::VisitFunctionDecl(FunctionDecl *D) {
   if (!ShouldSkipCheckingODR)
     Record.push_back(D->getODRHash());
 
-  if (D->isDefaulted()) {
-    if (auto *FDI = D->getDefaultedFunctionInfo()) {
+  if (D->isDefaulted() || D->isDeletedAsWritten()) {
+    if (auto *FDI = D->getDefalutedOrDeletedInfo()) {
+      // Store both that there is an DefaultedOrDeletedInfo and whether it
+      // contains a DeletedMessage.
+      StringLiteral *DeletedMessage = FDI->getDeletedMessage();
+      Record.push_back(1 | (DeletedMessage ? 2 : 0));
+      if (DeletedMessage)
+        Record.AddStmt(DeletedMessage);
+
       Record.push_back(FDI->getUnqualifiedLookups().size());
       for (DeclAccessPair P : FDI->getUnqualifiedLookups()) {
         Record.AddDeclRef(P.getDecl());
diff --git a/clang/test/AST/ast-dump-cxx2c-delete-with-message.cpp b/clang/test/AST/ast-dump-cxx2c-delete-with-message.cpp
new file mode 100644
index 000000000000..ea16b97da23e
--- /dev/null
+++ b/clang/test/AST/ast-dump-cxx2c-delete-with-message.cpp
@@ -0,0 +1,23 @@
+// Without serialization:
+// RUN: %clang_cc1 -ast-dump %s | FileCheck %s
+//
+// With serialization:
+// RUN: %clang_cc1 -emit-pch -o %t %s
+// RUN: %clang_cc1 -x c++ -include-pch %t -ast-dump-all  /dev/null | FileCheck %s
+
+struct S {
+  // CHECK:      CXXMethodDecl {{.*}} a 'void ()' delete
+  // CHECK-NEXT:   delete message: StringLiteral {{.*}} "foo"
+  void a() = delete("foo");
+
+  // CHECK:      FunctionTemplateDecl {{.*}} b
+  // CHECK-NEXT:   TemplateTypeParmDecl
+  // CHECK-NEXT:   CXXMethodDecl {{.*}} b 'void ()' delete
+  // CHECK-NEXT:     delete message: StringLiteral {{.*}} "bar"
+  template <typename>
+  void b() = delete("bar");
+};
+
+// CHECK:      FunctionDecl {{.*}} c 'void ()' delete
+// CHECK-NEXT:   delete message: StringLiteral {{.*}} "baz"
+void c() = delete("baz");
diff --git a/clang/test/AST/ast-print-cxx2c-delete-with-message.cpp b/clang/test/AST/ast-print-cxx2c-delete-with-message.cpp
new file mode 100644
index 000000000000..11e037e4d744
--- /dev/null
+++ b/clang/test/AST/ast-print-cxx2c-delete-with-message.cpp
@@ -0,0 +1,18 @@
+// Without serialization:
+// RUN: %clang_cc1 -ast-print %s | FileCheck %s
+//
+// With serialization:
+// RUN: %clang_cc1 -emit-pch -o %t %s
+// RUN: %clang_cc1 -x c++ -include-pch %t -ast-print  /dev/null | FileCheck %s
+
+// CHECK: struct S {
+struct S {
+  // CHECK-NEXT: void a() = delete("foo");
+  void a() = delete("foo");
+
+  // CHECK-NEXT: template <typename T> T b() = delete("bar");
+  template <typename T> T b() = delete("bar");
+};
+
+// CHECK: void c() = delete("baz");
+void c() = delete("baz");
diff --git a/clang/test/Lexer/cxx-features.cpp b/clang/test/Lexer/cxx-features.cpp
index 1de32498cd34..baaa9d4434e9 100644
--- a/clang/test/Lexer/cxx-features.cpp
+++ b/clang/test/Lexer/cxx-features.cpp
@@ -34,6 +34,10 @@
 
 // --- C++26 features ---
 
+#if check(deleted_function, 202403, 202403, 202403, 202403, 202403, 202403, 202403)
+#error "wrong value for __cpp_deleted_function"
+#endif
+
 #if check(placeholder_variables, 202306, 202306, 202306, 202306, 202306, 202306, 202306)
 #error "wrong value for __cpp_placeholder_variables"
 #endif
diff --git a/clang/test/PCH/cxx2a-defaulted-comparison.cpp b/clang/test/PCH/cxx2a-defaulted-comparison.cpp
index 8aeb1683961a..bdf649919659 100644
--- a/clang/test/PCH/cxx2a-defaulted-comparison.cpp
+++ b/clang/test/PCH/cxx2a-defaulted-comparison.cpp
@@ -22,7 +22,7 @@ namespace std {
   constexpr strong_ordering strong_ordering::less = {-1};
 }
 
-// Ensure that we can round-trip DefaultedFunctionInfo through an AST file.
+// Ensure that we can round-trip DefaultedOrDeletedInfo through an AST file.
 namespace LookupContext {
   struct A {};
 
diff --git a/clang/test/Parser/cxx2c-delete-with-message.cpp b/clang/test/Parser/cxx2c-delete-with-message.cpp
new file mode 100644
index 000000000000..1767a080a7dc
--- /dev/null
+++ b/clang/test/Parser/cxx2c-delete-with-message.cpp
@@ -0,0 +1,51 @@
+// RUN: %clang_cc1 -std=c++23 -fsyntax-only -verify=expected,pre26 -pedantic %s
+// RUN: %clang_cc1 -std=c++2c -fsyntax-only -verify=expected,compat -Wpre-c++26-compat %s
+// RUN: %clang_cc1 -std=c++2c -fsyntax-only -verify %s
+
+struct S {
+  void a() = delete;
+  void b() = delete(; // expected-error {{expected string literal}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+  void c() = delete(); // expected-error {{expected string literal}}
+  void d() = delete(42); // expected-error {{expected string literal}}
+  void e() = delete("foo"[0]); // expected-error {{expected ')'}} expected-note {{to match this '('}} // pre26-warning {{'= delete' with a message is a C++2c extension}} compat-warning {{'= delete' with a message is incompatible with C++ standards before C++2c}}
+  void f() = delete("foo"); // pre26-warning {{'= delete' with a message is a C++2c extension}} compat-warning {{'= delete' with a message is incompatible with C++ standards before C++2c}}
+
+  S() = delete("foo"); // pre26-warning {{'= delete' with a message is a C++2c extension}} compat-warning {{'= delete' with a message is incompatible with C++ standards before C++2c}}
+  ~S() = delete("foo"); // pre26-warning {{'= delete' with a message is a C++2c extension}} compat-warning {{'= delete' with a message is incompatible with C++ standards before C++2c}}
+  S(const S&) = delete("foo"); // pre26-warning {{'= delete' with a message is a C++2c extension}} compat-warning {{'= delete' with a message is incompatible with C++ standards before C++2c}}
+  S(S&&) = delete("foo"); // pre26-warning {{'= delete' with a message is a C++2c extension}} compat-warning {{'= delete' with a message is incompatible with C++ standards before C++2c}}
+  S& operator=(const S&) = delete("foo"); // pre26-warning {{'= delete' with a message is a C++2c extension}} compat-warning {{'= delete' with a message is incompatible with C++ standards before C++2c}}
+  S& operator=(S&&) = delete("foo"); // pre26-warning {{'= delete' with a message is a C++2c extension}} compat-warning {{'= delete' with a message is incompatible with C++ standards before C++2c}}
+};
+
+struct T {
+  T() = delete(); // expected-error {{expected string literal}}
+  ~T() = delete(); // expected-error {{expected string literal}}
+  T(const T&) = delete(); // expected-error {{expected string literal}}
+  T(T&&) = delete(); // expected-error {{expected string literal}}
+  T& operator=(const T&) = delete(); // expected-error {{expected string literal}}
+  T& operator=(T&&) = delete(); // expected-error {{expected string literal}}
+};
+
+void a() = delete;
+void b() = delete(; // expected-error {{expected string literal}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+void c() = delete(); // expected-error {{expected string literal}}
+void d() = delete(42); // expected-error {{expected string literal}}
+void e() = delete("foo"[0]); // expected-error {{expected ')'}} expected-note {{to match this '('}} // pre26-warning {{'= delete' with a message is a C++2c extension}} compat-warning {{'= delete' with a message is incompatible with C++ standards before C++2c}}
+void f() = delete("foo"); // pre26-warning {{'= delete' with a message is a C++2c extension}} compat-warning {{'= delete' with a message is incompatible with C++ standards before C++2c}}
+
+constexpr const char *getMsg() { return "this is a message"; }
+void func() = delete(getMsg()); // expected-error {{expected string literal}}
+
+namespace CWG2876 {
+using T = void ();
+using U = int;
+
+T a = delete ("hello"); // expected-error {{only functions can have deleted definitions}}
+U b = delete ("hello"), c, d = delete ("hello"); // expected-error 2 {{only functions can have deleted definitions}}
+
+struct C {
+  T e = delete ("hello"); // expected-error {{'= delete' is a function definition and must occur in a standalone declaration}}
+  U f = delete ("hello"); // expected-error {{cannot delete expression of type 'const char[6]'}}
+};
+}
diff --git a/clang/test/SemaCXX/cxx2c-delete-with-message.cpp b/clang/test/SemaCXX/cxx2c-delete-with-message.cpp
new file mode 100644
index 000000000000..22e65d902ecd
--- /dev/null
+++ b/clang/test/SemaCXX/cxx2c-delete-with-message.cpp
@@ -0,0 +1,273 @@
+// RUN: %clang_cc1 -std=c++2c -fsyntax-only -verify %s
+
+struct S {
+  void f() = delete("deleted (1)"); // expected-note {{explicitly marked deleted}}
+
+  template <typename T>
+  T g() = delete("deleted (2)"); // expected-note {{explicitly deleted}}
+};
+
+template <typename T>
+struct TS {
+  T f() = delete("deleted (3)"); // expected-note {{explicitly marked deleted}}
+
+  template <typename U>
+  T g(U) = delete("deleted (4)"); // expected-note {{explicitly deleted}}
+};
+
+void f() = delete("deleted (5)"); // expected-note {{explicitly deleted}}
+
+template <typename T>
+T g() = delete("deleted (6)"); // expected-note {{explicitly deleted}}
+
+template <typename T, typename U>
+struct TSp {
+  T f() = delete("deleted (7)"); // expected-note 2 {{explicitly marked deleted}}
+};
+
+template <typename T>
+struct TSp<T, int> {
+  T f() = delete("deleted (8)"); // expected-note {{explicitly marked deleted}}
+};
+
+template <>
+struct TSp<int, int> {
+  int f() = delete("deleted (9)"); // expected-note {{explicitly marked deleted}}
+};
+
+void u1() = delete(L"\xFFFFFFFF"); // expected-error {{an unevaluated string literal cannot have an encoding prefix}} \
+                                   // expected-error {{invalid escape sequence '\xFFFFFFFF' in an unevaluated string literal}}
+void u2() = delete(u"\U000317FF"); // expected-error {{an unevaluated string literal cannot have an encoding prefix}}
+
+void u3() = delete("Ω"); // expected-note {{explicitly deleted}}
+void u4() = delete("\u1234"); // expected-note {{explicitly deleted}}
+
+void u5() = delete("\x1ff"       // expected-error {{hex escape sequence out of range}} \
+                                 // expected-error {{invalid escape sequence '\x1ff' in an unevaluated string literal}}
+                     "0\x123"    // expected-error {{invalid escape sequence '\x123' in an unevaluated string literal}}
+                     "fx\xfffff" // expected-error {{invalid escape sequence '\xfffff' in an unevaluated string literal}}
+                     "goop");
+
+void u6() = delete("\'\"\?\\\a\b\f\n\r\t\v"); // expected-note {{explicitly deleted}}
+void u7() = delete("\xFF"); // expected-error {{invalid escape sequence '\xFF' in an unevaluated string literal}}
+void u8() = delete("\123"); // expected-error {{invalid escape sequence '\123' in an unevaluated string literal}}
+void u9() = delete("\pOh no, a Pascal string!"); // expected-warning {{unknown escape sequence '\p'}} \
+                                                 // expected-error {{invalid escape sequence '\p' in an unevaluated string literal}}
+// expected-note@+1 {{explicitly deleted}}
+void u10() = delete(R"(a
+\tb
+c
+)");
+
+void u11() = delete("\u0080\u0081\u0082\u0083\u0099\u009A\u009B\u009C\u009D\u009E\u009F"); // expected-note {{explicitly deleted}}
+
+
+//! Contains RTL/LTR marks
+void u12() = delete("\u200Eabc\u200Fdef\u200Fgh"); // expected-note {{explicitly deleted}}
+
+//! Contains ZWJ/regional indicators
+void u13() = delete("🏳️‍🌈 🏴󠁧󠁢󠁥󠁮󠁧󠁿 🇪🇺"); // expected-note {{explicitly deleted}}
+
+void h() {
+  S{}.f(); // expected-error {{attempt to use a deleted function: deleted (1)}}
+  S{}.g<int>(); // expected-error {{call to deleted member function 'g': deleted (2)}}
+  TS<int>{}.f(); // expected-error {{attempt to use a deleted function: deleted (3)}}
+  TS<int>{}.g<int>(0); // expected-error {{call to deleted member function 'g': deleted (4)}}
+  f(); // expected-error {{call to deleted function 'f': deleted (5)}}
+  g<int>(); // expected-error {{call to deleted function 'g': deleted (6)}}
+  TSp<double, double>{}.f(); // expected-error {{attempt to use a deleted function: deleted (7)}}
+  TSp<int, double>{}.f(); // expected-error {{attempt to use a deleted function: deleted (7)}}
+  TSp<double, int>{}.f(); // expected-error {{attempt to use a deleted function: deleted (8)}}
+  TSp<int, int>{}.f(); // expected-error {{attempt to use a deleted function: deleted (9)}}
+  u3(); // expected-error {{call to deleted function 'u3': Ω}}
+  u4(); // expected-error {{call to deleted function 'u4': ሴ}}
+  u6(); // expected-error {{call to deleted function 'u6': '"?\<U+0007><U+0008>}}
+  u10(); // expected-error {{call to deleted function 'u10': a\n\tb\nc\n}}
+  u11(); // expected-error {{call to deleted function 'u11': <U+0080><U+0081><U+0082><U+0083><U+0099><U+009A><U+009B><U+009C><U+009D><U+009E><U+009F>}}
+  u12(); // expected-error {{call to deleted function 'u12': ‎abc‏def‏gh}}
+  u13(); // expected-error {{call to deleted function 'u13': 🏳️‍🌈 🏴󠁧󠁢󠁥󠁮󠁧󠁿 🇪🇺}}
+}
+
+struct C {
+  C() = delete("deleted (C, Constructor)"); // expected-note {{explicitly marked deleted}}
+  C(int) = delete("deleted (C, C(int))"); // expected-note {{explicitly marked deleted}}
+  C(const C&) = delete("deleted (C, Copy Constructor)"); // expected-note {{explicitly marked deleted}}
+  C(C&&) = delete("deleted (C, Move Constructor)"); // expected-note {{explicitly marked deleted}}
+  C& operator=(const C&) = delete("deleted (C, Copy Assignment)"); // expected-note 2 {{explicitly deleted}}
+  C& operator=(C&&) = delete("deleted (C, Move Assignment)"); // expected-note {{explicitly deleted}} expected-note {{not viable}}
+  ~C() = delete("deleted (C, Destructor)"); // expected-note {{explicitly marked deleted}}
+  void* operator new(__SIZE_TYPE__) = delete("deleted (C, New)"); // expected-note {{explicitly deleted}}
+  void operator delete(void*) = delete("deleted (C, Delete)"); // expected-note {{explicitly marked deleted}}
+};
+
+template <typename T, typename U>
+struct TC {
+  TC() = delete("deleted (TC, Constructor)"); // expected-note {{explicitly marked deleted}}
+  TC(int) = delete("deleted (TC, TC(int))"); // expected-note {{explicitly marked deleted}}
+  TC(const TC&) = delete("deleted (TC, Copy Constructor)"); // expected-note {{explicitly marked deleted}}
+  TC(TC&&) = delete("deleted (TC, Move Constructor)"); // expected-note {{explicitly marked deleted}}
+  TC& operator=(const TC&) = delete("deleted (TC, Copy Assignment)"); // expected-note 2 {{explicitly deleted}}
+  TC& operator=(TC&&) = delete("deleted (TC, Move Assignment)");  // expected-note {{explicitly deleted}} expected-note {{not viable}}
+  ~TC() = delete("deleted (TC, Destructor)"); // expected-note {{explicitly marked deleted}}
+  void* operator new(__SIZE_TYPE__) = delete("deleted (TC, New)"); // expected-note {{explicitly deleted}}
+  void operator delete(void*) = delete("deleted (TC, Delete)"); // expected-note {{explicitly marked deleted}}
+};
+
+template <typename T>
+struct TC<T, int> {
+  TC() = delete("deleted (TC<T, int>, Constructor)"); // expected-note {{explicitly marked deleted}}
+  TC(int) = delete("deleted (TC<T, int>, TC(int))"); // expected-note {{explicitly marked deleted}}
+  TC(const TC&) = delete("deleted (TC<T, int>, Copy Constructor)"); // expected-note {{explicitly marked deleted}}
+  TC(TC&&) = delete("deleted (TC<T, int>, Move Constructor)"); // expected-note {{explicitly marked deleted}}
+  TC& operator=(const TC&) = delete("deleted (TC<T, int>, Copy Assignment)"); // expected-note 2 {{explicitly deleted}}
+  TC& operator=(TC&&) = delete("deleted (TC<T, int>, Move Assignment)"); // expected-note {{explicitly deleted}} expected-note {{not viable}}
+  ~TC() = delete("deleted (TC<T, int>, Destructor)"); // expected-note {{explicitly marked deleted}}
+  void* operator new(__SIZE_TYPE__) = delete("deleted (TC<T, int>, New)"); // expected-note {{explicitly deleted}}
+  void operator delete(void*) = delete("deleted (TC<T, int>, Delete)"); // expected-note {{explicitly marked deleted}}
+};
+
+template <>
+struct TC<int, int> {
+  TC() = delete("deleted (TC<int, int>, Constructor)"); // expected-note {{explicitly marked deleted}}
+  TC(int) = delete("deleted (TC<int, int>, TC(int))"); // expected-note {{explicitly marked deleted}}
+  TC(const TC&) = delete("deleted (TC<int, int>, Copy Constructor)"); // expected-note {{explicitly marked deleted}}
+  TC(TC&&) = delete("deleted (TC<int, int>, Move Constructor)"); // expected-note {{explicitly marked deleted}}
+  TC& operator=(const TC&) = delete("deleted (TC<int, int>, Copy Assignment)"); // expected-note 2 {{explicitly deleted}}
+  TC& operator=(TC&&) = delete("deleted (TC<int, int>, Move Assignment)"); // expected-note {{explicitly deleted}} expected-note {{not viable}}
+  ~TC() = delete("deleted (TC<int, int>, Destructor)"); // expected-note {{explicitly marked deleted}}
+  void* operator new(__SIZE_TYPE__) = delete("deleted (TC<int, int>, New)"); // expected-note {{explicitly deleted}}
+  void operator delete(void*) = delete("deleted (TC<int, int>, Delete)"); // expected-note {{explicitly marked deleted}}
+};
+
+void special_members(
+  C& c1,
+  C& c2,
+  TC<double, double>& tc1,
+  TC<double, double>& tc2,
+  TC<double, int>& tc_int1,
+  TC<double, int>& tc_int2,
+  TC<int, int>& tc_int_int1,
+  TC<int, int>& tc_int_int2
+) {
+  C{}; // expected-error {{call to deleted constructor of 'C': deleted (C, Constructor)}}
+  C{c1}; // expected-error {{call to deleted constructor of 'C': deleted (C, Copy Constructor)}}
+  C{static_cast<C&&>(c1)}; // expected-error {{call to deleted constructor of 'C': deleted (C, Move Constructor)}}
+  c1 = c2; // expected-error {{overload resolution selected deleted operator '=': deleted (C, Copy Assignment)}}
+  c1 = static_cast<C&&>(c2); // expected-error {{overload resolution selected deleted operator '=': deleted (C, Move Assignment)}}
+  c1.~C(); // expected-error {{attempt to use a deleted function: deleted (C, Destructor)}}
+  new C{}; // expected-error {{call to deleted function 'operator new': deleted (C, New)}}
+  delete &c2; // expected-error {{attempt to use a deleted function: deleted (C, Delete)}}
+
+  TC<double, double>{}; // expected-error {{call to deleted constructor of 'TC<double, double>': deleted (TC, Constructor)}}
+  TC<double, double>{tc1}; // expected-error {{call to deleted constructor of 'TC<double, double>': deleted (TC, Copy Constructor)}}
+  TC<double, double>{static_cast<TC<double, double>&&>(tc1)}; // expected-error {{call to deleted constructor of 'TC<double, double>': deleted (TC, Move Constructor)}}
+  tc1 = tc2; // expected-error {{overload resolution selected deleted operator '=': deleted (TC, Copy Assignment)}}
+  tc1 = static_cast<TC<double, double>&&>(tc2); // expected-error {{overload resolution selected deleted operator '=': deleted (TC, Move Assignment)}}
+  tc1.~TC(); // expected-error {{attempt to use a deleted function: deleted (TC, Destructor)}}
+  new TC<double, double>{}; // expected-error {{call to deleted function 'operator new': deleted (TC, New)}}
+  delete &tc2; // expected-error {{attempt to use a deleted function: deleted (TC, Delete)}}
+
+  TC<double, int>{}; // expected-error {{call to deleted constructor of 'TC<double, int>': deleted (TC<T, int>, Constructor)}}
+  TC<double, int>{tc_int1}; // expected-error {{call to deleted constructor of 'TC<double, int>': deleted (TC<T, int>, Copy Constructor)}}
+  TC<double, int>{static_cast<TC<double, int>&&>(tc_int1)}; // expected-error {{call to deleted constructor of 'TC<double, int>': deleted (TC<T, int>, Move Constructor)}}
+  tc_int1 = tc_int2; // expected-error {{overload resolution selected deleted operator '=': deleted (TC<T, int>, Copy Assignment)}}
+  tc_int1 = static_cast<TC<double, int>&&>(tc_int2); // expected-error {{overload resolution selected deleted operator '=': deleted (TC<T, int>, Move Assignment)}}
+  tc_int1.~TC(); // expected-error {{attempt to use a deleted function: deleted (TC<T, int>, Destructor)}}
+  new TC<double, int>{}; // expected-error {{call to deleted function 'operator new': deleted (TC<T, int>, New)}}
+  delete &tc_int2; // expected-error {{attempt to use a deleted function: deleted (TC<T, int>, Delete)}}
+
+  TC<int, int>{}; // expected-error {{call to deleted constructor of 'TC<int, int>': deleted (TC<int, int>, Constructor)}}
+  TC<int, int>{tc_int_int1}; // expected-error {{call to deleted constructor of 'TC<int, int>': deleted (TC<int, int>, Copy Constructor)}}
+  TC<int, int>{static_cast<TC<int, int>&&>(tc_int_int1)}; // expected-error {{call to deleted constructor of 'TC<int, int>': deleted (TC<int, int>, Move Constructor)}}
+  tc_int_int1 = tc_int_int2; // expected-error {{overload resolution selected deleted operator '=': deleted (TC<int, int>, Copy Assignment)}}
+  tc_int_int1 = static_cast<TC<int, int>&&>(tc_int_int2); // expected-error {{overload resolution selected deleted operator '=': deleted (TC<int, int>, Move Assignment)}}
+  tc_int_int1.~TC(); // expected-error {{attempt to use a deleted function: deleted (TC<int, int>, Destructor)}}
+  new TC<int, int>{}; // expected-error {{call to deleted function 'operator new': deleted (TC<int, int>, New)}}
+  delete &tc_int_int2; // expected-error {{attempt to use a deleted function: deleted (TC<int, int>, Delete)}}
+}
+
+C conv1() { return 1; } // expected-error {{conversion function from 'int' to 'C' invokes a deleted function: deleted (C, C(int))}}
+TC<double, double> conv2() { return 1; } // expected-error {{conversion function from 'int' to 'TC<double, double>' invokes a deleted function: deleted (TC, TC(int))}}
+TC<double, int> conv3() { return 1; } // expected-error {{conversion function from 'int' to 'TC<double, int>' invokes a deleted function: deleted (TC<T, int>, TC(int))}}
+TC<int, int> conv4() { return 1; } // expected-error {{conversion function from 'int' to 'TC<int, int>' invokes a deleted function: deleted (TC<int, int>, TC(int))}}
+
+struct O {
+  int x;
+  int operator+() = delete("deleted (O, +)"); // expected-note {{explicitly deleted}}
+  O* operator->() = delete("deleted (O, ->)"); // expected-note {{explicitly deleted}}
+  int operator-(O) = delete("deleted (O, -)"); // expected-note {{explicitly deleted}}
+  int operator[](O) = delete("deleted (O, [])"); // expected-note {{explicitly deleted}}
+  int operator()(O) = delete("deleted (O, ())"); // expected-note {{explicitly marked deleted}} expected-note {{explicitly deleted}}
+  explicit operator bool() = delete("deleted (O, operator bool)"); // expected-note {{explicitly marked deleted}} expected-note {{explicitly deleted}}
+};
+
+template <typename T, typename U>
+struct TO {
+  T x;
+  T operator+() = delete("deleted (TO, +)"); // expected-note {{explicitly deleted}}
+  T* operator->() = delete("deleted (TO, ->)"); // expected-note {{explicitly deleted}}
+  T operator-(TO) = delete("deleted (TO, -)"); // expected-note {{explicitly deleted}}
+  T operator[](TO) = delete("deleted (TO, [])"); // expected-note {{explicitly deleted}}
+  T operator()(TO) = delete("deleted (TO, ())"); // expected-note {{explicitly marked deleted}} expected-note {{explicitly deleted}}
+  explicit operator bool() = delete("deleted (TO, operator bool)"); // expected-note {{explicitly marked deleted}} expected-note {{explicitly deleted}}
+};
+
+template <typename T>
+struct TO<T, int> {
+  T x;
+  T operator+() = delete("deleted (TO<T, int>, +)"); // expected-note {{explicitly deleted}}
+  T* operator->() = delete("deleted (TO<T, int>, ->)"); // expected-note {{explicitly deleted}}
+  T operator-(TO) = delete("deleted (TO<T, int>, -)"); // expected-note {{explicitly deleted}}
+  T operator[](TO) = delete("deleted (TO<T, int>, [])"); // expected-note {{explicitly deleted}}
+  T operator()(TO) = delete("deleted (TO<T, int>, ())"); // expected-note {{explicitly marked deleted}} expected-note {{explicitly deleted}}
+  explicit operator bool() = delete("deleted (TO<T, int>, operator bool)"); // expected-note {{explicitly marked deleted}} expected-note {{explicitly deleted}}
+};
+
+template <>
+struct TO<int, int> {
+  int x;
+  int operator+() = delete("deleted (TO<int, int>, +)"); // expected-note {{explicitly deleted}}
+  int* operator->() = delete("deleted (TO<int, int>, ->)"); // expected-note {{explicitly deleted}}
+  int operator-(TO) = delete("deleted (TO<int, int>, -)"); // expected-note {{explicitly deleted}}
+  int operator[](TO) = delete("deleted (TO<int, int>, [])"); // expected-note {{explicitly deleted}}
+  int operator()(TO) = delete("deleted (TO<int, int>, ())"); // expected-note {{explicitly marked deleted}} expected-note {{explicitly deleted}}
+  explicit operator bool() = delete("deleted (TO<int, int>, operator bool)"); // expected-note {{explicitly marked deleted}} expected-note {{explicitly deleted}}
+};
+
+void operators() {
+  O o;
+  +o; // expected-error {{overload resolution selected deleted operator '+': deleted (O, +)}}
+  o->x; // expected-error {{overload resolution selected deleted operator '->': deleted (O, ->)}}
+  o - o; // expected-error {{overload resolution selected deleted operator '-': deleted (O, -)}}
+  o[o]; // expected-error {{overload resolution selected deleted operator '[]': deleted (O, [])}}
+  o(o); // expected-error {{call to deleted function call operator in type 'O': deleted (O, ())}} expected-error {{attempt to use a deleted function: deleted (O, ())}}
+  if (o) {} // expected-error {{attempt to use a deleted function: deleted (O, operator bool)}}
+  static_cast<bool>(o); // expected-error {{static_cast from 'O' to 'bool' uses deleted function: deleted (O, operator bool)}}
+
+  TO<double, double> to;
+  +to; // expected-error {{overload resolution selected deleted operator '+': deleted (TO, +)}}
+  to->x; // expected-error {{overload resolution selected deleted operator '->': deleted (TO, ->)}}
+  to - to; // expected-error {{overload resolution selected deleted operator '-': deleted (TO, -)}}
+  to[to]; // expected-error {{overload resolution selected deleted operator '[]': deleted (TO, [])}}
+  to(to); // expected-error {{call to deleted function call operator in type 'TO<double, double>': deleted (TO, ())}} expected-error {{attempt to use a deleted function: deleted (TO, ())}}
+  if (to) {} // expected-error {{attempt to use a deleted function: deleted (TO, operator bool)}}
+  static_cast<bool>(to); // expected-error {{static_cast from 'TO<double, double>' to 'bool' uses deleted function: deleted (TO, operator bool)}}
+
+  TO<double, int> to_int;
+  +to_int; // expected-error {{overload resolution selected deleted operator '+': deleted (TO<T, int>, +)}}
+  to_int->x; // expected-error {{overload resolution selected deleted operator '->': deleted (TO<T, int>, ->)}}
+  to_int - to_int; // expected-error {{overload resolution selected deleted operator '-': deleted (TO<T, int>, -)}}
+  to_int[to_int]; // expected-error {{overload resolution selected deleted operator '[]': deleted (TO<T, int>, [])}}
+  to_int(to_int); // expected-error {{call to deleted function call operator in type 'TO<double, int>': deleted (TO<T, int>, ())}} expected-error {{attempt to use a deleted function: deleted (TO<T, int>, ())}}
+  if (to_int) {} // expected-error {{attempt to use a deleted function: deleted (TO<T, int>, operator bool)}}
+  static_cast<bool>(to_int); // expected-error {{static_cast from 'TO<double, int>' to 'bool' uses deleted function: deleted (TO<T, int>, operator bool)}}
+
+  TO<int, int> to_int_int;
+  +to_int_int; // expected-error {{overload resolution selected deleted operator '+': deleted (TO<int, int>, +)}}
+  to_int_int->x; // expected-error {{overload resolution selected deleted operator '->': deleted (TO<int, int>, ->)}}
+  to_int_int - to_int_int; // expected-error {{overload resolution selected deleted operator '-': deleted (TO<int, int>, -)}}
+  to_int_int[to_int_int]; // expected-error {{overload resolution selected deleted operator '[]': deleted (TO<int, int>, [])}}
+  to_int_int(to_int_int); // expected-error {{call to deleted function call operator in type 'TO<int, int>': deleted (TO<int, int>, ())}} expected-error {{attempt to use a deleted function: deleted (TO<int, int>, ())}}
+  if (to_int_int) {} // expected-error {{attempt to use a deleted function: deleted (TO<int, int>, operator bool)}}
+  static_cast<bool>(to_int_int); // expected-error {{static_cast from 'TO<int, int>' to 'bool' uses deleted function: deleted (TO<int, int>, operator bool)}}
+};
diff --git a/clang/www/cxx_status.html b/clang/www/cxx_status.html
index 130148c7420f..c233171e63c8 100755
--- a/clang/www/cxx_status.html
+++ b/clang/www/cxx_status.html
@@ -197,7 +197,7 @@ C++23, informally referred to as C++26.</p>
  <tr>
   <td><tt>= delete("should have a reason");</tt></td>
   <td><a href="https://wg21.link/P2573R2">P2573R2</a></td>
-  <td class="none" align="center">No</td>
+  <td class="unreleased" align="center">Clang 19</td>
  </tr>
  <tr>
   <td>Variadic friends</td>
-- 
GitLab


From 8891fd5acbe441d24a1734aa144f3f3dca075620 Mon Sep 17 00:00:00 2001
From: Kai Sasaki <lewuathe@gmail.com>
Date: Sun, 14 Apr 2024 19:52:03 +0900
Subject: [PATCH 166/464] [mlir][complex] Fastmath flag support for
 complex.tanh (#88571)

---
 .../ComplexToStandard/ComplexToStandard.cpp   | 14 ++++++++------
 .../convert-to-standard.mlir                  | 19 +++++++++++++++++++
 2 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
index 0664b053fc9e..49eb575212ff 100644
--- a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
+++ b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
@@ -973,6 +973,7 @@ struct TanhOpConversion : public OpConversionPattern<complex::TanhOp> {
     auto loc = op.getLoc();
     auto type = cast<ComplexType>(adaptor.getComplex().getType());
     auto elementType = cast<FloatType>(type.getElementType());
+    arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
 
     // The hyperbolic tangent for complex number can be calculated as follows.
     // tanh(x + i * y) = (tanh(x) + i * tan(y)) / (1 + tanh(x) * tan(y))
@@ -981,17 +982,18 @@ struct TanhOpConversion : public OpConversionPattern<complex::TanhOp> {
         rewriter.create<complex::ReOp>(loc, elementType, adaptor.getComplex());
     Value imag =
         rewriter.create<complex::ImOp>(loc, elementType, adaptor.getComplex());
-    Value tanhA = rewriter.create<math::TanhOp>(loc, real);
-    Value cosB = rewriter.create<math::CosOp>(loc, imag);
-    Value sinB = rewriter.create<math::SinOp>(loc, imag);
-    Value tanB = rewriter.create<arith::DivFOp>(loc, sinB, cosB);
+    Value tanhA = rewriter.create<math::TanhOp>(loc, real, fmf);
+    Value cosB = rewriter.create<math::CosOp>(loc, imag, fmf);
+    Value sinB = rewriter.create<math::SinOp>(loc, imag, fmf);
+    Value tanB = rewriter.create<arith::DivFOp>(loc, sinB, cosB, fmf);
     Value numerator =
         rewriter.create<complex::CreateOp>(loc, type, tanhA, tanB);
     Value one = rewriter.create<arith::ConstantOp>(
         loc, elementType, rewriter.getFloatAttr(elementType, 1));
-    Value mul = rewriter.create<arith::MulFOp>(loc, tanhA, tanB);
+    Value mul = rewriter.create<arith::MulFOp>(loc, tanhA, tanB, fmf);
     Value denominator = rewriter.create<complex::CreateOp>(loc, type, one, mul);
-    rewriter.replaceOpWithNewOp<complex::DivOp>(op, numerator, denominator);
+    rewriter.replaceOpWithNewOp<complex::DivOp>(op, numerator, denominator,
+                                                fmf);
     return success();
   }
 };
diff --git a/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir b/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir
index b22c1acacaea..e0e7cdadd317 100644
--- a/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir
+++ b/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir
@@ -2085,3 +2085,22 @@ func.func @complex_tan_with_fmf(%arg: complex<f32>) -> complex<f32> {
 // CHECK: %[[RESULT:.*]] = complex.create %[[RESULT_REAL_WITH_SPECIAL_CASES]], %[[RESULT_IMAG_WITH_SPECIAL_CASES]] : complex<f32>
 // CHECK: return %[[RESULT]] : complex<f32>
 
+
+// -----
+
+// CHECK-LABEL: func @complex_tanh_with_fmf
+// CHECK-SAME: %[[ARG:.*]]: complex<f32>
+func.func @complex_tanh_with_fmf(%arg: complex<f32>) -> complex<f32> {
+  %tanh = complex.tanh %arg fastmath<nnan,contract> : complex<f32>
+  return %tanh : complex<f32>
+}
+// CHECK: %[[REAL:.*]] = complex.re %[[ARG]] : complex<f32>
+// CHECK: %[[IMAG:.*]] = complex.im %[[ARG]] : complex<f32>
+// CHECK: %[[TANH_A:.*]] = math.tanh %[[REAL]] fastmath<nnan,contract> : f32
+// CHECK: %[[COS_B:.*]] = math.cos %[[IMAG]] fastmath<nnan,contract> : f32
+// CHECK: %[[SIN_B:.*]] = math.sin %[[IMAG]] fastmath<nnan,contract> : f32
+// CHECK: %[[TAN_B:.*]] = arith.divf %[[SIN_B]], %[[COS_B]] fastmath<nnan,contract> : f32
+// CHECK: %[[NUM:.*]] = complex.create %[[TANH_A]], %[[TAN_B]] : complex<f32>
+// CHECK: %[[ONE:.*]] = arith.constant 1.000000e+00 : f32
+// CHECK: %[[MUL:.*]] = arith.mulf %[[TANH_A]], %[[TAN_B]] fastmath<nnan,contract> : f32
+// CHECK: %[[DENOM:.*]] = complex.create %[[ONE]], %[[MUL]] : complex<f32>
\ No newline at end of file
-- 
GitLab


From 6c2cc8240e11721cac466cfce89bc0f87a5019be Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Sun, 14 Apr 2024 12:09:14 +0100
Subject: [PATCH 167/464] [AArch64] Improve cost of non-zero lane splats

This adds a cost for non-zero lane splats, which is not included by default in
SK_Broadcast but can be handled by aarch64 dup lane instruction.
---
 llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp | 5 ++++-
 llvm/test/Analysis/CostModel/AArch64/shuffle-other.ll  | 6 +++---
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index bd943de06b4b..e80931a03f30 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -3939,7 +3939,10 @@ InstructionCost AArch64TTIImpl::getShuffleCost(
       LT.second.getVectorNumElements() == Mask.size() &&
       (Kind == TTI::SK_PermuteTwoSrc || Kind == TTI::SK_PermuteSingleSrc) &&
       (isZIPMask(Mask, LT.second, Unused) ||
-       isUZPMask(Mask, LT.second, Unused)))
+       isUZPMask(Mask, LT.second, Unused) ||
+       // Check for non-zero lane splats
+       all_of(drop_begin(Mask),
+              [&Mask](int M) { return M < 0 || M == Mask[0]; })))
     return 1;
 
   if (Kind == TTI::SK_Broadcast || Kind == TTI::SK_Transpose ||
diff --git a/llvm/test/Analysis/CostModel/AArch64/shuffle-other.ll b/llvm/test/Analysis/CostModel/AArch64/shuffle-other.ll
index 6c45ebcb69f4..d67f05636610 100644
--- a/llvm/test/Analysis/CostModel/AArch64/shuffle-other.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/shuffle-other.ll
@@ -361,9 +361,9 @@ define void @uzp() {
 define void @multipart() {
 ; CHECK-LABEL: 'multipart'
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16a = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %v16b = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v16c = shufflevector <16 x i16> undef, <16 x i16> undef, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %v16d = shufflevector <16 x i16> undef, <16 x i16> undef, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16b = shufflevector <8 x i16> undef, <8 x i16> undef, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16c = shufflevector <16 x i16> undef, <16 x i16> undef, <16 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16d = shufflevector <16 x i16> undef, <16 x i16> undef, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32a = shufflevector <4 x i32> undef, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 0, i32 1>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32a4 = shufflevector <16 x i32> undef, <16 x i32> undef, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v32idrev = shufflevector <16 x i32> undef, <16 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 15, i32 14, i32 13, i32 12, i32 16, i32 17, i32 18, i32 19, i32 31, i32 30, i32 29, i32 28>
-- 
GitLab


From 9b832b726c9c9bb0672c5f0912f6f131e3e27a10 Mon Sep 17 00:00:00 2001
From: Hristo Hristov <hghristov.rmm@gmail.com>
Date: Sun, 14 Apr 2024 14:37:51 +0300
Subject: [PATCH 168/464] [libc++] Deprecated `shared_ptr` Atomic Access APIs
 as per P0718R2 & Implemented P2869R3: Remove Deprecated `shared_ptr` Atomic
 Access APIs from C++26 (#87111)

Implements https://wg21.link/P2869R4
Implements deprecations as per https://wg21.link/P0718R2
---
 libcxx/docs/ReleaseNotes/19.rst               |  4 +++
 libcxx/docs/Status/Cxx20.rst                  |  2 ++
 libcxx/docs/Status/Cxx20Papers.csv            |  2 +-
 libcxx/docs/Status/Cxx2cPapers.csv            |  2 +-
 libcxx/docs/UsingLibcxx.rst                   |  4 +++
 libcxx/include/__memory/shared_ptr.h          | 29 ++++++++++------
 libcxx/include/memory                         | 22 ++++++------
 libcxx/modules/std/memory.inc                 |  6 ++++
 ..._exchange_strong.depr_in_cxx20..verify.cpp | 32 +++++++++++++++++
 ...e_strong_explicit.depr_in_cxx20.verify.cpp | 34 +++++++++++++++++++
 ...re_exchange_weak.depr_in_cxx20..verify.cpp | 31 +++++++++++++++++
 ...ge_weak_explicit.depr_in_cxx20..verify.cpp | 34 +++++++++++++++++++
 .../atomic_exchange.depr_in_cxx20..verify.cpp | 29 ++++++++++++++++
 ...xchange_explicit.verify.depr_in_cxx20..cpp | 30 ++++++++++++++++
 ...mic_is_lock_free.depr_in_cxx20..verify.cpp | 28 +++++++++++++++
 .../atomic_load.depr_in_cxx20..verify.cpp     | 28 +++++++++++++++
 ...ic_load_explicit.depr_in_cxx20..verify.cpp | 29 ++++++++++++++++
 .../atomic_store.depr_in_cxx20..verify.cpp    | 28 +++++++++++++++
 ...c_store_explicit.depr_in_cxx20..verify.cpp | 29 ++++++++++++++++
 .../atomic_compare_exchange_strong.pass.cpp   |  6 ++--
 ..._compare_exchange_strong_explicit.pass.cpp |  6 ++--
 .../atomic_compare_exchange_weak.pass.cpp     |  6 ++--
 ...ic_compare_exchange_weak_explicit.pass.cpp |  6 ++--
 .../atomic_exchange.pass.cpp                  |  6 ++--
 .../atomic_exchange_explicit.pass.cpp         |  6 ++--
 .../atomic_is_lock_free.pass.cpp              |  6 ++--
 .../atomic_load.pass.cpp                      |  6 ++--
 .../atomic_load_explicit.pass.cpp             |  6 ++--
 .../atomic_store.pass.cpp                     |  6 ++--
 .../atomic_store_explicit.pass.cpp            |  6 ++--
 30 files changed, 423 insertions(+), 46 deletions(-)
 create mode 100644 libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_strong.depr_in_cxx20..verify.cpp
 create mode 100644 libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_strong_explicit.depr_in_cxx20.verify.cpp
 create mode 100644 libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_weak.depr_in_cxx20..verify.cpp
 create mode 100644 libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_weak_explicit.depr_in_cxx20..verify.cpp
 create mode 100644 libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_exchange.depr_in_cxx20..verify.cpp
 create mode 100644 libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_exchange_explicit.verify.depr_in_cxx20..cpp
 create mode 100644 libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_is_lock_free.depr_in_cxx20..verify.cpp
 create mode 100644 libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_load.depr_in_cxx20..verify.cpp
 create mode 100644 libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_load_explicit.depr_in_cxx20..verify.cpp
 create mode 100644 libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_store.depr_in_cxx20..verify.cpp
 create mode 100644 libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_store_explicit.depr_in_cxx20..verify.cpp

diff --git a/libcxx/docs/ReleaseNotes/19.rst b/libcxx/docs/ReleaseNotes/19.rst
index 390de789ddb6..e5db17daa482 100644
--- a/libcxx/docs/ReleaseNotes/19.rst
+++ b/libcxx/docs/ReleaseNotes/19.rst
@@ -43,6 +43,7 @@ Implemented Papers
 - P2819R2 - Add ``tuple`` protocol to ``complex``
 - P2495R3 - Interfacing ``stringstream``\s with ``string_view``
 - P2867R2 - Remove Deprecated ``strstream``\s From C++26
+- P2869R4 - Remove Deprecated ``shared_ptr`` Atomic Access APIs from C++26
 - P2872R3 - Remove ``wstring_convert`` From C++26
 - P3142R0 - Printing Blank Lines with ``println`` (as DR against C++23)
 - P2302R4 - ``std::ranges::contains``
@@ -63,6 +64,9 @@ Improvements and New Features
 
 - The ``_LIBCPP_ENABLE_CXX26_REMOVED_STRSTREAM`` macro has been added to make the declarations in ``<strstream>`` available.
 
+- The ``_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS`` macro has been added to make the declarations in ``<memory>``
+  available.
+
 - The ``_LIBCPP_ENABLE_CXX26_REMOVED_WSTRING_CONVERT`` macro has been added to make the declarations in ``<locale>``
   available.
 
diff --git a/libcxx/docs/Status/Cxx20.rst b/libcxx/docs/Status/Cxx20.rst
index 2deb82547d63..23289dc6e596 100644
--- a/libcxx/docs/Status/Cxx20.rst
+++ b/libcxx/docs/Status/Cxx20.rst
@@ -59,6 +59,8 @@ Paper Status
       * ``GPS clock`` not done
       * ``UTC clock`` not done
 
+   .. [#note-P0718] P0718: Implemented deprecation of ``shared_ptr`` atomic access APIs only.
+
 .. _issues-status-cxx20:
 
 Library Working Group Issues Status
diff --git a/libcxx/docs/Status/Cxx20Papers.csv b/libcxx/docs/Status/Cxx20Papers.csv
index 6e4678da2888..d31720b7576d 100644
--- a/libcxx/docs/Status/Cxx20Papers.csv
+++ b/libcxx/docs/Status/Cxx20Papers.csv
@@ -12,7 +12,7 @@
 "`P0600R1 <https://wg21.link/P0600R1>`__","LWG","nodiscard in the Library","Albuquerque","|Complete|","16.0"
 "`P0616R0 <https://wg21.link/P0616R0>`__","LWG","de-pessimize legacy <numeric> algorithms with std::move","Albuquerque","|Complete|","12.0"
 "`P0653R2 <https://wg21.link/P0653R2>`__","LWG","Utility to convert a pointer to a raw pointer","Albuquerque","|Complete|","6.0"
-"`P0718R2 <https://wg21.link/P0718R2>`__","LWG","Atomic shared_ptr","Albuquerque","",""
+"`P0718R2 <https://wg21.link/P0718R2>`__","LWG","Atomic shared_ptr","Albuquerque","|Partial| [#note-P0718]_",""
 "`P0767R1 <https://wg21.link/P0767R1>`__","CWG","Deprecate POD","Albuquerque","|Complete|","7.0"
 "`P0768R1 <https://wg21.link/P0768R1>`__","CWG","Library Support for the Spaceship (Comparison) Operator","Albuquerque","|Complete|",""
 "`P0777R1 <https://wg21.link/P0777R1>`__","LWG","Treating Unnecessary ``decay``\ ","Albuquerque","|Complete|","7.0"
diff --git a/libcxx/docs/Status/Cxx2cPapers.csv b/libcxx/docs/Status/Cxx2cPapers.csv
index 8516070c72b8..1b00a05ad3d6 100644
--- a/libcxx/docs/Status/Cxx2cPapers.csv
+++ b/libcxx/docs/Status/Cxx2cPapers.csv
@@ -48,7 +48,7 @@
 "","","","","","",""
 "`P2875R4 <https://wg21.link/P2875R4>`__","LWG","Undeprecate ``polymorphic_allocator::destroy`` for C++26","Tokyo March 2024","|Complete|","15.0",""
 "`P2867R2 <https://wg21.link/P2867R2>`__","LWG","Remove Deprecated ``strstreams`` From C++26","Tokyo March 2024","|Complete|","19.0",""
-"`P2869R4 <https://wg21.link/P2869R4>`__","LWG","Remove Deprecated ``shared_ptr`` Atomic Access APIs from C++26","Tokyo March 2024","","",""
+"`P2869R4 <https://wg21.link/P2869R4>`__","LWG","Remove Deprecated ``shared_ptr`` Atomic Access APIs from C++26","Tokyo March 2024","|Complete|","19.0",""
 "`P2872R3 <https://wg21.link/P2872R3>`__","LWG","Remove ``wstring_convert`` From C++26","Tokyo March 2024","|Complete|","19.0",""
 "`P3107R5 <https://wg21.link/P3107R5>`__","LWG","Permit an efficient implementation of ``std::print``","Tokyo March 2024","","","|format| |DR|"
 "`P3142R0 <https://wg21.link/P3142R0>`__","LWG","Printing Blank Lines with ``println``","Tokyo March 2024","|Complete|","19.0","|format|"
diff --git a/libcxx/docs/UsingLibcxx.rst b/libcxx/docs/UsingLibcxx.rst
index bc7817d14d04..c0e85ad4d5e2 100644
--- a/libcxx/docs/UsingLibcxx.rst
+++ b/libcxx/docs/UsingLibcxx.rst
@@ -277,10 +277,14 @@ C++26 Specific Configuration Macros
 **_LIBCPP_ENABLE_CXX26_REMOVED_STRSTREAM**:
   This macro is used to re-enable all named declarations in ``<strstream>``.
 
+**_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS**:
+  This macro is used to re-enable all ``shared_ptr`` atomic access APIs in ``<memory>``.
+
 **_LIBCPP_ENABLE_CXX26_REMOVED_WSTRING_CONVERT**:
   This macro is used to re-enable the ``wstring_convert`` and ``wbuffer_convert``
   in ``<locale>``.
 
+
 Libc++ Extensions
 =================
 
diff --git a/libcxx/include/__memory/shared_ptr.h b/libcxx/include/__memory/shared_ptr.h
index a8ff189df2aa..992b1ba43f10 100644
--- a/libcxx/include/__memory/shared_ptr.h
+++ b/libcxx/include/__memory/shared_ptr.h
@@ -1580,13 +1580,15 @@ private:
 
 _LIBCPP_EXPORTED_FROM_ABI __sp_mut& __get_sp_mut(const void*);
 
+#  if _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS)
+
 template <class _Tp>
-inline _LIBCPP_HIDE_FROM_ABI bool atomic_is_lock_free(const shared_ptr<_Tp>*) {
+_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI bool atomic_is_lock_free(const shared_ptr<_Tp>*) {
   return false;
 }
 
 template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> atomic_load(const shared_ptr<_Tp>* __p) {
+_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> atomic_load(const shared_ptr<_Tp>* __p) {
   __sp_mut& __m = std::__get_sp_mut(__p);
   __m.lock();
   shared_ptr<_Tp> __q = *__p;
@@ -1595,12 +1597,13 @@ _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> atomic_load(const shared_ptr<_Tp>* __p) {
 }
 
 template <class _Tp>
-inline _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> atomic_load_explicit(const shared_ptr<_Tp>* __p, memory_order) {
+_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp>
+atomic_load_explicit(const shared_ptr<_Tp>* __p, memory_order) {
   return std::atomic_load(__p);
 }
 
 template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI void atomic_store(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r) {
+_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI void atomic_store(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r) {
   __sp_mut& __m = std::__get_sp_mut(__p);
   __m.lock();
   __p->swap(__r);
@@ -1608,12 +1611,14 @@ _LIBCPP_HIDE_FROM_ABI void atomic_store(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __
 }
 
 template <class _Tp>
-inline _LIBCPP_HIDE_FROM_ABI void atomic_store_explicit(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r, memory_order) {
+_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI void
+atomic_store_explicit(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r, memory_order) {
   std::atomic_store(__p, __r);
 }
 
 template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> atomic_exchange(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r) {
+_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp>
+atomic_exchange(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r) {
   __sp_mut& __m = std::__get_sp_mut(__p);
   __m.lock();
   __p->swap(__r);
@@ -1622,13 +1627,13 @@ _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp> atomic_exchange(shared_ptr<_Tp>* __p, shar
 }
 
 template <class _Tp>
-inline _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp>
+_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI shared_ptr<_Tp>
 atomic_exchange_explicit(shared_ptr<_Tp>* __p, shared_ptr<_Tp> __r, memory_order) {
   return std::atomic_exchange(__p, __r);
 }
 
 template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI bool
+_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI bool
 atomic_compare_exchange_strong(shared_ptr<_Tp>* __p, shared_ptr<_Tp>* __v, shared_ptr<_Tp> __w) {
   shared_ptr<_Tp> __temp;
   __sp_mut& __m = std::__get_sp_mut(__p);
@@ -1646,23 +1651,25 @@ atomic_compare_exchange_strong(shared_ptr<_Tp>* __p, shared_ptr<_Tp>* __v, share
 }
 
 template <class _Tp>
-inline _LIBCPP_HIDE_FROM_ABI bool
+_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI bool
 atomic_compare_exchange_weak(shared_ptr<_Tp>* __p, shared_ptr<_Tp>* __v, shared_ptr<_Tp> __w) {
   return std::atomic_compare_exchange_strong(__p, __v, __w);
 }
 
 template <class _Tp>
-inline _LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong_explicit(
+_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_strong_explicit(
     shared_ptr<_Tp>* __p, shared_ptr<_Tp>* __v, shared_ptr<_Tp> __w, memory_order, memory_order) {
   return std::atomic_compare_exchange_strong(__p, __v, __w);
 }
 
 template <class _Tp>
-inline _LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak_explicit(
+_LIBCPP_DEPRECATED_IN_CXX20 _LIBCPP_HIDE_FROM_ABI bool atomic_compare_exchange_weak_explicit(
     shared_ptr<_Tp>* __p, shared_ptr<_Tp>* __v, shared_ptr<_Tp> __w, memory_order, memory_order) {
   return std::atomic_compare_exchange_weak(__p, __v, __w);
 }
 
+#  endif // _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS)
+
 #endif // !defined(_LIBCPP_HAS_NO_THREADS)
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/memory b/libcxx/include/memory
index a8c0264eb9eb..30fadce760d7 100644
--- a/libcxx/include/memory
+++ b/libcxx/include/memory
@@ -830,34 +830,34 @@ public:
 };
 
 template<class T>
-    bool atomic_is_lock_free(const shared_ptr<T>* p);
+    bool atomic_is_lock_free(const shared_ptr<T>* p);                                     // Deprecated in C++20, removed in C++26
 template<class T>
-    shared_ptr<T> atomic_load(const shared_ptr<T>* p);
+    shared_ptr<T> atomic_load(const shared_ptr<T>* p);                                    // Deprecated in C++20, removed in C++26
 template<class T>
-    shared_ptr<T> atomic_load_explicit(const shared_ptr<T>* p, memory_order mo);
+    shared_ptr<T> atomic_load_explicit(const shared_ptr<T>* p, memory_order mo);          // Deprecated in C++20, removed in C++26
 template<class T>
-    void atomic_store(shared_ptr<T>* p, shared_ptr<T> r);
+    void atomic_store(shared_ptr<T>* p, shared_ptr<T> r);                                 // Deprecated in C++20, removed in C++26
 template<class T>
-    void atomic_store_explicit(shared_ptr<T>* p, shared_ptr<T> r, memory_order mo);
+    void atomic_store_explicit(shared_ptr<T>* p, shared_ptr<T> r, memory_order mo);       // Deprecated in C++20, removed in C++26
 template<class T>
-    shared_ptr<T> atomic_exchange(shared_ptr<T>* p, shared_ptr<T> r);
+    shared_ptr<T> atomic_exchange(shared_ptr<T>* p, shared_ptr<T> r);                     // Deprecated in C++20, removed in C++26
 template<class T>
     shared_ptr<T>
-    atomic_exchange_explicit(shared_ptr<T>* p, shared_ptr<T> r, memory_order mo);
+    atomic_exchange_explicit(shared_ptr<T>* p, shared_ptr<T> r, memory_order mo);         // Deprecated in C++20, removed in C++26
 template<class T>
     bool
-    atomic_compare_exchange_weak(shared_ptr<T>* p, shared_ptr<T>* v, shared_ptr<T> w);
+    atomic_compare_exchange_weak(shared_ptr<T>* p, shared_ptr<T>* v, shared_ptr<T> w);    // Deprecated in C++20, removed in C++26
 template<class T>
     bool
-    atomic_compare_exchange_strong( shared_ptr<T>* p, shared_ptr<T>* v, shared_ptr<T> w);
+    atomic_compare_exchange_strong( shared_ptr<T>* p, shared_ptr<T>* v, shared_ptr<T> w); // Deprecated in C++20, removed in C++26
 template<class T>
     bool
-    atomic_compare_exchange_weak_explicit(shared_ptr<T>* p, shared_ptr<T>* v,
+    atomic_compare_exchange_weak_explicit(shared_ptr<T>* p, shared_ptr<T>* v,             // Deprecated in C++20, removed in C++26
                                           shared_ptr<T> w, memory_order success,
                                           memory_order failure);
 template<class T>
     bool
-    atomic_compare_exchange_strong_explicit(shared_ptr<T>* p, shared_ptr<T>* v,
+    atomic_compare_exchange_strong_explicit(shared_ptr<T>* p, shared_ptr<T>* v,           // Deprecated in C++20, removed in C++26
                                             shared_ptr<T> w, memory_order success,
                                             memory_order failure);
 // Hash support
diff --git a/libcxx/modules/std/memory.inc b/libcxx/modules/std/memory.inc
index 56c621c0cf17..b23c27707afd 100644
--- a/libcxx/modules/std/memory.inc
+++ b/libcxx/modules/std/memory.inc
@@ -190,6 +190,9 @@ export namespace std {
   //  using std::inout_ptr;
 
 #ifndef _LIBCPP_HAS_NO_THREADS
+
+#  if _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS)
+
   // [depr.util.smartptr.shared.atomic]
   using std::atomic_is_lock_free;
 
@@ -206,5 +209,8 @@ export namespace std {
   using std::atomic_compare_exchange_strong_explicit;
   using std::atomic_compare_exchange_weak;
   using std::atomic_compare_exchange_weak_explicit;
+
+#  endif // _LIBCPP_STD_VER < 26 || defined(_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS)
+
 #endif // _LIBCPP_HAS_NO_THREADS
 } // namespace std
diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_strong.depr_in_cxx20..verify.cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_strong.depr_in_cxx20..verify.cpp
new file mode 100644
index 000000000000..3ca01cad451a
--- /dev/null
+++ b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_strong.depr_in_cxx20..verify.cpp
@@ -0,0 +1,32 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
+// UNSUPPORTED: no-threads
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// <memory>
+
+// shared_ptr
+
+// template <class T>
+// bool
+// atomic_compare_exchange_strong(shared_ptr<T>* p, shared_ptr<T>* v,
+//                                shared_ptr<T> w);                     // Deprecated in C++20, removed in C++26
+
+#include <atomic>
+#include <memory>
+#include <tuple>
+
+void test() {
+  std::shared_ptr<int> p(new int(4));
+  std::shared_ptr<int> v(new int(3));
+  std::shared_ptr<int> w(new int(2));
+  // expected-warning@+1 {{'atomic_compare_exchange_strong<int>' is deprecated}}
+  std::ignore = std::atomic_compare_exchange_strong(&p, &v, w);
+}
diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_strong_explicit.depr_in_cxx20.verify.cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_strong_explicit.depr_in_cxx20.verify.cpp
new file mode 100644
index 000000000000..93384772c8af
--- /dev/null
+++ b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_strong_explicit.depr_in_cxx20.verify.cpp
@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
+// UNSUPPORTED: no-threads
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// <memory>
+
+// shared_ptr
+
+// template <class T>
+// bool
+// atomic_compare_exchange_strong_explicit(shared_ptr<T>* p, shared_ptr<T>* v,
+//                                         shared_ptr<T> w, memory_order success,
+//                                         memory_order failure);                 // Deprecated in C++20, removed in C++26
+
+#include <atomic>
+#include <memory>
+#include <tuple>
+
+void test() {
+  std::shared_ptr<int> p(new int(4));
+  std::shared_ptr<int> v(new int(3));
+  std::shared_ptr<int> w(new int(2));
+  // expected-warning@+2 {{'atomic_compare_exchange_strong_explicit<int>' is deprecated}}
+  std::ignore =
+      std::atomic_compare_exchange_strong_explicit(&p, &v, w, std::memory_order_seq_cst, std::memory_order_seq_cst);
+}
diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_weak.depr_in_cxx20..verify.cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_weak.depr_in_cxx20..verify.cpp
new file mode 100644
index 000000000000..30ca70523fb6
--- /dev/null
+++ b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_weak.depr_in_cxx20..verify.cpp
@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
+// UNSUPPORTED: no-threads
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// <memory>
+
+// shared_ptr
+
+// template <class T>
+// bool
+// atomic_compare_exchange_weak(shared_ptr<T>* p, shared_ptr<T>* v,
+//                              shared_ptr<T> w);                    // Deprecated in C++20, removed in C++26
+
+#include <memory>
+#include <tuple>
+
+int main(int, char**) {
+  std::shared_ptr<int> p(new int(4));
+  std::shared_ptr<int> v(new int(3));
+  std::shared_ptr<int> w(new int(2));
+  // expected-warning@+1 {{'atomic_compare_exchange_weak<int>' is deprecated}}
+  std::ignore = std::atomic_compare_exchange_weak(&p, &v, w);
+}
diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_weak_explicit.depr_in_cxx20..verify.cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_weak_explicit.depr_in_cxx20..verify.cpp
new file mode 100644
index 000000000000..ae6e98d0692d
--- /dev/null
+++ b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_compare_exchange_weak_explicit.depr_in_cxx20..verify.cpp
@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
+// UNSUPPORTED: no-threads
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// <memory>
+
+// shared_ptr
+
+// template <class T>
+// bool
+// atomic_compare_exchange_weak_explicit(shared_ptr<T>* p, shared_ptr<T>* v,
+//                                       shared_ptr<T> w, memory_order success,
+//                                       memory_order failure);                 // Deprecated in C++20, removed in C++26
+
+#include <atomic>
+#include <memory>
+#include <tuple>
+
+void test() {
+  std::shared_ptr<int> p(new int(4));
+  std::shared_ptr<int> v(new int(3));
+  std::shared_ptr<int> w(new int(2));
+  // expected-warning@+2 {{'atomic_compare_exchange_weak_explicit<int>' is deprecated}}
+  std::ignore =
+      std::atomic_compare_exchange_weak_explicit(&p, &v, w, std::memory_order_seq_cst, std::memory_order_seq_cst);
+}
diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_exchange.depr_in_cxx20..verify.cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_exchange.depr_in_cxx20..verify.cpp
new file mode 100644
index 000000000000..0e4bc1bc50b1
--- /dev/null
+++ b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_exchange.depr_in_cxx20..verify.cpp
@@ -0,0 +1,29 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
+// UNSUPPORTED: no-threads
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// <memory>
+
+// shared_ptr
+
+// template <class T>
+// shared_ptr<T>
+// atomic_exchange(shared_ptr<T>* p, shared_ptr<T> r)   // Deprecated in C++20, removed in C++26
+
+#include <memory>
+#include <tuple>
+
+void test() {
+  std::shared_ptr<int> p(new int(4));
+  std::shared_ptr<int> r(new int(3));
+  // expected-warning@+1 {{'atomic_exchange<int>' is deprecated}}
+  std::ignore = std::atomic_exchange(&p, r);
+}
diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_exchange_explicit.verify.depr_in_cxx20..cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_exchange_explicit.verify.depr_in_cxx20..cpp
new file mode 100644
index 000000000000..5a6a2bfe4c1e
--- /dev/null
+++ b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_exchange_explicit.verify.depr_in_cxx20..cpp
@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
+// UNSUPPORTED: no-threads
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// <memory>
+
+// shared_ptr
+
+// template <class T>
+// shared_ptr<T>
+// atomic_exchange_explicit(shared_ptr<T>* p, shared_ptr<T> r)  // Deprecated in C++20, removed in C++26
+
+#include <atomic>
+#include <memory>
+#include <tuple>
+
+void test() {
+  std::shared_ptr<int> p(new int(4));
+  std::shared_ptr<int> r(new int(3));
+  // expected-warning@+1 {{'atomic_exchange_explicit<int>' is deprecated}}
+  std::ignore = std::atomic_exchange_explicit(&p, r, std::memory_order_seq_cst);
+}
diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_is_lock_free.depr_in_cxx20..verify.cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_is_lock_free.depr_in_cxx20..verify.cpp
new file mode 100644
index 000000000000..6799d31ce1fa
--- /dev/null
+++ b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_is_lock_free.depr_in_cxx20..verify.cpp
@@ -0,0 +1,28 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
+// UNSUPPORTED: no-threads
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// <memory>
+
+// shared_ptr
+
+// template<class T>
+// bool
+// atomic_is_lock_free(const shared_ptr<T>* p);    // Deprecated in C++20, removed in C++26
+
+#include <memory>
+#include <tuple>
+
+void test() {
+  const std::shared_ptr<int> p(new int(3));
+  // expected-warning@+1 {{'atomic_is_lock_free<int>' is deprecated}}
+  std::ignore = std::atomic_is_lock_free(&p);
+}
diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_load.depr_in_cxx20..verify.cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_load.depr_in_cxx20..verify.cpp
new file mode 100644
index 000000000000..768ad0ded983
--- /dev/null
+++ b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_load.depr_in_cxx20..verify.cpp
@@ -0,0 +1,28 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
+// UNSUPPORTED: no-threads
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// <memory>
+
+// shared_ptr
+
+// template <class T>
+// shared_ptr<T>
+// atomic_load(const shared_ptr<T>* p)      // Deprecated in C++20, removed in C++26
+
+#include <memory>
+#include <tuple>
+
+void test() {
+  std::shared_ptr<int> p(new int(3));
+  // expected-warning@+1 {{'atomic_load<int>' is deprecated}}
+  std::ignore = std::atomic_load(&p);
+}
diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_load_explicit.depr_in_cxx20..verify.cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_load_explicit.depr_in_cxx20..verify.cpp
new file mode 100644
index 000000000000..dc56193324f0
--- /dev/null
+++ b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_load_explicit.depr_in_cxx20..verify.cpp
@@ -0,0 +1,29 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
+// UNSUPPORTED: no-threads
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// <memory>
+
+// shared_ptr
+
+// template <class T>
+// shared_ptr<T>
+// atomic_load_explicit(const shared_ptr<T>* p, memory_order mo)     // Deprecated in C++20, removed in C++26
+
+#include <atomic>
+#include <memory>
+#include <tuple>
+
+void test() {
+  const std::shared_ptr<int> p(new int(3));
+  // expected-warning@+1 {{'atomic_load_explicit<int>' is deprecated}}
+  std::ignore = std::atomic_load_explicit(&p, std::memory_order_relaxed);
+}
diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_store.depr_in_cxx20..verify.cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_store.depr_in_cxx20..verify.cpp
new file mode 100644
index 000000000000..38b8df6f2439
--- /dev/null
+++ b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_store.depr_in_cxx20..verify.cpp
@@ -0,0 +1,28 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
+// UNSUPPORTED: no-threads
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// <memory>
+
+// shared_ptr
+
+// template <class T>
+// void
+// atomic_store(shared_ptr<T>* p, shared_ptr<T> r)   // Deprecated in C++20, removed in C++26
+
+#include <memory>
+
+void test() {
+  std::shared_ptr<int> p;
+  std::shared_ptr<int> r(new int(3));
+  // expected-warning@+1 {{'atomic_store<int>' is deprecated}}
+  std::atomic_store(&p, r);
+}
diff --git a/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_store_explicit.depr_in_cxx20..verify.cpp b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_store_explicit.depr_in_cxx20..verify.cpp
new file mode 100644
index 000000000000..18f59c461869
--- /dev/null
+++ b/libcxx/test/libcxx/depr/depr.util.smartptr.shared.atomic/atomic_store_explicit.depr_in_cxx20..verify.cpp
@@ -0,0 +1,29 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
+// UNSUPPORTED: no-threads
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// <memory>
+
+// shared_ptr
+
+// template <class T>
+// void
+// atomic_store_explicit(shared_ptr<T>* p, shared_ptr<T> r, memory_order mo)  // Deprecated in C++20, removed in C++26
+
+#include <atomic>
+#include <memory>
+
+void test() {
+  std::shared_ptr<int> p;
+  std::shared_ptr<int> r(new int(3));
+  // expected-warning@+1 {{'atomic_store_explicit<int>' is deprecated}}
+  std::atomic_store_explicit(&p, r, std::memory_order_seq_cst);
+}
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_strong.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_strong.pass.cpp
index 37be6ceea3e0..ad9d2a9d9ccc 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_strong.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_strong.pass.cpp
@@ -5,7 +5,9 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-//
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
 // UNSUPPORTED: no-threads
 
 // <memory>
@@ -15,7 +17,7 @@
 // template <class T>
 // bool
 // atomic_compare_exchange_strong(shared_ptr<T>* p, shared_ptr<T>* v,
-//                                shared_ptr<T> w);
+//                                shared_ptr<T> w);                     // Deprecated in C++20, removed in C++26
 
 // UNSUPPORTED: c++03
 
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_strong_explicit.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_strong_explicit.pass.cpp
index 3965863b86cc..26496df96d47 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_strong_explicit.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_strong_explicit.pass.cpp
@@ -5,7 +5,9 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-//
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
 // UNSUPPORTED: no-threads
 
 // <memory>
@@ -16,7 +18,7 @@
 // bool
 // atomic_compare_exchange_strong_explicit(shared_ptr<T>* p, shared_ptr<T>* v,
 //                                         shared_ptr<T> w, memory_order success,
-//                                         memory_order failure);
+//                                         memory_order failure);                 // Deprecated in C++20, removed in C++26
 
 // UNSUPPORTED: c++03
 
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_weak.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_weak.pass.cpp
index 6dd04f924a10..5cdc0ea3312f 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_weak.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_weak.pass.cpp
@@ -5,7 +5,9 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-//
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
 // UNSUPPORTED: no-threads
 
 // <memory>
@@ -15,7 +17,7 @@
 // template <class T>
 // bool
 // atomic_compare_exchange_weak(shared_ptr<T>* p, shared_ptr<T>* v,
-//                              shared_ptr<T> w);
+//                              shared_ptr<T> w);                    // Deprecated in C++20, removed in C++26
 
 // UNSUPPORTED: c++03
 
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_weak_explicit.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_weak_explicit.pass.cpp
index 4837fa9793a5..e4cc4cf77a64 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_weak_explicit.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_compare_exchange_weak_explicit.pass.cpp
@@ -5,7 +5,9 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-//
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
 // UNSUPPORTED: no-threads
 
 // <memory>
@@ -16,7 +18,7 @@
 // bool
 // atomic_compare_exchange_weak_explicit(shared_ptr<T>* p, shared_ptr<T>* v,
 //                                       shared_ptr<T> w, memory_order success,
-//                                       memory_order failure);
+//                                       memory_order failure);                 // Deprecated in C++20, removed in C++26
 
 // UNSUPPORTED: c++03
 
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_exchange.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_exchange.pass.cpp
index f488e0ed1d14..db02c9d9f75b 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_exchange.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_exchange.pass.cpp
@@ -5,7 +5,9 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-//
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
 // UNSUPPORTED: no-threads
 
 // <memory>
@@ -14,7 +16,7 @@
 
 // template <class T>
 // shared_ptr<T>
-// atomic_exchange(shared_ptr<T>* p, shared_ptr<T> r)
+// atomic_exchange(shared_ptr<T>* p, shared_ptr<T> r)   // Deprecated in C++20, removed in C++26
 
 // UNSUPPORTED: c++03
 
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_exchange_explicit.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_exchange_explicit.pass.cpp
index 1945f7bba6dc..704232e71fa4 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_exchange_explicit.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_exchange_explicit.pass.cpp
@@ -5,7 +5,9 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-//
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
 // UNSUPPORTED: no-threads
 
 // <memory>
@@ -14,7 +16,7 @@
 
 // template <class T>
 // shared_ptr<T>
-// atomic_exchange_explicit(shared_ptr<T>* p, shared_ptr<T> r)
+// atomic_exchange_explicit(shared_ptr<T>* p, shared_ptr<T> r)  // Deprecated in C++20, removed in C++26
 
 // UNSUPPORTED: c++03
 
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_is_lock_free.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_is_lock_free.pass.cpp
index 37f7d12eda2c..8f3b5bbb5155 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_is_lock_free.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_is_lock_free.pass.cpp
@@ -5,7 +5,9 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-//
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
 // UNSUPPORTED: no-threads
 
 // <memory>
@@ -14,7 +16,7 @@
 
 // template<class T>
 // bool
-// atomic_is_lock_free(const shared_ptr<T>* p);
+// atomic_is_lock_free(const shared_ptr<T>* p);    // Deprecated in C++20, removed in C++26
 
 // UNSUPPORTED: c++03
 
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_load.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_load.pass.cpp
index 1b9a15ac92ac..9d81e81b4392 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_load.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_load.pass.cpp
@@ -5,7 +5,9 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-//
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
 // UNSUPPORTED: no-threads
 
 // <memory>
@@ -14,7 +16,7 @@
 
 // template <class T>
 // shared_ptr<T>
-// atomic_load(const shared_ptr<T>* p)
+// atomic_load(const shared_ptr<T>* p)      // Deprecated in C++20, removed in C++26
 
 // UNSUPPORTED: c++03
 
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_load_explicit.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_load_explicit.pass.cpp
index 5c2970133328..f4875dce38a8 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_load_explicit.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_load_explicit.pass.cpp
@@ -5,7 +5,9 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-//
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
 // UNSUPPORTED: no-threads
 
 // <memory>
@@ -14,7 +16,7 @@
 
 // template <class T>
 // shared_ptr<T>
-// atomic_load_explicit(const shared_ptr<T>* p, memory_order mo)
+// atomic_load_explicit(const shared_ptr<T>* p, memory_order mo)     // Deprecated in C++20, removed in C++26
 
 // UNSUPPORTED: c++03
 
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_store.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_store.pass.cpp
index 5b7bd5fad69c..836dcb16ff32 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_store.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_store.pass.cpp
@@ -5,7 +5,9 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-//
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
 // UNSUPPORTED: no-threads
 
 // <memory>
@@ -14,7 +16,7 @@
 
 // template <class T>
 // void
-// atomic_store(shared_ptr<T>* p, shared_ptr<T> r)
+// atomic_store(shared_ptr<T>* p, shared_ptr<T> r)   // Deprecated in C++20, removed in C++26
 
 // UNSUPPORTED: c++03
 
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_store_explicit.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_store_explicit.pass.cpp
index 571219042130..85b31c017d73 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_store_explicit.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared.atomic/atomic_store_explicit.pass.cpp
@@ -5,7 +5,9 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-//
+
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX26_REMOVED_SHARED_PTR_ATOMICS
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
 // UNSUPPORTED: no-threads
 
 // <memory>
@@ -14,7 +16,7 @@
 
 // template <class T>
 // void
-// atomic_store_explicit(shared_ptr<T>* p, shared_ptr<T> r, memory_order mo)
+// atomic_store_explicit(shared_ptr<T>* p, shared_ptr<T> r, memory_order mo)  // Deprecated in C++20, removed in C++26
 
 // UNSUPPORTED: c++03
 
-- 
GitLab


From 6e934b7cfba8e0c167d2b287a30dfc1b720397c1 Mon Sep 17 00:00:00 2001
From: Jie Fu <jiefu@tencent.com>
Date: Sun, 14 Apr 2024 20:34:26 +0800
Subject: [PATCH 169/464] [clang] Fix -Wunused-variable in SemaCast.cpp (NFC)

llvm-project/clang/lib/Sema/SemaCast.cpp:503:23:
error: unused variable 'Res' [-Werror,-Wunused-variable]
    OverloadingResult Res =
                      ^
1 error generated.
---
 clang/lib/Sema/SemaCast.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/lib/Sema/SemaCast.cpp b/clang/lib/Sema/SemaCast.cpp
index aac43e2cad60..b0c28531fe87 100644
--- a/clang/lib/Sema/SemaCast.cpp
+++ b/clang/lib/Sema/SemaCast.cpp
@@ -500,7 +500,7 @@ static bool tryDiagnoseOverloadedCast(Sema &S, CastType CT,
 
   case OR_Deleted: {
     OverloadCandidateSet::iterator Best;
-    OverloadingResult Res =
+    [[maybe_unused]] OverloadingResult Res =
         candidates.BestViableFunction(S, range.getBegin(), Best);
     assert(Res == OR_Deleted && "Inconsistent overload resolution");
 
-- 
GitLab


From 3c4d9559495e60dcbb431ab80d806096559a7486 Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Sun, 14 Apr 2024 06:36:40 -0700
Subject: [PATCH 170/464] [mlir] Add lib to tests for shared build (#88574)

These resulted in link failures:

```
/usr/bin/ld:
tools/mlir/test/CAPI/CMakeFiles/mlir-capi-translation-test.dir/translation.c.o:
in function `main':
translation.c:(.text.main+0x58): undefined reference to
`LLVMContextCreate'
/usr/bin/ld: translation.c:(.text.main+0x9b): undefined reference to
`LLVMDumpModule'
/usr/bin/ld: translation.c:(.text.main+0xa3): undefined reference to
`LLVMDisposeModule'
/usr/bin/ld: translation.c:(.text.main+0xb3): undefined reference to
`LLVMContextDispose'
```

Found in mlir-hs. Not sure why this hasn't been flagged elsewhere.
---
 mlir/test/CAPI/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mlir/test/CAPI/CMakeLists.txt b/mlir/test/CAPI/CMakeLists.txt
index 79b61fdef38b..b9cd63ef7c67 100644
--- a/mlir/test/CAPI/CMakeLists.txt
+++ b/mlir/test/CAPI/CMakeLists.txt
@@ -12,6 +12,7 @@ function(_add_capi_test_executable name)
   llvm_update_compile_flags(${name})
   if(MLIR_BUILD_MLIR_C_DYLIB)
     target_link_libraries(${name} PRIVATE
+      LLVMCore
       MLIR-C)
   else()
     target_link_libraries(${name} PRIVATE
-- 
GitLab


From 7cfe73624780010ec81ca11c41ebbf214400abdd Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Sun, 14 Apr 2024 17:51:15 +0400
Subject: [PATCH 171/464] [clang] Fix name conflict with `sys/mac.h` on AIX
 (#88644)

Fixes clang-ppc64-aix bot failure after #88559
(0a6f6df5b0c3d0f2a42f013bf5cafb9b5020dcac)
https://lab.llvm.org/buildbot/#/builders/214/builds/11887

---------

Co-authored-by: Joseph Huber <huberjn@outlook.com>
---
 clang/include/clang/Basic/Cuda.h | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/clang/include/clang/Basic/Cuda.h b/clang/include/clang/Basic/Cuda.h
index acc6bb6581d8..38f30543a0f6 100644
--- a/clang/include/clang/Basic/Cuda.h
+++ b/clang/include/clang/Basic/Cuda.h
@@ -50,6 +50,10 @@ const char *CudaVersionToString(CudaVersion V);
 // Input is "Major.Minor"
 CudaVersion CudaStringToVersion(const llvm::Twine &S);
 
+// We have a name conflict with sys/mac.h on AIX
+#ifdef SM_32
+#undef SM_32
+#endif
 enum class CudaArch {
   UNUSED,
   UNKNOWN,
-- 
GitLab


From d5c654b5b7c8bb81affdc69976ef9bc5ad5b4302 Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser@berlin.de>
Date: Sun, 14 Apr 2024 15:52:56 +0200
Subject: [PATCH 172/464] [libc++][RFC] Only include what is required
 by-version in the umbrella headers (#83740)

This is a relatively low cost way of reducing the include sizes in older
language modes compared to the effect. For example, in C++14 mode the
include time of `<algorithm>` is reduced from 198ms to 127ms.
---
 libcxx/include/algorithm                      | 259 +++++++++---------
 libcxx/include/bit                            |  36 ++-
 libcxx/include/charconv                       |  32 ++-
 libcxx/include/chrono                         |  75 ++---
 libcxx/include/compare                        |  36 ++-
 libcxx/include/concepts                       |  52 ++--
 libcxx/include/coroutine                      |   4 +
 libcxx/include/module.modulemap               |   5 +-
 libcxx/include/streambuf                      |   1 +
 .../libcxx/algorithms/half_positive.pass.cpp  |   4 +-
 libcxx/test/libcxx/numerics/bit.ops.pass.cpp  |   4 +-
 .../test/libcxx/time/convert_to_tm.pass.cpp   |   1 +
 .../test/libcxx/transitive_includes/cxx03.csv |  12 +-
 .../test/libcxx/transitive_includes/cxx11.csv |  13 +-
 .../test/libcxx/transitive_includes/cxx14.csv |  12 +
 .../test/libcxx/transitive_includes/cxx17.csv |  12 +
 .../test/libcxx/transitive_includes/cxx20.csv |  12 +
 .../test/libcxx/transitive_includes/cxx23.csv |  13 +
 .../test/libcxx/transitive_includes/cxx26.csv |  13 +
 19 files changed, 374 insertions(+), 222 deletions(-)

diff --git a/libcxx/include/algorithm b/libcxx/include/algorithm
index 0f62de7fa83f..869fc19737b5 100644
--- a/libcxx/include/algorithm
+++ b/libcxx/include/algorithm
@@ -1794,15 +1794,11 @@ template <class BidirectionalIterator, class Compare>
 */
 
 #include <__config>
-#include <version>
 
 #include <__algorithm/adjacent_find.h>
 #include <__algorithm/all_of.h>
 #include <__algorithm/any_of.h>
 #include <__algorithm/binary_search.h>
-#include <__algorithm/clamp.h>
-#include <__algorithm/comp.h>
-#include <__algorithm/comp_ref_type.h>
 #include <__algorithm/copy.h>
 #include <__algorithm/copy_backward.h>
 #include <__algorithm/copy_if.h>
@@ -1818,18 +1814,9 @@ template <class BidirectionalIterator, class Compare>
 #include <__algorithm/find_first_of.h>
 #include <__algorithm/find_if.h>
 #include <__algorithm/find_if_not.h>
-#include <__algorithm/fold.h>
 #include <__algorithm/for_each.h>
-#include <__algorithm/for_each_n.h>
 #include <__algorithm/generate.h>
 #include <__algorithm/generate_n.h>
-#include <__algorithm/half_positive.h>
-#include <__algorithm/in_found_result.h>
-#include <__algorithm/in_fun_result.h>
-#include <__algorithm/in_in_out_result.h>
-#include <__algorithm/in_in_result.h>
-#include <__algorithm/in_out_out_result.h>
-#include <__algorithm/in_out_result.h>
 #include <__algorithm/includes.h>
 #include <__algorithm/inplace_merge.h>
 #include <__algorithm/is_heap.h>
@@ -1840,7 +1827,6 @@ template <class BidirectionalIterator, class Compare>
 #include <__algorithm/is_sorted_until.h>
 #include <__algorithm/iter_swap.h>
 #include <__algorithm/lexicographical_compare.h>
-#include <__algorithm/lexicographical_compare_three_way.h>
 #include <__algorithm/lower_bound.h>
 #include <__algorithm/make_heap.h>
 #include <__algorithm/max.h>
@@ -1848,7 +1834,6 @@ template <class BidirectionalIterator, class Compare>
 #include <__algorithm/merge.h>
 #include <__algorithm/min.h>
 #include <__algorithm/min_element.h>
-#include <__algorithm/min_max_result.h>
 #include <__algorithm/minmax.h>
 #include <__algorithm/minmax_element.h>
 #include <__algorithm/mismatch.h>
@@ -1864,112 +1849,7 @@ template <class BidirectionalIterator, class Compare>
 #include <__algorithm/partition_point.h>
 #include <__algorithm/pop_heap.h>
 #include <__algorithm/prev_permutation.h>
-#include <__algorithm/pstl_any_all_none_of.h>
-#include <__algorithm/pstl_copy.h>
-#include <__algorithm/pstl_count.h>
-#include <__algorithm/pstl_equal.h>
-#include <__algorithm/pstl_fill.h>
-#include <__algorithm/pstl_find.h>
-#include <__algorithm/pstl_for_each.h>
-#include <__algorithm/pstl_generate.h>
-#include <__algorithm/pstl_is_partitioned.h>
-#include <__algorithm/pstl_merge.h>
-#include <__algorithm/pstl_move.h>
-#include <__algorithm/pstl_replace.h>
-#include <__algorithm/pstl_rotate_copy.h>
-#include <__algorithm/pstl_sort.h>
-#include <__algorithm/pstl_stable_sort.h>
-#include <__algorithm/pstl_transform.h>
 #include <__algorithm/push_heap.h>
-#include <__algorithm/ranges_adjacent_find.h>
-#include <__algorithm/ranges_all_of.h>
-#include <__algorithm/ranges_any_of.h>
-#include <__algorithm/ranges_binary_search.h>
-#include <__algorithm/ranges_clamp.h>
-#include <__algorithm/ranges_contains.h>
-#include <__algorithm/ranges_contains_subrange.h>
-#include <__algorithm/ranges_copy.h>
-#include <__algorithm/ranges_copy_backward.h>
-#include <__algorithm/ranges_copy_if.h>
-#include <__algorithm/ranges_copy_n.h>
-#include <__algorithm/ranges_count.h>
-#include <__algorithm/ranges_count_if.h>
-#include <__algorithm/ranges_ends_with.h>
-#include <__algorithm/ranges_equal.h>
-#include <__algorithm/ranges_equal_range.h>
-#include <__algorithm/ranges_fill.h>
-#include <__algorithm/ranges_fill_n.h>
-#include <__algorithm/ranges_find.h>
-#include <__algorithm/ranges_find_end.h>
-#include <__algorithm/ranges_find_first_of.h>
-#include <__algorithm/ranges_find_if.h>
-#include <__algorithm/ranges_find_if_not.h>
-#include <__algorithm/ranges_for_each.h>
-#include <__algorithm/ranges_for_each_n.h>
-#include <__algorithm/ranges_generate.h>
-#include <__algorithm/ranges_generate_n.h>
-#include <__algorithm/ranges_includes.h>
-#include <__algorithm/ranges_inplace_merge.h>
-#include <__algorithm/ranges_is_heap.h>
-#include <__algorithm/ranges_is_heap_until.h>
-#include <__algorithm/ranges_is_partitioned.h>
-#include <__algorithm/ranges_is_permutation.h>
-#include <__algorithm/ranges_is_sorted.h>
-#include <__algorithm/ranges_is_sorted_until.h>
-#include <__algorithm/ranges_lexicographical_compare.h>
-#include <__algorithm/ranges_lower_bound.h>
-#include <__algorithm/ranges_make_heap.h>
-#include <__algorithm/ranges_max.h>
-#include <__algorithm/ranges_max_element.h>
-#include <__algorithm/ranges_merge.h>
-#include <__algorithm/ranges_min.h>
-#include <__algorithm/ranges_min_element.h>
-#include <__algorithm/ranges_minmax.h>
-#include <__algorithm/ranges_minmax_element.h>
-#include <__algorithm/ranges_mismatch.h>
-#include <__algorithm/ranges_move.h>
-#include <__algorithm/ranges_move_backward.h>
-#include <__algorithm/ranges_next_permutation.h>
-#include <__algorithm/ranges_none_of.h>
-#include <__algorithm/ranges_nth_element.h>
-#include <__algorithm/ranges_partial_sort.h>
-#include <__algorithm/ranges_partial_sort_copy.h>
-#include <__algorithm/ranges_partition.h>
-#include <__algorithm/ranges_partition_copy.h>
-#include <__algorithm/ranges_partition_point.h>
-#include <__algorithm/ranges_pop_heap.h>
-#include <__algorithm/ranges_prev_permutation.h>
-#include <__algorithm/ranges_push_heap.h>
-#include <__algorithm/ranges_remove.h>
-#include <__algorithm/ranges_remove_copy.h>
-#include <__algorithm/ranges_remove_copy_if.h>
-#include <__algorithm/ranges_remove_if.h>
-#include <__algorithm/ranges_replace.h>
-#include <__algorithm/ranges_replace_copy.h>
-#include <__algorithm/ranges_replace_copy_if.h>
-#include <__algorithm/ranges_replace_if.h>
-#include <__algorithm/ranges_reverse.h>
-#include <__algorithm/ranges_reverse_copy.h>
-#include <__algorithm/ranges_rotate.h>
-#include <__algorithm/ranges_rotate_copy.h>
-#include <__algorithm/ranges_sample.h>
-#include <__algorithm/ranges_search.h>
-#include <__algorithm/ranges_search_n.h>
-#include <__algorithm/ranges_set_difference.h>
-#include <__algorithm/ranges_set_intersection.h>
-#include <__algorithm/ranges_set_symmetric_difference.h>
-#include <__algorithm/ranges_set_union.h>
-#include <__algorithm/ranges_shuffle.h>
-#include <__algorithm/ranges_sort.h>
-#include <__algorithm/ranges_sort_heap.h>
-#include <__algorithm/ranges_stable_partition.h>
-#include <__algorithm/ranges_stable_sort.h>
-#include <__algorithm/ranges_starts_with.h>
-#include <__algorithm/ranges_swap_ranges.h>
-#include <__algorithm/ranges_transform.h>
-#include <__algorithm/ranges_unique.h>
-#include <__algorithm/ranges_unique_copy.h>
-#include <__algorithm/ranges_upper_bound.h>
 #include <__algorithm/remove.h>
 #include <__algorithm/remove_copy.h>
 #include <__algorithm/remove_copy_if.h>
@@ -1982,17 +1862,13 @@ template <class BidirectionalIterator, class Compare>
 #include <__algorithm/reverse_copy.h>
 #include <__algorithm/rotate.h>
 #include <__algorithm/rotate_copy.h>
-#include <__algorithm/sample.h>
 #include <__algorithm/search.h>
 #include <__algorithm/search_n.h>
 #include <__algorithm/set_difference.h>
 #include <__algorithm/set_intersection.h>
 #include <__algorithm/set_symmetric_difference.h>
 #include <__algorithm/set_union.h>
-#include <__algorithm/shift_left.h>
-#include <__algorithm/shift_right.h>
 #include <__algorithm/shuffle.h>
-#include <__algorithm/sift_down.h>
 #include <__algorithm/sort.h>
 #include <__algorithm/sort_heap.h>
 #include <__algorithm/stable_partition.h>
@@ -2001,9 +1877,138 @@ template <class BidirectionalIterator, class Compare>
 #include <__algorithm/transform.h>
 #include <__algorithm/unique.h>
 #include <__algorithm/unique_copy.h>
-#include <__algorithm/unwrap_iter.h>
 #include <__algorithm/upper_bound.h>
 
+#if _LIBCPP_STD_VER >= 17
+#  include <__algorithm/clamp.h>
+#  include <__algorithm/for_each_n.h>
+#  include <__algorithm/pstl_any_all_none_of.h>
+#  include <__algorithm/pstl_copy.h>
+#  include <__algorithm/pstl_count.h>
+#  include <__algorithm/pstl_equal.h>
+#  include <__algorithm/pstl_fill.h>
+#  include <__algorithm/pstl_find.h>
+#  include <__algorithm/pstl_for_each.h>
+#  include <__algorithm/pstl_generate.h>
+#  include <__algorithm/pstl_is_partitioned.h>
+#  include <__algorithm/pstl_merge.h>
+#  include <__algorithm/pstl_move.h>
+#  include <__algorithm/pstl_replace.h>
+#  include <__algorithm/pstl_rotate_copy.h>
+#  include <__algorithm/pstl_sort.h>
+#  include <__algorithm/pstl_stable_sort.h>
+#  include <__algorithm/pstl_transform.h>
+#  include <__algorithm/sample.h>
+#endif // _LIBCPP_STD_VER >= 17
+
+#if _LIBCPP_STD_VER >= 20
+#  include <__algorithm/in_found_result.h>
+#  include <__algorithm/in_fun_result.h>
+#  include <__algorithm/in_in_out_result.h>
+#  include <__algorithm/in_in_result.h>
+#  include <__algorithm/in_out_out_result.h>
+#  include <__algorithm/in_out_result.h>
+#  include <__algorithm/lexicographical_compare_three_way.h>
+#  include <__algorithm/min_max_result.h>
+#  include <__algorithm/ranges_adjacent_find.h>
+#  include <__algorithm/ranges_all_of.h>
+#  include <__algorithm/ranges_any_of.h>
+#  include <__algorithm/ranges_binary_search.h>
+#  include <__algorithm/ranges_clamp.h>
+#  include <__algorithm/ranges_contains.h>
+#  include <__algorithm/ranges_copy.h>
+#  include <__algorithm/ranges_copy_backward.h>
+#  include <__algorithm/ranges_copy_if.h>
+#  include <__algorithm/ranges_copy_n.h>
+#  include <__algorithm/ranges_count.h>
+#  include <__algorithm/ranges_count_if.h>
+#  include <__algorithm/ranges_equal.h>
+#  include <__algorithm/ranges_equal_range.h>
+#  include <__algorithm/ranges_fill.h>
+#  include <__algorithm/ranges_fill_n.h>
+#  include <__algorithm/ranges_find.h>
+#  include <__algorithm/ranges_find_end.h>
+#  include <__algorithm/ranges_find_first_of.h>
+#  include <__algorithm/ranges_find_if.h>
+#  include <__algorithm/ranges_find_if_not.h>
+#  include <__algorithm/ranges_for_each.h>
+#  include <__algorithm/ranges_for_each_n.h>
+#  include <__algorithm/ranges_generate.h>
+#  include <__algorithm/ranges_generate_n.h>
+#  include <__algorithm/ranges_includes.h>
+#  include <__algorithm/ranges_inplace_merge.h>
+#  include <__algorithm/ranges_is_heap.h>
+#  include <__algorithm/ranges_is_heap_until.h>
+#  include <__algorithm/ranges_is_partitioned.h>
+#  include <__algorithm/ranges_is_permutation.h>
+#  include <__algorithm/ranges_is_sorted.h>
+#  include <__algorithm/ranges_is_sorted_until.h>
+#  include <__algorithm/ranges_lexicographical_compare.h>
+#  include <__algorithm/ranges_lower_bound.h>
+#  include <__algorithm/ranges_make_heap.h>
+#  include <__algorithm/ranges_max.h>
+#  include <__algorithm/ranges_max_element.h>
+#  include <__algorithm/ranges_merge.h>
+#  include <__algorithm/ranges_min.h>
+#  include <__algorithm/ranges_min_element.h>
+#  include <__algorithm/ranges_minmax.h>
+#  include <__algorithm/ranges_minmax_element.h>
+#  include <__algorithm/ranges_mismatch.h>
+#  include <__algorithm/ranges_move.h>
+#  include <__algorithm/ranges_move_backward.h>
+#  include <__algorithm/ranges_next_permutation.h>
+#  include <__algorithm/ranges_none_of.h>
+#  include <__algorithm/ranges_nth_element.h>
+#  include <__algorithm/ranges_partial_sort.h>
+#  include <__algorithm/ranges_partial_sort_copy.h>
+#  include <__algorithm/ranges_partition.h>
+#  include <__algorithm/ranges_partition_copy.h>
+#  include <__algorithm/ranges_partition_point.h>
+#  include <__algorithm/ranges_pop_heap.h>
+#  include <__algorithm/ranges_prev_permutation.h>
+#  include <__algorithm/ranges_push_heap.h>
+#  include <__algorithm/ranges_remove.h>
+#  include <__algorithm/ranges_remove_copy.h>
+#  include <__algorithm/ranges_remove_copy_if.h>
+#  include <__algorithm/ranges_remove_if.h>
+#  include <__algorithm/ranges_replace.h>
+#  include <__algorithm/ranges_replace_copy.h>
+#  include <__algorithm/ranges_replace_copy_if.h>
+#  include <__algorithm/ranges_replace_if.h>
+#  include <__algorithm/ranges_reverse.h>
+#  include <__algorithm/ranges_reverse_copy.h>
+#  include <__algorithm/ranges_rotate.h>
+#  include <__algorithm/ranges_rotate_copy.h>
+#  include <__algorithm/ranges_sample.h>
+#  include <__algorithm/ranges_search.h>
+#  include <__algorithm/ranges_search_n.h>
+#  include <__algorithm/ranges_set_difference.h>
+#  include <__algorithm/ranges_set_intersection.h>
+#  include <__algorithm/ranges_set_symmetric_difference.h>
+#  include <__algorithm/ranges_set_union.h>
+#  include <__algorithm/ranges_shuffle.h>
+#  include <__algorithm/ranges_sort.h>
+#  include <__algorithm/ranges_sort_heap.h>
+#  include <__algorithm/ranges_stable_partition.h>
+#  include <__algorithm/ranges_stable_sort.h>
+#  include <__algorithm/ranges_swap_ranges.h>
+#  include <__algorithm/ranges_transform.h>
+#  include <__algorithm/ranges_unique.h>
+#  include <__algorithm/ranges_unique_copy.h>
+#  include <__algorithm/ranges_upper_bound.h>
+#  include <__algorithm/shift_left.h>
+#  include <__algorithm/shift_right.h>
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+#  include <__algorithm/fold.h>
+#  include <__algorithm/ranges_contains_subrange.h>
+#  include <__algorithm/ranges_ends_with.h>
+#  include <__algorithm/ranges_starts_with.h>
+#endif // _LIBCPP_STD_VER >= 23
+
+#include <version>
+
 // standard-mandated includes
 
 // [algorithm.syn]
@@ -2013,6 +2018,10 @@ template <class BidirectionalIterator, class Compare>
 #  pragma GCC system_header
 #endif
 
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER == 14
+#  include <execution>
+#endif
+
 #if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
 #  include <atomic>
 #  include <bit>
diff --git a/libcxx/include/bit b/libcxx/include/bit
index b8e4bdc2dfe2..94387d101a39 100644
--- a/libcxx/include/bit
+++ b/libcxx/include/bit
@@ -61,26 +61,36 @@ namespace std {
 
 */
 
-#include <__bit/bit_cast.h>
-#include <__bit/bit_ceil.h>
-#include <__bit/bit_floor.h>
-#include <__bit/bit_log2.h>
-#include <__bit/bit_width.h>
-#include <__bit/blsr.h>
-#include <__bit/byteswap.h>
-#include <__bit/countl.h>
-#include <__bit/countr.h>
-#include <__bit/endian.h>
-#include <__bit/has_single_bit.h>
-#include <__bit/popcount.h>
-#include <__bit/rotate.h>
 #include <__config>
+
+#if _LIBCPP_STD_VER >= 20
+#  include <__bit/bit_cast.h>
+#  include <__bit/bit_ceil.h>
+#  include <__bit/bit_floor.h>
+#  include <__bit/bit_log2.h>
+#  include <__bit/bit_width.h>
+#  include <__bit/countl.h>
+#  include <__bit/countr.h>
+#  include <__bit/endian.h>
+#  include <__bit/has_single_bit.h>
+#  include <__bit/popcount.h>
+#  include <__bit/rotate.h>
+#endif
+
+#if _LIBCPP_STD_VER >= 23
+#  include <__bit/byteswap.h>
+#endif
+
 #include <version>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
 #endif
 
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 17
+#  include <cstdint>
+#endif
+
 #if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
 #  include <cstdlib>
 #  include <iosfwd>
diff --git a/libcxx/include/charconv b/libcxx/include/charconv
index 5bc7b9011be0..a2e270e9316d 100644
--- a/libcxx/include/charconv
+++ b/libcxx/include/charconv
@@ -69,18 +69,21 @@ namespace std {
 
 */
 
-#include <__charconv/chars_format.h>
-#include <__charconv/from_chars_integral.h>
-#include <__charconv/from_chars_result.h>
-#include <__charconv/tables.h>
-#include <__charconv/to_chars.h>
-#include <__charconv/to_chars_base_10.h>
-#include <__charconv/to_chars_floating_point.h>
-#include <__charconv/to_chars_integral.h>
-#include <__charconv/to_chars_result.h>
-#include <__charconv/traits.h>
 #include <__config>
-#include <__system_error/errc.h>
+
+#if _LIBCPP_STD_VER >= 17
+#  include <__charconv/chars_format.h>
+#  include <__charconv/from_chars_integral.h>
+#  include <__charconv/from_chars_result.h>
+#  include <__charconv/tables.h>
+#  include <__charconv/to_chars.h>
+#  include <__charconv/to_chars_base_10.h>
+#  include <__charconv/to_chars_floating_point.h>
+#  include <__charconv/to_chars_integral.h>
+#  include <__charconv/to_chars_result.h>
+#  include <__charconv/traits.h>
+#endif // _LIBCPP_STD_VER >= 17
+
 #include <version>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -91,6 +94,13 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 _LIBCPP_END_NAMESPACE_STD
 
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 14
+#  include <cerrno>
+#  include <cstddef>
+#  include <initializer_list>
+#  include <new>
+#endif
+
 #if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
 #  include <cmath>
 #  include <concepts>
diff --git a/libcxx/include/chrono b/libcxx/include/chrono
index 5eddd050196d..513ae52006e8 100644
--- a/libcxx/include/chrono
+++ b/libcxx/include/chrono
@@ -880,28 +880,48 @@ constexpr chrono::year                                  operator ""y(unsigned lo
 
 // clang-format on
 
-#include <__chrono/calendar.h>
-#include <__chrono/convert_to_timespec.h>
-#include <__chrono/convert_to_tm.h>
-#include <__chrono/day.h>
+#include <__config>
+
 #include <__chrono/duration.h>
 #include <__chrono/file_clock.h>
-#include <__chrono/hh_mm_ss.h>
 #include <__chrono/high_resolution_clock.h>
-#include <__chrono/literals.h>
-#include <__chrono/month.h>
-#include <__chrono/month_weekday.h>
-#include <__chrono/monthday.h>
 #include <__chrono/steady_clock.h>
-#include <__chrono/sys_info.h>
 #include <__chrono/system_clock.h>
 #include <__chrono/time_point.h>
-#include <__chrono/weekday.h>
-#include <__chrono/year.h>
-#include <__chrono/year_month.h>
-#include <__chrono/year_month_day.h>
-#include <__chrono/year_month_weekday.h>
-#include <__config>
+
+#if _LIBCPP_STD_VER >= 20
+#  include <__chrono/calendar.h>
+#  include <__chrono/day.h>
+#  include <__chrono/hh_mm_ss.h>
+#  include <__chrono/literals.h>
+#  include <__chrono/month.h>
+#  include <__chrono/month_weekday.h>
+#  include <__chrono/monthday.h>
+#  include <__chrono/sys_info.h>
+#  include <__chrono/weekday.h>
+#  include <__chrono/year.h>
+#  include <__chrono/year_month.h>
+#  include <__chrono/year_month_day.h>
+#  include <__chrono/year_month_weekday.h>
+
+#  if !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#    include <__chrono/formatter.h>
+#    include <__chrono/ostream.h>
+#    include <__chrono/parser_std_format_spec.h>
+#    include <__chrono/statically_widen.h>
+#  endif
+
+#  if !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&                            \
+      !defined(_LIBCPP_HAS_NO_LOCALIZATION)
+#    include <__chrono/leap_second.h>
+#    include <__chrono/time_zone.h>
+#    include <__chrono/time_zone_link.h>
+#    include <__chrono/tzdb.h>
+#    include <__chrono/tzdb_list.h>
+#  endif
+
+#endif
+
 #include <version>
 
 // standard-mandated includes
@@ -909,26 +929,17 @@ constexpr chrono::year                                  operator ""y(unsigned lo
 // [time.syn]
 #include <compare>
 
-#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) && _LIBCPP_STD_VER >= 20
-#  include <__chrono/formatter.h>
-#  include <__chrono/ostream.h>
-#  include <__chrono/parser_std_format_spec.h>
-#  include <__chrono/statically_widen.h>
-#endif
-
-#if !defined(_LIBCPP_HAS_NO_TIME_ZONE_DATABASE) && !defined(_LIBCPP_HAS_NO_FILESYSTEM) &&                              \
-    !defined(_LIBCPP_HAS_NO_LOCALIZATION)
-#  include <__chrono/leap_second.h>
-#  include <__chrono/time_zone.h>
-#  include <__chrono/time_zone_link.h>
-#  include <__chrono/tzdb.h>
-#  include <__chrono/tzdb_list.h>
-#endif
-
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
 #endif
 
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 17
+#  include <cstdint>
+#  include <stdexcept>
+#  include <string_view>
+#  include <vector>
+#endif
+
 #if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
 #  include <bit>
 #  include <concepts>
diff --git a/libcxx/include/compare b/libcxx/include/compare
index 93953254b784..8a41835b1489 100644
--- a/libcxx/include/compare
+++ b/libcxx/include/compare
@@ -140,26 +140,36 @@ namespace std {
 }
 */
 
-#include <__compare/common_comparison_category.h>
-#include <__compare/compare_partial_order_fallback.h>
-#include <__compare/compare_strong_order_fallback.h>
-#include <__compare/compare_three_way.h>
-#include <__compare/compare_three_way_result.h>
-#include <__compare/compare_weak_order_fallback.h>
-#include <__compare/is_eq.h>
-#include <__compare/ordering.h>
-#include <__compare/partial_order.h>
-#include <__compare/strong_order.h>
-#include <__compare/synth_three_way.h>
-#include <__compare/three_way_comparable.h>
-#include <__compare/weak_order.h>
 #include <__config>
+
+#if _LIBCPP_STD_VER >= 20
+#  include <__compare/common_comparison_category.h>
+#  include <__compare/compare_partial_order_fallback.h>
+#  include <__compare/compare_strong_order_fallback.h>
+#  include <__compare/compare_three_way.h>
+#  include <__compare/compare_three_way_result.h>
+#  include <__compare/compare_weak_order_fallback.h>
+#  include <__compare/is_eq.h>
+#  include <__compare/ordering.h>
+#  include <__compare/partial_order.h>
+#  include <__compare/strong_order.h>
+#  include <__compare/synth_three_way.h>
+#  include <__compare/three_way_comparable.h>
+#  include <__compare/weak_order.h>
+#endif // _LIBCPP_STD_VER >= 20
+
 #include <version>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
 #endif
 
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 17
+#  include <cstddef>
+#  include <cstdint>
+#  include <limits>
+#endif
+
 #if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
 #  include <cmath>
 #  include <type_traits>
diff --git a/libcxx/include/concepts b/libcxx/include/concepts
index e10f5ab5ad8a..e89d216a5937 100644
--- a/libcxx/include/concepts
+++ b/libcxx/include/concepts
@@ -129,31 +129,39 @@ namespace std {
 
 */
 
-#include <__concepts/arithmetic.h>
-#include <__concepts/assignable.h>
-#include <__concepts/boolean_testable.h>
-#include <__concepts/class_or_enum.h>
-#include <__concepts/common_reference_with.h>
-#include <__concepts/common_with.h>
-#include <__concepts/constructible.h>
-#include <__concepts/convertible_to.h>
-#include <__concepts/copyable.h>
-#include <__concepts/derived_from.h>
-#include <__concepts/destructible.h>
-#include <__concepts/different_from.h>
-#include <__concepts/equality_comparable.h>
-#include <__concepts/invocable.h>
-#include <__concepts/movable.h>
-#include <__concepts/predicate.h>
-#include <__concepts/regular.h>
-#include <__concepts/relation.h>
-#include <__concepts/same_as.h>
-#include <__concepts/semiregular.h>
-#include <__concepts/swappable.h>
-#include <__concepts/totally_ordered.h>
 #include <__config>
+
+#if _LIBCPP_STD_VER >= 20
+#  include <__concepts/arithmetic.h>
+#  include <__concepts/assignable.h>
+#  include <__concepts/boolean_testable.h>
+#  include <__concepts/class_or_enum.h>
+#  include <__concepts/common_reference_with.h>
+#  include <__concepts/common_with.h>
+#  include <__concepts/constructible.h>
+#  include <__concepts/convertible_to.h>
+#  include <__concepts/copyable.h>
+#  include <__concepts/derived_from.h>
+#  include <__concepts/destructible.h>
+#  include <__concepts/different_from.h>
+#  include <__concepts/equality_comparable.h>
+#  include <__concepts/invocable.h>
+#  include <__concepts/movable.h>
+#  include <__concepts/predicate.h>
+#  include <__concepts/regular.h>
+#  include <__concepts/relation.h>
+#  include <__concepts/same_as.h>
+#  include <__concepts/semiregular.h>
+#  include <__concepts/swappable.h>
+#  include <__concepts/totally_ordered.h>
+#endif // _LIBCPP_STD_VER >= 20
+
 #include <version>
 
+#if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 17
+#  include <cstddef>
+#endif
+
 #if _LIBCPP_STD_VER <= 20 && !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES)
 #  include <type_traits>
 #endif
diff --git a/libcxx/include/coroutine b/libcxx/include/coroutine
index b1ba83b541b4..d1a44e2e12dd 100644
--- a/libcxx/include/coroutine
+++ b/libcxx/include/coroutine
@@ -39,10 +39,14 @@ struct suspend_always;
  */
 
 #include <__config>
+
+#if _LIBCPP_STD_VER >= 20
 #include <__coroutine/coroutine_handle.h>
 #include <__coroutine/coroutine_traits.h>
 #include <__coroutine/noop_coroutine_handle.h>
 #include <__coroutine/trivial_awaitables.h>
+#endif // _LIBCPP_STD_VER >= 20
+
 #include <version>
 
 // standard-mandated includes
diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap
index 372219e8f836..ccee7029824e 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap
@@ -1123,7 +1123,10 @@ module std_private_charconv_to_chars                [system] { header "__charcon
 module std_private_charconv_to_chars_base_10        [system] { header "__charconv/to_chars_base_10.h" }
 module std_private_charconv_to_chars_floating_point [system] { header "__charconv/to_chars_floating_point.h" }
 module std_private_charconv_to_chars_integral       [system] { header "__charconv/to_chars_integral.h" }
-module std_private_charconv_to_chars_result         [system] { header "__charconv/to_chars_result.h" }
+module std_private_charconv_to_chars_result         [system] {
+  header "__charconv/to_chars_result.h"
+  export *
+}
 module std_private_charconv_traits                  [system] { header "__charconv/traits.h" }
 
 module std_private_chrono_calendar               [system] { header "__chrono/calendar.h" }
diff --git a/libcxx/include/streambuf b/libcxx/include/streambuf
index aec537866c20..7964758c908f 100644
--- a/libcxx/include/streambuf
+++ b/libcxx/include/streambuf
@@ -109,6 +109,7 @@ protected:
 
 #include <__config>
 #include <__fwd/streambuf.h>
+#include <__locale>
 #include <__type_traits/is_same.h>
 #include <climits>
 #include <ios>
diff --git a/libcxx/test/libcxx/algorithms/half_positive.pass.cpp b/libcxx/test/libcxx/algorithms/half_positive.pass.cpp
index 82d18cba37f3..40f469741957 100644
--- a/libcxx/test/libcxx/algorithms/half_positive.pass.cpp
+++ b/libcxx/test/libcxx/algorithms/half_positive.pass.cpp
@@ -11,10 +11,10 @@
 // __half_positive divides an integer number by 2 as unsigned number for known types.
 // It can be an important optimization for lower bound, for example.
 
-#include <algorithm>
+#include <__algorithm/half_positive.h>
 #include <cassert>
+#include <cstddef>
 #include <limits>
-#include <type_traits>
 
 #include "test_macros.h"
 #include "user_defined_integral.h"
diff --git a/libcxx/test/libcxx/numerics/bit.ops.pass.cpp b/libcxx/test/libcxx/numerics/bit.ops.pass.cpp
index d3ca8b2f8030..7f502d6e01d1 100644
--- a/libcxx/test/libcxx/numerics/bit.ops.pass.cpp
+++ b/libcxx/test/libcxx/numerics/bit.ops.pass.cpp
@@ -9,7 +9,9 @@
 // Test the __XXXX routines in the <bit> header.
 // These are not supposed to be exhaustive tests, just sanity checks.
 
-#include <bit>
+#include <__bit/bit_log2.h>
+#include <__bit/countl.h>
+#include <__bit/rotate.h>
 #include <cassert>
 
 #include "test_macros.h"
diff --git a/libcxx/test/libcxx/time/convert_to_tm.pass.cpp b/libcxx/test/libcxx/time/convert_to_tm.pass.cpp
index 708d2c3ff7c2..908a38dec83d 100644
--- a/libcxx/test/libcxx/time/convert_to_tm.pass.cpp
+++ b/libcxx/test/libcxx/time/convert_to_tm.pass.cpp
@@ -15,6 +15,7 @@
 // Most of the code is tested indirectly in the chrono formatters. This only
 // tests the hour overflow.
 
+#include <__chrono/convert_to_tm.h>
 #include <chrono>
 #include <cassert>
 #include <format>
diff --git a/libcxx/test/libcxx/transitive_includes/cxx03.csv b/libcxx/test/libcxx/transitive_includes/cxx03.csv
index 2e246644f626..c2250899a800 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx03.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx03.csv
@@ -7,7 +7,6 @@ algorithm cstdint
 algorithm cstdlib
 algorithm cstring
 algorithm cwchar
-algorithm execution
 algorithm initializer_list
 algorithm iosfwd
 algorithm iterator
@@ -777,10 +776,21 @@ stop_token limits
 stop_token ratio
 stop_token type_traits
 stop_token version
+streambuf cctype
 streambuf climits
+streambuf clocale
+streambuf cstddef
 streambuf cstdint
+streambuf cstdlib
+streambuf cstring
+streambuf cwchar
+streambuf initializer_list
 streambuf ios
 streambuf iosfwd
+streambuf limits
+streambuf new
+streambuf string
+streambuf typeinfo
 streambuf version
 string algorithm
 string climits
diff --git a/libcxx/test/libcxx/transitive_includes/cxx11.csv b/libcxx/test/libcxx/transitive_includes/cxx11.csv
index e074bf1f7dcc..3e929e8f9409 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx11.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx11.csv
@@ -7,7 +7,6 @@ algorithm cstdint
 algorithm cstdlib
 algorithm cstring
 algorithm cwchar
-algorithm execution
 algorithm initializer_list
 algorithm iosfwd
 algorithm iterator
@@ -783,10 +782,22 @@ stop_token limits
 stop_token ratio
 stop_token type_traits
 stop_token version
+streambuf cctype
 streambuf climits
+streambuf clocale
+streambuf cstddef
 streambuf cstdint
+streambuf cstdlib
+streambuf cstring
+streambuf cwchar
+streambuf initializer_list
 streambuf ios
 streambuf iosfwd
+streambuf limits
+streambuf new
+streambuf string
+streambuf tuple
+streambuf typeinfo
 streambuf version
 string algorithm
 string climits
diff --git a/libcxx/test/libcxx/transitive_includes/cxx14.csv b/libcxx/test/libcxx/transitive_includes/cxx14.csv
index 88f9c24f0864..422db19b6bb8 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx14.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx14.csv
@@ -785,10 +785,22 @@ stop_token limits
 stop_token ratio
 stop_token type_traits
 stop_token version
+streambuf cctype
 streambuf climits
+streambuf clocale
+streambuf cstddef
 streambuf cstdint
+streambuf cstdlib
+streambuf cstring
+streambuf cwchar
+streambuf initializer_list
 streambuf ios
 streambuf iosfwd
+streambuf limits
+streambuf new
+streambuf string
+streambuf tuple
+streambuf typeinfo
 streambuf version
 string algorithm
 string climits
diff --git a/libcxx/test/libcxx/transitive_includes/cxx17.csv b/libcxx/test/libcxx/transitive_includes/cxx17.csv
index 88f9c24f0864..422db19b6bb8 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx17.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx17.csv
@@ -785,10 +785,22 @@ stop_token limits
 stop_token ratio
 stop_token type_traits
 stop_token version
+streambuf cctype
 streambuf climits
+streambuf clocale
+streambuf cstddef
 streambuf cstdint
+streambuf cstdlib
+streambuf cstring
+streambuf cwchar
+streambuf initializer_list
 streambuf ios
 streambuf iosfwd
+streambuf limits
+streambuf new
+streambuf string
+streambuf tuple
+streambuf typeinfo
 streambuf version
 string algorithm
 string climits
diff --git a/libcxx/test/libcxx/transitive_includes/cxx20.csv b/libcxx/test/libcxx/transitive_includes/cxx20.csv
index 27f59660fb98..6b80790a9d19 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx20.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx20.csv
@@ -795,10 +795,22 @@ stop_token limits
 stop_token ratio
 stop_token type_traits
 stop_token version
+streambuf cctype
 streambuf climits
+streambuf clocale
+streambuf cstddef
 streambuf cstdint
+streambuf cstdlib
+streambuf cstring
+streambuf cwchar
+streambuf initializer_list
 streambuf ios
 streambuf iosfwd
+streambuf limits
+streambuf new
+streambuf string
+streambuf tuple
+streambuf typeinfo
 streambuf version
 string algorithm
 string climits
diff --git a/libcxx/test/libcxx/transitive_includes/cxx23.csv b/libcxx/test/libcxx/transitive_includes/cxx23.csv
index 9ae422a31f07..ea01e4134585 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx23.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx23.csv
@@ -541,9 +541,22 @@ stop_token ctime
 stop_token limits
 stop_token ratio
 stop_token version
+streambuf cctype
 streambuf climits
+streambuf clocale
+streambuf cstddef
+streambuf cstdint
+streambuf cstdlib
+streambuf cstring
+streambuf cwchar
+streambuf initializer_list
 streambuf ios
 streambuf iosfwd
+streambuf limits
+streambuf new
+streambuf string
+streambuf tuple
+streambuf typeinfo
 streambuf version
 string climits
 string compare
diff --git a/libcxx/test/libcxx/transitive_includes/cxx26.csv b/libcxx/test/libcxx/transitive_includes/cxx26.csv
index 9ae422a31f07..ea01e4134585 100644
--- a/libcxx/test/libcxx/transitive_includes/cxx26.csv
+++ b/libcxx/test/libcxx/transitive_includes/cxx26.csv
@@ -541,9 +541,22 @@ stop_token ctime
 stop_token limits
 stop_token ratio
 stop_token version
+streambuf cctype
 streambuf climits
+streambuf clocale
+streambuf cstddef
+streambuf cstdint
+streambuf cstdlib
+streambuf cstring
+streambuf cwchar
+streambuf initializer_list
 streambuf ios
 streambuf iosfwd
+streambuf limits
+streambuf new
+streambuf string
+streambuf tuple
+streambuf typeinfo
 streambuf version
 string climits
 string compare
-- 
GitLab


From 00162162dd853795c532afa5dec4dc4e798d4a4b Mon Sep 17 00:00:00 2001
From: YunQiang Su <syq@debian.org>
Date: Sun, 14 Apr 2024 23:16:52 +0800
Subject: [PATCH 173/464] CompilerRT: Normalize
 COMPILER_RT_DEFAULT_TARGET_TRIPLE (#88407)

If LLVM is configured with -DLLVM_DEFAULT_TARGET_TRIPLE, and the
argument is not normalized, such as Debian-style vendor-less triple,
clang will try to find libclang_rt in lib/<normalized_triple>, while
libclang_rt is placed into lib/<triple_arg>.

Let's also place libclang_rt into lib/<normalized_triple>.
---
 .../cmake/Modules/CompilerRTUtils.cmake        | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/compiler-rt/cmake/Modules/CompilerRTUtils.cmake b/compiler-rt/cmake/Modules/CompilerRTUtils.cmake
index e8e5f612d5b0..75f111efeb10 100644
--- a/compiler-rt/cmake/Modules/CompilerRTUtils.cmake
+++ b/compiler-rt/cmake/Modules/CompilerRTUtils.cmake
@@ -362,10 +362,22 @@ macro(construct_compiler_rt_default_triple)
       message(FATAL_ERROR "CMAKE_C_COMPILER_TARGET must also be set when COMPILER_RT_DEFAULT_TARGET_ONLY is ON")
     endif()
     message(STATUS "cmake c compiler target: ${CMAKE_C_COMPILER_TARGET}")
-    set(COMPILER_RT_DEFAULT_TARGET_TRIPLE ${CMAKE_C_COMPILER_TARGET})
+    if ("${CMAKE_C_COMPILER_ID}" MATCHES "Clang")
+      execute_process(COMMAND ${CMAKE_C_COMPILER} --target=${CMAKE_C_COMPILER_TARGET} -print-effective-triple
+                      OUTPUT_VARIABLE COMPILER_RT_DEFAULT_TARGET_TRIPLE
+                      OUTPUT_STRIP_TRAILING_WHITESPACE)
+    else()
+      set(COMPILER_RT_DEFAULT_TARGET_TRIPLE ${CMAKE_C_COMPILER_TARGET})
+    endif()
   else()
-    set(COMPILER_RT_DEFAULT_TARGET_TRIPLE ${LLVM_TARGET_TRIPLE} CACHE STRING
-          "Default triple for which compiler-rt runtimes will be built.")
+    if ("${CMAKE_C_COMPILER_ID}" MATCHES "Clang")
+      execute_process(COMMAND ${CMAKE_C_COMPILER} --target=${LLVM_TARGET_TRIPLE} -print-effective-triple
+                      OUTPUT_VARIABLE COMPILER_RT_DEFAULT_TARGET_TRIPLE
+                      OUTPUT_STRIP_TRAILING_WHITESPACE)
+    else()
+      set(COMPILER_RT_DEFAULT_TARGET_TRIPLE ${LLVM_TARGET_TRIPLE} CACHE STRING
+            "Default triple for which compiler-rt runtimes will be built.")
+    endif()
   endif()
 
   string(REPLACE "-" ";" LLVM_TARGET_TRIPLE_LIST ${COMPILER_RT_DEFAULT_TARGET_TRIPLE})
-- 
GitLab


From 5927492e8e68c1ee5a0d84cf6c402a900aac4a3c Mon Sep 17 00:00:00 2001
From: YunQiang Su <syq@debian.org>
Date: Mon, 15 Apr 2024 00:12:42 +0800
Subject: [PATCH 174/464] Revert "CompilerRT: Normalize
 COMPILER_RT_DEFAULT_TARGET_TRIPLE" (#88663)

Reverts llvm/llvm-project#88407
---
 .../cmake/Modules/CompilerRTUtils.cmake        | 18 +++---------------
 1 file changed, 3 insertions(+), 15 deletions(-)

diff --git a/compiler-rt/cmake/Modules/CompilerRTUtils.cmake b/compiler-rt/cmake/Modules/CompilerRTUtils.cmake
index 75f111efeb10..e8e5f612d5b0 100644
--- a/compiler-rt/cmake/Modules/CompilerRTUtils.cmake
+++ b/compiler-rt/cmake/Modules/CompilerRTUtils.cmake
@@ -362,22 +362,10 @@ macro(construct_compiler_rt_default_triple)
       message(FATAL_ERROR "CMAKE_C_COMPILER_TARGET must also be set when COMPILER_RT_DEFAULT_TARGET_ONLY is ON")
     endif()
     message(STATUS "cmake c compiler target: ${CMAKE_C_COMPILER_TARGET}")
-    if ("${CMAKE_C_COMPILER_ID}" MATCHES "Clang")
-      execute_process(COMMAND ${CMAKE_C_COMPILER} --target=${CMAKE_C_COMPILER_TARGET} -print-effective-triple
-                      OUTPUT_VARIABLE COMPILER_RT_DEFAULT_TARGET_TRIPLE
-                      OUTPUT_STRIP_TRAILING_WHITESPACE)
-    else()
-      set(COMPILER_RT_DEFAULT_TARGET_TRIPLE ${CMAKE_C_COMPILER_TARGET})
-    endif()
+    set(COMPILER_RT_DEFAULT_TARGET_TRIPLE ${CMAKE_C_COMPILER_TARGET})
   else()
-    if ("${CMAKE_C_COMPILER_ID}" MATCHES "Clang")
-      execute_process(COMMAND ${CMAKE_C_COMPILER} --target=${LLVM_TARGET_TRIPLE} -print-effective-triple
-                      OUTPUT_VARIABLE COMPILER_RT_DEFAULT_TARGET_TRIPLE
-                      OUTPUT_STRIP_TRAILING_WHITESPACE)
-    else()
-      set(COMPILER_RT_DEFAULT_TARGET_TRIPLE ${LLVM_TARGET_TRIPLE} CACHE STRING
-            "Default triple for which compiler-rt runtimes will be built.")
-    endif()
+    set(COMPILER_RT_DEFAULT_TARGET_TRIPLE ${LLVM_TARGET_TRIPLE} CACHE STRING
+          "Default triple for which compiler-rt runtimes will be built.")
   endif()
 
   string(REPLACE "-" ";" LLVM_TARGET_TRIPLE_LIST ${COMPILER_RT_DEFAULT_TARGET_TRIPLE})
-- 
GitLab


From b8d0cba14bcfc5c1c2f7a878ad9eaa12b6a590b6 Mon Sep 17 00:00:00 2001
From: Cyndy Ishida <cyndy_ishida@apple.com>
Date: Sun, 14 Apr 2024 09:23:42 -0700
Subject: [PATCH 175/464] [TextAPI] Apply NFC code fixups (#88639)

* Remove unnecessary cast
* Annotate `#endif`
---
 llvm/include/llvm/TextAPI/Utils.h             | 2 +-
 llvm/lib/TextAPI/BinaryReader/DylibReader.cpp | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/TextAPI/Utils.h b/llvm/include/llvm/TextAPI/Utils.h
index 319e0abce25b..ebfe88984f80 100644
--- a/llvm/include/llvm/TextAPI/Utils.h
+++ b/llvm/include/llvm/TextAPI/Utils.h
@@ -21,7 +21,7 @@
 
 #if !defined(PATH_MAX)
 #define PATH_MAX 1024
-#endif
+#endif // !defined(PATH_MAX)
 
 #define MACCATALYST_PREFIX_PATH "/System/iOSSupport"
 #define DRIVERKIT_PREFIX_PATH "/System/DriverKit"
diff --git a/llvm/lib/TextAPI/BinaryReader/DylibReader.cpp b/llvm/lib/TextAPI/BinaryReader/DylibReader.cpp
index f92a2d19a63f..ec0271f774ec 100644
--- a/llvm/lib/TextAPI/BinaryReader/DylibReader.cpp
+++ b/llvm/lib/TextAPI/BinaryReader/DylibReader.cpp
@@ -484,7 +484,7 @@ accumulateLocs(MachOObjectFile &Obj,
     auto Sym = parseSymbol(Name);
 
     if (!File.empty() && Line != 0)
-      LocMap.insert({Sym.Name.str(), RecordLoc(File, Line)});
+      LocMap.insert({Sym.Name, RecordLoc(File, Line)});
   }
 
   return LocMap;
-- 
GitLab


From 48e955496bee035121ebfe5a837ebcbc2ba8846c Mon Sep 17 00:00:00 2001
From: Marc Auberer <marc.auberer@chillibits.com>
Date: Sun, 14 Apr 2024 21:37:39 +0200
Subject: [PATCH 176/464] [Docs] Update python dependencies for documentation
 (#87877)

This has no particular reason, but imo we should update dependencies
every now and then to reduce the gap if breaking changes happen
somewhere.
---
 llvm/docs/requirements-hashed.txt | 24 ++++++++++++------------
 llvm/docs/requirements.txt        |  8 ++++----
 2 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/llvm/docs/requirements-hashed.txt b/llvm/docs/requirements-hashed.txt
index 19d1cfdfc572..64cfc8fc7979 100644
--- a/llvm/docs/requirements-hashed.txt
+++ b/llvm/docs/requirements-hashed.txt
@@ -124,9 +124,9 @@ docutils==0.20.1 \
     #   myst-parser
     #   recommonmark
     #   sphinx
-furo==2023.8.19 \
-    --hash=sha256:12f99f87a1873b6746228cfde18f77244e6c1ffb85d7fed95e638aae70d80590 \
-    --hash=sha256:e671ee638ab3f1b472f4033b0167f502ab407830e0db0f843b1c1028119c9cd1
+furo==2024.1.29 \
+    --hash=sha256:3548be2cef45a32f8cdc0272d415fcb3e5fa6a0eb4ddfe21df3ecf1fe45a13cf \
+    --hash=sha256:4d6b2fe3f10a6e36eb9cc24c1e7beb38d7a23fc7b3c382867503b7fcac8a1e02
     # via -r requirements.txt
 idna==3.6 \
     --hash=sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca \
@@ -304,9 +304,9 @@ soupsieve==2.5 \
     --hash=sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690 \
     --hash=sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7
     # via beautifulsoup4
-sphinx==7.1.2 \
-    --hash=sha256:780f4d32f1d7d1126576e0e5ecc19dc32ab76cd24e950228dcf7b1f6d3d9e22f \
-    --hash=sha256:d170a81825b2fcacb6dfd5a0d7f578a053e45d3f2b153fecc948c37344eb4cbe
+sphinx==7.2.6 \
+    --hash=sha256:1e09160a40b956dc623c910118fa636da93bd3ca0b9876a7b3df90f07d691560 \
+    --hash=sha256:9a5160e1ea90688d5963ba09a2dcd8bdd526620edbb65c328728f1b2228d5ab5
     # via
     #   -r requirements.txt
     #   furo
@@ -318,9 +318,9 @@ sphinx==7.1.2 \
     #   sphinxcontrib-htmlhelp
     #   sphinxcontrib-qthelp
     #   sphinxcontrib-serializinghtml
-sphinx-automodapi==0.16.0 \
-    --hash=sha256:68fc47064804604b90aa27c047016e86aaf970981d90a0082d5b5dd2e9d38afd \
-    --hash=sha256:6c673ef93066408e5ad3e2fa3533044d432a47fe6a826212b9ebf5f52a872554
+sphinx-automodapi==0.17.0 \
+    --hash=sha256:4d029cb79eef29413e94ab01bb0177ebd2d5ba86e9789b73575afe9c06ae1501 \
+    --hash=sha256:7ccdadad57add4aa9149d9f2bb5cf28c8f8b590280b4735b1156ea8355c423a1
     # via -r requirements.txt
 sphinx-basic-ng==1.0.0b2 \
     --hash=sha256:9ec55a47c90c8c002b5960c57492ec3021f5193cb26cebc2dc4ea226848651c9 \
@@ -334,9 +334,9 @@ sphinx-markdown-tables==0.0.17 \
     --hash=sha256:2bd0c30779653e4dd120300cbd9ca412c480738cc2241f6dea477a883f299e04 \
     --hash=sha256:6bc6d3d400eaccfeebd288446bc08dd83083367c58b85d40fe6c12d77ef592f1
     # via -r requirements.txt
-sphinxcontrib-applehelp==1.0.4 \
-    --hash=sha256:29d341f67fb0f6f586b23ad80e072c8e6ad0b48417db2bde114a4c9746feb228 \
-    --hash=sha256:828f867945bbe39817c210a1abfd1bc4895c8b73fcaade56d45357a348a07d7e
+sphinxcontrib-applehelp==1.0.8 \
+    --hash=sha256:c40a4f96f3776c4393d933412053962fac2b84f4c99a7982ba42e09576a70619 \
+    --hash=sha256:cb61eb0ec1b61f349e5cc36b2028e9e7ca765be05e49641c97241274753067b4
     # via
     #   -r requirements.txt
     #   sphinx
diff --git a/llvm/docs/requirements.txt b/llvm/docs/requirements.txt
index d66569c1b3e7..960a6eb644e1 100644
--- a/llvm/docs/requirements.txt
+++ b/llvm/docs/requirements.txt
@@ -1,9 +1,9 @@
-sphinx==7.1.2
+sphinx==7.2.6
 docutils==0.20.1
 sphinx-markdown-tables==0.0.17
 recommonmark==0.7.1
-sphinx-automodapi==0.16.0
+sphinx-automodapi==0.17.0
 sphinx-bootstrap-theme==0.8.1
-sphinxcontrib-applehelp==1.0.4
-furo==2023.8.19
+sphinxcontrib-applehelp==1.0.8
+furo==2024.1.29
 myst-parser==2.0.0
-- 
GitLab


From 84ec2d3b38635dead7690f97f215ff4c473184f0 Mon Sep 17 00:00:00 2001
From: Maksim Levental <maksim.levental@gmail.com>
Date: Sun, 14 Apr 2024 17:20:05 -0500
Subject: [PATCH 177/464] [mlir][python] add to CODEOWNERS (#88561)

---
 .github/CODEOWNERS | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index 76b8266cae87..c4727a0c267d 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -115,6 +115,10 @@ clang/test/AST/Interp/ @tbaederr
 /mlir/**/LLVMIR/**/BasicPtxBuilderInterface* @grypp
 /mlir/**/NVVM*/ @grypp
 
+# MLIR Python Bindings
+/mlir/test/python/ @makslevental @stellaraccident
+/mlir/python/ @makslevental @stellaraccident
+
 # BOLT
 /bolt/ @aaupov @maksfb @rafaelauler @ayermolo @dcci
 
-- 
GitLab


From a5f54175dcf120180c3d91bbc13062bbf8f42f61 Mon Sep 17 00:00:00 2001
From: Zentrik <Zentrik@users.noreply.github.com>
Date: Sun, 14 Apr 2024 23:21:06 +0100
Subject: [PATCH 178/464] Include cmath to fix build error on mac os 10.14
 (#88665)

This fixes #88664.
---
 clang-tools-extra/clang-tidy/modernize/UseStdNumbersCheck.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/clang-tools-extra/clang-tidy/modernize/UseStdNumbersCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseStdNumbersCheck.cpp
index b299afd540b9..1548fc454cfb 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseStdNumbersCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseStdNumbersCheck.cpp
@@ -29,6 +29,7 @@
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/MathExtras.h"
 #include <array>
+#include <cmath>
 #include <cstdint>
 #include <cstdlib>
 #include <initializer_list>
-- 
GitLab


From 9f43a41db37253685c5ed428c215528eb92bbd43 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sun, 14 Apr 2024 17:04:03 -0700
Subject: [PATCH 179/464] [Driver,test] Make Android runtime tests resisent to
 #87866 change

Suggested by YunQiang Su <syq@debian.org>
---
 clang/test/Driver/linux-ld.c     | 2 ++
 clang/test/Driver/sanitizer-ld.c | 6 ++++++
 2 files changed, 8 insertions(+)

diff --git a/clang/test/Driver/linux-ld.c b/clang/test/Driver/linux-ld.c
index 4020b138dc8f..f4e40439a5f3 100644
--- a/clang/test/Driver/linux-ld.c
+++ b/clang/test/Driver/linux-ld.c
@@ -92,6 +92,7 @@
 // RUN: %clang -### %s -no-pie 2>&1 \
 // RUN:     --target=arm-linux-androideabi \
 // RUN:     --sysroot=%S/Inputs/basic_android_tree/sysroot \
+// RUN:     -resource-dir=%S/Inputs/resource_dir \
 // RUN:     --rtlib=compiler-rt \
 // RUN:   | FileCheck --check-prefix=CHECK-LD-RT-ANDROID %s
 // CHECK-LD-RT-ANDROID-NOT: warning:
@@ -262,6 +263,7 @@
 // RUN: %clang -static -### %s -no-pie 2>&1 \
 // RUN:     --target=aarch64-linux-android -rtlib=platform --unwindlib=platform \
 // RUN:     --sysroot=%S/Inputs/basic_linux_tree \
+// RUN:     -resource-dir=%S/Inputs/resource_dir \
 // RUN:   | FileCheck --check-prefix=CHECK-CLANG-ANDROID-STATIC %s
 // CHECK-CLANG-ANDROID-STATIC: "{{.*}}ld{{(.exe)?}}" "--sysroot=[[SYSROOT:[^"]+]]"
 // CHECK-CLANG-ANDROID-STATIC: "--start-group" "{{[^"]*}}{{/|\\\\}}libclang_rt.builtins-aarch64-android.a" "-l:libunwind.a" "-lc" "--end-group"
diff --git a/clang/test/Driver/sanitizer-ld.c b/clang/test/Driver/sanitizer-ld.c
index 53e536d77292..f4ac484d6e4c 100644
--- a/clang/test/Driver/sanitizer-ld.c
+++ b/clang/test/Driver/sanitizer-ld.c
@@ -177,6 +177,7 @@
 // RUN: %clang -### %s 2>&1 \
 // RUN:     --target=arm-linux-androideabi -fuse-ld=ld -fsanitize=address \
 // RUN:     --sysroot=%S/Inputs/basic_android_tree/sysroot \
+// RUN:     -resource-dir=%S/Inputs/resource_dir \
 // RUN:   | FileCheck --check-prefix=CHECK-ASAN-ANDROID %s
 //
 // CHECK-ASAN-ANDROID: "{{(.*[^.0-9A-Z_a-z])?}}ld.lld{{(.exe)?}}"
@@ -191,6 +192,7 @@
 // RUN: %clang -### %s 2>&1 \
 // RUN:     --target=arm-linux-androideabi -fuse-ld=ld -fsanitize=address \
 // RUN:     --sysroot=%S/Inputs/basic_android_tree/sysroot \
+// RUN:     -resource-dir=%S/Inputs/resource_dir \
 // RUN:     -static-libsan \
 // RUN:   | FileCheck --check-prefix=CHECK-ASAN-ANDROID-STATICLIBASAN %s
 //
@@ -203,6 +205,7 @@
 // RUN: %clang -### %s 2>&1 \
 // RUN:     --target=arm-linux-androideabi -fuse-ld=ld -fsanitize=undefined \
 // RUN:     --sysroot=%S/Inputs/basic_android_tree/sysroot \
+// RUN:     -resource-dir=%S/Inputs/resource_dir \
 // RUN:   | FileCheck --check-prefix=CHECK-UBSAN-ANDROID %s
 //
 // CHECK-UBSAN-ANDROID: "{{(.*[^.0-9A-Z_a-z])?}}ld.lld{{(.exe)?}}"
@@ -217,6 +220,7 @@
 // RUN: %clang -### %s 2>&1 \
 // RUN:     --target=arm-linux-androideabi -fuse-ld=ld -fsanitize=undefined \
 // RUN:     --sysroot=%S/Inputs/basic_android_tree/sysroot \
+// RUN:     -resource-dir=%S/Inputs/resource_dir \
 // RUN:     -static-libsan \
 // RUN:   | FileCheck --check-prefix=CHECK-UBSAN-ANDROID-STATICLIBASAN %s
 //
@@ -230,6 +234,7 @@
 // RUN: %clang -### %s 2>&1 \
 // RUN:     --target=i686-linux-android -fuse-ld=ld -fsanitize=address \
 // RUN:     --sysroot=%S/Inputs/basic_android_tree/sysroot \
+// RUN:     -resource-dir=%S/Inputs/resource_dir \
 // RUN:   | FileCheck --check-prefix=CHECK-ASAN-ANDROID-X86 %s
 //
 // CHECK-ASAN-ANDROID-X86: "{{(.*[^.0-9A-Z_a-z])?}}ld.lld{{(.exe)?}}"
@@ -252,6 +257,7 @@
 // RUN: %clang -### %s 2>&1 \
 // RUN:     --target=arm-linux-androideabi -fuse-ld=ld -fsanitize=address \
 // RUN:     --sysroot=%S/Inputs/basic_android_tree/sysroot \
+// RUN:     -resource-dir=%S/Inputs/resource_dir \
 // RUN:     -shared \
 // RUN:   | FileCheck --check-prefix=CHECK-ASAN-ANDROID-SHARED %s
 //
-- 
GitLab


From 1693009679313282afbed38778dd3fad62641e1b Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Sun, 14 Apr 2024 17:05:55 -0700
Subject: [PATCH 180/464] [Driver] Ensure ToolChain::LibraryPaths is not empty
 for non-Darwin

Follow-up to #81037.

ToolChain::LibraryPaths holds the new compiler-rt library directory
(e.g. `/tmp/Debug/lib/clang/19/lib/x86_64-unknown-linux-gnu`). However,
it might be empty when the directory does not exist (due to the `if
(getVFS().exists(P))` change in https://reviews.llvm.org/D158475).

If neither the old/new compiler-rt library directories exists, we would
suggest the undesired old compiler-rt file name:

```
% /tmp/Debug/bin/clang++ a.cc -fsanitize=memory -o a
ld.lld: error: cannot open /tmp/Debug/lib/clang/19/lib/linux/libclang_rt.msan-x86_64.a: No such file or directory
clang++: error: linker command failed with exit code 1 (use -v to see invocation)
```

With this change, we will correctly suggest the new compiler-rt file name.

Fix #87150

Pull Request: https://github.com/llvm/llvm-project/pull/87866
---
 clang/lib/Driver/ToolChain.cpp                |  8 ++++-
 clang/test/Driver/arm-compiler-rt.c           | 14 ++++-----
 clang/test/Driver/cl-link.c                   | 16 +++++-----
 clang/test/Driver/compiler-rt-unwind.c        |  6 ++--
 clang/test/Driver/coverage-ld.c               |  8 ++---
 clang/test/Driver/instrprof-ld.c              | 16 +++++-----
 clang/test/Driver/linux-ld.c                  |  6 ++--
 clang/test/Driver/mingw-sanitizers.c          | 16 +++++-----
 clang/test/Driver/msp430-toolchain.c          |  4 +--
 .../Driver/print-libgcc-file-name-clangrt.c   | 12 ++++----
 clang/test/Driver/print-runtime-dir.c         |  6 ----
 clang/test/Driver/riscv32-toolchain-extra.c   |  6 ++--
 clang/test/Driver/riscv32-toolchain.c         |  6 ++--
 clang/test/Driver/riscv64-toolchain-extra.c   |  6 ++--
 clang/test/Driver/riscv64-toolchain.c         |  6 ++--
 clang/test/Driver/sanitizer-ld.c              | 30 +++++++++----------
 clang/test/Driver/wasm-toolchain.c            | 18 +++++------
 clang/test/Driver/wasm-toolchain.cpp          | 16 +++++-----
 clang/test/Driver/windows-cross.c             | 18 +++++------
 clang/test/Driver/zos-ld.c                    | 12 ++++----
 .../test/Driver/msvc-dependent-lib-flags.f90  |  8 ++---
 21 files changed, 119 insertions(+), 119 deletions(-)

diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index 03450fc0f57b..237092ed07e5 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -796,7 +796,13 @@ ToolChain::getTargetSubDirPath(StringRef BaseDir) const {
 std::optional<std::string> ToolChain::getRuntimePath() const {
   SmallString<128> P(D.ResourceDir);
   llvm::sys::path::append(P, "lib");
-  return getTargetSubDirPath(P);
+  if (auto Ret = getTargetSubDirPath(P))
+    return Ret;
+  // Darwin does not use per-target runtime directory.
+  if (Triple.isOSDarwin())
+    return {};
+  llvm::sys::path::append(P, Triple.str());
+  return std::string(P);
 }
 
 std::optional<std::string> ToolChain::getStdlibPath() const {
diff --git a/clang/test/Driver/arm-compiler-rt.c b/clang/test/Driver/arm-compiler-rt.c
index 5e9e528400d0..cb6c29f48a78 100644
--- a/clang/test/Driver/arm-compiler-rt.c
+++ b/clang/test/Driver/arm-compiler-rt.c
@@ -10,47 +10,47 @@
 // RUN:     -resource-dir=%S/Inputs/resource_dir_with_arch_subdir \
 // RUN:     -rtlib=compiler-rt -### %s 2>&1 \
 // RUN:   | FileCheck %s -check-prefix ARM-GNUEABI
-// ARM-GNUEABI: "{{.*[/\\]}}libclang_rt.builtins-arm.a"
+// ARM-GNUEABI: "{{.*[/\\]}}libclang_rt.builtins.a"
 
 // RUN: %clang -target arm-linux-gnueabi \
 // RUN:     --sysroot=%S/Inputs/resource_dir_with_arch_subdir \
 // RUN:     -resource-dir=%S/Inputs/resource_dir_with_arch_subdir \
 // RUN:     -rtlib=compiler-rt -mfloat-abi=hard -### %s 2>&1 \
 // RUN:   | FileCheck %s -check-prefix ARM-GNUEABI-ABI
-// ARM-GNUEABI-ABI: "{{.*[/\\]}}libclang_rt.builtins-armhf.a"
+// ARM-GNUEABI-ABI: "{{.*[/\\]}}libclang_rt.builtins.a"
 
 // RUN: %clang -target arm-linux-gnueabihf \
 // RUN:     --sysroot=%S/Inputs/resource_dir_with_arch_subdir \
 // RUN:     -resource-dir=%S/Inputs/resource_dir_with_arch_subdir \
 // RUN:     -rtlib=compiler-rt -### %s 2>&1 \
 // RUN:   | FileCheck %s -check-prefix ARM-GNUEABIHF
-// ARM-GNUEABIHF: "{{.*[/\\]}}libclang_rt.builtins-armhf.a"
+// ARM-GNUEABIHF: "{{.*[/\\]}}libclang_rt.builtins.a"
 
 // RUN: %clang -target arm-linux-gnueabihf \
 // RUN:     --sysroot=%S/Inputs/resource_dir_with_arch_subdir \
 // RUN:     -resource-dir=%S/Inputs/resource_dir_with_arch_subdir \
 // RUN:     -rtlib=compiler-rt -mfloat-abi=soft -### %s 2>&1 \
 // RUN:   | FileCheck %s -check-prefix ARM-GNUEABIHF-ABI
-// ARM-GNUEABIHF-ABI: "{{.*[/\\]}}libclang_rt.builtins-arm.a"
+// ARM-GNUEABIHF-ABI: "{{.*[/\\]}}libclang_rt.builtins.a"
 
 // RUN: %clang -target arm-windows-itanium \
 // RUN:     --sysroot=%S/Inputs/resource_dir_with_arch_subdir \
 // RUN:     -resource-dir=%S/Inputs/resource_dir_with_arch_subdir \
 // RUN:     -rtlib=compiler-rt -### %s 2>&1 \
 // RUN:   | FileCheck %s -check-prefix ARM-WINDOWS
-// ARM-WINDOWS: "{{.*[/\\]}}clang_rt.builtins-arm.lib"
+// ARM-WINDOWS: "{{.*[/\\]}}clang_rt.builtins.lib"
 
 // RUN: %clang -target arm-linux-androideabi \
 // RUN:     --sysroot=%S/Inputs/resource_dir_with_arch_subdir \
 // RUN:     -resource-dir=%S/Inputs/resource_dir_with_arch_subdir \
 // RUN:     -rtlib=compiler-rt -### %s 2>&1 \
 // RUN:   | FileCheck %s -check-prefix ARM-ANDROID
-// ARM-ANDROID: "{{.*[/\\]}}libclang_rt.builtins-arm-android.a"
+// ARM-ANDROID: "{{.*[/\\]}}libclang_rt.builtins.a"
 
 // RUN: not %clang --target=arm-linux-androideabi \
 // RUN:     --sysroot=%S/Inputs/resource_dir_with_arch_subdir \
 // RUN:     -resource-dir=%S/Inputs/resource_dir_with_arch_subdir \
 // RUN:     -rtlib=compiler-rt -mfloat-abi=hard -### %s 2>&1 \
 // RUN:   | FileCheck %s -check-prefix ARM-ANDROIDHF
-// ARM-ANDROIDHF: "{{.*[/\\]}}libclang_rt.builtins-armhf-android.a"
+// ARM-ANDROIDHF: "{{.*[/\\]}}libclang_rt.builtins.a"
 
diff --git a/clang/test/Driver/cl-link.c b/clang/test/Driver/cl-link.c
index 444f0c01b3f9..ffd0b5ac4bad 100644
--- a/clang/test/Driver/cl-link.c
+++ b/clang/test/Driver/cl-link.c
@@ -13,20 +13,20 @@
 // ASAN: link.exe
 // ASAN: "-debug"
 // ASAN: "-incremental:no"
-// ASAN: "{{[^"]*}}clang_rt.asan-i386.lib"
-// ASAN: "-wholearchive:{{.*}}clang_rt.asan-i386.lib"
-// ASAN: "{{[^"]*}}clang_rt.asan_cxx-i386.lib"
-// ASAN: "-wholearchive:{{.*}}clang_rt.asan_cxx-i386.lib"
+// ASAN: "{{[^"]*}}clang_rt.asan.lib"
+// ASAN: "-wholearchive:{{.*}}clang_rt.asan.lib"
+// ASAN: "{{[^"]*}}clang_rt.asan_cxx.lib"
+// ASAN: "-wholearchive:{{.*}}clang_rt.asan_cxx.lib"
 // ASAN: "{{.*}}cl-link{{.*}}.obj"
 
 // RUN: %clang_cl -m32 -arch:IA32 --target=i386-pc-win32 /MD /Tc%s -fuse-ld=link -### -fsanitize=address 2>&1 | FileCheck --check-prefix=ASAN-MD %s
 // ASAN-MD: link.exe
 // ASAN-MD: "-debug"
 // ASAN-MD: "-incremental:no"
-// ASAN-MD: "{{.*}}clang_rt.asan_dynamic-i386.lib"
-// ASAN-MD: "{{[^"]*}}clang_rt.asan_dynamic_runtime_thunk-i386.lib"
+// ASAN-MD: "{{.*}}clang_rt.asan_dynamic.lib"
+// ASAN-MD: "{{[^"]*}}clang_rt.asan_dynamic_runtime_thunk.lib"
 // ASAN-MD: "-include:___asan_seh_interceptor"
-// ASAN-MD: "-wholearchive:{{.*}}clang_rt.asan_dynamic_runtime_thunk-i386.lib"
+// ASAN-MD: "-wholearchive:{{.*}}clang_rt.asan_dynamic_runtime_thunk.lib"
 // ASAN-MD: "{{.*}}cl-link{{.*}}.obj"
 
 // RUN: %clang_cl /LD -fuse-ld=link -### /Tc%s 2>&1 | FileCheck --check-prefix=DLL %s
@@ -40,7 +40,7 @@
 // ASAN-DLL: "-dll"
 // ASAN-DLL: "-debug"
 // ASAN-DLL: "-incremental:no"
-// ASAN-DLL: "{{.*}}clang_rt.asan_dll_thunk-i386.lib"
+// ASAN-DLL: "{{.*}}clang_rt.asan_dll_thunk.lib"
 // ASAN-DLL: "{{.*}}cl-link{{.*}}.obj"
 
 // RUN: %clang_cl /Zi /Tc%s -fuse-ld=link -### 2>&1 | FileCheck --check-prefix=DEBUG %s
diff --git a/clang/test/Driver/compiler-rt-unwind.c b/clang/test/Driver/compiler-rt-unwind.c
index 7f4e3f22ab19..c5040d7fd900 100644
--- a/clang/test/Driver/compiler-rt-unwind.c
+++ b/clang/test/Driver/compiler-rt-unwind.c
@@ -98,14 +98,14 @@
 // RUN:     --target=x86_64-w64-mingw32 -rtlib=compiler-rt --unwindlib=libunwind \
 // RUN:     -shared-libgcc \
 // RUN:   | FileCheck --check-prefix=MINGW-RTLIB-COMPILER-RT-SHARED-UNWINDLIB-COMPILER-RT %s
-// MINGW-RTLIB-COMPILER-RT-SHARED-UNWINDLIB-COMPILER-RT: "{{.*}}libclang_rt.builtins-x86_64.a"
+// MINGW-RTLIB-COMPILER-RT-SHARED-UNWINDLIB-COMPILER-RT: "{{.*}}libclang_rt.builtins.a"
 // MINGW-RTLIB-COMPILER-RT-SHARED-UNWINDLIB-COMPILER-RT-SAME: "-l:libunwind.dll.a"
 //
 // RUN: %clang -### %s 2>&1 \
 // RUN:     --target=x86_64-w64-mingw32 -rtlib=compiler-rt --unwindlib=libunwind \
 // RUN:     -static-libgcc \
 // RUN:   | FileCheck --check-prefix=MINGW-RTLIB-COMPILER-RT-STATIC-UNWINDLIB-COMPILER-RT %s
-// MINGW-RTLIB-COMPILER-RT-STATIC-UNWINDLIB-COMPILER-RT: "{{.*}}libclang_rt.builtins-x86_64.a"
+// MINGW-RTLIB-COMPILER-RT-STATIC-UNWINDLIB-COMPILER-RT: "{{.*}}libclang_rt.builtins.a"
 // MINGW-RTLIB-COMPILER-RT-STATIC-UNWINDLIB-COMPILER-RT-SAME: "-l:libunwind.a"
 //
 // RUN: %clang -### %s 2>&1 \
@@ -114,5 +114,5 @@
 // RUN: %clangxx -### %s 2>&1 \
 // RUN:     --target=x86_64-w64-mingw32 -rtlib=compiler-rt --unwindlib=libunwind \
 // RUN:   | FileCheck --check-prefix=MINGW-RTLIB-COMPILER-RT-UNWINDLIB-COMPILER-RT %s
-// MINGW-RTLIB-COMPILER-RT-UNWINDLIB-COMPILER-RT: "{{.*}}libclang_rt.builtins-x86_64.a"
+// MINGW-RTLIB-COMPILER-RT-UNWINDLIB-COMPILER-RT: "{{.*}}libclang_rt.builtins.a"
 // MINGW-RTLIB-COMPILER-RT-UNWINDLIB-COMPILER-RT-SAME: "-lunwind"
diff --git a/clang/test/Driver/coverage-ld.c b/clang/test/Driver/coverage-ld.c
index acb08eb5db59..be1d8320ab8b 100644
--- a/clang/test/Driver/coverage-ld.c
+++ b/clang/test/Driver/coverage-ld.c
@@ -33,7 +33,7 @@
 // RUN:   | FileCheck --check-prefix=CHECK-FREEBSD-X86-64 %s
 //
 // CHECK-FREEBSD-X86-64: "{{(.*[^-.0-9A-Z_a-z])?}}ld{{(.exe)?}}"
-// CHECK-FREEBSD-X86-64: "{{.*}}/Inputs/resource_dir{{/|\\\\}}lib{{/|\\\\}}freebsd{{/|\\\\}}libclang_rt.profile-x86_64.a"
+// CHECK-FREEBSD-X86-64: "{{.*}}/Inputs/resource_dir{{/|\\\\}}lib{{/|\\\\}}x86_64-unknown-freebsd{{/|\\\\}}libclang_rt.profile.a"
 //
 // RUN: %clang -### %s 2>&1 \
 // RUN:     --target=x86_64-unknown-netbsd --coverage -fuse-ld=ld \
@@ -42,7 +42,7 @@
 // RUN:   | FileCheck --check-prefix=CHECK-NETBSD-X86-64 %s
 
 // CHECK-NETBSD-X86-64: "{{(.*[^-.0-9A-Z_a-z])?}}ld{{(.exe)?}}"
-// CHECK-NETBSD-X86-64: "{{.*}}/Inputs/resource_dir{{/|\\\\}}lib{{/|\\\\}}netbsd{{/|\\\\}}libclang_rt.profile-x86_64.a"
+// CHECK-NETBSD-X86-64: "{{.*}}/Inputs/resource_dir{{/|\\\\}}lib{{/|\\\\}}x86_64-unknown-netbsd{{/|\\\\}}libclang_rt.profile.a"
 
 // RUN: %clang -### %s 2>&1 \
 // RUN:     --target=x86_64-unknown-openbsd --coverage -fuse-ld=ld \
@@ -51,7 +51,7 @@
 // RUN:   | FileCheck --check-prefix=CHECK-OPENBSD-X86-64 %s
 
 // CHECK-OPENBSD-X86-64: "{{(.*[^-.0-9A-Z_a-z])?}}ld{{(.exe)?}}"
-// CHECK-OPENBSD-X86-64: "{{.*}}/Inputs/resource_dir{{/|\\\\}}lib{{/|\\\\}}openbsd{{/|\\\\}}libclang_rt.profile-x86_64.a"
+// CHECK-OPENBSD-X86-64: "{{.*}}/Inputs/resource_dir{{/|\\\\}}lib{{/|\\\\}}x86_64-unknown-openbsd{{/|\\\\}}libclang_rt.profile.a"
 
 // RUN: %clang -### %s 2>&1 \
 // RUN:     --target=arm-linux-androideabi --coverage -fuse-ld=ld \
@@ -60,4 +60,4 @@
 // RUN:   | FileCheck --check-prefix=CHECK-ANDROID-ARM %s
 //
 // CHECK-ANDROID-ARM: "{{(.*[^.0-9A-Z_a-z])?}}ld.lld{{(.exe)?}}"
-// CHECK-ANDROID-ARM: "{{.*}}/Inputs/resource_dir{{/|\\\\}}lib{{/|\\\\}}linux{{/|\\\\}}libclang_rt.profile-arm-android.a"
+// CHECK-ANDROID-ARM: "{{.*}}/Inputs/resource_dir{{/|\\\\}}lib{{/|\\\\}}arm-unknown-linux-android{{/|\\\\}}libclang_rt.profile.a"
diff --git a/clang/test/Driver/instrprof-ld.c b/clang/test/Driver/instrprof-ld.c
index 674580b349d4..a96bba4a1e76 100644
--- a/clang/test/Driver/instrprof-ld.c
+++ b/clang/test/Driver/instrprof-ld.c
@@ -34,7 +34,7 @@
 // RUN:   | FileCheck --check-prefix=CHECK-FREEBSD-X86-64 %s
 //
 // CHECK-FREEBSD-X86-64: "{{(.*[^-.0-9A-Z_a-z])?}}ld{{(.exe)?}}"
-// CHECK-FREEBSD-X86-64: "{{.*}}/Inputs/resource_dir{{/|\\\\}}lib{{/|\\\\}}freebsd{{/|\\\\}}libclang_rt.profile-x86_64.a"
+// CHECK-FREEBSD-X86-64: "{{.*}}/Inputs/resource_dir{{/|\\\\}}lib{{/|\\\\}}x86_64-unknown-freebsd{{/|\\\\}}libclang_rt.profile.a"
 //
 // RUN: %clang -### %s 2>&1 \
 // RUN:     --target=x86_64-unknown-netbsd -fprofile-instr-generate -fuse-ld=ld \
@@ -43,7 +43,7 @@
 // RUN:   | FileCheck --check-prefix=CHECK-NETBSD-X86-64 %s
 
 // CHECK-NETBSD-X86-64: "{{(.*[^-.0-9A-Z_a-z])?}}ld{{(.exe)?}}"
-// CHECK-NETBSD-X86-64: "{{.*}}/Inputs/resource_dir{{/|\\\\}}lib{{/|\\\\}}netbsd{{/|\\\\}}libclang_rt.profile-x86_64.a"
+// CHECK-NETBSD-X86-64: "{{.*}}/Inputs/resource_dir{{/|\\\\}}lib{{/|\\\\}}x86_64-unknown-netbsd{{/|\\\\}}libclang_rt.profile.a"
 
 // RUN: %clang -### %s 2>&1 \
 // RUN:     --target=x86_64-unknown-openbsd -fprofile-instr-generate -fuse-ld=ld \
@@ -52,7 +52,7 @@
 // RUN:   | FileCheck --check-prefix=CHECK-OPENBSD-X86-64 %s
 
 // CHECK-OPENBSD-X86-64: "{{(.*[^-.0-9A-Z_a-z])?}}ld{{(.exe)?}}"
-// CHECK-OPENBSD-X86-64: "{{.*}}/Inputs/resource_dir{{/|\\\\}}lib{{/|\\\\}}openbsd{{/|\\\\}}libclang_rt.profile-x86_64.a"
+// CHECK-OPENBSD-X86-64: "{{.*}}/Inputs/resource_dir{{/|\\\\}}lib{{/|\\\\}}x86_64-unknown-openbsd{{/|\\\\}}libclang_rt.profile.a"
 
 // RUN: %clang -### %s 2>&1 \
 // RUN:     -shared \
@@ -72,7 +72,7 @@
 // RUN:   | FileCheck --check-prefix=CHECK-LINUX-X86-64-SHARED %s
 //
 // CHECK-LINUX-X86-64-SHARED: "{{(.*[^-.0-9A-Z_a-z])?}}ld{{(.exe)?}}"
-// CHECK-LINUX-X86-64-SHARED: "{{.*}}/Inputs/resource_dir{{/|\\\\}}lib{{.*}}linux{{.*}}libclang_rt.profile.a" {{.*}} "-lc"
+// CHECK-LINUX-X86-64-SHARED: "{{.*}}/Inputs/resource_dir{{/|\\\\}}lib{{.*}}x86_64-unknown-linux{{.*}}libclang_rt.profile.a" {{.*}} "-lc"
 //
 // RUN: %clang -### %s 2>&1 \
 // RUN:     -shared \
@@ -82,7 +82,7 @@
 // RUN:   | FileCheck --check-prefix=CHECK-FREEBSD-X86-64-SHARED %s
 //
 // CHECK-FREEBSD-X86-64-SHARED: "{{(.*[^-.0-9A-Z_a-z])?}}ld{{(.exe)?}}"
-// CHECK-FREEBSD-X86-64-SHARED: "{{.*}}/Inputs/resource_dir{{/|\\\\}}lib{{/|\\\\}}freebsd{{/|\\\\}}libclang_rt.profile-x86_64.a"
+// CHECK-FREEBSD-X86-64-SHARED: "{{.*}}/Inputs/resource_dir{{/|\\\\}}lib{{/|\\\\}}x86_64-unknown-freebsd{{/|\\\\}}libclang_rt.profile.a"
 //
 // RUN: %clang -### %s 2>&1 \
 // RUN:     -shared \
@@ -92,7 +92,7 @@
 // RUN:   | FileCheck --check-prefix=CHECK-NETBSD-X86-64-SHARED %s
 
 // CHECK-NETBSD-X86-64-SHARED: "{{(.*[^-.0-9A-Z_a-z])?}}ld{{(.exe)?}}"
-// CHECK-NETBSD-X86-64-SHARED: "{{.*}}/Inputs/resource_dir{{/|\\\\}}lib{{/|\\\\}}netbsd{{/|\\\\}}libclang_rt.profile-x86_64.a"
+// CHECK-NETBSD-X86-64-SHARED: "{{.*}}/Inputs/resource_dir{{/|\\\\}}lib{{/|\\\\}}x86_64-unknown-netbsd{{/|\\\\}}libclang_rt.profile.a"
 
 // RUN: %clang -### %s 2>&1 \
 // RUN:     -shared \
@@ -102,7 +102,7 @@
 // RUN:   | FileCheck --check-prefix=CHECK-OPENBSD-X86-64-SHARED %s
 
 // CHECK-OPENBSD-X86-64-SHARED: "{{(.*[^-.0-9A-Z_a-z])?}}ld{{(.exe)?}}"
-// CHECK-OPENBSD-X86-64-SHARED: "{{.*}}/Inputs/resource_dir{{/|\\\\}}lib{{/|\\\\}}openbsd{{/|\\\\}}libclang_rt.profile-x86_64.a"
+// CHECK-OPENBSD-X86-64-SHARED: "{{.*}}/Inputs/resource_dir{{/|\\\\}}lib{{/|\\\\}}x86_64-unknown-openbsd{{/|\\\\}}libclang_rt.profile.a"
 
 // RUN: %clang -### %s 2>&1 \
 // RUN:     --target=x86_64-apple-darwin14 -fprofile-instr-generate -fuse-ld=ld \
@@ -174,7 +174,7 @@
 // RUN:   | FileCheck --check-prefix=CHECK-MINGW-X86-64 %s
 //
 // CHECK-MINGW-X86-64: "{{(.*[^.0-9A-Z_a-z])?}}ld{{(.exe)?}}"
-// CHECK-MINGW-X86-64: "{{.*}}/Inputs/resource_dir{{/|\\\\}}lib{{/|\\\\}}windows{{/|\\\\}}libclang_rt.profile-x86_64.a"
+// CHECK-MINGW-X86-64: "{{.*}}/Inputs/resource_dir{{/|\\\\}}lib{{/|\\\\}}x86_64-unknown-windows-gnu{{/|\\\\}}libclang_rt.profile.a"
 
 // Test instrumented profiling dependent-lib flags
 //
diff --git a/clang/test/Driver/linux-ld.c b/clang/test/Driver/linux-ld.c
index f4e40439a5f3..d918f4f2d7db 100644
--- a/clang/test/Driver/linux-ld.c
+++ b/clang/test/Driver/linux-ld.c
@@ -100,9 +100,9 @@
 // CHECK-LD-RT-ANDROID: "--eh-frame-hdr"
 // CHECK-LD-RT-ANDROID: "-m" "armelf_linux_eabi"
 // CHECK-LD-RT-ANDROID: "-dynamic-linker"
-// CHECK-LD-RT-ANDROID: libclang_rt.builtins-arm-android.a"
+// CHECK-LD-RT-ANDROID: libclang_rt.builtins.a"
 // CHECK-LD-RT-ANDROID: "-lc"
-// CHECK-LD-RT-ANDROID: libclang_rt.builtins-arm-android.a"
+// CHECK-LD-RT-ANDROID: libclang_rt.builtins.a"
 //
 // RUN: %clang -### %s -no-pie 2>&1 \
 // RUN:     --target=x86_64-unknown-linux -rtlib=platform --unwindlib=platform \
@@ -266,7 +266,7 @@
 // RUN:     -resource-dir=%S/Inputs/resource_dir \
 // RUN:   | FileCheck --check-prefix=CHECK-CLANG-ANDROID-STATIC %s
 // CHECK-CLANG-ANDROID-STATIC: "{{.*}}ld{{(.exe)?}}" "--sysroot=[[SYSROOT:[^"]+]]"
-// CHECK-CLANG-ANDROID-STATIC: "--start-group" "{{[^"]*}}{{/|\\\\}}libclang_rt.builtins-aarch64-android.a" "-l:libunwind.a" "-lc" "--end-group"
+// CHECK-CLANG-ANDROID-STATIC: "--start-group" "{{[^"]*}}{{/|\\\\}}libclang_rt.builtins.a" "-l:libunwind.a" "-lc" "--end-group"
 //
 // RUN: %clang -### %s 2>&1      \
 // RUN:     --target=x86_64-unknown-linux -rtlib=platform --unwindlib=platform \
diff --git a/clang/test/Driver/mingw-sanitizers.c b/clang/test/Driver/mingw-sanitizers.c
index d165648a8fdf..2325f8f0f1f2 100644
--- a/clang/test/Driver/mingw-sanitizers.c
+++ b/clang/test/Driver/mingw-sanitizers.c
@@ -4,17 +4,17 @@
 //
 // ASAN-ALL-NOT:"-l{{[^"]+"]}}"
 // ASAN-ALL-NOT:"[[INPUT]]"
-// ASAN-I686:   "{{[^"]*}}libclang_rt.asan_dynamic-i386.dll.a"
-// ASAN-X86_64: "{{[^"]*}}libclang_rt.asan_dynamic-x86_64.dll.a"
+// ASAN-I686:   "{{[^"]*}}libclang_rt.asan_dynamic.dll.a"
+// ASAN-X86_64: "{{[^"]*}}libclang_rt.asan_dynamic.dll.a"
 // ASAN-ALL:    "-lcomponent"
 // ASAN-ALL:    "[[INPUT]]"
-// ASAN-I686:   "{{[^"]*}}libclang_rt.asan_dynamic-i386.dll.a"
-// ASAN-I686:   "{{[^"]*}}libclang_rt.asan_dynamic_runtime_thunk-i386.a"
+// ASAN-I686:   "{{[^"]*}}libclang_rt.asan_dynamic.dll.a"
+// ASAN-I686:   "{{[^"]*}}libclang_rt.asan_dynamic_runtime_thunk.a"
 // ASAN-I686:   "--require-defined" "___asan_seh_interceptor"
-// ASAN-I686:   "--whole-archive" "{{[^"]*}}libclang_rt.asan_dynamic_runtime_thunk-i386.a" "--no-whole-archive"
-// ASAN-X86_64: "{{[^"]*}}libclang_rt.asan_dynamic-x86_64.dll.a"
-// ASAN-X86_64: "{{[^"]*}}libclang_rt.asan_dynamic_runtime_thunk-x86_64.a"
+// ASAN-I686:   "--whole-archive" "{{[^"]*}}libclang_rt.asan_dynamic_runtime_thunk.a" "--no-whole-archive"
+// ASAN-X86_64: "{{[^"]*}}libclang_rt.asan_dynamic.dll.a"
+// ASAN-X86_64: "{{[^"]*}}libclang_rt.asan_dynamic_runtime_thunk.a"
 // ASAN-X86_64: "--require-defined" "__asan_seh_interceptor"
-// ASAN-X86_64: "--whole-archive" "{{[^"]*}}libclang_rt.asan_dynamic_runtime_thunk-x86_64.a" "--no-whole-archive"
+// ASAN-X86_64: "--whole-archive" "{{[^"]*}}libclang_rt.asan_dynamic_runtime_thunk.a" "--no-whole-archive"
 
 // RUN: %clang -target x86_64-windows-gnu %s -### -fsanitize=vptr
diff --git a/clang/test/Driver/msp430-toolchain.c b/clang/test/Driver/msp430-toolchain.c
index ef6780c38f2e..3c3042b482ef 100644
--- a/clang/test/Driver/msp430-toolchain.c
+++ b/clang/test/Driver/msp430-toolchain.c
@@ -103,8 +103,8 @@
 // LIBS-COMPILER-RT-POS: "{{.*}}/Inputs/basic_msp430_tree/lib/gcc/msp430-elf/8.3.1/430{{/|\\\\}}crtbegin_no_eh.o"
 // LIBS-COMPILER-RT-POS: "-L{{.*}}/Inputs/basic_msp430_tree/lib/gcc/msp430-elf/8.3.1/430"
 // LIBS-COMPILER-RT-POS: "-L{{.*}}/Inputs/basic_msp430_tree{{/|\\\\}}msp430-elf{{/|\\\\}}lib/430"
-// LIBS-COMPILER-RT-POS: "{{[^"]*}}libclang_rt.builtins-msp430.a" "--start-group" "-lmul_none" "-lc" "{{[^"]*}}libclang_rt.builtins-msp430.a" "-lcrt" "-lnosys" "--end-group" "{{[^"]*}}libclang_rt.builtins-msp430.a"
-// LIBS-COMPILER-RT-POS: "{{.*}}/Inputs/basic_msp430_tree/lib/gcc/msp430-elf/8.3.1/430{{/|\\\\}}crtend_no_eh.o" "{{[^"]*}}libclang_rt.builtins-msp430.a"
+// LIBS-COMPILER-RT-POS: "{{[^"]*}}libclang_rt.builtins.a" "--start-group" "-lmul_none" "-lc" "{{[^"]*}}libclang_rt.builtins.a" "-lcrt" "-lnosys" "--end-group" "{{[^"]*}}libclang_rt.builtins.a"
+// LIBS-COMPILER-RT-POS: "{{.*}}/Inputs/basic_msp430_tree/lib/gcc/msp430-elf/8.3.1/430{{/|\\\\}}crtend_no_eh.o" "{{[^"]*}}libclang_rt.builtins.a"
 // LIBS-COMPILER-RT-NEG-NOT: crtbegin.o
 // LIBS-COMPILER-RT-NEG-NOT: -lssp_nonshared
 // LIBS-COMPILER-RT-NEG-NOT: -lssp
diff --git a/clang/test/Driver/print-libgcc-file-name-clangrt.c b/clang/test/Driver/print-libgcc-file-name-clangrt.c
index ed740e0d2917..a902eedc8520 100644
--- a/clang/test/Driver/print-libgcc-file-name-clangrt.c
+++ b/clang/test/Driver/print-libgcc-file-name-clangrt.c
@@ -5,14 +5,14 @@
 // RUN:     --sysroot=%S/Inputs/resource_dir_with_arch_subdir \
 // RUN:     -resource-dir=%S/Inputs/resource_dir_with_arch_subdir 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-X8664 %s
-// CHECK-CLANGRT-X8664: libclang_rt.builtins-x86_64.a
+// CHECK-CLANGRT-X8664: libclang_rt.builtins.a
 
 // RUN: %clang -rtlib=compiler-rt -print-libgcc-file-name \
 // RUN:     --target=i386-pc-linux \
 // RUN:     --sysroot=%S/Inputs/resource_dir_with_arch_subdir \
 // RUN:     -resource-dir=%S/Inputs/resource_dir_with_arch_subdir 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-I386 %s
-// CHECK-CLANGRT-I386: libclang_rt.builtins-i386.a
+// CHECK-CLANGRT-I386: libclang_rt.builtins.a
 
 // Check whether alternate arch values map to the correct library.
 //
@@ -27,28 +27,28 @@
 // RUN:     --sysroot=%S/Inputs/resource_dir_with_arch_subdir \
 // RUN:     -resource-dir=%S/Inputs/resource_dir_with_arch_subdir 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-ARM %s
-// CHECK-CLANGRT-ARM: libclang_rt.builtins-arm.a
+// CHECK-CLANGRT-ARM: libclang_rt.builtins.a
 
 // RUN: %clang -rtlib=compiler-rt -print-libgcc-file-name \
 // RUN:     --target=arm-linux-androideabi \
 // RUN:     --sysroot=%S/Inputs/resource_dir_with_arch_subdir \
 // RUN:     -resource-dir=%S/Inputs/resource_dir_with_arch_subdir 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-ARM-ANDROID %s
-// CHECK-CLANGRT-ARM-ANDROID: libclang_rt.builtins-arm-android.a
+// CHECK-CLANGRT-ARM-ANDROID: libclang_rt.builtins.a
 
 // RUN: %clang -rtlib=compiler-rt -print-libgcc-file-name \
 // RUN:     --target=arm-linux-gnueabihf \
 // RUN:     --sysroot=%S/Inputs/resource_dir_with_arch_subdir \
 // RUN:     -resource-dir=%S/Inputs/resource_dir_with_arch_subdir 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-ARMHF %s
-// CHECK-CLANGRT-ARMHF: libclang_rt.builtins-armhf.a
+// CHECK-CLANGRT-ARMHF: libclang_rt.builtins.a
 
 // RUN: %clang -rtlib=compiler-rt -print-libgcc-file-name \
 // RUN:     --target=arm-linux-gnueabi -mfloat-abi=hard \
 // RUN:     --sysroot=%S/Inputs/resource_dir_with_arch_subdir \
 // RUN:     -resource-dir=%S/Inputs/resource_dir_with_arch_subdir 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-CLANGRT-ARM-ABI %s
-// CHECK-CLANGRT-ARM-ABI: libclang_rt.builtins-armhf.a
+// CHECK-CLANGRT-ARM-ABI: libclang_rt.builtins.a
 
 // RUN: %clang -rtlib=compiler-rt -print-libgcc-file-name \
 // RUN:     --target=armv7m-none-eabi \
diff --git a/clang/test/Driver/print-runtime-dir.c b/clang/test/Driver/print-runtime-dir.c
index 550ffef1aaf6..ac1ff7e634b8 100644
--- a/clang/test/Driver/print-runtime-dir.c
+++ b/clang/test/Driver/print-runtime-dir.c
@@ -1,9 +1,3 @@
-// Default directory layout
-// RUN: %clang -print-runtime-dir --target=x86_64-pc-windows-msvc \
-// RUN:   -resource-dir=%S/Inputs/resource_dir \
-// RUN:   | FileCheck --check-prefix=PRINT-RUNTIME-DIR -DFILE=%S/Inputs/resource_dir %s
-// PRINT-RUNTIME-DIR: [[FILE]]{{/|\\}}lib{{/|\\}}windows
-
 // Per-target directory layout
 // RUN: %clang -print-runtime-dir --target=x86_64-pc-windows-msvc \
 // RUN:   -resource-dir=%S/Inputs/resource_dir_with_per_target_subdir \
diff --git a/clang/test/Driver/riscv32-toolchain-extra.c b/clang/test/Driver/riscv32-toolchain-extra.c
index 2d38aa3b545f..aab6b36f3cfc 100644
--- a/clang/test/Driver/riscv32-toolchain-extra.c
+++ b/clang/test/Driver/riscv32-toolchain-extra.c
@@ -29,8 +29,8 @@
 // C-RV32-BAREMETAL-ILP32-NOGCC: "-internal-isystem" "{{.*}}/riscv32-nogcc/bin/../riscv32-unknown-elf/include"
 // C-RV32-BAREMETAL-ILP32-NOGCC: "{{.*}}/riscv32-nogcc/bin/riscv32-unknown-elf-ld"
 // C-RV32-BAREMETAL-ILP32-NOGCC: "{{.*}}/riscv32-nogcc/bin/../riscv32-unknown-elf/lib/crt0.o"
-// C-RV32-BAREMETAL-ILP32-NOGCC: "{{.*}}/riscv32-nogcc/{{.*}}/lib/clang_rt.crtbegin-riscv32.o"
+// C-RV32-BAREMETAL-ILP32-NOGCC: "{{.*}}/riscv32-nogcc/{{.*}}/riscv32-unknown-unknown-elf/clang_rt.crtbegin.o"
 // C-RV32-BAREMETAL-ILP32-NOGCC: "{{.*}}/riscv32-nogcc/bin/../riscv32-unknown-elf/lib"
 // C-RV32-BAREMETAL-ILP32-NOGCC: "--start-group" "-lc" "-lgloss" "--end-group"
-// C-RV32-BAREMETAL-ILP32-NOGCC: "{{.*}}/riscv32-nogcc/{{.*}}/lib/libclang_rt.builtins-riscv32.a"
-// C-RV32-BAREMETAL-ILP32-NOGCC: "{{.*}}/riscv32-nogcc/{{.*}}/lib/clang_rt.crtend-riscv32.o"
+// C-RV32-BAREMETAL-ILP32-NOGCC: "{{.*}}/riscv32-nogcc/{{.*}}/riscv32-unknown-unknown-elf/libclang_rt.builtins.a"
+// C-RV32-BAREMETAL-ILP32-NOGCC: "{{.*}}/riscv32-nogcc/{{.*}}/riscv32-unknown-unknown-elf/clang_rt.crtend.o"
diff --git a/clang/test/Driver/riscv32-toolchain.c b/clang/test/Driver/riscv32-toolchain.c
index bb2533cdf1bc..322a6ca2840f 100644
--- a/clang/test/Driver/riscv32-toolchain.c
+++ b/clang/test/Driver/riscv32-toolchain.c
@@ -195,9 +195,9 @@
 // RUN:   --target=riscv32-unknown-elf --rtlib=compiler-rt --unwindlib=compiler-rt 2>&1 \
 // RUN:   | FileCheck -check-prefix=C-RV32-RTLIB-COMPILERRT-ILP32 %s
 // C-RV32-RTLIB-COMPILERRT-ILP32: "{{.*}}crt0.o"
-// C-RV32-RTLIB-COMPILERRT-ILP32: "{{.*}}clang_rt.crtbegin-riscv32.o"
-// C-RV32-RTLIB-COMPILERRT-ILP32: "--start-group" "-lc" "-lgloss" "--end-group" "{{.*}}libclang_rt.builtins-riscv32.a"
-// C-RV32-RTLIB-COMPILERRT-ILP32: "{{.*}}clang_rt.crtend-riscv32.o"
+// C-RV32-RTLIB-COMPILERRT-ILP32: "{{.*}}clang_rt.crtbegin.o"
+// C-RV32-RTLIB-COMPILERRT-ILP32: "--start-group" "-lc" "-lgloss" "--end-group" "{{.*}}libclang_rt.builtins.a"
+// C-RV32-RTLIB-COMPILERRT-ILP32: "{{.*}}clang_rt.crtend.o"
 
 // RUN: %clang -### %s --target=riscv32 \
 // RUN:   --gcc-toolchain=%S/Inputs/basic_riscv32_tree --sysroot= \
diff --git a/clang/test/Driver/riscv64-toolchain-extra.c b/clang/test/Driver/riscv64-toolchain-extra.c
index a6ec9b16cc5c..d8d9b5844167 100644
--- a/clang/test/Driver/riscv64-toolchain-extra.c
+++ b/clang/test/Driver/riscv64-toolchain-extra.c
@@ -29,8 +29,8 @@
 // C-RV64-BAREMETAL-LP64-NOGCC: "-internal-isystem" "{{.*}}/riscv64-nogcc/bin/../riscv64-unknown-elf/include"
 // C-RV64-BAREMETAL-LP64-NOGCC: "{{.*}}/riscv64-nogcc/bin/riscv64-unknown-elf-ld"
 // C-RV64-BAREMETAL-LP64-NOGCC: "{{.*}}/riscv64-nogcc/bin/../riscv64-unknown-elf/lib/crt0.o"
-// C-RV64-BAREMETAL-LP64-NOGCC: "{{.*}}/riscv64-nogcc/{{.*}}/lib/clang_rt.crtbegin-riscv64.o"
+// C-RV64-BAREMETAL-LP64-NOGCC: "{{.*}}/riscv64-nogcc/{{.*}}/riscv64-unknown-unknown-elf/clang_rt.crtbegin.o"
 // C-RV64-BAREMETAL-LP64-NOGCC: "{{.*}}/riscv64-nogcc/bin/../riscv64-unknown-elf/lib"
 // C-RV64-BAREMETAL-LP64-NOGCC: "--start-group" "-lc" "-lgloss" "--end-group"
-// C-RV64-BAREMETAL-LP64-NOGCC: "{{.*}}/riscv64-nogcc/{{.*}}/lib/libclang_rt.builtins-riscv64.a"
-// C-RV64-BAREMETAL-LP64-NOGCC: "{{.*}}/riscv64-nogcc/{{.*}}/lib/clang_rt.crtend-riscv64.o"
+// C-RV64-BAREMETAL-LP64-NOGCC: "{{.*}}/riscv64-nogcc/{{.*}}/riscv64-unknown-unknown-elf/libclang_rt.builtins.a"
+// C-RV64-BAREMETAL-LP64-NOGCC: "{{.*}}/riscv64-nogcc/{{.*}}/riscv64-unknown-unknown-elf/clang_rt.crtend.o"
diff --git a/clang/test/Driver/riscv64-toolchain.c b/clang/test/Driver/riscv64-toolchain.c
index 381ee58c470c..b3216de30754 100644
--- a/clang/test/Driver/riscv64-toolchain.c
+++ b/clang/test/Driver/riscv64-toolchain.c
@@ -151,9 +151,9 @@
 // RUN:   --target=riscv64-unknown-elf --rtlib=compiler-rt --unwindlib=compiler-rt 2>&1 \
 // RUN:   | FileCheck -check-prefix=C-RV64-RTLIB-COMPILERRT-LP64 %s
 // C-RV64-RTLIB-COMPILERRT-LP64: "{{.*}}crt0.o"
-// C-RV64-RTLIB-COMPILERRT-LP64: "{{.*}}clang_rt.crtbegin-riscv64.o"
-// C-RV64-RTLIB-COMPILERRT-LP64: "--start-group" "-lc" "-lgloss" "--end-group" "{{.*}}libclang_rt.builtins-riscv64.a"
-// C-RV64-RTLIB-COMPILERRT-LP64: "{{.*}}clang_rt.crtend-riscv64.o"
+// C-RV64-RTLIB-COMPILERRT-LP64: "{{.*}}clang_rt.crtbegin.o"
+// C-RV64-RTLIB-COMPILERRT-LP64: "--start-group" "-lc" "-lgloss" "--end-group" "{{.*}}libclang_rt.builtins.a"
+// C-RV64-RTLIB-COMPILERRT-LP64: "{{.*}}clang_rt.crtend.o"
 
 // RUN: %clang -### %s --target=riscv64 \
 // RUN:   --gcc-toolchain=%S/Inputs/basic_riscv64_tree --sysroot= \
diff --git a/clang/test/Driver/sanitizer-ld.c b/clang/test/Driver/sanitizer-ld.c
index f4ac484d6e4c..f5657e47626e 100644
--- a/clang/test/Driver/sanitizer-ld.c
+++ b/clang/test/Driver/sanitizer-ld.c
@@ -111,7 +111,7 @@
 // CHECK-ASAN-FREEBSD: "{{(.*[^-.0-9A-Z_a-z])?}}ld{{(.exe)?}}"
 // CHECK-ASAN-FREEBSD-NOT: "-lc"
 // CHECK-ASAN-FREEBSD-NOT: libclang_rt.asan_cxx
-// CHECK-ASAN-FREEBSD: freebsd{{/|\\+}}libclang_rt.asan-i386.a"
+// CHECK-ASAN-FREEBSD: freebsd{{/|\\+}}libclang_rt.asan.a"
 // CHECK-ASAN-FREEBSD-NOT: libclang_rt.asan_cxx
 // CHECK-ASAN-FREEBSD-NOT: "--dynamic-list"
 // CHECK-ASAN-FREEBSD: "--export-dynamic"
@@ -135,8 +135,8 @@
 //
 // CHECK-ASAN-LINUX-CXX: "{{(.*[^-.0-9A-Z_a-z])?}}ld{{(.exe)?}}"
 // CHECK-ASAN-LINUX-CXX-NOT: "-lc"
-// CHECK-ASAN-LINUX-CXX: "--whole-archive" "{{.*}}libclang_rt.asan-i386.a" "--no-whole-archive"
-// CHECK-ASAN-LINUX-CXX: "--whole-archive" "{{.*}}libclang_rt.asan_cxx-i386.a" "--no-whole-archive"
+// CHECK-ASAN-LINUX-CXX: "--whole-archive" "{{.*}}libclang_rt.asan.a" "--no-whole-archive"
+// CHECK-ASAN-LINUX-CXX: "--whole-archive" "{{.*}}libclang_rt.asan_cxx.a" "--no-whole-archive"
 // CHECK-ASAN-LINUX-CXX-NOT: "--dynamic-list"
 // CHECK-ASAN-LINUX-CXX: "--export-dynamic"
 // CHECK-ASAN-LINUX-CXX: stdc++
@@ -163,7 +163,7 @@
 //
 // CHECK-ASAN-ARM: "{{(.*[^.0-9A-Z_a-z])?}}ld{{(.exe)?}}"
 // CHECK-ASAN-ARM-NOT: "-lc"
-// CHECK-ASAN-ARM: libclang_rt.asan-arm.a"
+// CHECK-ASAN-ARM: libclang_rt.asan.a"
 //
 // RUN: %clang -### %s 2>&1 \
 // RUN:     --target=armv7l-linux-gnueabi -fuse-ld=ld -fsanitize=address \
@@ -172,7 +172,7 @@
 //
 // CHECK-ASAN-ARMv7: "{{(.*[^.0-9A-Z_a-z])?}}ld{{(.exe)?}}"
 // CHECK-ASAN-ARMv7-NOT: "-lc"
-// CHECK-ASAN-ARMv7: libclang_rt.asan-arm.a"
+// CHECK-ASAN-ARMv7: libclang_rt.asan.a"
 
 // RUN: %clang -### %s 2>&1 \
 // RUN:     --target=arm-linux-androideabi -fuse-ld=ld -fsanitize=address \
@@ -185,7 +185,7 @@
 // CHECK-ASAN-ANDROID-NOT: "-lc"
 // CHECK-ASAN-ANDROID-NOT: "-lpthread"
 // CHECK-ASAN-ANDROID-NOT: "-lresolv"
-// CHECK-ASAN-ANDROID: libclang_rt.asan-arm-android.so"
+// CHECK-ASAN-ANDROID: libclang_rt.asan.so"
 // CHECK-ASAN-ANDROID-NOT: "-lpthread"
 // CHECK-ASAN-ANDROID-NOT: "-lresolv"
 
@@ -197,7 +197,7 @@
 // RUN:   | FileCheck --check-prefix=CHECK-ASAN-ANDROID-STATICLIBASAN %s
 //
 // CHECK-ASAN-ANDROID-STATICLIBASAN: "{{(.*[^.0-9A-Z_a-z])?}}ld.lld{{(.exe)?}}"
-// CHECK-ASAN-ANDROID-STATICLIBASAN: libclang_rt.asan-arm-android.a"
+// CHECK-ASAN-ANDROID-STATICLIBASAN: libclang_rt.asan.a"
 // CHECK-ASAN-ANDROID-STATICLIBASAN-NOT: "-lpthread"
 // CHECK-ASAN-ANDROID-STATICLIBASAN-NOT: "-lrt"
 // CHECK-ASAN-ANDROID-STATICLIBASAN-NOT: "-lresolv"
@@ -213,7 +213,7 @@
 // CHECK-UBSAN-ANDROID-NOT: "-lc"
 // CHECK-UBSAN-ANDROID-NOT: "-lpthread"
 // CHECK-UBSAN-ANDROID-NOT: "-lresolv"
-// CHECK-UBSAN-ANDROID: libclang_rt.ubsan_standalone-arm-android.so"
+// CHECK-UBSAN-ANDROID: libclang_rt.ubsan_standalone.so"
 // CHECK-UBSAN-ANDROID-NOT: "-lpthread"
 // CHECK-UBSAN-ANDROID-NOT: "-lresolv"
 
@@ -225,7 +225,7 @@
 // RUN:   | FileCheck --check-prefix=CHECK-UBSAN-ANDROID-STATICLIBASAN %s
 //
 // CHECK-UBSAN-ANDROID-STATICLIBASAN: "{{(.*[^.0-9A-Z_a-z])?}}ld.lld{{(.exe)?}}"
-// CHECK-UBSAN-ANDROID-STATICLIBASAN: libclang_rt.ubsan_standalone-arm-android.a"
+// CHECK-UBSAN-ANDROID-STATICLIBASAN: libclang_rt.ubsan_standalone.a"
 // CHECK-UBSAN-ANDROID-STATICLIBASAN-NOT: "-lpthread"
 // CHECK-UBSAN-ANDROID-STATICLIBASAN-NOT: "-lrt"
 // CHECK-UBSAN-ANDROID-STATICLIBASAN-NOT: "-lresolv"
@@ -242,7 +242,7 @@
 // CHECK-ASAN-ANDROID-X86-NOT: "-lc"
 // CHECK-ASAN-ANDROID-X86-NOT: "-lpthread"
 // CHECK-ASAN-ANDROID-X86-NOT: "-lresolv"
-// CHECK-ASAN-ANDROID-X86: libclang_rt.asan-i686-android.so"
+// CHECK-ASAN-ANDROID-X86: libclang_rt.asan.so"
 // CHECK-ASAN-ANDROID-X86-NOT: "-lpthread"
 // CHECK-ASAN-ANDROID-X86-NOT: "-lresolv"
 //
@@ -263,7 +263,7 @@
 //
 // CHECK-ASAN-ANDROID-SHARED: "{{(.*[^.0-9A-Z_a-z])?}}ld.lld{{(.exe)?}}"
 // CHECK-ASAN-ANDROID-SHARED-NOT: "-lc"
-// CHECK-ASAN-ANDROID-SHARED: libclang_rt.asan-arm-android.so"
+// CHECK-ASAN-ANDROID-SHARED: libclang_rt.asan.so"
 // CHECK-ASAN-ANDROID-SHARED-NOT: "-lpthread"
 // CHECK-ASAN-ANDROID-SHARED-NOT: "-lresolv"
 
@@ -353,7 +353,7 @@
 // CHECK-UBSAN-LINUX: "{{.*}}ld{{(.exe)?}}"
 // CHECK-UBSAN-LINUX-NOT: libclang_rt.asan
 // CHECK-UBSAN-LINUX-NOT: libclang_rt.ubsan_standalone_cxx
-// CHECK-UBSAN-LINUX: "--whole-archive" "{{.*}}libclang_rt.ubsan_standalone-x32.a" "--no-whole-archive"
+// CHECK-UBSAN-LINUX: "--whole-archive" "{{.*}}libclang_rt.ubsan_standalone.a" "--no-whole-archive"
 // CHECK-UBSAN-LINUX-NOT: libclang_rt.asan
 // CHECK-UBSAN-LINUX-NOT: libclang_rt.ubsan_standalone_cxx
 // CHECK-UBSAN-LINUX-NOT: "-lstdc++"
@@ -684,7 +684,7 @@
 // RUN:     --sysroot=%S/Inputs/basic_android_tree \
 // RUN:   | FileCheck --check-prefix=CHECK-CFI-CROSS-DSO-DIAG-ANDROID %s
 // CHECK-CFI-CROSS-DSO-DIAG-ANDROID: "{{.*}}ld{{(.exe)?}}"
-// CHECK-CFI-CROSS-DSO-DIAG-ANDROID: "{{[^"]*}}libclang_rt.ubsan_standalone-aarch64-android.so"
+// CHECK-CFI-CROSS-DSO-DIAG-ANDROID: "{{[^"]*}}libclang_rt.ubsan_standalone.so"
 // CHECK-CFI-CROSS-DSO-DIAG-ANDROID: "--export-dynamic-symbol=__cfi_check"
 
 // RUN: %clangxx -fsanitize=address -### %s 2>&1 \
@@ -935,7 +935,7 @@
 // CHECK-SCUDO-ANDROID: "-pie"
 // CHECK-SCUDO-ANDROID-NOT: "-lpthread"
 // CHECK-SCUDO-ANDROID-NOT: "-lresolv"
-// CHECK-SCUDO-ANDROID: libclang_rt.scudo_standalone-arm-android.so"
+// CHECK-SCUDO-ANDROID: libclang_rt.scudo_standalone.so"
 // CHECK-SCUDO-ANDROID-NOT: "-lpthread"
 // CHECK-SCUDO-ANDROID-NOT: "-lresolv"
 
@@ -946,7 +946,7 @@
 // RUN:   | FileCheck --check-prefix=CHECK-SCUDO-ANDROID-STATIC %s
 // CHECK-SCUDO-ANDROID-STATIC: "{{(.*[^.0-9A-Z_a-z])?}}ld.lld{{(.exe)?}}"
 // CHECK-SCUDO-ANDROID-STATIC: "-pie"
-// CHECK-SCUDO-ANDROID-STATIC: "--whole-archive" "{{.*}}libclang_rt.scudo_standalone-arm-android.a" "--no-whole-archive"
+// CHECK-SCUDO-ANDROID-STATIC: "--whole-archive" "{{.*}}libclang_rt.scudo_standalone.a" "--no-whole-archive"
 // CHECK-SCUDO-ANDROID-STATIC-NOT: "-lstdc++"
 // CHECK-SCUDO-ANDROID-STATIC-NOT: "-lpthread"
 // CHECK-SCUDO-ANDROID-STATIC-NOT: "-lrt"
diff --git a/clang/test/Driver/wasm-toolchain.c b/clang/test/Driver/wasm-toolchain.c
index 88590a3ba4c4..dabf0ac2433b 100644
--- a/clang/test/Driver/wasm-toolchain.c
+++ b/clang/test/Driver/wasm-toolchain.c
@@ -17,42 +17,42 @@
 // RUN: %clang -### --target=wasm32-unknown-unknown --sysroot=/foo %s 2>&1 \
 // RUN:   | FileCheck -check-prefix=LINK %s
 // LINK: "-cc1" {{.*}} "-o" "[[temp:[^"]*]]"
-// LINK: wasm-ld{{.*}}" "-L/foo/lib" "crt1.o" "[[temp]]" "-lc" "{{.*[/\\]}}libclang_rt.builtins-wasm32.a" "-o" "a.out"
+// LINK: wasm-ld{{.*}}" "-L/foo/lib" "crt1.o" "[[temp]]" "-lc" "{{.*[/\\]}}libclang_rt.builtins.a" "-o" "a.out"
 
 // A basic C link command-line with optimization with unknown OS.
 
 // RUN: %clang -### -O2 --target=wasm32-unknown-unknown --sysroot=/foo %s 2>&1 \
 // RUN:   | FileCheck -check-prefix=LINK_OPT %s
 // LINK_OPT: "-cc1" {{.*}} "-o" "[[temp:[^"]*]]"
-// LINK_OPT: wasm-ld{{.*}}" "-L/foo/lib" "crt1.o" "[[temp]]" "-lc" "{{.*[/\\]}}libclang_rt.builtins-wasm32.a" "-o" "a.out"
+// LINK_OPT: wasm-ld{{.*}}" "-L/foo/lib" "crt1.o" "[[temp]]" "-lc" "{{.*[/\\]}}libclang_rt.builtins.a" "-o" "a.out"
 
 // A basic C link command-line with known OS.
 
 // RUN: %clang -### --target=wasm32-wasi --sysroot=/foo %s 2>&1 \
 // RUN:   | FileCheck -check-prefix=LINK_KNOWN %s
 // LINK_KNOWN: "-cc1" {{.*}} "-o" "[[temp:[^"]*]]"
-// LINK_KNOWN: wasm-ld{{.*}}" "-L/foo/lib/wasm32-wasi" "crt1.o" "[[temp]]" "-lc" "{{.*[/\\]}}libclang_rt.builtins-wasm32.a" "-o" "a.out"
+// LINK_KNOWN: wasm-ld{{.*}}" "-L/foo/lib/wasm32-wasi" "crt1.o" "[[temp]]" "-lc" "{{.*[/\\]}}libclang_rt.builtins.a" "-o" "a.out"
 
 // -shared should be passed through to `wasm-ld` and include crt1-reactor.o with a known OS.
 
 // RUN: %clang -### -shared -mexec-model=reactor --target=wasm32-wasi --sysroot=/foo %s 2>&1 \
 // RUN:   | FileCheck -check-prefix=LINK_KNOWN_SHARED %s
 // LINK_KNOWN_SHARED: "-cc1" {{.*}} "-o" "[[temp:[^"]*]]"
-// LINK_KNOWN_SHARED: wasm-ld{{.*}}" "-L/foo/lib/wasm32-wasi" "crt1-reactor.o" "--entry" "_initialize" "-shared" "[[temp]]" "-lc" "{{.*[/\\]}}libclang_rt.builtins-wasm32.a" "-o" "a.out"
+// LINK_KNOWN_SHARED: wasm-ld{{.*}}" "-L/foo/lib/wasm32-wasi" "crt1-reactor.o" "--entry" "_initialize" "-shared" "[[temp]]" "-lc" "{{.*[/\\]}}libclang_rt.builtins.a" "-o" "a.out"
 
 // -shared should be passed through to `wasm-ld` and include crt1-reactor.o with an unknown OS.
 
 // RUN: %clang -### -shared -mexec-model=reactor --target=wasm32-unknown-unknown --sysroot=/foo %s 2>&1 \
 // RUN:   | FileCheck -check-prefix=LINK_UNKNOWN_SHARED %s
 // LINK_UNKNOWN_SHARED: "-cc1" {{.*}} "-o" "[[temp:[^"]*]]"
-// LINK_UNKNOWN_SHARED: wasm-ld{{.*}}" "crt1-reactor.o" "--entry" "_initialize" "-shared" "[[temp]]" "-lc" "{{.*[/\\]}}libclang_rt.builtins-wasm32.a" "-o" "a.out"
+// LINK_UNKNOWN_SHARED: wasm-ld{{.*}}" "crt1-reactor.o" "--entry" "_initialize" "-shared" "[[temp]]" "-lc" "{{.*[/\\]}}libclang_rt.builtins.a" "-o" "a.out"
 
 // A basic C link command-line with optimization with known OS.
 
 // RUN: %clang -### -O2 --target=wasm32-wasi --sysroot=/foo %s 2>&1 \
 // RUN:   | FileCheck -check-prefix=LINK_OPT_KNOWN %s
 // LINK_OPT_KNOWN: "-cc1" {{.*}} "-o" "[[temp:[^"]*]]"
-// LINK_OPT_KNOWN: wasm-ld{{.*}}" "-L/foo/lib/wasm32-wasi" "crt1.o" "[[temp]]" "-lc" "{{.*[/\\]}}libclang_rt.builtins-wasm32.a" "-o" "a.out"
+// LINK_OPT_KNOWN: wasm-ld{{.*}}" "-L/foo/lib/wasm32-wasi" "crt1.o" "[[temp]]" "-lc" "{{.*[/\\]}}libclang_rt.builtins.a" "-o" "a.out"
 
 // A basic C compile command-line with known OS.
 
@@ -180,12 +180,12 @@
 // RUN: %clang -### %s --target=wasm32-unknown-unknown --sysroot=%s/no-sysroot-there -mexec-model=command 2>&1 \
 // RUN:   | FileCheck -check-prefix=CHECK-COMMAND %s
 // CHECK-COMMAND: "-cc1" {{.*}} "-o" "[[temp:[^"]*]]"
-// CHECK-COMMAND: wasm-ld{{.*}}" "crt1.o" "[[temp]]" "-lc" "{{.*[/\\]}}libclang_rt.builtins-wasm32.a" "-o" "a.out"
+// CHECK-COMMAND: wasm-ld{{.*}}" "crt1.o" "[[temp]]" "-lc" "{{.*[/\\]}}libclang_rt.builtins.a" "-o" "a.out"
 
 // RUN: %clang -### %s --target=wasm32-unknown-unknown --sysroot=%s/no-sysroot-there -mexec-model=reactor 2>&1 \
 // RUN:   | FileCheck -check-prefix=CHECK-REACTOR %s
 // CHECK-REACTOR: "-cc1" {{.*}} "-o" "[[temp:[^"]*]]"
-// CHECK-REACTOR: wasm-ld{{.*}}" "crt1-reactor.o" "--entry" "_initialize" "[[temp]]" "-lc" "{{.*[/\\]}}libclang_rt.builtins-wasm32.a" "-o" "a.out"
+// CHECK-REACTOR: wasm-ld{{.*}}" "crt1-reactor.o" "--entry" "_initialize" "[[temp]]" "-lc" "{{.*[/\\]}}libclang_rt.builtins.a" "-o" "a.out"
 
 // -fPIC implies +mutable-globals
 
@@ -204,7 +204,7 @@
 // RUN: %clang -### -O2 --target=wasm32-wasip2 %s --sysroot /foo 2>&1 \
 // RUN:   | FileCheck -check-prefix=LINK_WASIP2 %s
 // LINK_WASIP2: "-cc1" {{.*}} "-o" "[[temp:[^"]*]]"
-// LINK_WASIP2: wasm-component-ld{{.*}}" "-L/foo/lib/wasm32-wasip2" "crt1.o" "[[temp]]" "-lc" "{{.*[/\\]}}libclang_rt.builtins-wasm32.a" "-o" "a.out"
+// LINK_WASIP2: wasm-component-ld{{.*}}" "-L/foo/lib/wasm32-wasip2" "crt1.o" "[[temp]]" "-lc" "{{.*[/\\]}}libclang_rt.builtins.a" "-o" "a.out"
 
 // Test that on `wasm32-wasip2` the `wasm-component-ld` programs is told where
 // to find `wasm-ld` by default.
diff --git a/clang/test/Driver/wasm-toolchain.cpp b/clang/test/Driver/wasm-toolchain.cpp
index 4af011097021..ba1c55b33edc 100644
--- a/clang/test/Driver/wasm-toolchain.cpp
+++ b/clang/test/Driver/wasm-toolchain.cpp
@@ -17,48 +17,48 @@
 // RUN: %clangxx -### --target=wasm32-unknown-unknown --sysroot=/foo --stdlib=libc++ %s 2>&1 \
 // RUN:   | FileCheck -check-prefix=LINK %s
 // LINK: "-cc1" {{.*}} "-o" "[[temp:[^"]*]]"
-// LINK: wasm-ld{{.*}}" "-L/foo/lib" "crt1.o" "[[temp]]" "-lc++" "-lc++abi" "-lc" "{{.*[/\\]}}libclang_rt.builtins-wasm32.a" "-o" "a.out"
+// LINK: wasm-ld{{.*}}" "-L/foo/lib" "crt1.o" "[[temp]]" "-lc++" "-lc++abi" "-lc" "{{.*[/\\]}}libclang_rt.builtins.a" "-o" "a.out"
 
 // RUN: %clangxx -### --target=wasm32-unknown-unknown --sysroot=/foo --stdlib=libstdc++ %s 2>&1 \
 // RUN:   | FileCheck -check-prefix=LINK_STDCXX %s
 // LINK_STDCXX: "-cc1" {{.*}} "-o" "[[temp:[^"]*]]"
-// LINK_STDCXX: wasm-ld{{.*}}" "-L/foo/lib" "crt1.o" "[[temp]]" "-lstdc++" "-lc" "{{.*[/\\]}}libclang_rt.builtins-wasm32.a" "-o" "a.out"
+// LINK_STDCXX: wasm-ld{{.*}}" "-L/foo/lib" "crt1.o" "[[temp]]" "-lstdc++" "-lc" "{{.*[/\\]}}libclang_rt.builtins.a" "-o" "a.out"
 
 // A basic C++ link command-line with optimization with unknown OS.
 
 // RUN: %clangxx -### -O2 --target=wasm32-unknown-unknown --sysroot=/foo %s --stdlib=libc++ 2>&1 \
 // RUN:   | FileCheck -check-prefix=LINK_OPT %s
 // LINK_OPT: "-cc1" {{.*}} "-o" "[[temp:[^"]*]]"
-// LINK_OPT: wasm-ld{{.*}}" "-L/foo/lib" "crt1.o" "[[temp]]" "-lc++" "-lc++abi" "-lc" "{{.*[/\\]}}libclang_rt.builtins-wasm32.a" "-o" "a.out"
+// LINK_OPT: wasm-ld{{.*}}" "-L/foo/lib" "crt1.o" "[[temp]]" "-lc++" "-lc++abi" "-lc" "{{.*[/\\]}}libclang_rt.builtins.a" "-o" "a.out"
 
 // RUN: %clangxx -### -O2 --target=wasm32-unknown-unknown --sysroot=/foo %s --stdlib=libstdc++ 2>&1 \
 // RUN:   | FileCheck -check-prefix=LINK_OPT_STDCXX %s
 // LINK_OPT_STDCXX: "-cc1" {{.*}} "-o" "[[temp:[^"]*]]"
-// LINK_OPT_STDCXX: wasm-ld{{.*}}" "-L/foo/lib" "crt1.o" "[[temp]]" "-lstdc++" "-lc" "{{.*[/\\]}}libclang_rt.builtins-wasm32.a" "-o" "a.out"
+// LINK_OPT_STDCXX: wasm-ld{{.*}}" "-L/foo/lib" "crt1.o" "[[temp]]" "-lstdc++" "-lc" "{{.*[/\\]}}libclang_rt.builtins.a" "-o" "a.out"
 
 // A basic C++ link command-line with known OS.
 
 // RUN: %clangxx -### --target=wasm32-wasi --sysroot=/foo --stdlib=libc++ %s 2>&1 \
 // RUN:   | FileCheck -check-prefix=LINK_KNOWN %s
 // LINK_KNOWN: "-cc1" {{.*}} "-o" "[[temp:[^"]*]]"
-// LINK_KNOWN: wasm-ld{{.*}}" "-L/foo/lib/wasm32-wasi" "crt1.o" "[[temp]]" "-lc++" "-lc++abi" "-lc" "{{.*[/\\]}}libclang_rt.builtins-wasm32.a" "-o" "a.out"
+// LINK_KNOWN: wasm-ld{{.*}}" "-L/foo/lib/wasm32-wasi" "crt1.o" "[[temp]]" "-lc++" "-lc++abi" "-lc" "{{.*[/\\]}}libclang_rt.builtins.a" "-o" "a.out"
 
 // RUN: %clangxx -### --target=wasm32-wasi --sysroot=/foo --stdlib=libstdc++ %s 2>&1 \
 // RUN:   | FileCheck -check-prefix=LINK_KNOWN_STDCXX %s
 // LINK_KNOWN_STDCXX: "-cc1" {{.*}} "-o" "[[temp:[^"]*]]"
-// LINK_KNOWN_STDCXX: wasm-ld{{.*}}" "-L/foo/lib/wasm32-wasi" "crt1.o" "[[temp]]" "-lstdc++" "-lc" "{{.*[/\\]}}libclang_rt.builtins-wasm32.a" "-o" "a.out"
+// LINK_KNOWN_STDCXX: wasm-ld{{.*}}" "-L/foo/lib/wasm32-wasi" "crt1.o" "[[temp]]" "-lstdc++" "-lc" "{{.*[/\\]}}libclang_rt.builtins.a" "-o" "a.out"
 
 // A basic C++ link command-line with optimization with known OS.
 
 // RUN: %clangxx -### -O2 --target=wasm32-wasi --sysroot=/foo %s --stdlib=libc++ 2>&1 \
 // RUN:   | FileCheck -check-prefix=LINK_OPT_KNOWN %s
 // LINK_OPT_KNOWN: "-cc1" {{.*}} "-o" "[[temp:[^"]*]]"
-// LINK_OPT_KNOWN: wasm-ld{{.*}}" "-L/foo/lib/wasm32-wasi" "crt1.o" "[[temp]]" "-lc++" "-lc++abi" "-lc" "{{.*[/\\]}}libclang_rt.builtins-wasm32.a" "-o" "a.out"
+// LINK_OPT_KNOWN: wasm-ld{{.*}}" "-L/foo/lib/wasm32-wasi" "crt1.o" "[[temp]]" "-lc++" "-lc++abi" "-lc" "{{.*[/\\]}}libclang_rt.builtins.a" "-o" "a.out"
 
 // RUN: %clangxx -### -O2 --target=wasm32-wasi --sysroot=/foo %s --stdlib=libstdc++ 2>&1 \
 // RUN:   | FileCheck -check-prefix=LINK_OPT_KNOWN_STDCXX %s
 // LINK_OPT_KNOWN_STDCXX: "-cc1" {{.*}} "-o" "[[temp:[^"]*]]"
-// LINK_OPT_KNOWN_STDCXX: wasm-ld{{.*}}" "-L/foo/lib/wasm32-wasi" "crt1.o" "[[temp]]" "-lstdc++" "-lc" "{{.*[/\\]}}libclang_rt.builtins-wasm32.a" "-o" "a.out"
+// LINK_OPT_KNOWN_STDCXX: wasm-ld{{.*}}" "-L/foo/lib/wasm32-wasi" "crt1.o" "[[temp]]" "-lstdc++" "-lc" "{{.*[/\\]}}libclang_rt.builtins.a" "-o" "a.out"
 
 // A basic C++ compile command-line with known OS.
 
diff --git a/clang/test/Driver/windows-cross.c b/clang/test/Driver/windows-cross.c
index 75490b992d78..f6e831f00e13 100644
--- a/clang/test/Driver/windows-cross.c
+++ b/clang/test/Driver/windows-cross.c
@@ -11,32 +11,32 @@
 // RUN: %clang -### -target armv7-windows-itanium --sysroot %s/Inputs/Windows/ARM/8.1 -B %S/Inputs/Windows/ARM/8.1/usr/bin -fuse-ld=ld -rtlib=compiler-rt -stdlib=libstdc++ -o /dev/null %s 2>&1 \
 // RUN:   | FileCheck %s --check-prefix CHECK-RTLIB
 
-// CHECK-RTLIB: {{[/\\]}}ld" "--sysroot={{.*}}/Inputs/Windows/ARM/8.1" "-m" "thumb2pe" "-Bdynamic" "--entry" "mainCRTStartup" "--allow-multiple-definition" "-o" "{{[^"]*}}" "{{.*}}.o" "-lmsvcrt" "{{.*[\\/]}}clang_rt.builtins-arm.lib"
+// CHECK-RTLIB: {{[/\\]}}ld" "--sysroot={{.*}}/Inputs/Windows/ARM/8.1" "-m" "thumb2pe" "-Bdynamic" "--entry" "mainCRTStartup" "--allow-multiple-definition" "-o" "{{[^"]*}}" "{{.*}}.o" "-lmsvcrt" "{{.*[\\/]}}clang_rt.builtins.lib"
 
 // RUN: %clang -### -target armv7-windows-itanium --sysroot %S/Inputs/Windows/ARM/8.1 -B %S/Inputs/Windows/ARM/8.1/usr/bin -fuse-ld=ld -rtlib=compiler-rt -stdlib=libc++ -o /dev/null %s 2>&1 \
 // RUN:   | FileCheck %s --check-prefix CHECK-C-LIBCXX
 
-// CHECK-C-LIBCXX: {{[/\\]}}ld" "--sysroot={{.*}}/Inputs/Windows/ARM/8.1" "-m" "thumb2pe" "-Bdynamic" "--entry" "mainCRTStartup" "--allow-multiple-definition" "-o" "{{[^"]*}}" "{{.*}}.o" "-lmsvcrt" "{{.*[\\/]}}clang_rt.builtins-arm.lib"
+// CHECK-C-LIBCXX: {{[/\\]}}ld" "--sysroot={{.*}}/Inputs/Windows/ARM/8.1" "-m" "thumb2pe" "-Bdynamic" "--entry" "mainCRTStartup" "--allow-multiple-definition" "-o" "{{[^"]*}}" "{{.*}}.o" "-lmsvcrt" "{{.*[\\/]}}clang_rt.builtins.lib"
 
 // RUN: %clangxx -### -target armv7-windows-itanium --sysroot %S/Inputs/Windows/ARM/8.1 -B %S/Inputs/Windows/ARM/8.1/usr/bin -fuse-ld=ld -rtlib=compiler-rt -stdlib=libc++ -o /dev/null %s 2>&1 \
 // RUN:   | FileCheck %s --check-prefix CHECK-LIBCXX
 
-// CHECK-LIBCXX: {{[/\\]}}ld" "--sysroot={{.*}}/Inputs/Windows/ARM/8.1" "-m" "thumb2pe" "-Bdynamic" "--entry" "mainCRTStartup" "--allow-multiple-definition" "-o" "{{[^"]*}}" "{{.*}}.o" "-lc++" "-lmsvcrt" "{{.*[\\/]}}clang_rt.builtins-arm.lib"
+// CHECK-LIBCXX: {{[/\\]}}ld" "--sysroot={{.*}}/Inputs/Windows/ARM/8.1" "-m" "thumb2pe" "-Bdynamic" "--entry" "mainCRTStartup" "--allow-multiple-definition" "-o" "{{[^"]*}}" "{{.*}}.o" "-lc++" "-lmsvcrt" "{{.*[\\/]}}clang_rt.builtins.lib"
 
 // RUN: %clang -### -target armv7-windows-itanium --sysroot %S/Inputs/Windows/ARM/8.1 -B %S/Inputs/Windows/ARM/8.1/usr/bin -fuse-ld=ld -shared -rtlib=compiler-rt -stdlib=libc++ -o shared.dll %s 2>&1 \
 // RUN:   | FileCheck %s --check-prefix CHECK-SHARED
 
-// CHECK-SHARED: {{[/\\]}}ld" "--sysroot={{.*}}/Inputs/Windows/ARM/8.1" "-m" "thumb2pe" "-shared" "-Bdynamic" "--enable-auto-image-base" "--entry" "_DllMainCRTStartup" "--allow-multiple-definition" "-o" "shared.dll" "--out-implib" "shared.lib" "{{.*}}.o" "-lmsvcrt" "{{.*[\\/]}}clang_rt.builtins-arm.lib"
+// CHECK-SHARED: {{[/\\]}}ld" "--sysroot={{.*}}/Inputs/Windows/ARM/8.1" "-m" "thumb2pe" "-shared" "-Bdynamic" "--enable-auto-image-base" "--entry" "_DllMainCRTStartup" "--allow-multiple-definition" "-o" "shared.dll" "--out-implib" "shared.lib" "{{.*}}.o" "-lmsvcrt" "{{.*[\\/]}}clang_rt.builtins.lib"
 
 // RUN: %clang -### -target armv7-windows-itanium --sysroot %S/Inputs/Windows/ARM/8.1 -B %S/Inputs/Windows/ARM/8.1/usr/bin -fuse-ld=ld -shared -rtlib=compiler-rt -stdlib=libc++ -static -o shared.dll %s 2>&1 \
 // RUN:   | FileCheck %s --check-prefix CHECK-SHARED-STATIC
 
-// CHECK-SHARED-STATIC: {{[/\\]}}ld" "--sysroot={{.*}}/Inputs/Windows/ARM/8.1" "-m" "thumb2pe" "-shared" "-Bstatic" "--enable-auto-image-base" "--entry" "_DllMainCRTStartup" "--allow-multiple-definition" "-o" "shared.dll" "--out-implib" "shared.lib" "{{.*}}.o" "-lmsvcrt" "{{.*[\\/]}}clang_rt.builtins-arm.lib"
+// CHECK-SHARED-STATIC: {{[/\\]}}ld" "--sysroot={{.*}}/Inputs/Windows/ARM/8.1" "-m" "thumb2pe" "-shared" "-Bstatic" "--enable-auto-image-base" "--entry" "_DllMainCRTStartup" "--allow-multiple-definition" "-o" "shared.dll" "--out-implib" "shared.lib" "{{.*}}.o" "-lmsvcrt" "{{.*[\\/]}}clang_rt.builtins.lib"
 
 // RUN: %clang -### -target armv7-windows-itanium --sysroot %s/Inputs/Windows/ARM/8.1 -B %S/Inputs/Windows/ARM/8.1/usr/bin -fuse-ld=ld -shared -rtlib=compiler-rt -stdlib=libc++ -nostartfiles -o shared.dll %s 2>&1 \
 // RUN:   | FileCheck %s --check-prefix CHECK-NOSTARTFILES
 
-// CHECK-NOSTARTFILES: {{[/\\]}}ld" "--sysroot={{.*}}/Inputs/Windows/ARM/8.1" "-m" "thumb2pe" "-shared" "-Bdynamic" "--enable-auto-image-base" "--entry" "_DllMainCRTStartup" "--allow-multiple-definition" "-o" "shared.dll" "--out-implib" "shared.lib" "{{.*}}.o" "-lmsvcrt" "{{.*[\\/]}}clang_rt.builtins-arm.lib"
+// CHECK-NOSTARTFILES: {{[/\\]}}ld" "--sysroot={{.*}}/Inputs/Windows/ARM/8.1" "-m" "thumb2pe" "-shared" "-Bdynamic" "--enable-auto-image-base" "--entry" "_DllMainCRTStartup" "--allow-multiple-definition" "-o" "shared.dll" "--out-implib" "shared.lib" "{{.*}}.o" "-lmsvcrt" "{{.*[\\/]}}clang_rt.builtins.lib"
 
 // RUN: %clang -### -target armv7-windows-itanium --sysroot %S/Inputs/Windows/ARM/8.1 -B %S/Inputs/Windows/ARM/8.1/usr/bin -fuse-ld=ld -shared -rtlib=compiler-rt -stdlib=libc++ -nostartfiles -nodefaultlibs -o shared.dll %s 2>&1 \
 // RUN:   | FileCheck %s --check-prefix CHECK-STANDALONE
@@ -52,19 +52,19 @@
 // RUN:    | FileCheck %s --check-prefix CHECK-SANITIZE-ADDRESS
 
 // CHECK-SANITIZE-ADDRESS: "-fsanitize=address"
-// CHECK-SANITIZE-ADDRESS: "{{.*}}clang_rt.asan_dll_thunk-arm.lib"
+// CHECK-SANITIZE-ADDRESS: "{{.*}}clang_rt.asan_dll_thunk.lib"
 
 // RUN: %clang -### -target armv7-windows-itanium --sysroot %S/Inputs/Windows/ARM/8.1 -B %S/Inputs/Windows/ARM/8.1/usr/bin -fuse-ld=lld-link2 -o test.exe -fsanitize=address -x c++ %s 2>&1 \
 // RUN:    | FileCheck %s --check-prefix CHECK-SANITIZE-ADDRESS-EXE
 
 // CHECK-SANITIZE-ADDRESS-EXE: "-fsanitize=address"
-// CHECK-SANITIZE-ADDRESS-EXE: "{{.*}}clang_rt.asan_dynamic-arm.lib" "{{.*}}clang_rt.asan_dynamic_runtime_thunk-arm.lib" "--undefined" "__asan_seh_interceptor"
+// CHECK-SANITIZE-ADDRESS-EXE: "{{.*}}clang_rt.asan_dynamic.lib" "{{.*}}clang_rt.asan_dynamic_runtime_thunk.lib" "--undefined" "__asan_seh_interceptor"
 
 // RUN: %clang -### -target i686-windows-itanium -B %S/Inputs/Windows/ARM/8.1/usr/bin -fuse-ld=lld-link2 -o test.exe -fsanitize=address -x c++ %s 2>&1 \
 // RUN:    | FileCheck %s --check-prefix CHECK-SANITIZE-ADDRESS-EXE-X86
 
 // CHECK-SANITIZE-ADDRESS-EXE-X86: "-fsanitize=address"
-// CHECK-SANITIZE-ADDRESS-EXE-X86: "{{.*}}clang_rt.asan_dynamic-i386.lib" "{{.*}}clang_rt.asan_dynamic_runtime_thunk-i386.lib" "--undefined" "___asan_seh_interceptor"
+// CHECK-SANITIZE-ADDRESS-EXE-X86: "{{.*}}clang_rt.asan_dynamic.lib" "{{.*}}clang_rt.asan_dynamic_runtime_thunk.lib" "--undefined" "___asan_seh_interceptor"
 
 // RUN: not %clang -### --target=armv7-windows-itanium --sysroot %S/Inputs/Windows/ARM/8.1 -B %S/Inputs/Windows/ARM/8.1/usr/bin -fuse-ld=lld-link2 -shared -o shared.dll -fsanitize=tsan -x c++ %s 2>&1 \
 // RUN:    | FileCheck %s --check-prefix CHECK-SANITIZE-TSAN
diff --git a/clang/test/Driver/zos-ld.c b/clang/test/Driver/zos-ld.c
index 4d4decdd0e65..87d169936e12 100644
--- a/clang/test/Driver/zos-ld.c
+++ b/clang/test/Driver/zos-ld.c
@@ -14,7 +14,7 @@
 // C-LD-SAME: "-S" "//'SYS1.CSSLIB'"
 // C-LD-SAME: "//'CEE.SCEELIB(CELQS001)'"
 // C-LD-SAME: "//'CEE.SCEELIB(CELQS003)'"
-// C-LD-SAME: "[[RESOURCE_DIR]]{{/|\\\\}}lib{{/|\\\\}}zos{{/|\\\\}}libclang_rt.builtins-s390x.a"
+// C-LD-SAME: "[[RESOURCE_DIR]]{{/|\\\\}}lib{{/|\\\\}}s390x-ibm-zos{{/|\\\\}}libclang_rt.builtins.a"
 
 // 2. General C link for dll
 // RUN: %clang -### --shared --target=s390x-ibm-zos %s 2>&1 \
@@ -30,7 +30,7 @@
 // C-LD-DLL-SAME: "-S" "//'SYS1.CSSLIB'"
 // C-LD-DLL-SAME: "//'CEE.SCEELIB(CELQS001)'"
 // C-LD-DLL-SAME: "//'CEE.SCEELIB(CELQS003)'"
-// C-LD-DLL-SAME: "[[RESOURCE_DIR]]{{/|\\\\}}lib{{/|\\\\}}zos{{/|\\\\}}libclang_rt.builtins-s390x.a"
+// C-LD-DLL-SAME: "[[RESOURCE_DIR]]{{/|\\\\}}lib{{/|\\\\}}s390x-ibm-zos{{/|\\\\}}libclang_rt.builtins.a"
 
 // 3. General C++ link for executable
 // RUN: %clangxx -### --target=s390x-ibm-zos %s 2>&1 \
@@ -52,7 +52,7 @@
 // CXX-LD-SAME: "//'CEE.SCEELIB(CRTDQCXA)'"
 // CXX-LD-SAME: "//'CEE.SCEELIB(CRTDQXLA)'"
 // CXX-LD-SAME: "//'CEE.SCEELIB(CRTDQUNW)'"
-// CXX-LD-SAME: "[[RESOURCE_DIR]]{{/|\\\\}}lib{{/|\\\\}}zos{{/|\\\\}}libclang_rt.builtins-s390x.a"
+// CXX-LD-SAME: "[[RESOURCE_DIR]]{{/|\\\\}}lib{{/|\\\\}}s390x-ibm-zos{{/|\\\\}}libclang_rt.builtins.a"
 
 // 4. General C++ link for dll
 // RUN: %clangxx -### --shared --target=s390x-ibm-zos %s 2>&1 \
@@ -74,7 +74,7 @@
 // CXX-LD-DLL-SAME: "//'CEE.SCEELIB(CRTDQCXA)'"
 // CXX-LD-DLL-SAME: "//'CEE.SCEELIB(CRTDQXLA)'"
 // CXX-LD-DLL-SAME: "//'CEE.SCEELIB(CRTDQUNW)'"
-// CXX-LD-DLL-SAME: "[[RESOURCE_DIR]]{{/|\\\\}}lib{{/|\\\\}}zos{{/|\\\\}}libclang_rt.builtins-s390x.a"
+// CXX-LD-DLL-SAME: "[[RESOURCE_DIR]]{{/|\\\\}}lib{{/|\\\\}}s390x-ibm-zos{{/|\\\\}}libclang_rt.builtins.a"
 
 // 5. C++ link for executable w/ -mzos-hlq-le=, -mzos-hlq-csslib=
 // RUN: %clangxx -### --target=s390x-ibm-zos %s 2>&1 \
@@ -97,7 +97,7 @@
 // CXX-LD5-SAME: "//'AAAA.SCEELIB(CRTDQCXA)'"
 // CXX-LD5-SAME: "//'AAAA.SCEELIB(CRTDQXLA)'"
 // CXX-LD5-SAME: "//'AAAA.SCEELIB(CRTDQUNW)'"
-// CXX-LD5-SAME: "[[RESOURCE_DIR]]{{/|\\\\}}lib{{/|\\\\}}zos{{/|\\\\}}libclang_rt.builtins-s390x.a"
+// CXX-LD5-SAME: "[[RESOURCE_DIR]]{{/|\\\\}}lib{{/|\\\\}}s390x-ibm-zos{{/|\\\\}}libclang_rt.builtins.a"
 
 // 6. C++ link for executable w/ -mzos-hlq-clang=
 // RUN: %clangxx -### --target=s390x-ibm-zos %s 2>&1 \
@@ -120,4 +120,4 @@
 // CXX-LD6-SAME: "//'AAAA.SCEELIB(CRTDQCXA)'"
 // CXX-LD6-SAME: "//'AAAA.SCEELIB(CRTDQXLA)'"
 // CXX-LD6-SAME: "//'AAAA.SCEELIB(CRTDQUNW)'"
-// CXX-LD6-SAME: "[[RESOURCE_DIR]]{{/|\\\\}}lib{{/|\\\\}}zos{{/|\\\\}}libclang_rt.builtins-s390x.a"
+// CXX-LD6-SAME: "[[RESOURCE_DIR]]{{/|\\\\}}lib{{/|\\\\}}s390x-ibm-zos{{/|\\\\}}libclang_rt.builtins.a"
diff --git a/flang/test/Driver/msvc-dependent-lib-flags.f90 b/flang/test/Driver/msvc-dependent-lib-flags.f90
index 7c1f962e339f..643dbe9e949c 100644
--- a/flang/test/Driver/msvc-dependent-lib-flags.f90
+++ b/flang/test/Driver/msvc-dependent-lib-flags.f90
@@ -4,7 +4,7 @@
 ! RUN: %flang -### --target=aarch64-windows-msvc -fms-runtime-lib=dll_dbg %S/Inputs/hello.f90 -v 2>&1 | FileCheck %s --check-prefixes=MSVC-DLL-DEBUG
 
 ! MSVC: -fc1
-! MSVC-SAME: --dependent-lib=clang_rt.builtins-aarch64.lib
+! MSVC-SAME: --dependent-lib=clang_rt.builtins.lib
 ! MSVC-SAME: -D_MT
 ! MSVC-SAME: --dependent-lib=libcmt
 ! MSVC-SAME: --dependent-lib=Fortran_main.static.lib
@@ -12,7 +12,7 @@
 ! MSVC-SAME: --dependent-lib=FortranDecimal.static.lib
 
 ! MSVC-DEBUG: -fc1
-! MSVC-DEBUG-SAME: --dependent-lib=clang_rt.builtins-aarch64.lib
+! MSVC-DEBUG-SAME: --dependent-lib=clang_rt.builtins.lib
 ! MSVC-DEBUG-SAME: -D_MT
 ! MSVC-DEBUG-SAME: -D_DEBUG
 ! MSVC-DEBUG-SAME: --dependent-lib=libcmtd
@@ -21,7 +21,7 @@
 ! MSVC-DEBUG-SAME: --dependent-lib=FortranDecimal.static_dbg.lib
 
 ! MSVC-DLL: -fc1
-! MSVC-DLL-SAME: --dependent-lib=clang_rt.builtins-aarch64.lib
+! MSVC-DLL-SAME: --dependent-lib=clang_rt.builtins.lib
 ! MSVC-DLL-SAME: -D_MT
 ! MSVC-DLL-SAME: -D_DLL
 ! MSVC-DLL-SAME: --dependent-lib=msvcrt
@@ -30,7 +30,7 @@
 ! MSVC-DLL-SAME: --dependent-lib=FortranDecimal.dynamic.lib
 
 ! MSVC-DLL-DEBUG: -fc1
-! MSVC-DLL-DEBUG-SAME: --dependent-lib=clang_rt.builtins-aarch64.lib
+! MSVC-DLL-DEBUG-SAME: --dependent-lib=clang_rt.builtins.lib
 ! MSVC-DLL-DEBUG-SAME: -D_MT
 ! MSVC-DLL-DEBUG-SAME: -D_DEBUG
 ! MSVC-DLL-DEBUG-SAME: -D_DLL
-- 
GitLab


From e8232c325f7c56c175b27a3538699c31519c8f5e Mon Sep 17 00:00:00 2001
From: Andreas Jonson <andjo403@hotmail.com>
Date: Mon, 15 Apr 2024 02:54:37 +0200
Subject: [PATCH 181/464] [ADT] In FoldingSet calulation for APInt add bit
 width to avoid crash (#88635)

Multiple ranges attributes with the same lower and upper limit but
different bit width caused a crash.
---
 llvm/include/llvm/ADT/FoldingSet.h | 1 +
 llvm/test/Bitcode/attributes.ll    | 5 +++++
 2 files changed, 6 insertions(+)

diff --git a/llvm/include/llvm/ADT/FoldingSet.h b/llvm/include/llvm/ADT/FoldingSet.h
index ddc3e52255d6..28a84d965883 100644
--- a/llvm/include/llvm/ADT/FoldingSet.h
+++ b/llvm/include/llvm/ADT/FoldingSet.h
@@ -356,6 +356,7 @@ public:
     AddInteger(unsigned(I >> 32));
   }
   void AddInteger(const APInt &Int) {
+    AddInteger(Int.getBitWidth());
     const auto *Parts = Int.getRawData();
     for (int i = 0, N = Int.getNumWords(); i < N; ++i) {
       AddInteger(Parts[i]);
diff --git a/llvm/test/Bitcode/attributes.ll b/llvm/test/Bitcode/attributes.ll
index 26163b4d38c8..0c753e58d435 100644
--- a/llvm/test/Bitcode/attributes.ll
+++ b/llvm/test/Bitcode/attributes.ll
@@ -531,6 +531,11 @@ define range(i32 -1, 42) i32 @range_attribute(<4 x i32> range(i32 -1, 42) %a) {
   ret i32 0
 }
 
+; CHECK: define range(i32 0, 42) i32 @range_attribute_same_range_other_bitwidth(i8 range(i8 0, 42) %a)
+define range(i32 0, 42) i32 @range_attribute_same_range_other_bitwidth(i8 range(i8 0, 42) %a) {
+  ret i32 0
+}
+
 ; CHECK: define void @wide_range_attribute(i128 range(i128 618970019642690137449562111, 618970019642690137449562114) %a)
 define void @wide_range_attribute(i128 range(i128 618970019642690137449562111, 618970019642690137449562114) %a) {
   ret void
-- 
GitLab


From 8e4b0890a61088ae55b4bccbb59e5c5e10d28385 Mon Sep 17 00:00:00 2001
From: wanglei <wanglei@loongson.cn>
Date: Mon, 15 Apr 2024 09:01:04 +0800
Subject: [PATCH 182/464] [LoongArch] Return true from
 shouldConsiderGEPOffsetSplit (#88371)

If not performing gep splits can prevent important optimizations, such
as preventing the element indices / member offsets from being
(partially) folded into load/store instruction immediates.
---
 llvm/lib/Target/LoongArch/LoongArchISelLowering.h         | 1 +
 llvm/test/Transforms/CodeGenPrepare/LoongArch/splitgep.ll | 6 +++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
index 9e9ac0b82692..4bb6c049f8d7 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h
@@ -238,6 +238,7 @@ public:
   bool isShuffleMaskLegal(ArrayRef<int> Mask, EVT VT) const override {
     return false;
   }
+  bool shouldConsiderGEPOffsetSplit() const override { return true; }
 
 private:
   /// Target-specific function used to lower LoongArch calling conventions.
diff --git a/llvm/test/Transforms/CodeGenPrepare/LoongArch/splitgep.ll b/llvm/test/Transforms/CodeGenPrepare/LoongArch/splitgep.ll
index ca2a16a0ca7e..20cc25e95adf 100644
--- a/llvm/test/Transforms/CodeGenPrepare/LoongArch/splitgep.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/LoongArch/splitgep.ll
@@ -4,9 +4,9 @@
 define void @test(ptr %sp, ptr %t, i32 %n) {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    %s = load ptr, ptr %sp, align 8
-; CHECK-NEXT:    br label %while_cond
-
+; CHECK-NEXT:    %splitgep1 = getelementptr i8, ptr %t, i64 80000
+; CHECK-NEXT:    %s = load ptr, ptr %sp
+; CHECK-NEXT:    %splitgep = getelementptr i8, ptr %s, i64 80000
 entry:
   %s = load ptr, ptr %sp
   br label %while_cond
-- 
GitLab


From dd4b40ffe3bd9da2e3d998b674305ae1f612ad51 Mon Sep 17 00:00:00 2001
From: paperchalice <liujunchang97@outlook.com>
Date: Mon, 15 Apr 2024 09:14:30 +0800
Subject: [PATCH 183/464] [CodeGen] Let `CodeGenPassBuilder` know concrete
 target machine (#88614)

Many backends passes accept only `<Target>TargetMachine`, this can avoid
cast to derived type.
---
 llvm/include/llvm/Passes/CodeGenPassBuilder.h | 107 +++++++++---------
 llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp |   5 +-
 2 files changed, 59 insertions(+), 53 deletions(-)

diff --git a/llvm/include/llvm/Passes/CodeGenPassBuilder.h b/llvm/include/llvm/Passes/CodeGenPassBuilder.h
index ed5d8affa793..2e94a1950213 100644
--- a/llvm/include/llvm/Passes/CodeGenPassBuilder.h
+++ b/llvm/include/llvm/Passes/CodeGenPassBuilder.h
@@ -109,9 +109,9 @@ namespace llvm {
 /// construction. The \c MachinePassRegistry.def file specifies how to construct
 /// all of the built-in passes, and those may reference these members during
 /// construction.
-template <typename DerivedT> class CodeGenPassBuilder {
+template <typename DerivedT, typename TargetMachineT> class CodeGenPassBuilder {
 public:
-  explicit CodeGenPassBuilder(LLVMTargetMachine &TM,
+  explicit CodeGenPassBuilder(TargetMachineT &TM,
                               const CGPassBuilderOption &Opts,
                               PassInstrumentationCallbacks *PIC)
       : TM(TM), Opt(Opts), PIC(PIC) {
@@ -237,7 +237,7 @@ protected:
     const DerivedT &PB;
   };
 
-  LLVMTargetMachine &TM;
+  TargetMachineT &TM;
   CGPassBuilderOption Opt;
   PassInstrumentationCallbacks *PIC;
 
@@ -498,8 +498,8 @@ private:
   mutable bool Stopped = true;
 };
 
-template <typename Derived>
-Error CodeGenPassBuilder<Derived>::buildPipeline(
+template <typename Derived, typename TargetMachineT>
+Error CodeGenPassBuilder<Derived, TargetMachineT>::buildPipeline(
     ModulePassManager &MPM, raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut,
     CodeGenFileType FileType) const {
   auto StartStopInfo = TargetPassConfig::getStartStopInfo(*PIC);
@@ -542,8 +542,8 @@ Error CodeGenPassBuilder<Derived>::buildPipeline(
   return verifyStartStop(*StartStopInfo);
 }
 
-template <typename Derived>
-void CodeGenPassBuilder<Derived>::setStartStopPasses(
+template <typename Derived, typename TargetMachineT>
+void CodeGenPassBuilder<Derived, TargetMachineT>::setStartStopPasses(
     const TargetPassConfig::StartStopInfo &Info) const {
   if (!Info.StartPass.empty()) {
     Started = false;
@@ -585,8 +585,8 @@ void CodeGenPassBuilder<Derived>::setStartStopPasses(
   }
 }
 
-template <typename Derived>
-Error CodeGenPassBuilder<Derived>::verifyStartStop(
+template <typename Derived, typename TargetMachineT>
+Error CodeGenPassBuilder<Derived, TargetMachineT>::verifyStartStop(
     const TargetPassConfig::StartStopInfo &Info) const {
   if (Started && Stopped)
     return Error::success();
@@ -604,8 +604,9 @@ Error CodeGenPassBuilder<Derived>::verifyStartStop(
   return Error::success();
 }
 
-template <typename Derived>
-void CodeGenPassBuilder<Derived>::addISelPasses(AddIRPass &addPass) const {
+template <typename Derived, typename TargetMachineT>
+void CodeGenPassBuilder<Derived, TargetMachineT>::addISelPasses(
+    AddIRPass &addPass) const {
   derived().addGlobalMergePass(addPass);
   if (TM.useEmulatedTLS())
     addPass(LowerEmuTLSPass());
@@ -620,8 +621,9 @@ void CodeGenPassBuilder<Derived>::addISelPasses(AddIRPass &addPass) const {
 
 /// Add common target configurable passes that perform LLVM IR to IR transforms
 /// following machine independent optimization.
-template <typename Derived>
-void CodeGenPassBuilder<Derived>::addIRPasses(AddIRPass &addPass) const {
+template <typename Derived, typename TargetMachineT>
+void CodeGenPassBuilder<Derived, TargetMachineT>::addIRPasses(
+    AddIRPass &addPass) const {
   // Before running any passes, run the verifier to determine if the input
   // coming from the front-end and/or optimizer is valid.
   if (!Opt.DisableVerify)
@@ -686,8 +688,8 @@ void CodeGenPassBuilder<Derived>::addIRPasses(AddIRPass &addPass) const {
 
 /// Turn exception handling constructs into something the code generators can
 /// handle.
-template <typename Derived>
-void CodeGenPassBuilder<Derived>::addPassesToHandleExceptions(
+template <typename Derived, typename TargetMachineT>
+void CodeGenPassBuilder<Derived, TargetMachineT>::addPassesToHandleExceptions(
     AddIRPass &addPass) const {
   const MCAsmInfo *MCAI = TM.getMCAsmInfo();
   assert(MCAI && "No MCAsmInfo");
@@ -733,8 +735,9 @@ void CodeGenPassBuilder<Derived>::addPassesToHandleExceptions(
 
 /// Add pass to prepare the LLVM IR for code generation. This should be done
 /// before exception handling preparation passes.
-template <typename Derived>
-void CodeGenPassBuilder<Derived>::addCodeGenPrepare(AddIRPass &addPass) const {
+template <typename Derived, typename TargetMachineT>
+void CodeGenPassBuilder<Derived, TargetMachineT>::addCodeGenPrepare(
+    AddIRPass &addPass) const {
   if (getOptLevel() != CodeGenOptLevel::None && !Opt.DisableCGP)
     addPass(CodeGenPreparePass(&TM));
   // TODO: Default ctor'd RewriteSymbolPass is no-op.
@@ -743,8 +746,9 @@ void CodeGenPassBuilder<Derived>::addCodeGenPrepare(AddIRPass &addPass) const {
 
 /// Add common passes that perform LLVM IR to IR transforms in preparation for
 /// instruction selection.
-template <typename Derived>
-void CodeGenPassBuilder<Derived>::addISelPrepare(AddIRPass &addPass) const {
+template <typename Derived, typename TargetMachineT>
+void CodeGenPassBuilder<Derived, TargetMachineT>::addISelPrepare(
+    AddIRPass &addPass) const {
   derived().addPreISel(addPass);
 
   addPass(CallBrPreparePass());
@@ -763,8 +767,8 @@ void CodeGenPassBuilder<Derived>::addISelPrepare(AddIRPass &addPass) const {
     addPass(VerifierPass());
 }
 
-template <typename Derived>
-Error CodeGenPassBuilder<Derived>::addCoreISelPasses(
+template <typename Derived, typename TargetMachineT>
+Error CodeGenPassBuilder<Derived, TargetMachineT>::addCoreISelPasses(
     AddMachinePass &addPass) const {
   // Enable FastISel with -fast-isel, but allow that to be overridden.
   TM.setO0WantsFastISel(Opt.EnableFastISelOption.value_or(true));
@@ -846,17 +850,18 @@ Error CodeGenPassBuilder<Derived>::addCoreISelPasses(
 /// with nontrivial configuration or multiple passes are broken out below in
 /// add%Stage routines.
 ///
-/// Any CodeGenPassBuilder<Derived>::addXX routine may be overriden by the
-/// Target. The addPre/Post methods with empty header implementations allow
-/// injecting target-specific fixups just before or after major stages.
-/// Additionally, targets have the flexibility to change pass order within a
-/// stage by overriding default implementation of add%Stage routines below. Each
-/// technique has maintainability tradeoffs because alternate pass orders are
-/// not well supported. addPre/Post works better if the target pass is easily
-/// tied to a common pass. But if it has subtle dependencies on multiple passes,
-/// the target should override the stage instead.
-template <typename Derived>
-Error CodeGenPassBuilder<Derived>::addMachinePasses(
+/// Any CodeGenPassBuilder<Derived, TargetMachine>::addXX routine may be
+/// overriden by the Target. The addPre/Post methods with empty header
+/// implementations allow injecting target-specific fixups just before or after
+/// major stages. Additionally, targets have the flexibility to change pass
+/// order within a stage by overriding default implementation of add%Stage
+/// routines below. Each technique has maintainability tradeoffs because
+/// alternate pass orders are not well supported. addPre/Post works better if
+/// the target pass is easily tied to a common pass. But if it has subtle
+/// dependencies on multiple passes, the target should override the stage
+/// instead.
+template <typename Derived, typename TargetMachineT>
+Error CodeGenPassBuilder<Derived, TargetMachineT>::addMachinePasses(
     AddMachinePass &addPass) const {
   // Add passes that optimize machine instructions in SSA form.
   if (getOptLevel() != CodeGenOptLevel::None) {
@@ -962,8 +967,8 @@ Error CodeGenPassBuilder<Derived>::addMachinePasses(
 }
 
 /// Add passes that optimize machine instructions in SSA form.
-template <typename Derived>
-void CodeGenPassBuilder<Derived>::addMachineSSAOptimization(
+template <typename Derived, typename TargetMachineT>
+void CodeGenPassBuilder<Derived, TargetMachineT>::addMachineSSAOptimization(
     AddMachinePass &addPass) const {
   // Pre-ra tail duplication.
   addPass(EarlyTailDuplicatePass());
@@ -1014,8 +1019,8 @@ void CodeGenPassBuilder<Derived>::addMachineSSAOptimization(
 /// A target that uses the standard regalloc pass order for fast or optimized
 /// allocation may still override this for per-target regalloc
 /// selection. But -regalloc=... always takes precedence.
-template <typename Derived>
-void CodeGenPassBuilder<Derived>::addTargetRegisterAllocator(
+template <typename Derived, typename TargetMachineT>
+void CodeGenPassBuilder<Derived, TargetMachineT>::addTargetRegisterAllocator(
     AddMachinePass &addPass, bool Optimized) const {
   if (Optimized)
     addPass(RAGreedyPass());
@@ -1026,22 +1031,22 @@ void CodeGenPassBuilder<Derived>::addTargetRegisterAllocator(
 /// Find and instantiate the register allocation pass requested by this target
 /// at the current optimization level.  Different register allocators are
 /// defined as separate passes because they may require different analysis.
-template <typename Derived>
-void CodeGenPassBuilder<Derived>::addRegAllocPass(AddMachinePass &addPass,
-                                                  bool Optimized) const {
+template <typename Derived, typename TargetMachineT>
+void CodeGenPassBuilder<Derived, TargetMachineT>::addRegAllocPass(
+    AddMachinePass &addPass, bool Optimized) const {
   // TODO: Parse Opt.RegAlloc to add register allocator.
 }
 
-template <typename Derived>
-Error CodeGenPassBuilder<Derived>::addRegAssignmentFast(
+template <typename Derived, typename TargetMachineT>
+Error CodeGenPassBuilder<Derived, TargetMachineT>::addRegAssignmentFast(
     AddMachinePass &addPass) const {
   // TODO: Ensure allocator is default or fast.
   addRegAllocPass(addPass, false);
   return Error::success();
 }
 
-template <typename Derived>
-Error CodeGenPassBuilder<Derived>::addRegAssignmentOptimized(
+template <typename Derived, typename TargetMachineT>
+Error CodeGenPassBuilder<Derived, TargetMachineT>::addRegAssignmentOptimized(
     AddMachinePass &addPass) const {
   // Add the selected register allocation pass.
   addRegAllocPass(addPass, true);
@@ -1062,8 +1067,8 @@ Error CodeGenPassBuilder<Derived>::addRegAssignmentOptimized(
 
 /// Add the minimum set of target-independent passes that are required for
 /// register allocation. No coalescing or scheduling.
-template <typename Derived>
-Error CodeGenPassBuilder<Derived>::addFastRegAlloc(
+template <typename Derived, typename TargetMachineT>
+Error CodeGenPassBuilder<Derived, TargetMachineT>::addFastRegAlloc(
     AddMachinePass &addPass) const {
   addPass(PHIEliminationPass());
   addPass(TwoAddressInstructionPass());
@@ -1073,8 +1078,8 @@ Error CodeGenPassBuilder<Derived>::addFastRegAlloc(
 /// Add standard target-independent passes that are tightly coupled with
 /// optimized register allocation, including coalescing, machine instruction
 /// scheduling, and register allocation itself.
-template <typename Derived>
-void CodeGenPassBuilder<Derived>::addOptimizedRegAlloc(
+template <typename Derived, typename TargetMachineT>
+void CodeGenPassBuilder<Derived, TargetMachineT>::addOptimizedRegAlloc(
     AddMachinePass &addPass) const {
   addPass(DetectDeadLanesPass());
 
@@ -1121,8 +1126,8 @@ void CodeGenPassBuilder<Derived>::addOptimizedRegAlloc(
 //===---------------------------------------------------------------------===//
 
 /// Add passes that optimize machine instructions after register allocation.
-template <typename Derived>
-void CodeGenPassBuilder<Derived>::addMachineLateOptimization(
+template <typename Derived, typename TargetMachineT>
+void CodeGenPassBuilder<Derived, TargetMachineT>::addMachineLateOptimization(
     AddMachinePass &addPass) const {
   // Branch folding must be run after regalloc and prolog/epilog insertion.
   addPass(BranchFolderPass());
@@ -1142,8 +1147,8 @@ void CodeGenPassBuilder<Derived>::addMachineLateOptimization(
 }
 
 /// Add standard basic block placement passes.
-template <typename Derived>
-void CodeGenPassBuilder<Derived>::addBlockPlacement(
+template <typename Derived, typename TargetMachineT>
+void CodeGenPassBuilder<Derived, TargetMachineT>::addBlockPlacement(
     AddMachinePass &addPass) const {
   addPass(MachineBlockPlacementPass());
   // Run a separate pass to collect block placement statistics.
diff --git a/llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp b/llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp
index 453754381034..7c1fb0b99f41 100644
--- a/llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp
+++ b/llvm/lib/Target/X86/X86CodeGenPassBuilder.cpp
@@ -19,9 +19,10 @@ using namespace llvm;
 
 namespace {
 
-class X86CodeGenPassBuilder : public CodeGenPassBuilder<X86CodeGenPassBuilder> {
+class X86CodeGenPassBuilder
+    : public CodeGenPassBuilder<X86CodeGenPassBuilder, X86TargetMachine> {
 public:
-  explicit X86CodeGenPassBuilder(LLVMTargetMachine &TM,
+  explicit X86CodeGenPassBuilder(X86TargetMachine &TM,
                                  const CGPassBuilderOption &Opts,
                                  PassInstrumentationCallbacks *PIC)
       : CodeGenPassBuilder(TM, Opts, PIC) {}
-- 
GitLab


From 7476b412d46322f52a7f58ad1d94a4398df941e2 Mon Sep 17 00:00:00 2001
From: Chen Zheng <czhengsz@cn.ibm.com>
Date: Sun, 14 Apr 2024 21:22:48 -0400
Subject: [PATCH 184/464] [NFC] add myself as the code owner of the PowerPC
 target

Norminated by Nemanja in https://discourse.llvm.org/t/powerpc-code-owner/78142
---
 llvm/CODE_OWNERS.TXT | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/CODE_OWNERS.TXT b/llvm/CODE_OWNERS.TXT
index 48026bd2f74e..d1620d1cbf87 100644
--- a/llvm/CODE_OWNERS.TXT
+++ b/llvm/CODE_OWNERS.TXT
@@ -79,8 +79,8 @@ N: Hal Finkel
 E: hfinkel@anl.gov
 D: The loop reroller and alias analysis
 
-N: Nemanja Ivanovic
-E: nemanja.i.ibm@gmail.com
+N: Zheng Chen
+E: czhengsz@cn.ibm.com
 D: PowerPC Backend
 
 N: Dan Gohman
-- 
GitLab


From a9d7ad23fa625e7f52f2641dbb0f90e8ea12bebf Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Mon, 15 Apr 2024 10:22:11 +0900
Subject: [PATCH 185/464] [InstCombine] Relax shamt assertion in fsh fold

Allow the result of the comparison to contain poison elements,
which happens if one of the elements in the input vector is
poison.
---
 llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp |  4 ++--
 llvm/test/Transforms/InstCombine/rotate.ll           | 11 +++++++++++
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index b35ab8751a21..20f51c8af617 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -1979,8 +1979,8 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
       if (ModuloC != ShAmtC)
         return replaceOperand(*II, 2, ModuloC);
 
-      assert(ConstantExpr::getICmp(ICmpInst::ICMP_UGT, WidthC, ShAmtC) ==
-                 ConstantInt::getTrue(CmpInst::makeCmpResultType(Ty)) &&
+      assert(match(ConstantExpr::getICmp(ICmpInst::ICMP_UGT, WidthC, ShAmtC),
+                   m_One()) &&
              "Shift amount expected to be modulo bitwidth");
 
       // Canonicalize funnel shift right by constant to funnel shift left. This
diff --git a/llvm/test/Transforms/InstCombine/rotate.ll b/llvm/test/Transforms/InstCombine/rotate.ll
index edd9bf39ca07..6c70c791fd88 100644
--- a/llvm/test/Transforms/InstCombine/rotate.ll
+++ b/llvm/test/Transforms/InstCombine/rotate.ll
@@ -147,6 +147,17 @@ define <2 x i32> @rotr_v2i32_constant_nonsplat_undef0(<2 x i32> %x) {
   ret <2 x i32> %r
 }
 
+define <2 x i32> @rotr_v2i32_constant_nonsplat_poison0(<2 x i32> %x) {
+; CHECK-LABEL: @rotr_v2i32_constant_nonsplat_poison0(
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i32> @llvm.fshl.v2i32(<2 x i32> [[X:%.*]], <2 x i32> [[X]], <2 x i32> <i32 poison, i32 19>)
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %shl = shl <2 x i32> %x, <i32 poison, i32 19>
+  %shr = lshr <2 x i32> %x, <i32 15, i32 13>
+  %r = or <2 x i32> %shl, %shr
+  ret <2 x i32> %r
+}
+
 define <2 x i32> @rotr_v2i32_constant_nonsplat_undef1(<2 x i32> %x) {
 ; CHECK-LABEL: @rotr_v2i32_constant_nonsplat_undef1(
 ; CHECK-NEXT:    [[R:%.*]] = call <2 x i32> @llvm.fshl.v2i32(<2 x i32> [[X:%.*]], <2 x i32> [[X]], <2 x i32> <i32 17, i32 0>)
-- 
GitLab


From 9c3475acc9b11d08f0e0b3e230429e9f552d5f80 Mon Sep 17 00:00:00 2001
From: Billy Zhu <billyzhu@modular.com>
Date: Sun, 14 Apr 2024 19:17:20 -0700
Subject: [PATCH 186/464] [MLIR][LLVM] Fix recursive DI type export memory leak
 (#88122)

Followup to discussion
https://github.com/llvm/llvm-project/pull/87295#discussion_r1556016667.

The export cache should not cache temporary nodes.
---
 mlir/lib/Target/LLVMIR/DebugTranslation.cpp | 3 ++-
 mlir/test/Target/LLVMIR/llvmir-debug.mlir   | 8 ++++++--
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/mlir/lib/Target/LLVMIR/DebugTranslation.cpp b/mlir/lib/Target/LLVMIR/DebugTranslation.cpp
index 46e2e7f2ba5d..2de5e372d88c 100644
--- a/mlir/lib/Target/LLVMIR/DebugTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/DebugTranslation.cpp
@@ -352,7 +352,8 @@ llvm::DINode *DebugTranslation::translate(DINodeAttr attr) {
                      DISubroutineTypeAttr>(
                    [&](auto attr) { return translateImpl(attr); });
 
-  attrToNode.insert({attr, node});
+  if (node && !node->isTemporary())
+    attrToNode.insert({attr, node});
   return node;
 }
 
diff --git a/mlir/test/Target/LLVMIR/llvmir-debug.mlir b/mlir/test/Target/LLVMIR/llvmir-debug.mlir
index f4c18bf6bd53..8ab1a1b290da 100644
--- a/mlir/test/Target/LLVMIR/llvmir-debug.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir-debug.mlir
@@ -442,12 +442,16 @@ llvm.mlir.global @global_variable() {dbg_expr = #di_global_variable_expression}
 #di_subprogram = #llvm.di_subprogram<scope = #di_file, file = #di_file, subprogramFlags = Optimized, type = #di_subroutine_type>
 #di_composite_type = #llvm.di_composite_type<tag = DW_TAG_class_type, recId = distinct[0]<>, scope = #di_subprogram>
 
-#di_global_variable = #llvm.di_global_variable<file = #di_file, line = 1, type = #di_composite_type>
+// Use the inner type standalone outside too. Ensures it's not cached wrong.
+#di_var_type = #llvm.di_subroutine_type<types = #di_composite_type, #di_composite_type_inner>
+#di_global_variable = #llvm.di_global_variable<file = #di_file, line = 1, type = #di_var_type>
 #di_global_variable_expression = #llvm.di_global_variable_expression<var = #di_global_variable>
 
 llvm.mlir.global @global_variable() {dbg_expr = #di_global_variable_expression} : !llvm.struct<()>
 
-// CHECK: distinct !DIGlobalVariable({{.*}}type: ![[COMP:[0-9]+]],
+// CHECK: distinct !DIGlobalVariable({{.*}}type: ![[VAR:[0-9]+]],
+// CHECK: ![[VAR]] = !DISubroutineType(types: ![[COMPS:[0-9]+]])
+// CHECK: ![[COMPS]] = !{![[COMP:[0-9]+]],
 // CHECK: ![[COMP]] = distinct !DICompositeType({{.*}}scope: ![[SCOPE:[0-9]+]],
 // CHECK: ![[SCOPE]] = !DISubprogram({{.*}}type: ![[SUBROUTINE:[0-9]+]],
 // CHECK: ![[SUBROUTINE]] = !DISubroutineType(types: ![[SR_TYPES:[0-9]+]])
-- 
GitLab


From 37b7207651b44743909a427b5509bed5e6c21b59 Mon Sep 17 00:00:00 2001
From: Allen <zhongyunde@huawei.com>
Date: Mon, 15 Apr 2024 10:19:49 +0800
Subject: [PATCH 187/464] [SimplifyCFG] Fix crash when there is unreachable
 large index (#88616)

The large case index out of scope is dead code, but it is still be
created for TableContents in SwitchLookupTable::SwitchLookupTable,
so make sure the table size after growing should not get smaller.

Fix https://github.com/llvm/llvm-project/issues/88607
---
 llvm/lib/Transforms/Utils/SimplifyCFG.cpp     |  4 ++-
 .../Transforms/SimplifyCFG/switch_mask.ll     | 25 +++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index 55bbffb18879..da3eb229c041 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -6783,9 +6783,11 @@ static bool SwitchToLookupTable(SwitchInst *SI, IRBuilder<> &Builder,
           return SwitchLookupTable::WouldFitInRegister(
               DL, UpperBound, KV.second /* ResultType */);
         })) {
+      // There may be some case index larger than the UpperBound (unreachable
+      // case), so make sure the table size does not get smaller.
+      TableSize = std::max(UpperBound, TableSize);
       // The default branch is unreachable after we enlarge the lookup table.
       // Adjust DefaultIsReachable to reuse code path.
-      TableSize = UpperBound;
       DefaultIsReachable = false;
     }
   }
diff --git a/llvm/test/Transforms/SimplifyCFG/switch_mask.ll b/llvm/test/Transforms/SimplifyCFG/switch_mask.ll
index 17f7b583ee84..51c15774d476 100644
--- a/llvm/test/Transforms/SimplifyCFG/switch_mask.ll
+++ b/llvm/test/Transforms/SimplifyCFG/switch_mask.ll
@@ -214,3 +214,28 @@ lor.end:                                          ; preds = %default, %for.end,
   %retval.0.i.i = phi i32 [ 1, %default ], [ 0, %for.end ], [ 0, %for.end ], [ 0, %for.end ]
   ret void
 }
+
+define i1 @pr88607() {
+; CHECK-LABEL: @pr88607(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND:%.*]] = select i1 false, i32 4, i32 1
+; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = select i1 false, i32 2, i32 [[COND]]
+; CHECK-NEXT:    ret i1 false
+;
+entry:
+  %cond = select i1 false, i32 4, i32 1
+  %spec.select = select i1 false, i32 2, i32 %cond
+  switch i32 %spec.select, label %lor.rhs [
+  i32 0, label %exit
+  i32 5, label %exit ; unreachable large case index
+  i32 1, label %exit
+  ]
+
+lor.rhs:                                        ; preds = %entry
+  br label %exit
+
+exit: ; preds = %lor.rhs, %entry, %entry, %entry, %entry
+  %res.ph = phi i1 [ false, %entry ], [ false, %lor.rhs ], [ false, %entry ], [ false, %entry ]
+  ret i1 %res.ph
+}
+
-- 
GitLab


From ccea9f2ae81d140a2367a82069cd00d6b08fe855 Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Wed, 10 Apr 2024 15:29:58 -0500
Subject: [PATCH 188/464] [ValueTracking] Add tests for `computeKnownBits` of
 `llvm.vector.reduce.{or,and}`; NFC

---
 .../test/Transforms/InstCombine/known-bits.ll | 52 +++++++++++++++++++
 1 file changed, 52 insertions(+)

diff --git a/llvm/test/Transforms/InstCombine/known-bits.ll b/llvm/test/Transforms/InstCombine/known-bits.ll
index fea2f647ee2a..cdc3e991a3fb 100644
--- a/llvm/test/Transforms/InstCombine/known-bits.ll
+++ b/llvm/test/Transforms/InstCombine/known-bits.ll
@@ -1195,5 +1195,57 @@ define i1 @extract_value_smul_fail(i8 %xx, i8 %yy) {
   ret i1 %r
 }
 
+define i8 @known_reduce_or(<2 x i8> %xx) {
+; CHECK-LABEL: @known_reduce_or(
+; CHECK-NEXT:    [[X:%.*]] = or <2 x i8> [[XX:%.*]], <i8 5, i8 3>
+; CHECK-NEXT:    [[V:%.*]] = call i8 @llvm.vector.reduce.or.v2i8(<2 x i8> [[X]])
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[V]], 1
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %x = or <2 x i8> %xx, <i8 5, i8 3>
+  %v = call i8 @llvm.vector.reduce.or(<2 x i8> %x)
+  %r = and i8 %v, 1
+  ret i8 %r
+}
+
+define i8 @known_reduce_or_fail(<2 x i8> %xx) {
+; CHECK-LABEL: @known_reduce_or_fail(
+; CHECK-NEXT:    [[X:%.*]] = or <2 x i8> [[XX:%.*]], <i8 5, i8 3>
+; CHECK-NEXT:    [[V:%.*]] = call i8 @llvm.vector.reduce.or.v2i8(<2 x i8> [[X]])
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[V]], 4
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %x = or <2 x i8> %xx, <i8 5, i8 3>
+  %v = call i8 @llvm.vector.reduce.or(<2 x i8> %x)
+  %r = and i8 %v, 4
+  ret i8 %r
+}
+
+define i8 @known_reduce_and(<2 x i8> %xx) {
+; CHECK-LABEL: @known_reduce_and(
+; CHECK-NEXT:    [[X:%.*]] = or <2 x i8> [[XX:%.*]], <i8 5, i8 3>
+; CHECK-NEXT:    [[V:%.*]] = call i8 @llvm.vector.reduce.or.v2i8(<2 x i8> [[X]])
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[V]], 1
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %x = or <2 x i8> %xx, <i8 5, i8 3>
+  %v = call i8 @llvm.vector.reduce.or(<2 x i8> %x)
+  %r = and i8 %v, 1
+  ret i8 %r
+}
+
+define i8 @known_reduce_and_fail(<2 x i8> %xx) {
+; CHECK-LABEL: @known_reduce_and_fail(
+; CHECK-NEXT:    [[X:%.*]] = or <2 x i8> [[XX:%.*]], <i8 5, i8 3>
+; CHECK-NEXT:    [[V:%.*]] = call i8 @llvm.vector.reduce.or.v2i8(<2 x i8> [[X]])
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[V]], 2
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %x = or <2 x i8> %xx, <i8 5, i8 3>
+  %v = call i8 @llvm.vector.reduce.or(<2 x i8> %x)
+  %r = and i8 %v, 2
+  ret i8 %r
+}
+
 declare void @use(i1)
 declare void @sink(i8)
-- 
GitLab


From 6063e3c40867e9b9865aee326ff36cb2ebbcf939 Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Wed, 10 Apr 2024 14:12:50 -0500
Subject: [PATCH 189/464] [ValueTracking] Implement `computeKnownBits` for
 `llvm.vector.reduce.{or,and}`

---
 llvm/lib/Analysis/ValueTracking.cpp            |  6 ++++--
 llvm/test/Transforms/InstCombine/known-bits.ll | 10 ++--------
 2 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index b83e7e6769c2..359a9dc7907b 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -1624,8 +1624,10 @@ static void computeKnownBitsFromOperator(const Operator *I,
         computeKnownBits(I->getOperand(1), Known2, Depth + 1, Q);
         Known = KnownBits::ssub_sat(Known, Known2);
         break;
-        // for min/max reduce, any bit common to each element in the input vec
-        // is set in the output.
+        // for min/max/and/or reduce, any bit common to each element in the
+        // input vec is set in the output.
+      case Intrinsic::vector_reduce_and:
+      case Intrinsic::vector_reduce_or:
       case Intrinsic::vector_reduce_umax:
       case Intrinsic::vector_reduce_umin:
       case Intrinsic::vector_reduce_smax:
diff --git a/llvm/test/Transforms/InstCombine/known-bits.ll b/llvm/test/Transforms/InstCombine/known-bits.ll
index cdc3e991a3fb..ce415dd406c0 100644
--- a/llvm/test/Transforms/InstCombine/known-bits.ll
+++ b/llvm/test/Transforms/InstCombine/known-bits.ll
@@ -1197,10 +1197,7 @@ define i1 @extract_value_smul_fail(i8 %xx, i8 %yy) {
 
 define i8 @known_reduce_or(<2 x i8> %xx) {
 ; CHECK-LABEL: @known_reduce_or(
-; CHECK-NEXT:    [[X:%.*]] = or <2 x i8> [[XX:%.*]], <i8 5, i8 3>
-; CHECK-NEXT:    [[V:%.*]] = call i8 @llvm.vector.reduce.or.v2i8(<2 x i8> [[X]])
-; CHECK-NEXT:    [[R:%.*]] = and i8 [[V]], 1
-; CHECK-NEXT:    ret i8 [[R]]
+; CHECK-NEXT:    ret i8 1
 ;
   %x = or <2 x i8> %xx, <i8 5, i8 3>
   %v = call i8 @llvm.vector.reduce.or(<2 x i8> %x)
@@ -1223,10 +1220,7 @@ define i8 @known_reduce_or_fail(<2 x i8> %xx) {
 
 define i8 @known_reduce_and(<2 x i8> %xx) {
 ; CHECK-LABEL: @known_reduce_and(
-; CHECK-NEXT:    [[X:%.*]] = or <2 x i8> [[XX:%.*]], <i8 5, i8 3>
-; CHECK-NEXT:    [[V:%.*]] = call i8 @llvm.vector.reduce.or.v2i8(<2 x i8> [[X]])
-; CHECK-NEXT:    [[R:%.*]] = and i8 [[V]], 1
-; CHECK-NEXT:    ret i8 [[R]]
+; CHECK-NEXT:    ret i8 1
 ;
   %x = or <2 x i8> %xx, <i8 5, i8 3>
   %v = call i8 @llvm.vector.reduce.or(<2 x i8> %x)
-- 
GitLab


From 44b1523b9a4e7e21677395e75352d9a62efe2790 Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Wed, 10 Apr 2024 15:30:11 -0500
Subject: [PATCH 190/464] [ValueTracking] Add tests for `computeKnownBits` of
 `llvm.vector.reduce.xor`; NFC

---
 .../test/Transforms/InstCombine/known-bits.ll | 151 ++++++++++++++++++
 1 file changed, 151 insertions(+)

diff --git a/llvm/test/Transforms/InstCombine/known-bits.ll b/llvm/test/Transforms/InstCombine/known-bits.ll
index ce415dd406c0..fa19adbd5580 100644
--- a/llvm/test/Transforms/InstCombine/known-bits.ll
+++ b/llvm/test/Transforms/InstCombine/known-bits.ll
@@ -1241,5 +1241,156 @@ define i8 @known_reduce_and_fail(<2 x i8> %xx) {
   ret i8 %r
 }
 
+define i8 @known_reduce_xor_even(<2 x i8> %xx) {
+; CHECK-LABEL: @known_reduce_xor_even(
+; CHECK-NEXT:    [[X:%.*]] = or <2 x i8> [[XX:%.*]], <i8 5, i8 3>
+; CHECK-NEXT:    [[V:%.*]] = call i8 @llvm.vector.reduce.xor.v2i8(<2 x i8> [[X]])
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[V]], 1
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %x = or <2 x i8> %xx, <i8 5, i8 3>
+  %v = call i8 @llvm.vector.reduce.xor(<2 x i8> %x)
+  %r = and i8 %v, 1
+  ret i8 %r
+}
+
+define i8 @known_reduce_xor_even2(<2 x i8> %xx) {
+; CHECK-LABEL: @known_reduce_xor_even2(
+; CHECK-NEXT:    [[X:%.*]] = and <2 x i8> [[XX:%.*]], <i8 15, i8 15>
+; CHECK-NEXT:    [[V:%.*]] = call i8 @llvm.vector.reduce.xor.v2i8(<2 x i8> [[X]])
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[V]], 16
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %x = and <2 x i8> %xx, <i8 15, i8 15>
+  %v = call i8 @llvm.vector.reduce.xor(<2 x i8> %x)
+  %r = and i8 %v, 16
+  ret i8 %r
+}
+
+define i8 @known_reduce_xor_even_fail(<2 x i8> %xx) {
+; CHECK-LABEL: @known_reduce_xor_even_fail(
+; CHECK-NEXT:    [[X:%.*]] = or <2 x i8> [[XX:%.*]], <i8 5, i8 3>
+; CHECK-NEXT:    [[V:%.*]] = call i8 @llvm.vector.reduce.xor.v2i8(<2 x i8> [[X]])
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[V]], 2
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %x = or <2 x i8> %xx, <i8 5, i8 3>
+  %v = call i8 @llvm.vector.reduce.xor(<2 x i8> %x)
+  %r = and i8 %v, 2
+  ret i8 %r
+}
+
+define i8 @known_reduce_xor_odd(<3 x i8> %xx) {
+; CHECK-LABEL: @known_reduce_xor_odd(
+; CHECK-NEXT:    [[X:%.*]] = or <3 x i8> [[XX:%.*]], <i8 5, i8 3, i8 9>
+; CHECK-NEXT:    [[V:%.*]] = call i8 @llvm.vector.reduce.xor.v3i8(<3 x i8> [[X]])
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[V]], 1
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %x = or <3 x i8> %xx, <i8 5, i8 3, i8 9>
+  %v = call i8 @llvm.vector.reduce.xor.v3i8(<3 x i8> %x)
+  %r = and i8 %v, 1
+  ret i8 %r
+}
+
+define i8 @known_reduce_xor_odd2(<3 x i8> %xx) {
+; CHECK-LABEL: @known_reduce_xor_odd2(
+; CHECK-NEXT:    [[X:%.*]] = and <3 x i8> [[XX:%.*]], <i8 15, i8 15, i8 31>
+; CHECK-NEXT:    [[V:%.*]] = call i8 @llvm.vector.reduce.xor.v3i8(<3 x i8> [[X]])
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[V]], 32
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %x = and <3 x i8> %xx, <i8 15, i8 15, i8 31>
+  %v = call i8 @llvm.vector.reduce.xor.v3i8(<3 x i8> %x)
+  %r = and i8 %v, 32
+  ret i8 %r
+}
+
+define i8 @known_reduce_xor_odd2_fail(<3 x i8> %xx) {
+; CHECK-LABEL: @known_reduce_xor_odd2_fail(
+; CHECK-NEXT:    [[X:%.*]] = and <3 x i8> [[XX:%.*]], <i8 15, i8 15, i8 31>
+; CHECK-NEXT:    [[V:%.*]] = call i8 @llvm.vector.reduce.xor.v3i8(<3 x i8> [[X]])
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[V]], 16
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %x = and <3 x i8> %xx, <i8 15, i8 15, i8 31>
+  %v = call i8 @llvm.vector.reduce.xor.v3i8(<3 x i8> %x)
+  %r = and i8 %v, 16
+  ret i8 %r
+}
+
+define i8 @known_reduce_xor_odd_fail(<3 x i8> %xx) {
+; CHECK-LABEL: @known_reduce_xor_odd_fail(
+; CHECK-NEXT:    [[X:%.*]] = or <3 x i8> [[XX:%.*]], <i8 5, i8 3, i8 9>
+; CHECK-NEXT:    [[V:%.*]] = call i8 @llvm.vector.reduce.xor.v3i8(<3 x i8> [[X]])
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[V]], 2
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %x = or <3 x i8> %xx, <i8 5, i8 3, i8 9>
+  %v = call i8 @llvm.vector.reduce.xor.v3i8(<3 x i8> %x)
+  %r = and i8 %v, 2
+  ret i8 %r
+}
+
+define i8 @nonzero_reduce_xor_vscale_even(<vscale x 2 x i8> %xx) {
+; CHECK-LABEL: @nonzero_reduce_xor_vscale_even(
+; CHECK-NEXT:    [[X:%.*]] = or <vscale x 2 x i8> [[XX:%.*]], shufflevector (<vscale x 2 x i8> insertelement (<vscale x 2 x i8> poison, i8 1, i64 0), <vscale x 2 x i8> poison, <vscale x 2 x i32> zeroinitializer)
+; CHECK-NEXT:    [[V:%.*]] = call i8 @llvm.vector.reduce.xor.nxv2i8(<vscale x 2 x i8> [[X]])
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[V]], 1
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %one = insertelement <vscale x 2 x i8> poison, i8 1, i64 0
+  %ones = shufflevector <vscale x 2 x i8> %one, <vscale x 2 x i8> poison, <vscale x 2 x i32> zeroinitializer
+  %x = or <vscale x 2 x i8> %xx, %ones
+  %v = call i8 @llvm.vector.reduce.xor.nxv2i8(<vscale x 2 x i8> %x)
+  %r = and i8 %v, 1
+  ret i8 %r
+}
+
+define i8 @nonzero_reduce_xor_vscale_odd_fail(<vscale x 3 x i8> %xx) {
+; CHECK-LABEL: @nonzero_reduce_xor_vscale_odd_fail(
+; CHECK-NEXT:    [[X:%.*]] = or <vscale x 3 x i8> [[XX:%.*]], shufflevector (<vscale x 3 x i8> insertelement (<vscale x 3 x i8> poison, i8 1, i64 0), <vscale x 3 x i8> poison, <vscale x 3 x i32> zeroinitializer)
+; CHECK-NEXT:    [[V:%.*]] = call i8 @llvm.vector.reduce.xor.nxv3i8(<vscale x 3 x i8> [[X]])
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[V]], 1
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %one = insertelement <vscale x 3 x i8> poison, i8 1, i64 0
+  %ones = shufflevector <vscale x 3 x i8> %one, <vscale x 3 x i8> poison, <vscale x 3 x i32> zeroinitializer
+  %x = or <vscale x 3 x i8> %xx, %ones
+  %v = call i8 @llvm.vector.reduce.xor.nxv3i8(<vscale x 3 x i8> %x)
+  %r = and i8 %v, 1
+  ret i8 %r
+}
+
+define i8 @nonzero_reduce_xor_vscale_even_fail(<vscale x 2 x i8> %xx) {
+; CHECK-LABEL: @nonzero_reduce_xor_vscale_even_fail(
+; CHECK-NEXT:    [[X:%.*]] = or <vscale x 2 x i8> [[XX:%.*]], shufflevector (<vscale x 2 x i8> insertelement (<vscale x 2 x i8> poison, i8 1, i64 0), <vscale x 2 x i8> poison, <vscale x 2 x i32> zeroinitializer)
+; CHECK-NEXT:    [[V:%.*]] = call i8 @llvm.vector.reduce.xor.nxv2i8(<vscale x 2 x i8> [[X]])
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[V]], 2
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %one = insertelement <vscale x 2 x i8> poison, i8 1, i64 0
+  %ones = shufflevector <vscale x 2 x i8> %one, <vscale x 2 x i8> poison, <vscale x 2 x i32> zeroinitializer
+  %x = or <vscale x 2 x i8> %xx, %ones
+  %v = call i8 @llvm.vector.reduce.xor.nxv2i8(<vscale x 2 x i8> %x)
+  %r = and i8 %v, 2
+  ret i8 %r
+}
+
+define i8 @nonzero_reduce_xor_vscale_odd(<vscale x 3 x i8> %xx) {
+; CHECK-LABEL: @nonzero_reduce_xor_vscale_odd(
+; CHECK-NEXT:    [[X:%.*]] = and <vscale x 3 x i8> [[XX:%.*]], shufflevector (<vscale x 3 x i8> insertelement (<vscale x 3 x i8> poison, i8 1, i64 0), <vscale x 3 x i8> poison, <vscale x 3 x i32> zeroinitializer)
+; CHECK-NEXT:    [[V:%.*]] = call i8 @llvm.vector.reduce.xor.nxv3i8(<vscale x 3 x i8> [[X]])
+; CHECK-NEXT:    [[R:%.*]] = and i8 [[V]], 2
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %one = insertelement <vscale x 3 x i8> poison, i8 1, i64 0
+  %ones = shufflevector <vscale x 3 x i8> %one, <vscale x 3 x i8> poison, <vscale x 3 x i32> zeroinitializer
+  %x = and <vscale x 3 x i8> %xx, %ones
+  %v = call i8 @llvm.vector.reduce.xor.nxv3i8(<vscale x 3 x i8> %x)
+  %r = and i8 %v, 2
+  ret i8 %r
+}
+
 declare void @use(i1)
 declare void @sink(i8)
-- 
GitLab


From 6c71707872e4d2f9aef19d0dbd502752b7fdfba0 Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Wed, 10 Apr 2024 14:15:54 -0500
Subject: [PATCH 191/464] [ValueTracking] Implement `computeKnownBits` for
 `llvm.vector.reduce.xor`

---
 llvm/lib/Analysis/ValueTracking.cpp           | 15 ++++++++++
 .../test/Transforms/InstCombine/known-bits.ll | 30 ++++---------------
 2 files changed, 21 insertions(+), 24 deletions(-)

diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 359a9dc7907b..378e4aa9974e 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -1634,6 +1634,21 @@ static void computeKnownBitsFromOperator(const Operator *I,
       case Intrinsic::vector_reduce_smin:
         computeKnownBits(I->getOperand(0), Known, Depth + 1, Q);
         break;
+      case Intrinsic::vector_reduce_xor: {
+        computeKnownBits(I->getOperand(0), Known, Depth + 1, Q);
+        // The zeros common to all vecs are zero in the output.
+        // If the number of elements is odd, then the common ones remain. If the
+        // number of elements is even, then the common ones becomes zeros.
+        auto *VecTy = cast<VectorType>(I->getOperand(0)->getType());
+        // Even, so the ones become zeros.
+        bool EvenCnt = VecTy->getElementCount().isKnownEven();
+        if (EvenCnt)
+          Known.Zero |= Known.One;
+        // Maybe even element count so need to clear ones.
+        if (VecTy->isScalableTy() || EvenCnt)
+          Known.One.clearAllBits();
+        break;
+      }
       case Intrinsic::umin:
         computeKnownBits(I->getOperand(0), Known, Depth + 1, Q);
         computeKnownBits(I->getOperand(1), Known2, Depth + 1, Q);
diff --git a/llvm/test/Transforms/InstCombine/known-bits.ll b/llvm/test/Transforms/InstCombine/known-bits.ll
index fa19adbd5580..85a21332b078 100644
--- a/llvm/test/Transforms/InstCombine/known-bits.ll
+++ b/llvm/test/Transforms/InstCombine/known-bits.ll
@@ -1243,10 +1243,7 @@ define i8 @known_reduce_and_fail(<2 x i8> %xx) {
 
 define i8 @known_reduce_xor_even(<2 x i8> %xx) {
 ; CHECK-LABEL: @known_reduce_xor_even(
-; CHECK-NEXT:    [[X:%.*]] = or <2 x i8> [[XX:%.*]], <i8 5, i8 3>
-; CHECK-NEXT:    [[V:%.*]] = call i8 @llvm.vector.reduce.xor.v2i8(<2 x i8> [[X]])
-; CHECK-NEXT:    [[R:%.*]] = and i8 [[V]], 1
-; CHECK-NEXT:    ret i8 [[R]]
+; CHECK-NEXT:    ret i8 0
 ;
   %x = or <2 x i8> %xx, <i8 5, i8 3>
   %v = call i8 @llvm.vector.reduce.xor(<2 x i8> %x)
@@ -1256,10 +1253,7 @@ define i8 @known_reduce_xor_even(<2 x i8> %xx) {
 
 define i8 @known_reduce_xor_even2(<2 x i8> %xx) {
 ; CHECK-LABEL: @known_reduce_xor_even2(
-; CHECK-NEXT:    [[X:%.*]] = and <2 x i8> [[XX:%.*]], <i8 15, i8 15>
-; CHECK-NEXT:    [[V:%.*]] = call i8 @llvm.vector.reduce.xor.v2i8(<2 x i8> [[X]])
-; CHECK-NEXT:    [[R:%.*]] = and i8 [[V]], 16
-; CHECK-NEXT:    ret i8 [[R]]
+; CHECK-NEXT:    ret i8 0
 ;
   %x = and <2 x i8> %xx, <i8 15, i8 15>
   %v = call i8 @llvm.vector.reduce.xor(<2 x i8> %x)
@@ -1282,10 +1276,7 @@ define i8 @known_reduce_xor_even_fail(<2 x i8> %xx) {
 
 define i8 @known_reduce_xor_odd(<3 x i8> %xx) {
 ; CHECK-LABEL: @known_reduce_xor_odd(
-; CHECK-NEXT:    [[X:%.*]] = or <3 x i8> [[XX:%.*]], <i8 5, i8 3, i8 9>
-; CHECK-NEXT:    [[V:%.*]] = call i8 @llvm.vector.reduce.xor.v3i8(<3 x i8> [[X]])
-; CHECK-NEXT:    [[R:%.*]] = and i8 [[V]], 1
-; CHECK-NEXT:    ret i8 [[R]]
+; CHECK-NEXT:    ret i8 1
 ;
   %x = or <3 x i8> %xx, <i8 5, i8 3, i8 9>
   %v = call i8 @llvm.vector.reduce.xor.v3i8(<3 x i8> %x)
@@ -1295,10 +1286,7 @@ define i8 @known_reduce_xor_odd(<3 x i8> %xx) {
 
 define i8 @known_reduce_xor_odd2(<3 x i8> %xx) {
 ; CHECK-LABEL: @known_reduce_xor_odd2(
-; CHECK-NEXT:    [[X:%.*]] = and <3 x i8> [[XX:%.*]], <i8 15, i8 15, i8 31>
-; CHECK-NEXT:    [[V:%.*]] = call i8 @llvm.vector.reduce.xor.v3i8(<3 x i8> [[X]])
-; CHECK-NEXT:    [[R:%.*]] = and i8 [[V]], 32
-; CHECK-NEXT:    ret i8 [[R]]
+; CHECK-NEXT:    ret i8 0
 ;
   %x = and <3 x i8> %xx, <i8 15, i8 15, i8 31>
   %v = call i8 @llvm.vector.reduce.xor.v3i8(<3 x i8> %x)
@@ -1334,10 +1322,7 @@ define i8 @known_reduce_xor_odd_fail(<3 x i8> %xx) {
 
 define i8 @nonzero_reduce_xor_vscale_even(<vscale x 2 x i8> %xx) {
 ; CHECK-LABEL: @nonzero_reduce_xor_vscale_even(
-; CHECK-NEXT:    [[X:%.*]] = or <vscale x 2 x i8> [[XX:%.*]], shufflevector (<vscale x 2 x i8> insertelement (<vscale x 2 x i8> poison, i8 1, i64 0), <vscale x 2 x i8> poison, <vscale x 2 x i32> zeroinitializer)
-; CHECK-NEXT:    [[V:%.*]] = call i8 @llvm.vector.reduce.xor.nxv2i8(<vscale x 2 x i8> [[X]])
-; CHECK-NEXT:    [[R:%.*]] = and i8 [[V]], 1
-; CHECK-NEXT:    ret i8 [[R]]
+; CHECK-NEXT:    ret i8 0
 ;
   %one = insertelement <vscale x 2 x i8> poison, i8 1, i64 0
   %ones = shufflevector <vscale x 2 x i8> %one, <vscale x 2 x i8> poison, <vscale x 2 x i32> zeroinitializer
@@ -1379,10 +1364,7 @@ define i8 @nonzero_reduce_xor_vscale_even_fail(<vscale x 2 x i8> %xx) {
 
 define i8 @nonzero_reduce_xor_vscale_odd(<vscale x 3 x i8> %xx) {
 ; CHECK-LABEL: @nonzero_reduce_xor_vscale_odd(
-; CHECK-NEXT:    [[X:%.*]] = and <vscale x 3 x i8> [[XX:%.*]], shufflevector (<vscale x 3 x i8> insertelement (<vscale x 3 x i8> poison, i8 1, i64 0), <vscale x 3 x i8> poison, <vscale x 3 x i32> zeroinitializer)
-; CHECK-NEXT:    [[V:%.*]] = call i8 @llvm.vector.reduce.xor.nxv3i8(<vscale x 3 x i8> [[X]])
-; CHECK-NEXT:    [[R:%.*]] = and i8 [[V]], 2
-; CHECK-NEXT:    ret i8 [[R]]
+; CHECK-NEXT:    ret i8 0
 ;
   %one = insertelement <vscale x 3 x i8> poison, i8 1, i64 0
   %ones = shufflevector <vscale x 3 x i8> %one, <vscale x 3 x i8> poison, <vscale x 3 x i32> zeroinitializer
-- 
GitLab


From 3921dffac29d8ea1c07bc6a7253876247a6eb1a3 Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Thu, 11 Apr 2024 11:18:33 -0500
Subject: [PATCH 192/464] [ValueTracking] Add tests for `isKnownNonZero` of
 `llvm.vector.reduce.or`; NFC

---
 .../Transforms/InstSimplify/known-non-zero.ll | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/llvm/test/Transforms/InstSimplify/known-non-zero.ll b/llvm/test/Transforms/InstSimplify/known-non-zero.ll
index d9b8f5eed323..b0b825efaeee 100644
--- a/llvm/test/Transforms/InstSimplify/known-non-zero.ll
+++ b/llvm/test/Transforms/InstSimplify/known-non-zero.ll
@@ -377,3 +377,29 @@ define <2 x i1> @insert_nonzero_any_idx_fail(<2 x i8> %xx, i8 %yy, i32 %idx) {
   %r = icmp eq <2 x i8> %ins, zeroinitializer
   ret <2 x i1> %r
 }
+
+define i1 @nonzero_reduce_or(<2 x i8> %xx) {
+; CHECK-LABEL: @nonzero_reduce_or(
+; CHECK-NEXT:    [[X:%.*]] = add nuw <2 x i8> [[XX:%.*]], <i8 1, i8 1>
+; CHECK-NEXT:    [[V:%.*]] = call i8 @llvm.vector.reduce.or.v2i8(<2 x i8> [[X]])
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[V]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %x = add nuw <2 x i8> %xx, <i8 1, i8 1>
+  %v = call i8 @llvm.vector.reduce.or(<2 x i8> %x)
+  %r = icmp eq i8 %v, 0
+  ret i1 %r
+}
+
+define i1 @nonzero_reduce_or_fail(<2 x i8> %xx) {
+; CHECK-LABEL: @nonzero_reduce_or_fail(
+; CHECK-NEXT:    [[X:%.*]] = add nsw <2 x i8> [[XX:%.*]], <i8 1, i8 1>
+; CHECK-NEXT:    [[V:%.*]] = call i8 @llvm.vector.reduce.or.v2i8(<2 x i8> [[X]])
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[V]], 0
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %x = add nsw <2 x i8> %xx, <i8 1, i8 1>
+  %v = call i8 @llvm.vector.reduce.or(<2 x i8> %x)
+  %r = icmp eq i8 %v, 0
+  ret i1 %r
+}
-- 
GitLab


From 1e16a35fbcc6b21d1bee597c7e76f2444b7000e5 Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Thu, 11 Apr 2024 11:19:15 -0500
Subject: [PATCH 193/464] [ValueTracking] Implement `isKnownNonZero` for
 `llvm.vector.reduce.or`

Closes #88320
---
 llvm/lib/Analysis/ValueTracking.cpp                 | 3 ++-
 llvm/test/Transforms/InstSimplify/known-non-zero.ll | 5 +----
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 378e4aa9974e..5c2806078261 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -2918,7 +2918,8 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
         return isNonZeroAdd(DemandedElts, Depth, Q, BitWidth,
                             II->getArgOperand(0), II->getArgOperand(1),
                             /*NSW=*/true, /* NUW=*/false);
-        // umin/smin/smax/smin of all non-zero elements is always non-zero.
+        // umin/smin/smax/smin/or of all non-zero elements is always non-zero.
+      case Intrinsic::vector_reduce_or:
       case Intrinsic::vector_reduce_umax:
       case Intrinsic::vector_reduce_umin:
       case Intrinsic::vector_reduce_smax:
diff --git a/llvm/test/Transforms/InstSimplify/known-non-zero.ll b/llvm/test/Transforms/InstSimplify/known-non-zero.ll
index b0b825efaeee..fd2862eb04a2 100644
--- a/llvm/test/Transforms/InstSimplify/known-non-zero.ll
+++ b/llvm/test/Transforms/InstSimplify/known-non-zero.ll
@@ -380,10 +380,7 @@ define <2 x i1> @insert_nonzero_any_idx_fail(<2 x i8> %xx, i8 %yy, i32 %idx) {
 
 define i1 @nonzero_reduce_or(<2 x i8> %xx) {
 ; CHECK-LABEL: @nonzero_reduce_or(
-; CHECK-NEXT:    [[X:%.*]] = add nuw <2 x i8> [[XX:%.*]], <i8 1, i8 1>
-; CHECK-NEXT:    [[V:%.*]] = call i8 @llvm.vector.reduce.or.v2i8(<2 x i8> [[X]])
-; CHECK-NEXT:    [[R:%.*]] = icmp eq i8 [[V]], 0
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    ret i1 false
 ;
   %x = add nuw <2 x i8> %xx, <i8 1, i8 1>
   %v = call i8 @llvm.vector.reduce.or(<2 x i8> %x)
-- 
GitLab


From f811d7b50957b801788d7b171ddeb25b1fda415a Mon Sep 17 00:00:00 2001
From: Chuanqi Xu <yedeng.yd@linux.alibaba.com>
Date: Mon, 15 Apr 2024 11:55:45 +0800
Subject: [PATCH 194/464] [C++20] [Modules] Introduce
 -fexperimental-modules-reduced-bmi (#85050)

This is the driver part of
https://github.com/llvm/llvm-project/pull/75894.

This patch introduces '-fexperimental-modules-reduced-bmi' to enable
generating the reduced BMI.

This patch did:
- When `-fexperimental-modules-reduced-bmi` is specified but
`--precompile` is not specified for a module unit, we'll skip the
precompile phase to avoid unnecessary two-phase compilation phases. Then
if `-c` is specified, we will generate the reduced BMI in CodeGenAction
as a by-product.
- When `-fexperimental-modules-reduced-bmi` is specified and
`--precompile` is specified, we will generate the reduced BMI in
GenerateModuleInterfaceAction as a by-product.
- When `-fexperimental-modules-reduced-bmi` is specified for a
non-module unit. We don't do anything nor try to give a warn. This is
more user friendly so that the end users can try to test and experiment
with the feature without asking help from the build systems.

The core design idea is that users should be able to enable this easily
with the existing cmake mechanisms.

The future plan for the flag is:
- Add this to clang19 and make it opt-in for 1~2 releases. It depends on
the testing feedback to decide how long we like to make it opt-in.
- Then we can announce the existing BMI generating may be deprecated and
suggesting people (end users or build systems) to enable this for 1~2
releases.
- Finally we will enable this by default. When that time comes, the term
`BMI` will refer to the reduced BMI today and the existing BMI will only
be meaningful to build systems which loves to support two phase
compilations.

I'll send release notes and document in seperate commits after this get
landed.
---
 clang/include/clang/CodeGen/CodeGenAction.h   |  2 +
 clang/include/clang/Driver/Options.td         |  6 +++
 .../include/clang/Frontend/FrontendOptions.h  | 10 +++-
 clang/lib/CodeGen/CodeGenAction.cpp           | 19 +++++++
 clang/lib/Driver/Driver.cpp                   | 12 ++++-
 clang/lib/Driver/ToolChains/Clang.cpp         | 18 +++++++
 clang/lib/Frontend/FrontendActions.cpp        |  7 +++
 .../test/Driver/module-fgen-reduced-bmi.cppm  | 51 +++++++++++++++++++
 clang/test/Modules/modules-reduced-bmi.cppm   | 36 +++++++++++++
 9 files changed, 159 insertions(+), 2 deletions(-)
 create mode 100644 clang/test/Driver/module-fgen-reduced-bmi.cppm
 create mode 100644 clang/test/Modules/modules-reduced-bmi.cppm

diff --git a/clang/include/clang/CodeGen/CodeGenAction.h b/clang/include/clang/CodeGen/CodeGenAction.h
index 7ad2988e589e..186dbb43f01e 100644
--- a/clang/include/clang/CodeGen/CodeGenAction.h
+++ b/clang/include/clang/CodeGen/CodeGenAction.h
@@ -57,6 +57,8 @@ private:
   bool loadLinkModules(CompilerInstance &CI);
 
 protected:
+  bool BeginSourceFileAction(CompilerInstance &CI) override;
+
   /// Create a new code generation action.  If the optional \p _VMContext
   /// parameter is supplied, the action uses it without taking ownership,
   /// otherwise it creates a fresh LLVM context and takes ownership.
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 9a0b5d3304ca..e24626913add 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -3036,6 +3036,7 @@ defm prebuilt_implicit_modules : BoolFOption<"prebuilt-implicit-modules",
 
 def fmodule_output_EQ : Joined<["-"], "fmodule-output=">,
   Flags<[NoXarchOption]>, Visibility<[ClangOption, CC1Option]>,
+  MarshallingInfoString<FrontendOpts<"ModuleOutputPath">>,
   HelpText<"Save intermediate module file results when compiling a standard C++ module unit.">;
 def fmodule_output : Flag<["-"], "fmodule-output">, Flags<[NoXarchOption]>,
   Visibility<[ClangOption, CC1Option]>,
@@ -3049,6 +3050,11 @@ defm skip_odr_check_in_gmf : BoolOption<"f", "skip-odr-check-in-gmf",
           "Perform ODR checks for decls in the global module fragment.">>,
   Group<f_Group>;
 
+def modules_reduced_bmi : Flag<["-"], "fexperimental-modules-reduced-bmi">,
+  Group<f_Group>, Visibility<[ClangOption, CC1Option]>,
+  HelpText<"Generate the reduced BMI">,
+  MarshallingInfoFlag<FrontendOpts<"GenReducedBMI">>;
+
 def fmodules_prune_interval : Joined<["-"], "fmodules-prune-interval=">, Group<i_Group>,
   Visibility<[ClangOption, CC1Option]>, MetaVarName<"<seconds>">,
   HelpText<"Specify the interval (in seconds) between attempts to prune the module cache">,
diff --git a/clang/include/clang/Frontend/FrontendOptions.h b/clang/include/clang/Frontend/FrontendOptions.h
index 5ee4d471670f..a738c1f37576 100644
--- a/clang/include/clang/Frontend/FrontendOptions.h
+++ b/clang/include/clang/Frontend/FrontendOptions.h
@@ -404,6 +404,10 @@ public:
   LLVM_PREFERRED_TYPE(bool)
   unsigned EmitPrettySymbolGraphs : 1;
 
+  /// Whether to generate reduced BMI for C++20 named modules.
+  LLVM_PREFERRED_TYPE(bool)
+  unsigned GenReducedBMI : 1;
+
   CodeCompleteOptions CodeCompleteOpts;
 
   /// Specifies the output format of the AST.
@@ -568,6 +572,9 @@ public:
   /// Path which stores the output files for -ftime-trace
   std::string TimeTracePath;
 
+  /// Output Path for module output file.
+  std::string ModuleOutputPath;
+
 public:
   FrontendOptions()
       : DisableFree(false), RelocatablePCH(false), ShowHelp(false),
@@ -582,7 +589,8 @@ public:
         AllowPCMWithCompilerErrors(false), ModulesShareFileManager(true),
         EmitSymbolGraph(false), EmitExtensionSymbolGraphs(false),
         EmitSymbolGraphSymbolLabelsForTesting(false),
-        EmitPrettySymbolGraphs(false), TimeTraceGranularity(500) {}
+        EmitPrettySymbolGraphs(false), GenReducedBMI(false),
+        TimeTraceGranularity(500) {}
 
   /// getInputKindForExtension - Return the appropriate input kind for a file
   /// extension. For example, "c" would return Language::C.
diff --git a/clang/lib/CodeGen/CodeGenAction.cpp b/clang/lib/CodeGen/CodeGenAction.cpp
index bb9aaba025fa..1a6b628016f7 100644
--- a/clang/lib/CodeGen/CodeGenAction.cpp
+++ b/clang/lib/CodeGen/CodeGenAction.cpp
@@ -25,8 +25,11 @@
 #include "clang/CodeGen/ModuleBuilder.h"
 #include "clang/Driver/DriverDiagnostic.h"
 #include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/FrontendActions.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
+#include "clang/Frontend/MultiplexConsumer.h"
 #include "clang/Lex/Preprocessor.h"
+#include "clang/Serialization/ASTWriter.h"
 #include "llvm/ADT/Hashing.h"
 #include "llvm/Bitcode/BitcodeReader.h"
 #include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
@@ -1003,6 +1006,12 @@ CodeGenerator *CodeGenAction::getCodeGenerator() const {
   return BEConsumer->getCodeGenerator();
 }
 
+bool CodeGenAction::BeginSourceFileAction(CompilerInstance &CI) {
+  if (CI.getFrontendOpts().GenReducedBMI)
+    CI.getLangOpts().setCompilingModule(LangOptions::CMK_ModuleInterface);
+  return true;
+}
+
 static std::unique_ptr<raw_pwrite_stream>
 GetOutputStream(CompilerInstance &CI, StringRef InFile, BackendAction Action) {
   switch (Action) {
@@ -1061,6 +1070,16 @@ CodeGenAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
     CI.getPreprocessor().addPPCallbacks(std::move(Callbacks));
   }
 
+  if (CI.getFrontendOpts().GenReducedBMI &&
+      !CI.getFrontendOpts().ModuleOutputPath.empty()) {
+    std::vector<std::unique_ptr<ASTConsumer>> Consumers(2);
+    Consumers[0] = std::make_unique<ReducedBMIGenerator>(
+        CI.getPreprocessor(), CI.getModuleCache(),
+        CI.getFrontendOpts().ModuleOutputPath);
+    Consumers[1] = std::move(Result);
+    return std::make_unique<MultiplexConsumer>(std::move(Consumers));
+  }
+
   return std::move(Result);
 }
 
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index e7335a61b10c..0da92001e08c 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -4756,6 +4756,14 @@ Action *Driver::ConstructPhaseAction(
     if (Args.hasArg(options::OPT_extract_api))
       return C.MakeAction<ExtractAPIJobAction>(Input, types::TY_API_INFO);
 
+    // With 'fexperimental-modules-reduced-bmi', we don't want to run the
+    // precompile phase unless the user specified '--precompile'. In the case
+    // the '--precompile' flag is enabled, we will try to emit the reduced BMI
+    // as a by product in GenerateModuleInterfaceAction.
+    if (Args.hasArg(options::OPT_modules_reduced_bmi) &&
+        !Args.getLastArg(options::OPT__precompile))
+      return Input;
+
     types::ID OutputTy = getPrecompiledType(Input->getType());
     assert(OutputTy != types::TY_INVALID &&
            "Cannot precompile this input type!");
@@ -5916,8 +5924,10 @@ const char *Driver::GetNamedOutputPath(Compilation &C, const JobAction &JA,
   // If we're emitting a module output with the specified option
   // `-fmodule-output`.
   if (!AtTopLevel && isa<PrecompileJobAction>(JA) &&
-      JA.getType() == types::TY_ModuleFile && SpecifiedModuleOutput)
+      JA.getType() == types::TY_ModuleFile && SpecifiedModuleOutput) {
+    assert(!C.getArgs().hasArg(options::OPT_modules_reduced_bmi));
     return GetModuleOutputPath(C, JA, BaseInput);
+  }
 
   // Output to a temporary file?
   if ((!AtTopLevel && !isSaveTempsEnabled() &&
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 766a9b91e3c0..6d52eced1042 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -4045,6 +4045,24 @@ static bool RenderModulesOptions(Compilation &C, const Driver &D,
   // module fragment.
   CmdArgs.push_back("-fskip-odr-check-in-gmf");
 
+  if (Args.hasArg(options::OPT_modules_reduced_bmi) &&
+      (Input.getType() == driver::types::TY_CXXModule ||
+       Input.getType() == driver::types::TY_PP_CXXModule)) {
+    CmdArgs.push_back("-fexperimental-modules-reduced-bmi");
+
+    if (Args.hasArg(options::OPT_fmodule_output_EQ))
+      Args.AddLastArg(CmdArgs, options::OPT_fmodule_output_EQ);
+    else
+      CmdArgs.push_back(Args.MakeArgString(
+          "-fmodule-output=" +
+          getCXX20NamedModuleOutputPath(Args, Input.getBaseInput())));
+  }
+
+  // Noop if we see '-fexperimental-modules-reduced-bmi' with other translation
+  // units than module units. This is more user friendly to allow end uers to
+  // enable this feature without asking for help from build systems.
+  Args.ClaimAllArgs(options::OPT_modules_reduced_bmi);
+
   // We need to include the case the input file is a module file here.
   // Since the default compilation model for C++ module interface unit will
   // create temporary module file and compile the temporary module file
diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp
index 642b14d8b09d..04eb10413267 100644
--- a/clang/lib/Frontend/FrontendActions.cpp
+++ b/clang/lib/Frontend/FrontendActions.cpp
@@ -281,6 +281,13 @@ GenerateModuleInterfaceAction::CreateASTConsumer(CompilerInstance &CI,
   if (Consumers.empty())
     return nullptr;
 
+  if (CI.getFrontendOpts().GenReducedBMI &&
+      !CI.getFrontendOpts().ModuleOutputPath.empty()) {
+    Consumers.push_back(std::make_unique<ReducedBMIGenerator>(
+        CI.getPreprocessor(), CI.getModuleCache(),
+        CI.getFrontendOpts().ModuleOutputPath));
+  }
+
   return std::make_unique<MultiplexConsumer>(std::move(Consumers));
 }
 
diff --git a/clang/test/Driver/module-fgen-reduced-bmi.cppm b/clang/test/Driver/module-fgen-reduced-bmi.cppm
new file mode 100644
index 000000000000..1223189fb49b
--- /dev/null
+++ b/clang/test/Driver/module-fgen-reduced-bmi.cppm
@@ -0,0 +1,51 @@
+// It is annoying to handle different slash direction
+// in Windows and Linux. So we disable the test on Windows
+// here.
+// REQUIRES: !system-windows
+// On AIX, the default output for `-c` may be `.s` instead of `.o`,
+// which makes the test fail. So disable the test on AIX.
+// UNSUPPORTED: system-aix
+//
+// RUN: rm -rf %t && split-file %s %t && cd %t
+//
+// RUN: %clang -std=c++20 Hello.cppm -fmodule-output=Hello.pcm \
+// RUN:     -fexperimental-modules-reduced-bmi -c -o Hello.o -### 2>&1 | FileCheck Hello.cppm
+//
+// RUN: %clang -std=c++20 Hello.cppm \
+// RUN:     -fexperimental-modules-reduced-bmi -c -o Hello.o -### 2>&1 | \
+// RUN:         FileCheck Hello.cppm --check-prefix=CHECK-UNSPECIFIED
+//
+// RUN: %clang -std=c++20 Hello.cppm \
+// RUN:     -fexperimental-modules-reduced-bmi -c -### 2>&1 | \
+// RUN:         FileCheck Hello.cppm --check-prefix=CHECK-NO-O
+//
+// RUN: %clang -std=c++20 Hello.cppm \
+// RUN:     -fexperimental-modules-reduced-bmi -c -o AnotherName.o -### 2>&1 | \
+// RUN:         FileCheck Hello.cppm --check-prefix=CHECK-ANOTHER-NAME
+//
+// RUN: %clang -std=c++20 Hello.cppm --precompile -fexperimental-modules-reduced-bmi \
+// RUN:     -o Hello.full.pcm -### 2>&1 | FileCheck Hello.cppm \
+// RUN:     --check-prefix=CHECK-EMIT-MODULE-INTERFACE
+//
+// RUN: %clang -std=c++20 Hello.cc -fexperimental-modules-reduced-bmi -Wall -Werror \
+// RUN:     -c -o Hello.o -### 2>&1 | FileCheck Hello.cc
+
+//--- Hello.cppm
+export module Hello;
+
+// Test that we won't generate the emit-module-interface as 2 phase compilation model.
+// CHECK-NOT: -emit-module-interface
+// CHECK: "-fexperimental-modules-reduced-bmi"
+
+// CHECK-UNSPECIFIED: -fmodule-output=Hello.pcm
+
+// CHECK-NO-O: -fmodule-output=Hello.pcm
+// CHECK-ANOTHER-NAME: -fmodule-output=AnotherName.pcm
+
+// With `-emit-module-interface` specified, we should still see the `-emit-module-interface`
+// flag.
+// CHECK-EMIT-MODULE-INTERFACE: -emit-module-interface
+
+//--- Hello.cc
+
+// CHECK-NOT: "-fexperimental-modules-reduced-bmi"
diff --git a/clang/test/Modules/modules-reduced-bmi.cppm b/clang/test/Modules/modules-reduced-bmi.cppm
new file mode 100644
index 000000000000..9b84220ae030
--- /dev/null
+++ b/clang/test/Modules/modules-reduced-bmi.cppm
@@ -0,0 +1,36 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+//
+// RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-reduced-module-interface -o %t/a.reduced.pcm
+// RUN: %clang_cc1 -std=c++20 %t/a.cppm -fexperimental-modules-reduced-bmi -fmodule-output=%t/a.pcm \
+// RUN:     -S -emit-llvm -o %t/a.ll
+//
+// Test that the generated BMI from `-fexperimental-modules-reduced-bmi -fmodule-output=` is same with
+// `-emit-reduced-module-interface`.
+// RUN: diff %t/a.reduced.pcm %t/a.pcm
+//
+// Test that we can consume the produced BMI correctly.
+// RUN: %clang_cc1 -std=c++20 %t/b.cppm -fmodule-file=a=%t/a.pcm -fsyntax-only -verify
+//
+// RUN: rm -f %t/a.pcm
+// RUN: %clang_cc1 -std=c++20 %t/a.cppm -fexperimental-modules-reduced-bmi -fmodule-output=%t/a.pcm \
+// RUN:     -emit-module-interface -o %t/a.full.pcm
+// RUN: diff %t/a.reduced.pcm %t/a.pcm
+// RUN: not diff %t/a.pcm %t/a.full.pcm
+//
+// RUN: %clang_cc1 -std=c++20 %t/b.cppm -fmodule-file=a=%t/a.pcm -fsyntax-only -verify
+// RUN: %clang_cc1 -std=c++20 %t/b.cppm -fmodule-file=a=%t/a.full.pcm -fsyntax-only -verify
+
+//--- a.cppm
+export module a;
+export int a() {
+    return 43;
+}
+
+//--- b.cppm
+// Test that we can consume the produced BMI correctly as a smocking test.
+// expected-no-diagnostics
+export module b;
+import a;
+export int b() { return a(); }
-- 
GitLab


From c7902d87a5a050b816edfe99e7e093ae63f4e564 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Mon, 15 Apr 2024 13:33:29 +0900
Subject: [PATCH 195/464] [InstCombine] Add test for isKnownNonZero() undef
 miscompile (NFC)

---
 llvm/test/Transforms/InstCombine/add.ll | 26 +++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/llvm/test/Transforms/InstCombine/add.ll b/llvm/test/Transforms/InstCombine/add.ll
index 23eee8547597..515b80d9d6ea 100644
--- a/llvm/test/Transforms/InstCombine/add.ll
+++ b/llvm/test/Transforms/InstCombine/add.ll
@@ -3093,6 +3093,32 @@ define <2 x i32> @dec_zext_add_nonzero_vec(<2 x i8> %x) {
   ret <2 x i32> %c
 }
 
+define <2 x i32> @dec_zext_add_nonzero_vec_undef0(<2 x i8> %x) {
+; CHECK-LABEL: @dec_zext_add_nonzero_vec_undef0(
+; CHECK-NEXT:    [[O:%.*]] = or <2 x i8> [[X:%.*]], <i8 8, i8 undef>
+; CHECK-NEXT:    [[C:%.*]] = zext <2 x i8> [[O]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[C]]
+;
+  %o = or <2 x i8> %x, <i8 8, i8 undef>
+  %a = add <2 x i8> %o, <i8 -1, i8 -1>
+  %b = zext <2 x i8> %a to <2 x i32>
+  %c = add <2 x i32> %b, <i32 1, i32 1>
+  ret <2 x i32> %c
+}
+
+define <2 x i32> @dec_zext_add_nonzero_poison0(<2 x i8> %x) {
+; CHECK-LABEL: @dec_zext_add_nonzero_poison0(
+; CHECK-NEXT:    [[O:%.*]] = or <2 x i8> [[X:%.*]], <i8 8, i8 poison>
+; CHECK-NEXT:    [[C:%.*]] = zext <2 x i8> [[O]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[C]]
+;
+  %o = or <2 x i8> %x, <i8 8, i8 poison>
+  %a = add <2 x i8> %o, <i8 -1, i8 -1>
+  %b = zext <2 x i8> %a to <2 x i32>
+  %c = add <2 x i32> %b, <i32 1, i32 1>
+  ret <2 x i32> %c
+}
+
 define <2 x i32> @dec_zext_add_nonzero_vec_poison1(<2 x i8> %x) {
 ; CHECK-LABEL: @dec_zext_add_nonzero_vec_poison1(
 ; CHECK-NEXT:    [[O:%.*]] = or <2 x i8> [[X:%.*]], <i8 8, i8 8>
-- 
GitLab


From 52a1998f15ab0e5b9ff7afa8b92cc714463d5dd8 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Mon, 15 Apr 2024 13:36:57 +0900
Subject: [PATCH 196/464] [ValueTracking] Don't accept undef in
 isKnownNonZero()

As the undef can be replaced with a zero value, this is not legal
in the general case. We can only allow poison values. This matches
what the other ValueTracking helpers like computeKnownBits() do.
---
 llvm/lib/Analysis/ValueTracking.cpp          |  4 ++--
 llvm/test/Transforms/InstCombine/add.ll      |  6 +++++-
 llvm/test/Transforms/InstSimplify/vec-cmp.ll | 17 ++++++++++++++---
 3 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 5c2806078261..5beea614e332 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -3034,7 +3034,7 @@ bool isKnownNonZero(const Value *V, const APInt &DemandedElts, unsigned Depth,
       // Must be non-zero due to null test above.
       return true;
 
-    // For constant vectors, check that all elements are undefined or known
+    // For constant vectors, check that all elements are poison or known
     // non-zero to determine that the whole vector is known non-zero.
     if (auto *VecTy = dyn_cast<FixedVectorType>(Ty)) {
       for (unsigned i = 0, e = VecTy->getNumElements(); i != e; ++i) {
@@ -3043,7 +3043,7 @@ bool isKnownNonZero(const Value *V, const APInt &DemandedElts, unsigned Depth,
         Constant *Elt = C->getAggregateElement(i);
         if (!Elt || Elt->isNullValue())
           return false;
-        if (!isa<UndefValue>(Elt) && !isa<ConstantInt>(Elt))
+        if (!isa<PoisonValue>(Elt) && !isa<ConstantInt>(Elt))
           return false;
       }
       return true;
diff --git a/llvm/test/Transforms/InstCombine/add.ll b/llvm/test/Transforms/InstCombine/add.ll
index 515b80d9d6ea..408b0c6559b0 100644
--- a/llvm/test/Transforms/InstCombine/add.ll
+++ b/llvm/test/Transforms/InstCombine/add.ll
@@ -3093,10 +3093,14 @@ define <2 x i32> @dec_zext_add_nonzero_vec(<2 x i8> %x) {
   ret <2 x i32> %c
 }
 
+; Negative test: Folding this with undef is not safe.
+
 define <2 x i32> @dec_zext_add_nonzero_vec_undef0(<2 x i8> %x) {
 ; CHECK-LABEL: @dec_zext_add_nonzero_vec_undef0(
 ; CHECK-NEXT:    [[O:%.*]] = or <2 x i8> [[X:%.*]], <i8 8, i8 undef>
-; CHECK-NEXT:    [[C:%.*]] = zext <2 x i8> [[O]] to <2 x i32>
+; CHECK-NEXT:    [[A:%.*]] = add <2 x i8> [[O]], <i8 -1, i8 -1>
+; CHECK-NEXT:    [[B:%.*]] = zext <2 x i8> [[A]] to <2 x i32>
+; CHECK-NEXT:    [[C:%.*]] = add nuw nsw <2 x i32> [[B]], <i32 1, i32 1>
 ; CHECK-NEXT:    ret <2 x i32> [[C]]
 ;
   %o = or <2 x i8> %x, <i8 8, i8 undef>
diff --git a/llvm/test/Transforms/InstSimplify/vec-cmp.ll b/llvm/test/Transforms/InstSimplify/vec-cmp.ll
index 6391e9aaa046..dce1261d7031 100644
--- a/llvm/test/Transforms/InstSimplify/vec-cmp.ll
+++ b/llvm/test/Transforms/InstSimplify/vec-cmp.ll
@@ -21,13 +21,24 @@ define <2 x i1> @nonzero_vec_nonsplat(<2 x i32> %x) {
 
 define <2 x i1> @nonzero_vec_undef_elt(<2 x i32> %x) {
 ; CHECK-LABEL: @nonzero_vec_undef_elt(
-; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+; CHECK-NEXT:    [[Y:%.*]] = or <2 x i32> [[X:%.*]], <i32 undef, i32 1>
+; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x i32> [[Y]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[C]]
 ;
   %y = or <2 x i32> %x, <i32 undef, i32 1>
   %c = icmp eq <2 x i32> %y, zeroinitializer
   ret <2 x i1> %c
 }
 
+define <2 x i1> @nonzero_vec_poison_elt(<2 x i32> %x) {
+; CHECK-LABEL: @nonzero_vec_poison_elt(
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
+;
+  %y = or <2 x i32> %x, <i32 poison, i32 1>
+  %c = icmp eq <2 x i32> %y, zeroinitializer
+  ret <2 x i1> %c
+}
+
 define <2 x i1> @may_be_zero_vec(<2 x i32> %x) {
 ; CHECK-LABEL: @may_be_zero_vec(
 ; CHECK-NEXT:    [[Y:%.*]] = or <2 x i32> [[X:%.*]], <i32 0, i32 1>
@@ -45,7 +56,7 @@ define <2 x i1> @nonzero_vec_mul_nuw(<2 x i32> %x, <2 x i32> %y) {
 ; CHECK-NEXT:    ret <2 x i1> zeroinitializer
 ;
   %xnz = or <2 x i32> %x, <i32 1, i32 2>
-  %ynz = or <2 x i32> %y, <i32 3, i32 undef>
+  %ynz = or <2 x i32> %y, <i32 3, i32 poison>
   %m = mul nuw <2 x i32> %xnz, %ynz
   %c = icmp eq <2 x i32> %m, zeroinitializer
   ret <2 x i1> %c
@@ -56,7 +67,7 @@ define <2 x i1> @nonzero_vec_mul_nsw(<2 x i32> %x, <2 x i32> %y) {
 ; CHECK-LABEL: @nonzero_vec_mul_nsw(
 ; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
 ;
-  %xnz = or <2 x i32> %x, <i32 undef, i32 2>
+  %xnz = or <2 x i32> %x, <i32 poison, i32 2>
   %ynz = or <2 x i32> %y, <i32 3, i32 4>
   %m = mul nsw <2 x i32> %xnz, %ynz
   %c = icmp ne <2 x i32> %m, zeroinitializer
-- 
GitLab


From 7177dc2ef7f3a50a1d8b892d7bd298f3d52a1aab Mon Sep 17 00:00:00 2001
From: fengfeng <153487255+fengfeng09@users.noreply.github.com>
Date: Mon, 15 Apr 2024 12:59:48 +0800
Subject: [PATCH 197/464] [SDAG] Apply or-disjoint in
 SelectionDAG::isBaseWithConstantOffset (#88493)

Signed-off-by: feng.feng <feng.feng@iluvatar.com>
---
 llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 11 ++---------
 .../AVR/base-with-add-like-constant-offset.ll  | 18 ++++++++++++++++++
 2 files changed, 20 insertions(+), 9 deletions(-)
 create mode 100644 llvm/test/CodeGen/AVR/base-with-add-like-constant-offset.ll

diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 412e1de9fc41..fc972664e5f0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -5203,15 +5203,8 @@ bool SelectionDAG::isADDLike(SDValue Op) const {
 }
 
 bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const {
-  if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) ||
-      !isa<ConstantSDNode>(Op.getOperand(1)))
-    return false;
-
-  if (Op.getOpcode() == ISD::OR &&
-      !MaskedValueIsZero(Op.getOperand(0), Op.getConstantOperandAPInt(1)))
-    return false;
-
-  return true;
+  return Op.getNumOperands() == 2 && isa<ConstantSDNode>(Op.getOperand(1)) &&
+         (Op.getOpcode() == ISD::ADD || isADDLike(Op));
 }
 
 bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const {
diff --git a/llvm/test/CodeGen/AVR/base-with-add-like-constant-offset.ll b/llvm/test/CodeGen/AVR/base-with-add-like-constant-offset.ll
new file mode 100644
index 000000000000..278dc4893a68
--- /dev/null
+++ b/llvm/test/CodeGen/AVR/base-with-add-like-constant-offset.ll
@@ -0,0 +1,18 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=avr %s -start-before=avr-isel -o - | FileCheck %s
+
+define void @test(i16 %x, ptr addrspace(1) %o) {
+; CHECK-LABEL: test:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    mov r30, r22
+; CHECK-NEXT:    mov r31, r23
+; CHECK-NEXT:    std Z+11, r25
+; CHECK-NEXT:    std Z+10, r24
+; CHECK-NEXT:    ret
+  %int = ptrtoint ptr addrspace(1) %o to i16
+  %or = or disjoint i16 %int, 10
+  %addr = inttoptr i16 %or to ptr addrspace(1)
+  store i16 %x, ptr addrspace(1) %addr
+  ret void
+}
+
-- 
GitLab


From 6b80e2fef5b0e99f81d1f4a7322b110f9ead3521 Mon Sep 17 00:00:00 2001
From: laichunfeng <laichunfeng@tencent.com>
Date: Mon, 15 Apr 2024 13:11:45 +0800
Subject: [PATCH 198/464] [mi-sched] Suppress register pressure with i64.
 (#88256)

Machine scheduler will suppress register pressure when the scheduling
window is too small, but now it doesn't consider i64 register type,
and this MR extends it into i64 register type, so architecture like
RISCV64 that only supports i64 interger register will have the same
behavior like RISCV32.
---
 llvm/lib/CodeGen/MachineScheduler.cpp         |   6 +-
 .../LoongArch/atomicrmw-uinc-udec-wrap.ll     |  16 +-
 llvm/test/CodeGen/LoongArch/vector-fp-imm.ll  |  18 +-
 .../CodeGen/RISCV/atomicrmw-uinc-udec-wrap.ll |   8 +-
 llvm/test/CodeGen/RISCV/bfloat-convert.ll     |   6 +-
 .../calling-conv-lp64-lp64f-lp64d-common.ll   |   6 +-
 llvm/test/CodeGen/RISCV/double-convert.ll     |  28 ++--
 llvm/test/CodeGen/RISCV/float-convert.ll      |  16 +-
 llvm/test/CodeGen/RISCV/half-convert.ll       | 132 +++++++--------
 .../RISCV/rvv/fixed-vectors-fp-shuffles.ll    |  46 ++---
 .../rvv/fixed-vectors-insert-subvector.ll     |   4 +-
 .../RISCV/rvv/fixed-vectors-setcc-fp-vp.ll    | 157 ++++++------------
 .../RISCV/rvv/fixed-vectors-setcc-int-vp.ll   | 154 ++++++-----------
 .../CodeGen/RISCV/rvv/vpscatter-sdnode.ll     |  24 +--
 14 files changed, 255 insertions(+), 366 deletions(-)

diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp
index 203bdffc49ae..78d581c8cead 100644
--- a/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -3281,14 +3281,16 @@ void GenericScheduler::initPolicy(MachineBasicBlock::iterator Begin,
 
   // Avoid setting up the register pressure tracker for small regions to save
   // compile time. As a rough heuristic, only track pressure when the number of
-  // schedulable instructions exceeds half the integer register file.
+  // schedulable instructions exceeds half the allocatable integer register file
+  // that is the largest legal integer regiser type.
   RegionPolicy.ShouldTrackPressure = true;
-  for (unsigned VT = MVT::i32; VT > (unsigned)MVT::i1; --VT) {
+  for (unsigned VT = MVT::i64; VT > (unsigned)MVT::i1; --VT) {
     MVT::SimpleValueType LegalIntVT = (MVT::SimpleValueType)VT;
     if (TLI->isTypeLegal(LegalIntVT)) {
       unsigned NIntRegs = Context->RegClassInfo->getNumAllocatableRegs(
         TLI->getRegClassFor(LegalIntVT));
       RegionPolicy.ShouldTrackPressure = NumRegionInstrs > (NIntRegs / 2);
+      break;
     }
   }
 
diff --git a/llvm/test/CodeGen/LoongArch/atomicrmw-uinc-udec-wrap.ll b/llvm/test/CodeGen/LoongArch/atomicrmw-uinc-udec-wrap.ll
index bf48c0df3e49..7cde034726e0 100644
--- a/llvm/test/CodeGen/LoongArch/atomicrmw-uinc-udec-wrap.ll
+++ b/llvm/test/CodeGen/LoongArch/atomicrmw-uinc-udec-wrap.ll
@@ -4,12 +4,12 @@
 define i8 @atomicrmw_uinc_wrap_i8(ptr %ptr, i8 %val) {
 ; LA64-LABEL: atomicrmw_uinc_wrap_i8:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    slli.d $a4, $a0, 3
 ; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
-; LA64-NEXT:    ori $a3, $zero, 255
-; LA64-NEXT:    sll.w $a4, $a3, $a2
+; LA64-NEXT:    andi $a2, $a4, 24
+; LA64-NEXT:    ori $a5, $zero, 255
 ; LA64-NEXT:    ld.w $a3, $a0, 0
-; LA64-NEXT:    andi $a2, $a2, 24
+; LA64-NEXT:    sll.w $a4, $a5, $a4
 ; LA64-NEXT:    nor $a4, $a4, $zero
 ; LA64-NEXT:    andi $a1, $a1, 255
 ; LA64-NEXT:    .p2align 4, , 16
@@ -54,13 +54,13 @@ define i8 @atomicrmw_uinc_wrap_i8(ptr %ptr, i8 %val) {
 define i16 @atomicrmw_uinc_wrap_i16(ptr %ptr, i16 %val) {
 ; LA64-LABEL: atomicrmw_uinc_wrap_i16:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    slli.d $a2, $a0, 3
+; LA64-NEXT:    slli.d $a4, $a0, 3
 ; LA64-NEXT:    bstrins.d $a0, $zero, 1, 0
+; LA64-NEXT:    andi $a2, $a4, 24
 ; LA64-NEXT:    lu12i.w $a3, 15
-; LA64-NEXT:    ori $a3, $a3, 4095
-; LA64-NEXT:    sll.w $a4, $a3, $a2
+; LA64-NEXT:    ori $a5, $a3, 4095
 ; LA64-NEXT:    ld.w $a3, $a0, 0
-; LA64-NEXT:    andi $a2, $a2, 24
+; LA64-NEXT:    sll.w $a4, $a5, $a4
 ; LA64-NEXT:    nor $a4, $a4, $zero
 ; LA64-NEXT:    bstrpick.d $a1, $a1, 15, 0
 ; LA64-NEXT:    .p2align 4, , 16
diff --git a/llvm/test/CodeGen/LoongArch/vector-fp-imm.ll b/llvm/test/CodeGen/LoongArch/vector-fp-imm.ll
index d03af114bcee..18d17751a771 100644
--- a/llvm/test/CodeGen/LoongArch/vector-fp-imm.ll
+++ b/llvm/test/CodeGen/LoongArch/vector-fp-imm.ll
@@ -124,10 +124,10 @@ define void @test_f2(ptr %P, ptr %S) nounwind {
 ; LA64F:       # %bb.0:
 ; LA64F-NEXT:    fld.s $fa0, $a0, 4
 ; LA64F-NEXT:    fld.s $fa1, $a0, 0
-; LA64F-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI1_0)
-; LA64F-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI1_0)
-; LA64F-NEXT:    fld.s $fa2, $a0, 0
 ; LA64F-NEXT:    addi.w $a0, $zero, 1
+; LA64F-NEXT:    pcalau12i $a2, %pc_hi20(.LCPI1_0)
+; LA64F-NEXT:    addi.d $a2, $a2, %pc_lo12(.LCPI1_0)
+; LA64F-NEXT:    fld.s $fa2, $a2, 0
 ; LA64F-NEXT:    movgr2fr.w $fa3, $a0
 ; LA64F-NEXT:    ffint.s.w $fa3, $fa3
 ; LA64F-NEXT:    fadd.s $fa1, $fa1, $fa3
@@ -140,10 +140,10 @@ define void @test_f2(ptr %P, ptr %S) nounwind {
 ; LA64D:       # %bb.0:
 ; LA64D-NEXT:    fld.s $fa0, $a0, 4
 ; LA64D-NEXT:    fld.s $fa1, $a0, 0
-; LA64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI1_0)
-; LA64D-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI1_0)
-; LA64D-NEXT:    fld.s $fa2, $a0, 0
 ; LA64D-NEXT:    addi.w $a0, $zero, 1
+; LA64D-NEXT:    pcalau12i $a2, %pc_hi20(.LCPI1_0)
+; LA64D-NEXT:    addi.d $a2, $a2, %pc_lo12(.LCPI1_0)
+; LA64D-NEXT:    fld.s $fa2, $a2, 0
 ; LA64D-NEXT:    movgr2fr.w $fa3, $a0
 ; LA64D-NEXT:    ffint.s.w $fa3, $fa3
 ; LA64D-NEXT:    fadd.s $fa1, $fa1, $fa3
@@ -527,10 +527,10 @@ define void @test_d2(ptr %P, ptr %S) nounwind {
 ; LA64D:       # %bb.0:
 ; LA64D-NEXT:    fld.d $fa0, $a0, 8
 ; LA64D-NEXT:    fld.d $fa1, $a0, 0
-; LA64D-NEXT:    pcalau12i $a0, %pc_hi20(.LCPI4_0)
-; LA64D-NEXT:    addi.d $a0, $a0, %pc_lo12(.LCPI4_0)
-; LA64D-NEXT:    fld.d $fa2, $a0, 0
 ; LA64D-NEXT:    addi.d $a0, $zero, 1
+; LA64D-NEXT:    pcalau12i $a2, %pc_hi20(.LCPI4_0)
+; LA64D-NEXT:    addi.d $a2, $a2, %pc_lo12(.LCPI4_0)
+; LA64D-NEXT:    fld.d $fa2, $a2, 0
 ; LA64D-NEXT:    movgr2fr.d $fa3, $a0
 ; LA64D-NEXT:    ffint.d.l $fa3, $fa3
 ; LA64D-NEXT:    fadd.d $fa1, $fa1, $fa3
diff --git a/llvm/test/CodeGen/RISCV/atomicrmw-uinc-udec-wrap.ll b/llvm/test/CodeGen/RISCV/atomicrmw-uinc-udec-wrap.ll
index 5914e45a1533..f96e1bad2e38 100644
--- a/llvm/test/CodeGen/RISCV/atomicrmw-uinc-udec-wrap.ll
+++ b/llvm/test/CodeGen/RISCV/atomicrmw-uinc-udec-wrap.ll
@@ -127,11 +127,11 @@ define i8 @atomicrmw_uinc_wrap_i8(ptr %ptr, i8 %val) {
 ; RV64IA-LABEL: atomicrmw_uinc_wrap_i8:
 ; RV64IA:       # %bb.0:
 ; RV64IA-NEXT:    andi a2, a0, -4
-; RV64IA-NEXT:    slli a0, a0, 3
-; RV64IA-NEXT:    li a3, 255
-; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    slli a4, a0, 3
+; RV64IA-NEXT:    andi a0, a4, 24
+; RV64IA-NEXT:    li a5, 255
 ; RV64IA-NEXT:    lw a3, 0(a2)
-; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    sllw a4, a5, a4
 ; RV64IA-NEXT:    not a4, a4
 ; RV64IA-NEXT:    andi a1, a1, 255
 ; RV64IA-NEXT:  .LBB0_1: # %atomicrmw.start
diff --git a/llvm/test/CodeGen/RISCV/bfloat-convert.ll b/llvm/test/CodeGen/RISCV/bfloat-convert.ll
index d5041c2a7ca7..9e2b0b5c3cbb 100644
--- a/llvm/test/CodeGen/RISCV/bfloat-convert.ll
+++ b/llvm/test/CodeGen/RISCV/bfloat-convert.ll
@@ -84,12 +84,12 @@ define i16 @fcvt_si_bf16_sat(bfloat %a) nounwind {
 ; CHECK64ZFBFMIN:       # %bb.0: # %start
 ; CHECK64ZFBFMIN-NEXT:    fcvt.s.bf16 fa5, fa0
 ; CHECK64ZFBFMIN-NEXT:    feq.s a0, fa5, fa5
+; CHECK64ZFBFMIN-NEXT:    neg a0, a0
 ; CHECK64ZFBFMIN-NEXT:    lui a1, %hi(.LCPI1_0)
 ; CHECK64ZFBFMIN-NEXT:    flw fa4, %lo(.LCPI1_0)(a1)
 ; CHECK64ZFBFMIN-NEXT:    lui a1, 815104
 ; CHECK64ZFBFMIN-NEXT:    fmv.w.x fa3, a1
 ; CHECK64ZFBFMIN-NEXT:    fmax.s fa5, fa5, fa3
-; CHECK64ZFBFMIN-NEXT:    neg a0, a0
 ; CHECK64ZFBFMIN-NEXT:    fmin.s fa5, fa5, fa4
 ; CHECK64ZFBFMIN-NEXT:    fcvt.l.s a1, fa5, rtz
 ; CHECK64ZFBFMIN-NEXT:    and a0, a0, a1
@@ -187,10 +187,10 @@ define i16 @fcvt_ui_bf16_sat(bfloat %a) nounwind {
 ;
 ; RV64ID-LABEL: fcvt_ui_bf16_sat:
 ; RV64ID:       # %bb.0: # %start
-; RV64ID-NEXT:    lui a0, %hi(.LCPI3_0)
-; RV64ID-NEXT:    flw fa5, %lo(.LCPI3_0)(a0)
 ; RV64ID-NEXT:    fmv.x.w a0, fa0
 ; RV64ID-NEXT:    slli a0, a0, 16
+; RV64ID-NEXT:    lui a1, %hi(.LCPI3_0)
+; RV64ID-NEXT:    flw fa5, %lo(.LCPI3_0)(a1)
 ; RV64ID-NEXT:    fmv.w.x fa4, a0
 ; RV64ID-NEXT:    fmv.w.x fa3, zero
 ; RV64ID-NEXT:    fmax.s fa4, fa4, fa3
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-lp64-lp64f-lp64d-common.ll b/llvm/test/CodeGen/RISCV/calling-conv-lp64-lp64f-lp64d-common.ll
index d84711294330..67123466354c 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-lp64-lp64f-lp64d-common.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-lp64-lp64f-lp64d-common.ll
@@ -140,11 +140,11 @@ define i64 @caller_large_scalars() nounwind {
 ; RV64I-NEXT:    sd a0, 0(sp)
 ; RV64I-NEXT:    sd zero, 56(sp)
 ; RV64I-NEXT:    sd zero, 48(sp)
-; RV64I-NEXT:    li a0, 1
-; RV64I-NEXT:    sd a0, 32(sp)
+; RV64I-NEXT:    sd zero, 40(sp)
+; RV64I-NEXT:    li a2, 1
 ; RV64I-NEXT:    addi a0, sp, 32
 ; RV64I-NEXT:    mv a1, sp
-; RV64I-NEXT:    sd zero, 40(sp)
+; RV64I-NEXT:    sd a2, 32(sp)
 ; RV64I-NEXT:    call callee_large_scalars
 ; RV64I-NEXT:    ld ra, 72(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 80
diff --git a/llvm/test/CodeGen/RISCV/double-convert.ll b/llvm/test/CodeGen/RISCV/double-convert.ll
index da882cafd997..c147d6ec6d9b 100644
--- a/llvm/test/CodeGen/RISCV/double-convert.ll
+++ b/llvm/test/CodeGen/RISCV/double-convert.ll
@@ -1651,8 +1651,8 @@ define signext i16 @fcvt_w_s_sat_i16(double %a) nounwind {
 ; RV64IFD-NEXT:    lui a0, %hi(.LCPI26_1)
 ; RV64IFD-NEXT:    fld fa4, %lo(.LCPI26_1)(a0)
 ; RV64IFD-NEXT:    feq.d a0, fa0, fa0
-; RV64IFD-NEXT:    fmax.d fa5, fa0, fa5
 ; RV64IFD-NEXT:    neg a0, a0
+; RV64IFD-NEXT:    fmax.d fa5, fa0, fa5
 ; RV64IFD-NEXT:    fmin.d fa5, fa5, fa4
 ; RV64IFD-NEXT:    fcvt.l.d a1, fa5, rtz
 ; RV64IFD-NEXT:    and a0, a0, a1
@@ -1680,12 +1680,12 @@ define signext i16 @fcvt_w_s_sat_i16(double %a) nounwind {
 ; RV64IZFINXZDINX-NEXT:    ld a1, %lo(.LCPI26_0)(a1)
 ; RV64IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI26_1)
 ; RV64IZFINXZDINX-NEXT:    ld a2, %lo(.LCPI26_1)(a2)
-; RV64IZFINXZDINX-NEXT:    fmax.d a1, a0, a1
-; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
-; RV64IZFINXZDINX-NEXT:    neg a0, a0
-; RV64IZFINXZDINX-NEXT:    fmin.d a1, a1, a2
-; RV64IZFINXZDINX-NEXT:    fcvt.l.d a1, a1, rtz
-; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    feq.d a3, a0, a0
+; RV64IZFINXZDINX-NEXT:    neg a3, a3
+; RV64IZFINXZDINX-NEXT:    fmax.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    fmin.d a0, a0, a2
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a0, a0, rtz
+; RV64IZFINXZDINX-NEXT:    and a0, a3, a0
 ; RV64IZFINXZDINX-NEXT:    ret
 ;
 ; RV32I-LABEL: fcvt_w_s_sat_i16:
@@ -2026,8 +2026,8 @@ define signext i8 @fcvt_w_s_sat_i8(double %a) nounwind {
 ; RV64IFD-NEXT:    lui a0, %hi(.LCPI30_1)
 ; RV64IFD-NEXT:    fld fa4, %lo(.LCPI30_1)(a0)
 ; RV64IFD-NEXT:    feq.d a0, fa0, fa0
-; RV64IFD-NEXT:    fmax.d fa5, fa0, fa5
 ; RV64IFD-NEXT:    neg a0, a0
+; RV64IFD-NEXT:    fmax.d fa5, fa0, fa5
 ; RV64IFD-NEXT:    fmin.d fa5, fa5, fa4
 ; RV64IFD-NEXT:    fcvt.l.d a1, fa5, rtz
 ; RV64IFD-NEXT:    and a0, a0, a1
@@ -2055,12 +2055,12 @@ define signext i8 @fcvt_w_s_sat_i8(double %a) nounwind {
 ; RV64IZFINXZDINX-NEXT:    ld a1, %lo(.LCPI30_0)(a1)
 ; RV64IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI30_1)
 ; RV64IZFINXZDINX-NEXT:    ld a2, %lo(.LCPI30_1)(a2)
-; RV64IZFINXZDINX-NEXT:    fmax.d a1, a0, a1
-; RV64IZFINXZDINX-NEXT:    feq.d a0, a0, a0
-; RV64IZFINXZDINX-NEXT:    neg a0, a0
-; RV64IZFINXZDINX-NEXT:    fmin.d a1, a1, a2
-; RV64IZFINXZDINX-NEXT:    fcvt.l.d a1, a1, rtz
-; RV64IZFINXZDINX-NEXT:    and a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    feq.d a3, a0, a0
+; RV64IZFINXZDINX-NEXT:    neg a3, a3
+; RV64IZFINXZDINX-NEXT:    fmax.d a0, a0, a1
+; RV64IZFINXZDINX-NEXT:    fmin.d a0, a0, a2
+; RV64IZFINXZDINX-NEXT:    fcvt.l.d a0, a0, rtz
+; RV64IZFINXZDINX-NEXT:    and a0, a3, a0
 ; RV64IZFINXZDINX-NEXT:    ret
 ;
 ; RV32I-LABEL: fcvt_w_s_sat_i8:
diff --git a/llvm/test/CodeGen/RISCV/float-convert.ll b/llvm/test/CodeGen/RISCV/float-convert.ll
index 2c7315fbe59f..653b64ec7304 100644
--- a/llvm/test/CodeGen/RISCV/float-convert.ll
+++ b/llvm/test/CodeGen/RISCV/float-convert.ll
@@ -1424,12 +1424,12 @@ define signext i16 @fcvt_w_s_sat_i16(float %a) nounwind {
 ; RV64IF-LABEL: fcvt_w_s_sat_i16:
 ; RV64IF:       # %bb.0: # %start
 ; RV64IF-NEXT:    feq.s a0, fa0, fa0
+; RV64IF-NEXT:    neg a0, a0
 ; RV64IF-NEXT:    lui a1, %hi(.LCPI24_0)
 ; RV64IF-NEXT:    flw fa5, %lo(.LCPI24_0)(a1)
 ; RV64IF-NEXT:    lui a1, 815104
 ; RV64IF-NEXT:    fmv.w.x fa4, a1
 ; RV64IF-NEXT:    fmax.s fa4, fa0, fa4
-; RV64IF-NEXT:    neg a0, a0
 ; RV64IF-NEXT:    fmin.s fa5, fa4, fa5
 ; RV64IF-NEXT:    fcvt.l.s a1, fa5, rtz
 ; RV64IF-NEXT:    and a0, a0, a1
@@ -1450,15 +1450,15 @@ define signext i16 @fcvt_w_s_sat_i16(float %a) nounwind {
 ;
 ; RV64IZFINX-LABEL: fcvt_w_s_sat_i16:
 ; RV64IZFINX:       # %bb.0: # %start
-; RV64IZFINX-NEXT:    lui a1, 815104
+; RV64IZFINX-NEXT:    feq.s a1, a0, a0
 ; RV64IZFINX-NEXT:    lui a2, %hi(.LCPI24_0)
 ; RV64IZFINX-NEXT:    lw a2, %lo(.LCPI24_0)(a2)
-; RV64IZFINX-NEXT:    fmax.s a1, a0, a1
-; RV64IZFINX-NEXT:    feq.s a0, a0, a0
-; RV64IZFINX-NEXT:    neg a0, a0
-; RV64IZFINX-NEXT:    fmin.s a1, a1, a2
-; RV64IZFINX-NEXT:    fcvt.l.s a1, a1, rtz
-; RV64IZFINX-NEXT:    and a0, a0, a1
+; RV64IZFINX-NEXT:    neg a1, a1
+; RV64IZFINX-NEXT:    lui a3, 815104
+; RV64IZFINX-NEXT:    fmax.s a0, a0, a3
+; RV64IZFINX-NEXT:    fmin.s a0, a0, a2
+; RV64IZFINX-NEXT:    fcvt.l.s a0, a0, rtz
+; RV64IZFINX-NEXT:    and a0, a1, a0
 ; RV64IZFINX-NEXT:    ret
 ;
 ; RV32I-LABEL: fcvt_w_s_sat_i16:
diff --git a/llvm/test/CodeGen/RISCV/half-convert.ll b/llvm/test/CodeGen/RISCV/half-convert.ll
index 16c096290720..277749c75bbb 100644
--- a/llvm/test/CodeGen/RISCV/half-convert.ll
+++ b/llvm/test/CodeGen/RISCV/half-convert.ll
@@ -210,12 +210,12 @@ define i16 @fcvt_si_h_sat(half %a) nounwind {
 ; RV64IZFH:       # %bb.0: # %start
 ; RV64IZFH-NEXT:    fcvt.s.h fa5, fa0
 ; RV64IZFH-NEXT:    feq.s a0, fa5, fa5
+; RV64IZFH-NEXT:    neg a0, a0
 ; RV64IZFH-NEXT:    lui a1, %hi(.LCPI1_0)
 ; RV64IZFH-NEXT:    flw fa4, %lo(.LCPI1_0)(a1)
 ; RV64IZFH-NEXT:    lui a1, 815104
 ; RV64IZFH-NEXT:    fmv.w.x fa3, a1
 ; RV64IZFH-NEXT:    fmax.s fa5, fa5, fa3
-; RV64IZFH-NEXT:    neg a0, a0
 ; RV64IZFH-NEXT:    fmin.s fa5, fa5, fa4
 ; RV64IZFH-NEXT:    fcvt.l.s a1, fa5, rtz
 ; RV64IZFH-NEXT:    and a0, a0, a1
@@ -240,12 +240,12 @@ define i16 @fcvt_si_h_sat(half %a) nounwind {
 ; RV64IDZFH:       # %bb.0: # %start
 ; RV64IDZFH-NEXT:    fcvt.s.h fa5, fa0
 ; RV64IDZFH-NEXT:    feq.s a0, fa5, fa5
+; RV64IDZFH-NEXT:    neg a0, a0
 ; RV64IDZFH-NEXT:    lui a1, %hi(.LCPI1_0)
 ; RV64IDZFH-NEXT:    flw fa4, %lo(.LCPI1_0)(a1)
 ; RV64IDZFH-NEXT:    lui a1, 815104
 ; RV64IDZFH-NEXT:    fmv.w.x fa3, a1
 ; RV64IDZFH-NEXT:    fmax.s fa5, fa5, fa3
-; RV64IDZFH-NEXT:    neg a0, a0
 ; RV64IDZFH-NEXT:    fmin.s fa5, fa5, fa4
 ; RV64IDZFH-NEXT:    fcvt.l.s a1, fa5, rtz
 ; RV64IDZFH-NEXT:    and a0, a0, a1
@@ -268,15 +268,15 @@ define i16 @fcvt_si_h_sat(half %a) nounwind {
 ; RV64IZHINX-LABEL: fcvt_si_h_sat:
 ; RV64IZHINX:       # %bb.0: # %start
 ; RV64IZHINX-NEXT:    fcvt.s.h a0, a0
-; RV64IZHINX-NEXT:    lui a1, 815104
+; RV64IZHINX-NEXT:    feq.s a1, a0, a0
 ; RV64IZHINX-NEXT:    lui a2, %hi(.LCPI1_0)
 ; RV64IZHINX-NEXT:    lw a2, %lo(.LCPI1_0)(a2)
-; RV64IZHINX-NEXT:    fmax.s a1, a0, a1
-; RV64IZHINX-NEXT:    feq.s a0, a0, a0
-; RV64IZHINX-NEXT:    neg a0, a0
-; RV64IZHINX-NEXT:    fmin.s a1, a1, a2
-; RV64IZHINX-NEXT:    fcvt.l.s a1, a1, rtz
-; RV64IZHINX-NEXT:    and a0, a0, a1
+; RV64IZHINX-NEXT:    neg a1, a1
+; RV64IZHINX-NEXT:    lui a3, 815104
+; RV64IZHINX-NEXT:    fmax.s a0, a0, a3
+; RV64IZHINX-NEXT:    fmin.s a0, a0, a2
+; RV64IZHINX-NEXT:    fcvt.l.s a0, a0, rtz
+; RV64IZHINX-NEXT:    and a0, a1, a0
 ; RV64IZHINX-NEXT:    ret
 ;
 ; RV32IZDINXZHINX-LABEL: fcvt_si_h_sat:
@@ -296,15 +296,15 @@ define i16 @fcvt_si_h_sat(half %a) nounwind {
 ; RV64IZDINXZHINX-LABEL: fcvt_si_h_sat:
 ; RV64IZDINXZHINX:       # %bb.0: # %start
 ; RV64IZDINXZHINX-NEXT:    fcvt.s.h a0, a0
-; RV64IZDINXZHINX-NEXT:    lui a1, 815104
+; RV64IZDINXZHINX-NEXT:    feq.s a1, a0, a0
 ; RV64IZDINXZHINX-NEXT:    lui a2, %hi(.LCPI1_0)
 ; RV64IZDINXZHINX-NEXT:    lw a2, %lo(.LCPI1_0)(a2)
-; RV64IZDINXZHINX-NEXT:    fmax.s a1, a0, a1
-; RV64IZDINXZHINX-NEXT:    feq.s a0, a0, a0
-; RV64IZDINXZHINX-NEXT:    neg a0, a0
-; RV64IZDINXZHINX-NEXT:    fmin.s a1, a1, a2
-; RV64IZDINXZHINX-NEXT:    fcvt.l.s a1, a1, rtz
-; RV64IZDINXZHINX-NEXT:    and a0, a0, a1
+; RV64IZDINXZHINX-NEXT:    neg a1, a1
+; RV64IZDINXZHINX-NEXT:    lui a3, 815104
+; RV64IZDINXZHINX-NEXT:    fmax.s a0, a0, a3
+; RV64IZDINXZHINX-NEXT:    fmin.s a0, a0, a2
+; RV64IZDINXZHINX-NEXT:    fcvt.l.s a0, a0, rtz
+; RV64IZDINXZHINX-NEXT:    and a0, a1, a0
 ; RV64IZDINXZHINX-NEXT:    ret
 ;
 ; RV32I-LABEL: fcvt_si_h_sat:
@@ -420,12 +420,12 @@ define i16 @fcvt_si_h_sat(half %a) nounwind {
 ; RV64ID-LP64-NEXT:    call __extendhfsf2
 ; RV64ID-LP64-NEXT:    fmv.w.x fa5, a0
 ; RV64ID-LP64-NEXT:    feq.s a0, fa5, fa5
+; RV64ID-LP64-NEXT:    neg a0, a0
 ; RV64ID-LP64-NEXT:    lui a1, %hi(.LCPI1_0)
 ; RV64ID-LP64-NEXT:    flw fa4, %lo(.LCPI1_0)(a1)
 ; RV64ID-LP64-NEXT:    lui a1, 815104
 ; RV64ID-LP64-NEXT:    fmv.w.x fa3, a1
 ; RV64ID-LP64-NEXT:    fmax.s fa5, fa5, fa3
-; RV64ID-LP64-NEXT:    neg a0, a0
 ; RV64ID-LP64-NEXT:    fmin.s fa5, fa5, fa4
 ; RV64ID-LP64-NEXT:    fcvt.l.s a1, fa5, rtz
 ; RV64ID-LP64-NEXT:    and a0, a0, a1
@@ -458,12 +458,12 @@ define i16 @fcvt_si_h_sat(half %a) nounwind {
 ; RV64ID-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64ID-NEXT:    call __extendhfsf2
 ; RV64ID-NEXT:    feq.s a0, fa0, fa0
+; RV64ID-NEXT:    neg a0, a0
 ; RV64ID-NEXT:    lui a1, %hi(.LCPI1_0)
 ; RV64ID-NEXT:    flw fa5, %lo(.LCPI1_0)(a1)
 ; RV64ID-NEXT:    lui a1, 815104
 ; RV64ID-NEXT:    fmv.w.x fa4, a1
 ; RV64ID-NEXT:    fmax.s fa4, fa0, fa4
-; RV64ID-NEXT:    neg a0, a0
 ; RV64ID-NEXT:    fmin.s fa5, fa4, fa5
 ; RV64ID-NEXT:    fcvt.l.s a1, fa5, rtz
 ; RV64ID-NEXT:    and a0, a0, a1
@@ -490,12 +490,12 @@ define i16 @fcvt_si_h_sat(half %a) nounwind {
 ; CHECK64-IZFHMIN:       # %bb.0: # %start
 ; CHECK64-IZFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; CHECK64-IZFHMIN-NEXT:    feq.s a0, fa5, fa5
+; CHECK64-IZFHMIN-NEXT:    neg a0, a0
 ; CHECK64-IZFHMIN-NEXT:    lui a1, %hi(.LCPI1_0)
 ; CHECK64-IZFHMIN-NEXT:    flw fa4, %lo(.LCPI1_0)(a1)
 ; CHECK64-IZFHMIN-NEXT:    lui a1, 815104
 ; CHECK64-IZFHMIN-NEXT:    fmv.w.x fa3, a1
 ; CHECK64-IZFHMIN-NEXT:    fmax.s fa5, fa5, fa3
-; CHECK64-IZFHMIN-NEXT:    neg a0, a0
 ; CHECK64-IZFHMIN-NEXT:    fmin.s fa5, fa5, fa4
 ; CHECK64-IZFHMIN-NEXT:    fcvt.l.s a1, fa5, rtz
 ; CHECK64-IZFHMIN-NEXT:    and a0, a0, a1
@@ -518,15 +518,15 @@ define i16 @fcvt_si_h_sat(half %a) nounwind {
 ; CHECK64-IZHINXMIN-LABEL: fcvt_si_h_sat:
 ; CHECK64-IZHINXMIN:       # %bb.0: # %start
 ; CHECK64-IZHINXMIN-NEXT:    fcvt.s.h a0, a0
-; CHECK64-IZHINXMIN-NEXT:    lui a1, 815104
+; CHECK64-IZHINXMIN-NEXT:    feq.s a1, a0, a0
 ; CHECK64-IZHINXMIN-NEXT:    lui a2, %hi(.LCPI1_0)
 ; CHECK64-IZHINXMIN-NEXT:    lw a2, %lo(.LCPI1_0)(a2)
-; CHECK64-IZHINXMIN-NEXT:    fmax.s a1, a0, a1
-; CHECK64-IZHINXMIN-NEXT:    feq.s a0, a0, a0
-; CHECK64-IZHINXMIN-NEXT:    neg a0, a0
-; CHECK64-IZHINXMIN-NEXT:    fmin.s a1, a1, a2
-; CHECK64-IZHINXMIN-NEXT:    fcvt.l.s a1, a1, rtz
-; CHECK64-IZHINXMIN-NEXT:    and a0, a0, a1
+; CHECK64-IZHINXMIN-NEXT:    neg a1, a1
+; CHECK64-IZHINXMIN-NEXT:    lui a3, 815104
+; CHECK64-IZHINXMIN-NEXT:    fmax.s a0, a0, a3
+; CHECK64-IZHINXMIN-NEXT:    fmin.s a0, a0, a2
+; CHECK64-IZHINXMIN-NEXT:    fcvt.l.s a0, a0, rtz
+; CHECK64-IZHINXMIN-NEXT:    and a0, a1, a0
 ; CHECK64-IZHINXMIN-NEXT:    ret
 ;
 ; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_si_h_sat:
@@ -546,15 +546,15 @@ define i16 @fcvt_si_h_sat(half %a) nounwind {
 ; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_si_h_sat:
 ; CHECK64-IZDINXZHINXMIN:       # %bb.0: # %start
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    lui a1, 815104
+; CHECK64-IZDINXZHINXMIN-NEXT:    feq.s a1, a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    lui a2, %hi(.LCPI1_0)
 ; CHECK64-IZDINXZHINXMIN-NEXT:    lw a2, %lo(.LCPI1_0)(a2)
-; CHECK64-IZDINXZHINXMIN-NEXT:    fmax.s a1, a0, a1
-; CHECK64-IZDINXZHINXMIN-NEXT:    feq.s a0, a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    neg a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    fmin.s a1, a1, a2
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.l.s a1, a1, rtz
-; CHECK64-IZDINXZHINXMIN-NEXT:    and a0, a0, a1
+; CHECK64-IZDINXZHINXMIN-NEXT:    neg a1, a1
+; CHECK64-IZDINXZHINXMIN-NEXT:    lui a3, 815104
+; CHECK64-IZDINXZHINXMIN-NEXT:    fmax.s a0, a0, a3
+; CHECK64-IZDINXZHINXMIN-NEXT:    fmin.s a0, a0, a2
+; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.l.s a0, a0, rtz
+; CHECK64-IZDINXZHINXMIN-NEXT:    and a0, a1, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    ret
 start:
   %0 = tail call i16 @llvm.fptosi.sat.i16.f16(half %a)
@@ -6377,12 +6377,12 @@ define signext i16 @fcvt_w_s_sat_i16(half %a) nounwind {
 ; RV64IZFH:       # %bb.0: # %start
 ; RV64IZFH-NEXT:    fcvt.s.h fa5, fa0
 ; RV64IZFH-NEXT:    feq.s a0, fa5, fa5
+; RV64IZFH-NEXT:    neg a0, a0
 ; RV64IZFH-NEXT:    lui a1, %hi(.LCPI32_0)
 ; RV64IZFH-NEXT:    flw fa4, %lo(.LCPI32_0)(a1)
 ; RV64IZFH-NEXT:    lui a1, 815104
 ; RV64IZFH-NEXT:    fmv.w.x fa3, a1
 ; RV64IZFH-NEXT:    fmax.s fa5, fa5, fa3
-; RV64IZFH-NEXT:    neg a0, a0
 ; RV64IZFH-NEXT:    fmin.s fa5, fa5, fa4
 ; RV64IZFH-NEXT:    fcvt.l.s a1, fa5, rtz
 ; RV64IZFH-NEXT:    and a0, a0, a1
@@ -6407,12 +6407,12 @@ define signext i16 @fcvt_w_s_sat_i16(half %a) nounwind {
 ; RV64IDZFH:       # %bb.0: # %start
 ; RV64IDZFH-NEXT:    fcvt.s.h fa5, fa0
 ; RV64IDZFH-NEXT:    feq.s a0, fa5, fa5
+; RV64IDZFH-NEXT:    neg a0, a0
 ; RV64IDZFH-NEXT:    lui a1, %hi(.LCPI32_0)
 ; RV64IDZFH-NEXT:    flw fa4, %lo(.LCPI32_0)(a1)
 ; RV64IDZFH-NEXT:    lui a1, 815104
 ; RV64IDZFH-NEXT:    fmv.w.x fa3, a1
 ; RV64IDZFH-NEXT:    fmax.s fa5, fa5, fa3
-; RV64IDZFH-NEXT:    neg a0, a0
 ; RV64IDZFH-NEXT:    fmin.s fa5, fa5, fa4
 ; RV64IDZFH-NEXT:    fcvt.l.s a1, fa5, rtz
 ; RV64IDZFH-NEXT:    and a0, a0, a1
@@ -6435,15 +6435,15 @@ define signext i16 @fcvt_w_s_sat_i16(half %a) nounwind {
 ; RV64IZHINX-LABEL: fcvt_w_s_sat_i16:
 ; RV64IZHINX:       # %bb.0: # %start
 ; RV64IZHINX-NEXT:    fcvt.s.h a0, a0
-; RV64IZHINX-NEXT:    lui a1, 815104
+; RV64IZHINX-NEXT:    feq.s a1, a0, a0
 ; RV64IZHINX-NEXT:    lui a2, %hi(.LCPI32_0)
 ; RV64IZHINX-NEXT:    lw a2, %lo(.LCPI32_0)(a2)
-; RV64IZHINX-NEXT:    fmax.s a1, a0, a1
-; RV64IZHINX-NEXT:    feq.s a0, a0, a0
-; RV64IZHINX-NEXT:    neg a0, a0
-; RV64IZHINX-NEXT:    fmin.s a1, a1, a2
-; RV64IZHINX-NEXT:    fcvt.l.s a1, a1, rtz
-; RV64IZHINX-NEXT:    and a0, a0, a1
+; RV64IZHINX-NEXT:    neg a1, a1
+; RV64IZHINX-NEXT:    lui a3, 815104
+; RV64IZHINX-NEXT:    fmax.s a0, a0, a3
+; RV64IZHINX-NEXT:    fmin.s a0, a0, a2
+; RV64IZHINX-NEXT:    fcvt.l.s a0, a0, rtz
+; RV64IZHINX-NEXT:    and a0, a1, a0
 ; RV64IZHINX-NEXT:    ret
 ;
 ; RV32IZDINXZHINX-LABEL: fcvt_w_s_sat_i16:
@@ -6463,15 +6463,15 @@ define signext i16 @fcvt_w_s_sat_i16(half %a) nounwind {
 ; RV64IZDINXZHINX-LABEL: fcvt_w_s_sat_i16:
 ; RV64IZDINXZHINX:       # %bb.0: # %start
 ; RV64IZDINXZHINX-NEXT:    fcvt.s.h a0, a0
-; RV64IZDINXZHINX-NEXT:    lui a1, 815104
+; RV64IZDINXZHINX-NEXT:    feq.s a1, a0, a0
 ; RV64IZDINXZHINX-NEXT:    lui a2, %hi(.LCPI32_0)
 ; RV64IZDINXZHINX-NEXT:    lw a2, %lo(.LCPI32_0)(a2)
-; RV64IZDINXZHINX-NEXT:    fmax.s a1, a0, a1
-; RV64IZDINXZHINX-NEXT:    feq.s a0, a0, a0
-; RV64IZDINXZHINX-NEXT:    neg a0, a0
-; RV64IZDINXZHINX-NEXT:    fmin.s a1, a1, a2
-; RV64IZDINXZHINX-NEXT:    fcvt.l.s a1, a1, rtz
-; RV64IZDINXZHINX-NEXT:    and a0, a0, a1
+; RV64IZDINXZHINX-NEXT:    neg a1, a1
+; RV64IZDINXZHINX-NEXT:    lui a3, 815104
+; RV64IZDINXZHINX-NEXT:    fmax.s a0, a0, a3
+; RV64IZDINXZHINX-NEXT:    fmin.s a0, a0, a2
+; RV64IZDINXZHINX-NEXT:    fcvt.l.s a0, a0, rtz
+; RV64IZDINXZHINX-NEXT:    and a0, a1, a0
 ; RV64IZDINXZHINX-NEXT:    ret
 ;
 ; RV32I-LABEL: fcvt_w_s_sat_i16:
@@ -6591,12 +6591,12 @@ define signext i16 @fcvt_w_s_sat_i16(half %a) nounwind {
 ; RV64ID-LP64-NEXT:    call __extendhfsf2
 ; RV64ID-LP64-NEXT:    fmv.w.x fa5, a0
 ; RV64ID-LP64-NEXT:    feq.s a0, fa5, fa5
+; RV64ID-LP64-NEXT:    neg a0, a0
 ; RV64ID-LP64-NEXT:    lui a1, %hi(.LCPI32_0)
 ; RV64ID-LP64-NEXT:    flw fa4, %lo(.LCPI32_0)(a1)
 ; RV64ID-LP64-NEXT:    lui a1, 815104
 ; RV64ID-LP64-NEXT:    fmv.w.x fa3, a1
 ; RV64ID-LP64-NEXT:    fmax.s fa5, fa5, fa3
-; RV64ID-LP64-NEXT:    neg a0, a0
 ; RV64ID-LP64-NEXT:    fmin.s fa5, fa5, fa4
 ; RV64ID-LP64-NEXT:    fcvt.l.s a1, fa5, rtz
 ; RV64ID-LP64-NEXT:    and a0, a0, a1
@@ -6629,12 +6629,12 @@ define signext i16 @fcvt_w_s_sat_i16(half %a) nounwind {
 ; RV64ID-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
 ; RV64ID-NEXT:    call __extendhfsf2
 ; RV64ID-NEXT:    feq.s a0, fa0, fa0
+; RV64ID-NEXT:    neg a0, a0
 ; RV64ID-NEXT:    lui a1, %hi(.LCPI32_0)
 ; RV64ID-NEXT:    flw fa5, %lo(.LCPI32_0)(a1)
 ; RV64ID-NEXT:    lui a1, 815104
 ; RV64ID-NEXT:    fmv.w.x fa4, a1
 ; RV64ID-NEXT:    fmax.s fa4, fa0, fa4
-; RV64ID-NEXT:    neg a0, a0
 ; RV64ID-NEXT:    fmin.s fa5, fa4, fa5
 ; RV64ID-NEXT:    fcvt.l.s a1, fa5, rtz
 ; RV64ID-NEXT:    and a0, a0, a1
@@ -6661,12 +6661,12 @@ define signext i16 @fcvt_w_s_sat_i16(half %a) nounwind {
 ; CHECK64-IZFHMIN:       # %bb.0: # %start
 ; CHECK64-IZFHMIN-NEXT:    fcvt.s.h fa5, fa0
 ; CHECK64-IZFHMIN-NEXT:    feq.s a0, fa5, fa5
+; CHECK64-IZFHMIN-NEXT:    neg a0, a0
 ; CHECK64-IZFHMIN-NEXT:    lui a1, %hi(.LCPI32_0)
 ; CHECK64-IZFHMIN-NEXT:    flw fa4, %lo(.LCPI32_0)(a1)
 ; CHECK64-IZFHMIN-NEXT:    lui a1, 815104
 ; CHECK64-IZFHMIN-NEXT:    fmv.w.x fa3, a1
 ; CHECK64-IZFHMIN-NEXT:    fmax.s fa5, fa5, fa3
-; CHECK64-IZFHMIN-NEXT:    neg a0, a0
 ; CHECK64-IZFHMIN-NEXT:    fmin.s fa5, fa5, fa4
 ; CHECK64-IZFHMIN-NEXT:    fcvt.l.s a1, fa5, rtz
 ; CHECK64-IZFHMIN-NEXT:    and a0, a0, a1
@@ -6689,15 +6689,15 @@ define signext i16 @fcvt_w_s_sat_i16(half %a) nounwind {
 ; CHECK64-IZHINXMIN-LABEL: fcvt_w_s_sat_i16:
 ; CHECK64-IZHINXMIN:       # %bb.0: # %start
 ; CHECK64-IZHINXMIN-NEXT:    fcvt.s.h a0, a0
-; CHECK64-IZHINXMIN-NEXT:    lui a1, 815104
+; CHECK64-IZHINXMIN-NEXT:    feq.s a1, a0, a0
 ; CHECK64-IZHINXMIN-NEXT:    lui a2, %hi(.LCPI32_0)
 ; CHECK64-IZHINXMIN-NEXT:    lw a2, %lo(.LCPI32_0)(a2)
-; CHECK64-IZHINXMIN-NEXT:    fmax.s a1, a0, a1
-; CHECK64-IZHINXMIN-NEXT:    feq.s a0, a0, a0
-; CHECK64-IZHINXMIN-NEXT:    neg a0, a0
-; CHECK64-IZHINXMIN-NEXT:    fmin.s a1, a1, a2
-; CHECK64-IZHINXMIN-NEXT:    fcvt.l.s a1, a1, rtz
-; CHECK64-IZHINXMIN-NEXT:    and a0, a0, a1
+; CHECK64-IZHINXMIN-NEXT:    neg a1, a1
+; CHECK64-IZHINXMIN-NEXT:    lui a3, 815104
+; CHECK64-IZHINXMIN-NEXT:    fmax.s a0, a0, a3
+; CHECK64-IZHINXMIN-NEXT:    fmin.s a0, a0, a2
+; CHECK64-IZHINXMIN-NEXT:    fcvt.l.s a0, a0, rtz
+; CHECK64-IZHINXMIN-NEXT:    and a0, a1, a0
 ; CHECK64-IZHINXMIN-NEXT:    ret
 ;
 ; CHECK32-IZDINXZHINXMIN-LABEL: fcvt_w_s_sat_i16:
@@ -6717,15 +6717,15 @@ define signext i16 @fcvt_w_s_sat_i16(half %a) nounwind {
 ; CHECK64-IZDINXZHINXMIN-LABEL: fcvt_w_s_sat_i16:
 ; CHECK64-IZDINXZHINXMIN:       # %bb.0: # %start
 ; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.s.h a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    lui a1, 815104
+; CHECK64-IZDINXZHINXMIN-NEXT:    feq.s a1, a0, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    lui a2, %hi(.LCPI32_0)
 ; CHECK64-IZDINXZHINXMIN-NEXT:    lw a2, %lo(.LCPI32_0)(a2)
-; CHECK64-IZDINXZHINXMIN-NEXT:    fmax.s a1, a0, a1
-; CHECK64-IZDINXZHINXMIN-NEXT:    feq.s a0, a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    neg a0, a0
-; CHECK64-IZDINXZHINXMIN-NEXT:    fmin.s a1, a1, a2
-; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.l.s a1, a1, rtz
-; CHECK64-IZDINXZHINXMIN-NEXT:    and a0, a0, a1
+; CHECK64-IZDINXZHINXMIN-NEXT:    neg a1, a1
+; CHECK64-IZDINXZHINXMIN-NEXT:    lui a3, 815104
+; CHECK64-IZDINXZHINXMIN-NEXT:    fmax.s a0, a0, a3
+; CHECK64-IZDINXZHINXMIN-NEXT:    fmin.s a0, a0, a2
+; CHECK64-IZDINXZHINXMIN-NEXT:    fcvt.l.s a0, a0, rtz
+; CHECK64-IZDINXZHINXMIN-NEXT:    and a0, a1, a0
 ; CHECK64-IZDINXZHINXMIN-NEXT:    ret
 start:
   %0 = tail call i16 @llvm.fptosi.sat.i16.f16(half %a)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-shuffles.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-shuffles.ll
index 59eb4b89a2f5..b0f6bebea038 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-shuffles.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-shuffles.ll
@@ -125,35 +125,20 @@ define <4 x double> @vrgather_shuffle_xv_v4f64(<4 x double> %x) {
 }
 
 define <4 x double> @vrgather_shuffle_vx_v4f64(<4 x double> %x) {
-; RV32-LABEL: vrgather_shuffle_vx_v4f64:
-; RV32:       # %bb.0:
-; RV32-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; RV32-NEXT:    vid.v v12
-; RV32-NEXT:    li a0, 3
-; RV32-NEXT:    lui a1, %hi(.LCPI8_0)
-; RV32-NEXT:    addi a1, a1, %lo(.LCPI8_0)
-; RV32-NEXT:    vlse64.v v10, (a1), zero
-; RV32-NEXT:    vmul.vx v12, v12, a0
-; RV32-NEXT:    vmv.v.i v0, 3
-; RV32-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
-; RV32-NEXT:    vrgatherei16.vv v10, v8, v12, v0.t
-; RV32-NEXT:    vmv.v.v v8, v10
-; RV32-NEXT:    ret
-;
-; RV64-LABEL: vrgather_shuffle_vx_v4f64:
-; RV64:       # %bb.0:
-; RV64-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
-; RV64-NEXT:    vid.v v12
-; RV64-NEXT:    lui a0, %hi(.LCPI8_0)
-; RV64-NEXT:    addi a0, a0, %lo(.LCPI8_0)
-; RV64-NEXT:    vlse64.v v10, (a0), zero
-; RV64-NEXT:    li a0, 3
-; RV64-NEXT:    vmul.vx v12, v12, a0
-; RV64-NEXT:    vmv.v.i v0, 3
-; RV64-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
-; RV64-NEXT:    vrgatherei16.vv v10, v8, v12, v0.t
-; RV64-NEXT:    vmv.v.v v8, v10
-; RV64-NEXT:    ret
+; CHECK-LABEL: vrgather_shuffle_vx_v4f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; CHECK-NEXT:    vid.v v12
+; CHECK-NEXT:    li a0, 3
+; CHECK-NEXT:    lui a1, %hi(.LCPI8_0)
+; CHECK-NEXT:    addi a1, a1, %lo(.LCPI8_0)
+; CHECK-NEXT:    vlse64.v v10, (a1), zero
+; CHECK-NEXT:    vmul.vx v12, v12, a0
+; CHECK-NEXT:    vmv.v.i v0, 3
+; CHECK-NEXT:    vsetvli zero, zero, e64, m2, ta, mu
+; CHECK-NEXT:    vrgatherei16.vv v10, v8, v12, v0.t
+; CHECK-NEXT:    vmv.v.v v8, v10
+; CHECK-NEXT:    ret
   %s = shufflevector <4 x double> %x, <4 x double> <double 2.0, double 2.0, double 2.0, double 2.0>, <4 x i32> <i32 0, i32 3, i32 6, i32 5>
   ret <4 x double> %s
 }
@@ -279,3 +264,6 @@ define <8 x double> @splice_binary2(<8 x double> %x, <8 x double> %y) {
   %s = shufflevector <8 x double> %x, <8 x double> %y, <8 x i32> <i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4>
   ret <8 x double> %s
 }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; RV32: {{.*}}
+; RV64: {{.*}}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-insert-subvector.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-insert-subvector.ll
index 9f0240c53b21..1d3c22a02efc 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-insert-subvector.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-insert-subvector.ll
@@ -774,9 +774,9 @@ define void @insert_v2i64_nxv16i64_hi(ptr %psv, ptr %out) {
 ; RV64VLS-NEXT:    vl1re64.v v8, (a0)
 ; RV64VLS-NEXT:    addi a0, sp, 128
 ; RV64VLS-NEXT:    vs1r.v v8, (a0)
-; RV64VLS-NEXT:    addi a0, sp, 192
-; RV64VLS-NEXT:    vl8re64.v v8, (a0)
 ; RV64VLS-NEXT:    addi a0, sp, 64
+; RV64VLS-NEXT:    addi a2, sp, 192
+; RV64VLS-NEXT:    vl8re64.v v8, (a2)
 ; RV64VLS-NEXT:    vl8re64.v v16, (a0)
 ; RV64VLS-NEXT:    addi a0, a1, 128
 ; RV64VLS-NEXT:    vs8r.v v8, (a0)
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-setcc-fp-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-setcc-fp-vp.ll
index 6ba90b00fdba..4598bf67a236 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-setcc-fp-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-setcc-fp-vp.ll
@@ -1142,109 +1142,57 @@ define <8 x i1> @fcmp_uno_vf_swap_v8f16(<8 x half> %va, half %b, <8 x i1> %m, i3
 declare <128 x i1> @llvm.vp.fcmp.v128f16(<128 x half>, <128 x half>, metadata, <128 x i1>, i32)
 
 define <128 x i1> @fcmp_oeq_vv_v128f16(<128 x half> %va, <128 x half> %vb, <128 x i1> %m, i32 zeroext %evl) {
-; ZVFH32-LABEL: fcmp_oeq_vv_v128f16:
-; ZVFH32:       # %bb.0:
-; ZVFH32-NEXT:    addi sp, sp, -16
-; ZVFH32-NEXT:    .cfi_def_cfa_offset 16
-; ZVFH32-NEXT:    csrr a1, vlenb
-; ZVFH32-NEXT:    slli a1, a1, 4
-; ZVFH32-NEXT:    sub sp, sp, a1
-; ZVFH32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
-; ZVFH32-NEXT:    addi a1, a0, 128
-; ZVFH32-NEXT:    li a3, 64
-; ZVFH32-NEXT:    vsetvli zero, a3, e16, m8, ta, ma
-; ZVFH32-NEXT:    vle16.v v24, (a1)
-; ZVFH32-NEXT:    csrr a1, vlenb
-; ZVFH32-NEXT:    slli a1, a1, 3
-; ZVFH32-NEXT:    add a1, sp, a1
-; ZVFH32-NEXT:    addi a1, a1, 16
-; ZVFH32-NEXT:    vs8r.v v24, (a1) # Unknown-size Folded Spill
-; ZVFH32-NEXT:    vle16.v v24, (a0)
-; ZVFH32-NEXT:    addi a0, sp, 16
-; ZVFH32-NEXT:    vs8r.v v24, (a0) # Unknown-size Folded Spill
-; ZVFH32-NEXT:    vsetivli zero, 8, e8, m1, ta, ma
-; ZVFH32-NEXT:    vslidedown.vi v7, v0, 8
-; ZVFH32-NEXT:    mv a0, a2
-; ZVFH32-NEXT:    bltu a2, a3, .LBB43_2
-; ZVFH32-NEXT:  # %bb.1:
-; ZVFH32-NEXT:    li a0, 64
-; ZVFH32-NEXT:  .LBB43_2:
-; ZVFH32-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
-; ZVFH32-NEXT:    addi a0, sp, 16
-; ZVFH32-NEXT:    vl8r.v v24, (a0) # Unknown-size Folded Reload
-; ZVFH32-NEXT:    vmfeq.vv v6, v8, v24, v0.t
-; ZVFH32-NEXT:    addi a0, a2, -64
-; ZVFH32-NEXT:    sltu a1, a2, a0
-; ZVFH32-NEXT:    addi a1, a1, -1
-; ZVFH32-NEXT:    and a0, a1, a0
-; ZVFH32-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
-; ZVFH32-NEXT:    vmv1r.v v0, v7
-; ZVFH32-NEXT:    csrr a0, vlenb
-; ZVFH32-NEXT:    slli a0, a0, 3
-; ZVFH32-NEXT:    add a0, sp, a0
-; ZVFH32-NEXT:    addi a0, a0, 16
-; ZVFH32-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
-; ZVFH32-NEXT:    vmfeq.vv v24, v16, v8, v0.t
-; ZVFH32-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; ZVFH32-NEXT:    vslideup.vi v6, v24, 8
-; ZVFH32-NEXT:    vmv.v.v v0, v6
-; ZVFH32-NEXT:    csrr a0, vlenb
-; ZVFH32-NEXT:    slli a0, a0, 4
-; ZVFH32-NEXT:    add sp, sp, a0
-; ZVFH32-NEXT:    addi sp, sp, 16
-; ZVFH32-NEXT:    ret
-;
-; ZVFH64-LABEL: fcmp_oeq_vv_v128f16:
-; ZVFH64:       # %bb.0:
-; ZVFH64-NEXT:    addi sp, sp, -16
-; ZVFH64-NEXT:    .cfi_def_cfa_offset 16
-; ZVFH64-NEXT:    csrr a1, vlenb
-; ZVFH64-NEXT:    slli a1, a1, 4
-; ZVFH64-NEXT:    sub sp, sp, a1
-; ZVFH64-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
-; ZVFH64-NEXT:    addi a1, a0, 128
-; ZVFH64-NEXT:    li a3, 64
-; ZVFH64-NEXT:    vsetvli zero, a3, e16, m8, ta, ma
-; ZVFH64-NEXT:    vle16.v v24, (a1)
-; ZVFH64-NEXT:    csrr a1, vlenb
-; ZVFH64-NEXT:    slli a1, a1, 3
-; ZVFH64-NEXT:    add a1, sp, a1
-; ZVFH64-NEXT:    addi a1, a1, 16
-; ZVFH64-NEXT:    vs8r.v v24, (a1) # Unknown-size Folded Spill
-; ZVFH64-NEXT:    vle16.v v24, (a0)
-; ZVFH64-NEXT:    addi a0, sp, 16
-; ZVFH64-NEXT:    vs8r.v v24, (a0) # Unknown-size Folded Spill
-; ZVFH64-NEXT:    vsetivli zero, 8, e8, m1, ta, ma
-; ZVFH64-NEXT:    mv a0, a2
-; ZVFH64-NEXT:    vslidedown.vi v7, v0, 8
-; ZVFH64-NEXT:    bltu a2, a3, .LBB43_2
-; ZVFH64-NEXT:  # %bb.1:
-; ZVFH64-NEXT:    li a0, 64
-; ZVFH64-NEXT:  .LBB43_2:
-; ZVFH64-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
-; ZVFH64-NEXT:    addi a0, sp, 16
-; ZVFH64-NEXT:    vl8r.v v24, (a0) # Unknown-size Folded Reload
-; ZVFH64-NEXT:    vmfeq.vv v6, v8, v24, v0.t
-; ZVFH64-NEXT:    addi a0, a2, -64
-; ZVFH64-NEXT:    sltu a1, a2, a0
-; ZVFH64-NEXT:    addi a1, a1, -1
-; ZVFH64-NEXT:    and a0, a1, a0
-; ZVFH64-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
-; ZVFH64-NEXT:    vmv1r.v v0, v7
-; ZVFH64-NEXT:    csrr a0, vlenb
-; ZVFH64-NEXT:    slli a0, a0, 3
-; ZVFH64-NEXT:    add a0, sp, a0
-; ZVFH64-NEXT:    addi a0, a0, 16
-; ZVFH64-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
-; ZVFH64-NEXT:    vmfeq.vv v24, v16, v8, v0.t
-; ZVFH64-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; ZVFH64-NEXT:    vslideup.vi v6, v24, 8
-; ZVFH64-NEXT:    vmv.v.v v0, v6
-; ZVFH64-NEXT:    csrr a0, vlenb
-; ZVFH64-NEXT:    slli a0, a0, 4
-; ZVFH64-NEXT:    add sp, sp, a0
-; ZVFH64-NEXT:    addi sp, sp, 16
-; ZVFH64-NEXT:    ret
+; ZVFH-LABEL: fcmp_oeq_vv_v128f16:
+; ZVFH:       # %bb.0:
+; ZVFH-NEXT:    addi sp, sp, -16
+; ZVFH-NEXT:    .cfi_def_cfa_offset 16
+; ZVFH-NEXT:    csrr a1, vlenb
+; ZVFH-NEXT:    slli a1, a1, 4
+; ZVFH-NEXT:    sub sp, sp, a1
+; ZVFH-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
+; ZVFH-NEXT:    addi a1, a0, 128
+; ZVFH-NEXT:    li a3, 64
+; ZVFH-NEXT:    vsetvli zero, a3, e16, m8, ta, ma
+; ZVFH-NEXT:    vle16.v v24, (a1)
+; ZVFH-NEXT:    csrr a1, vlenb
+; ZVFH-NEXT:    slli a1, a1, 3
+; ZVFH-NEXT:    add a1, sp, a1
+; ZVFH-NEXT:    addi a1, a1, 16
+; ZVFH-NEXT:    vs8r.v v24, (a1) # Unknown-size Folded Spill
+; ZVFH-NEXT:    vle16.v v24, (a0)
+; ZVFH-NEXT:    addi a0, sp, 16
+; ZVFH-NEXT:    vs8r.v v24, (a0) # Unknown-size Folded Spill
+; ZVFH-NEXT:    vsetivli zero, 8, e8, m1, ta, ma
+; ZVFH-NEXT:    vslidedown.vi v7, v0, 8
+; ZVFH-NEXT:    mv a0, a2
+; ZVFH-NEXT:    bltu a2, a3, .LBB43_2
+; ZVFH-NEXT:  # %bb.1:
+; ZVFH-NEXT:    li a0, 64
+; ZVFH-NEXT:  .LBB43_2:
+; ZVFH-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
+; ZVFH-NEXT:    addi a0, sp, 16
+; ZVFH-NEXT:    vl8r.v v24, (a0) # Unknown-size Folded Reload
+; ZVFH-NEXT:    vmfeq.vv v6, v8, v24, v0.t
+; ZVFH-NEXT:    addi a0, a2, -64
+; ZVFH-NEXT:    sltu a1, a2, a0
+; ZVFH-NEXT:    addi a1, a1, -1
+; ZVFH-NEXT:    and a0, a1, a0
+; ZVFH-NEXT:    vsetvli zero, a0, e16, m8, ta, ma
+; ZVFH-NEXT:    vmv1r.v v0, v7
+; ZVFH-NEXT:    csrr a0, vlenb
+; ZVFH-NEXT:    slli a0, a0, 3
+; ZVFH-NEXT:    add a0, sp, a0
+; ZVFH-NEXT:    addi a0, a0, 16
+; ZVFH-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
+; ZVFH-NEXT:    vmfeq.vv v24, v16, v8, v0.t
+; ZVFH-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; ZVFH-NEXT:    vslideup.vi v6, v24, 8
+; ZVFH-NEXT:    vmv.v.v v0, v6
+; ZVFH-NEXT:    csrr a0, vlenb
+; ZVFH-NEXT:    slli a0, a0, 4
+; ZVFH-NEXT:    add sp, sp, a0
+; ZVFH-NEXT:    addi sp, sp, 16
+; ZVFH-NEXT:    ret
 ;
 ; ZVFHMIN32-LABEL: fcmp_oeq_vv_v128f16:
 ; ZVFHMIN32:       # %bb.0:
@@ -2956,3 +2904,6 @@ define <32 x i1> @fcmp_oeq_vv_v32f64(<32 x double> %va, <32 x double> %vb, <32 x
   %v = call <32 x i1> @llvm.vp.fcmp.v32f64(<32 x double> %va, <32 x double> %vb, metadata !"oeq", <32 x i1> %m, i32 %evl)
   ret <32 x i1> %v
 }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; ZVFH32: {{.*}}
+; ZVFH64: {{.*}}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-setcc-int-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-setcc-int-vp.ll
index 981715bd2b99..21bbca00921d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-setcc-int-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-setcc-int-vp.ll
@@ -1243,109 +1243,57 @@ define <8 x i1> @icmp_sle_vi_swap_v8i32(<8 x i32> %va, <8 x i1> %m, i32 zeroext
 declare <64 x i1> @llvm.vp.icmp.v64i32(<64 x i32>, <64 x i32>, metadata, <64 x i1>, i32)
 
 define <64 x i1> @icmp_eq_vv_v64i32(<64 x i32> %va, <64 x i32> %vb, <64 x i1> %m, i32 zeroext %evl) {
-; RV32-LABEL: icmp_eq_vv_v64i32:
-; RV32:       # %bb.0:
-; RV32-NEXT:    addi sp, sp, -16
-; RV32-NEXT:    .cfi_def_cfa_offset 16
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 4
-; RV32-NEXT:    sub sp, sp, a1
-; RV32-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
-; RV32-NEXT:    addi a1, a0, 128
-; RV32-NEXT:    li a3, 32
-; RV32-NEXT:    vsetvli zero, a3, e32, m8, ta, ma
-; RV32-NEXT:    vle32.v v24, (a1)
-; RV32-NEXT:    csrr a1, vlenb
-; RV32-NEXT:    slli a1, a1, 3
-; RV32-NEXT:    add a1, sp, a1
-; RV32-NEXT:    addi a1, a1, 16
-; RV32-NEXT:    vs8r.v v24, (a1) # Unknown-size Folded Spill
-; RV32-NEXT:    vle32.v v24, (a0)
-; RV32-NEXT:    addi a0, sp, 16
-; RV32-NEXT:    vs8r.v v24, (a0) # Unknown-size Folded Spill
-; RV32-NEXT:    vsetivli zero, 4, e8, mf2, ta, ma
-; RV32-NEXT:    vslidedown.vi v7, v0, 4
-; RV32-NEXT:    mv a0, a2
-; RV32-NEXT:    bltu a2, a3, .LBB99_2
-; RV32-NEXT:  # %bb.1:
-; RV32-NEXT:    li a0, 32
-; RV32-NEXT:  .LBB99_2:
-; RV32-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; RV32-NEXT:    addi a0, sp, 16
-; RV32-NEXT:    vl8r.v v24, (a0) # Unknown-size Folded Reload
-; RV32-NEXT:    vmseq.vv v6, v8, v24, v0.t
-; RV32-NEXT:    addi a0, a2, -32
-; RV32-NEXT:    sltu a1, a2, a0
-; RV32-NEXT:    addi a1, a1, -1
-; RV32-NEXT:    and a0, a1, a0
-; RV32-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; RV32-NEXT:    vmv1r.v v0, v7
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 3
-; RV32-NEXT:    add a0, sp, a0
-; RV32-NEXT:    addi a0, a0, 16
-; RV32-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
-; RV32-NEXT:    vmseq.vv v24, v16, v8, v0.t
-; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; RV32-NEXT:    vslideup.vi v6, v24, 4
-; RV32-NEXT:    vmv1r.v v0, v6
-; RV32-NEXT:    csrr a0, vlenb
-; RV32-NEXT:    slli a0, a0, 4
-; RV32-NEXT:    add sp, sp, a0
-; RV32-NEXT:    addi sp, sp, 16
-; RV32-NEXT:    ret
-;
-; RV64-LABEL: icmp_eq_vv_v64i32:
-; RV64:       # %bb.0:
-; RV64-NEXT:    addi sp, sp, -16
-; RV64-NEXT:    .cfi_def_cfa_offset 16
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    slli a1, a1, 4
-; RV64-NEXT:    sub sp, sp, a1
-; RV64-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
-; RV64-NEXT:    addi a1, a0, 128
-; RV64-NEXT:    li a3, 32
-; RV64-NEXT:    vsetvli zero, a3, e32, m8, ta, ma
-; RV64-NEXT:    vle32.v v24, (a1)
-; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    slli a1, a1, 3
-; RV64-NEXT:    add a1, sp, a1
-; RV64-NEXT:    addi a1, a1, 16
-; RV64-NEXT:    vs8r.v v24, (a1) # Unknown-size Folded Spill
-; RV64-NEXT:    vle32.v v24, (a0)
-; RV64-NEXT:    addi a0, sp, 16
-; RV64-NEXT:    vs8r.v v24, (a0) # Unknown-size Folded Spill
-; RV64-NEXT:    vsetivli zero, 4, e8, mf2, ta, ma
-; RV64-NEXT:    mv a0, a2
-; RV64-NEXT:    vslidedown.vi v7, v0, 4
-; RV64-NEXT:    bltu a2, a3, .LBB99_2
-; RV64-NEXT:  # %bb.1:
-; RV64-NEXT:    li a0, 32
-; RV64-NEXT:  .LBB99_2:
-; RV64-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; RV64-NEXT:    addi a0, sp, 16
-; RV64-NEXT:    vl8r.v v24, (a0) # Unknown-size Folded Reload
-; RV64-NEXT:    vmseq.vv v6, v8, v24, v0.t
-; RV64-NEXT:    addi a0, a2, -32
-; RV64-NEXT:    sltu a1, a2, a0
-; RV64-NEXT:    addi a1, a1, -1
-; RV64-NEXT:    and a0, a1, a0
-; RV64-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; RV64-NEXT:    vmv1r.v v0, v7
-; RV64-NEXT:    csrr a0, vlenb
-; RV64-NEXT:    slli a0, a0, 3
-; RV64-NEXT:    add a0, sp, a0
-; RV64-NEXT:    addi a0, a0, 16
-; RV64-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
-; RV64-NEXT:    vmseq.vv v24, v16, v8, v0.t
-; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; RV64-NEXT:    vslideup.vi v6, v24, 4
-; RV64-NEXT:    vmv1r.v v0, v6
-; RV64-NEXT:    csrr a0, vlenb
-; RV64-NEXT:    slli a0, a0, 4
-; RV64-NEXT:    add sp, sp, a0
-; RV64-NEXT:    addi sp, sp, 16
-; RV64-NEXT:    ret
+; CHECK-LABEL: icmp_eq_vv_v64i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi sp, sp, -16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    slli a1, a1, 4
+; CHECK-NEXT:    sub sp, sp, a1
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
+; CHECK-NEXT:    addi a1, a0, 128
+; CHECK-NEXT:    li a3, 32
+; CHECK-NEXT:    vsetvli zero, a3, e32, m8, ta, ma
+; CHECK-NEXT:    vle32.v v24, (a1)
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    slli a1, a1, 3
+; CHECK-NEXT:    add a1, sp, a1
+; CHECK-NEXT:    addi a1, a1, 16
+; CHECK-NEXT:    vs8r.v v24, (a1) # Unknown-size Folded Spill
+; CHECK-NEXT:    vle32.v v24, (a0)
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vs8r.v v24, (a0) # Unknown-size Folded Spill
+; CHECK-NEXT:    vsetivli zero, 4, e8, mf2, ta, ma
+; CHECK-NEXT:    vslidedown.vi v7, v0, 4
+; CHECK-NEXT:    mv a0, a2
+; CHECK-NEXT:    bltu a2, a3, .LBB99_2
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    li a0, 32
+; CHECK-NEXT:  .LBB99_2:
+; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
+; CHECK-NEXT:    addi a0, sp, 16
+; CHECK-NEXT:    vl8r.v v24, (a0) # Unknown-size Folded Reload
+; CHECK-NEXT:    vmseq.vv v6, v8, v24, v0.t
+; CHECK-NEXT:    addi a0, a2, -32
+; CHECK-NEXT:    sltu a1, a2, a0
+; CHECK-NEXT:    addi a1, a1, -1
+; CHECK-NEXT:    and a0, a1, a0
+; CHECK-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
+; CHECK-NEXT:    vmv1r.v v0, v7
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 3
+; CHECK-NEXT:    add a0, sp, a0
+; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    vl8r.v v8, (a0) # Unknown-size Folded Reload
+; CHECK-NEXT:    vmseq.vv v24, v16, v8, v0.t
+; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vi v6, v24, 4
+; CHECK-NEXT:    vmv1r.v v0, v6
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 4
+; CHECK-NEXT:    add sp, sp, a0
+; CHECK-NEXT:    addi sp, sp, 16
+; CHECK-NEXT:    ret
   %v = call <64 x i1> @llvm.vp.icmp.v64i32(<64 x i32> %va, <64 x i32> %vb, metadata !"eq", <64 x i1> %m, i32 %evl)
   ret <64 x i1> %v
 }
diff --git a/llvm/test/CodeGen/RISCV/rvv/vpscatter-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vpscatter-sdnode.ll
index 38e4aab4deb3..351fc500145e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vpscatter-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vpscatter-sdnode.ll
@@ -2219,8 +2219,8 @@ define void @vpscatter_baseidx_nxv16i16_nxv16f64(<vscale x 16 x double> %val, pt
 ; RV64-NEXT:    vsll.vi v16, v16, 3
 ; RV64-NEXT:    addi a1, sp, 16
 ; RV64-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
-; RV64-NEXT:    csrr a1, vlenb
 ; RV64-NEXT:    vsext.vf4 v16, v24
+; RV64-NEXT:    csrr a1, vlenb
 ; RV64-NEXT:    vsll.vi v24, v16, 3
 ; RV64-NEXT:    mv a3, a2
 ; RV64-NEXT:    bltu a2, a1, .LBB100_2
@@ -2286,29 +2286,31 @@ define void @vpscatter_baseidx_sext_nxv16i16_nxv16f64(<vscale x 16 x double> %va
 ; RV64-NEXT:    addi sp, sp, -16
 ; RV64-NEXT:    .cfi_def_cfa_offset 16
 ; RV64-NEXT:    csrr a3, vlenb
-; RV64-NEXT:    slli a3, a3, 4
+; RV64-NEXT:    li a4, 10
+; RV64-NEXT:    mul a3, a3, a4
 ; RV64-NEXT:    sub sp, sp, a3
-; RV64-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
+; RV64-NEXT:    .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x0a, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 10 * vlenb
 ; RV64-NEXT:    vl4re16.v v24, (a1)
+; RV64-NEXT:    addi a1, sp, 16
+; RV64-NEXT:    vs1r.v v0, (a1) # Unknown-size Folded Spill
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    slli a1, a1, 3
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
 ; RV64-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
 ; RV64-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
+; RV64-NEXT:    vsext.vf4 v0, v24
 ; RV64-NEXT:    vsext.vf4 v16, v26
 ; RV64-NEXT:    vsll.vi v16, v16, 3
-; RV64-NEXT:    addi a1, sp, 16
-; RV64-NEXT:    vs8r.v v16, (a1) # Unknown-size Folded Spill
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    vsext.vf4 v16, v24
-; RV64-NEXT:    vsll.vi v24, v16, 3
+; RV64-NEXT:    vsll.vi v24, v0, 3
 ; RV64-NEXT:    mv a3, a2
 ; RV64-NEXT:    bltu a2, a1, .LBB101_2
 ; RV64-NEXT:  # %bb.1:
 ; RV64-NEXT:    mv a3, a1
 ; RV64-NEXT:  .LBB101_2:
 ; RV64-NEXT:    vsetvli zero, a3, e64, m8, ta, ma
+; RV64-NEXT:    addi a3, sp, 16
+; RV64-NEXT:    vl1r.v v0, (a3) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsoxei64.v v8, (a0), v24, v0.t
 ; RV64-NEXT:    sub a3, a2, a1
 ; RV64-NEXT:    sltu a2, a2, a3
@@ -2319,15 +2321,13 @@ define void @vpscatter_baseidx_sext_nxv16i16_nxv16f64(<vscale x 16 x double> %va
 ; RV64-NEXT:    vslidedown.vx v0, v0, a1
 ; RV64-NEXT:    vsetvli zero, a2, e64, m8, ta, ma
 ; RV64-NEXT:    csrr a1, vlenb
-; RV64-NEXT:    slli a1, a1, 3
 ; RV64-NEXT:    add a1, sp, a1
 ; RV64-NEXT:    addi a1, a1, 16
 ; RV64-NEXT:    vl8r.v v8, (a1) # Unknown-size Folded Reload
-; RV64-NEXT:    addi a1, sp, 16
-; RV64-NEXT:    vl8r.v v16, (a1) # Unknown-size Folded Reload
 ; RV64-NEXT:    vsoxei64.v v8, (a0), v16, v0.t
 ; RV64-NEXT:    csrr a0, vlenb
-; RV64-NEXT:    slli a0, a0, 4
+; RV64-NEXT:    li a1, 10
+; RV64-NEXT:    mul a0, a0, a1
 ; RV64-NEXT:    add sp, sp, a0
 ; RV64-NEXT:    addi sp, sp, 16
 ; RV64-NEXT:    ret
-- 
GitLab


From 2cc0c2104909558680409f8a8f39755936305e72 Mon Sep 17 00:00:00 2001
From: Christian Sigg <csigg@google.com>
Date: Mon, 15 Apr 2024 07:39:02 +0200
Subject: [PATCH 199/464] [clang][bazel] Fix BUILD after
 f811d7b50957b801788d7b171ddeb25b1fda415a.

---
 utils/bazel/llvm-project-overlay/clang/BUILD.bazel | 1 +
 1 file changed, 1 insertion(+)

diff --git a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
index 2c3c39e5b53d..c2f77e3abca0 100644
--- a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
@@ -1928,6 +1928,7 @@ cc_library(
         ":frontend",
         ":lex",
         ":sema",
+        ":serialization",
         ":type_nodes_gen",
         "//llvm:AllTargetsAsmParsers",
         "//llvm:Analysis",
-- 
GitLab


From 46131aaf616c5cd97df0ec376a7e6ba475e1913c Mon Sep 17 00:00:00 2001
From: Mikhail Goncharov <goncharov.mikhail@gmail.com>
Date: Mon, 15 Apr 2024 08:41:32 +0200
Subject: [PATCH 200/464] Revert "Reapply "[Clang][Sema] Fix crash when 'this'
 is used in a dependent class scope function template specialization that
 instantiates to a static member function (#87541)" (#88311)"

This reverts commit aa80f3ec48419a73aafcc2ff947c6dd1e3734481.

See
https://github.com/llvm/llvm-project/pull/88311#issuecomment-2052291140.

There is a fix forward proposed but I am reverting this commit to fix
trunk.
---
 clang/docs/ReleaseNotes.rst                   |   2 -
 clang/include/clang/Sema/Sema.h               |  16 +--
 clang/lib/Sema/SemaExpr.cpp                   |  28 ++++-
 clang/lib/Sema/SemaExprCXX.cpp                |  61 +++++-----
 clang/lib/Sema/SemaExprMember.cpp             |  63 +++--------
 .../lib/Sema/SemaTemplateInstantiateDecl.cpp  |   8 --
 clang/lib/Sema/TreeTransform.h                |  65 +++++------
 .../SemaTemplate/instantiate-using-decl.cpp   |   2 +-
 ...ms-function-specialization-class-scope.cpp | 106 +-----------------
 9 files changed, 99 insertions(+), 252 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index ade8f4e93d5a..de5fa99bff5d 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -529,8 +529,6 @@ Bug Fixes to C++ Support
 - Fix an issue caused by not handling invalid cases when substituting into the parameter mapping of a constraint. Fixes (#GH86757).
 - Fixed a bug that prevented member function templates of class templates declared with a deduced return type
   from being explicitly specialized for a given implicit instantiation of the class template.
-- Fixed a crash when ``this`` is used in a dependent class scope function template specialization
-  that instantiates to a static member function.
 
 - Fix crash when inheriting from a cv-qualified type. Fixes:
   (`#35603 <https://github.com/llvm/llvm-project/issues/35603>`_)
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index c6e0332c3176..c6035445e901 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -5452,8 +5452,7 @@ public:
 
   ExprResult BuildDeclarationNameExpr(const CXXScopeSpec &SS, LookupResult &R,
                                       bool NeedsADL,
-                                      bool AcceptInvalidDecl = false,
-                                      bool NeedUnresolved = false);
+                                      bool AcceptInvalidDecl = false);
   ExprResult BuildDeclarationNameExpr(
       const CXXScopeSpec &SS, const DeclarationNameInfo &NameInfo, NamedDecl *D,
       NamedDecl *FoundD = nullptr,
@@ -6596,10 +6595,7 @@ public:
                             SourceLocation RParenLoc);
 
   //// ActOnCXXThis -  Parse 'this' pointer.
-  ExprResult ActOnCXXThis(SourceLocation Loc);
-
-  /// Check whether the type of 'this' is valid in the current context.
-  bool CheckCXXThisType(SourceLocation Loc, QualType Type);
+  ExprResult ActOnCXXThis(SourceLocation loc);
 
   /// Build a CXXThisExpr and mark it referenced in the current context.
   Expr *BuildCXXThisExpr(SourceLocation Loc, QualType Type, bool IsImplicit);
@@ -7022,14 +7018,10 @@ private:
   ///@{
 
 public:
-  /// Check whether an expression might be an implicit class member access.
-  bool isPotentialImplicitMemberAccess(const CXXScopeSpec &SS, LookupResult &R,
-                                       bool IsAddressOfOperand);
-
   ExprResult BuildPossibleImplicitMemberExpr(
       const CXXScopeSpec &SS, SourceLocation TemplateKWLoc, LookupResult &R,
-      const TemplateArgumentListInfo *TemplateArgs, const Scope *S);
-
+      const TemplateArgumentListInfo *TemplateArgs, const Scope *S,
+      UnresolvedLookupExpr *AsULE = nullptr);
   ExprResult
   BuildImplicitMemberExpr(const CXXScopeSpec &SS, SourceLocation TemplateKWLoc,
                           LookupResult &R,
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 505d068ac42e..c5395fb2068a 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -2917,9 +2917,26 @@ Sema::ActOnIdExpression(Scope *S, CXXScopeSpec &SS,
   // to get this right here so that we don't end up making a
   // spuriously dependent expression if we're inside a dependent
   // instance method.
-  if (isPotentialImplicitMemberAccess(SS, R, IsAddressOfOperand))
-    return BuildPossibleImplicitMemberExpr(SS, TemplateKWLoc, R, TemplateArgs,
-                                           S);
+  if (getLangOpts().CPlusPlus && !R.empty() &&
+      (*R.begin())->isCXXClassMember()) {
+    bool MightBeImplicitMember;
+    if (!IsAddressOfOperand)
+      MightBeImplicitMember = true;
+    else if (!SS.isEmpty())
+      MightBeImplicitMember = false;
+    else if (R.isOverloadedResult())
+      MightBeImplicitMember = false;
+    else if (R.isUnresolvableResult())
+      MightBeImplicitMember = true;
+    else
+      MightBeImplicitMember = isa<FieldDecl>(R.getFoundDecl()) ||
+                              isa<IndirectFieldDecl>(R.getFoundDecl()) ||
+                              isa<MSPropertyDecl>(R.getFoundDecl());
+
+    if (MightBeImplicitMember)
+      return BuildPossibleImplicitMemberExpr(SS, TemplateKWLoc,
+                                             R, TemplateArgs, S);
+  }
 
   if (TemplateArgs || TemplateKWLoc.isValid()) {
 
@@ -3430,11 +3447,10 @@ static bool ShouldLookupResultBeMultiVersionOverload(const LookupResult &R) {
 
 ExprResult Sema::BuildDeclarationNameExpr(const CXXScopeSpec &SS,
                                           LookupResult &R, bool NeedsADL,
-                                          bool AcceptInvalidDecl,
-                                          bool NeedUnresolved) {
+                                          bool AcceptInvalidDecl) {
   // If this is a single, fully-resolved result and we don't need ADL,
   // just build an ordinary singleton decl ref.
-  if (!NeedUnresolved && !NeedsADL && R.isSingleResult() &&
+  if (!NeedsADL && R.isSingleResult() &&
       !R.getAsSingle<FunctionTemplateDecl>() &&
       !ShouldLookupResultBeMultiVersionOverload(R))
     return BuildDeclarationNameExpr(SS, R.getLookupNameInfo(), R.getFoundDecl(),
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index 25f23a3abf17..fa7b92a43346 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -1416,42 +1416,26 @@ bool Sema::CheckCXXThisCapture(SourceLocation Loc, const bool Explicit,
 }
 
 ExprResult Sema::ActOnCXXThis(SourceLocation Loc) {
-  // C++20 [expr.prim.this]p1:
-  //   The keyword this names a pointer to the object for which an
-  //   implicit object member function is invoked or a non-static
-  //   data member's initializer is evaluated.
+  /// C++ 9.3.2: In the body of a non-static member function, the keyword this
+  /// is a non-lvalue expression whose value is the address of the object for
+  /// which the function is called.
   QualType ThisTy = getCurrentThisType();
 
-  if (CheckCXXThisType(Loc, ThisTy))
-    return ExprError();
+  if (ThisTy.isNull()) {
+    DeclContext *DC = getFunctionLevelDeclContext();
 
-  return BuildCXXThisExpr(Loc, ThisTy, /*IsImplicit=*/false);
-}
+    if (const auto *Method = dyn_cast<CXXMethodDecl>(DC);
+        Method && Method->isExplicitObjectMemberFunction()) {
+      return Diag(Loc, diag::err_invalid_this_use) << 1;
+    }
 
-bool Sema::CheckCXXThisType(SourceLocation Loc, QualType Type) {
-  if (!Type.isNull())
-    return false;
+    if (isLambdaCallWithExplicitObjectParameter(CurContext))
+      return Diag(Loc, diag::err_invalid_this_use) << 1;
 
-  // C++20 [expr.prim.this]p3:
-  //   If a declaration declares a member function or member function template
-  //   of a class X, the expression this is a prvalue of type
-  //   "pointer to cv-qualifier-seq X" wherever X is the current class between
-  //   the optional cv-qualifier-seq and the end of the function-definition,
-  //   member-declarator, or declarator. It shall not appear within the
-  //   declaration of either a static member function or an explicit object
-  //   member function of the current class (although its type and value
-  //   category are defined within such member functions as they are within
-  //   an implicit object member function).
-  DeclContext *DC = getFunctionLevelDeclContext();
-  if (const auto *Method = dyn_cast<CXXMethodDecl>(DC);
-      Method && Method->isExplicitObjectMemberFunction()) {
-    Diag(Loc, diag::err_invalid_this_use) << 1;
-  } else if (isLambdaCallWithExplicitObjectParameter(CurContext)) {
-    Diag(Loc, diag::err_invalid_this_use) << 1;
-  } else {
-    Diag(Loc, diag::err_invalid_this_use) << 0;
+    return Diag(Loc, diag::err_invalid_this_use) << 0;
   }
-  return true;
+
+  return BuildCXXThisExpr(Loc, ThisTy, /*IsImplicit=*/false);
 }
 
 Expr *Sema::BuildCXXThisExpr(SourceLocation Loc, QualType Type,
@@ -8658,8 +8642,21 @@ static ExprResult attemptRecovery(Sema &SemaRef,
 
       // Detect and handle the case where the decl might be an implicit
       // member.
-      if (SemaRef.isPotentialImplicitMemberAccess(
-              NewSS, R, Consumer.isAddressOfOperand()))
+      bool MightBeImplicitMember;
+      if (!Consumer.isAddressOfOperand())
+        MightBeImplicitMember = true;
+      else if (!NewSS.isEmpty())
+        MightBeImplicitMember = false;
+      else if (R.isOverloadedResult())
+        MightBeImplicitMember = false;
+      else if (R.isUnresolvableResult())
+        MightBeImplicitMember = true;
+      else
+        MightBeImplicitMember = isa<FieldDecl>(ND) ||
+                                isa<IndirectFieldDecl>(ND) ||
+                                isa<MSPropertyDecl>(ND);
+
+      if (MightBeImplicitMember)
         return SemaRef.BuildPossibleImplicitMemberExpr(
             NewSS, /*TemplateKWLoc*/ SourceLocation(), R,
             /*TemplateArgs*/ nullptr, /*S*/ nullptr);
diff --git a/clang/lib/Sema/SemaExprMember.cpp b/clang/lib/Sema/SemaExprMember.cpp
index eeac753a3489..32998ae60eaf 100644
--- a/clang/lib/Sema/SemaExprMember.cpp
+++ b/clang/lib/Sema/SemaExprMember.cpp
@@ -61,10 +61,6 @@ enum IMAKind {
   /// The reference is a contextually-permitted abstract member reference.
   IMA_Abstract,
 
-  /// Whether the context is static is dependent on the enclosing template (i.e.
-  /// in a dependent class scope explicit specialization).
-  IMA_Dependent,
-
   /// The reference may be to an unresolved using declaration and the
   /// context is not an instance method.
   IMA_Unresolved_StaticOrExplicitContext,
@@ -95,18 +91,10 @@ static IMAKind ClassifyImplicitMemberAccess(Sema &SemaRef,
 
   DeclContext *DC = SemaRef.getFunctionLevelDeclContext();
 
-  bool couldInstantiateToStatic = false;
-  bool isStaticOrExplicitContext = SemaRef.CXXThisTypeOverride.isNull();
-
-  if (auto *MD = dyn_cast<CXXMethodDecl>(DC)) {
-    if (MD->isImplicitObjectMemberFunction()) {
-      isStaticOrExplicitContext = false;
-      // A dependent class scope function template explicit specialization
-      // that is neither declared 'static' nor with an explicit object
-      // parameter could instantiate to a static or non-static member function.
-      couldInstantiateToStatic = MD->getDependentSpecializationInfo();
-    }
-  }
+  bool isStaticOrExplicitContext =
+      SemaRef.CXXThisTypeOverride.isNull() &&
+      (!isa<CXXMethodDecl>(DC) || cast<CXXMethodDecl>(DC)->isStatic() ||
+       cast<CXXMethodDecl>(DC)->isExplicitObjectMemberFunction());
 
   if (R.isUnresolvableResult())
     return isStaticOrExplicitContext ? IMA_Unresolved_StaticOrExplicitContext
@@ -135,9 +123,6 @@ static IMAKind ClassifyImplicitMemberAccess(Sema &SemaRef,
   if (Classes.empty())
     return IMA_Static;
 
-  if (couldInstantiateToStatic)
-    return IMA_Dependent;
-
   // C++11 [expr.prim.general]p12:
   //   An id-expression that denotes a non-static data member or non-static
   //   member function of a class can only be used:
@@ -278,52 +263,32 @@ static void diagnoseInstanceReference(Sema &SemaRef,
   }
 }
 
-bool Sema::isPotentialImplicitMemberAccess(const CXXScopeSpec &SS,
-                                           LookupResult &R,
-                                           bool IsAddressOfOperand) {
-  if (!getLangOpts().CPlusPlus)
-    return false;
-  else if (R.empty() || !R.begin()->isCXXClassMember())
-    return false;
-  else if (!IsAddressOfOperand)
-    return true;
-  else if (!SS.isEmpty())
-    return false;
-  else if (R.isOverloadedResult())
-    return false;
-  else if (R.isUnresolvableResult())
-    return true;
-  else
-    return isa<FieldDecl, IndirectFieldDecl, MSPropertyDecl>(R.getFoundDecl());
-}
-
 /// Builds an expression which might be an implicit member expression.
 ExprResult Sema::BuildPossibleImplicitMemberExpr(
     const CXXScopeSpec &SS, SourceLocation TemplateKWLoc, LookupResult &R,
-    const TemplateArgumentListInfo *TemplateArgs, const Scope *S) {
-  switch (IMAKind Classification = ClassifyImplicitMemberAccess(*this, R)) {
+    const TemplateArgumentListInfo *TemplateArgs, const Scope *S,
+    UnresolvedLookupExpr *AsULE) {
+  switch (ClassifyImplicitMemberAccess(*this, R)) {
   case IMA_Instance:
+    return BuildImplicitMemberExpr(SS, TemplateKWLoc, R, TemplateArgs, true, S);
+
   case IMA_Mixed:
   case IMA_Mixed_Unrelated:
   case IMA_Unresolved:
-    return BuildImplicitMemberExpr(
-        SS, TemplateKWLoc, R, TemplateArgs,
-        /*IsKnownInstance=*/Classification == IMA_Instance, S);
+    return BuildImplicitMemberExpr(SS, TemplateKWLoc, R, TemplateArgs, false,
+                                   S);
+
   case IMA_Field_Uneval_Context:
     Diag(R.getNameLoc(), diag::warn_cxx98_compat_non_static_member_use)
       << R.getLookupNameInfo().getName();
     [[fallthrough]];
   case IMA_Static:
   case IMA_Abstract:
-  case IMA_Dependent:
   case IMA_Mixed_StaticOrExplicitContext:
   case IMA_Unresolved_StaticOrExplicitContext:
     if (TemplateArgs || TemplateKWLoc.isValid())
-      return BuildTemplateIdExpr(SS, TemplateKWLoc, R, /*RequiresADL=*/false,
-                                 TemplateArgs);
-    return BuildDeclarationNameExpr(
-        SS, R, /*NeedsADL=*/false, /*AcceptInvalidDecl=*/false,
-        /*NeedUnresolved=*/Classification == IMA_Dependent);
+      return BuildTemplateIdExpr(SS, TemplateKWLoc, R, false, TemplateArgs);
+    return AsULE ? AsULE : BuildDeclarationNameExpr(SS, R, false);
 
   case IMA_Error_StaticOrExplicitContext:
   case IMA_Error_Unrelated:
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index 21a139e434cb..02d0b35ce940 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -5097,14 +5097,6 @@ void Sema::InstantiateFunctionDefinition(SourceLocation PointOfInstantiation,
   EnterExpressionEvaluationContext EvalContext(
       *this, Sema::ExpressionEvaluationContext::PotentiallyEvaluated);
 
-  Qualifiers ThisTypeQuals;
-  CXXRecordDecl *ThisContext = nullptr;
-  if (CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(Function)) {
-    ThisContext = Method->getParent();
-    ThisTypeQuals = Method->getMethodQualifiers();
-  }
-  CXXThisScopeRAII ThisScope(*this, ThisContext, ThisTypeQuals);
-
   // Introduce a new scope where local variable instantiations will be
   // recorded, unless we're actually a member function within a local
   // class, in which case we need to merge our results with the parent
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index ee6bd228b459..fdf84c512c49 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -795,9 +795,6 @@ public:
       ParenExpr *PE, DependentScopeDeclRefExpr *DRE, bool IsAddressOfOperand,
       TypeSourceInfo **RecoveryTSI);
 
-  ExprResult TransformUnresolvedLookupExpr(UnresolvedLookupExpr *E,
-                                           bool IsAddressOfOperand);
-
   StmtResult TransformOMPExecutableDirective(OMPExecutableDirective *S);
 
 // FIXME: We use LLVM_ATTRIBUTE_NOINLINE because inlining causes a ridiculous
@@ -3312,13 +3309,12 @@ public:
 
   /// Build a new C++ "this" expression.
   ///
-  /// By default, performs semantic analysis to build a new "this" expression.
-  /// Subclasses may override this routine to provide different behavior.
+  /// By default, builds a new "this" expression without performing any
+  /// semantic analysis. Subclasses may override this routine to provide
+  /// different behavior.
   ExprResult RebuildCXXThisExpr(SourceLocation ThisLoc,
                                 QualType ThisType,
                                 bool isImplicit) {
-    if (getSema().CheckCXXThisType(ThisLoc, ThisType))
-      return ExprError();
     return getSema().BuildCXXThisExpr(ThisLoc, ThisType, isImplicit);
   }
 
@@ -11369,11 +11365,7 @@ template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformAddressOfOperand(Expr *E) {
   if (DependentScopeDeclRefExpr *DRE = dyn_cast<DependentScopeDeclRefExpr>(E))
-    return getDerived().TransformDependentScopeDeclRefExpr(
-        DRE, /*IsAddressOfOperand=*/true, nullptr);
-  else if (UnresolvedLookupExpr *ULE = dyn_cast<UnresolvedLookupExpr>(E))
-    return getDerived().TransformUnresolvedLookupExpr(
-        ULE, /*IsAddressOfOperand=*/true);
+    return getDerived().TransformDependentScopeDeclRefExpr(DRE, true, nullptr);
   else
     return getDerived().TransformExpr(E);
 }
@@ -13079,16 +13071,10 @@ bool TreeTransform<Derived>::TransformOverloadExprDecls(OverloadExpr *Old,
   return false;
 }
 
-template <typename Derived>
-ExprResult TreeTransform<Derived>::TransformUnresolvedLookupExpr(
-    UnresolvedLookupExpr *Old) {
-  return TransformUnresolvedLookupExpr(Old, /*IsAddressOfOperand=*/false);
-}
-
-template <typename Derived>
+template<typename Derived>
 ExprResult
-TreeTransform<Derived>::TransformUnresolvedLookupExpr(UnresolvedLookupExpr *Old,
-                                                      bool IsAddressOfOperand) {
+TreeTransform<Derived>::TransformUnresolvedLookupExpr(
+                                                  UnresolvedLookupExpr *Old) {
   LookupResult R(SemaRef, Old->getName(), Old->getNameLoc(),
                  Sema::LookupOrdinaryName);
 
@@ -13120,8 +13106,26 @@ TreeTransform<Derived>::TransformUnresolvedLookupExpr(UnresolvedLookupExpr *Old,
     R.setNamingClass(NamingClass);
   }
 
-  // Rebuild the template arguments, if any.
   SourceLocation TemplateKWLoc = Old->getTemplateKeywordLoc();
+
+  // If we have neither explicit template arguments, nor the template keyword,
+  // it's a normal declaration name or member reference.
+  if (!Old->hasExplicitTemplateArgs() && !TemplateKWLoc.isValid()) {
+    NamedDecl *D = R.getAsSingle<NamedDecl>();
+    // In a C++11 unevaluated context, an UnresolvedLookupExpr might refer to an
+    // instance member. In other contexts, BuildPossibleImplicitMemberExpr will
+    // give a good diagnostic.
+    if (D && D->isCXXInstanceMember()) {
+      return SemaRef.BuildPossibleImplicitMemberExpr(SS, TemplateKWLoc, R,
+                                                     /*TemplateArgs=*/nullptr,
+                                                     /*Scope=*/nullptr);
+    }
+
+    return getDerived().RebuildDeclarationNameExpr(SS, R, Old->requiresADL());
+  }
+
+  // If we have template arguments, rebuild them, then rebuild the
+  // templateid expression.
   TemplateArgumentListInfo TransArgs(Old->getLAngleLoc(), Old->getRAngleLoc());
   if (Old->hasExplicitTemplateArgs() &&
       getDerived().TransformTemplateArguments(Old->getTemplateArgs(),
@@ -13131,23 +13135,6 @@ TreeTransform<Derived>::TransformUnresolvedLookupExpr(UnresolvedLookupExpr *Old,
     return ExprError();
   }
 
-  // An UnresolvedLookupExpr can refer to a class member. This occurs e.g. when
-  // a non-static data member is named in an unevaluated operand, or when
-  // a member is named in a dependent class scope function template explicit
-  // specialization that is neither declared static nor with an explicit object
-  // parameter.
-  if (SemaRef.isPotentialImplicitMemberAccess(SS, R, IsAddressOfOperand))
-    return SemaRef.BuildPossibleImplicitMemberExpr(
-        SS, TemplateKWLoc, R,
-        Old->hasExplicitTemplateArgs() ? &TransArgs : nullptr,
-        /*S=*/nullptr);
-
-  // If we have neither explicit template arguments, nor the template keyword,
-  // it's a normal declaration name or member reference.
-  if (!Old->hasExplicitTemplateArgs() && !TemplateKWLoc.isValid())
-    return getDerived().RebuildDeclarationNameExpr(SS, R, Old->requiresADL());
-
-  // If we have template arguments, then rebuild the template-id expression.
   return getDerived().RebuildTemplateIdExpr(SS, TemplateKWLoc, R,
                                             Old->requiresADL(), &TransArgs);
 }
diff --git a/clang/test/SemaTemplate/instantiate-using-decl.cpp b/clang/test/SemaTemplate/instantiate-using-decl.cpp
index 28d837643851..0bbb3ca9c88c 100644
--- a/clang/test/SemaTemplate/instantiate-using-decl.cpp
+++ b/clang/test/SemaTemplate/instantiate-using-decl.cpp
@@ -121,7 +121,7 @@ template <typename Scalar> struct Derived : Base<Scalar> {
     (void)&field;
     // expected-error@+1 {{call to non-static member function without an object argument}}
     (void)method;
-    // expected-error@+1 {{must explicitly qualify name of member function when taking its address}}
+    // expected-error@+1 {{call to non-static member function without an object argument}}
     (void)&method;
     // expected-error@+1 {{call to non-static member function without an object argument}}
     method();
diff --git a/clang/test/SemaTemplate/ms-function-specialization-class-scope.cpp b/clang/test/SemaTemplate/ms-function-specialization-class-scope.cpp
index f7b59f2ac715..dcab9bfaeabc 100644
--- a/clang/test/SemaTemplate/ms-function-specialization-class-scope.cpp
+++ b/clang/test/SemaTemplate/ms-function-specialization-class-scope.cpp
@@ -1,6 +1,7 @@
-// RUN: %clang_cc1 -fms-extensions -fsyntax-only -Wno-unused-value -verify %s
-// RUN: %clang_cc1 -fms-extensions -fdelayed-template-parsing -fsyntax-only -Wno-unused-value -verify %s
+// RUN: %clang_cc1 -fms-extensions -fsyntax-only -verify %s
+// RUN: %clang_cc1 -fms-extensions -fdelayed-template-parsing -fsyntax-only -verify %s
 
+// expected-no-diagnostics
 class A {
 public:
   template<class U> A(U p) {}
@@ -75,104 +76,3 @@ struct S {
   int f<0>(int);
 };
 }
-
-namespace UsesThis {
-  template<typename T>
-  struct A {
-    int x;
-
-    static inline int y;
-
-    template<typename U = void>
-    static void f();
-
-    template<typename U = void>
-    void g();
-
-    template<typename U>
-    static auto h() -> A*;
-
-    void i();
-
-    static void j();
-
-    template<>
-    void f<int>() {
-      this->x; // expected-error {{invalid use of 'this' outside of a non-static member function}}
-      x; // expected-error {{invalid use of member 'x' in static member function}}
-      A::x; // expected-error {{invalid use of member 'x' in static member function}}
-      +x; // expected-error {{invalid use of member 'x' in static member function}}
-      +A::x; // expected-error {{invalid use of member 'x' in static member function}}
-      &x; // expected-error {{invalid use of member 'x' in static member function}}
-      &A::x;
-      this->y; // expected-error {{invalid use of 'this' outside of a non-static member function}}
-      y;
-      A::y;
-      +y;
-      +A::y;
-      &y;
-      &A::y;
-      f();
-      f<void>();
-      g(); // expected-error {{call to non-static member function without an object argument}}
-      g<void>(); // expected-error {{call to non-static member function without an object argument}}
-      i(); // expected-error {{call to non-static member function without an object argument}}
-      j();
-      &i; // expected-error 2{{must explicitly qualify name of member function when taking its address}}
-      &j;
-      &A::i;
-      &A::j;
-    }
-
-    template<>
-    void g<int>() {
-      this->x;
-      x;
-      A::x;
-      +x;
-      +A::x;
-      &x;
-      &A::x;
-      this->y;
-      y;
-      A::y;
-      +y;
-      +A::y;
-      &y;
-      &A::y;
-      f();
-      f<void>();
-      g();
-      g<void>();
-      i();
-      j();
-      &i; // expected-error 2{{must explicitly qualify name of member function when taking its address}}
-      &j;
-      &A::i;
-      &A::j;
-    }
-
-    template<>
-    auto h<int>() -> decltype(this); // expected-error {{'this' cannot be used in a static member function declaration}}
-  };
-
-  template struct A<int>; // expected-note 3{{in instantiation of}}
-
-  template <typename T>
-  struct Foo {
-    template <typename X>
-    int bar(X x) {
-      return 0;
-    }
-
-    template <>
-    int bar(int x) {
-      return bar(5.0); // ok
-    }
-  };
-
-  void call() {
-    Foo<double> f;
-    f.bar(1);
-  }
-}
-- 
GitLab


From a71565d75e23fc28076aa5bf1c5cf4432623afc5 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Mon, 15 Apr 2024 15:42:38 +0900
Subject: [PATCH 201/464] [ARM] Don't include IRBuilder.h in ARMISelLowering.h
 (NFC)

Just a forward declaration is sufficient.
---
 llvm/lib/Target/ARM/ARMISelLowering.h  | 2 +-
 llvm/lib/Target/ARM/ARMParallelDSP.cpp | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h
index b13ddf697cb8..26ef295e3d3f 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.h
+++ b/llvm/lib/Target/ARM/ARMISelLowering.h
@@ -27,7 +27,6 @@
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/CallingConv.h"
 #include "llvm/IR/Function.h"
-#include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InlineAsm.h"
 #include "llvm/Support/CodeGen.h"
 #include <optional>
@@ -42,6 +41,7 @@ class FunctionLoweringInfo;
 class GlobalValue;
 class InstrItineraryData;
 class Instruction;
+class IRBuilderBase;
 class MachineBasicBlock;
 class MachineInstr;
 class SelectionDAG;
diff --git a/llvm/lib/Target/ARM/ARMParallelDSP.cpp b/llvm/lib/Target/ARM/ARMParallelDSP.cpp
index 1efda5d1c937..8c1b3325f155 100644
--- a/llvm/lib/Target/ARM/ARMParallelDSP.cpp
+++ b/llvm/lib/Target/ARM/ARMParallelDSP.cpp
@@ -26,6 +26,7 @@
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicsARM.h"
+#include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/NoFolder.h"
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/Pass.h"
-- 
GitLab


From 302d0f3476c0fc68d796e7b8a0d4237121ec33fe Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Mon, 15 Apr 2024 08:33:14 +0100
Subject: [PATCH 202/464] [VectorCombine] Add some tests for converting
 shuffles to identities. NFC

---
 .../AArch64/shuffletoidentity.ll              | 647 ++++++++++++++++++
 1 file changed, 647 insertions(+)
 create mode 100644 llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll

diff --git a/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll b/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll
new file mode 100644
index 000000000000..d96dfec84916
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/AArch64/shuffletoidentity.ll
@@ -0,0 +1,647 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes='vector-combine' -S %s | FileCheck %s
+
+target triple = "aarch64"
+
+define <8 x i8> @trivial(<8 x i8> %a) {
+; CHECK-LABEL: @trivial(
+; CHECK-NEXT:    [[AB:%.*]] = shufflevector <8 x i8> [[A:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[AT:%.*]] = shufflevector <8 x i8> [[A]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i8> [[AT]], <4 x i8> [[AB]], <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x i8> [[R]]
+;
+  %ab = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %at = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %r = shufflevector <4 x i8> %at, <4 x i8> %ab, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x i8> %r
+}
+
+define <8 x i8> @add(<8 x i8> %a, <8 x i8> %b) {
+; CHECK-LABEL: @add(
+; CHECK-NEXT:    [[AB:%.*]] = shufflevector <8 x i8> [[A:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[AT:%.*]] = shufflevector <8 x i8> [[A]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[BB:%.*]] = shufflevector <8 x i8> [[B:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[BT:%.*]] = shufflevector <8 x i8> [[B]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[ABT:%.*]] = add <4 x i8> [[AT]], [[BT]]
+; CHECK-NEXT:    [[ABB:%.*]] = add <4 x i8> [[AB]], [[BB]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i8> [[ABT]], <4 x i8> [[ABB]], <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x i8> [[R]]
+;
+  %ab = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %at = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %bb = shufflevector <8 x i8> %b, <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %bt = shufflevector <8 x i8> %b, <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %abt = add <4 x i8> %at, %bt
+  %abb = add <4 x i8> %ab, %bb
+  %r = shufflevector <4 x i8> %abt, <4 x i8> %abb, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x i8> %r
+}
+
+; Different opcodes
+define <8 x i8> @wrong_addsub(<8 x i8> %a, <8 x i8> %b) {
+; CHECK-LABEL: @wrong_addsub(
+; CHECK-NEXT:    [[AB:%.*]] = shufflevector <8 x i8> [[A:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[AT:%.*]] = shufflevector <8 x i8> [[A]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[BB:%.*]] = shufflevector <8 x i8> [[B:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[BT:%.*]] = shufflevector <8 x i8> [[B]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[ABT:%.*]] = add <4 x i8> [[AT]], [[BT]]
+; CHECK-NEXT:    [[ABB:%.*]] = sub <4 x i8> [[AB]], [[BB]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i8> [[ABT]], <4 x i8> [[ABB]], <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x i8> [[R]]
+;
+  %ab = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %at = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %bb = shufflevector <8 x i8> %b, <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %bt = shufflevector <8 x i8> %b, <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %abt = add <4 x i8> %at, %bt
+  %abb = sub <4 x i8> %ab, %bb
+  %r = shufflevector <4 x i8> %abt, <4 x i8> %abb, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x i8> %r
+}
+
+; Different lanes that do not make an identity
+define <8 x i8> @wrong_lanes(<8 x i8> %a, <8 x i8> %b) {
+; CHECK-LABEL: @wrong_lanes(
+; CHECK-NEXT:    [[AB:%.*]] = shufflevector <8 x i8> [[A:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[AT:%.*]] = shufflevector <8 x i8> [[A]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[BB:%.*]] = shufflevector <8 x i8> [[B:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[BT:%.*]] = shufflevector <8 x i8> [[B]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[ABT:%.*]] = add <4 x i8> [[AT]], [[BT]]
+; CHECK-NEXT:    [[ABB:%.*]] = add <4 x i8> [[AB]], [[BB]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i8> [[ABT]], <4 x i8> [[ABB]], <8 x i32> <i32 6, i32 7, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x i8> [[R]]
+;
+  %ab = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %at = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %bb = shufflevector <8 x i8> %b, <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %bt = shufflevector <8 x i8> %b, <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %abt = add <4 x i8> %at, %bt
+  %abb = add <4 x i8> %ab, %bb
+  %r = shufflevector <4 x i8> %abt, <4 x i8> %abb, <8 x i32> <i32 6, i32 7, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x i8> %r
+}
+
+define <8 x half> @fadd(<8 x half> %a, <8 x half> %b) {
+; CHECK-LABEL: @fadd(
+; CHECK-NEXT:    [[AB:%.*]] = shufflevector <8 x half> [[A:%.*]], <8 x half> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[AT:%.*]] = shufflevector <8 x half> [[A]], <8 x half> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[BB:%.*]] = shufflevector <8 x half> [[B:%.*]], <8 x half> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[BT:%.*]] = shufflevector <8 x half> [[B]], <8 x half> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[ABT:%.*]] = fadd <4 x half> [[AT]], [[BT]]
+; CHECK-NEXT:    [[ABB:%.*]] = fadd <4 x half> [[AB]], [[BB]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x half> [[ABT]], <4 x half> [[ABB]], <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x half> [[R]]
+;
+  %ab = shufflevector <8 x half> %a, <8 x half> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %at = shufflevector <8 x half> %a, <8 x half> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %bb = shufflevector <8 x half> %b, <8 x half> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %bt = shufflevector <8 x half> %b, <8 x half> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %abt = fadd <4 x half> %at, %bt
+  %abb = fadd <4 x half> %ab, %bb
+  %r = shufflevector <4 x half> %abt, <4 x half> %abb, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x half> %r
+}
+
+define <8 x half> @fneg(<8 x half> %a, <8 x half> %b) {
+; CHECK-LABEL: @fneg(
+; CHECK-NEXT:    [[AB:%.*]] = shufflevector <8 x half> [[A:%.*]], <8 x half> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[AT:%.*]] = shufflevector <8 x half> [[A]], <8 x half> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[ABT:%.*]] = fneg <4 x half> [[AT]]
+; CHECK-NEXT:    [[ABB:%.*]] = fneg <4 x half> [[AB]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x half> [[ABT]], <4 x half> [[ABB]], <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x half> [[R]]
+;
+  %ab = shufflevector <8 x half> %a, <8 x half> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %at = shufflevector <8 x half> %a, <8 x half> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %abt = fneg <4 x half> %at
+  %abb = fneg <4 x half> %ab
+  %r = shufflevector <4 x half> %abt, <4 x half> %abb, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x half> %r
+}
+
+define <8 x i8> @abs(<8 x i8> %a) {
+; CHECK-LABEL: @abs(
+; CHECK-NEXT:    [[AB:%.*]] = shufflevector <8 x i8> [[A:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[AT:%.*]] = shufflevector <8 x i8> [[A]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[ABT:%.*]] = call <4 x i8> @llvm.abs.v4i8(<4 x i8> [[AT]], i1 true)
+; CHECK-NEXT:    [[ABB:%.*]] = call <4 x i8> @llvm.abs.v4i8(<4 x i8> [[AB]], i1 false)
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i8> [[ABT]], <4 x i8> [[ABB]], <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x i8> [[R]]
+;
+  %ab = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %at = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %abt = call <4 x i8> @llvm.abs.v4i8(<4 x i8> %at, i1 true)
+  %abb = call <4 x i8> @llvm.abs.v4i8(<4 x i8> %ab, i1 false)
+  %r = shufflevector <4 x i8> %abt, <4 x i8> %abb, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x i8> %r
+}
+
+define <8 x half> @splat0(<8 x half> %a, <8 x half> %b) {
+; CHECK-LABEL: @splat0(
+; CHECK-NEXT:    [[AB:%.*]] = shufflevector <8 x half> [[A:%.*]], <8 x half> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[AT:%.*]] = shufflevector <8 x half> [[A]], <8 x half> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[BS:%.*]] = shufflevector <8 x half> [[B:%.*]], <8 x half> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[ABT:%.*]] = fadd <4 x half> [[AT]], [[BS]]
+; CHECK-NEXT:    [[ABB:%.*]] = fadd <4 x half> [[AB]], [[BS]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x half> [[ABT]], <4 x half> [[ABB]], <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x half> [[R]]
+;
+  %ab = shufflevector <8 x half> %a, <8 x half> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %at = shufflevector <8 x half> %a, <8 x half> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %bs = shufflevector <8 x half> %b, <8 x half> poison, <4 x i32> zeroinitializer
+  %abt = fadd <4 x half> %at, %bs
+  %abb = fadd <4 x half> %ab, %bs
+  %r = shufflevector <4 x half> %abt, <4 x half> %abb, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x half> %r
+}
+
+define <8 x half> @splat2(<8 x half> %a, <8 x half> %b) {
+; CHECK-LABEL: @splat2(
+; CHECK-NEXT:    [[AB:%.*]] = shufflevector <8 x half> [[A:%.*]], <8 x half> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[AT:%.*]] = shufflevector <8 x half> [[A]], <8 x half> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[BS:%.*]] = shufflevector <8 x half> [[B:%.*]], <8 x half> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    [[ABT:%.*]] = fadd <4 x half> [[AT]], [[BS]]
+; CHECK-NEXT:    [[ABB:%.*]] = fadd <4 x half> [[AB]], [[BS]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x half> [[ABT]], <4 x half> [[ABB]], <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x half> [[R]]
+;
+  %ab = shufflevector <8 x half> %a, <8 x half> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %at = shufflevector <8 x half> %a, <8 x half> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %bs = shufflevector <8 x half> %b, <8 x half> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+  %abt = fadd <4 x half> %at, %bs
+  %abb = fadd <4 x half> %ab, %bs
+  %r = shufflevector <4 x half> %abt, <4 x half> %abb, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x half> %r
+}
+
+define <8 x half> @splatandidentity(<8 x half> %a, <8 x half> %b) {
+; CHECK-LABEL: @splatandidentity(
+; CHECK-NEXT:    [[AB:%.*]] = shufflevector <8 x half> [[A:%.*]], <8 x half> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[AT:%.*]] = shufflevector <8 x half> [[A]], <8 x half> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[BS:%.*]] = shufflevector <8 x half> [[A]], <8 x half> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[ABT:%.*]] = fadd <4 x half> [[AT]], [[BS]]
+; CHECK-NEXT:    [[ABB:%.*]] = fadd <4 x half> [[AB]], [[BS]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x half> [[ABT]], <4 x half> [[ABB]], <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x half> [[R]]
+;
+  %ab = shufflevector <8 x half> %a, <8 x half> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %at = shufflevector <8 x half> %a, <8 x half> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %bs = shufflevector <8 x half> %a, <8 x half> poison, <4 x i32> zeroinitializer
+  %abt = fadd <4 x half> %at, %bs
+  %abb = fadd <4 x half> %ab, %bs
+  %r = shufflevector <4 x half> %abt, <4 x half> %abb, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x half> %r
+}
+
+define <8 x half> @splattwice(<8 x half> %a, <8 x half> %b) {
+; CHECK-LABEL: @splattwice(
+; CHECK-NEXT:    [[AS:%.*]] = shufflevector <8 x half> [[A:%.*]], <8 x half> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BS:%.*]] = shufflevector <8 x half> [[B:%.*]], <8 x half> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[AB1:%.*]] = fadd <4 x half> [[AS]], [[BS]]
+; CHECK-NEXT:    [[AB2:%.*]] = fadd <4 x half> [[AS]], [[BS]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x half> [[AB1]], <4 x half> [[AB2]], <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x half> [[R]]
+;
+  %as = shufflevector <8 x half> %a, <8 x half> poison, <4 x i32> zeroinitializer
+  %bs = shufflevector <8 x half> %b, <8 x half> poison, <4 x i32> zeroinitializer
+  %ab1 = fadd <4 x half> %as, %bs
+  %ab2 = fadd <4 x half> %as, %bs
+  %r = shufflevector <4 x half> %ab1, <4 x half> %ab2, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x half> %r
+}
+
+define <8 x i8> @undeflane(<8 x i8> %a, <8 x i8> %b) {
+; CHECK-LABEL: @undeflane(
+; CHECK-NEXT:    [[AB:%.*]] = shufflevector <8 x i8> [[A:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[AT:%.*]] = shufflevector <8 x i8> [[A]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[BB:%.*]] = shufflevector <8 x i8> [[B:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[BT:%.*]] = shufflevector <8 x i8> [[B]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[ABT:%.*]] = add <4 x i8> [[AT]], [[BT]]
+; CHECK-NEXT:    [[ABB:%.*]] = add <4 x i8> [[AB]], [[BB]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i8> [[ABT]], <4 x i8> [[ABB]], <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 poison, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x i8> [[R]]
+;
+  %ab = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %at = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %bb = shufflevector <8 x i8> %b, <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %bt = shufflevector <8 x i8> %b, <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %abt = add <4 x i8> %at, %bt
+  %abb = add <4 x i8> %ab, %bb
+  %r = shufflevector <4 x i8> %abt, <4 x i8> %abb, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 poison, i32 1, i32 0>
+  ret <8 x i8> %r
+}
+
+define <8 x i8> @constantsplat(<8 x i8> %a) {
+; CHECK-LABEL: @constantsplat(
+; CHECK-NEXT:    [[AB:%.*]] = shufflevector <8 x i8> [[A:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[AT:%.*]] = shufflevector <8 x i8> [[A]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[ABT:%.*]] = add <4 x i8> [[AT]], <i8 10, i8 10, i8 10, i8 10>
+; CHECK-NEXT:    [[ABB:%.*]] = add <4 x i8> [[AB]], <i8 10, i8 10, i8 10, i8 10>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i8> [[ABT]], <4 x i8> [[ABB]], <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x i8> [[R]]
+;
+  %ab = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %at = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %abt = add <4 x i8> %at, <i8 10, i8 10, i8 10, i8 10>
+  %abb = add <4 x i8> %ab, <i8 10, i8 10, i8 10, i8 10>
+  %r = shufflevector <4 x i8> %abt, <4 x i8> %abb, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x i8> %r
+}
+
+define <8 x i8> @constantdiff(<8 x i8> %a) {
+; CHECK-LABEL: @constantdiff(
+; CHECK-NEXT:    [[AB:%.*]] = shufflevector <8 x i8> [[A:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[AT:%.*]] = shufflevector <8 x i8> [[A]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[ABT:%.*]] = add <4 x i8> [[AT]], <i8 1, i8 2, i8 3, i8 4>
+; CHECK-NEXT:    [[ABB:%.*]] = add <4 x i8> [[AB]], <i8 5, i8 6, i8 7, i8 8>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i8> [[ABT]], <4 x i8> [[ABB]], <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x i8> [[R]]
+;
+  %ab = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %at = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %abt = add <4 x i8> %at, <i8 1, i8 2, i8 3, i8 4>
+  %abb = add <4 x i8> %ab, <i8 5, i8 6, i8 7, i8 8>
+  %r = shufflevector <4 x i8> %abt, <4 x i8> %abb, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x i8> %r
+}
+
+define <8 x i8> @constantdiff2(<8 x i8> %a) {
+; CHECK-LABEL: @constantdiff2(
+; CHECK-NEXT:    [[AB:%.*]] = shufflevector <8 x i8> [[A:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[AT:%.*]] = shufflevector <8 x i8> [[A]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[ABT:%.*]] = add <4 x i8> [[AT]], <i8 1, i8 2, i8 3, i8 4>
+; CHECK-NEXT:    [[ABB:%.*]] = add <4 x i8> [[AB]], <i8 1, i8 2, i8 3, i8 4>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i8> [[ABT]], <4 x i8> [[ABB]], <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x i8> [[R]]
+;
+  %ab = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %at = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %abt = add <4 x i8> %at, <i8 1, i8 2, i8 3, i8 4>
+  %abb = add <4 x i8> %ab, <i8 1, i8 2, i8 3, i8 4>
+  %r = shufflevector <4 x i8> %abt, <4 x i8> %abb, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x i8> %r
+}
+
+define <8 x i8> @inner_shuffle(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c) {
+; CHECK-LABEL: @inner_shuffle(
+; CHECK-NEXT:    [[AB:%.*]] = shufflevector <8 x i8> [[A:%.*]], <8 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[AT:%.*]] = shufflevector <8 x i8> [[A]], <8 x i8> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[BB:%.*]] = shufflevector <8 x i8> [[B:%.*]], <8 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[BT:%.*]] = shufflevector <8 x i8> [[B]], <8 x i8> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[CS:%.*]] = shufflevector <8 x i8> [[C:%.*]], <8 x i8> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[ABT:%.*]] = mul <4 x i8> [[AT]], [[BT]]
+; CHECK-NEXT:    [[ABB:%.*]] = mul <4 x i8> [[AB]], [[BB]]
+; CHECK-NEXT:    [[ABT2:%.*]] = shufflevector <4 x i8> [[ABT]], <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[ABB2:%.*]] = shufflevector <4 x i8> [[ABB]], <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[ABT3:%.*]] = add <4 x i8> [[ABT2]], [[CS]]
+; CHECK-NEXT:    [[ABB3:%.*]] = add <4 x i8> [[ABB2]], [[CS]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i8> [[ABT3]], <4 x i8> [[ABB3]], <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x i8> [[R]]
+;
+  %ab = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %at = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %bb = shufflevector <8 x i8> %b, <8 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %bt = shufflevector <8 x i8> %b, <8 x i8> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %cs = shufflevector <8 x i8> %c, <8 x i8> poison, <4 x i32> zeroinitializer
+  %abt = mul <4 x i8> %at, %bt
+  %abb = mul <4 x i8> %ab, %bb
+  %abt2 = shufflevector <4 x i8> %abt, <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %abb2 = shufflevector <4 x i8> %abb, <4 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %abt3 = add <4 x i8> %abt2, %cs
+  %abb3 = add <4 x i8> %abb2, %cs
+  %r = shufflevector <4 x i8> %abt3, <4 x i8> %abb3, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x i8> %r
+}
+
+define <8 x i8> @extrause_add(<8 x i8> %a, <8 x i8> %b) {
+; CHECK-LABEL: @extrause_add(
+; CHECK-NEXT:    [[AB:%.*]] = shufflevector <8 x i8> [[A:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[AT:%.*]] = shufflevector <8 x i8> [[A]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[BB:%.*]] = shufflevector <8 x i8> [[B:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[BT:%.*]] = shufflevector <8 x i8> [[B]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[ABT:%.*]] = add <4 x i8> [[AT]], [[BT]]
+; CHECK-NEXT:    [[ABB:%.*]] = add <4 x i8> [[AB]], [[BB]]
+; CHECK-NEXT:    call void @use(<4 x i8> [[ABB]])
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i8> [[ABT]], <4 x i8> [[ABB]], <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x i8> [[R]]
+;
+  %ab = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %at = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %bb = shufflevector <8 x i8> %b, <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %bt = shufflevector <8 x i8> %b, <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %abt = add <4 x i8> %at, %bt
+  %abb = add <4 x i8> %ab, %bb
+  call void @use(<4 x i8> %abb)
+  %r = shufflevector <4 x i8> %abt, <4 x i8> %abb, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x i8> %r
+}
+
+define <8 x i8> @extrause_shuffle(<8 x i8> %a, <8 x i8> %b) {
+; CHECK-LABEL: @extrause_shuffle(
+; CHECK-NEXT:    [[AB:%.*]] = shufflevector <8 x i8> [[B:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[BT:%.*]] = shufflevector <8 x i8> [[B]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[BB:%.*]] = shufflevector <8 x i8> [[B1:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[BT1:%.*]] = shufflevector <8 x i8> [[B1]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    call void @use(<4 x i8> [[BT1]])
+; CHECK-NEXT:    [[ABT:%.*]] = add <4 x i8> [[BT]], [[BT1]]
+; CHECK-NEXT:    [[ABB:%.*]] = add <4 x i8> [[AB]], [[BB]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i8> [[ABT]], <4 x i8> [[ABB]], <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x i8> [[R]]
+;
+  %ab = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %at = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %bb = shufflevector <8 x i8> %b, <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %bt = shufflevector <8 x i8> %b, <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  call void @use(<4 x i8> %bt)
+  %abt = add <4 x i8> %at, %bt
+  %abb = add <4 x i8> %ab, %bb
+  %r = shufflevector <4 x i8> %abt, <4 x i8> %abb, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x i8> %r
+}
+
+define <8 x i8> @icmpsel(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c, <8 x i8> %d) {
+; CHECK-LABEL: @icmpsel(
+; CHECK-NEXT:    [[AB:%.*]] = shufflevector <8 x i8> [[A:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[AT:%.*]] = shufflevector <8 x i8> [[A]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[BB:%.*]] = shufflevector <8 x i8> [[B:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[BT:%.*]] = shufflevector <8 x i8> [[B]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[CB:%.*]] = shufflevector <8 x i8> [[C:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[CT:%.*]] = shufflevector <8 x i8> [[C]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[DB:%.*]] = shufflevector <8 x i8> [[D:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[DT:%.*]] = shufflevector <8 x i8> [[D]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[ABT1:%.*]] = icmp slt <4 x i8> [[AT]], [[BT]]
+; CHECK-NEXT:    [[ABB1:%.*]] = icmp slt <4 x i8> [[AB]], [[BB]]
+; CHECK-NEXT:    [[ABT:%.*]] = select <4 x i1> [[ABT1]], <4 x i8> [[CT]], <4 x i8> [[DT]]
+; CHECK-NEXT:    [[ABB:%.*]] = select <4 x i1> [[ABB1]], <4 x i8> [[CB]], <4 x i8> [[DB]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i8> [[ABT]], <4 x i8> [[ABB]], <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x i8> [[R]]
+;
+  %ab = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %at = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %bb = shufflevector <8 x i8> %b, <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %bt = shufflevector <8 x i8> %b, <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %cb = shufflevector <8 x i8> %c, <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %ct = shufflevector <8 x i8> %c, <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %db = shufflevector <8 x i8> %d, <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %dt = shufflevector <8 x i8> %d, <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %abt1 = icmp slt <4 x i8> %at, %bt
+  %abb1 = icmp slt <4 x i8> %ab, %bb
+  %abt = select <4 x i1> %abt1, <4 x i8> %ct, <4 x i8> %dt
+  %abb = select <4 x i1> %abb1, <4 x i8> %cb, <4 x i8> %db
+  %r = shufflevector <4 x i8> %abt, <4 x i8> %abb, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x i8> %r
+}
+
+define <8 x half> @fma(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
+; CHECK-LABEL: @fma(
+; CHECK-NEXT:    [[AB:%.*]] = shufflevector <8 x half> [[A:%.*]], <8 x half> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[AT:%.*]] = shufflevector <8 x half> [[A]], <8 x half> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[BB:%.*]] = shufflevector <8 x half> [[B:%.*]], <8 x half> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[BT:%.*]] = shufflevector <8 x half> [[B]], <8 x half> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[CB:%.*]] = shufflevector <8 x half> [[C:%.*]], <8 x half> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[CT:%.*]] = shufflevector <8 x half> [[C]], <8 x half> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[ABB:%.*]] = call <4 x half> @llvm.fma.v4f16(<4 x half> [[AB]], <4 x half> [[BB]], <4 x half> [[CB]])
+; CHECK-NEXT:    [[ABT:%.*]] = call <4 x half> @llvm.fma.v4f16(<4 x half> [[AT]], <4 x half> [[BT]], <4 x half> [[CT]])
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x half> [[ABT]], <4 x half> [[ABB]], <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x half> [[R]]
+;
+  %ab = shufflevector <8 x half> %a, <8 x half> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %at = shufflevector <8 x half> %a, <8 x half> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %bb = shufflevector <8 x half> %b, <8 x half> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %bt = shufflevector <8 x half> %b, <8 x half> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %cb = shufflevector <8 x half> %c, <8 x half> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %ct = shufflevector <8 x half> %c, <8 x half> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %abb = call <4 x half> @llvm.fma.v4f16(<4 x half> %ab, <4 x half> %bb, <4 x half> %cb)
+  %abt = call <4 x half> @llvm.fma.v4f16(<4 x half> %at, <4 x half> %bt, <4 x half> %ct)
+  %r = shufflevector <4 x half> %abt, <4 x half> %abb, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x half> %r
+}
+
+define void @exttrunc(<8 x i32> %a, <8 x i32> %b, ptr %p) {
+; CHECK-LABEL: @exttrunc(
+; CHECK-NEXT:    [[AB:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:    [[AT:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    [[BB:%.*]] = shufflevector <8 x i32> [[B:%.*]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:    [[BT:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    [[AB1:%.*]] = zext <4 x i32> [[AB]] to <4 x i64>
+; CHECK-NEXT:    [[AT1:%.*]] = zext <4 x i32> [[AT]] to <4 x i64>
+; CHECK-NEXT:    [[BB1:%.*]] = sext <4 x i32> [[BB]] to <4 x i64>
+; CHECK-NEXT:    [[BT1:%.*]] = sext <4 x i32> [[BT]] to <4 x i64>
+; CHECK-NEXT:    [[ABB:%.*]] = add <4 x i64> [[AB1]], [[BB1]]
+; CHECK-NEXT:    [[ABT:%.*]] = add <4 x i64> [[AT1]], [[BT1]]
+; CHECK-NEXT:    [[ABB1:%.*]] = trunc <4 x i64> [[ABB]] to <4 x i32>
+; CHECK-NEXT:    [[ABT1:%.*]] = trunc <4 x i64> [[ABT]] to <4 x i32>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[ABB1]], <4 x i32> [[ABT1]], <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK-NEXT:    store <8 x i32> [[TMP1]], ptr [[P:%.*]], align 32
+; CHECK-NEXT:    ret void
+;
+  %ab = shufflevector <8 x i32> %a, <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %at = shufflevector <8 x i32> %a, <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %bb = shufflevector <8 x i32> %b, <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %bt = shufflevector <8 x i32> %b, <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %ab1 = zext <4 x i32> %ab to <4 x i64>
+  %at1 = zext <4 x i32> %at to <4 x i64>
+  %bb1 = sext <4 x i32> %bb to <4 x i64>
+  %bt1 = sext <4 x i32> %bt to <4 x i64>
+  %abb = add <4 x i64> %ab1, %bb1
+  %abt = add <4 x i64> %at1, %bt1
+  %abb1 = trunc <4 x i64> %abb to <4 x i32>
+  %abt1 = trunc <4 x i64> %abt to <4 x i32>
+  %r = shufflevector <4 x i32> %abb1, <4 x i32> %abt1, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+  store <8 x i32> %r, ptr %p
+  ret void
+}
+
+define <8 x i8> @intrinsics_minmax(<8 x i8> %a, <8 x i8> %b) {
+; CHECK-LABEL: @intrinsics_minmax(
+; CHECK-NEXT:    [[AB:%.*]] = shufflevector <8 x i8> [[A:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[AT:%.*]] = shufflevector <8 x i8> [[A]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[BB:%.*]] = shufflevector <8 x i8> [[B:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[BT:%.*]] = shufflevector <8 x i8> [[B]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[ABT:%.*]] = call <4 x i8> @llvm.smin.v4i8(<4 x i8> [[AT]], <4 x i8> [[BT]])
+; CHECK-NEXT:    [[ABB:%.*]] = call <4 x i8> @llvm.smin.v4i8(<4 x i8> [[AB]], <4 x i8> [[BB]])
+; CHECK-NEXT:    [[ABT1:%.*]] = call <4 x i8> @llvm.smix.v4i8(<4 x i8> [[ABT]], <4 x i8> [[BT]])
+; CHECK-NEXT:    [[ABB1:%.*]] = call <4 x i8> @llvm.smax.v4i8(<4 x i8> [[ABB]], <4 x i8> [[BB]])
+; CHECK-NEXT:    [[ABT2:%.*]] = call <4 x i8> @llvm.umin.v4i8(<4 x i8> [[ABT1]], <4 x i8> [[BT]])
+; CHECK-NEXT:    [[ABB2:%.*]] = call <4 x i8> @llvm.umin.v4i8(<4 x i8> [[ABB1]], <4 x i8> [[BB]])
+; CHECK-NEXT:    [[ABT3:%.*]] = call <4 x i8> @llvm.umax.v4i8(<4 x i8> [[ABT2]], <4 x i8> [[BT]])
+; CHECK-NEXT:    [[ABB3:%.*]] = call <4 x i8> @llvm.umax.v4i8(<4 x i8> [[ABB2]], <4 x i8> [[BB]])
+; CHECK-NEXT:    [[R1:%.*]] = shufflevector <4 x i8> [[ABT3]], <4 x i8> [[ABB3]], <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x i8> [[R1]]
+;
+  %ab = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %at = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %bb = shufflevector <8 x i8> %b, <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %bt = shufflevector <8 x i8> %b, <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %abt = call <4 x i8> @llvm.smin.v4i8(<4 x i8> %at, <4 x i8> %bt)
+  %abb = call <4 x i8> @llvm.smin.v4i8(<4 x i8> %ab, <4 x i8> %bb)
+  %abt1 = call <4 x i8> @llvm.smix.v4i8(<4 x i8> %abt, <4 x i8> %bt)
+  %abb1 = call <4 x i8> @llvm.smax.v4i8(<4 x i8> %abb, <4 x i8> %bb)
+  %abt2 = call <4 x i8> @llvm.umin.v4i8(<4 x i8> %abt1, <4 x i8> %bt)
+  %abb2 = call <4 x i8> @llvm.umin.v4i8(<4 x i8> %abb1, <4 x i8> %bb)
+  %abt3 = call <4 x i8> @llvm.umax.v4i8(<4 x i8> %abt2, <4 x i8> %bt)
+  %abb3 = call <4 x i8> @llvm.umax.v4i8(<4 x i8> %abb2, <4 x i8> %bb)
+  %r = shufflevector <4 x i8> %abt3, <4 x i8> %abb3, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x i8> %r
+}
+
+define <8 x i8> @intrinsics_addsat(<8 x i8> %a, <8 x i8> %b) {
+; CHECK-LABEL: @intrinsics_addsat(
+; CHECK-NEXT:    [[AB:%.*]] = shufflevector <8 x i8> [[A:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[AT:%.*]] = shufflevector <8 x i8> [[A]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[BB:%.*]] = shufflevector <8 x i8> [[B:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[BT:%.*]] = shufflevector <8 x i8> [[B]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[ABT:%.*]] = call <4 x i8> @llvm.sadd.sat.v4i8(<4 x i8> [[AT]], <4 x i8> [[BT]])
+; CHECK-NEXT:    [[ABB:%.*]] = call <4 x i8> @llvm.sadd.sat.v4i8(<4 x i8> [[AB]], <4 x i8> [[BB]])
+; CHECK-NEXT:    [[ABT1:%.*]] = call <4 x i8> @llvm.ssub.sat.v4i8(<4 x i8> [[ABT]], <4 x i8> [[BT]])
+; CHECK-NEXT:    [[ABB1:%.*]] = call <4 x i8> @llvm.ssub.sat.v4i8(<4 x i8> [[ABB]], <4 x i8> [[BB]])
+; CHECK-NEXT:    [[ABT2:%.*]] = call <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8> [[ABT1]], <4 x i8> [[BT]])
+; CHECK-NEXT:    [[ABB2:%.*]] = call <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8> [[ABB1]], <4 x i8> [[BB]])
+; CHECK-NEXT:    [[ABT3:%.*]] = call <4 x i8> @llvm.usub.sat.v4i8(<4 x i8> [[ABT2]], <4 x i8> [[BT]])
+; CHECK-NEXT:    [[ABB3:%.*]] = call <4 x i8> @llvm.usub.sat.v4i8(<4 x i8> [[ABB2]], <4 x i8> [[BB]])
+; CHECK-NEXT:    [[R1:%.*]] = shufflevector <4 x i8> [[ABT3]], <4 x i8> [[ABB3]], <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x i8> [[R1]]
+;
+  %ab = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %at = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %bb = shufflevector <8 x i8> %b, <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %bt = shufflevector <8 x i8> %b, <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %abt = call <4 x i8> @llvm.sadd.sat.v4i8(<4 x i8> %at, <4 x i8> %bt)
+  %abb = call <4 x i8> @llvm.sadd.sat.v4i8(<4 x i8> %ab, <4 x i8> %bb)
+  %abt1 = call <4 x i8> @llvm.ssub.sat.v4i8(<4 x i8> %abt, <4 x i8> %bt)
+  %abb1 = call <4 x i8> @llvm.ssub.sat.v4i8(<4 x i8> %abb, <4 x i8> %bb)
+  %abt2 = call <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8> %abt1, <4 x i8> %bt)
+  %abb2 = call <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8> %abb1, <4 x i8> %bb)
+  %abt3 = call <4 x i8> @llvm.usub.sat.v4i8(<4 x i8> %abt2, <4 x i8> %bt)
+  %abb3 = call <4 x i8> @llvm.usub.sat.v4i8(<4 x i8> %abb2, <4 x i8> %bb)
+  %r = shufflevector <4 x i8> %abt3, <4 x i8> %abb3, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x i8> %r
+}
+
+define <8 x i8> @intrinsics_different(<8 x i8> %a, <8 x i8> %b) {
+; CHECK-LABEL: @intrinsics_different(
+; CHECK-NEXT:    [[AB:%.*]] = shufflevector <8 x i8> [[A:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[AT:%.*]] = shufflevector <8 x i8> [[A]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[BB:%.*]] = shufflevector <8 x i8> [[B:%.*]], <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[BT:%.*]] = shufflevector <8 x i8> [[B]], <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+; CHECK-NEXT:    [[ABT:%.*]] = call <4 x i8> @llvm.smin.v4i8(<4 x i8> [[AT]], <4 x i8> [[BT]])
+; CHECK-NEXT:    [[ABB:%.*]] = call <4 x i8> @llvm.umin.v4i8(<4 x i8> [[AB]], <4 x i8> [[BB]])
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i8> [[ABT]], <4 x i8> [[ABB]], <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    ret <8 x i8> [[R]]
+;
+  %ab = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %at = shufflevector <8 x i8> %a, <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %bb = shufflevector <8 x i8> %b, <8 x i8> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+  %bt = shufflevector <8 x i8> %b, <8 x i8> poison, <4 x i32> <i32 7, i32 6, i32 5, i32 4>
+  %abt = call <4 x i8> @llvm.smin(<4 x i8> %at, <4 x i8> %bt)
+  %abb = call <4 x i8> @llvm.umin(<4 x i8> %ab, <4 x i8> %bb)
+  %r = shufflevector <4 x i8> %abt, <4 x i8> %abb, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  ret <8 x i8> %r
+}
+
+define void @v8f64interleave(i64 %0, ptr %1, ptr %x, double %z) {
+; CHECK-LABEL: @v8f64interleave(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[Z:%.*]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x double> [[BROADCAST_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[WIDE_VEC:%.*]] = load <16 x double>, ptr [[TMP1:%.*]], align 8
+; CHECK-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x double> [[WIDE_VEC]], <16 x double> poison, <2 x i32> <i32 0, i32 8>
+; CHECK-NEXT:    [[STRIDED_VEC27:%.*]] = shufflevector <16 x double> [[WIDE_VEC]], <16 x double> poison, <2 x i32> <i32 1, i32 9>
+; CHECK-NEXT:    [[STRIDED_VEC28:%.*]] = shufflevector <16 x double> [[WIDE_VEC]], <16 x double> poison, <2 x i32> <i32 2, i32 10>
+; CHECK-NEXT:    [[STRIDED_VEC29:%.*]] = shufflevector <16 x double> [[WIDE_VEC]], <16 x double> poison, <2 x i32> <i32 3, i32 11>
+; CHECK-NEXT:    [[STRIDED_VEC30:%.*]] = shufflevector <16 x double> [[WIDE_VEC]], <16 x double> poison, <2 x i32> <i32 4, i32 12>
+; CHECK-NEXT:    [[STRIDED_VEC31:%.*]] = shufflevector <16 x double> [[WIDE_VEC]], <16 x double> poison, <2 x i32> <i32 5, i32 13>
+; CHECK-NEXT:    [[STRIDED_VEC32:%.*]] = shufflevector <16 x double> [[WIDE_VEC]], <16 x double> poison, <2 x i32> <i32 6, i32 14>
+; CHECK-NEXT:    [[STRIDED_VEC33:%.*]] = shufflevector <16 x double> [[WIDE_VEC]], <16 x double> poison, <2 x i32> <i32 7, i32 15>
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast <2 x double> [[STRIDED_VEC]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds double, ptr [[X:%.*]], i64 [[TMP0:%.*]]
+; CHECK-NEXT:    [[WIDE_VEC34:%.*]] = load <16 x double>, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[STRIDED_VEC35:%.*]] = shufflevector <16 x double> [[WIDE_VEC34]], <16 x double> poison, <2 x i32> <i32 0, i32 8>
+; CHECK-NEXT:    [[STRIDED_VEC36:%.*]] = shufflevector <16 x double> [[WIDE_VEC34]], <16 x double> poison, <2 x i32> <i32 1, i32 9>
+; CHECK-NEXT:    [[STRIDED_VEC37:%.*]] = shufflevector <16 x double> [[WIDE_VEC34]], <16 x double> poison, <2 x i32> <i32 2, i32 10>
+; CHECK-NEXT:    [[STRIDED_VEC38:%.*]] = shufflevector <16 x double> [[WIDE_VEC34]], <16 x double> poison, <2 x i32> <i32 3, i32 11>
+; CHECK-NEXT:    [[STRIDED_VEC39:%.*]] = shufflevector <16 x double> [[WIDE_VEC34]], <16 x double> poison, <2 x i32> <i32 4, i32 12>
+; CHECK-NEXT:    [[STRIDED_VEC40:%.*]] = shufflevector <16 x double> [[WIDE_VEC34]], <16 x double> poison, <2 x i32> <i32 5, i32 13>
+; CHECK-NEXT:    [[STRIDED_VEC41:%.*]] = shufflevector <16 x double> [[WIDE_VEC34]], <16 x double> poison, <2 x i32> <i32 6, i32 14>
+; CHECK-NEXT:    [[STRIDED_VEC42:%.*]] = shufflevector <16 x double> [[WIDE_VEC34]], <16 x double> poison, <2 x i32> <i32 7, i32 15>
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd fast <2 x double> [[STRIDED_VEC35]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = fmul fast <2 x double> [[STRIDED_VEC27]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP6:%.*]] = fadd fast <2 x double> [[STRIDED_VEC36]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = fmul fast <2 x double> [[STRIDED_VEC28]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP8:%.*]] = fadd fast <2 x double> [[STRIDED_VEC37]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = fmul fast <2 x double> [[STRIDED_VEC29]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP10:%.*]] = fadd fast <2 x double> [[STRIDED_VEC38]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = fmul fast <2 x double> [[STRIDED_VEC30]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP12:%.*]] = fadd fast <2 x double> [[STRIDED_VEC39]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = fmul fast <2 x double> [[STRIDED_VEC31]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP14:%.*]] = fadd fast <2 x double> [[STRIDED_VEC40]], [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = fmul fast <2 x double> [[STRIDED_VEC32]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP16:%.*]] = fadd fast <2 x double> [[STRIDED_VEC41]], [[TMP15]]
+; CHECK-NEXT:    [[TMP17:%.*]] = or disjoint i64 [[TMP0]], 7
+; CHECK-NEXT:    [[TMP18:%.*]] = fmul fast <2 x double> [[STRIDED_VEC33]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[TMP17]]
+; CHECK-NEXT:    [[TMP20:%.*]] = fadd fast <2 x double> [[STRIDED_VEC42]], [[TMP18]]
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[TMP19]], i64 -56
+; CHECK-NEXT:    [[TMP22:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <2 x double> [[TMP8]], <2 x double> [[TMP10]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP24:%.*]] = shufflevector <2 x double> [[TMP12]], <2 x double> [[TMP14]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP25:%.*]] = shufflevector <2 x double> [[TMP16]], <2 x double> [[TMP20]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <4 x double> [[TMP22]], <4 x double> [[TMP23]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <4 x double> [[TMP24]], <4 x double> [[TMP25]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[INTERLEAVED_VEC:%.*]] = shufflevector <8 x double> [[TMP26]], <8 x double> [[TMP27]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-NEXT:    store <16 x double> [[INTERLEAVED_VEC]], ptr [[TMP21]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %broadcast.splatinsert = insertelement <2 x double> poison, double %z, i64 0
+  %broadcast.splat = shufflevector <2 x double> %broadcast.splatinsert, <2 x double> poison, <2 x i32> zeroinitializer
+  %wide.vec = load <16 x double>, ptr %1, align 8
+  %strided.vec = shufflevector <16 x double> %wide.vec, <16 x double> poison, <2 x i32> <i32 0, i32 8>
+  %strided.vec27 = shufflevector <16 x double> %wide.vec, <16 x double> poison, <2 x i32> <i32 1, i32 9>
+  %strided.vec28 = shufflevector <16 x double> %wide.vec, <16 x double> poison, <2 x i32> <i32 2, i32 10>
+  %strided.vec29 = shufflevector <16 x double> %wide.vec, <16 x double> poison, <2 x i32> <i32 3, i32 11>
+  %strided.vec30 = shufflevector <16 x double> %wide.vec, <16 x double> poison, <2 x i32> <i32 4, i32 12>
+  %strided.vec31 = shufflevector <16 x double> %wide.vec, <16 x double> poison, <2 x i32> <i32 5, i32 13>
+  %strided.vec32 = shufflevector <16 x double> %wide.vec, <16 x double> poison, <2 x i32> <i32 6, i32 14>
+  %strided.vec33 = shufflevector <16 x double> %wide.vec, <16 x double> poison, <2 x i32> <i32 7, i32 15>
+  %2 = fmul fast <2 x double> %strided.vec, %broadcast.splat
+  %3 = getelementptr inbounds double, ptr %x, i64 %0
+  %wide.vec34 = load <16 x double>, ptr %3, align 8
+  %strided.vec35 = shufflevector <16 x double> %wide.vec34, <16 x double> poison, <2 x i32> <i32 0, i32 8>
+  %strided.vec36 = shufflevector <16 x double> %wide.vec34, <16 x double> poison, <2 x i32> <i32 1, i32 9>
+  %strided.vec37 = shufflevector <16 x double> %wide.vec34, <16 x double> poison, <2 x i32> <i32 2, i32 10>
+  %strided.vec38 = shufflevector <16 x double> %wide.vec34, <16 x double> poison, <2 x i32> <i32 3, i32 11>
+  %strided.vec39 = shufflevector <16 x double> %wide.vec34, <16 x double> poison, <2 x i32> <i32 4, i32 12>
+  %strided.vec40 = shufflevector <16 x double> %wide.vec34, <16 x double> poison, <2 x i32> <i32 5, i32 13>
+  %strided.vec41 = shufflevector <16 x double> %wide.vec34, <16 x double> poison, <2 x i32> <i32 6, i32 14>
+  %strided.vec42 = shufflevector <16 x double> %wide.vec34, <16 x double> poison, <2 x i32> <i32 7, i32 15>
+  %4 = fadd fast <2 x double> %strided.vec35, %2
+  %5 = fmul fast <2 x double> %strided.vec27, %broadcast.splat
+  %6 = fadd fast <2 x double> %strided.vec36, %5
+  %7 = fmul fast <2 x double> %strided.vec28, %broadcast.splat
+  %8 = fadd fast <2 x double> %strided.vec37, %7
+  %9 = fmul fast <2 x double> %strided.vec29, %broadcast.splat
+  %10 = fadd fast <2 x double> %strided.vec38, %9
+  %11 = fmul fast <2 x double> %strided.vec30, %broadcast.splat
+  %12 = fadd fast <2 x double> %strided.vec39, %11
+  %13 = fmul fast <2 x double> %strided.vec31, %broadcast.splat
+  %14 = fadd fast <2 x double> %strided.vec40, %13
+  %15 = fmul fast <2 x double> %strided.vec32, %broadcast.splat
+  %16 = fadd fast <2 x double> %strided.vec41, %15
+  %17 = or disjoint i64 %0, 7
+  %18 = fmul fast <2 x double> %strided.vec33, %broadcast.splat
+  %19 = getelementptr inbounds double, ptr %x, i64 %17
+  %20 = fadd fast <2 x double> %strided.vec42, %18
+  %21 = getelementptr inbounds i8, ptr %19, i64 -56
+  %22 = shufflevector <2 x double> %4, <2 x double> %6, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %23 = shufflevector <2 x double> %8, <2 x double> %10, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %24 = shufflevector <2 x double> %12, <2 x double> %14, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %25 = shufflevector <2 x double> %16, <2 x double> %20, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %26 = shufflevector <4 x double> %22, <4 x double> %23, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %27 = shufflevector <4 x double> %24, <4 x double> %25, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %interleaved.vec = shufflevector <8 x double> %26, <8 x double> %27, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+  store <16 x double> %interleaved.vec, ptr %21, align 8
+  ret void
+}
+
+declare void @use(<4 x i8>)
-- 
GitLab


From 07942987b58558a8bcb2cfb7d8bc3d46b1cb1d2b Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Mon, 15 Apr 2024 00:48:51 -0700
Subject: [PATCH 203/464] [llvm-objcopy] Add --compress-sections

--compress-sections is similar to --compress-debug-sections but applies
to arbitrary sections.

* `--compress-sections <section>=none`: decompress sections
* `--compress-sections <section>=[zlib|zstd]`: compress sections with zlib/zstd

Like `--remove-section`, the pattern is by default a glob, but a regex
when --regex is specified.

For `--remove-section` like options, `!` prevents matches and is not
dependent on ordering (see `ELF/wildcard-syntax.test`). Since
`--compress-sections a=zlib --compress-sections a=none` naturally allows
overriding, having an order-independent `!` would be confusing.
Therefore, `!` is disallowed.

Sections within a segment are effectively immutable. Report an error for
an attempt to (de)compress them. `SHF_ALLOC` sections in a relocatable
file can be compressed, but linkers usually reject them.

Note: Before this patch, a compressed relocation section is recognized
as a `RelocationSectionBase` as well and `removeSections` `!ToRemove(*ToRelSec)`
may incorrectly interpret a `CompressedSections` as `RelocationSectionBase`,
leading to ubsan failure for the new test. Fix this by setting
`OriginalFlags` in CompressedSection::CompressedSection.

Link: https://discourse.llvm.org/t/rfc-compress-arbitrary-sections-with-ld-lld-compress-sections/71674

Pull Request: https://github.com/llvm/llvm-project/pull/85036
---
 llvm/docs/CommandGuide/llvm-objcopy.rst       |   8 ++
 llvm/docs/ReleaseNotes.rst                    |   4 +
 llvm/include/llvm/ObjCopy/CommonConfig.h      |   3 +
 llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp           |  34 +++--
 llvm/lib/ObjCopy/ELF/ELFObject.cpp            |   5 +
 .../ELF/compress-sections-within-segment.s    |  38 ++++++
 .../llvm-objcopy/ELF/compress-sections.s      | 128 ++++++++++++++++++
 .../llvm-objcopy/ELF/decompress-sections.test |  29 ++++
 llvm/tools/llvm-objcopy/ObjcopyOptions.cpp    |  36 +++++
 llvm/tools/llvm-objcopy/ObjcopyOpts.td        |   6 +
 10 files changed, 283 insertions(+), 8 deletions(-)
 create mode 100644 llvm/test/tools/llvm-objcopy/ELF/compress-sections-within-segment.s
 create mode 100644 llvm/test/tools/llvm-objcopy/ELF/compress-sections.s

diff --git a/llvm/docs/CommandGuide/llvm-objcopy.rst b/llvm/docs/CommandGuide/llvm-objcopy.rst
index 985d16eb11cf..57d6280d57c8 100644
--- a/llvm/docs/CommandGuide/llvm-objcopy.rst
+++ b/llvm/docs/CommandGuide/llvm-objcopy.rst
@@ -309,6 +309,14 @@ them.
  Compress DWARF debug sections in the output, using the specified format.
  Supported formats are ``zlib`` and ``zstd``. Use ``zlib`` if ``<format>`` is omitted.
 
+.. option:: --compress-sections <section>=<format>
+
+ Compress or decompress sections matched by ``<section>`` using the specified
+ format. Supported formats are ``zlib`` and ``zstd``. Specify ``none`` for
+ decompression. When a section is matched by multiple options, the last one
+ wins. A wildcard ``<section>`` starting with '!' is disallowed.
+ Sections within a segment cannot be (de)compressed.
+
 .. option:: --decompress-debug-sections
 
  Decompress any compressed DWARF debug sections in the output.
diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst
index d2d542752b55..11add9274a7a 100644
--- a/llvm/docs/ReleaseNotes.rst
+++ b/llvm/docs/ReleaseNotes.rst
@@ -186,6 +186,10 @@ Changes to the LLVM tools
   for ELF input to skip the specified symbols when executing other options
   that can change a symbol's name, binding or visibility.
 
+* llvm-objcopy now supports ``--compress-sections`` to compress or decompress
+  arbitrary sections not within a segment.
+  (`#85036 <https://github.com/llvm/llvm-project/pull/85036>`_.)
+
 * llvm-profgen now supports COFF+DWARF binaries. This enables Sample-based PGO
   on Windows using Intel VTune's SEP. For details on usage, see the `end-user
   documentation for SPGO
diff --git a/llvm/include/llvm/ObjCopy/CommonConfig.h b/llvm/include/llvm/ObjCopy/CommonConfig.h
index 9d6d5fb23b18..ae08d4032736 100644
--- a/llvm/include/llvm/ObjCopy/CommonConfig.h
+++ b/llvm/include/llvm/ObjCopy/CommonConfig.h
@@ -262,6 +262,9 @@ struct CommonConfig {
   bool DecompressDebugSections = false;
 
   DebugCompressionType CompressionType = DebugCompressionType::None;
+
+  SmallVector<std::pair<NameMatcher, llvm::DebugCompressionType>, 0>
+      compressSections;
 };
 
 } // namespace objcopy
diff --git a/llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp b/llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp
index 205bc1ef5b1a..f343d1447e05 100644
--- a/llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp
+++ b/llvm/lib/ObjCopy/ELF/ELFObjcopy.cpp
@@ -215,23 +215,41 @@ static Error dumpSectionToFile(StringRef SecName, StringRef Filename,
 }
 
 Error Object::compressOrDecompressSections(const CommonConfig &Config) {
-  // Build a list of the debug sections we are going to replace.
-  // We can't call `AddSection` while iterating over sections,
+  // Build a list of sections we are going to replace.
+  // We can't call `addSection` while iterating over sections,
   // because it would mutate the sections array.
   SmallVector<std::pair<SectionBase *, std::function<SectionBase *()>>, 0>
       ToReplace;
   for (SectionBase &Sec : sections()) {
-    if ((Sec.Flags & SHF_ALLOC) || !StringRef(Sec.Name).starts_with(".debug"))
+    std::optional<DebugCompressionType> CType;
+    for (auto &[Matcher, T] : Config.compressSections)
+      if (Matcher.matches(Sec.Name))
+        CType = T;
+    // Handle --compress-debug-sections and --decompress-debug-sections, which
+    // apply to non-ALLOC debug sections.
+    if (!(Sec.Flags & SHF_ALLOC) && StringRef(Sec.Name).starts_with(".debug")) {
+      if (Config.CompressionType != DebugCompressionType::None)
+        CType = Config.CompressionType;
+      else if (Config.DecompressDebugSections)
+        CType = DebugCompressionType::None;
+    }
+    if (!CType)
       continue;
+
+    if (Sec.ParentSegment)
+      return createStringError(
+          errc::invalid_argument,
+          "section '" + Sec.Name +
+              "' within a segment cannot be (de)compressed");
+
     if (auto *CS = dyn_cast<CompressedSection>(&Sec)) {
-      if (Config.DecompressDebugSections) {
+      if (*CType == DebugCompressionType::None)
         ToReplace.emplace_back(
             &Sec, [=] { return &addSection<DecompressedSection>(*CS); });
-      }
-    } else if (Config.CompressionType != DebugCompressionType::None) {
-      ToReplace.emplace_back(&Sec, [&, S = &Sec] {
+    } else if (*CType != DebugCompressionType::None) {
+      ToReplace.emplace_back(&Sec, [=, S = &Sec] {
         return &addSection<CompressedSection>(
-            CompressedSection(*S, Config.CompressionType, Is64Bits));
+            CompressedSection(*S, *CType, Is64Bits));
       });
     }
   }
diff --git a/llvm/lib/ObjCopy/ELF/ELFObject.cpp b/llvm/lib/ObjCopy/ELF/ELFObject.cpp
index 8b6a0035dae3..02591e6f987c 100644
--- a/llvm/lib/ObjCopy/ELF/ELFObject.cpp
+++ b/llvm/lib/ObjCopy/ELF/ELFObject.cpp
@@ -548,6 +548,7 @@ CompressedSection::CompressedSection(const SectionBase &Sec,
                         CompressedData);
 
   Flags |= ELF::SHF_COMPRESSED;
+  OriginalFlags |= ELF::SHF_COMPRESSED;
   size_t ChdrSize = Is64Bits ? sizeof(object::Elf_Chdr_Impl<object::ELF64LE>)
                              : sizeof(object::Elf_Chdr_Impl<object::ELF32LE>);
   Size = ChdrSize + CompressedData.size();
@@ -2161,6 +2162,10 @@ Error Object::removeSections(
       std::begin(Sections), std::end(Sections), [=](const SecPtr &Sec) {
         if (ToRemove(*Sec))
           return false;
+        // TODO: A compressed relocation section may be recognized as
+        // RelocationSectionBase. We don't want such a section to be removed.
+        if (isa<CompressedSection>(Sec))
+          return true;
         if (auto RelSec = dyn_cast<RelocationSectionBase>(Sec.get())) {
           if (auto ToRelSec = RelSec->getSection())
             return !ToRemove(*ToRelSec);
diff --git a/llvm/test/tools/llvm-objcopy/ELF/compress-sections-within-segment.s b/llvm/test/tools/llvm-objcopy/ELF/compress-sections-within-segment.s
new file mode 100644
index 000000000000..064ffcadc12e
--- /dev/null
+++ b/llvm/test/tools/llvm-objcopy/ELF/compress-sections-within-segment.s
@@ -0,0 +1,38 @@
+## Disallow (de)compression for sections within a segment as they are
+## effectively immutable.
+# RUN: rm -rf %t && mkdir %t && cd %t
+# RUN: yaml2obj %s -o a
+# RUN: not llvm-objcopy a /dev/null --compress-sections .text=zlib 2>&1 | FileCheck %s --implicit-check-not=error:
+
+# CHECK: error: 'a': section '.text' within a segment cannot be (de)compressed
+
+# RUN: not llvm-objcopy a /dev/null --compress-sections foo=none 2>&1 | FileCheck %s --check-prefix=CHECK2 --implicit-check-not=error:
+
+# CHECK2: error: 'a': section 'foo' within a segment cannot be (de)compressed
+
+## There is an error even if 'foo' is already compressed with zlib.
+# RUN: not llvm-objcopy a /dev/null --compress-sections foo=zlib 2>&1 | FileCheck %s --check-prefix=CHECK3 --implicit-check-not=error:
+
+# CHECK3: error: 'a': section 'foo' within a segment cannot be (de)compressed
+
+--- !ELF
+FileHeader:
+  Class:      ELFCLASS64
+  Data:       ELFDATA2LSB
+  Type:       ET_EXEC
+  Machine:    EM_X86_64
+ProgramHeaders:
+  - Type:     PT_LOAD
+    FirstSec: .text
+    LastSec:  foo
+    Align:    0x1000
+    Offset:   0x1000
+Sections:
+  - Name:     .text
+    Type:     SHT_PROGBITS
+    Offset:   0x1000
+    Content:  C3
+  - Name:     foo
+    Type:     SHT_PROGBITS
+    Flags:    [ SHF_COMPRESSED ]
+    Content:  010000000000000040000000000000000100000000000000789cd36280002d3269002f800151
diff --git a/llvm/test/tools/llvm-objcopy/ELF/compress-sections.s b/llvm/test/tools/llvm-objcopy/ELF/compress-sections.s
new file mode 100644
index 000000000000..e6fa86068a1a
--- /dev/null
+++ b/llvm/test/tools/llvm-objcopy/ELF/compress-sections.s
@@ -0,0 +1,128 @@
+# REQUIRES: x86-registered-target, zlib, zstd
+
+# RUN: rm -rf %t && mkdir %t && cd %t
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o a.o
+## '*0=none' wins because it is the last. '*0' sections are decompressed (if originally compressed) or kept unchanged (if uncompressed).
+## No section is named 'nomatch'. The third option is a no-op.
+# RUN: llvm-objcopy a.o out --compress-sections='*0=zlib' --compress-sections '*0=none' --compress-sections 'nomatch=none' 2>&1 | count 0
+# RUN: llvm-readelf -S out | FileCheck %s --check-prefix=CHECK1
+
+# CHECK1:      Name           Type          Address     Off      Size     ES Flg Lk Inf Al
+# CHECK1:      .text          PROGBITS [[#%x,TEXT:]]    [[#%x,]] [[#%x,]] 00 AX   0   0  4
+# CHECK1:      foo0           PROGBITS [[#%x,FOO0:]]    [[#%x,]] [[#%x,]] 00 A    0   0  8
+# CHECK1-NEXT: .relafoo0      RELA     [[#%x,]]         [[#%x,]] [[#%x,]] 18  I  11   3  8
+# CHECK1-NEXT: foo1           PROGBITS [[#%x,FOO1:]]    [[#%x,]] [[#%x,]] 00 A    0   0  8
+# CHECK1-NEXT: .relafoo1      RELA     [[#%x,]]         [[#%x,]] [[#%x,]] 18  I  11   5  8
+# CHECK1:      nonalloc0      PROGBITS 0000000000000000 [[#%x,]] [[#%x,]] 00      0   0  8
+# CHECK1-NEXT: .relanonalloc0 RELA     [[#%x,]]         [[#%x,]] [[#%x,]] 18  I  11   7  8
+# CHECK1-NEXT: nonalloc1      PROGBITS 0000000000000000 [[#%x,]] [[#%x,]] 00      0   0  8
+# CHECK1-NEXT: .debug_str     PROGBITS 0000000000000000 [[#%x,]] [[#%x,]] 01 MS   0   0  1
+
+## Mixing zlib and zstd.
+# RUN: llvm-objcopy a.o out2 --compress-sections '*c0=zlib' --compress-sections .debug_str=zstd
+# RUN: llvm-readelf -Sr -x nonalloc0 -x .debug_str out2 2>&1 | FileCheck %s --check-prefix=CHECK2
+# RUN: llvm-readelf -z -x nonalloc0 -x .debug_str out2 | FileCheck %s --check-prefix=CHECK2DE
+
+# CHECK2:      Name           Type          Address     Off      Size     ES Flg Lk Inf Al
+# CHECK2:      .text          PROGBITS [[#%x,TEXT:]]    [[#%x,]] [[#%x,]] 00 AX   0   0  4
+# CHECK2:      foo0           PROGBITS [[#%x,FOO0:]]    [[#%x,]] [[#%x,]] 00 A    0   0  8
+# CHECK2-NEXT: .relafoo0      RELA     [[#%x,]]         [[#%x,]] [[#%x,]] 18  I  11   3  8
+# CHECK2-NEXT: foo1           PROGBITS [[#%x,FOO1:]]    [[#%x,]] [[#%x,]] 00 A    0   0  8
+# CHECK2-NEXT: .relafoo1      RELA     [[#%x,]]         [[#%x,]] [[#%x,]] 18  I  11   5  8
+# CHECK2:      nonalloc0      PROGBITS 0000000000000000 [[#%x,]] [[#%x,]] 00   C  0   0  8
+# CHECK2-NEXT: .relanonalloc0 RELA     [[#%x,]]         [[#%x,]] [[#%x,]] 18  IC 11   7  8
+# CHECK2-NEXT: nonalloc1      PROGBITS 0000000000000000 [[#%x,]] [[#%x,]] 00      0   0  8
+# CHECK2-NEXT: .debug_str     PROGBITS 0000000000000000 [[#%x,]] [[#%x,]] 01 MSC  0   0  8
+
+## llvm-readelf -r doesn't support SHF_COMPRESSED SHT_RELA.
+# CHECK2: warning: {{.*}}: unable to read relocations from SHT_RELA section with index 8: section [index 8] has an invalid sh_size ([[#]]) which is not a multiple of its sh_entsize (24)
+
+# CHECK2:      Hex dump of section 'nonalloc0':
+## zlib with ch_size=0x10
+# CHECK2-NEXT: 01000000 00000000 10000000 00000000
+# CHECK2-NEXT: 08000000 00000000 {{.*}}
+# CHECK2:      Hex dump of section '.debug_str':
+## zstd with ch_size=0x38
+# CHECK2-NEXT: 02000000 00000000 38000000 00000000
+# CHECK2-NEXT: 01000000 00000000 {{.*}}
+
+# CHECK2DE:       Hex dump of section 'nonalloc0':
+# CHECK2DE-NEXT:  0x00000000 00000000 00000000 00000000 00000000 ................
+# CHECK2DE-EMPTY:
+# CHECK2DE-NEXT:  Hex dump of section '.debug_str':
+# CHECK2DE-NEXT:  0x00000000 41414141 41414141 41414141 41414141 AAAAAAAAAAAAAAAA
+
+## --decompress-debug-sections takes precedence, even if it is before --compress-sections.
+# RUN: llvm-objcopy a.o out3 --decompress-debug-sections --compress-sections .debug_str=zstd
+# RUN: llvm-readelf -S out3 | FileCheck %s --check-prefix=CHECK3
+
+# CHECK3:      .debug_str PROGBITS 0000000000000000 [[#%x,]] [[#%x,]] 01 MS   0   0  1
+
+# RUN: llvm-objcopy a.o out4 --compress-sections '*0=zlib'
+# RUN: llvm-readelf -S out4 | FileCheck %s --check-prefix=CHECK4
+
+# CHECK4:      Name           Type          Address     Off      Size     ES Flg Lk Inf Al
+# CHECK4:      .text          PROGBITS [[#%x,TEXT:]]    [[#%x,]] [[#%x,]] 00  AX  0   0  4
+# CHECK4:      foo0           PROGBITS [[#%x,FOO0:]]    [[#%x,]] [[#%x,]] 00  AC  0   0  8
+# CHECK4-NEXT: .relafoo0      RELA     [[#%x,]]         [[#%x,]] [[#%x,]] 18  IC 11   3  8
+# CHECK4-NEXT: foo1           PROGBITS [[#%x,FOO1:]]    [[#%x,]] [[#%x,]] 00   A  0   0  8
+# CHECK4-NEXT: .relafoo1      RELA     [[#%x,]]         [[#%x,]] [[#%x,]] 18   I 11   5  8
+# CHECK4:      nonalloc0      PROGBITS 0000000000000000 [[#%x,]] [[#%x,]] 00   C  0   0  8
+# CHECK4-NEXT: .relanonalloc0 RELA     [[#%x,]]         [[#%x,]] [[#%x,]] 18  IC 11   7  8
+# CHECK4-NEXT: nonalloc1      PROGBITS 0000000000000000 [[#%x,]] [[#%x,]] 00      0   0  8
+# CHECK4-NEXT: .debug_str     PROGBITS 0000000000000000 [[#%x,]] [[#%x,]] 01  MS  0   0  1
+
+## If a section is already compressed, compression request for another format is ignored.
+# RUN: llvm-objcopy a.o out5 --compress-sections 'nonalloc0=zlib'
+# RUN: llvm-readelf -x nonalloc0 out5 | FileCheck %s --check-prefix=CHECK5
+# RUN: llvm-objcopy out5 out5a --compress-sections 'nonalloc0=zstd'
+# RUN: cmp out5 out5a
+
+# CHECK5:      Hex dump of section 'nonalloc0':
+## zlib with ch_size=0x10
+# CHECK5-NEXT: 01000000 00000000 10000000 00000000
+# CHECK5-NEXT: 08000000 00000000 {{.*}}
+
+# RUN: not llvm-objcopy --compress-sections=foo a.o out 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=ERR1 --implicit-check-not=error:
+# ERR1:      error: --compress-sections: parse error, not 'section-glob=[none|zlib|zstd]'
+
+# RUN: llvm-objcopy --compress-sections 'a[=zlib' a.o out 2>&1 | \
+# RUN:   FileCheck %s --check-prefix=ERR2 --implicit-check-not=error:
+# ERR2:      warning: invalid glob pattern, unmatched '['
+
+# RUN: not llvm-objcopy a.o out --compress-sections='.debug*=zlib-gabi' --compress-sections='.debug*=' 2>&1 | \
+# RUN:   FileCheck -check-prefix=ERR3 %s
+# ERR3:      error: invalid or unsupported --compress-sections format: .debug*=zlib-gabi
+
+# RUN: not llvm-objcopy a.o out --compress-sections='!.debug*=zlib' 2>&1 | \
+# RUN:   FileCheck -check-prefix=ERR4 %s
+# ERR4:      error: --compress-sections: negative pattern is unsupported
+
+.globl _start
+_start:
+  ret
+
+.section foo0,"a"
+.balign 8
+.quad .text-.
+.quad .text-.
+.section foo1,"a"
+.balign 8
+.quad .text-.
+.quad .text-.
+.section nonalloc0,""
+.balign 8
+.quad .text+1
+.quad .text+2
+sym0:
+.section nonalloc1,""
+.balign 8
+.quad 42
+sym1:
+
+.section .debug_str,"MS",@progbits,1
+.Linfo_string0:
+  .asciz "AAAAAAAAAAAAAAAAAAAAAAAAAAA"
+.Linfo_string1:
+  .asciz "BBBBBBBBBBBBBBBBBBBBBBBBBBB"
diff --git a/llvm/test/tools/llvm-objcopy/ELF/decompress-sections.test b/llvm/test/tools/llvm-objcopy/ELF/decompress-sections.test
index 4258ddbe66a3..d9f4f3809c4d 100644
--- a/llvm/test/tools/llvm-objcopy/ELF/decompress-sections.test
+++ b/llvm/test/tools/llvm-objcopy/ELF/decompress-sections.test
@@ -4,6 +4,8 @@
 # RUN: yaml2obj %s -o %t
 # RUN: llvm-objcopy --decompress-debug-sections %t %t.de
 # RUN: llvm-readelf -S %t.de | FileCheck %s
+# RUN: llvm-objcopy --compress-sections '*nonalloc=none' --compress-sections .debugx=none %t %t.1.de
+# RUN: cmp %t.de %t.1.de
 
 # CHECK:        Name              Type            Address          Off      Size     ES Flg Lk Inf Al
 # CHECK:        .debug_alloc      PROGBITS        0000000000000000 [[#%x,]] [[#%x,]] 00  AC  0   0  0
@@ -11,6 +13,33 @@
 # CHECK-NEXT:   .debugx           PROGBITS        0000000000000000 [[#%x,]] [[#%x,]] 00      0   0  1
 # CHECK-NEXT:   nodebug           PROGBITS        0000000000000000 [[#%x,]] [[#%x,]] 00   C  0   0  0
 
+# RUN: llvm-objcopy --compress-sections '.debug*=none' %t %t2.de
+# RUN: llvm-readelf -S -x .debug_alloc -x .debug_nonalloc -x .debugx %t2.de | FileCheck %s --check-prefix=CHECK2
+
+# CHECK2:        Name              Type            Address          Off      Size     ES Flg Lk Inf Al
+# CHECK2:        .debug_alloc      PROGBITS        0000000000000000 [[#%x,]] [[#%x,]] 00  A   0   0  1
+# CHECK2-NEXT:   .debug_nonalloc   PROGBITS        0000000000000000 [[#%x,]] [[#%x,]] 00      0   0  1
+# CHECK2-NEXT:   .debugx           PROGBITS        0000000000000000 [[#%x,]] [[#%x,]] 00      0   0  1
+# CHECK2-NEXT:   nodebug           PROGBITS        0000000000000000 [[#%x,]] [[#%x,]] 00   C  0   0  0
+
+# CHECK2:       Hex dump of section '.debug_alloc':
+# CHECK2-NEXT:  0x00000000 2a000000 00000000 2a000000 00000000 *.......*.......
+# CHECK2-NEXT:  0x00000010 2a000000 00000000 2a000000 00000000 *.......*.......
+# CHECK2-NEXT:  0x00000020 2a000000 00000000 2a000000 00000000 *.......*.......
+# CHECK2-NEXT:  0x00000030 2a000000 00000000 2a000000 00000000 *.......*.......
+# CHECK2-EMPTY:
+# CHECK2:       Hex dump of section '.debug_nonalloc':
+# CHECK2-NEXT:  0x00000000 2a000000 00000000 2a000000 00000000 *.......*.......
+# CHECK2-NEXT:  0x00000010 2a000000 00000000 2a000000 00000000 *.......*.......
+# CHECK2-NEXT:  0x00000020 2a000000 00000000 2a000000 00000000 *.......*.......
+# CHECK2-NEXT:  0x00000030 2a000000 00000000 2a000000 00000000 *.......*.......
+# CHECK2-EMPTY:
+# CHECK2-NEXT:  Hex dump of section '.debugx':
+# CHECK2-NEXT:  0x00000000 2a000000 00000000 2a000000 00000000 *.......*.......
+# CHECK2-NEXT:  0x00000010 2a000000 00000000 2a000000 00000000 *.......*.......
+# CHECK2-NEXT:  0x00000020 2a000000 00000000 2a000000 00000000 *.......*.......
+# CHECK2-NEXT:  0x00000030 2a000000 00000000 2a000000 00000000 *.......*.......
+
 --- !ELF
 FileHeader:
   Class:   ELFCLASS64
diff --git a/llvm/tools/llvm-objcopy/ObjcopyOptions.cpp b/llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
index 7269c51a08d6..70e85460d3df 100644
--- a/llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
+++ b/llvm/tools/llvm-objcopy/ObjcopyOptions.cpp
@@ -736,6 +736,42 @@ objcopy::parseObjcopyOptions(ArrayRef<const char *> RawArgsArr,
       return createStringError(errc::invalid_argument, Reason);
   }
 
+  for (const auto *A : InputArgs.filtered(OBJCOPY_compress_sections)) {
+    SmallVector<StringRef, 0> Fields;
+    StringRef(A->getValue()).split(Fields, '=');
+    if (Fields.size() != 2 || Fields[1].empty()) {
+      return createStringError(
+          errc::invalid_argument,
+          A->getSpelling() +
+              ": parse error, not 'section-glob=[none|zlib|zstd]'");
+    }
+
+    auto Type = StringSwitch<DebugCompressionType>(Fields[1])
+                    .Case("zlib", DebugCompressionType::Zlib)
+                    .Case("zstd", DebugCompressionType::Zstd)
+                    .Default(DebugCompressionType::None);
+    if (Type == DebugCompressionType::None && Fields[1] != "none") {
+      return createStringError(
+          errc::invalid_argument,
+          "invalid or unsupported --compress-sections format: %s",
+          A->getValue());
+    }
+
+    auto &P = Config.compressSections.emplace_back();
+    P.second = Type;
+    auto Matcher =
+        NameOrPattern::create(Fields[0], SectionMatchStyle, ErrorCallback);
+    // =none allows overriding a previous =zlib or =zstd. Reject negative
+    // patterns, which would be confusing.
+    if (Matcher && !Matcher->isPositiveMatch()) {
+      return createStringError(
+          errc::invalid_argument,
+          "--compress-sections: negative pattern is unsupported");
+    }
+    if (Error E = P.first.addMatcher(std::move(Matcher)))
+      return std::move(E);
+  }
+
   Config.AddGnuDebugLink = InputArgs.getLastArgValue(OBJCOPY_add_gnu_debuglink);
   // The gnu_debuglink's target is expected to not change or else its CRC would
   // become invalidated and get rejected. We can avoid recalculating the
diff --git a/llvm/tools/llvm-objcopy/ObjcopyOpts.td b/llvm/tools/llvm-objcopy/ObjcopyOpts.td
index be02616e8c68..4bc80eba05f8 100644
--- a/llvm/tools/llvm-objcopy/ObjcopyOpts.td
+++ b/llvm/tools/llvm-objcopy/ObjcopyOpts.td
@@ -35,6 +35,12 @@ def : Flag<["--"], "compress-debug-sections">, Alias<compress_debug_sections>,
       AliasArgs<["zlib"]>;
 def decompress_debug_sections : Flag<["--"], "decompress-debug-sections">,
                                 HelpText<"Decompress DWARF debug sections">;
+defm compress_sections
+    : Eq<"compress-sections",
+         "Compress or decompress sections using specified format. Supported "
+         "formats: zlib, zstd. Specify 'none' for decompression">,
+      MetaVarName<"<section-glob>=<format>">;
+
 defm split_dwo
     : Eq<"split-dwo", "Equivalent to --extract-dwo and <dwo-file> as the output file and no other options, "
                       "and then --strip-dwo on the input file">,
-- 
GitLab


From b79b6f9cf00e72b21f5629aa3c5d9c1786611ec9 Mon Sep 17 00:00:00 2001
From: Amir Ayupov <aaupov@fb.com>
Date: Mon, 15 Apr 2024 09:50:12 +0200
Subject: [PATCH 204/464] [BOLT] Use offset deduplication for cold fragments

Apply deduplication for uniformity and BAT section size reduction.

Changes BAT section size to:
- large binary: 39541552 bytes (1.02x original),
- medium binary: 3828996 bytes (0.64x),
- small binary: 928 bytes (0.65x).

Test Plan: Updated bolt-address-translation.test

Reviewers: rafaelauler, dcci, ayermolo, JDevlieghere, maksfb

Reviewed By: maksfb

Pull Request: https://github.com/llvm/llvm-project/pull/87853
---
 bolt/docs/BAT.md                              |  9 +-
 .../bolt/Profile/BoltAddressTranslation.h     |  6 +-
 bolt/lib/Profile/BoltAddressTranslation.cpp   | 86 ++++++++++---------
 bolt/test/X86/bolt-address-translation.test   |  2 +-
 4 files changed, 53 insertions(+), 50 deletions(-)

diff --git a/bolt/docs/BAT.md b/bolt/docs/BAT.md
index f23ef1abf876..7ffb5d7c0081 100644
--- a/bolt/docs/BAT.md
+++ b/bolt/docs/BAT.md
@@ -81,9 +81,10 @@ Hot indices are delta encoded, implicitly starting at zero.
 | `FuncHash` | 8b | Function hash for input function | Hot |
 | `NumBlocks` | ULEB128 | Number of basic blocks in the original function | Hot |
 | `NumSecEntryPoints` | ULEB128 | Number of secondary entry points in the original function | Hot |
+| `ColdInputSkew` | ULEB128 | Skew to apply to all input offsets | Cold |
 | `NumEntries` | ULEB128 | Number of address translation entries for a function | Both |
-| `EqualElems` | ULEB128 | Number of equal offsets in the beginning of a function | Hot |
-| `BranchEntries` | Bitmask, `alignTo(EqualElems, 8)` bits | If `EqualElems` is non-zero, bitmask denoting entries with `BRANCHENTRY` bit | Hot |
+| `EqualElems` | ULEB128 | Number of equal offsets in the beginning of a function | Both |
+| `BranchEntries` | Bitmask, `alignTo(EqualElems, 8)` bits | If `EqualElems` is non-zero, bitmask denoting entries with `BRANCHENTRY` bit | Both |
 
 Function header is followed by *Address Translation Table* with `NumEntries`
 total entries, and *Secondary Entry Points* table with `NumSecEntryPoints`
@@ -99,8 +100,8 @@ entry is encoded. Input offsets implicitly start at zero.
 | `BBHash` | Optional, 8b | Basic block hash in input binary | BB |
 | `BBIdx`  | Optional, Delta, ULEB128 | Basic block index in input binary | BB |
 
-For hot fragments, the table omits the first `EqualElems` input offsets
-where the input offset equals output offset.
+The table omits the first `EqualElems` input offsets where the input offset
+equals output offset.
 
 `BRANCHENTRY` bit denotes whether a given offset pair is a control flow source
 (branch or call instruction). If not set, it signifies a control flow target
diff --git a/bolt/include/bolt/Profile/BoltAddressTranslation.h b/bolt/include/bolt/Profile/BoltAddressTranslation.h
index eef05e8a0e68..68b993ee363c 100644
--- a/bolt/include/bolt/Profile/BoltAddressTranslation.h
+++ b/bolt/include/bolt/Profile/BoltAddressTranslation.h
@@ -149,9 +149,9 @@ private:
   /// entries in function address translation map.
   APInt calculateBranchEntriesBitMask(MapTy &Map, size_t EqualElems);
 
-  /// Calculate the number of equal offsets (output = input) in the beginning
-  /// of the function.
-  size_t getNumEqualOffsets(const MapTy &Map) const;
+  /// Calculate the number of equal offsets (output = input - skew) in the
+  /// beginning of the function.
+  size_t getNumEqualOffsets(const MapTy &Map, uint32_t Skew) const;
 
   std::map<uint64_t, MapTy> Maps;
 
diff --git a/bolt/lib/Profile/BoltAddressTranslation.cpp b/bolt/lib/Profile/BoltAddressTranslation.cpp
index 0141ce189acd..ac6e3d01f271 100644
--- a/bolt/lib/Profile/BoltAddressTranslation.cpp
+++ b/bolt/lib/Profile/BoltAddressTranslation.cpp
@@ -153,12 +153,13 @@ APInt BoltAddressTranslation::calculateBranchEntriesBitMask(MapTy &Map,
   return BitMask;
 }
 
-size_t BoltAddressTranslation::getNumEqualOffsets(const MapTy &Map) const {
+size_t BoltAddressTranslation::getNumEqualOffsets(const MapTy &Map,
+                                                  uint32_t Skew) const {
   size_t EqualOffsets = 0;
   for (const std::pair<const uint32_t, uint32_t> &KeyVal : Map) {
     const uint32_t OutputOffset = KeyVal.first;
     const uint32_t InputOffset = KeyVal.second >> 1;
-    if (OutputOffset == InputOffset)
+    if (OutputOffset == InputOffset - Skew)
       ++EqualOffsets;
     else
       break;
@@ -196,12 +197,17 @@ void BoltAddressTranslation::writeMaps(std::map<uint64_t, MapTy> &Maps,
         SecondaryEntryPointsMap.count(Address)
             ? SecondaryEntryPointsMap[Address].size()
             : 0;
+    uint32_t Skew = 0;
     if (Cold) {
       auto HotEntryIt = Maps.find(ColdPartSource[Address]);
       assert(HotEntryIt != Maps.end());
       size_t HotIndex = std::distance(Maps.begin(), HotEntryIt);
       encodeULEB128(HotIndex - PrevIndex, OS);
       PrevIndex = HotIndex;
+      // Skew of all input offsets for cold fragments is simply the first input
+      // offset.
+      Skew = Map.begin()->second >> 1;
+      encodeULEB128(Skew, OS);
     } else {
       // Function hash
       size_t BFHash = getBFHash(HotInputAddress);
@@ -217,24 +223,21 @@ void BoltAddressTranslation::writeMaps(std::map<uint64_t, MapTy> &Maps,
                         << '\n');
     }
     encodeULEB128(NumEntries, OS);
-    // For hot fragments only: encode the number of equal offsets
-    // (output = input) in the beginning of the function. Only encode one offset
-    // in these cases.
-    const size_t EqualElems = Cold ? 0 : getNumEqualOffsets(Map);
-    if (!Cold) {
-      encodeULEB128(EqualElems, OS);
-      if (EqualElems) {
-        const size_t BranchEntriesBytes = alignTo(EqualElems, 8) / 8;
-        APInt BranchEntries = calculateBranchEntriesBitMask(Map, EqualElems);
-        OS.write(reinterpret_cast<const char *>(BranchEntries.getRawData()),
-                 BranchEntriesBytes);
-        LLVM_DEBUG({
-          dbgs() << "BranchEntries: ";
-          SmallString<8> BitMaskStr;
-          BranchEntries.toString(BitMaskStr, 2, false);
-          dbgs() << BitMaskStr << '\n';
-        });
-      }
+    // Encode the number of equal offsets (output = input - skew) in the
+    // beginning of the function. Only encode one offset in these cases.
+    const size_t EqualElems = getNumEqualOffsets(Map, Skew);
+    encodeULEB128(EqualElems, OS);
+    if (EqualElems) {
+      const size_t BranchEntriesBytes = alignTo(EqualElems, 8) / 8;
+      APInt BranchEntries = calculateBranchEntriesBitMask(Map, EqualElems);
+      OS.write(reinterpret_cast<const char *>(BranchEntries.getRawData()),
+               BranchEntriesBytes);
+      LLVM_DEBUG({
+        dbgs() << "BranchEntries: ";
+        SmallString<8> BitMaskStr;
+        BranchEntries.toString(BitMaskStr, 2, false);
+        dbgs() << BitMaskStr << '\n';
+      });
     }
     const BBHashMapTy &BBHashMap = getBBHashMap(HotInputAddress);
     size_t Index = 0;
@@ -315,10 +318,12 @@ void BoltAddressTranslation::parseMaps(std::vector<uint64_t> &HotFuncs,
     uint64_t HotAddress = Cold ? 0 : Address;
     PrevAddress = Address;
     uint32_t SecondaryEntryPoints = 0;
+    uint64_t ColdInputSkew = 0;
     if (Cold) {
       HotIndex += DE.getULEB128(&Offset, &Err);
       HotAddress = HotFuncs[HotIndex];
       ColdPartSource.emplace(Address, HotAddress);
+      ColdInputSkew = DE.getULEB128(&Offset, &Err);
     } else {
       HotFuncs.push_back(Address);
       // Function hash
@@ -339,28 +344,25 @@ void BoltAddressTranslation::parseMaps(std::vector<uint64_t> &HotFuncs,
                             getULEB128Size(SecondaryEntryPoints)));
     }
     const uint32_t NumEntries = DE.getULEB128(&Offset, &Err);
-    // Equal offsets, hot fragments only.
-    size_t EqualElems = 0;
+    // Equal offsets.
+    const size_t EqualElems = DE.getULEB128(&Offset, &Err);
     APInt BEBitMask;
-    if (!Cold) {
-      EqualElems = DE.getULEB128(&Offset, &Err);
-      LLVM_DEBUG(dbgs() << formatv("Equal offsets: {0}, {1} bytes\n",
-                                   EqualElems, getULEB128Size(EqualElems)));
-      if (EqualElems) {
-        const size_t BranchEntriesBytes = alignTo(EqualElems, 8) / 8;
-        BEBitMask = APInt(alignTo(EqualElems, 8), 0);
-        LoadIntFromMemory(
-            BEBitMask,
-            reinterpret_cast<const uint8_t *>(
-                DE.getBytes(&Offset, BranchEntriesBytes, &Err).data()),
-            BranchEntriesBytes);
-        LLVM_DEBUG({
-          dbgs() << "BEBitMask: ";
-          SmallString<8> BitMaskStr;
-          BEBitMask.toString(BitMaskStr, 2, false);
-          dbgs() << BitMaskStr << ", " << BranchEntriesBytes << " bytes\n";
-        });
-      }
+    LLVM_DEBUG(dbgs() << formatv("Equal offsets: {0}, {1} bytes\n", EqualElems,
+                                 getULEB128Size(EqualElems)));
+    if (EqualElems) {
+      const size_t BranchEntriesBytes = alignTo(EqualElems, 8) / 8;
+      BEBitMask = APInt(alignTo(EqualElems, 8), 0);
+      LoadIntFromMemory(
+          BEBitMask,
+          reinterpret_cast<const uint8_t *>(
+              DE.getBytes(&Offset, BranchEntriesBytes, &Err).data()),
+          BranchEntriesBytes);
+      LLVM_DEBUG({
+        dbgs() << "BEBitMask: ";
+        SmallString<8> BitMaskStr;
+        BEBitMask.toString(BitMaskStr, 2, false);
+        dbgs() << BitMaskStr << ", " << BranchEntriesBytes << " bytes\n";
+      });
     }
     MapTy Map;
 
@@ -375,7 +377,7 @@ void BoltAddressTranslation::parseMaps(std::vector<uint64_t> &HotFuncs,
       PrevAddress = OutputAddress;
       int64_t InputDelta = 0;
       if (J < EqualElems) {
-        InputOffset = (OutputOffset << 1) | BEBitMask[J];
+        InputOffset = (OutputOffset + ColdInputSkew << 1) | BEBitMask[J];
       } else {
         InputDelta = DE.getSLEB128(&Offset, &Err);
         InputOffset += InputDelta;
diff --git a/bolt/test/X86/bolt-address-translation.test b/bolt/test/X86/bolt-address-translation.test
index 63234b4c1d21..e6b21c14077b 100644
--- a/bolt/test/X86/bolt-address-translation.test
+++ b/bolt/test/X86/bolt-address-translation.test
@@ -37,7 +37,7 @@
 # CHECK:      BOLT: 3 out of 7 functions were overwritten.
 # CHECK:      BOLT-INFO: Wrote 6 BAT maps
 # CHECK:      BOLT-INFO: Wrote 3 function and 58 basic block hashes
-# CHECK:      BOLT-INFO: BAT section size (bytes): 924
+# CHECK:      BOLT-INFO: BAT section size (bytes): 928
 #
 # usqrt mappings (hot part). We match against any key (left side containing
 # the bolted binary offsets) because BOLT may change where it puts instructions
-- 
GitLab


From f76808351677f4e361f2acd9c4b3f385901d37ef Mon Sep 17 00:00:00 2001
From: Vyacheslav Levytskyy <vyacheslav.levytskyy@intel.com>
Date: Mon, 15 Apr 2024 09:59:47 +0200
Subject: [PATCH 205/464] [SPIR-V] Update type inference and instruction
 selection (#88254)

This PR contains a series of fixes which are to improve type inference
and instruction selection.

Namely, it includes:
* fix OpSelect to support operands of a pointer type, according to the
SPIR-V specification (previously only integer/float/vectors of integer
or float were supported) -- a new test case is added and existing test
case is updated;
* fix TableGen typo's in definition of register classes and introduce a
new reg class that is a vector of pointers;
* fix usage of a machine function context when there is a need to switch
between different machine functions to infer/validate correct types;
* add usage of TypedPointerType instead of PointerType so that later
stages of type inference are able to distinguish pointer types by their
element types, effectively supporting hierarchy of pointer/pointee types
and avoiding more complicated recursive type matching on level of
machine instructions in favor of direct pointer comparison using LLVM's
`Type *` values;
* extracting detailed information about operand types using known type
rules for some llvm instructions (for instance, by deducing PHI's
operand pointee types if PHI's results type was deducted on previous
stages of type inference), and adding correspondent
`Intrinsic::spv_assign_ptr_type` to keep type info along consequent
passes,
* ensure that OpConstantComposite reuses a constant when it's already
created and available in the same machine function -- otherwise there is
a crash while building a dependency graph, the corresponding test case
is attached,
* implement deduction of function's return type for opaque pointers, a
new test case is attached,
* make 'emit intrinsics' a module pass to resolve function return types
over the module -- first types for all functions of the module must be
calculated, and only after that it's feasible to deduct function return
types on this earlier stage of translation.
---
 llvm/lib/Target/SPIRV/SPIRV.h                 |   2 +-
 llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp   |  18 +-
 llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 167 +++++++++++++++++-
 llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp |  11 +-
 llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h   |  22 ++-
 llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp   |  23 ++-
 llvm/lib/Target/SPIRV/SPIRVInstrInfo.cpp      |   2 +-
 llvm/lib/Target/SPIRV/SPIRVInstrInfo.td       |  13 +-
 llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp  |   2 +-
 llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp   |  13 +-
 .../Target/SPIRV/SPIRVRegisterBankInfo.cpp    |   2 +
 llvm/lib/Target/SPIRV/SPIRVRegisterBanks.td   |   1 +
 llvm/lib/Target/SPIRV/SPIRVRegisterInfo.td    |  19 +-
 llvm/lib/Target/SPIRV/SPIRVUtils.cpp          |   3 +-
 llvm/lib/Target/SPIRV/SPIRVUtils.h            |   7 +
 llvm/test/CodeGen/SPIRV/const-composite.ll    |  26 +++
 .../CodeGen/SPIRV/instructions/ret-type.ll    |  82 +++++++++
 .../CodeGen/SPIRV/instructions/select-phi.ll  |  58 ++++++
 .../test/CodeGen/SPIRV/instructions/select.ll |  15 ++
 19 files changed, 448 insertions(+), 38 deletions(-)
 create mode 100644 llvm/test/CodeGen/SPIRV/const-composite.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/instructions/ret-type.ll
 create mode 100644 llvm/test/CodeGen/SPIRV/instructions/select-phi.ll

diff --git a/llvm/lib/Target/SPIRV/SPIRV.h b/llvm/lib/Target/SPIRV/SPIRV.h
index 6979107349d9..fb8580cd47c0 100644
--- a/llvm/lib/Target/SPIRV/SPIRV.h
+++ b/llvm/lib/Target/SPIRV/SPIRV.h
@@ -24,7 +24,7 @@ FunctionPass *createSPIRVStripConvergenceIntrinsicsPass();
 FunctionPass *createSPIRVRegularizerPass();
 FunctionPass *createSPIRVPreLegalizerPass();
 FunctionPass *createSPIRVPostLegalizerPass();
-FunctionPass *createSPIRVEmitIntrinsicsPass(SPIRVTargetMachine *TM);
+ModulePass *createSPIRVEmitIntrinsicsPass(SPIRVTargetMachine *TM);
 InstructionSelector *
 createSPIRVInstructionSelector(const SPIRVTargetMachine &TM,
                                const SPIRVSubtarget &Subtarget,
diff --git a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
index 9e4ba2191366..c107b99cf4cb 100644
--- a/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVCallLowering.cpp
@@ -383,7 +383,16 @@ bool SPIRVCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
   if (F.isDeclaration())
     GR->add(&F, &MIRBuilder.getMF(), FuncVReg);
   FunctionType *FTy = getOriginalFunctionType(F);
-  SPIRVType *RetTy = GR->getOrCreateSPIRVType(FTy->getReturnType(), MIRBuilder);
+  Type *FRetTy = FTy->getReturnType();
+  if (isUntypedPointerTy(FRetTy)) {
+    if (Type *FRetElemTy = GR->findDeducedElementType(&F)) {
+      TypedPointerType *DerivedTy =
+          TypedPointerType::get(FRetElemTy, getPointerAddressSpace(FRetTy));
+      GR->addReturnType(&F, DerivedTy);
+      FRetTy = DerivedTy;
+    }
+  }
+  SPIRVType *RetTy = GR->getOrCreateSPIRVType(FRetTy, MIRBuilder);
   FTy = fixFunctionTypeIfPtrArgs(GR, F, FTy, RetTy, ArgTypeVRegs);
   SPIRVType *FuncTy = GR->getOrCreateOpTypeFunctionWithArgs(
       FTy, RetTy, ArgTypeVRegs, MIRBuilder);
@@ -505,8 +514,13 @@ bool SPIRVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
     // TODO: support constexpr casts and indirect calls.
     if (CF == nullptr)
       return false;
-    if (FunctionType *FTy = getOriginalFunctionType(*CF))
+    if (FunctionType *FTy = getOriginalFunctionType(*CF)) {
       OrigRetTy = FTy->getReturnType();
+      if (isUntypedPointerTy(OrigRetTy)) {
+        if (auto *DerivedRetTy = GR->findReturnType(CF))
+          OrigRetTy = DerivedRetTy;
+      }
+    }
   }
 
   MachineRegisterInfo *MRI = MIRBuilder.getMRI();
diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
index e8ce5a35b457..472bc8638c9a 100644
--- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp
@@ -51,7 +51,7 @@ void initializeSPIRVEmitIntrinsicsPass(PassRegistry &);
 
 namespace {
 class SPIRVEmitIntrinsics
-    : public FunctionPass,
+    : public ModulePass,
       public InstVisitor<SPIRVEmitIntrinsics, Instruction *> {
   SPIRVTargetMachine *TM = nullptr;
   SPIRVGlobalRegistry *GR = nullptr;
@@ -61,6 +61,9 @@ class SPIRVEmitIntrinsics
   DenseMap<Instruction *, Type *> AggrConstTypes;
   DenseSet<Instruction *> AggrStores;
 
+  // a registry of created Intrinsic::spv_assign_ptr_type instructions
+  DenseMap<Value *, CallInst *> AssignPtrTypeInstr;
+
   // deduce element type of untyped pointers
   Type *deduceElementType(Value *I);
   Type *deduceElementTypeHelper(Value *I);
@@ -75,6 +78,9 @@ class SPIRVEmitIntrinsics
   Type *deduceNestedTypeHelper(User *U, Type *Ty,
                                std::unordered_set<Value *> &Visited);
 
+  // deduce Types of operands of the Instruction if possible
+  void deduceOperandElementType(Instruction *I);
+
   void preprocessCompositeConstants(IRBuilder<> &B);
   void preprocessUndefs(IRBuilder<> &B);
 
@@ -111,10 +117,10 @@ class SPIRVEmitIntrinsics
 
 public:
   static char ID;
-  SPIRVEmitIntrinsics() : FunctionPass(ID) {
+  SPIRVEmitIntrinsics() : ModulePass(ID) {
     initializeSPIRVEmitIntrinsicsPass(*PassRegistry::getPassRegistry());
   }
-  SPIRVEmitIntrinsics(SPIRVTargetMachine *_TM) : FunctionPass(ID), TM(_TM) {
+  SPIRVEmitIntrinsics(SPIRVTargetMachine *_TM) : ModulePass(ID), TM(_TM) {
     initializeSPIRVEmitIntrinsicsPass(*PassRegistry::getPassRegistry());
   }
   Instruction *visitInstruction(Instruction &I) { return &I; }
@@ -130,7 +136,15 @@ public:
   Instruction *visitAllocaInst(AllocaInst &I);
   Instruction *visitAtomicCmpXchgInst(AtomicCmpXchgInst &I);
   Instruction *visitUnreachableInst(UnreachableInst &I);
-  bool runOnFunction(Function &F) override;
+
+  StringRef getPassName() const override { return "SPIRV emit intrinsics"; }
+
+  bool runOnModule(Module &M) override;
+  bool runOnFunction(Function &F);
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    ModulePass::getAnalysisUsage(AU);
+  }
 };
 } // namespace
 
@@ -269,6 +283,12 @@ Type *SPIRVEmitIntrinsics::deduceElementTypeHelper(
       if (Ty)
         break;
     }
+  } else if (auto *Ref = dyn_cast<SelectInst>(I)) {
+    for (Value *Op : {Ref->getTrueValue(), Ref->getFalseValue()}) {
+      Ty = deduceElementTypeByUsersDeep(Op, Visited);
+      if (Ty)
+        break;
+    }
   }
 
   // remember the found relationship
@@ -368,6 +388,112 @@ Type *SPIRVEmitIntrinsics::deduceElementType(Value *I) {
   return IntegerType::getInt8Ty(I->getContext());
 }
 
+// If the Instruction has Pointer operands with unresolved types, this function
+// tries to deduce them. If the Instruction has Pointer operands with known
+// types which differ from expected, this function tries to insert a bitcast to
+// resolve the issue.
+void SPIRVEmitIntrinsics::deduceOperandElementType(Instruction *I) {
+  SmallVector<std::pair<Value *, unsigned>> Ops;
+  Type *KnownElemTy = nullptr;
+  // look for known basic patterns of type inference
+  if (auto *Ref = dyn_cast<PHINode>(I)) {
+    if (!isPointerTy(I->getType()) ||
+        !(KnownElemTy = GR->findDeducedElementType(I)))
+      return;
+    for (unsigned i = 0; i < Ref->getNumIncomingValues(); i++) {
+      Value *Op = Ref->getIncomingValue(i);
+      if (isPointerTy(Op->getType()))
+        Ops.push_back(std::make_pair(Op, i));
+    }
+  } else if (auto *Ref = dyn_cast<SelectInst>(I)) {
+    if (!isPointerTy(I->getType()) ||
+        !(KnownElemTy = GR->findDeducedElementType(I)))
+      return;
+    for (unsigned i = 0; i < Ref->getNumOperands(); i++) {
+      Value *Op = Ref->getOperand(i);
+      if (isPointerTy(Op->getType()))
+        Ops.push_back(std::make_pair(Op, i));
+    }
+  } else if (auto *Ref = dyn_cast<ReturnInst>(I)) {
+    Type *RetTy = F->getReturnType();
+    if (!isPointerTy(RetTy))
+      return;
+    Value *Op = Ref->getReturnValue();
+    if (!Op)
+      return;
+    if (!(KnownElemTy = GR->findDeducedElementType(F))) {
+      if (Type *OpElemTy = GR->findDeducedElementType(Op)) {
+        GR->addDeducedElementType(F, OpElemTy);
+        TypedPointerType *DerivedTy =
+            TypedPointerType::get(OpElemTy, getPointerAddressSpace(RetTy));
+        GR->addReturnType(F, DerivedTy);
+      }
+      return;
+    }
+    Ops.push_back(std::make_pair(Op, 0));
+  } else if (auto *Ref = dyn_cast<ICmpInst>(I)) {
+    if (!isPointerTy(Ref->getOperand(0)->getType()))
+      return;
+    Value *Op0 = Ref->getOperand(0);
+    Value *Op1 = Ref->getOperand(1);
+    Type *ElemTy0 = GR->findDeducedElementType(Op0);
+    Type *ElemTy1 = GR->findDeducedElementType(Op1);
+    if (ElemTy0) {
+      KnownElemTy = ElemTy0;
+      Ops.push_back(std::make_pair(Op1, 1));
+    } else if (ElemTy1) {
+      KnownElemTy = ElemTy1;
+      Ops.push_back(std::make_pair(Op0, 0));
+    }
+  }
+
+  // There is no enough info to deduce types or all is valid.
+  if (!KnownElemTy || Ops.size() == 0)
+    return;
+
+  LLVMContext &Ctx = F->getContext();
+  IRBuilder<> B(Ctx);
+  for (auto &OpIt : Ops) {
+    Value *Op = OpIt.first;
+    if (Op->use_empty())
+      continue;
+    Type *Ty = GR->findDeducedElementType(Op);
+    if (Ty == KnownElemTy)
+      continue;
+    if (Instruction *User = dyn_cast<Instruction>(Op->use_begin()->get()))
+      setInsertPointSkippingPhis(B, User->getNextNode());
+    else
+      B.SetInsertPoint(I);
+    Value *OpTyVal = Constant::getNullValue(KnownElemTy);
+    Type *OpTy = Op->getType();
+    if (!Ty) {
+      GR->addDeducedElementType(Op, KnownElemTy);
+      // check if there is existing Intrinsic::spv_assign_ptr_type instruction
+      auto It = AssignPtrTypeInstr.find(Op);
+      if (It == AssignPtrTypeInstr.end()) {
+        CallInst *CI =
+            buildIntrWithMD(Intrinsic::spv_assign_ptr_type, {OpTy}, OpTyVal, Op,
+                            {B.getInt32(getPointerAddressSpace(OpTy))}, B);
+        AssignPtrTypeInstr[Op] = CI;
+      } else {
+        It->second->setArgOperand(
+            1,
+            MetadataAsValue::get(
+                Ctx, MDNode::get(Ctx, ValueAsMetadata::getConstant(OpTyVal))));
+      }
+    } else {
+      SmallVector<Type *, 2> Types = {OpTy, OpTy};
+      MetadataAsValue *VMD = MetadataAsValue::get(
+          Ctx, MDNode::get(Ctx, ValueAsMetadata::getConstant(OpTyVal)));
+      SmallVector<Value *, 2> Args = {Op, VMD,
+                                      B.getInt32(getPointerAddressSpace(OpTy))};
+      CallInst *PtrCastI =
+          B.CreateIntrinsic(Intrinsic::spv_ptrcast, {Types}, Args);
+      I->setOperand(OpIt.second, PtrCastI);
+    }
+  }
+}
+
 void SPIRVEmitIntrinsics::replaceMemInstrUses(Instruction *Old,
                                               Instruction *New,
                                               IRBuilder<> &B) {
@@ -630,6 +756,7 @@ void SPIRVEmitIntrinsics::replacePointerOperandWithPtrCast(
         ExpectedElementTypeConst, Pointer, {B.getInt32(AddressSpace)}, B);
     GR->addDeducedElementType(CI, ExpectedElementType);
     GR->addDeducedElementType(Pointer, ExpectedElementType);
+    AssignPtrTypeInstr[Pointer] = CI;
     return;
   }
 
@@ -914,6 +1041,7 @@ void SPIRVEmitIntrinsics::insertAssignPtrTypeIntrs(Instruction *I,
   CallInst *CI = buildIntrWithMD(Intrinsic::spv_assign_ptr_type, {I->getType()},
                                  EltTyConst, I, {B.getInt32(AddressSpace)}, B);
   GR->addDeducedElementType(CI, ElemTy);
+  AssignPtrTypeInstr[I] = CI;
 }
 
 void SPIRVEmitIntrinsics::insertAssignTypeIntrs(Instruction *I,
@@ -1070,6 +1198,7 @@ void SPIRVEmitIntrinsics::processParamTypes(Function *F, IRBuilder<> &B) {
             {B.getInt32(getPointerAddressSpace(Arg->getType()))}, B);
         GR->addDeducedElementType(AssignPtrTyCI, ElemTy);
         GR->addDeducedElementType(Arg, ElemTy);
+        AssignPtrTypeInstr[Arg] = AssignPtrTyCI;
       }
     }
   }
@@ -1114,6 +1243,10 @@ bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) {
     insertAssignTypeIntrs(I, B);
     insertPtrCastOrAssignTypeInstr(I, B);
   }
+
+  for (auto &I : instructions(Func))
+    deduceOperandElementType(&I);
+
   for (auto *I : Worklist) {
     TrackConstants = true;
     if (!I->getType()->isVoidTy() || isa<StoreInst>(I))
@@ -1126,13 +1259,29 @@ bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) {
     processInstrAfterVisit(I, B);
   }
 
-  // check if function parameter types are set
-  if (!F->isIntrinsic())
-    processParamTypes(F, B);
-
   return true;
 }
 
-FunctionPass *llvm::createSPIRVEmitIntrinsicsPass(SPIRVTargetMachine *TM) {
+bool SPIRVEmitIntrinsics::runOnModule(Module &M) {
+  bool Changed = false;
+
+  for (auto &F : M) {
+    Changed |= runOnFunction(F);
+  }
+
+  for (auto &F : M) {
+    // check if function parameter types are set
+    if (!F.isDeclaration() && !F.isIntrinsic()) {
+      const SPIRVSubtarget &ST = TM->getSubtarget<SPIRVSubtarget>(F);
+      GR = ST.getSPIRVGlobalRegistry();
+      IRBuilder<> B(F.getContext());
+      processParamTypes(&F, B);
+    }
+  }
+
+  return Changed;
+}
+
+ModulePass *llvm::createSPIRVEmitIntrinsicsPass(SPIRVTargetMachine *TM) {
   return new SPIRVEmitIntrinsics(TM);
 }
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
index 70197e948c65..05e41e06248e 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp
@@ -23,7 +23,6 @@
 #include "llvm/ADT/APInt.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Type.h"
-#include "llvm/IR/TypedPointerType.h"
 #include "llvm/Support/Casting.h"
 #include <cassert>
 
@@ -61,7 +60,6 @@ SPIRVType *SPIRVGlobalRegistry::assignVectTypeToVReg(
 SPIRVType *SPIRVGlobalRegistry::assignTypeToVReg(
     const Type *Type, Register VReg, MachineIRBuilder &MIRBuilder,
     SPIRV::AccessQualifier::AccessQualifier AccessQual, bool EmitIR) {
-
   SPIRVType *SpirvType =
       getOrCreateSPIRVType(Type, MIRBuilder, AccessQual, EmitIR);
   assignSPIRVTypeToVReg(SpirvType, VReg, MIRBuilder.getMF());
@@ -726,7 +724,8 @@ SPIRVType *SPIRVGlobalRegistry::getOpTypeStruct(const StructType *Ty,
                                                 bool EmitIR) {
   SmallVector<Register, 4> FieldTypes;
   for (const auto &Elem : Ty->elements()) {
-    SPIRVType *ElemTy = findSPIRVType(Elem, MIRBuilder);
+    SPIRVType *ElemTy =
+        findSPIRVType(toTypedPointer(Elem, Ty->getContext()), MIRBuilder);
     assert(ElemTy && ElemTy->getOpcode() != SPIRV::OpTypeVoid &&
            "Invalid struct element type");
     FieldTypes.push_back(getSPIRVTypeID(ElemTy));
@@ -919,8 +918,10 @@ SPIRVType *SPIRVGlobalRegistry::restOfCreateSPIRVType(
   return SpirvType;
 }
 
-SPIRVType *SPIRVGlobalRegistry::getSPIRVTypeForVReg(Register VReg) const {
-  auto t = VRegToTypeMap.find(CurMF);
+SPIRVType *
+SPIRVGlobalRegistry::getSPIRVTypeForVReg(Register VReg,
+                                         const MachineFunction *MF) const {
+  auto t = VRegToTypeMap.find(MF ? MF : CurMF);
   if (t != VRegToTypeMap.end()) {
     auto tt = t->second.find(VReg);
     if (tt != t->second.end())
diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
index 2e3e69456ac2..55979ba403a0 100644
--- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
+++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h
@@ -21,6 +21,7 @@
 #include "SPIRVInstrInfo.h"
 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
 #include "llvm/IR/Constant.h"
+#include "llvm/IR/TypedPointerType.h"
 
 namespace llvm {
 using SPIRVType = const MachineInstr;
@@ -58,6 +59,9 @@ class SPIRVGlobalRegistry {
   SmallPtrSet<const Type *, 4> TypesInProcessing;
   DenseMap<const Type *, SPIRVType *> ForwardPointerTypes;
 
+  // if a function returns a pointer, this is to map it into TypedPointerType
+  DenseMap<const Function *, TypedPointerType *> FunResPointerTypes;
+
   // Number of bits pointers and size_t integers require.
   const unsigned PointerSize;
 
@@ -134,6 +138,16 @@ public:
   void setBound(unsigned V) { Bound = V; }
   unsigned getBound() { return Bound; }
 
+  // Add a record to the map of function return pointer types.
+  void addReturnType(const Function *ArgF, TypedPointerType *DerivedTy) {
+    FunResPointerTypes[ArgF] = DerivedTy;
+  }
+  // Find a record in the map of function return pointer types.
+  const TypedPointerType *findReturnType(const Function *ArgF) {
+    auto It = FunResPointerTypes.find(ArgF);
+    return It == FunResPointerTypes.end() ? nullptr : It->second;
+  }
+
   // Deduced element types of untyped pointers and composites:
   // - Add a record to the map of deduced element types.
   void addDeducedElementType(Value *Val, Type *Ty) { DeducedElTys[Val] = Ty; }
@@ -276,8 +290,12 @@ public:
           SPIRV::AccessQualifier::ReadWrite);
 
   // Return the SPIR-V type instruction corresponding to the given VReg, or
-  // nullptr if no such type instruction exists.
-  SPIRVType *getSPIRVTypeForVReg(Register VReg) const;
+  // nullptr if no such type instruction exists. The second argument MF
+  // allows to search for the association in a context of the machine functions
+  // than the current one, without switching between different "current" machine
+  // functions.
+  SPIRVType *getSPIRVTypeForVReg(Register VReg,
+                                 const MachineFunction *MF = nullptr) const;
 
   // Whether the given VReg has a SPIR-V type mapped to it yet.
   bool hasSPIRVTypeForVReg(Register VReg) const {
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
index 8db54c74f236..b8296c3f6eea 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp
@@ -88,19 +88,24 @@ static void validatePtrTypes(const SPIRVSubtarget &STI,
                              MachineRegisterInfo *MRI, SPIRVGlobalRegistry &GR,
                              MachineInstr &I, unsigned OpIdx,
                              SPIRVType *ResType, const Type *ResTy = nullptr) {
+  // Get operand type
+  MachineFunction *MF = I.getParent()->getParent();
   Register OpReg = I.getOperand(OpIdx).getReg();
   SPIRVType *TypeInst = MRI->getVRegDef(OpReg);
-  SPIRVType *OpType = GR.getSPIRVTypeForVReg(
+  Register OpTypeReg =
       TypeInst && TypeInst->getOpcode() == SPIRV::OpFunctionParameter
           ? TypeInst->getOperand(1).getReg()
-          : OpReg);
+          : OpReg;
+  SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpTypeReg, MF);
   if (!ResType || !OpType || OpType->getOpcode() != SPIRV::OpTypePointer)
     return;
-  SPIRVType *ElemType = GR.getSPIRVTypeForVReg(OpType->getOperand(2).getReg());
+  // Get operand's pointee type
+  Register ElemTypeReg = OpType->getOperand(2).getReg();
+  SPIRVType *ElemType = GR.getSPIRVTypeForVReg(ElemTypeReg, MF);
   if (!ElemType)
     return;
-  bool IsSameMF =
-      ElemType->getParent()->getParent() == ResType->getParent()->getParent();
+  // Check if we need a bitcast to make a statement valid
+  bool IsSameMF = MF == ResType->getParent()->getParent();
   bool IsEqualTypes = IsSameMF ? ElemType == ResType
                                : GR.getTypeForSPIRVType(ElemType) == ResTy;
   if (IsEqualTypes)
@@ -156,7 +161,8 @@ void validateFunCallMachineDef(const SPIRVSubtarget &STI,
     SPIRVType *DefPtrType = DefMRI->getVRegDef(FunDef->getOperand(1).getReg());
     SPIRVType *DefElemType =
         DefPtrType && DefPtrType->getOpcode() == SPIRV::OpTypePointer
-            ? GR.getSPIRVTypeForVReg(DefPtrType->getOperand(2).getReg())
+            ? GR.getSPIRVTypeForVReg(DefPtrType->getOperand(2).getReg(),
+                                     DefPtrType->getParent()->getParent())
             : nullptr;
     if (DefElemType) {
       const Type *DefElemTy = GR.getTypeForSPIRVType(DefElemType);
@@ -177,7 +183,7 @@ void validateFunCallMachineDef(const SPIRVSubtarget &STI,
 // with a processed definition. Return Function pointer if it's a forward
 // call (ahead of definition), and nullptr otherwise.
 const Function *validateFunCall(const SPIRVSubtarget &STI,
-                                MachineRegisterInfo *MRI,
+                                MachineRegisterInfo *CallMRI,
                                 SPIRVGlobalRegistry &GR,
                                 MachineInstr &FunCall) {
   const GlobalValue *GV = FunCall.getOperand(2).getGlobal();
@@ -186,7 +192,8 @@ const Function *validateFunCall(const SPIRVSubtarget &STI,
       const_cast<MachineInstr *>(GR.getFunctionDefinition(F));
   if (!FunDef)
     return F;
-  validateFunCallMachineDef(STI, MRI, MRI, GR, FunCall, FunDef);
+  MachineRegisterInfo *DefMRI = &FunDef->getParent()->getParent()->getRegInfo();
+  validateFunCallMachineDef(STI, DefMRI, CallMRI, GR, FunCall, FunDef);
   return nullptr;
 }
 
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.cpp b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.cpp
index e3f76419f131..aacfecc1e313 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.cpp
@@ -248,7 +248,7 @@ void SPIRVInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
 bool SPIRVInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
   if (MI.getOpcode() == SPIRV::GET_ID || MI.getOpcode() == SPIRV::GET_fID ||
       MI.getOpcode() == SPIRV::GET_pID || MI.getOpcode() == SPIRV::GET_vfID ||
-      MI.getOpcode() == SPIRV::GET_vID) {
+      MI.getOpcode() == SPIRV::GET_vID || MI.getOpcode() == SPIRV::GET_vpID) {
     auto &MRI = MI.getMF()->getRegInfo();
     MRI.replaceRegWith(MI.getOperand(0).getReg(), MI.getOperand(1).getReg());
     MI.eraseFromParent();
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
index 99c57dac4141..a3f981457c8d 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
+++ b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td
@@ -22,6 +22,7 @@ let isCodeGenOnly=1 in {
   def GET_pID: Pseudo<(outs pID:$dst_id), (ins ANYID:$src)>;
   def GET_vID: Pseudo<(outs vID:$dst_id), (ins ANYID:$src)>;
   def GET_vfID: Pseudo<(outs vfID:$dst_id), (ins ANYID:$src)>;
+  def GET_vpID: Pseudo<(outs vpID:$dst_id), (ins ANYID:$src)>;
 }
 
 def SPVTypeBin : SDTypeProfile<1, 2, []>;
@@ -55,7 +56,7 @@ multiclass BinOpTypedGen<string name, bits<16> opCode, SDNode node, bit genF = 0
   }
 }
 
-multiclass TernOpTypedGen<string name, bits<16> opCode, SDNode node, bit genI = 1, bit genF = 0, bit genV = 0> {
+multiclass TernOpTypedGen<string name, bits<16> opCode, SDNode node, bit genP = 1, bit genI = 1, bit genF = 0, bit genV = 0> {
   if genF then {
     def SFSCond: TernOpTyped<name, opCode, ID, fID, node>;
     def SFVCond: TernOpTyped<name, opCode, vID, fID, node>;
@@ -64,6 +65,10 @@ multiclass TernOpTypedGen<string name, bits<16> opCode, SDNode node, bit genI =
     def SISCond: TernOpTyped<name, opCode, ID, ID, node>;
     def SIVCond: TernOpTyped<name, opCode, vID, ID, node>;
   }
+  if genP then {
+    def SPSCond: TernOpTyped<name, opCode, ID, pID, node>;
+    def SPVCond: TernOpTyped<name, opCode, vID, pID, node>;
+  }
   if genV then {
     if genF then {
       def VFSCond: TernOpTyped<name, opCode, ID, vfID, node>;
@@ -73,6 +78,10 @@ multiclass TernOpTypedGen<string name, bits<16> opCode, SDNode node, bit genI =
       def VISCond: TernOpTyped<name, opCode, ID, vID, node>;
       def VIVCond: TernOpTyped<name, opCode, vID, vID, node>;
     }
+    if genP then {
+      def VPSCond: TernOpTyped<name, opCode, ID, vpID, node>;
+      def VPVCond: TernOpTyped<name, opCode, vID, vpID, node>;
+    }
   }
 }
 
@@ -552,7 +561,7 @@ def OpLogicalOr: BinOp<"OpLogicalOr", 166>;
 def OpLogicalAnd: BinOp<"OpLogicalAnd", 167>;
 def OpLogicalNot: UnOp<"OpLogicalNot", 168>;
 
-defm OpSelect: TernOpTypedGen<"OpSelect", 169, select, 1, 1, 1>;
+defm OpSelect: TernOpTypedGen<"OpSelect", 169, select, 1, 1, 1, 1>;
 
 def OpIEqual: BinOp<"OpIEqual", 170>;
 def OpINotEqual: BinOp<"OpINotEqual", 171>;
diff --git a/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp
index b9d66de9555b..f069a92ac686 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPostLegalizer.cpp
@@ -56,7 +56,7 @@ extern void processInstr(MachineInstr &MI, MachineIRBuilder &MIB,
 static bool isMetaInstrGET(unsigned Opcode) {
   return Opcode == SPIRV::GET_ID || Opcode == SPIRV::GET_fID ||
          Opcode == SPIRV::GET_pID || Opcode == SPIRV::GET_vID ||
-         Opcode == SPIRV::GET_vfID;
+         Opcode == SPIRV::GET_vfID || Opcode == SPIRV::GET_vpID;
 }
 
 static bool mayBeInserted(unsigned Opcode) {
diff --git a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
index 7e155a36aadb..2c964595fc39 100644
--- a/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp
@@ -64,9 +64,16 @@ static void addConstantsToTrack(MachineFunction &MF, SPIRVGlobalRegistry *GR) {
             auto *BuildVec = MRI.getVRegDef(MI.getOperand(2).getReg());
             assert(BuildVec &&
                    BuildVec->getOpcode() == TargetOpcode::G_BUILD_VECTOR);
-            for (unsigned i = 0; i < ConstVec->getNumElements(); ++i)
-              GR->add(ConstVec->getElementAsConstant(i), &MF,
-                      BuildVec->getOperand(1 + i).getReg());
+            for (unsigned i = 0; i < ConstVec->getNumElements(); ++i) {
+              // Ensure that OpConstantComposite reuses a constant when it's
+              // already created and available in the same machine function.
+              Constant *ElemConst = ConstVec->getElementAsConstant(i);
+              Register ElemReg = GR->find(ElemConst, &MF);
+              if (!ElemReg.isValid())
+                GR->add(ElemConst, &MF, BuildVec->getOperand(1 + i).getReg());
+              else
+                BuildVec->getOperand(1 + i).setReg(ElemReg);
+            }
           }
           GR->add(Const, &MF, MI.getOperand(2).getReg());
         } else {
diff --git a/llvm/lib/Target/SPIRV/SPIRVRegisterBankInfo.cpp b/llvm/lib/Target/SPIRV/SPIRVRegisterBankInfo.cpp
index 9bf9d7fe5b39..5983c9229cb3 100644
--- a/llvm/lib/Target/SPIRV/SPIRVRegisterBankInfo.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVRegisterBankInfo.cpp
@@ -39,6 +39,8 @@ SPIRVRegisterBankInfo::getRegBankFromRegClass(const TargetRegisterClass &RC,
     return SPIRV::vIDRegBank;
   case SPIRV::vfIDRegClassID:
     return SPIRV::vfIDRegBank;
+  case SPIRV::vpIDRegClassID:
+    return SPIRV::vpIDRegBank;
   case SPIRV::ANYIDRegClassID:
   case SPIRV::ANYRegClassID:
     return SPIRV::IDRegBank;
diff --git a/llvm/lib/Target/SPIRV/SPIRVRegisterBanks.td b/llvm/lib/Target/SPIRV/SPIRVRegisterBanks.td
index 90c7f3a6e672..c7f1e172f3d4 100644
--- a/llvm/lib/Target/SPIRV/SPIRVRegisterBanks.td
+++ b/llvm/lib/Target/SPIRV/SPIRVRegisterBanks.td
@@ -12,4 +12,5 @@ def IDRegBank : RegisterBank<"IDBank", [ID]>;
 def fIDRegBank : RegisterBank<"fIDBank", [fID]>;
 def vIDRegBank : RegisterBank<"vIDBank", [vID]>;
 def vfIDRegBank : RegisterBank<"vfIDBank", [vfID]>;
+def vpIDRegBank : RegisterBank<"vpIDBank", [vpID]>;
 def TYPERegBank : RegisterBank<"TYPEBank", [TYPE]>;
diff --git a/llvm/lib/Target/SPIRV/SPIRVRegisterInfo.td b/llvm/lib/Target/SPIRV/SPIRVRegisterInfo.td
index d0b64b6895d0..6d2bfb91a97f 100644
--- a/llvm/lib/Target/SPIRV/SPIRVRegisterInfo.td
+++ b/llvm/lib/Target/SPIRV/SPIRVRegisterInfo.td
@@ -12,6 +12,17 @@
 
 let Namespace = "SPIRV" in {
   def p0 : PtrValueType <i32, 0>;
+
+  class P0Vec<ValueType scalar>
+      : PtrValueType <scalar, 0> {
+    let nElem = 2;
+    let ElementType = p0;
+    let isInteger = false;
+    let isFP = false;
+    let isVector = true;
+  }
+
+  def v2p0 : P0Vec<i32>;
   // All registers are for 32-bit identifiers, so have a single dummy register
 
   // Class for registers that are the result of OpTypeXXX instructions
@@ -21,14 +32,16 @@ let Namespace = "SPIRV" in {
   // Class for every other non-type ID
   def ID0 : Register<"ID0">;
   def ID : RegisterClass<"SPIRV", [i32], 32, (add ID0)>;
-  def fID0 : Register<"FID0">;
+  def fID0 : Register<"fID0">;
   def fID : RegisterClass<"SPIRV", [f32], 32, (add fID0)>;
   def pID0 : Register<"pID0">;
   def pID : RegisterClass<"SPIRV", [p0], 32, (add pID0)>;
-  def vID0 : Register<"pID0">;
+  def vID0 : Register<"vID0">;
   def vID : RegisterClass<"SPIRV", [v2i32], 32, (add vID0)>;
-  def vfID0 : Register<"pID0">;
+  def vfID0 : Register<"vfID0">;
   def vfID : RegisterClass<"SPIRV", [v2f32], 32, (add vfID0)>;
+  def vpID0 : Register<"vpID0">;
+  def vpID : RegisterClass<"SPIRV", [v2p0], 32, (add vpID0)>;
 
   def ANYID : RegisterClass<"SPIRV", [i32, f32, p0, v2i32, v2f32], 32, (add ID, fID, pID, vID, vfID)>;
 
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
index 299a4341193b..2e44c208ed8e 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp
@@ -251,7 +251,8 @@ bool isSpvIntrinsic(const MachineInstr &MI, Intrinsic::ID IntrinsicID) {
 }
 
 Type *getMDOperandAsType(const MDNode *N, unsigned I) {
-  return cast<ValueAsMetadata>(N->getOperand(I))->getType();
+  Type *ElementTy = cast<ValueAsMetadata>(N->getOperand(I))->getType();
+  return toTypedPointer(ElementTy, N->getContext());
 }
 
 // The set of names is borrowed from the SPIR-V translator.
diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.h b/llvm/lib/Target/SPIRV/SPIRVUtils.h
index c2c3475e1a93..cd1a2af09147 100644
--- a/llvm/lib/Target/SPIRV/SPIRVUtils.h
+++ b/llvm/lib/Target/SPIRV/SPIRVUtils.h
@@ -149,5 +149,12 @@ inline Type *reconstructFunctionType(Function *F) {
   return FunctionType::get(F->getReturnType(), ArgTys, F->isVarArg());
 }
 
+inline Type *toTypedPointer(Type *Ty, LLVMContext &Ctx) {
+  return isUntypedPointerTy(Ty)
+             ? TypedPointerType::get(IntegerType::getInt8Ty(Ctx),
+                                     getPointerAddressSpace(Ty))
+             : Ty;
+}
+
 } // namespace llvm
 #endif // LLVM_LIB_TARGET_SPIRV_SPIRVUTILS_H
diff --git a/llvm/test/CodeGen/SPIRV/const-composite.ll b/llvm/test/CodeGen/SPIRV/const-composite.ll
new file mode 100644
index 000000000000..4e304bb95167
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/const-composite.ll
@@ -0,0 +1,26 @@
+; This test is to ensure that OpConstantComposite reuses a constant when it's
+; already created and available in the same machine function. In this test case
+; it's `1` that is passed implicitly as a part of the `foo` function argument
+; and also takes part in a composite constant creation.
+
+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s --check-prefix=CHECK-SPIRV
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv64-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-SPIRV: %[[#type_int32:]] = OpTypeInt 32 0
+; CHECK-SPIRV: %[[#const1:]] = OpConstant %[[#type_int32]] 1
+; CHECK-SPIRV: OpTypeArray %[[#]] %[[#const1:]]
+; CHECK-SPIRV: %[[#const0:]] = OpConstant %[[#type_int32]] 0
+; CHECK-SPIRV: OpConstantComposite %[[#]] %[[#const0]] %[[#const1]]
+
+%struct = type { [1 x i64] }
+
+define spir_kernel void @foo(ptr noundef byval(%struct) %arg) {
+entry:
+  call spir_func void @bar(<2 x i32> noundef <i32 0, i32 1>)
+  ret void
+}
+
+define spir_func void @bar(<2 x i32> noundef) {
+entry:
+  ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/instructions/ret-type.ll b/llvm/test/CodeGen/SPIRV/instructions/ret-type.ll
new file mode 100644
index 000000000000..bf71eb5628e2
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/instructions/ret-type.ll
@@ -0,0 +1,82 @@
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown --translator-compatibility-mode %s -o - -filetype=obj | spirv-val %}
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: OpName %[[Test1:.*]] "test1"
+; CHECK-DAG: OpName %[[Foo:.*]] "foo"
+; CHECK-DAG: OpName %[[Bar:.*]] "bar"
+; CHECK-DAG: OpName %[[Test2:.*]] "test2"
+
+; CHECK-DAG: %[[Long:.*]] = OpTypeInt 64 0
+; CHECK-DAG: %[[Array:.*]] = OpTypeArray %[[Long]] %[[#]]
+; CHECK-DAG: %[[Struct1:.*]] = OpTypeStruct %[[Array]]
+; CHECK-DAG: %[[Struct2:.*]] = OpTypeStruct %[[Struct1]]
+; CHECK-DAG: %[[StructPtr:.*]] = OpTypePointer Function %[[Struct2]]
+; CHECK-DAG: %[[Bool:.*]] = OpTypeBool
+; CHECK-DAG: %[[FooType:.*]] = OpTypeFunction %[[StructPtr:.*]] %[[StructPtr]] %[[StructPtr]] %[[Bool]]
+; CHECK-DAG: %[[Char:.*]] = OpTypeInt 8 0
+; CHECK-DAG: %[[CharPtr:.*]] = OpTypePointer Function %[[Char]]
+
+; CHECK: %[[Test1]] = OpFunction
+; CHECK: OpFunctionCall %[[StructPtr:.*]] %[[Foo]]
+; CHECK: OpFunctionCall %[[StructPtr:.*]] %[[Bar]]
+; CHECK: OpFunctionEnd
+
+; CHECK: %[[Foo]] = OpFunction %[[StructPtr:.*]] None %[[FooType]]
+; CHECK: %[[Arg1:.*]] = OpFunctionParameter %[[StructPtr]]
+; CHECK: %[[Arg2:.*]] = OpFunctionParameter
+; CHECK: %[[Sw:.*]] = OpFunctionParameter
+; CHECK: %[[Res:.*]] = OpInBoundsPtrAccessChain %[[StructPtr]] %[[Arg1]] %[[#]]
+; CHECK: OpReturnValue %[[Res]]
+; CHECK: OpReturnValue %[[Arg2]]
+
+; CHECK: %[[Bar]] = OpFunction %[[StructPtr:.*]] None %[[#]]
+; CHECK: %[[BarArg:.*]] = OpFunctionParameter
+; CHECK: %[[BarRes:.*]] = OpInBoundsPtrAccessChain %[[CharPtr]] %[[BarArg]] %[[#]]
+; CHECK: %[[BarResCasted:.*]] = OpBitcast %[[StructPtr]] %[[BarRes]]
+; CHECK: %[[BarResStruct:.*]] = OpInBoundsPtrAccessChain %[[StructPtr]] %[[#]] %[[#]]
+; CHECK: OpReturnValue %[[BarResStruct]]
+; CHECK: OpReturnValue %[[BarResCasted]]
+
+; CHECK: %[[Test2]] = OpFunction
+; CHECK: OpFunctionCall %[[StructPtr:.*]] %[[Foo]]
+; CHECK: OpFunctionCall %[[StructPtr:.*]] %[[Bar]]
+; CHECK: OpFunctionEnd
+
+%struct = type { %array }
+%array = type { [1 x i64] }
+
+define spir_func void @test1(ptr %arg1, ptr %arg2, i1 %sw) {
+entry:
+  %r1 = call ptr @foo(ptr %arg1, ptr %arg2, i1 %sw)
+  %r2 = call ptr @bar(ptr %arg1, i1 %sw)
+  ret void
+}
+
+define spir_func ptr @foo(ptr %arg1, ptr %arg2, i1 %sw) {
+entry:
+  br i1 %sw, label %exit, label %sw1
+sw1:
+  %result = getelementptr inbounds %struct, ptr %arg1, i64 100
+  ret ptr %result
+exit:
+  ret ptr %arg2
+}
+
+define spir_func ptr @bar(ptr %arg1, i1 %sw) {
+entry:
+  %charptr = getelementptr inbounds i8, ptr %arg1, i64 0
+  br i1 %sw, label %exit, label %sw1
+sw1:
+  %result = getelementptr inbounds %struct, ptr %arg1, i64 100
+  ret ptr %result
+exit:
+  ret ptr %charptr
+}
+
+define spir_func void @test2(ptr %arg1, ptr %arg2, i1 %sw) {
+entry:
+  %r1 = call ptr @foo(ptr %arg1, ptr %arg2, i1 %sw)
+  %r2 = call ptr @bar(ptr %arg1, i1 %sw)
+  ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/instructions/select-phi.ll b/llvm/test/CodeGen/SPIRV/instructions/select-phi.ll
new file mode 100644
index 000000000000..afc75c616f02
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/instructions/select-phi.ll
@@ -0,0 +1,58 @@
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown --translator-compatibility-mode %s -o - -filetype=obj | spirv-val %}
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+
+; CHECK-DAG: %[[Char:.*]] = OpTypeInt 8 0
+; CHECK-DAG: %[[Long:.*]] = OpTypeInt 32 0
+; CHECK-DAG: %[[Array:.*]] = OpTypeArray %[[Long]] %[[#]]
+; CHECK-DAG: %[[Struct:.*]] = OpTypeStruct %[[Array]]
+; CHECK-DAG: %[[StructPtr:.*]] = OpTypePointer Function %[[Struct]]
+; CHECK-DAG: %[[CharPtr:.*]] = OpTypePointer Function %[[Char]]
+
+; CHECK: %[[Branch1:.*]] = OpLabel
+; CHECK: %[[Res1:.*]] = OpVariable %[[StructPtr]] Function
+; CHECK: OpBranchConditional %[[#]] %[[#]] %[[Branch2:.*]]
+; CHECK: %[[Res2:.*]] = OpInBoundsPtrAccessChain %[[CharPtr]] %[[#]] %[[#]]
+; CHECK: %[[Res2Casted:.*]] = OpBitcast %[[StructPtr]] %[[Res2]]
+; CHECK: OpBranchConditional %[[#]] %[[#]] %[[BranchSelect:.*]]
+; CHECK: %[[SelectRes:.*]] = OpSelect %[[CharPtr]] %[[#]] %[[#]] %[[#]]
+; CHECK: %[[SelectResCasted:.*]] = OpBitcast %[[StructPtr]] %[[SelectRes]]
+; CHECK: OpLabel
+; CHECK: OpPhi %[[StructPtr]] %[[Res1]] %[[Branch1]] %[[Res2Casted]] %[[Branch2]] %[[SelectResCasted]] %[[BranchSelect]]
+
+%struct = type { %array }
+%array = type { [1 x i64] }
+%array3 = type { [3 x i32] }
+
+define spir_kernel void @foo(ptr addrspace(1) noundef align 1 %arg1, ptr noundef byval(%struct) align 8 %arg2, i1 noundef zeroext %expected) {
+entry:
+  %agg = alloca %array3, align 8
+  %r0 = load i64, ptr %arg2, align 8
+  %add.ptr = getelementptr inbounds i8, ptr %agg, i64 12
+  %r1 = load i32, ptr %agg, align 4
+  %tobool0 = icmp slt i32 %r1, 0
+  br i1 %tobool0, label %exit, label %sw1
+
+sw1:                            ; preds = %entry
+  %incdec1 = getelementptr inbounds i8, ptr %agg, i64 4
+  %r2 = load i32, ptr %incdec1, align 4
+  %tobool1 = icmp slt i32 %r2, 0
+  br i1 %tobool1, label %exit, label %sw2
+
+sw2:                            ; preds = %sw1
+  %incdec2 = getelementptr inbounds i8, ptr %agg, i64 8
+  %r3 = load i32, ptr %incdec2, align 4
+  %tobool2 = icmp slt i32 %r3, 0
+  %spec.select = select i1 %tobool2, ptr %incdec2, ptr %add.ptr
+  br label %exit
+
+exit: ; preds = %sw2, %sw1, %entry
+  %retval.0 = phi ptr [ %agg, %entry ], [ %incdec1, %sw1 ], [ %spec.select, %sw2 ]
+  %add.ptr.i = getelementptr inbounds i8, ptr addrspace(1) %arg1, i64 %r0
+  %r4 = icmp eq ptr %retval.0, %add.ptr
+  %cmp = xor i1 %r4, %expected
+  %frombool6.i = zext i1 %cmp to i8
+  store i8 %frombool6.i, ptr addrspace(1) %add.ptr.i, align 1
+  %r5 = icmp eq ptr %add.ptr, %retval.0
+  ret void
+}
diff --git a/llvm/test/CodeGen/SPIRV/instructions/select.ll b/llvm/test/CodeGen/SPIRV/instructions/select.ll
index f54ef21f2085..c4176b17abb4 100644
--- a/llvm/test/CodeGen/SPIRV/instructions/select.ll
+++ b/llvm/test/CodeGen/SPIRV/instructions/select.ll
@@ -1,6 +1,8 @@
 ; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
 
 ; CHECK-DAG:  OpName [[SCALARi32:%.+]] "select_i32"
+; CHECK-DAG:  OpName [[SCALARPTR:%.+]] "select_ptr"
 ; CHECK-DAG:  OpName [[VEC2i32:%.+]] "select_i32v2"
 ; CHECK-DAG:  OpName [[VEC2i32v2:%.+]] "select_v2i32v2"
 
@@ -17,6 +19,19 @@ define i32 @select_i32(i1 %c, i32 %t, i32 %f) {
   ret i32 %r
 }
 
+; CHECK:      [[SCALARPTR]] = OpFunction
+; CHECK-NEXT: [[C:%.+]] = OpFunctionParameter
+; CHECK-NEXT: [[T:%.+]] = OpFunctionParameter
+; CHECK-NEXT: [[F:%.+]] = OpFunctionParameter
+; CHECK:      OpLabel
+; CHECK:      [[R:%.+]] = OpSelect {{%.+}} [[C]] [[T]] [[F]]
+; CHECK:      OpReturnValue [[R]]
+; CHECK-NEXT: OpFunctionEnd
+define ptr @select_ptr(i1 %c, ptr %t, ptr %f) {
+  %r = select i1 %c, ptr %t, ptr %f
+  ret ptr %r
+}
+
 ; CHECK:      [[VEC2i32]] = OpFunction
 ; CHECK-NEXT: [[C:%.+]] = OpFunctionParameter
 ; CHECK-NEXT: [[T:%.+]] = OpFunctionParameter
-- 
GitLab


From f7bf11cd8db29a178463f2a12b9f1589ecab64e2 Mon Sep 17 00:00:00 2001
From: wanglei <wanglei@loongson.cn>
Date: Mon, 15 Apr 2024 14:38:28 +0800
Subject: [PATCH 206/464] [LoongArch] Pre-commit test for #88694. NFC

---
 llvm/test/CodeGen/LoongArch/gep-imm.ll | 52 ++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)
 create mode 100644 llvm/test/CodeGen/LoongArch/gep-imm.ll

diff --git a/llvm/test/CodeGen/LoongArch/gep-imm.ll b/llvm/test/CodeGen/LoongArch/gep-imm.ll
new file mode 100644
index 000000000000..52d25f9d6389
--- /dev/null
+++ b/llvm/test/CodeGen/LoongArch/gep-imm.ll
@@ -0,0 +1,52 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s
+
+define void @test(ptr %sp, ptr %t, i32 %n) {
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    move $a3, $zero
+; CHECK-NEXT:    ld.d $a4, $a0, 0
+; CHECK-NEXT:    lu12i.w $a0, 1
+; CHECK-NEXT:    ori $a5, $a0, 3904
+; CHECK-NEXT:    add.d $a0, $a1, $a5
+; CHECK-NEXT:    add.d $a1, $a4, $a5
+; CHECK-NEXT:    addi.w $a2, $a2, 0
+; CHECK-NEXT:    addi.w $a4, $a3, 0
+; CHECK-NEXT:    bge $a4, $a2, .LBB0_2
+; CHECK-NEXT:    .p2align 4, , 16
+; CHECK-NEXT:  .LBB0_1: # %while_body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    addi.d $a4, $a3, 1
+; CHECK-NEXT:    st.w $a4, $a1, 0
+; CHECK-NEXT:    st.w $a3, $a1, 4
+; CHECK-NEXT:    st.w $a4, $a0, 0
+; CHECK-NEXT:    st.w $a3, $a0, 4
+; CHECK-NEXT:    move $a3, $a4
+; CHECK-NEXT:    addi.w $a4, $a3, 0
+; CHECK-NEXT:    blt $a4, $a2, .LBB0_1
+; CHECK-NEXT:  .LBB0_2: # %while_end
+; CHECK-NEXT:    ret
+entry:
+  %s = load ptr, ptr %sp
+  br label %while_cond
+
+while_cond:
+  %phi = phi i32 [ 0, %entry ], [ %i, %while_body ]
+  %gep0 = getelementptr [65536 x i32], ptr %s, i64 0, i64 2000
+  %gep1 = getelementptr [65536 x i32], ptr %s, i64 0, i64 2001
+  %gep2 = getelementptr [65536 x i32], ptr %t, i64 0, i64 2000
+  %gep3 = getelementptr [65536 x i32], ptr %t, i64 0, i64 2001
+  %cmp = icmp slt i32 %phi, %n
+  br i1 %cmp, label %while_body, label %while_end
+
+while_body:
+  %i = add i32 %phi, 1
+  store i32 %i, ptr %gep0
+  store i32 %phi, ptr %gep1
+  store i32 %i, ptr %gep2
+  store i32 %phi, ptr %gep3
+  br label %while_cond
+
+while_end:
+  ret void
+}
-- 
GitLab


From 06714e1bfe8999c0acb5c9b94b60690f7b9d2904 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <ram.ramachandra@arm.com>
Date: Mon, 15 Apr 2024 09:36:40 +0100
Subject: [PATCH 207/464] llvm-readobj/ARM: activate big-endian attribute
 printing (#87806)

Activate attribute printing for big-endian encoding for ARM. The
supporting code already exists, and the patch is trivial.

Fixes #62400.
---
 .../ELF/ARM/attribute-big-endian.test            | 16 +++++++++++-----
 llvm/tools/llvm-readobj/ELFDumper.cpp            | 10 +++-------
 2 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/llvm/test/tools/llvm-readobj/ELF/ARM/attribute-big-endian.test b/llvm/test/tools/llvm-readobj/ELF/ARM/attribute-big-endian.test
index 7d20b310f9d9..3b94c7994cf3 100644
--- a/llvm/test/tools/llvm-readobj/ELF/ARM/attribute-big-endian.test
+++ b/llvm/test/tools/llvm-readobj/ELF/ARM/attribute-big-endian.test
@@ -1,17 +1,23 @@
-## We only implement attribute section printing for little-endian encoding.
-
 # RUN: yaml2obj %s -o %t.o
-# RUN: llvm-readobj -A %t.o 2>&1 | FileCheck %s -DFILE=%t.o
+# RUN: llvm-readelf -A %t.o 2>&1 | FileCheck %s
 
-# CHECK: warning: '[[FILE]]': attribute printing not implemented for big-endian ARM objects
+# CHECK:      BuildAttributes {
+# CHECK-NEXT:   FormatVersion: 0x41
+# CHECK-NEXT:   Section 1 {
+# CHECK-NEXT:     SectionLength: 22
+# CHECK-NEXT:     Vendor: armabi
+# CHECK-NEXT:   }
+# CHECK-NEXT: }
 
 --- !ELF
 FileHeader:
   Class:   ELFCLASS32
-## Test big-endian encoding.
   Data:    ELFDATA2MSB
   Type:    ET_REL
   Machine: EM_ARM
 Sections:
   - Name: .ARM.attributes
     Type: SHT_ARM_ATTRIBUTES
+    ContentArray: [ 0x41, 0x00, 0x00, 0x00, 0x16, 0x61, 0x72, 0x6D, 0x61, 0x62,
+    0x69, 0x00, 0x01, 0x0b, 0x00, 0x00, 0x00, 0x04, 0x01, 0x06, 0x01, 0x08,
+    0x01 ]
diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
index d3534820c117..1108672003fc 100644
--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
+++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
@@ -2833,13 +2833,9 @@ template <class ELFT> void ELFDumper<ELFT>::printArchSpecificInfo() {
                     llvm::endianness::little);
     break;
   case EM_ARM:
-    if (Obj.isLE())
-      printAttributes(ELF::SHT_ARM_ATTRIBUTES,
-                      std::make_unique<ARMAttributeParser>(&W),
-                      llvm::endianness::little);
-    else
-      reportUniqueWarning("attribute printing not implemented for big-endian "
-                          "ARM objects");
+    printAttributes(
+        ELF::SHT_ARM_ATTRIBUTES, std::make_unique<ARMAttributeParser>(&W),
+        Obj.isLE() ? llvm::endianness::little : llvm::endianness::big);
     break;
   case EM_RISCV:
     if (Obj.isLE())
-- 
GitLab


From 5fe146672d2b1c9f257a6ee045e0bc13fed1e504 Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333@gmail.com>
Date: Mon, 15 Apr 2024 16:40:16 +0800
Subject: [PATCH 208/464] [InstCombine] Simplify switch with selects (#84143)

An example from https://github.com/image-rs/image:
```
define void @test_ult_rhsc(i8 %x) {
  %val = add nsw i8 %x, -2
  %cmp = icmp ult i8 %val, 11
  %cond = select i1 %cmp, i8 %val, i8 6
  switch i8 %cond, label %bb1 [
  i8 0, label %bb2
  i8 10, label %bb3
  ]

bb1:
  call void @func1()
  unreachable
bb2:
  call void @func2()
  unreachable
bb3:
  call void @func3()
  unreachable
}
```

When `%cmp` evaluates to false, we can prove that the range of `%val` is
[11, umax]. Thus we can safely replace `%cond` with `%val` since both
`switch 6` and `switch %val` go to the default dest `%bb1`.

Alive2: https://alive2.llvm.org/ce/z/uSTj6w
Godbolt: https://godbolt.org/z/MGrG84bzr

This patch will benefit many rust applications and some C/C++
applications (e.g., cvc5).
---
 .../InstCombine/InstructionCombining.cpp      |  42 +++++
 .../Transforms/InstCombine/switch-select.ll   | 159 ++++++++++++++++++
 2 files changed, 201 insertions(+)
 create mode 100644 llvm/test/Transforms/InstCombine/switch-select.ll

diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 4cfafa7ac800..4c00f2a0ea17 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -3572,6 +3572,38 @@ Instruction *InstCombinerImpl::visitBranchInst(BranchInst &BI) {
   return nullptr;
 }
 
+// Replaces (switch (select cond, X, C)/(select cond, C, X)) with (switch X) if
+// we can prove that both (switch C) and (switch X) go to the default when cond
+// is false/true.
+static Value *simplifySwitchOnSelectUsingRanges(SwitchInst &SI,
+                                                SelectInst *Select,
+                                                bool IsTrueArm) {
+  unsigned CstOpIdx = IsTrueArm ? 1 : 2;
+  auto *C = dyn_cast<ConstantInt>(Select->getOperand(CstOpIdx));
+  if (!C)
+    return nullptr;
+
+  BasicBlock *CstBB = SI.findCaseValue(C)->getCaseSuccessor();
+  if (CstBB != SI.getDefaultDest())
+    return nullptr;
+  Value *X = Select->getOperand(3 - CstOpIdx);
+  ICmpInst::Predicate Pred;
+  const APInt *RHSC;
+  if (!match(Select->getCondition(),
+             m_ICmp(Pred, m_Specific(X), m_APInt(RHSC))))
+    return nullptr;
+  if (IsTrueArm)
+    Pred = ICmpInst::getInversePredicate(Pred);
+
+  // See whether we can replace the select with X
+  ConstantRange CR = ConstantRange::makeExactICmpRegion(Pred, *RHSC);
+  for (auto Case : SI.cases())
+    if (!CR.contains(Case.getCaseValue()->getValue()))
+      return nullptr;
+
+  return X;
+}
+
 Instruction *InstCombinerImpl::visitSwitchInst(SwitchInst &SI) {
   Value *Cond = SI.getCondition();
   Value *Op0;
@@ -3645,6 +3677,16 @@ Instruction *InstCombinerImpl::visitSwitchInst(SwitchInst &SI) {
     }
   }
 
+  // Fold switch(select cond, X, Y) into switch(X/Y) if possible
+  if (auto *Select = dyn_cast<SelectInst>(Cond)) {
+    if (Value *V =
+            simplifySwitchOnSelectUsingRanges(SI, Select, /*IsTrueArm=*/true))
+      return replaceOperand(SI, 0, V);
+    if (Value *V =
+            simplifySwitchOnSelectUsingRanges(SI, Select, /*IsTrueArm=*/false))
+      return replaceOperand(SI, 0, V);
+  }
+
   KnownBits Known = computeKnownBits(Cond, 0, &SI);
   unsigned LeadingKnownZeros = Known.countMinLeadingZeros();
   unsigned LeadingKnownOnes = Known.countMinLeadingOnes();
diff --git a/llvm/test/Transforms/InstCombine/switch-select.ll b/llvm/test/Transforms/InstCombine/switch-select.ll
new file mode 100644
index 000000000000..60757c5d2252
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/switch-select.ll
@@ -0,0 +1,159 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+define void @test_ult_rhsc(i8 %x) {
+; CHECK-LABEL: define void @test_ult_rhsc(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    switch i8 [[X]], label [[BB1:%.*]] [
+; CHECK-NEXT:      i8 2, label [[BB2:%.*]]
+; CHECK-NEXT:      i8 12, label [[BB3:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       bb1:
+; CHECK-NEXT:    call void @func1()
+; CHECK-NEXT:    unreachable
+; CHECK:       bb2:
+; CHECK-NEXT:    call void @func2()
+; CHECK-NEXT:    unreachable
+; CHECK:       bb3:
+; CHECK-NEXT:    call void @func3()
+; CHECK-NEXT:    unreachable
+;
+  %val = add nsw i8 %x, -2
+  %cmp = icmp ult i8 %val, 11
+  %cond = select i1 %cmp, i8 %val, i8 6
+  switch i8 %cond, label %bb1 [
+  i8 0, label %bb2
+  i8 10, label %bb3
+  ]
+
+bb1:
+  call void @func1()
+  unreachable
+bb2:
+  call void @func2()
+  unreachable
+bb3:
+  call void @func3()
+  unreachable
+}
+
+define void @test_eq_lhsc(i8 %x) {
+; CHECK-LABEL: define void @test_eq_lhsc(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    switch i8 [[X]], label [[BB1:%.*]] [
+; CHECK-NEXT:      i8 0, label [[BB2:%.*]]
+; CHECK-NEXT:      i8 10, label [[BB3:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       bb1:
+; CHECK-NEXT:    call void @func1()
+; CHECK-NEXT:    unreachable
+; CHECK:       bb2:
+; CHECK-NEXT:    call void @func2()
+; CHECK-NEXT:    unreachable
+; CHECK:       bb3:
+; CHECK-NEXT:    call void @func3()
+; CHECK-NEXT:    unreachable
+;
+  %cmp = icmp eq i8 %x, 4
+  %cond = select i1 %cmp, i8 6, i8 %x
+  switch i8 %cond, label %bb1 [
+  i8 0, label %bb2
+  i8 10, label %bb3
+  ]
+
+bb1:
+  call void @func1()
+  unreachable
+bb2:
+  call void @func2()
+  unreachable
+bb3:
+  call void @func3()
+  unreachable
+}
+
+define void @test_ult_rhsc_invalid_cond(i8 %x, i8 %y) {
+; CHECK-LABEL: define void @test_ult_rhsc_invalid_cond(
+; CHECK-SAME: i8 [[X:%.*]], i8 [[Y:%.*]]) {
+; CHECK-NEXT:    [[VAL:%.*]] = add nsw i8 [[X]], -2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[Y]], 11
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i8 [[VAL]], i8 6
+; CHECK-NEXT:    switch i8 [[COND]], label [[BB1:%.*]] [
+; CHECK-NEXT:      i8 0, label [[BB2:%.*]]
+; CHECK-NEXT:      i8 10, label [[BB3:%.*]]
+; CHECK-NEXT:      i8 13, label [[BB3]]
+; CHECK-NEXT:    ]
+; CHECK:       bb1:
+; CHECK-NEXT:    call void @func1()
+; CHECK-NEXT:    unreachable
+; CHECK:       bb2:
+; CHECK-NEXT:    call void @func2()
+; CHECK-NEXT:    unreachable
+; CHECK:       bb3:
+; CHECK-NEXT:    call void @func3()
+; CHECK-NEXT:    unreachable
+;
+  %val = add nsw i8 %x, -2
+  %cmp = icmp ult i8 %y, 11
+  %cond = select i1 %cmp, i8 %val, i8 6
+  switch i8 %cond, label %bb1 [
+  i8 0, label %bb2
+  i8 10, label %bb3
+  i8 13, label %bb3
+  ]
+
+bb1:
+  call void @func1()
+  unreachable
+bb2:
+  call void @func2()
+  unreachable
+bb3:
+  call void @func3()
+  unreachable
+}
+
+define void @test_ult_rhsc_fail(i8 %x) {
+; CHECK-LABEL: define void @test_ult_rhsc_fail(
+; CHECK-SAME: i8 [[X:%.*]]) {
+; CHECK-NEXT:    [[VAL:%.*]] = add nsw i8 [[X]], -2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[VAL]], 11
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i8 [[VAL]], i8 6
+; CHECK-NEXT:    switch i8 [[COND]], label [[BB1:%.*]] [
+; CHECK-NEXT:      i8 0, label [[BB2:%.*]]
+; CHECK-NEXT:      i8 10, label [[BB3:%.*]]
+; CHECK-NEXT:      i8 13, label [[BB3]]
+; CHECK-NEXT:    ]
+; CHECK:       bb1:
+; CHECK-NEXT:    call void @func1()
+; CHECK-NEXT:    unreachable
+; CHECK:       bb2:
+; CHECK-NEXT:    call void @func2()
+; CHECK-NEXT:    unreachable
+; CHECK:       bb3:
+; CHECK-NEXT:    call void @func3()
+; CHECK-NEXT:    unreachable
+;
+  %val = add nsw i8 %x, -2
+  %cmp = icmp ult i8 %val, 11
+  %cond = select i1 %cmp, i8 %val, i8 6
+  switch i8 %cond, label %bb1 [
+  i8 0, label %bb2
+  i8 10, label %bb3
+  i8 13, label %bb3
+  ]
+
+bb1:
+  call void @func1()
+  unreachable
+bb2:
+  call void @func2()
+  unreachable
+bb3:
+  call void @func3()
+  unreachable
+}
+
+declare void @func1()
+declare void @func2()
+declare void @func3()
-- 
GitLab


From 28d28d58bedf413671859b959b7ba06bccc5066d Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Mon, 15 Apr 2024 09:41:08 +0100
Subject: [PATCH 209/464] [AArch64][GlobalISel] Extend scalar lrint
 legalization. (#88360)

This extends the legalization of lrint, adding libcall support for
fp128. The old vector legal types were removed as they were not being
properly handled (vector lrint is a fairly new concept as far as I
understand). They can be re-added properly in a followup.
---
 .../CodeGen/GlobalISel/LegalizerHelper.cpp    | 22 +++++++++++++++++++
 .../AArch64/GISel/AArch64LegalizerInfo.cpp    | 20 +++--------------
 .../AArch64/GISel/AArch64RegisterBankInfo.cpp |  1 +
 .../GlobalISel/legalizer-info-validation.mir  |  4 ++--
 llvm/test/CodeGen/AArch64/lrint-conv-fp16.ll  |  2 ++
 llvm/test/CodeGen/AArch64/lrint-conv.ll       |  5 +----
 llvm/test/CodeGen/AArch64/vector-lrint.ll     |  3 +--
 7 files changed, 32 insertions(+), 25 deletions(-)

diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 95c6a359e52e..20a7fa253519 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -472,6 +472,8 @@ static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
     RTLIBCASE(NEARBYINT_F);
   case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
     RTLIBCASE(ROUNDEVEN_F);
+  case TargetOpcode::G_INTRINSIC_LRINT:
+    RTLIBCASE(LRINT_F);
   }
   llvm_unreachable("Unknown libcall function");
 }
@@ -1059,6 +1061,25 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
       return Status;
     break;
   }
+  case TargetOpcode::G_INTRINSIC_LRINT: {
+    LLT LLTy = MRI.getType(MI.getOperand(1).getReg());
+    unsigned Size = LLTy.getSizeInBits();
+    Type *HLTy = getFloatTypeForLLT(Ctx, LLTy);
+    Type *ITy = IntegerType::get(
+        Ctx, MRI.getType(MI.getOperand(0).getReg()).getSizeInBits());
+    if (!HLTy || (Size != 32 && Size != 64 && Size != 80 && Size != 128)) {
+      LLVM_DEBUG(dbgs() << "No libcall available for type " << LLTy << ".\n");
+      return UnableToLegalize;
+    }
+    auto Libcall = getRTLibDesc(MI.getOpcode(), Size);
+    LegalizeResult Status =
+        createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), ITy, 0},
+                      {{MI.getOperand(1).getReg(), HLTy, 0}}, LocObserver, &MI);
+    if (Status != Legalized)
+      return Status;
+    MI.eraseFromParent();
+    return Legalized;
+  }
   case TargetOpcode::G_FPOWI: {
     LLT LLTy = MRI.getType(MI.getOperand(0).getReg());
     unsigned Size = LLTy.getSizeInBits();
@@ -2639,6 +2660,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
 
   case TargetOpcode::G_FPTOSI:
   case TargetOpcode::G_FPTOUI:
+  case TargetOpcode::G_INTRINSIC_LRINT:
   case TargetOpcode::G_IS_FPCLASS:
     Observer.changingInstr(MI);
 
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 96ded69905f7..661ea151d1a0 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -263,23 +263,9 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
       .scalarize(0);
 
   getActionDefinitionsBuilder(G_INTRINSIC_LRINT)
-      // If we don't have full FP16 support, then scalarize the elements of
-      // vectors containing fp16 types.
-      .fewerElementsIf(
-          [=, &ST](const LegalityQuery &Query) {
-            const auto &Ty = Query.Types[0];
-            return Ty.isVector() && Ty.getElementType() == s16 &&
-                   !ST.hasFullFP16();
-          },
-          [=](const LegalityQuery &Query) { return std::make_pair(0, s16); })
-      // If we don't have full FP16 support, then widen s16 to s32 if we
-      // encounter it.
-      .widenScalarIf(
-          [=, &ST](const LegalityQuery &Query) {
-            return Query.Types[0] == s16 && !ST.hasFullFP16();
-          },
-          [=](const LegalityQuery &Query) { return std::make_pair(0, s32); })
-      .legalFor({s16, s32, s64, v2s32, v4s32, v2s64, v2s16, v4s16, v8s16});
+      .legalFor({{s64, MinFPScalar}, {s64, s32}, {s64, s64}})
+      .libcallFor({{s64, s128}})
+      .minScalarOrElt(1, MinFPScalar);
 
   getActionDefinitionsBuilder(
       {G_FCOS, G_FSIN, G_FPOW, G_FLOG, G_FLOG2, G_FLOG10,
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp
index 58d000b6b2a9..d39de770eaf1 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp
@@ -829,6 +829,7 @@ AArch64RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
   }
   case TargetOpcode::G_FPTOSI:
   case TargetOpcode::G_FPTOUI:
+  case TargetOpcode::G_INTRINSIC_LRINT:
     if (MRI.getType(MI.getOperand(0).getReg()).isVector())
       break;
     OpRegBankIdx = {PMI_FirstGPR, PMI_FirstFPR};
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index 200e9d19d58d..0793f3983c8e 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -154,8 +154,8 @@
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_INTRINSIC_LRINT (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. the first uncovered type index: 2, OK
+# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_INTRINSIC_ROUNDEVEN (opcode {{[0-9]+}}): 1 type index, 0 imm indices
 # DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
 # DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
diff --git a/llvm/test/CodeGen/AArch64/lrint-conv-fp16.ll b/llvm/test/CodeGen/AArch64/lrint-conv-fp16.ll
index 7557ceac1212..6771a862c386 100644
--- a/llvm/test/CodeGen/AArch64/lrint-conv-fp16.ll
+++ b/llvm/test/CodeGen/AArch64/lrint-conv-fp16.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
 ; RUN: llc < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK-NOFP16
 ; RUN: llc < %s -mtriple=aarch64 -mattr=+fullfp16 | FileCheck %s --check-prefixes=CHECK-FP16
+; RUN: llc < %s -mtriple=aarch64 -global-isel | FileCheck %s --check-prefixes=CHECK-NOFP16
+; RUN: llc < %s -mtriple=aarch64 -mattr=+fullfp16 -global-isel | FileCheck %s --check-prefixes=CHECK-FP16
 
 define i16 @testmhhs(half %x) {
 ; CHECK-NOFP16-LABEL: testmhhs:
diff --git a/llvm/test/CodeGen/AArch64/lrint-conv.ll b/llvm/test/CodeGen/AArch64/lrint-conv.ll
index b61d6f04b400..60393b4f18d9 100644
--- a/llvm/test/CodeGen/AArch64/lrint-conv.ll
+++ b/llvm/test/CodeGen/AArch64/lrint-conv.ll
@@ -1,9 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
 ; RUN: llc < %s -mtriple=aarch64 | FileCheck %s
-; RUN: llc < %s -mtriple=aarch64 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
-
-; CHECK-GI:       warning: Instruction selection used fallback path for testmswl
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for testmsll
+; RUN: llc < %s -mtriple=aarch64 -global-isel | FileCheck %s
 
 define i32 @testmsws(float %x) {
 ; CHECK-LABEL: testmsws:
diff --git a/llvm/test/CodeGen/AArch64/vector-lrint.ll b/llvm/test/CodeGen/AArch64/vector-lrint.ll
index b7fcd11ba8d1..db85b2342821 100644
--- a/llvm/test/CodeGen/AArch64/vector-lrint.ll
+++ b/llvm/test/CodeGen/AArch64/vector-lrint.ll
@@ -2,8 +2,7 @@
 ; RUN: llc < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-SD
 ; RUN: llc < %s -mtriple=aarch64 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
 
-; CHECK-GI:       warning: Instruction selection used fallback path for lrint_v1f16
-; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v2f16
+; CHECK-GI:       warning: Instruction selection used fallback path for lrint_v2f16
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v4f16
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v8f16
 ; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for lrint_v16i64_v16f16
-- 
GitLab


From 36230f90eef455b6af867ca7ad4ed25d00ad5d3b Mon Sep 17 00:00:00 2001
From: fengfeng <153487255+fengfeng09@users.noreply.github.com>
Date: Mon, 15 Apr 2024 17:01:15 +0800
Subject: [PATCH 210/464] [SelectionDAG] Propagate Disjoint flag. (#88370)

Signed-off-by: feng.feng <feng.feng@iluvatar.com>
---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp     |  3 ++-
 llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp  |  3 ++-
 .../CodeGen/X86/propagate-disjoint-in-shl-or.ll   | 15 +++++++++++++++
 3 files changed, 19 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/CodeGen/X86/propagate-disjoint-in-shl-or.ll

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index d9c6c28d5dac..dbbe8d9193c2 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -9530,7 +9530,8 @@ static SDValue combineShiftOfShiftedLogic(SDNode *Shift, SelectionDAG &DAG) {
   SDValue ShiftSumC = DAG.getConstant(*C0Val + C1Val, DL, ShiftAmtVT);
   SDValue NewShift1 = DAG.getNode(ShiftOpcode, DL, VT, X, ShiftSumC);
   SDValue NewShift2 = DAG.getNode(ShiftOpcode, DL, VT, Y, C1);
-  return DAG.getNode(LogicOpcode, DL, VT, NewShift1, NewShift2);
+  return DAG.getNode(LogicOpcode, DL, VT, NewShift1, NewShift2,
+                     LogicOp->getFlags());
 }
 
 /// Handle transforms common to the three shifts, when the shift amount is a
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 4939976b8e28..c938b3996be3 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -544,7 +544,8 @@ bool TargetLowering::ShrinkDemandedConstant(SDValue Op,
     if (!C.isSubsetOf(DemandedBits)) {
       EVT VT = Op.getValueType();
       SDValue NewC = TLO.DAG.getConstant(DemandedBits & C, DL, VT);
-      SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC);
+      SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC,
+                                      Op->getFlags());
       return TLO.CombineTo(Op, NewOp);
     }
 
diff --git a/llvm/test/CodeGen/X86/propagate-disjoint-in-shl-or.ll b/llvm/test/CodeGen/X86/propagate-disjoint-in-shl-or.ll
new file mode 100644
index 000000000000..e38840f3e461
--- /dev/null
+++ b/llvm/test/CodeGen/X86/propagate-disjoint-in-shl-or.ll
@@ -0,0 +1,15 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=x86_64 %s -start-before=x86-isel -o - | FileCheck %s
+
+define i32 @add_shl_or_disjoint(i32 %x) {
+; CHECK-LABEL: add_shl_or_disjoint:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT:    leal 165036822(,%rdi,4), %eax
+; CHECK-NEXT:    retq
+  %or = or disjoint i32 %x, 4027584529
+  %shl = shl i32 %or, 2
+  %add = add i32 %shl, 1234567890
+  ret i32 %add
+}
+
-- 
GitLab


From 2347020e4ce62ee7c7378824dc6ad2b94917a638 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?D=C3=A1vid=20Ferenc=20Szab=C3=B3?=
 <30732159+dfszabo@users.noreply.github.com>
Date: Mon, 15 Apr 2024 11:01:55 +0200
Subject: [PATCH 211/464] [GlobalISel] Fix fewerElementsVectorPhi to insert
 after G_PHIs (#87927)

Currently the inserted mergelike instructions will be inserted at the
location of the G_PHI. Seems like the behaviour was correct before, but
the rework done in https://reviews.llvm.org/D114198 forgot to include
the part which makes sure the instructions will be inserted after all
the G_PHIs.
---
 llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp           | 4 ++++
 llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp | 6 +++---
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 20a7fa253519..156353296cfc 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -4287,6 +4287,10 @@ LegalizerHelper::fewerElementsVectorPhi(GenericMachineInstr &MI,
     }
   }
 
+  // Set the insert point after the existing PHIs
+  MachineBasicBlock &MBB = *MI.getParent();
+  MIRBuilder.setInsertPt(MBB, MBB.getFirstNonPHI());
+
   // Merge small outputs into MI's def.
   if (NumLeftovers) {
     mergeMixedSubvectors(MI.getReg(0), OutputRegs);
diff --git a/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp b/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp
index 33155d2c9a96..831d7e6292e3 100644
--- a/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp
@@ -1556,12 +1556,12 @@ TEST_F(AArch64GISelMITest, FewerElementsPhi) {
   CHECK: [[PHI0:%[0-9]+]]:_(<2 x s32>) = G_PHI [[INITVAL_E01]]:_(<2 x s32>), %bb.0, [[MIDVAL_E01]]:_(<2 x s32>), %bb.1
   CHECK: [[PHI1:%[0-9]+]]:_(<2 x s32>) = G_PHI [[INITVAL_E23]]:_(<2 x s32>), %bb.0, [[MIDVAL_E23]]:_(<2 x s32>), %bb.1
   CHECK: [[PHI2:%[0-9]+]]:_(s32) = G_PHI [[INITVAL_E4]]:_(s32), %bb.0, [[MIDVAL_E4]]:_(s32), %bb.1
-  CHECK: [[UNMERGE0:%[0-9]+]]:_(s32), [[UNMERGE1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[PHI0]]:_(<2 x s32>)
-  CHECK: [[UNMERGE2:%[0-9]+]]:_(s32), [[UNMERGE3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[PHI1]]:_(<2 x s32>)
-  CHECK: [[BV:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[UNMERGE0]]:_(s32), [[UNMERGE1]]:_(s32), [[UNMERGE2]]:_(s32), [[UNMERGE3]]:_(s32), [[PHI2]]:_(s32)
 
   CHECK: [[OTHER_PHI:%[0-9]+]]:_(s64) = G_PHI
 
+  CHECK: [[UNMERGE0:%[0-9]+]]:_(s32), [[UNMERGE1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[PHI0]]:_(<2 x s32>)
+  CHECK: [[UNMERGE2:%[0-9]+]]:_(s32), [[UNMERGE3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[PHI1]]:_(<2 x s32>)
+  CHECK: [[BV:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[UNMERGE0]]:_(s32), [[UNMERGE1]]:_(s32), [[UNMERGE2]]:_(s32), [[UNMERGE3]]:_(s32), [[PHI2]]:_(s32)
   CHECK: [[USE_OP:%[0-9]+]]:_(<5 x s32>) = G_AND [[BV]]:_, [[BV]]:_
   )";
 
-- 
GitLab


From aa2741449c3609b2ae244d8d3f3e14ad16de72e4 Mon Sep 17 00:00:00 2001
From: Chuanqi Xu <yedeng.yd@linux.alibaba.com>
Date: Mon, 15 Apr 2024 16:56:44 +0800
Subject: [PATCH 212/464] Revert "[C++20] [Modules] Don't import non-inline
 function bodies even if it is marked as always_inline"

This reverts commit 1ecbab56dcbb78268c8d19af34a50591f90b12a0.

See the discussion in https://github.com/llvm/llvm-project/issues/86893.

The original commit receives too many complaints. Let's try to
workaround the issue to give better user experiences.
---
 clang/lib/CodeGen/CodeGenModule.cpp                  | 3 ++-
 clang/test/CodeGenCXX/module-funcs-from-imports.cppm | 6 +++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index 73a9cb9d6e04..e44749672d58 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -3952,7 +3952,8 @@ bool CodeGenModule::shouldEmitFunction(GlobalDecl GD) {
   // behavior may break ABI compatibility of the current unit.
   if (const Module *M = F->getOwningModule();
       M && M->getTopLevelModule()->isNamedModule() &&
-      getContext().getCurrentNamedModule() != M->getTopLevelModule())
+      getContext().getCurrentNamedModule() != M->getTopLevelModule() &&
+      !F->hasAttr<AlwaysInlineAttr>())
     return false;
 
   if (F->hasAttr<NoInlineAttr>())
diff --git a/clang/test/CodeGenCXX/module-funcs-from-imports.cppm b/clang/test/CodeGenCXX/module-funcs-from-imports.cppm
index 8d04328eaf3f..33cdf437110a 100644
--- a/clang/test/CodeGenCXX/module-funcs-from-imports.cppm
+++ b/clang/test/CodeGenCXX/module-funcs-from-imports.cppm
@@ -53,11 +53,11 @@ int use() {
     return exported_func() + always_inline_func();
 }
 
-// Checks that none of the function in the importees
+// Checks that none of the function (except the always_inline_func) in the importees
 // are generated in the importer's code.
 // CHECK-O0: define{{.*}}_Z3usev(
 // CHECK-O0: declare{{.*}}_ZW1M13exported_funcv(
-// CHECK-O0: declare{{.*}}_ZW1M18always_inline_funcv(
+// CHECK-O0: define{{.*}}available_externally{{.*}}_ZW1M18always_inline_funcv(
 // CHECK-O0-NOT: func_in_gmf
 // CHECK-O0-NOT: func_in_gmf_not_called
 // CHECK-O0-NOT: non_exported_func
@@ -68,7 +68,7 @@ int use() {
 // O0 to keep consistent ABI.
 // CHECK-O1: define{{.*}}_Z3usev(
 // CHECK-O1: declare{{.*}}_ZW1M13exported_funcv(
-// CHECK-O1: declare{{.*}}_ZW1M18always_inline_funcv(
+// CHECK-O1: define{{.*}}available_externally{{.*}}_ZW1M18always_inline_funcv(
 // CHECK-O1-NOT: func_in_gmf
 // CHECK-O1-NOT: func_in_gmf_not_called
 // CHECK-O1-NOT: non_exported_func
-- 
GitLab


From 7740e933c41f07444ead12fd0417c5e53c6a4c1a Mon Sep 17 00:00:00 2001
From: abidh <haqadeer@amd.com>
Date: Mon, 15 Apr 2024 10:18:31 +0100
Subject: [PATCH 213/464] [flang] Rename AddDebugFoundation to AddDebugInfo
 (NFC) (#88526)

As discussed in PR 86939, this PR renames the pass and updates the
references. The actual changes for debug info will come in separate PRs.
---
 flang/include/flang/Optimizer/Transforms/Passes.h |  2 +-
 .../include/flang/Optimizer/Transforms/Passes.td  |  6 +++---
 flang/include/flang/Tools/CLOptions.inc           |  8 ++++----
 .../{AddDebugFoundation.cpp => AddDebugInfo.cpp}  | 15 +++++++--------
 flang/lib/Optimizer/Transforms/CMakeLists.txt     |  2 +-
 flang/test/Driver/mlir-debug-pass-pipeline.f90    |  4 ++--
 .../test/Transforms/debug-line-table-existing.fir |  2 +-
 .../test/Transforms/debug-line-table-inc-file.fir |  2 +-
 .../Transforms/debug-line-table-inc-same-file.fir |  2 +-
 flang/test/Transforms/debug-line-table.fir        |  2 +-
 10 files changed, 22 insertions(+), 23 deletions(-)
 rename flang/lib/Optimizer/Transforms/{AddDebugFoundation.cpp => AddDebugInfo.cpp} (90%)

diff --git a/flang/include/flang/Optimizer/Transforms/Passes.h b/flang/include/flang/Optimizer/Transforms/Passes.h
index a60d39c8df08..d8840d9e967b 100644
--- a/flang/include/flang/Optimizer/Transforms/Passes.h
+++ b/flang/include/flang/Optimizer/Transforms/Passes.h
@@ -67,7 +67,7 @@ std::unique_ptr<mlir::Pass> createMemoryAllocationPass();
 std::unique_ptr<mlir::Pass> createStackArraysPass();
 std::unique_ptr<mlir::Pass> createAliasTagsPass();
 std::unique_ptr<mlir::Pass> createSimplifyIntrinsicsPass();
-std::unique_ptr<mlir::Pass> createAddDebugFoundationPass();
+std::unique_ptr<mlir::Pass> createAddDebugInfoPass();
 std::unique_ptr<mlir::Pass> createLoopVersioningPass();
 
 std::unique_ptr<mlir::Pass>
diff --git a/flang/include/flang/Optimizer/Transforms/Passes.td b/flang/include/flang/Optimizer/Transforms/Passes.td
index 978911c21794..187796d77cf5 100644
--- a/flang/include/flang/Optimizer/Transforms/Passes.td
+++ b/flang/include/flang/Optimizer/Transforms/Passes.td
@@ -201,12 +201,12 @@ def MemRefDataFlowOpt : Pass<"fir-memref-dataflow-opt", "::mlir::func::FuncOp">
 
 // This needs to be a "mlir::ModuleOp" pass, because we are creating debug for
 // the module in this pass.
-def AddDebugFoundation : Pass<"add-debug-foundation", "mlir::ModuleOp"> {
-  let summary = "Add the foundation for debug info";
+def AddDebugInfo : Pass<"add-debug-info", "mlir::ModuleOp"> {
+  let summary = "Add the debug info";
   let description = [{
     Add the foundation for emitting debug info that can be understood by llvm.
   }];
-  let constructor = "::fir::createAddDebugFoundationPass()";
+  let constructor = "::fir::createAddDebugInfoPass()";
   let dependentDialects = [
     "fir::FIROpsDialect", "mlir::func::FuncDialect", "mlir::LLVM::LLVMDialect"
   ];
diff --git a/flang/include/flang/Tools/CLOptions.inc b/flang/include/flang/Tools/CLOptions.inc
index bc6df2c83069..268d00b5a605 100644
--- a/flang/include/flang/Tools/CLOptions.inc
+++ b/flang/include/flang/Tools/CLOptions.inc
@@ -155,9 +155,9 @@ inline void addTargetRewritePass(mlir::PassManager &pm) {
   });
 }
 
-inline void addDebugFoundationPass(mlir::PassManager &pm) {
+inline void addDebugInfoPass(mlir::PassManager &pm) {
   addPassConditionally(pm, disableDebugFoundation,
-      [&]() { return fir::createAddDebugFoundationPass(); });
+      [&]() { return fir::createAddDebugInfoPass(); });
 }
 
 inline void addFIRToLLVMPass(
@@ -288,7 +288,7 @@ inline void createDebugPasses(
   // Currently only -g1, -g, -gline-tables-only supported
   switch (debugLevel) {
   case llvm::codegenoptions::DebugLineTablesOnly:
-    addDebugFoundationPass(pm);
+    addDebugInfoPass(pm);
     return;
   case llvm::codegenoptions::NoDebugInfo:
     return;
@@ -296,7 +296,7 @@ inline void createDebugPasses(
     // TODO: Add cases and passes for other debug options.
     // All other debug options not implemented yet, currently emits warning
     // and generates as much debug information as possible.
-    addDebugFoundationPass(pm);
+    addDebugInfoPass(pm);
     return;
   }
 }
diff --git a/flang/lib/Optimizer/Transforms/AddDebugFoundation.cpp b/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
similarity index 90%
rename from flang/lib/Optimizer/Transforms/AddDebugFoundation.cpp
rename to flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
index 678fbf6a7d23..b719f677230f 100644
--- a/flang/lib/Optimizer/Transforms/AddDebugFoundation.cpp
+++ b/flang/lib/Optimizer/Transforms/AddDebugInfo.cpp
@@ -1,4 +1,4 @@
-//===- AddDebugFoundation.cpp -- add basic debug linetable info -----------===//
+//===-------------- AddDebugInfo.cpp -- add debug info -------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -32,8 +32,8 @@
 #include "llvm/Support/raw_ostream.h"
 
 namespace fir {
-#define GEN_PASS_DEF_ADDDEBUGFOUNDATION
-#define GEN_PASS_DECL_ADDDEBUGFOUNDATION
+#define GEN_PASS_DEF_ADDDEBUGINFO
+#define GEN_PASS_DECL_ADDDEBUGINFO
 #include "flang/Optimizer/Transforms/Passes.h.inc"
 } // namespace fir
 
@@ -41,15 +41,14 @@ namespace fir {
 
 namespace {
 
-class AddDebugFoundationPass
-    : public fir::impl::AddDebugFoundationBase<AddDebugFoundationPass> {
+class AddDebugInfoPass : public fir::impl::AddDebugInfoBase<AddDebugInfoPass> {
 public:
   void runOnOperation() override;
 };
 
 } // namespace
 
-void AddDebugFoundationPass::runOnOperation() {
+void AddDebugInfoPass::runOnOperation() {
   mlir::ModuleOp module = getOperation();
   mlir::MLIRContext *context = &getContext();
   mlir::OpBuilder builder(context);
@@ -113,6 +112,6 @@ void AddDebugFoundationPass::runOnOperation() {
   });
 }
 
-std::unique_ptr<mlir::Pass> fir::createAddDebugFoundationPass() {
-  return std::make_unique<AddDebugFoundationPass>();
+std::unique_ptr<mlir::Pass> fir::createAddDebugInfoPass() {
+  return std::make_unique<AddDebugInfoPass>();
 }
diff --git a/flang/lib/Optimizer/Transforms/CMakeLists.txt b/flang/lib/Optimizer/Transforms/CMakeLists.txt
index ba2e26799615..d55655c53906 100644
--- a/flang/lib/Optimizer/Transforms/CMakeLists.txt
+++ b/flang/lib/Optimizer/Transforms/CMakeLists.txt
@@ -14,7 +14,7 @@ add_flang_library(FIRTransforms
   SimplifyRegionLite.cpp
   AlgebraicSimplification.cpp
   SimplifyIntrinsics.cpp
-  AddDebugFoundation.cpp
+  AddDebugInfo.cpp
   PolymorphicOpConversion.cpp
   LoopVersioning.cpp
   OMPDescriptorMapInfoGen.cpp
diff --git a/flang/test/Driver/mlir-debug-pass-pipeline.f90 b/flang/test/Driver/mlir-debug-pass-pipeline.f90
index e714a66b85f3..04d432f854ca 100644
--- a/flang/test/Driver/mlir-debug-pass-pipeline.f90
+++ b/flang/test/Driver/mlir-debug-pass-pipeline.f90
@@ -82,7 +82,7 @@ end program
 ! ALL-NEXT:   (S) 0 num-dce'd - Number of operations eliminated
 ! ALL-NEXT: TargetRewrite
 ! ALL-NEXT: ExternalNameConversion
-! DEBUG-NEXT: AddDebugFoundation
-! NO-DEBUG-NOT: AddDebugFoundation
+! DEBUG-NEXT: AddDebugInfo
+! NO-DEBUG-NOT: AddDebugInfo
 ! ALL: FIRToLLVMLowering
 ! ALL-NOT: LLVMIRLoweringPass
diff --git a/flang/test/Transforms/debug-line-table-existing.fir b/flang/test/Transforms/debug-line-table-existing.fir
index 3c81d75dbd66..534278ebc972 100644
--- a/flang/test/Transforms/debug-line-table-existing.fir
+++ b/flang/test/Transforms/debug-line-table-existing.fir
@@ -1,5 +1,5 @@
 
-// RUN: fir-opt --add-debug-foundation --mlir-print-debuginfo %s | FileCheck %s
+// RUN: fir-opt --add-debug-info --mlir-print-debuginfo %s | FileCheck %s
 // REQUIRES: system-linux
 
 // Test that there are no changes to a function with existed fused loc debug
diff --git a/flang/test/Transforms/debug-line-table-inc-file.fir b/flang/test/Transforms/debug-line-table-inc-file.fir
index f809ab99b472..be4f005bf664 100644
--- a/flang/test/Transforms/debug-line-table-inc-file.fir
+++ b/flang/test/Transforms/debug-line-table-inc-file.fir
@@ -1,5 +1,5 @@
 
-// RUN: fir-opt --add-debug-foundation --mlir-print-debuginfo %s | FileCheck %s
+// RUN: fir-opt --add-debug-info --mlir-print-debuginfo %s | FileCheck %s
 // REQUIRES: system-linux
 
 // Test for included functions that have a different debug location than the current file
diff --git a/flang/test/Transforms/debug-line-table-inc-same-file.fir b/flang/test/Transforms/debug-line-table-inc-same-file.fir
index 3ca00e68768c..4836f2e21dd9 100644
--- a/flang/test/Transforms/debug-line-table-inc-same-file.fir
+++ b/flang/test/Transforms/debug-line-table-inc-same-file.fir
@@ -1,5 +1,5 @@
 
-// RUN: fir-opt --add-debug-foundation --mlir-print-debuginfo %s | FileCheck %s
+// RUN: fir-opt --add-debug-info --mlir-print-debuginfo %s | FileCheck %s
 // REQUIRES: system-linux
 
 // Test that there is only one FileAttribute generated for multiple functions
diff --git a/flang/test/Transforms/debug-line-table.fir b/flang/test/Transforms/debug-line-table.fir
index 8e66fc1ab398..0ba88d3d9f7f 100644
--- a/flang/test/Transforms/debug-line-table.fir
+++ b/flang/test/Transforms/debug-line-table.fir
@@ -1,5 +1,5 @@
 
-// RUN: fir-opt --add-debug-foundation --mlir-print-debuginfo %s | FileCheck %s
+// RUN: fir-opt --add-debug-info --mlir-print-debuginfo %s | FileCheck %s
 
 module attributes { fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", llvm.data_layout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128", llvm.target_triple = "aarch64-unknown-linux-gnu"} {
   func.func @_QPsb() {
-- 
GitLab


From 8ddaf750746d7f9b5f7e878870b086edc0f55326 Mon Sep 17 00:00:00 2001
From: Johannes Reifferscheid <jreiffers@google.com>
Date: Mon, 15 Apr 2024 11:24:00 +0200
Subject: [PATCH 214/464] Fix rsqrt inaccuracies. (#88691)

The current lowering has issues with large/subnormal values. This ports
XLA's lowering and was verified using XLA's test suite and the
MLIR-based emitters.
---
 .../ComplexToStandard/ComplexToStandard.cpp   | 88 ++++++++++++++++---
 .../convert-to-standard.mlir                  | 17 +++-
 2 files changed, 92 insertions(+), 13 deletions(-)

diff --git a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
index 49eb575212ff..3ebee9baff31 100644
--- a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
+++ b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
@@ -27,9 +27,11 @@ using namespace mlir;
 
 namespace {
 
-// Returns the absolute value or its square root.
+enum class AbsFn { abs, sqrt, rsqrt };
+
+// Returns the absolute value, its square root or its reciprocal square root.
 Value computeAbs(Value real, Value imag, arith::FastMathFlags fmf,
-                 ImplicitLocOpBuilder &b, bool returnSqrt = false) {
+                 ImplicitLocOpBuilder &b, AbsFn fn = AbsFn::abs) {
   Value one = b.create<arith::ConstantOp>(real.getType(),
                                           b.getFloatAttr(real.getType(), 1.0));
 
@@ -43,7 +45,13 @@ Value computeAbs(Value real, Value imag, arith::FastMathFlags fmf,
   Value ratioSqPlusOne = b.create<arith::AddFOp>(ratioSq, one, fmf);
   Value result;
 
-  if (returnSqrt) {
+  if (fn == AbsFn::rsqrt) {
+    ratioSqPlusOne = b.create<math::RsqrtOp>(ratioSqPlusOne, fmf);
+    min = b.create<math::RsqrtOp>(min, fmf);
+    max = b.create<math::RsqrtOp>(max, fmf);
+  }
+
+  if (fn == AbsFn::sqrt) {
     Value quarter = b.create<arith::ConstantOp>(
         real.getType(), b.getFloatAttr(real.getType(), 0.25));
     // sqrt(sqrt(a*b)) would avoid the pow, but will overflow more easily.
@@ -863,7 +871,7 @@ struct SqrtOpConversion : public OpConversionPattern<complex::SqrtOp> {
 
     Value real = b.create<complex::ReOp>(elementType, adaptor.getComplex());
     Value imag = b.create<complex::ImOp>(elementType, adaptor.getComplex());
-    Value absSqrt = computeAbs(real, imag, fmf, b, /*returnSqrt=*/true);
+    Value absSqrt = computeAbs(real, imag, fmf, b, AbsFn::sqrt);
     Value argArg = b.create<math::Atan2Op>(imag, real, fmf);
     Value sqrtArg = b.create<arith::MulFOp>(argArg, half, fmf);
     Value cos = b.create<math::CosOp>(sqrtArg, fmf);
@@ -1147,18 +1155,74 @@ struct RsqrtOpConversion : public OpConversionPattern<complex::RsqrtOp> {
   LogicalResult
   matchAndRewrite(complex::RsqrtOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
-    mlir::ImplicitLocOpBuilder builder(op.getLoc(), rewriter);
+    mlir::ImplicitLocOpBuilder b(op.getLoc(), rewriter);
     auto type = cast<ComplexType>(adaptor.getComplex().getType());
     auto elementType = cast<FloatType>(type.getElementType());
 
-    Value c = builder.create<arith::ConstantOp>(
-        elementType, builder.getFloatAttr(elementType, -0.5));
-    Value d = builder.create<arith::ConstantOp>(
-        elementType, builder.getFloatAttr(elementType, 0));
+    arith::FastMathFlags fmf = op.getFastMathFlagsAttr().getValue();
+
+    auto cst = [&](APFloat v) {
+      return b.create<arith::ConstantOp>(elementType,
+                                         b.getFloatAttr(elementType, v));
+    };
+    const auto &floatSemantics = elementType.getFloatSemantics();
+    Value zero = cst(APFloat::getZero(floatSemantics));
+    Value inf = cst(APFloat::getInf(floatSemantics));
+    Value negHalf = b.create<arith::ConstantOp>(
+        elementType, b.getFloatAttr(elementType, -0.5));
+    Value nan = cst(APFloat::getNaN(floatSemantics));
+
+    Value real = b.create<complex::ReOp>(elementType, adaptor.getComplex());
+    Value imag = b.create<complex::ImOp>(elementType, adaptor.getComplex());
+    Value absRsqrt = computeAbs(real, imag, fmf, b, AbsFn::rsqrt);
+    Value argArg = b.create<math::Atan2Op>(imag, real, fmf);
+    Value rsqrtArg = b.create<arith::MulFOp>(argArg, negHalf, fmf);
+    Value cos = b.create<math::CosOp>(rsqrtArg, fmf);
+    Value sin = b.create<math::SinOp>(rsqrtArg, fmf);
+
+    Value resultReal = b.create<arith::MulFOp>(absRsqrt, cos, fmf);
+    Value resultImag = b.create<arith::MulFOp>(absRsqrt, sin, fmf);
+
+    if (!arith::bitEnumContainsAll(fmf, arith::FastMathFlags::nnan |
+                                            arith::FastMathFlags::ninf)) {
+      Value negOne = b.create<arith::ConstantOp>(
+          elementType, b.getFloatAttr(elementType, -1));
+
+      Value realSignedZero = b.create<math::CopySignOp>(zero, real, fmf);
+      Value imagSignedZero = b.create<math::CopySignOp>(zero, imag, fmf);
+      Value negImagSignedZero =
+          b.create<arith::MulFOp>(negOne, imagSignedZero, fmf);
 
-    rewriter.replaceOp(op,
-                       {powOpConversionImpl(builder, type, adaptor.getComplex(),
-                                            c, d, op.getFastmath())});
+      Value absReal = b.create<math::AbsFOp>(real, fmf);
+      Value absImag = b.create<math::AbsFOp>(imag, fmf);
+
+      Value absImagIsInf =
+          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absImag, inf, fmf);
+      Value realIsNan =
+          b.create<arith::CmpFOp>(arith::CmpFPredicate::UNO, real, real, fmf);
+      Value realIsInf =
+          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absReal, inf, fmf);
+      Value inIsNanInf = b.create<arith::AndIOp>(absImagIsInf, realIsNan);
+
+      Value resultIsZero = b.create<arith::OrIOp>(inIsNanInf, realIsInf);
+
+      resultReal =
+          b.create<arith::SelectOp>(resultIsZero, realSignedZero, resultReal);
+      resultImag = b.create<arith::SelectOp>(resultIsZero, negImagSignedZero,
+                                             resultImag);
+    }
+
+    Value isRealZero =
+        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, real, zero, fmf);
+    Value isImagZero =
+        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, imag, zero, fmf);
+    Value isZero = b.create<arith::AndIOp>(isRealZero, isImagZero);
+
+    resultReal = b.create<arith::SelectOp>(isZero, inf, resultReal);
+    resultImag = b.create<arith::SelectOp>(isZero, nan, resultImag);
+
+    rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
+                                                   resultImag);
     return success();
   }
 };
diff --git a/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir b/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir
index e0e7cdadd317..8b4ea9777f79 100644
--- a/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir
+++ b/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir
@@ -837,6 +837,21 @@ func.func @complex_rsqrt(%arg: complex<f32>) -> complex<f32> {
   return %rsqrt : complex<f32>
 }
 
+// CHECK-COUNT-5: arith.select
+// CHECK-NOT: arith.select
+
+// -----
+
+// CHECK-LABEL: func @complex_rsqrt_nnan_ninf
+// CHECK-SAME: %[[ARG:.*]]: complex<f32>
+func.func @complex_rsqrt_nnan_ninf(%arg: complex<f32>) -> complex<f32> {
+  %sqrt = complex.rsqrt %arg fastmath<nnan,ninf> : complex<f32>
+  return %sqrt : complex<f32>
+}
+
+// CHECK-COUNT-3: arith.select
+// CHECK-NOT: arith.select
+
 // -----
 
 // CHECK-LABEL:   func.func @complex_angle
@@ -2103,4 +2118,4 @@ func.func @complex_tanh_with_fmf(%arg: complex<f32>) -> complex<f32> {
 // CHECK: %[[NUM:.*]] = complex.create %[[TANH_A]], %[[TAN_B]] : complex<f32>
 // CHECK: %[[ONE:.*]] = arith.constant 1.000000e+00 : f32
 // CHECK: %[[MUL:.*]] = arith.mulf %[[TANH_A]], %[[TAN_B]] fastmath<nnan,contract> : f32
-// CHECK: %[[DENOM:.*]] = complex.create %[[ONE]], %[[MUL]] : complex<f32>
\ No newline at end of file
+// CHECK: %[[DENOM:.*]] = complex.create %[[ONE]], %[[MUL]] : complex<f32>
-- 
GitLab


From f2923e31b64907df44ae641114228ad6e283627f Mon Sep 17 00:00:00 2001
From: Sergio Afonso <safonsof@amd.com>
Date: Mon, 15 Apr 2024 10:33:54 +0100
Subject: [PATCH 215/464] [MLIR][OpenMP] Introduce the LoopWrapperInterface
 (#87232)

This patch defines a common interface to be shared by all OpenMP loop
wrapper operations. The main restrictions these operations must meet in
order to be considered a wrapper are:

- They contain a single region.
- Their region contains a single block.
- Their block only contains another loop wrapper or `omp.loop_nest` and
a terminator.

The new interface is attached to the `omp.parallel`, `omp.wsloop`,
`omp.simdloop`, `omp.distribute` and `omp.taskloop` operations. It is
not currently enforced that these operations meet the wrapper
restrictions, which would break existing OpenMP loop-generating code.
Rather, this will be introduced progressively in subsequent patches.
---
 .../mlir/Dialect/OpenMP/OpenMPInterfaces.h    |  3 +
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td | 16 +++--
 .../Dialect/OpenMP/OpenMPOpsInterfaces.td     | 67 +++++++++++++++++++
 mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp  | 18 +++++
 mlir/test/Dialect/OpenMP/invalid.mlir         | 16 ++++-
 5 files changed, 115 insertions(+), 5 deletions(-)

diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPInterfaces.h b/mlir/include/mlir/Dialect/OpenMP/OpenMPInterfaces.h
index b3184db88521..787c48b05c5c 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPInterfaces.h
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPInterfaces.h
@@ -21,6 +21,9 @@
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 
+#define GET_OP_FWD_DEFINES
+#include "mlir/Dialect/OpenMP/OpenMPOps.h.inc"
+
 #include "mlir/Dialect/OpenMP/OpenMPOpsInterfaces.h.inc"
 
 namespace mlir::omp {
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index 596b2f5e4444..3abdbe3adfd0 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -236,6 +236,7 @@ def PrivateClauseOp : OpenMP_Op<"private", [IsolatedFromAbove]> {
 
 def ParallelOp : OpenMP_Op<"parallel", [
                  AutomaticAllocationScope, AttrSizedOperandSegments,
+                 DeclareOpInterfaceMethods<LoopWrapperInterface>,
                  DeclareOpInterfaceMethods<OutlineableOpenMPOpInterface>,
                  RecursiveMemoryEffects, ReductionClauseInterface]> {
   let summary = "parallel construct";
@@ -530,8 +531,6 @@ def SingleOp : OpenMP_Op<"single", [AttrSizedOperandSegments]> {
 
 def LoopNestOp : OpenMP_Op<"loop_nest", [SameVariadicOperandSize,
                         AllTypesMatch<["lowerBound", "upperBound", "step"]>,
-                        ParentOneOf<["DistributeOp", "SimdLoopOp", "TaskloopOp",
-                                     "WsloopOp"]>,
                         RecursiveMemoryEffects]> {
   let summary = "rectangular loop nest";
   let description = [{
@@ -586,6 +585,10 @@ def LoopNestOp : OpenMP_Op<"loop_nest", [SameVariadicOperandSize,
 
     /// Returns the induction variables of the loop nest.
     ArrayRef<BlockArgument> getIVs() { return getRegion().getArguments(); }
+
+    /// Fills a list of wrapper operations around this loop nest. Wrappers
+    /// in the resulting vector will be sorted from innermost to outermost.
+    void gatherWrappers(SmallVectorImpl<LoopWrapperInterface> &wrappers);
   }];
 
   let hasCustomAssemblyFormat = 1;
@@ -598,6 +601,7 @@ def LoopNestOp : OpenMP_Op<"loop_nest", [SameVariadicOperandSize,
 
 def WsloopOp : OpenMP_Op<"wsloop", [AttrSizedOperandSegments,
                          AllTypesMatch<["lowerBound", "upperBound", "step"]>,
+                         DeclareOpInterfaceMethods<LoopWrapperInterface>,
                          RecursiveMemoryEffects, ReductionClauseInterface]> {
   let summary = "worksharing-loop construct";
   let description = [{
@@ -719,7 +723,9 @@ def WsloopOp : OpenMP_Op<"wsloop", [AttrSizedOperandSegments,
 //===----------------------------------------------------------------------===//
 
 def SimdLoopOp : OpenMP_Op<"simdloop", [AttrSizedOperandSegments,
-                         AllTypesMatch<["lowerBound", "upperBound", "step"]>]> {
+                         AllTypesMatch<["lowerBound", "upperBound", "step"]>,
+                         DeclareOpInterfaceMethods<LoopWrapperInterface>,
+                         RecursiveMemoryEffects]> {
  let summary = "simd loop construct";
   let description = [{
     The simd construct can be applied to a loop to indicate that the loop can be
@@ -833,7 +839,8 @@ def YieldOp : OpenMP_Op<"yield",
 // Distribute construct [2.9.4.1]
 //===----------------------------------------------------------------------===//
 def DistributeOp : OpenMP_Op<"distribute", [AttrSizedOperandSegments,
-                              MemoryEffects<[MemWrite]>]> {
+                             DeclareOpInterfaceMethods<LoopWrapperInterface>,
+                             RecursiveMemoryEffects]> {
   let summary = "distribute construct";
   let description = [{
     The distribute construct specifies that the iterations of one or more loops
@@ -1011,6 +1018,7 @@ def TaskOp : OpenMP_Op<"task", [AttrSizedOperandSegments,
 def TaskloopOp : OpenMP_Op<"taskloop", [AttrSizedOperandSegments,
                            AutomaticAllocationScope, RecursiveMemoryEffects,
                            AllTypesMatch<["lowerBound", "upperBound", "step"]>,
+                           DeclareOpInterfaceMethods<LoopWrapperInterface>,
                            ReductionClauseInterface]> {
   let summary = "taskloop construct";
   let description = [{
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td
index 2e37384ce3eb..ab9b78e755d9 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOpsInterfaces.td
@@ -69,6 +69,73 @@ def ReductionClauseInterface : OpInterface<"ReductionClauseInterface"> {
   ];
 }
 
+def LoopWrapperInterface : OpInterface<"LoopWrapperInterface"> {
+  let description = [{
+    OpenMP operations that can wrap a single loop nest. When taking a wrapper
+    role, these operations must only contain a single region with a single block
+    in which there's a single operation and a terminator. That nested operation
+    must be another loop wrapper or an `omp.loop_nest`.
+  }];
+
+  let cppNamespace = "::mlir::omp";
+
+  let methods = [
+    InterfaceMethod<
+      /*description=*/[{
+        Tell whether the operation could be taking the role of a loop wrapper.
+        That is, it has a single region with a single block in which there are
+        two operations: another wrapper or `omp.loop_nest` operation and a
+        terminator.
+      }],
+      /*retTy=*/"bool",
+      /*methodName=*/"isWrapper",
+      (ins ), [{}], [{
+        if ($_op->getNumRegions() != 1)
+          return false;
+
+        Region &r = $_op->getRegion(0);
+        if (!r.hasOneBlock())
+          return false;
+
+        if (::llvm::range_size(r.getOps()) != 2)
+          return false;
+
+        Operation &firstOp = *r.op_begin();
+        Operation &secondOp = *(std::next(r.op_begin()));
+        return ::llvm::isa<LoopNestOp, LoopWrapperInterface>(firstOp) &&
+               secondOp.hasTrait<OpTrait::IsTerminator>();
+      }]
+    >,
+    InterfaceMethod<
+      /*description=*/[{
+        If there is another loop wrapper immediately nested inside, return that
+        operation. Assumes this operation is taking a loop wrapper role.
+      }],
+      /*retTy=*/"::mlir::omp::LoopWrapperInterface",
+      /*methodName=*/"getNestedWrapper",
+      (ins), [{}], [{
+        assert($_op.isWrapper() && "Unexpected non-wrapper op");
+        Operation *nested = &*$_op->getRegion(0).op_begin();
+        return ::llvm::dyn_cast<LoopWrapperInterface>(nested);
+      }]
+    >,
+    InterfaceMethod<
+      /*description=*/[{
+        Return the loop nest nested directly or indirectly inside of this loop
+        wrapper. Assumes this operation is taking a loop wrapper role.
+      }],
+      /*retTy=*/"::mlir::Operation *",
+      /*methodName=*/"getWrappedLoop",
+      (ins), [{}], [{
+        assert($_op.isWrapper() && "Unexpected non-wrapper op");
+        if (LoopWrapperInterface nested = $_op.getNestedWrapper())
+          return nested.getWrappedLoop();
+        return &*$_op->getRegion(0).op_begin();
+      }]
+    >
+  ];
+}
+
 def DeclareTargetInterface : OpInterface<"DeclareTargetInterface"> {
   let description = [{
     OpenMP operations that support declare target have this interface.
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index 35fb174046a3..90b49b2528b7 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -1936,9 +1936,27 @@ LogicalResult LoopNestOp::verify() {
              << "range argument type does not match corresponding IV type";
   }
 
+  auto wrapper =
+      llvm::dyn_cast_if_present<LoopWrapperInterface>((*this)->getParentOp());
+
+  if (!wrapper || !wrapper.isWrapper())
+    return emitOpError() << "expects parent op to be a valid loop wrapper";
+
   return success();
 }
 
+void LoopNestOp::gatherWrappers(
+    SmallVectorImpl<LoopWrapperInterface> &wrappers) {
+  Operation *parent = (*this)->getParentOp();
+  while (auto wrapper =
+             llvm::dyn_cast_if_present<LoopWrapperInterface>(parent)) {
+    if (!wrapper.isWrapper())
+      break;
+    wrappers.push_back(wrapper);
+    parent = parent->getParentOp();
+  }
+}
+
 //===----------------------------------------------------------------------===//
 // WsloopOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir
index 3802fbde534d..88dca1b85ee5 100644
--- a/mlir/test/Dialect/OpenMP/invalid.mlir
+++ b/mlir/test/Dialect/OpenMP/invalid.mlir
@@ -88,7 +88,7 @@ func.func @proc_bind_once() {
 // -----
 
 func.func @invalid_parent(%lb : index, %ub : index, %step : index) {
-  // expected-error@+1 {{op expects parent op to be one of 'omp.distribute, omp.simdloop, omp.taskloop, omp.wsloop'}}
+  // expected-error@+1 {{op expects parent op to be a valid loop wrapper}}
   omp.loop_nest (%iv) : index = (%lb) to (%ub) step (%step) {
     omp.yield
   }
@@ -96,6 +96,20 @@ func.func @invalid_parent(%lb : index, %ub : index, %step : index) {
 
 // -----
 
+func.func @invalid_wrapper(%lb : index, %ub : index, %step : index) {
+  // TODO Remove induction variables from omp.wsloop.
+  omp.wsloop for (%iv) : index = (%lb) to (%ub) step (%step) {
+    %0 = arith.constant 0 : i32
+    // expected-error@+1 {{op expects parent op to be a valid loop wrapper}}
+    omp.loop_nest (%iv2) : index = (%lb) to (%ub) step (%step) {
+      omp.yield
+    }
+    omp.yield
+  }
+}
+
+// -----
+
 func.func @type_mismatch(%lb : index, %ub : index, %step : index) {
   // TODO Remove induction variables from omp.wsloop.
   omp.wsloop for (%iv) : index = (%lb) to (%ub) step (%step) {
-- 
GitLab


From c7bd284dc3ff21a5a14b4ccbbfa1988de8723c33 Mon Sep 17 00:00:00 2001
From: shamithoke <152091883+shamithoke@users.noreply.github.com>
Date: Mon, 15 Apr 2024 15:08:26 +0530
Subject: [PATCH 216/464] Support for i8/i16 for bitreverse using GFNI.
 (#88625)

In continuation to the PR #81764, this change extends the GFNI support to i8 and i16.

---------

Co-authored-by: shami <shami_thoke@yahoo.com>
---
 llvm/lib/Target/X86/X86ISelLowering.cpp    | 14 ++--
 llvm/test/CodeGen/X86/bitreverse.ll        | 57 ++++------------
 llvm/test/CodeGen/X86/vector-bitreverse.ll | 78 +++++-----------------
 3 files changed, 38 insertions(+), 111 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index edfa05a66288..87f2eaddedf8 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -1287,8 +1287,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     }
 
     if (Subtarget.hasGFNI()) {
-      setOperationAction(ISD::BITREVERSE,       MVT::i32, Custom);
-      setOperationAction(ISD::BITREVERSE,       MVT::i64, Custom);
+      setOperationAction(ISD::BITREVERSE, MVT::i8, Custom);
+      setOperationAction(ISD::BITREVERSE, MVT::i16, Custom);
+      setOperationAction(ISD::BITREVERSE, MVT::i32, Custom);
+      setOperationAction(ISD::BITREVERSE, MVT::i64, Custom);
     }
 
     // These might be better off as horizontal vector ops.
@@ -31317,16 +31319,18 @@ static SDValue LowerBITREVERSE(SDValue Op, const X86Subtarget &Subtarget,
   if (VT.is256BitVector() && !Subtarget.hasInt256())
     return splitVectorIntUnary(Op, DAG, DL);
 
-  // Lower i32/i64 as vXi8 BITREVERSE + BSWAP
+  // Lower i8/i16/i32/i64 as vXi8 BITREVERSE + BSWAP
   if (!VT.isVector()) {
-    assert((VT == MVT::i32 || VT == MVT::i64) && "Only tested for i32/i64");
+    assert(
+        (VT == MVT::i32 || VT == MVT::i64 || VT == MVT::i16 || VT == MVT::i8) &&
+        "Only tested for i8/i16/i32/i64");
     MVT VecVT = MVT::getVectorVT(VT, 128 / VT.getSizeInBits());
     SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, In);
     Res = DAG.getNode(ISD::BITREVERSE, DL, MVT::v16i8,
                       DAG.getBitcast(MVT::v16i8, Res));
     Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
                       DAG.getBitcast(VecVT, Res), DAG.getIntPtrConstant(0, DL));
-    return DAG.getNode(ISD::BSWAP, DL, VT, Res);
+    return (VT == MVT::i8) ? Res : DAG.getNode(ISD::BSWAP, DL, VT, Res);
   }
 
   assert(VT.isVector() && VT.getSizeInBits() >= 128);
diff --git a/llvm/test/CodeGen/X86/bitreverse.ll b/llvm/test/CodeGen/X86/bitreverse.ll
index 704563ab1bbf..4f2654843728 100644
--- a/llvm/test/CodeGen/X86/bitreverse.ll
+++ b/llvm/test/CodeGen/X86/bitreverse.ll
@@ -374,24 +374,10 @@ define i16 @test_bitreverse_i16(i16 %a) nounwind {
 ;
 ; GFNI-LABEL: test_bitreverse_i16:
 ; GFNI:       # %bb.0:
-; GFNI-NEXT:    # kill: def $edi killed $edi def $rdi
-; GFNI-NEXT:    rolw $8, %di
-; GFNI-NEXT:    movl %edi, %eax
-; GFNI-NEXT:    andl $3855, %eax # imm = 0xF0F
-; GFNI-NEXT:    shll $4, %eax
-; GFNI-NEXT:    shrl $4, %edi
-; GFNI-NEXT:    andl $3855, %edi # imm = 0xF0F
-; GFNI-NEXT:    orl %eax, %edi
-; GFNI-NEXT:    movl %edi, %eax
-; GFNI-NEXT:    andl $13107, %eax # imm = 0x3333
-; GFNI-NEXT:    shrl $2, %edi
-; GFNI-NEXT:    andl $13107, %edi # imm = 0x3333
-; GFNI-NEXT:    leal (%rdi,%rax,4), %eax
-; GFNI-NEXT:    movl %eax, %ecx
-; GFNI-NEXT:    andl $21845, %ecx # imm = 0x5555
-; GFNI-NEXT:    shrl %eax
-; GFNI-NEXT:    andl $21845, %eax # imm = 0x5555
-; GFNI-NEXT:    leal (%rax,%rcx,2), %eax
+; GFNI-NEXT:    vmovd %edi, %xmm0
+; GFNI-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
+; GFNI-NEXT:    vmovd %xmm0, %eax
+; GFNI-NEXT:    rolw $8, %ax
 ; GFNI-NEXT:    # kill: def $ax killed $ax killed $eax
 ; GFNI-NEXT:    retq
   %b = call i16 @llvm.bitreverse.i16(i16 %a)
@@ -446,19 +432,10 @@ define i8 @test_bitreverse_i8(i8 %a) {
 ;
 ; GFNI-LABEL: test_bitreverse_i8:
 ; GFNI:       # %bb.0:
-; GFNI-NEXT:    rolb $4, %dil
-; GFNI-NEXT:    movl %edi, %eax
-; GFNI-NEXT:    andb $51, %al
-; GFNI-NEXT:    shlb $2, %al
-; GFNI-NEXT:    shrb $2, %dil
-; GFNI-NEXT:    andb $51, %dil
-; GFNI-NEXT:    orb %dil, %al
-; GFNI-NEXT:    movl %eax, %ecx
-; GFNI-NEXT:    andb $85, %cl
-; GFNI-NEXT:    addb %cl, %cl
-; GFNI-NEXT:    shrb %al
-; GFNI-NEXT:    andb $85, %al
-; GFNI-NEXT:    orb %cl, %al
+; GFNI-NEXT:    vmovd %edi, %xmm0
+; GFNI-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
+; GFNI-NEXT:    vmovd %xmm0, %eax
+; GFNI-NEXT:    # kill: def $al killed $al killed $eax
 ; GFNI-NEXT:    retq
   %b = call i8 @llvm.bitreverse.i8(i8 %a)
   ret i8 %b
@@ -514,19 +491,11 @@ define i4 @test_bitreverse_i4(i4 %a) {
 ;
 ; GFNI-LABEL: test_bitreverse_i4:
 ; GFNI:       # %bb.0:
-; GFNI-NEXT:    # kill: def $edi killed $edi def $rdi
-; GFNI-NEXT:    movl %edi, %eax
-; GFNI-NEXT:    andb $8, %al
-; GFNI-NEXT:    leal (%rdi,%rdi), %ecx
-; GFNI-NEXT:    andb $4, %cl
-; GFNI-NEXT:    leal (,%rdi,8), %edx
-; GFNI-NEXT:    andb $8, %dl
-; GFNI-NEXT:    orb %cl, %dl
-; GFNI-NEXT:    shrb %dil
-; GFNI-NEXT:    andb $2, %dil
-; GFNI-NEXT:    orb %dil, %dl
-; GFNI-NEXT:    shrb $3, %al
-; GFNI-NEXT:    orb %dl, %al
+; GFNI-NEXT:    vmovd %edi, %xmm0
+; GFNI-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
+; GFNI-NEXT:    vmovd %xmm0, %eax
+; GFNI-NEXT:    shrb $4, %al
+; GFNI-NEXT:    # kill: def $al killed $al killed $eax
 ; GFNI-NEXT:    retq
   %b = call i4 @llvm.bitreverse.i4(i4 %a)
   ret i4 %b
diff --git a/llvm/test/CodeGen/X86/vector-bitreverse.ll b/llvm/test/CodeGen/X86/vector-bitreverse.ll
index b22b508db8b2..90cc3d5fdde8 100644
--- a/llvm/test/CodeGen/X86/vector-bitreverse.ll
+++ b/llvm/test/CodeGen/X86/vector-bitreverse.ll
@@ -61,36 +61,18 @@ define i8 @test_bitreverse_i8(i8 %a) nounwind {
 ;
 ; GFNISSE-LABEL: test_bitreverse_i8:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    rolb $4, %dil
-; GFNISSE-NEXT:    movl %edi, %eax
-; GFNISSE-NEXT:    andb $51, %al
-; GFNISSE-NEXT:    shlb $2, %al
-; GFNISSE-NEXT:    shrb $2, %dil
-; GFNISSE-NEXT:    andb $51, %dil
-; GFNISSE-NEXT:    orb %dil, %al
-; GFNISSE-NEXT:    movl %eax, %ecx
-; GFNISSE-NEXT:    andb $85, %cl
-; GFNISSE-NEXT:    addb %cl, %cl
-; GFNISSE-NEXT:    shrb %al
-; GFNISSE-NEXT:    andb $85, %al
-; GFNISSE-NEXT:    orb %cl, %al
+; GFNISSE-NEXT:    movd %edi, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; GFNISSE-NEXT:    movd %xmm0, %eax
+; GFNISSE-NEXT:    # kill: def $al killed $al killed $eax
 ; GFNISSE-NEXT:    retq
 ;
 ; GFNIAVX-LABEL: test_bitreverse_i8:
 ; GFNIAVX:       # %bb.0:
-; GFNIAVX-NEXT:    rolb $4, %dil
-; GFNIAVX-NEXT:    movl %edi, %eax
-; GFNIAVX-NEXT:    andb $51, %al
-; GFNIAVX-NEXT:    shlb $2, %al
-; GFNIAVX-NEXT:    shrb $2, %dil
-; GFNIAVX-NEXT:    andb $51, %dil
-; GFNIAVX-NEXT:    orb %dil, %al
-; GFNIAVX-NEXT:    movl %eax, %ecx
-; GFNIAVX-NEXT:    andb $85, %cl
-; GFNIAVX-NEXT:    addb %cl, %cl
-; GFNIAVX-NEXT:    shrb %al
-; GFNIAVX-NEXT:    andb $85, %al
-; GFNIAVX-NEXT:    orb %cl, %al
+; GFNIAVX-NEXT:    vmovd %edi, %xmm0
+; GFNIAVX-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; GFNIAVX-NEXT:    vmovd %xmm0, %eax
+; GFNIAVX-NEXT:    # kill: def $al killed $al killed $eax
 ; GFNIAVX-NEXT:    retq
   %b = call i8 @llvm.bitreverse.i8(i8 %a)
   ret i8 %b
@@ -153,47 +135,19 @@ define i16 @test_bitreverse_i16(i16 %a) nounwind {
 ;
 ; GFNISSE-LABEL: test_bitreverse_i16:
 ; GFNISSE:       # %bb.0:
-; GFNISSE-NEXT:    # kill: def $edi killed $edi def $rdi
-; GFNISSE-NEXT:    rolw $8, %di
-; GFNISSE-NEXT:    movl %edi, %eax
-; GFNISSE-NEXT:    andl $3855, %eax # imm = 0xF0F
-; GFNISSE-NEXT:    shll $4, %eax
-; GFNISSE-NEXT:    shrl $4, %edi
-; GFNISSE-NEXT:    andl $3855, %edi # imm = 0xF0F
-; GFNISSE-NEXT:    orl %eax, %edi
-; GFNISSE-NEXT:    movl %edi, %eax
-; GFNISSE-NEXT:    andl $13107, %eax # imm = 0x3333
-; GFNISSE-NEXT:    shrl $2, %edi
-; GFNISSE-NEXT:    andl $13107, %edi # imm = 0x3333
-; GFNISSE-NEXT:    leal (%rdi,%rax,4), %eax
-; GFNISSE-NEXT:    movl %eax, %ecx
-; GFNISSE-NEXT:    andl $21845, %ecx # imm = 0x5555
-; GFNISSE-NEXT:    shrl %eax
-; GFNISSE-NEXT:    andl $21845, %eax # imm = 0x5555
-; GFNISSE-NEXT:    leal (%rax,%rcx,2), %eax
+; GFNISSE-NEXT:    movd %edi, %xmm0
+; GFNISSE-NEXT:    gf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; GFNISSE-NEXT:    movd %xmm0, %eax
+; GFNISSE-NEXT:    rolw $8, %ax
 ; GFNISSE-NEXT:    # kill: def $ax killed $ax killed $eax
 ; GFNISSE-NEXT:    retq
 ;
 ; GFNIAVX-LABEL: test_bitreverse_i16:
 ; GFNIAVX:       # %bb.0:
-; GFNIAVX-NEXT:    # kill: def $edi killed $edi def $rdi
-; GFNIAVX-NEXT:    rolw $8, %di
-; GFNIAVX-NEXT:    movl %edi, %eax
-; GFNIAVX-NEXT:    andl $3855, %eax # imm = 0xF0F
-; GFNIAVX-NEXT:    shll $4, %eax
-; GFNIAVX-NEXT:    shrl $4, %edi
-; GFNIAVX-NEXT:    andl $3855, %edi # imm = 0xF0F
-; GFNIAVX-NEXT:    orl %eax, %edi
-; GFNIAVX-NEXT:    movl %edi, %eax
-; GFNIAVX-NEXT:    andl $13107, %eax # imm = 0x3333
-; GFNIAVX-NEXT:    shrl $2, %edi
-; GFNIAVX-NEXT:    andl $13107, %edi # imm = 0x3333
-; GFNIAVX-NEXT:    leal (%rdi,%rax,4), %eax
-; GFNIAVX-NEXT:    movl %eax, %ecx
-; GFNIAVX-NEXT:    andl $21845, %ecx # imm = 0x5555
-; GFNIAVX-NEXT:    shrl %eax
-; GFNIAVX-NEXT:    andl $21845, %eax # imm = 0x5555
-; GFNIAVX-NEXT:    leal (%rax,%rcx,2), %eax
+; GFNIAVX-NEXT:    vmovd %edi, %xmm0
+; GFNIAVX-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; GFNIAVX-NEXT:    vmovd %xmm0, %eax
+; GFNIAVX-NEXT:    rolw $8, %ax
 ; GFNIAVX-NEXT:    # kill: def $ax killed $ax killed $eax
 ; GFNIAVX-NEXT:    retq
   %b = call i16 @llvm.bitreverse.i16(i16 %a)
-- 
GitLab


From b4e7b56403387630d4c390ea4f53921312326e34 Mon Sep 17 00:00:00 2001
From: Johannes Reifferscheid <jreiffers@google.com>
Date: Mon, 15 Apr 2024 11:48:22 +0200
Subject: [PATCH 217/464] Revert "Fix rsqrt inaccuracies." (#88705)

Reverts llvm/llvm-project#88691
---
 .../ComplexToStandard/ComplexToStandard.cpp   | 88 +++----------------
 .../convert-to-standard.mlir                  | 17 +---
 2 files changed, 13 insertions(+), 92 deletions(-)

diff --git a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
index 3ebee9baff31..49eb575212ff 100644
--- a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
+++ b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
@@ -27,11 +27,9 @@ using namespace mlir;
 
 namespace {
 
-enum class AbsFn { abs, sqrt, rsqrt };
-
-// Returns the absolute value, its square root or its reciprocal square root.
+// Returns the absolute value or its square root.
 Value computeAbs(Value real, Value imag, arith::FastMathFlags fmf,
-                 ImplicitLocOpBuilder &b, AbsFn fn = AbsFn::abs) {
+                 ImplicitLocOpBuilder &b, bool returnSqrt = false) {
   Value one = b.create<arith::ConstantOp>(real.getType(),
                                           b.getFloatAttr(real.getType(), 1.0));
 
@@ -45,13 +43,7 @@ Value computeAbs(Value real, Value imag, arith::FastMathFlags fmf,
   Value ratioSqPlusOne = b.create<arith::AddFOp>(ratioSq, one, fmf);
   Value result;
 
-  if (fn == AbsFn::rsqrt) {
-    ratioSqPlusOne = b.create<math::RsqrtOp>(ratioSqPlusOne, fmf);
-    min = b.create<math::RsqrtOp>(min, fmf);
-    max = b.create<math::RsqrtOp>(max, fmf);
-  }
-
-  if (fn == AbsFn::sqrt) {
+  if (returnSqrt) {
     Value quarter = b.create<arith::ConstantOp>(
         real.getType(), b.getFloatAttr(real.getType(), 0.25));
     // sqrt(sqrt(a*b)) would avoid the pow, but will overflow more easily.
@@ -871,7 +863,7 @@ struct SqrtOpConversion : public OpConversionPattern<complex::SqrtOp> {
 
     Value real = b.create<complex::ReOp>(elementType, adaptor.getComplex());
     Value imag = b.create<complex::ImOp>(elementType, adaptor.getComplex());
-    Value absSqrt = computeAbs(real, imag, fmf, b, AbsFn::sqrt);
+    Value absSqrt = computeAbs(real, imag, fmf, b, /*returnSqrt=*/true);
     Value argArg = b.create<math::Atan2Op>(imag, real, fmf);
     Value sqrtArg = b.create<arith::MulFOp>(argArg, half, fmf);
     Value cos = b.create<math::CosOp>(sqrtArg, fmf);
@@ -1155,74 +1147,18 @@ struct RsqrtOpConversion : public OpConversionPattern<complex::RsqrtOp> {
   LogicalResult
   matchAndRewrite(complex::RsqrtOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
-    mlir::ImplicitLocOpBuilder b(op.getLoc(), rewriter);
+    mlir::ImplicitLocOpBuilder builder(op.getLoc(), rewriter);
     auto type = cast<ComplexType>(adaptor.getComplex().getType());
     auto elementType = cast<FloatType>(type.getElementType());
 
-    arith::FastMathFlags fmf = op.getFastMathFlagsAttr().getValue();
-
-    auto cst = [&](APFloat v) {
-      return b.create<arith::ConstantOp>(elementType,
-                                         b.getFloatAttr(elementType, v));
-    };
-    const auto &floatSemantics = elementType.getFloatSemantics();
-    Value zero = cst(APFloat::getZero(floatSemantics));
-    Value inf = cst(APFloat::getInf(floatSemantics));
-    Value negHalf = b.create<arith::ConstantOp>(
-        elementType, b.getFloatAttr(elementType, -0.5));
-    Value nan = cst(APFloat::getNaN(floatSemantics));
-
-    Value real = b.create<complex::ReOp>(elementType, adaptor.getComplex());
-    Value imag = b.create<complex::ImOp>(elementType, adaptor.getComplex());
-    Value absRsqrt = computeAbs(real, imag, fmf, b, AbsFn::rsqrt);
-    Value argArg = b.create<math::Atan2Op>(imag, real, fmf);
-    Value rsqrtArg = b.create<arith::MulFOp>(argArg, negHalf, fmf);
-    Value cos = b.create<math::CosOp>(rsqrtArg, fmf);
-    Value sin = b.create<math::SinOp>(rsqrtArg, fmf);
-
-    Value resultReal = b.create<arith::MulFOp>(absRsqrt, cos, fmf);
-    Value resultImag = b.create<arith::MulFOp>(absRsqrt, sin, fmf);
-
-    if (!arith::bitEnumContainsAll(fmf, arith::FastMathFlags::nnan |
-                                            arith::FastMathFlags::ninf)) {
-      Value negOne = b.create<arith::ConstantOp>(
-          elementType, b.getFloatAttr(elementType, -1));
-
-      Value realSignedZero = b.create<math::CopySignOp>(zero, real, fmf);
-      Value imagSignedZero = b.create<math::CopySignOp>(zero, imag, fmf);
-      Value negImagSignedZero =
-          b.create<arith::MulFOp>(negOne, imagSignedZero, fmf);
+    Value c = builder.create<arith::ConstantOp>(
+        elementType, builder.getFloatAttr(elementType, -0.5));
+    Value d = builder.create<arith::ConstantOp>(
+        elementType, builder.getFloatAttr(elementType, 0));
 
-      Value absReal = b.create<math::AbsFOp>(real, fmf);
-      Value absImag = b.create<math::AbsFOp>(imag, fmf);
-
-      Value absImagIsInf =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absImag, inf, fmf);
-      Value realIsNan =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::UNO, real, real, fmf);
-      Value realIsInf =
-          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absReal, inf, fmf);
-      Value inIsNanInf = b.create<arith::AndIOp>(absImagIsInf, realIsNan);
-
-      Value resultIsZero = b.create<arith::OrIOp>(inIsNanInf, realIsInf);
-
-      resultReal =
-          b.create<arith::SelectOp>(resultIsZero, realSignedZero, resultReal);
-      resultImag = b.create<arith::SelectOp>(resultIsZero, negImagSignedZero,
-                                             resultImag);
-    }
-
-    Value isRealZero =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, real, zero, fmf);
-    Value isImagZero =
-        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, imag, zero, fmf);
-    Value isZero = b.create<arith::AndIOp>(isRealZero, isImagZero);
-
-    resultReal = b.create<arith::SelectOp>(isZero, inf, resultReal);
-    resultImag = b.create<arith::SelectOp>(isZero, nan, resultImag);
-
-    rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
-                                                   resultImag);
+    rewriter.replaceOp(op,
+                       {powOpConversionImpl(builder, type, adaptor.getComplex(),
+                                            c, d, op.getFastmath())});
     return success();
   }
 };
diff --git a/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir b/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir
index 8b4ea9777f79..e0e7cdadd317 100644
--- a/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir
+++ b/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir
@@ -837,21 +837,6 @@ func.func @complex_rsqrt(%arg: complex<f32>) -> complex<f32> {
   return %rsqrt : complex<f32>
 }
 
-// CHECK-COUNT-5: arith.select
-// CHECK-NOT: arith.select
-
-// -----
-
-// CHECK-LABEL: func @complex_rsqrt_nnan_ninf
-// CHECK-SAME: %[[ARG:.*]]: complex<f32>
-func.func @complex_rsqrt_nnan_ninf(%arg: complex<f32>) -> complex<f32> {
-  %sqrt = complex.rsqrt %arg fastmath<nnan,ninf> : complex<f32>
-  return %sqrt : complex<f32>
-}
-
-// CHECK-COUNT-3: arith.select
-// CHECK-NOT: arith.select
-
 // -----
 
 // CHECK-LABEL:   func.func @complex_angle
@@ -2118,4 +2103,4 @@ func.func @complex_tanh_with_fmf(%arg: complex<f32>) -> complex<f32> {
 // CHECK: %[[NUM:.*]] = complex.create %[[TANH_A]], %[[TAN_B]] : complex<f32>
 // CHECK: %[[ONE:.*]] = arith.constant 1.000000e+00 : f32
 // CHECK: %[[MUL:.*]] = arith.mulf %[[TANH_A]], %[[TAN_B]] fastmath<nnan,contract> : f32
-// CHECK: %[[DENOM:.*]] = complex.create %[[ONE]], %[[MUL]] : complex<f32>
+// CHECK: %[[DENOM:.*]] = complex.create %[[ONE]], %[[MUL]] : complex<f32>
\ No newline at end of file
-- 
GitLab


From 45137766ca6be16b9bc6082eb9bedae7bc621437 Mon Sep 17 00:00:00 2001
From: Victor Campos <victor.campos@arm.com>
Date: Mon, 15 Apr 2024 10:58:30 +0100
Subject: [PATCH 218/464] [ARM][Thumb2] Mark BTI-clearing instructions as
 scheduling region boundaries (#87982)

Following https://github.com/llvm/llvm-project/pull/68313 this patch
extends the idea to M-profile PACBTI.

The Machine Scheduler can reorder instructions within a scheduling
region depending on the scheduling policy set. If a BTI-clearing
instruction happens to partake in one such region, it might be moved
around, therefore ending up where it shouldn't.

The solution is to mark all BTI-clearing instructions as scheduling
region boundaries. This essentially means that they must not be part of
any scheduling region, and as consequence never get moved:

 - PAC
 - PACBTI
 - BTI
 - SG

Note that PAC isn't BTI-clearing, but it's replaced by PACBTI late in
the compilation pipeline.

As far as I know, currently it isn't possible to organically obtain code
that's susceptible to the bug:

- Instructions that write to SP are region boundaries. PAC seems to
always be followed by the pushing of r12 to the stack, so essentially
PAC is always by itself in a scheduling region.
- CALL_BTI is expanded into a machine instruction bundle. Bundles are
unpacked only after the last machine scheduler run. Thus setjmp and BTI
can be separated only if someone deliberately run the scheduler once
more.
- The BTI insertion pass is run late in the pipeline, only after the
last machine scheduling has run. So once again it can be reordered only
if someone deliberately runs the scheduler again.

Nevertheless, one can reasonably argue that we should prevent the bug in
spite of the compiler not being able to produce the required conditions
for it. If things change, the compiler will be robust against this
issue.

The tests written for this are contrived: bogus MIR instructions have
been added adjacent to the BTI-clearing instructions in order to have
them inside non-trivial scheduling regions.
---
 llvm/lib/Target/ARM/Thumb2InstrInfo.cpp       |  19 ++
 llvm/lib/Target/ARM/Thumb2InstrInfo.h         |   4 +
 .../CodeGen/ARM/misched-branch-targets.mir    | 169 ++++++++++++++++++
 3 files changed, 192 insertions(+)
 create mode 100644 llvm/test/CodeGen/ARM/misched-branch-targets.mir

diff --git a/llvm/lib/Target/ARM/Thumb2InstrInfo.cpp b/llvm/lib/Target/ARM/Thumb2InstrInfo.cpp
index 083f25f49dec..fc2834cb0b45 100644
--- a/llvm/lib/Target/ARM/Thumb2InstrInfo.cpp
+++ b/llvm/lib/Target/ARM/Thumb2InstrInfo.cpp
@@ -286,6 +286,25 @@ MachineInstr *Thumb2InstrInfo::commuteInstructionImpl(MachineInstr &MI,
   return ARMBaseInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
 }
 
+bool Thumb2InstrInfo::isSchedulingBoundary(const MachineInstr &MI,
+                                           const MachineBasicBlock *MBB,
+                                           const MachineFunction &MF) const {
+  // BTI clearing instructions shall not take part in scheduling regions as
+  // they must stay in their intended place. Although PAC isn't BTI clearing,
+  // it can be transformed into PACBTI after the pre-RA Machine Scheduling
+  // has taken place, so its movement must also be restricted.
+  switch (MI.getOpcode()) {
+  case ARM::t2BTI:
+  case ARM::t2PAC:
+  case ARM::t2PACBTI:
+  case ARM::t2SG:
+    return true;
+  default:
+    break;
+  }
+  return ARMBaseInstrInfo::isSchedulingBoundary(MI, MBB, MF);
+}
+
 void llvm::emitT2RegPlusImmediate(MachineBasicBlock &MBB,
                                   MachineBasicBlock::iterator &MBBI,
                                   const DebugLoc &dl, Register DestReg,
diff --git a/llvm/lib/Target/ARM/Thumb2InstrInfo.h b/llvm/lib/Target/ARM/Thumb2InstrInfo.h
index 4bb412f09dcb..8915da8c5bf3 100644
--- a/llvm/lib/Target/ARM/Thumb2InstrInfo.h
+++ b/llvm/lib/Target/ARM/Thumb2InstrInfo.h
@@ -68,6 +68,10 @@ public:
                                        unsigned OpIdx1,
                                        unsigned OpIdx2) const override;
 
+  bool isSchedulingBoundary(const MachineInstr &MI,
+                            const MachineBasicBlock *MBB,
+                            const MachineFunction &MF) const override;
+
 private:
   void expandLoadStackGuard(MachineBasicBlock::iterator MI) const override;
 };
diff --git a/llvm/test/CodeGen/ARM/misched-branch-targets.mir b/llvm/test/CodeGen/ARM/misched-branch-targets.mir
new file mode 100644
index 000000000000..d828d9e51627
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/misched-branch-targets.mir
@@ -0,0 +1,169 @@
+# RUN: llc -o - -run-pass=machine-scheduler -misched=shuffle %s | FileCheck %s
+# RUN: llc -o - -run-pass=postmisched %s | FileCheck %s
+
+# REQUIRES: asserts
+# -misched=shuffle is only available with assertions enabled
+
+--- |
+  target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+  target triple = "thumbv8.1m.main-arm-none-eabi"
+
+  define i32 @foo_bti() #0 {
+  entry:
+    ret i32 0
+  }
+
+  define i32 @foo_pac() #0 {
+  entry:
+    ret i32 0
+  }
+
+  define i32 @foo_pacbti() #0 {
+  entry:
+    ret i32 0
+  }
+
+  define i32 @foo_setjmp() #0 {
+  entry:
+    ret i32 0
+  if.then:
+    ret i32 0
+  }
+
+  define i32 @foo_sg() #0 {
+  entry:
+    ret i32 0
+  }
+
+  declare i32 @setjmp(ptr noundef) #1
+  declare void @longjmp(ptr noundef, i32 noundef) #2
+
+  attributes #0 = { "frame-pointer"="all" "target-cpu"="cortex-m55" "target-features"="+armv8.1-m.main" }
+  attributes #1 = { nounwind returns_twice "frame-pointer"="all" "target-cpu"="cortex-m55" "target-features"="+armv8.1-m.main" }
+  attributes #2 = { noreturn nounwind "frame-pointer"="all" "target-cpu"="cortex-m55" "target-features"="+armv8.1-m.main" }
+
+...
+---
+name:            foo_bti
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $r0
+
+    t2BTI
+    renamable $r0, dead $cpsr = nsw tADDi8 killed renamable $r0, 1, 14 /* CC::al */, $noreg
+    tBX_RET 14 /* CC::al */, $noreg, implicit killed $r0
+
+...
+
+# CHECK-LABEL: name:            foo_bti
+# CHECK:       body:
+# CHECK-NEXT:   bb.0.entry:
+# CHECK-NEXT:     liveins: $r0
+# CHECK-NEXT:     {{^ +$}}
+# CHECK-NEXT:     t2BTI
+
+---
+name:            foo_pac
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $r0, $lr, $r12
+
+    frame-setup t2PAC implicit-def $r12, implicit $lr, implicit $sp
+    renamable $r2 = nsw t2ADDri $r0, 3, 14 /* CC::al */, $noreg, $noreg
+    $sp = frame-setup t2STMDB_UPD $sp, 14 /* CC::al */, $noreg, killed $r7, killed $lr
+    $r7 = frame-setup tMOVr killed $sp, 14 /* CC::al */, $noreg
+    early-clobber $sp = frame-setup t2STR_PRE killed $r12, $sp, -4, 14 /* CC::al */, $noreg
+    $r12, $sp = frame-destroy t2LDR_POST $sp, 4, 14 /* CC::al */, $noreg
+    $sp = frame-destroy t2LDMIA_UPD $sp, 14 /* CC::al */, $noreg, def $r7, def $lr
+    t2AUT implicit $r12, implicit $lr, implicit $sp
+    tBX_RET 14 /* CC::al */, $noreg, implicit $r0
+
+...
+
+# CHECK-LABEL: name:            foo_pac
+# CHECK:       body:
+# CHECK-NEXT:    bb.0.entry:
+# CHECK-NEXT:      liveins: $r0, $lr, $r12
+# CHECK-NEXT:      {{^ +$}}
+# CHECK-NEXT:      frame-setup t2PAC implicit-def $r12, implicit $lr, implicit $sp
+
+---
+name:            foo_pacbti
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $r0, $lr, $r12
+
+    frame-setup t2PACBTI implicit-def $r12, implicit $lr, implicit $sp
+    renamable $r2 = nsw t2ADDri $r0, 3, 14 /* CC::al */, $noreg, $noreg
+    $sp = frame-setup t2STMDB_UPD $sp, 14 /* CC::al */, $noreg, killed $r7, killed $lr
+    $r7 = frame-setup tMOVr killed $sp, 14 /* CC::al */, $noreg
+    early-clobber $sp = frame-setup t2STR_PRE killed $r12, $sp, -4, 14 /* CC::al */, $noreg
+    $r12, $sp = frame-destroy t2LDR_POST $sp, 4, 14 /* CC::al */, $noreg
+    $sp = frame-destroy t2LDMIA_UPD $sp, 14 /* CC::al */, $noreg, def $r7, def $lr
+    t2AUT implicit $r12, implicit $lr, implicit $sp
+    tBX_RET 14 /* CC::al */, $noreg, implicit $r0
+
+...
+
+# CHECK-LABEL: name:            foo_pacbti
+# CHECK:       body:
+# CHECK-NEXT:    bb.0.entry:
+# CHECK-NEXT:      liveins: $r0, $lr, $r12
+# CHECK-NEXT:      {{^ +$}}
+# CHECK-NEXT:      frame-setup t2PACBTI implicit-def $r12, implicit $lr, implicit $sp
+
+---
+name:            foo_setjmp
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    successors: %bb.1
+    liveins: $lr
+
+    frame-setup tPUSH 14 /* CC::al */, $noreg, $r7, killed $lr, implicit-def $sp, implicit $sp
+    $r7 = frame-setup tMOVr $sp, 14 /* CC::al */, $noreg
+    $sp = frame-setup tSUBspi $sp, 40, 14 /* CC::al */, $noreg
+    renamable $r0 = tMOVr $sp, 14 /* CC::al */, $noreg
+    tBL 14 /* CC::al */, $noreg, @setjmp, csr_aapcs, implicit-def dead $lr, implicit $sp, implicit killed $r0, implicit-def $sp, implicit-def $r0
+    t2BTI
+    renamable $r2 = nsw t2ADDri $r0, 3, 14 /* CC::al */, $noreg, $noreg
+    tCMPi8 killed renamable $r0, 0, 14 /* CC::al */, $noreg, implicit-def $cpsr
+    t2IT 0, 2, implicit-def $itstate
+    renamable $r0 = tMOVi8 $noreg, 0, 0 /* CC::eq */, $cpsr, implicit $itstate
+    $sp = frame-destroy tADDspi $sp, 40, 0 /* CC::eq */, $cpsr, implicit $itstate
+    frame-destroy tPOP_RET 0 /* CC::eq */, killed $cpsr, def $r7, def $pc, implicit killed $r0, implicit $sp, implicit killed $itstate
+
+  bb.1.if.then:
+    renamable $r0 = tMOVr $sp, 14 /* CC::al */, $noreg
+    renamable $r1, dead $cpsr = tMOVi8 1, 14 /* CC::al */, $noreg
+    tBL 14 /* CC::al */, $noreg, @longjmp, csr_aapcs, implicit-def dead $lr, implicit $sp, implicit killed $r0, implicit killed $r1, implicit-def $sp
+
+...
+
+# CHECK-LABEL: name:            foo_setjmp
+# CHECK:       body:
+# CHECK:         tBL 14 /* CC::al */, $noreg, @setjmp, csr_aapcs, implicit-def dead $lr, implicit $sp, implicit killed $r0, implicit-def $sp, implicit-def $r0
+# CHECK-NEXT:    t2BTI
+
+---
+name:            foo_sg
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $r0
+
+    t2SG 14 /* CC::al */, $noreg
+    renamable $r0, dead $cpsr = nsw tADDi8 killed renamable $r0, 1, 14 /* CC::al */, $noreg
+    tBX_RET 14 /* CC::al */, $noreg, implicit killed $r0
+
+...
+
+# CHECK-LABEL: name:            foo_sg
+# CHECK:       body:
+# CHECK-NEXT:   bb.0.entry:
+# CHECK-NEXT:     liveins: $r0
+# CHECK-NEXT:     {{^ +$}}
+# CHECK-NEXT:     t2SG
-- 
GitLab


From 0822780b7ffa552b351364be350a997a47f22250 Mon Sep 17 00:00:00 2001
From: wanglei <wanglei@loongson.cn>
Date: Mon, 15 Apr 2024 18:01:09 +0800
Subject: [PATCH 219/464] [LoongArch] Fix incorrect logic in
 isLegalAddressingMode() (#88694)

This will adress issue:
https://github.com/ClangBuiltLinux/linux/issues/2014
---
 .../Target/LoongArch/LoongArchISelLowering.cpp    | 15 +++++++--------
 llvm/test/CodeGen/LoongArch/gep-imm.ll            | 14 +++++---------
 2 files changed, 12 insertions(+), 17 deletions(-)

diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index c13b10a320f8..285d5c2a63b2 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -4876,20 +4876,19 @@ bool LoongArchTargetLowering::isLegalAddressingMode(const DataLayout &DL,
   if (AM.BaseGV)
     return false;
 
-  // Require a 12 or 14 bit signed offset.
-  if (!isInt<12>(AM.BaseOffs) || !isShiftedInt<14, 2>(AM.BaseOffs))
+  // Require a 12-bit signed offset or 14-bit signed offset left-shifted by 2
+  // with `UAL` feature.
+  if (!isInt<12>(AM.BaseOffs) &&
+      !(isShiftedInt<14, 2>(AM.BaseOffs) && Subtarget.hasUAL()))
     return false;
 
   switch (AM.Scale) {
   case 0:
-    // "i" is not allowed.
-    if (!AM.HasBaseReg)
-      return false;
-    // Otherwise we have "r+i".
+    // "r+i" or just "i", depending on HasBaseReg.
     break;
   case 1:
     // "r+r+i" is not allowed.
-    if (AM.HasBaseReg && AM.BaseOffs != 0)
+    if (AM.HasBaseReg && AM.BaseOffs)
       return false;
     // Otherwise we have "r+r" or "r+i".
     break;
@@ -4897,7 +4896,7 @@ bool LoongArchTargetLowering::isLegalAddressingMode(const DataLayout &DL,
     // "2*r+r" or "2*r+i" is not allowed.
     if (AM.HasBaseReg || AM.BaseOffs)
       return false;
-    // Otherwise we have "r+r".
+    // Allow "2*r" as "r+r".
     break;
   default:
     return false;
diff --git a/llvm/test/CodeGen/LoongArch/gep-imm.ll b/llvm/test/CodeGen/LoongArch/gep-imm.ll
index 52d25f9d6389..0eef7e4517f3 100644
--- a/llvm/test/CodeGen/LoongArch/gep-imm.ll
+++ b/llvm/test/CodeGen/LoongArch/gep-imm.ll
@@ -4,12 +4,8 @@
 define void @test(ptr %sp, ptr %t, i32 %n) {
 ; CHECK-LABEL: test:
 ; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    ld.d $a0, $a0, 0
 ; CHECK-NEXT:    move $a3, $zero
-; CHECK-NEXT:    ld.d $a4, $a0, 0
-; CHECK-NEXT:    lu12i.w $a0, 1
-; CHECK-NEXT:    ori $a5, $a0, 3904
-; CHECK-NEXT:    add.d $a0, $a1, $a5
-; CHECK-NEXT:    add.d $a1, $a4, $a5
 ; CHECK-NEXT:    addi.w $a2, $a2, 0
 ; CHECK-NEXT:    addi.w $a4, $a3, 0
 ; CHECK-NEXT:    bge $a4, $a2, .LBB0_2
@@ -17,10 +13,10 @@ define void @test(ptr %sp, ptr %t, i32 %n) {
 ; CHECK-NEXT:  .LBB0_1: # %while_body
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    addi.d $a4, $a3, 1
-; CHECK-NEXT:    st.w $a4, $a1, 0
-; CHECK-NEXT:    st.w $a3, $a1, 4
-; CHECK-NEXT:    st.w $a4, $a0, 0
-; CHECK-NEXT:    st.w $a3, $a0, 4
+; CHECK-NEXT:    stptr.w $a4, $a0, 8000
+; CHECK-NEXT:    stptr.w $a3, $a0, 8004
+; CHECK-NEXT:    stptr.w $a4, $a1, 8000
+; CHECK-NEXT:    stptr.w $a3, $a1, 8004
 ; CHECK-NEXT:    move $a3, $a4
 ; CHECK-NEXT:    addi.w $a4, $a3, 0
 ; CHECK-NEXT:    blt $a4, $a2, .LBB0_1
-- 
GitLab


From 33e60f358cc0c564e771c0234dd8a60f3bddce5d Mon Sep 17 00:00:00 2001
From: Johannes Reifferscheid <jreiffers@google.com>
Date: Mon, 15 Apr 2024 12:04:36 +0200
Subject: [PATCH 220/464] Fix rsqrt inaccuracies. (#88707)

The current lowering has issues with large/subnormal values. This po
XLA's lowering and was verified using XLA's test suite and the
MLIR-based emitters.

This updates https://github.com/llvm/llvm-project/pull/88691 to also
update the correctness test for rsqrt(0). I checked C++ and Python, they
both agree the result should be (inf, nan). Updated the correctness test
to match this.
---
 .../ComplexToStandard/ComplexToStandard.cpp   | 88 ++++++++++++++++---
 .../convert-to-standard.mlir                  | 17 +++-
 .../Dialect/Complex/CPU/correctness.mlir      |  2 +-
 3 files changed, 93 insertions(+), 14 deletions(-)

diff --git a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
index 49eb575212ff..3ebee9baff31 100644
--- a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
+++ b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
@@ -27,9 +27,11 @@ using namespace mlir;
 
 namespace {
 
-// Returns the absolute value or its square root.
+enum class AbsFn { abs, sqrt, rsqrt };
+
+// Returns the absolute value, its square root or its reciprocal square root.
 Value computeAbs(Value real, Value imag, arith::FastMathFlags fmf,
-                 ImplicitLocOpBuilder &b, bool returnSqrt = false) {
+                 ImplicitLocOpBuilder &b, AbsFn fn = AbsFn::abs) {
   Value one = b.create<arith::ConstantOp>(real.getType(),
                                           b.getFloatAttr(real.getType(), 1.0));
 
@@ -43,7 +45,13 @@ Value computeAbs(Value real, Value imag, arith::FastMathFlags fmf,
   Value ratioSqPlusOne = b.create<arith::AddFOp>(ratioSq, one, fmf);
   Value result;
 
-  if (returnSqrt) {
+  if (fn == AbsFn::rsqrt) {
+    ratioSqPlusOne = b.create<math::RsqrtOp>(ratioSqPlusOne, fmf);
+    min = b.create<math::RsqrtOp>(min, fmf);
+    max = b.create<math::RsqrtOp>(max, fmf);
+  }
+
+  if (fn == AbsFn::sqrt) {
     Value quarter = b.create<arith::ConstantOp>(
         real.getType(), b.getFloatAttr(real.getType(), 0.25));
     // sqrt(sqrt(a*b)) would avoid the pow, but will overflow more easily.
@@ -863,7 +871,7 @@ struct SqrtOpConversion : public OpConversionPattern<complex::SqrtOp> {
 
     Value real = b.create<complex::ReOp>(elementType, adaptor.getComplex());
     Value imag = b.create<complex::ImOp>(elementType, adaptor.getComplex());
-    Value absSqrt = computeAbs(real, imag, fmf, b, /*returnSqrt=*/true);
+    Value absSqrt = computeAbs(real, imag, fmf, b, AbsFn::sqrt);
     Value argArg = b.create<math::Atan2Op>(imag, real, fmf);
     Value sqrtArg = b.create<arith::MulFOp>(argArg, half, fmf);
     Value cos = b.create<math::CosOp>(sqrtArg, fmf);
@@ -1147,18 +1155,74 @@ struct RsqrtOpConversion : public OpConversionPattern<complex::RsqrtOp> {
   LogicalResult
   matchAndRewrite(complex::RsqrtOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
-    mlir::ImplicitLocOpBuilder builder(op.getLoc(), rewriter);
+    mlir::ImplicitLocOpBuilder b(op.getLoc(), rewriter);
     auto type = cast<ComplexType>(adaptor.getComplex().getType());
     auto elementType = cast<FloatType>(type.getElementType());
 
-    Value c = builder.create<arith::ConstantOp>(
-        elementType, builder.getFloatAttr(elementType, -0.5));
-    Value d = builder.create<arith::ConstantOp>(
-        elementType, builder.getFloatAttr(elementType, 0));
+    arith::FastMathFlags fmf = op.getFastMathFlagsAttr().getValue();
+
+    auto cst = [&](APFloat v) {
+      return b.create<arith::ConstantOp>(elementType,
+                                         b.getFloatAttr(elementType, v));
+    };
+    const auto &floatSemantics = elementType.getFloatSemantics();
+    Value zero = cst(APFloat::getZero(floatSemantics));
+    Value inf = cst(APFloat::getInf(floatSemantics));
+    Value negHalf = b.create<arith::ConstantOp>(
+        elementType, b.getFloatAttr(elementType, -0.5));
+    Value nan = cst(APFloat::getNaN(floatSemantics));
+
+    Value real = b.create<complex::ReOp>(elementType, adaptor.getComplex());
+    Value imag = b.create<complex::ImOp>(elementType, adaptor.getComplex());
+    Value absRsqrt = computeAbs(real, imag, fmf, b, AbsFn::rsqrt);
+    Value argArg = b.create<math::Atan2Op>(imag, real, fmf);
+    Value rsqrtArg = b.create<arith::MulFOp>(argArg, negHalf, fmf);
+    Value cos = b.create<math::CosOp>(rsqrtArg, fmf);
+    Value sin = b.create<math::SinOp>(rsqrtArg, fmf);
+
+    Value resultReal = b.create<arith::MulFOp>(absRsqrt, cos, fmf);
+    Value resultImag = b.create<arith::MulFOp>(absRsqrt, sin, fmf);
+
+    if (!arith::bitEnumContainsAll(fmf, arith::FastMathFlags::nnan |
+                                            arith::FastMathFlags::ninf)) {
+      Value negOne = b.create<arith::ConstantOp>(
+          elementType, b.getFloatAttr(elementType, -1));
+
+      Value realSignedZero = b.create<math::CopySignOp>(zero, real, fmf);
+      Value imagSignedZero = b.create<math::CopySignOp>(zero, imag, fmf);
+      Value negImagSignedZero =
+          b.create<arith::MulFOp>(negOne, imagSignedZero, fmf);
 
-    rewriter.replaceOp(op,
-                       {powOpConversionImpl(builder, type, adaptor.getComplex(),
-                                            c, d, op.getFastmath())});
+      Value absReal = b.create<math::AbsFOp>(real, fmf);
+      Value absImag = b.create<math::AbsFOp>(imag, fmf);
+
+      Value absImagIsInf =
+          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absImag, inf, fmf);
+      Value realIsNan =
+          b.create<arith::CmpFOp>(arith::CmpFPredicate::UNO, real, real, fmf);
+      Value realIsInf =
+          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absReal, inf, fmf);
+      Value inIsNanInf = b.create<arith::AndIOp>(absImagIsInf, realIsNan);
+
+      Value resultIsZero = b.create<arith::OrIOp>(inIsNanInf, realIsInf);
+
+      resultReal =
+          b.create<arith::SelectOp>(resultIsZero, realSignedZero, resultReal);
+      resultImag = b.create<arith::SelectOp>(resultIsZero, negImagSignedZero,
+                                             resultImag);
+    }
+
+    Value isRealZero =
+        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, real, zero, fmf);
+    Value isImagZero =
+        b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, imag, zero, fmf);
+    Value isZero = b.create<arith::AndIOp>(isRealZero, isImagZero);
+
+    resultReal = b.create<arith::SelectOp>(isZero, inf, resultReal);
+    resultImag = b.create<arith::SelectOp>(isZero, nan, resultImag);
+
+    rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
+                                                   resultImag);
     return success();
   }
 };
diff --git a/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir b/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir
index e0e7cdadd317..8b4ea9777f79 100644
--- a/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir
+++ b/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir
@@ -837,6 +837,21 @@ func.func @complex_rsqrt(%arg: complex<f32>) -> complex<f32> {
   return %rsqrt : complex<f32>
 }
 
+// CHECK-COUNT-5: arith.select
+// CHECK-NOT: arith.select
+
+// -----
+
+// CHECK-LABEL: func @complex_rsqrt_nnan_ninf
+// CHECK-SAME: %[[ARG:.*]]: complex<f32>
+func.func @complex_rsqrt_nnan_ninf(%arg: complex<f32>) -> complex<f32> {
+  %sqrt = complex.rsqrt %arg fastmath<nnan,ninf> : complex<f32>
+  return %sqrt : complex<f32>
+}
+
+// CHECK-COUNT-3: arith.select
+// CHECK-NOT: arith.select
+
 // -----
 
 // CHECK-LABEL:   func.func @complex_angle
@@ -2103,4 +2118,4 @@ func.func @complex_tanh_with_fmf(%arg: complex<f32>) -> complex<f32> {
 // CHECK: %[[NUM:.*]] = complex.create %[[TANH_A]], %[[TAN_B]] : complex<f32>
 // CHECK: %[[ONE:.*]] = arith.constant 1.000000e+00 : f32
 // CHECK: %[[MUL:.*]] = arith.mulf %[[TANH_A]], %[[TAN_B]] fastmath<nnan,contract> : f32
-// CHECK: %[[DENOM:.*]] = complex.create %[[ONE]], %[[MUL]] : complex<f32>
\ No newline at end of file
+// CHECK: %[[DENOM:.*]] = complex.create %[[ONE]], %[[MUL]] : complex<f32>
diff --git a/mlir/test/Integration/Dialect/Complex/CPU/correctness.mlir b/mlir/test/Integration/Dialect/Complex/CPU/correctness.mlir
index 441f7500538f..b0e414d15726 100644
--- a/mlir/test/Integration/Dialect/Complex/CPU/correctness.mlir
+++ b/mlir/test/Integration/Dialect/Complex/CPU/correctness.mlir
@@ -242,7 +242,7 @@ func.func @entry() {
     // CHECK-NEXT:  0.321
     // CHECK-NEXT:  -0.776
     (0.0, 0.0),
-    // CHECK-NEXT:  nan
+    // CHECK-NEXT:  inf
     // CHECK-NEXT:  nan
     (0.0, 1.0),
     // CHECK-NEXT:  0.707
-- 
GitLab


From 5b95c9e0cad189cc609c31029f6bc3a89bc7b612 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Markus=20B=C3=B6ck?= <markus.boeck02@gmail.com>
Date: Mon, 15 Apr 2024 12:13:06 +0200
Subject: [PATCH 221/464] [mlir][ODS] Make `prop-dict` behave closer to
 `attr-dict` (#88659)

`attr-dict` currently prints any attribute (inherent or discardable)
that does not occur elsewhere in the assembly format. `prop-dict` on the
other hand, always prints and parses all properties (including inherent
attributes stored as properties) as part of the property dictionary,
regardless of whether the properties or attributes are used elsewhere.
(with the exception of default-valued attributes implemented recently in
https://github.com/llvm/llvm-project/pull/87970).

This PR changes the behavior of `prop-dict` to only print and parse
attributes and properties that do not occur elsewhere in the assembly
format. This is achieved by 1) adding used attributes and properties to
the elision list when printing and 2) using a custom version of
`setPropertiesFromAttr` called `setPropertiesFromParsedAttr` that is
sensitive to the assembly format and auto-generated by ODS.

The current and new behavior of `prop-dict` and `attr-dict` were also
documented.

Happens to also fix https://github.com/llvm/llvm-project/issues/88506
---
 mlir/docs/DefiningDialects/Operations.md   |  12 ++-
 mlir/include/mlir/IR/OpDefinition.h        |  32 +++++-
 mlir/test/lib/Dialect/Test/TestDialect.cpp |  11 ++
 mlir/test/lib/Dialect/Test/TestOps.td      |  26 +++++
 mlir/test/mlir-tblgen/op-format.mlir       |  11 ++
 mlir/tools/mlir-tblgen/OpFormatGen.cpp     | 118 ++++++++++++++++++++-
 6 files changed, 202 insertions(+), 8 deletions(-)

diff --git a/mlir/docs/DefiningDialects/Operations.md b/mlir/docs/DefiningDialects/Operations.md
index b27330319f65..729393d53626 100644
--- a/mlir/docs/DefiningDialects/Operations.md
+++ b/mlir/docs/DefiningDialects/Operations.md
@@ -640,13 +640,23 @@ The available directives are as follows:
 
 *   `attr-dict`
 
-    -   Represents the attribute dictionary of the operation.
+    -   Represents the attribute dictionary of the operation. Any inherent 
+    -   attributes that are not used elsewhere in the format are printed as
+    -   part of the attribute dictionary unless a `prop-dict` is present.
+    -   Discardable attributes are always part of the `attr-dict`.  
 
 *   `attr-dict-with-keyword`
 
     -   Represents the attribute dictionary of the operation, but prefixes the
         dictionary with an `attributes` keyword.
 
+*   `prop-dict`
+
+    -   Represents the properties of the operation converted to a dictionary.
+    -   Any property or inherent attribute that are not used elsewhere in the
+    -   format are parsed and printed as part of this dictionary.
+    -   If present, the `attr-dict` will not contain any inherent attributes.
+
 *   `custom` < UserDirective > ( Params )
 
     -   Represents a custom directive implemented by the user in C++.
diff --git a/mlir/include/mlir/IR/OpDefinition.h b/mlir/include/mlir/IR/OpDefinition.h
index f4355c8ce26a..59f094d66909 100644
--- a/mlir/include/mlir/IR/OpDefinition.h
+++ b/mlir/include/mlir/IR/OpDefinition.h
@@ -1993,8 +1993,10 @@ public:
         p, ConcreteType::getPropertiesAsAttr(ctx, properties), elidedProps);
   }
 
-  /// Parser the properties. Unless overridden, this method will print by
-  /// converting the properties to an Attribute.
+  /// Parses 'prop-dict' for the operation. Unless overridden, the method will
+  /// parse the properties using the generic property dictionary using the
+  /// '<{ ... }>' syntax. The resulting properties are stored within the
+  /// property structure of 'result', accessible via 'getOrAddProperties'.
   template <typename T = ConcreteType>
   static ParseResult parseProperties(OpAsmParser &parser,
                                      OperationState &result) {
@@ -2002,7 +2004,31 @@ public:
       return parseProperties(
           parser, result.getOrAddProperties<InferredProperties<T>>());
     }
-    return genericParseProperties(parser, result.propertiesAttr);
+
+    Attribute propertyDictionary;
+    if (genericParseProperties(parser, propertyDictionary))
+      return failure();
+
+    // The generated 'setPropertiesFromParsedAttr', like
+    // 'setPropertiesFromAttr', expects a 'DictionaryAttr' that is not null.
+    // Use an empty dictionary in the case that the whole dictionary is
+    // optional.
+    if (!propertyDictionary)
+      propertyDictionary = DictionaryAttr::get(result.getContext());
+
+    auto emitError = [&]() {
+      return mlir::emitError(result.location, "invalid properties ")
+             << propertyDictionary << " for op " << result.name.getStringRef()
+             << ": ";
+    };
+
+    // Copy the data from the dictionary attribute into the property struct of
+    // the operation. This method is generated by ODS by default if there are
+    // any occurrences of 'prop-dict' in the assembly format and should set
+    // any properties that aren't parsed elsewhere.
+    return ConcreteOpType::setPropertiesFromParsedAttr(
+        result.getOrAddProperties<InferredProperties<T>>(), propertyDictionary,
+        emitError);
   }
 
 private:
diff --git a/mlir/test/lib/Dialect/Test/TestDialect.cpp b/mlir/test/lib/Dialect/Test/TestDialect.cpp
index 380c74a47e50..74378633032c 100644
--- a/mlir/test/lib/Dialect/Test/TestDialect.cpp
+++ b/mlir/test/lib/Dialect/Test/TestDialect.cpp
@@ -740,6 +740,17 @@ LogicalResult OpWithResultShapePerDimInterfaceOp::reifyResultShapes(
   return success();
 }
 
+LogicalResult TestOpWithPropertiesAndInferredType::inferReturnTypes(
+    MLIRContext *context, std::optional<Location>, ValueRange operands,
+    DictionaryAttr attributes, OpaqueProperties properties, RegionRange regions,
+    SmallVectorImpl<Type> &inferredReturnTypes) {
+
+  Adaptor adaptor(operands, attributes, properties, regions);
+  inferredReturnTypes.push_back(IntegerType::get(
+      context, adaptor.getLhs() + adaptor.getProperties().rhs));
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // Test SideEffect interfaces
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/lib/Dialect/Test/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td
index edca05fde5a5..c88f85b8b6b3 100644
--- a/mlir/test/lib/Dialect/Test/TestOps.td
+++ b/mlir/test/lib/Dialect/Test/TestOps.td
@@ -2852,6 +2852,23 @@ def TestOpWithProperties : TEST_Op<"with_properties"> {
   );
 }
 
+def TestOpWithPropertiesAndAttr
+  : TEST_Op<"with_properties_and_attr"> {
+  let assemblyFormat = "$lhs prop-dict attr-dict";
+
+  let arguments = (ins I32Attr:$lhs, IntProperty<"int64_t">:$rhs);
+}
+
+def TestOpWithPropertiesAndInferredType
+  : TEST_Op<"with_properties_and_inferred_type", [
+    DeclareOpInterfaceMethods<InferTypeOpInterface>
+  ]> {
+  let assemblyFormat = "$lhs prop-dict attr-dict";
+
+  let arguments = (ins I32Attr:$lhs, IntProperty<"int64_t">:$rhs);
+  let results = (outs AnyType:$result);
+}
+
 // Demonstrate how to wrap an existing C++ class named MyPropStruct.
 def MyStructProperty : Property<"MyPropStruct"> {
   let convertToAttribute = "$_storage.asAttribute($_ctxt)";
@@ -2871,6 +2888,15 @@ def TestOpUsingPropertyInCustom : TEST_Op<"using_property_in_custom"> {
   let arguments = (ins ArrayProperty<"int64_t", 3>:$prop);
 }
 
+def TestOpUsingPropertyInCustomAndOther
+  : TEST_Op<"using_property_in_custom_and_other"> {
+  let assemblyFormat = "custom<UsingPropertyInCustom>($prop) prop-dict attr-dict";
+  let arguments = (ins
+    ArrayProperty<"int64_t", 3>:$prop,
+    IntProperty<"int64_t">:$other
+  );
+}
+
 def TestOpUsingPropertyRefInCustom : TEST_Op<"using_property_ref_in_custom"> {
   let assemblyFormat = "custom<IntProperty>($first) `+` custom<SumProperty>($second, ref($first)) attr-dict";
   let arguments = (ins IntProperty<"int64_t">:$first, IntProperty<"int64_t">:$second);
diff --git a/mlir/test/mlir-tblgen/op-format.mlir b/mlir/test/mlir-tblgen/op-format.mlir
index 14e1cdb07db3..46d272649cae 100644
--- a/mlir/test/mlir-tblgen/op-format.mlir
+++ b/mlir/test/mlir-tblgen/op-format.mlir
@@ -480,6 +480,17 @@ test.format_infer_variadic_type_from_non_variadic %i64, %i64 : i64
 // CHECK: test.format_infer_type_variadic_operands(%[[I32]], %[[I32]] : i32, i32) (%[[I64]], %[[I64]] : i64, i64)
 %ignored_res13:4 = test.format_infer_type_variadic_operands(%i32, %i32 : i32, i32) (%i64, %i64 : i64, i64)
 
+// CHECK: test.with_properties_and_attr 16 < {rhs = 16 : i64}>
+test.with_properties_and_attr 16 <{rhs = 16 : i64}>
+
+// CHECK: test.with_properties_and_inferred_type 16 < {rhs = 16 : i64}>
+%should_be_i32 = test.with_properties_and_inferred_type 16 <{rhs = 16 : i64}>
+// Assert through the verifier that its inferred as i32.
+test.format_all_types_match_var %should_be_i32, %i32 : i32
+
+// CHECK: test.using_property_in_custom_and_other [1, 4, 20] < {other = 16 : i64}>
+test.using_property_in_custom_and_other [1, 4, 20] <{other = 16 : i64}>
+
 //===----------------------------------------------------------------------===//
 // Check DefaultValuedStrAttr
 //===----------------------------------------------------------------------===//
diff --git a/mlir/tools/mlir-tblgen/OpFormatGen.cpp b/mlir/tools/mlir-tblgen/OpFormatGen.cpp
index 5963b5e689da..806991035e66 100644
--- a/mlir/tools/mlir-tblgen/OpFormatGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpFormatGen.cpp
@@ -379,8 +379,11 @@ struct OperationFormat {
   std::vector<TypeResolution> operandTypes, resultTypes;
 
   /// The set of attributes explicitly used within the format.
-  SmallVector<const NamedAttribute *, 8> usedAttributes;
+  llvm::SmallSetVector<const NamedAttribute *, 8> usedAttributes;
   llvm::StringSet<> inferredAttributes;
+
+  /// The set of properties explicitly used within the format.
+  llvm::SmallSetVector<const NamedProperty *, 8> usedProperties;
 };
 } // namespace
 
@@ -1183,6 +1186,105 @@ static void genAttrParser(AttributeVariable *attr, MethodBody &body,
   }
 }
 
+// Generates the 'setPropertiesFromParsedAttr' used to set properties from a
+// 'prop-dict' dictionary attr.
+static void genParsedAttrPropertiesSetter(OperationFormat &fmt, Operator &op,
+                                          OpClass &opClass) {
+  // Not required unless 'prop-dict' is present.
+  if (!fmt.hasPropDict)
+    return;
+
+  SmallVector<MethodParameter> paramList;
+  paramList.emplace_back("Properties &", "prop");
+  paramList.emplace_back("::mlir::Attribute", "attr");
+  paramList.emplace_back("::llvm::function_ref<::mlir::InFlightDiagnostic()>",
+                         "emitError");
+
+  Method *method = opClass.addStaticMethod("::mlir::LogicalResult",
+                                           "setPropertiesFromParsedAttr",
+                                           std::move(paramList));
+  MethodBody &body = method->body().indent();
+
+  body << R"decl(
+::mlir::DictionaryAttr dict = ::llvm::dyn_cast<::mlir::DictionaryAttr>(attr);
+if (!dict) {
+  emitError() << "expected DictionaryAttr to set properties";
+  return ::mlir::failure();
+}
+)decl";
+
+  // TODO: properties might be optional as well.
+  const char *propFromAttrFmt = R"decl(
+auto setFromAttr = [] (auto &propStorage, ::mlir::Attribute propAttr,
+         ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {{
+  {0};
+};
+auto attr = dict.get("{1}");
+if (!attr) {{
+  emitError() << "expected key entry for {1} in DictionaryAttr to set "
+             "Properties.";
+  return ::mlir::failure();
+}
+if (::mlir::failed(setFromAttr(prop.{1}, attr, emitError)))
+  return ::mlir::failure();
+)decl";
+
+  // Generate the setter for any property not parsed elsewhere.
+  for (const NamedProperty &namedProperty : op.getProperties()) {
+    if (fmt.usedProperties.contains(&namedProperty))
+      continue;
+
+    auto scope = body.scope("{\n", "}\n", /*indent=*/true);
+
+    StringRef name = namedProperty.name;
+    const Property &prop = namedProperty.prop;
+    FmtContext fctx;
+    body << formatv(propFromAttrFmt,
+                    tgfmt(prop.getConvertFromAttributeCall(),
+                          &fctx.addSubst("_attr", "propAttr")
+                               .addSubst("_storage", "propStorage")
+                               .addSubst("_diag", "emitError")),
+                    name);
+  }
+
+  // Generate the setter for any attribute not parsed elsewhere.
+  for (const NamedAttribute &namedAttr : op.getAttributes()) {
+    if (fmt.usedAttributes.contains(&namedAttr))
+      continue;
+
+    const Attribute &attr = namedAttr.attr;
+    // Derived attributes do not need to be parsed.
+    if (attr.isDerivedAttr())
+      continue;
+
+    auto scope = body.scope("{\n", "}\n", /*indent=*/true);
+
+    // If the attribute has a default value or is optional, it does not need to
+    // be present in the parsed dictionary attribute.
+    bool isRequired = !attr.isOptional() && !attr.hasDefaultValue();
+    body << formatv(R"decl(
+auto &propStorage = prop.{0};
+auto attr = dict.get("{0}");
+if (attr || /*isRequired=*/{1}) {{
+  if (!attr) {{
+    emitError() << "expected key entry for {0} in DictionaryAttr to set "
+               "Properties.";
+    return ::mlir::failure();
+  }
+  auto convertedAttr = ::llvm::dyn_cast<std::remove_reference_t<decltype(propStorage)>>(attr);
+  if (convertedAttr) {{
+    propStorage = convertedAttr;
+  } else {{
+    emitError() << "Invalid attribute `{0}` in property conversion: " << attr;
+    return ::mlir::failure();
+  }
+}
+)decl",
+                    namedAttr.name, isRequired);
+  }
+  body << "return ::mlir::success();\n";
+}
+
 void OperationFormat::genParser(Operator &op, OpClass &opClass) {
   SmallVector<MethodParameter> paramList;
   paramList.emplace_back("::mlir::OpAsmParser &", "parser");
@@ -1214,6 +1316,8 @@ void OperationFormat::genParser(Operator &op, OpClass &opClass) {
   genParserTypeResolution(op, body);
 
   body << "  return ::mlir::success();\n";
+
+  genParsedAttrPropertiesSetter(*this, op, opClass);
 }
 
 void OperationFormat::genElementParser(FormatElement *element, MethodBody &body,
@@ -1776,6 +1880,11 @@ const char *enumAttrBeginPrinterCode = R"(
 static void genPropDictPrinter(OperationFormat &fmt, Operator &op,
                                MethodBody &body) {
   body << "  ::llvm::SmallVector<::llvm::StringRef, 2> elidedProps;\n";
+  for (const NamedProperty *namedProperty : fmt.usedProperties)
+    body << "  elidedProps.push_back(\"" << namedProperty->name << "\");\n";
+  for (const NamedAttribute *namedAttr : fmt.usedAttributes)
+    body << "  elidedProps.push_back(\"" << namedAttr->name << "\");\n";
+
   // Add code to check attributes for equality with the default value
   // for attributes with the elidePrintingDefaultValue bit set.
   for (const NamedAttribute &namedAttr : op.getAttributes()) {
@@ -2543,7 +2652,7 @@ private:
   llvm::DenseSet<const NamedTypeConstraint *> seenOperands;
   llvm::DenseSet<const NamedRegion *> seenRegions;
   llvm::DenseSet<const NamedSuccessor *> seenSuccessors;
-  llvm::DenseSet<const NamedProperty *> seenProperties;
+  llvm::SmallSetVector<const NamedProperty *, 8> seenProperties;
 };
 } // namespace
 
@@ -2589,7 +2698,8 @@ LogicalResult OpFormatParser::verify(SMLoc loc,
     return failure();
 
   // Collect the set of used attributes in the format.
-  fmt.usedAttributes = seenAttrs.takeVector();
+  fmt.usedAttributes = std::move(seenAttrs);
+  fmt.usedProperties = std::move(seenProperties);
 
   // Set whether prop-dict is used in the format
   fmt.hasPropDict = hasPropDict;
@@ -3042,7 +3152,7 @@ OpFormatParser::parseVariableImpl(SMLoc loc, StringRef name, Context ctx) {
         return emitError(loc, "property '" + name +
                                   "' must be bound before it is referenced");
     } else {
-      if (!seenProperties.insert(property).second)
+      if (!seenProperties.insert(property))
         return emitError(loc, "property '" + name + "' is already bound");
     }
 
-- 
GitLab


From 7dbba39e583a3fd64e7e6b947251c035e483f054 Mon Sep 17 00:00:00 2001
From: Julian Nagele <j.nagele@apple.com>
Date: Mon, 15 Apr 2024 11:25:05 +0100
Subject: [PATCH 222/464] Reapply "[TBAA] Add verifier for tbaa.struct metadata
 (#86709)"

This reverts commit b9cd48f96acdd07c627ccafbf4386a1f3dcd6c51.

-------------------------------------------------------------
Original commit message:

Adds logic to the IR verifier that checks whether !tbaa.struct nodes are
well-formed. That is, it checks that the operands of !tbaa.struct nodes
are in groups of three, that each group of three operands consists of
two integers and a valid tbaa node, and that the regions described by
the offset and size operands are non-overlapping.

PR: https://github.com/llvm/llvm-project/pull/86709
---
 llvm/include/llvm/IR/Verifier.h               |  1 +
 llvm/lib/IR/Verifier.cpp                      | 32 +++++++++++++++++++
 llvm/test/CodeGen/AArch64/arm64-abi_align.ll  |  4 ++-
 .../AMDGPU/mem-intrinsics.ll                  |  2 +-
 .../InstCombine/struct-assign-tbaa.ll         |  2 +-
 llvm/test/Transforms/SROA/tbaa-struct3.ll     |  2 +-
 .../Scalarizer/basic-inseltpoison.ll          |  3 +-
 llvm/test/Transforms/Scalarizer/basic.ll      |  3 +-
 llvm/test/Verifier/tbaa-struct.ll             | 14 ++++++--
 9 files changed, 54 insertions(+), 9 deletions(-)

diff --git a/llvm/include/llvm/IR/Verifier.h b/llvm/include/llvm/IR/Verifier.h
index b25f8eb77ee3..b7db6e0bbfb7 100644
--- a/llvm/include/llvm/IR/Verifier.h
+++ b/llvm/include/llvm/IR/Verifier.h
@@ -77,6 +77,7 @@ public:
   /// Visit an instruction and return true if it is valid, return false if an
   /// invalid TBAA is attached.
   bool visitTBAAMetadata(Instruction &I, const MDNode *MD);
+  bool visitTBAAStructMetadata(Instruction &I, const MDNode *MD);
 };
 
 /// Check a function for errors, useful for use when debugging a
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 25cb99f0f9bd..516d4a051556 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -5122,6 +5122,9 @@ void Verifier::visitInstruction(Instruction &I) {
   if (MDNode *TBAA = I.getMetadata(LLVMContext::MD_tbaa))
     TBAAVerifyHelper.visitTBAAMetadata(I, TBAA);
 
+  if (MDNode *TBAA = I.getMetadata(LLVMContext::MD_tbaa_struct))
+    TBAAVerifyHelper.visitTBAAStructMetadata(I, TBAA);
+
   if (MDNode *MD = I.getMetadata(LLVMContext::MD_noalias))
     visitAliasScopeListMetadata(MD);
   if (MDNode *MD = I.getMetadata(LLVMContext::MD_alias_scope))
@@ -7453,6 +7456,35 @@ bool TBAAVerifier::visitTBAAMetadata(Instruction &I, const MDNode *MD) {
   return true;
 }
 
+bool TBAAVerifier::visitTBAAStructMetadata(Instruction &I, const MDNode *MD) {
+  CheckTBAA(MD->getNumOperands() % 3 == 0,
+            "tbaa.struct operands must occur in groups of three", &I, MD);
+
+  // Each group of three operands must consist of two integers and a
+  // tbaa node. Moreover, the regions described by the offset and size
+  // operands must be non-overlapping.
+  std::optional<APInt> NextFree;
+  for (unsigned int Idx = 0; Idx < MD->getNumOperands(); Idx += 3) {
+    auto *OffsetCI =
+        mdconst::dyn_extract_or_null<ConstantInt>(MD->getOperand(Idx));
+    CheckTBAA(OffsetCI, "Offset must be a constant integer", &I, MD);
+
+    auto *SizeCI =
+        mdconst::dyn_extract_or_null<ConstantInt>(MD->getOperand(Idx + 1));
+    CheckTBAA(SizeCI, "Size must be a constant integer", &I, MD);
+
+    MDNode *TBAA = dyn_cast_or_null<MDNode>(MD->getOperand(Idx + 2));
+    CheckTBAA(TBAA, "TBAA tag missing", &I, MD);
+    visitTBAAMetadata(I, TBAA);
+
+    bool NonOverlapping = !NextFree || NextFree->ule(OffsetCI->getValue());
+    CheckTBAA(NonOverlapping, "Overlapping tbaa.struct regions", &I, MD);
+
+    NextFree = OffsetCI->getValue() + SizeCI->getValue();
+  }
+  return true;
+}
+
 char VerifierLegacyPass::ID = 0;
 INITIALIZE_PASS(VerifierLegacyPass, "verify", "Module Verifier", false, false)
 
diff --git a/llvm/test/CodeGen/AArch64/arm64-abi_align.ll b/llvm/test/CodeGen/AArch64/arm64-abi_align.ll
index 089e171e5a4a..c9fd2d38e27a 100644
--- a/llvm/test/CodeGen/AArch64/arm64-abi_align.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-abi_align.ll
@@ -518,4 +518,6 @@ attributes #5 = { nobuiltin }
 !1 = !{!"omnipotent char", !2}
 !2 = !{!"Simple C/C++ TBAA"}
 !3 = !{!"short", !1}
-!4 = !{i64 0, i64 4, !0, i64 4, i64 2, !3, i64 8, i64 4, !0, i64 12, i64 2, !3, i64 16, i64 4, !0, i64 20, i64 2, !3}
+!4 = !{i64 0, i64 4, !5, i64 4, i64 2, !6, i64 8, i64 4, !5, i64 12, i64 2, !6, i64 16, i64 4, !5, i64 20, i64 2, !6}
+!5 = !{!0, !0, i64 0}
+!6 = !{!3, !3, i64 0}
diff --git a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/mem-intrinsics.ll b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/mem-intrinsics.ll
index 50b0e7a0f547..2f264a2432fc 100644
--- a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/mem-intrinsics.ll
+++ b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/mem-intrinsics.ll
@@ -141,4 +141,4 @@ attributes #1 = { argmemonly nounwind }
 !5 = distinct !{!5, !"some domain"}
 !6 = !{!7}
 !7 = distinct !{!7, !5, !"some scope 2"}
-!8 = !{i64 0, i64 8, null}
+!8 = !{i64 0, i64 8, !0}
diff --git a/llvm/test/Transforms/InstCombine/struct-assign-tbaa.ll b/llvm/test/Transforms/InstCombine/struct-assign-tbaa.ll
index 996d2c0e67e1..d079c03f1dcb 100644
--- a/llvm/test/Transforms/InstCombine/struct-assign-tbaa.ll
+++ b/llvm/test/Transforms/InstCombine/struct-assign-tbaa.ll
@@ -75,7 +75,7 @@ entry:
 !1 = !{!"omnipotent char", !0}
 !2 = !{!5, !5, i64 0}
 !3 = !{i64 0, i64 4, !2}
-!4 = !{i64 0, i64 8, null}
+!4 = !{i64 0, i64 8, !2}
 !5 = !{!"float", !0}
 !6 = !{i64 0, i64 4, !2, i64 4, i64 4, !2}
 !7 = !{i64 0, i64 2, !2, i64 4, i64 6, !2}
diff --git a/llvm/test/Transforms/SROA/tbaa-struct3.ll b/llvm/test/Transforms/SROA/tbaa-struct3.ll
index 0fcd787fef97..61034de81e4b 100644
--- a/llvm/test/Transforms/SROA/tbaa-struct3.ll
+++ b/llvm/test/Transforms/SROA/tbaa-struct3.ll
@@ -539,7 +539,7 @@ declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias
 !6 = !{!5, !5, i64 0}
 !7 = !{i64 0, i64 8, !6, i64 8, i64 4, !1}
 !8 = !{i64 0, i64 4, !1, i64 4, i64 8, !6}
-!9 = !{i64 0, i64 8, !6, i64 4, i64 8, !1}
+!9 = !{i64 0, i64 8, !6, i64 8, i64 8, !1}
 !10 = !{i64 0, i64 2, !1, i64 2, i64 2, !1}
 !11 = !{i64 0, i64 1, !1, i64 1, i64 3, !1}
 !12 = !{i64 0, i64 2, !1, i64 2, i64 6, !1}
diff --git a/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll b/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll
index bbcdcb6f5867..73ae66dd76c6 100644
--- a/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll
+++ b/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll
@@ -836,5 +836,6 @@ define <2 x i32> @f23_crash(<2 x i32> %srcvec, i32 %v1) {
 !2 = !{ !"set2", !0 }
 !3 = !{ !3, !{!"llvm.loop.parallel_accesses", !13} }
 !4 = !{ float 4.0 }
-!5 = !{ i64 0, i64 8, null }
+!5 = !{ i64 0, i64 8, !6 }
+!6 = !{ !1, !1, i64 0 }
 !13 = distinct !{}
diff --git a/llvm/test/Transforms/Scalarizer/basic.ll b/llvm/test/Transforms/Scalarizer/basic.ll
index db7c5f535f7e..87a70ccd3fc7 100644
--- a/llvm/test/Transforms/Scalarizer/basic.ll
+++ b/llvm/test/Transforms/Scalarizer/basic.ll
@@ -870,5 +870,6 @@ define <2 x float> @f25(<2 x float> %src) {
 !2 = !{ !"set2", !0 }
 !3 = !{ !3, !{!"llvm.loop.parallel_accesses", !13} }
 !4 = !{ float 4.0 }
-!5 = !{ i64 0, i64 8, null }
+!5 = !{ i64 0, i64 8, !6 }
+!6 = !{ !1, !1, i64 0 }
 !13 = distinct !{}
diff --git a/llvm/test/Verifier/tbaa-struct.ll b/llvm/test/Verifier/tbaa-struct.ll
index b8ddc7cee496..14c19a19d5ae 100644
--- a/llvm/test/Verifier/tbaa-struct.ll
+++ b/llvm/test/Verifier/tbaa-struct.ll
@@ -1,28 +1,36 @@
-; RUN: llvm-as < %s 2>&1
-
-; FIXME: The verifer should reject the invalid !tbaa.struct nodes below.
+; RUN: not llvm-as < %s 2>&1 | FileCheck %s
 
 define void @test_overlapping_regions(ptr %a1) {
+; CHECK: Overlapping tbaa.struct regions
+; CHECK-NEXT:  %ld = load i8, ptr %a1, align 1, !tbaa.struct !0
   %ld = load i8, ptr %a1, align 1, !tbaa.struct !0
   ret void
 }
 
 define void @test_size_not_integer(ptr %a1) {
+; CHECK: Size must be a constant integer
+; CHECK-NEXT:  store i8 1, ptr %a1, align 1, !tbaa.struct !5
   store i8 1, ptr %a1, align 1, !tbaa.struct !5
   ret void
 }
 
 define void @test_offset_not_integer(ptr %a1, ptr %a2) {
+; CHECK: Offset must be a constant integer
+; CHECK-NEXT:  tail call void @llvm.memcpy.p0.p0.i64(ptr align 8 %a1, ptr align 8 %a2, i64 16, i1 false), !tbaa.struct !6
   tail call void @llvm.memcpy.p0.p0.i64(ptr align 8 %a1, ptr align 8 %a2, i64 16, i1 false), !tbaa.struct !6
   ret void
 }
 
 define void @test_tbaa_missing(ptr %a1, ptr %a2) {
+; CHECK: TBAA tag missing
+; CHECK-NEXT:  tail call void @llvm.memcpy.p0.p0.i64(ptr align 8 %a1, ptr align 8 %a2, i64 16, i1 false), !tbaa.struct !7
   tail call void @llvm.memcpy.p0.p0.i64(ptr align 8 %a1, ptr align 8 %a2, i64 16, i1 false), !tbaa.struct !7
   ret void
 }
 
 define void @test_tbaa_invalid(ptr %a1) {
+; CHECK: Old-style TBAA is no longer allowed, use struct-path TBAA instead
+; CHECK-NEXT:  store i8 1, ptr %a1, align 1, !tbaa.struct !8
   store i8 1, ptr %a1, align 1, !tbaa.struct !8
   ret void
 }
-- 
GitLab


From c006b9088fa564fe343550a8dd5f9b0ad0c15b75 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Mon, 15 Apr 2024 11:30:36 +0100
Subject: [PATCH 223/464] [VPlan] Add test for dump of recipe not inserted in
 VPlan. (NFC)

Add extra tests for printing recipes not inserted in a VPlan yet, e.g.
when using a debugger.

Guard against regressions in changes to printing, i.e.
https://github.com/llvm/llvm-project/pull/81411.
---
 .../Transforms/Vectorize/VPlanTest.cpp        | 163 +++++++++++++++++-
 1 file changed, 162 insertions(+), 1 deletion(-)

diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index 73687846e281..ff0f482b833c 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -1227,7 +1227,7 @@ TEST(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) {
 }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-TEST(VPRecipeTest, dump) {
+TEST(VPRecipeTest, dumpRecipeInPlan) {
   VPBasicBlock *VPBB0 = new VPBasicBlock("preheader");
   VPBasicBlock *VPBB1 = new VPBasicBlock();
   VPlan Plan(VPBB0, VPBB1);
@@ -1280,6 +1280,167 @@ TEST(VPRecipeTest, dump) {
 
   delete AI;
 }
+
+TEST(VPRecipeTest, dumpRecipeUnnamedVPValuesInPlan) {
+  VPBasicBlock *VPBB0 = new VPBasicBlock("preheader");
+  VPBasicBlock *VPBB1 = new VPBasicBlock();
+  VPlan Plan(VPBB0, VPBB1);
+
+  LLVMContext C;
+
+  IntegerType *Int32 = IntegerType::get(C, 32);
+  SmallVector<VPValue *, 2> Args;
+  VPValue *ExtVPV1 = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
+  VPValue *ExtVPV2 = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2));
+  Args.push_back(ExtVPV1);
+  Args.push_back(ExtVPV2);
+  VPInstruction *I1 = new VPInstruction(Instruction::Add, {ExtVPV1, ExtVPV2});
+  VPInstruction *I2 = new VPInstruction(Instruction::Mul, {I1, I1});
+  VPBB1->appendRecipe(I1);
+  VPBB1->appendRecipe(I2);
+
+  // Check printing I1.
+  {
+    // Use EXPECT_EXIT to capture stderr and compare against expected output.
+    //
+    // Test VPValue::dump().
+    VPValue *VPV = I1;
+    EXPECT_EXIT(
+        {
+          VPV->dump();
+          exit(0);
+        },
+        testing::ExitedWithCode(0), "EMIT vp<%1> = add ir<1>, ir<2>");
+
+    // Test VPRecipeBase::dump().
+    VPRecipeBase *R = I1;
+    EXPECT_EXIT(
+        {
+          R->dump();
+          exit(0);
+        },
+        testing::ExitedWithCode(0), "EMIT vp<%1> = add ir<1>, ir<2>");
+
+    // Test VPDef::dump().
+    VPDef *D = I1;
+    EXPECT_EXIT(
+        {
+          D->dump();
+          exit(0);
+        },
+        testing::ExitedWithCode(0), "EMIT vp<%1> = add ir<1>, ir<2>");
+  }
+  // Check printing I2.
+  {
+    // Use EXPECT_EXIT to capture stderr and compare against expected output.
+    //
+    // Test VPValue::dump().
+    VPValue *VPV = I2;
+    EXPECT_EXIT(
+        {
+          VPV->dump();
+          exit(0);
+        },
+        testing::ExitedWithCode(0), "EMIT vp<%2> = mul vp<%1>, vp<%1>");
+
+    // Test VPRecipeBase::dump().
+    VPRecipeBase *R = I2;
+    EXPECT_EXIT(
+        {
+          R->dump();
+          exit(0);
+        },
+        testing::ExitedWithCode(0), "EMIT vp<%2> = mul vp<%1>, vp<%1>");
+
+    // Test VPDef::dump().
+    VPDef *D = I2;
+    EXPECT_EXIT(
+        {
+          D->dump();
+          exit(0);
+        },
+        testing::ExitedWithCode(0), "EMIT vp<%2> = mul vp<%1>, vp<%1>");
+  }
+}
+
+TEST(VPRecipeTest, dumpRecipeUnnamedVPValuesNotInPlanOrBlock) {
+  LLVMContext C;
+  IntegerType *Int32 = IntegerType::get(C, 32);
+  VPValue *ExtVPV1 = new VPValue(ConstantInt::get(Int32, 1));
+  VPValue *ExtVPV2 = new VPValue(ConstantInt::get(Int32, 2));
+
+  VPInstruction *I1 = new VPInstruction(Instruction::Add, {ExtVPV1, ExtVPV2});
+  VPInstruction *I2 = new VPInstruction(Instruction::Mul, {I1, I1});
+
+  // Check printing I1.
+  {
+    // Use EXPECT_EXIT to capture stderr and compare against expected output.
+    //
+    // Test VPValue::dump().
+    VPValue *VPV = I1;
+    EXPECT_EXIT(
+        {
+          VPV->dump();
+          exit(0);
+        },
+        testing::ExitedWithCode(0), "EMIT <badref> = add ir<1>, ir<2>");
+
+    // Test VPRecipeBase::dump().
+    VPRecipeBase *R = I1;
+    EXPECT_EXIT(
+        {
+          R->dump();
+          exit(0);
+        },
+        testing::ExitedWithCode(0), "EMIT <badref> = add ir<1>, ir<2>");
+
+    // Test VPDef::dump().
+    VPDef *D = I1;
+    EXPECT_EXIT(
+        {
+          D->dump();
+          exit(0);
+        },
+        testing::ExitedWithCode(0), "EMIT <badref> = add ir<1>, ir<2>");
+  }
+  // Check printing I2.
+  {
+    // Use EXPECT_EXIT to capture stderr and compare against expected output.
+    //
+    // Test VPValue::dump().
+    VPValue *VPV = I2;
+    EXPECT_EXIT(
+        {
+          VPV->dump();
+          exit(0);
+        },
+        testing::ExitedWithCode(0), "EMIT <badref> = mul <badref>, <badref>");
+
+    // Test VPRecipeBase::dump().
+    VPRecipeBase *R = I2;
+    EXPECT_EXIT(
+        {
+          R->dump();
+          exit(0);
+        },
+        testing::ExitedWithCode(0), "EMIT <badref> = mul <badref>, <badref>");
+
+    // Test VPDef::dump().
+    VPDef *D = I2;
+    EXPECT_EXIT(
+        {
+          D->dump();
+          exit(0);
+        },
+        testing::ExitedWithCode(0), "EMIT <badref> = mul <badref>, <badref>");
+  }
+
+  delete I2;
+  delete I1;
+  delete ExtVPV2;
+  delete ExtVPV1;
+}
+
 #endif
 
 TEST(VPRecipeTest, CastVPReductionRecipeToVPUser) {
-- 
GitLab


From e060cb26d9b503ea5138909ec5c7099e7d4499c2 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Mon, 15 Apr 2024 11:47:44 +0100
Subject: [PATCH 224/464] [X86] Add isCompletePermute helper for single input
 shuffles where we don't lose/reuse any vector element. NFC

Matches purely reorder shuffles.
---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 34 +++++++++++++------------
 1 file changed, 18 insertions(+), 16 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 87f2eaddedf8..97a79d4b2bcb 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -3596,6 +3596,17 @@ static bool isUndefOrZeroInRange(ArrayRef<int> Mask, unsigned Pos,
   return llvm::all_of(Mask.slice(Pos, Size), isUndefOrZero);
 }
 
+/// Return true if every element of a single input is referenced by the shuffle
+/// mask. i.e. it just permutes them all.
+static bool isCompletePermute(ArrayRef<int> Mask) {
+  unsigned NumElts = Mask.size();
+  APInt DemandedElts = APInt::getZero(NumElts);
+  for (int M : Mask)
+    if (isInRange(M, 0, NumElts))
+      DemandedElts.setBit(M);
+  return DemandedElts.isAllOnes();
+}
+
 /// Helper function to test whether a shuffle mask could be
 /// simplified by widening the elements being shuffled.
 ///
@@ -46469,24 +46480,15 @@ static SDValue combineSetCCMOVMSK(SDValue EFLAGS, X86::CondCode &CC,
   if (NumElts <= CmpBits &&
       getTargetShuffleInputs(peekThroughBitcasts(Vec), ShuffleInputs,
                              ShuffleMask, DAG) &&
-      ShuffleInputs.size() == 1 && !isAnyZeroOrUndef(ShuffleMask) &&
+      ShuffleInputs.size() == 1 && isCompletePermute(ShuffleMask) &&
       ShuffleInputs[0].getValueSizeInBits() == VecVT.getSizeInBits() &&
       scaleShuffleElements(ShuffleMask, NumElts, ScaledMaskUnused)) {
-    unsigned NumShuffleElts = ShuffleMask.size();
-    APInt DemandedElts = APInt::getZero(NumShuffleElts);
-    for (int M : ShuffleMask) {
-      assert(0 <= M && M < (int)NumShuffleElts && "Bad unary shuffle index");
-      DemandedElts.setBit(M);
-    }
-    if (DemandedElts.isAllOnes()) {
-      SDLoc DL(EFLAGS);
-      SDValue Result = DAG.getBitcast(VecVT, ShuffleInputs[0]);
-      Result = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result);
-      Result =
-          DAG.getZExtOrTrunc(Result, DL, EFLAGS.getOperand(0).getValueType());
-      return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Result,
-                         EFLAGS.getOperand(1));
-    }
+    SDLoc DL(EFLAGS);
+    SDValue Result = DAG.getBitcast(VecVT, ShuffleInputs[0]);
+    Result = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Result);
+    Result =
+        DAG.getZExtOrTrunc(Result, DL, EFLAGS.getOperand(0).getValueType());
+    return DAG.getNode(X86ISD::CMP, DL, MVT::i32, Result, EFLAGS.getOperand(1));
   }
 
   // MOVMSKPS(V) !=/== 0 -> TESTPS(V,V)
-- 
GitLab


From 4859a593f4016f0a78e2fadf21722e370560c5e9 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Mon, 15 Apr 2024 11:50:56 +0100
Subject: [PATCH 225/464] [VPlan] Extend recipe printing tests with underlying
 IR values. (NFC)

Make tests added in c006b9088fa56 more interesting by using an
underlying IR value as well.
---
 .../Transforms/Vectorize/VPlanTest.cpp        | 24 ++++++++++++-------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
index ff0f482b833c..cb8737a9e64d 100644
--- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -1289,9 +1289,12 @@ TEST(VPRecipeTest, dumpRecipeUnnamedVPValuesInPlan) {
   LLVMContext C;
 
   IntegerType *Int32 = IntegerType::get(C, 32);
+  auto *AI =
+      BinaryOperator::CreateAdd(UndefValue::get(Int32), UndefValue::get(Int32));
+  AI->setName("a");
   SmallVector<VPValue *, 2> Args;
   VPValue *ExtVPV1 = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1));
-  VPValue *ExtVPV2 = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2));
+  VPValue *ExtVPV2 = Plan.getOrAddLiveIn(AI);
   Args.push_back(ExtVPV1);
   Args.push_back(ExtVPV2);
   VPInstruction *I1 = new VPInstruction(Instruction::Add, {ExtVPV1, ExtVPV2});
@@ -1310,7 +1313,7 @@ TEST(VPRecipeTest, dumpRecipeUnnamedVPValuesInPlan) {
           VPV->dump();
           exit(0);
         },
-        testing::ExitedWithCode(0), "EMIT vp<%1> = add ir<1>, ir<2>");
+        testing::ExitedWithCode(0), "EMIT vp<%1> = add ir<1>, ir<%a>");
 
     // Test VPRecipeBase::dump().
     VPRecipeBase *R = I1;
@@ -1319,7 +1322,7 @@ TEST(VPRecipeTest, dumpRecipeUnnamedVPValuesInPlan) {
           R->dump();
           exit(0);
         },
-        testing::ExitedWithCode(0), "EMIT vp<%1> = add ir<1>, ir<2>");
+        testing::ExitedWithCode(0), "EMIT vp<%1> = add ir<1>, ir<%a>");
 
     // Test VPDef::dump().
     VPDef *D = I1;
@@ -1328,7 +1331,7 @@ TEST(VPRecipeTest, dumpRecipeUnnamedVPValuesInPlan) {
           D->dump();
           exit(0);
         },
-        testing::ExitedWithCode(0), "EMIT vp<%1> = add ir<1>, ir<2>");
+        testing::ExitedWithCode(0), "EMIT vp<%1> = add ir<1>, ir<%a>");
   }
   // Check printing I2.
   {
@@ -1361,13 +1364,17 @@ TEST(VPRecipeTest, dumpRecipeUnnamedVPValuesInPlan) {
         },
         testing::ExitedWithCode(0), "EMIT vp<%2> = mul vp<%1>, vp<%1>");
   }
+  delete AI;
 }
 
 TEST(VPRecipeTest, dumpRecipeUnnamedVPValuesNotInPlanOrBlock) {
   LLVMContext C;
   IntegerType *Int32 = IntegerType::get(C, 32);
+  auto *AI =
+      BinaryOperator::CreateAdd(UndefValue::get(Int32), UndefValue::get(Int32));
+  AI->setName("a");
   VPValue *ExtVPV1 = new VPValue(ConstantInt::get(Int32, 1));
-  VPValue *ExtVPV2 = new VPValue(ConstantInt::get(Int32, 2));
+  VPValue *ExtVPV2 = new VPValue(AI);
 
   VPInstruction *I1 = new VPInstruction(Instruction::Add, {ExtVPV1, ExtVPV2});
   VPInstruction *I2 = new VPInstruction(Instruction::Mul, {I1, I1});
@@ -1383,7 +1390,7 @@ TEST(VPRecipeTest, dumpRecipeUnnamedVPValuesNotInPlanOrBlock) {
           VPV->dump();
           exit(0);
         },
-        testing::ExitedWithCode(0), "EMIT <badref> = add ir<1>, ir<2>");
+        testing::ExitedWithCode(0), "EMIT <badref> = add ir<1>, ir<%a>");
 
     // Test VPRecipeBase::dump().
     VPRecipeBase *R = I1;
@@ -1392,7 +1399,7 @@ TEST(VPRecipeTest, dumpRecipeUnnamedVPValuesNotInPlanOrBlock) {
           R->dump();
           exit(0);
         },
-        testing::ExitedWithCode(0), "EMIT <badref> = add ir<1>, ir<2>");
+        testing::ExitedWithCode(0), "EMIT <badref> = add ir<1>, ir<%a>");
 
     // Test VPDef::dump().
     VPDef *D = I1;
@@ -1401,7 +1408,7 @@ TEST(VPRecipeTest, dumpRecipeUnnamedVPValuesNotInPlanOrBlock) {
           D->dump();
           exit(0);
         },
-        testing::ExitedWithCode(0), "EMIT <badref> = add ir<1>, ir<2>");
+        testing::ExitedWithCode(0), "EMIT <badref> = add ir<1>, ir<%a>");
   }
   // Check printing I2.
   {
@@ -1439,6 +1446,7 @@ TEST(VPRecipeTest, dumpRecipeUnnamedVPValuesNotInPlanOrBlock) {
   delete I1;
   delete ExtVPV2;
   delete ExtVPV1;
+  delete AI;
 }
 
 #endif
-- 
GitLab


From f78b3466caa9296b32ec235dee87ace2dea94507 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Mon, 15 Apr 2024 12:51:29 +0200
Subject: [PATCH 226/464] ValueTracking: Treat poison more aggressively in
 computeKnownFPClass (#87990)

Assume no valid values, and the sign bit is 0.
---
 llvm/lib/Analysis/ValueTracking.cpp           |  6 ++
 llvm/test/CodeGen/AMDGPU/fold-fabs.ll         |  6 --
 llvm/test/Transforms/Attributor/nofpclass.ll  | 12 ++++
 llvm/unittests/Analysis/ValueTrackingTest.cpp | 55 +++++++++++++++++++
 4 files changed, 73 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 5beea614e332..b3abf016cfb9 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -4664,6 +4664,12 @@ void computeKnownFPClass(const Value *V, const APInt &DemandedElts,
     return;
   }
 
+  if (isa<PoisonValue>(V)) {
+    Known.KnownFPClasses = fcNone;
+    Known.SignBit = false;
+    return;
+  }
+
   // Try to handle fixed width vector constants
   auto *VFVTy = dyn_cast<FixedVectorType>(V->getType());
   const Constant *CV = dyn_cast<Constant>(V);
diff --git a/llvm/test/CodeGen/AMDGPU/fold-fabs.ll b/llvm/test/CodeGen/AMDGPU/fold-fabs.ll
index bb2bad9d3521..a04bf4454932 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-fabs.ll
+++ b/llvm/test/CodeGen/AMDGPU/fold-fabs.ll
@@ -99,12 +99,6 @@ define float @fold_abs_in_branch_poison(float %arg1, float %arg2) {
 ; GFX10-LABEL: fold_abs_in_branch_poison:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    v_add_f32_e64 v0, |s4|, |s4|
-; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, 1.0, v0
-; GFX10-NEXT:    s_cbranch_vccnz .LBB3_2
-; GFX10-NEXT:  ; %bb.1: ; %if
-; GFX10-NEXT:    v_mul_f32_e64 v0, 0x3e4ccccd, |s4|
-; GFX10-NEXT:  .LBB3_2: ; %exit
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %0 = fadd reassoc nnan nsz arcp contract afn float %arg1, %arg2
diff --git a/llvm/test/Transforms/Attributor/nofpclass.ll b/llvm/test/Transforms/Attributor/nofpclass.ll
index 7828629f9fc0..d2d11e0276c4 100644
--- a/llvm/test/Transforms/Attributor/nofpclass.ll
+++ b/llvm/test/Transforms/Attributor/nofpclass.ll
@@ -54,6 +54,18 @@ define float @returned_poison() {
   ret float poison
 }
 
+; Know nothing
+define float @returned_freeze_poison() {
+; CHECK-LABEL: define noundef float @returned_freeze_poison() {
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    [[FREEZE_POISON:%.*]] = freeze float poison
+; CHECK-NEXT:    ret float [[FREEZE_POISON]]
+;
+  call void @unknown()
+  %freeze.poison = freeze float poison
+  ret float %freeze.poison
+}
+
 define double @returned_snan() {
 ; CHECK-LABEL: define noundef nofpclass(qnan inf zero sub norm) double @returned_snan() {
 ; CHECK-NEXT:    call void @unknown()
diff --git a/llvm/unittests/Analysis/ValueTrackingTest.cpp b/llvm/unittests/Analysis/ValueTrackingTest.cpp
index 6b1044876d36..8ebd9b511f39 100644
--- a/llvm/unittests/Analysis/ValueTrackingTest.cpp
+++ b/llvm/unittests/Analysis/ValueTrackingTest.cpp
@@ -2035,6 +2035,61 @@ TEST_F(ComputeKnownFPClassTest, Constants) {
     ASSERT_TRUE(ConstAggZero.SignBit);
     EXPECT_FALSE(*ConstAggZero.SignBit);
   }
+
+  {
+    KnownFPClass Undef =
+        computeKnownFPClass(UndefValue::get(F32), M->getDataLayout(),
+                            fcAllFlags, 0, nullptr, nullptr, nullptr, nullptr);
+    EXPECT_EQ(fcAllFlags, Undef.KnownFPClasses);
+    EXPECT_FALSE(Undef.SignBit);
+  }
+
+  {
+    KnownFPClass Poison =
+        computeKnownFPClass(PoisonValue::get(F32), M->getDataLayout(),
+                            fcAllFlags, 0, nullptr, nullptr, nullptr, nullptr);
+    EXPECT_EQ(fcNone, Poison.KnownFPClasses);
+    ASSERT_TRUE(Poison.SignBit);
+    EXPECT_FALSE(*Poison.SignBit);
+  }
+
+  {
+    // Assume the poison element should be 0.
+    Constant *ZeroF32 = ConstantFP::getZero(F32);
+    Constant *PoisonF32 = PoisonValue::get(F32);
+
+    KnownFPClass PartiallyPoison = computeKnownFPClass(
+        ConstantVector::get({ZeroF32, PoisonF32}), M->getDataLayout(),
+        fcAllFlags, 0, nullptr, nullptr, nullptr, nullptr);
+    EXPECT_EQ(fcPosZero, PartiallyPoison.KnownFPClasses);
+    ASSERT_TRUE(PartiallyPoison.SignBit);
+    EXPECT_FALSE(*PartiallyPoison.SignBit);
+  }
+
+  {
+    // Assume the poison element should be 1.
+    Constant *NegZeroF32 = ConstantFP::getZero(F32, true);
+    Constant *PoisonF32 = PoisonValue::get(F32);
+
+    KnownFPClass PartiallyPoison = computeKnownFPClass(
+        ConstantVector::get({NegZeroF32, PoisonF32}), M->getDataLayout(),
+        fcAllFlags, 0, nullptr, nullptr, nullptr, nullptr);
+    EXPECT_EQ(fcNegZero, PartiallyPoison.KnownFPClasses);
+    ASSERT_TRUE(PartiallyPoison.SignBit);
+    EXPECT_TRUE(*PartiallyPoison.SignBit);
+  }
+
+  {
+    // Assume the poison element should be 1.
+    Constant *NegZeroF32 = ConstantFP::getZero(F32, true);
+    Constant *PoisonF32 = PoisonValue::get(F32);
+
+    KnownFPClass PartiallyPoison = computeKnownFPClass(
+        ConstantVector::get({PoisonF32, NegZeroF32}), M->getDataLayout(),
+        fcAllFlags, 0, nullptr, nullptr, nullptr, nullptr);
+    EXPECT_EQ(fcNegZero, PartiallyPoison.KnownFPClasses);
+    EXPECT_TRUE(PartiallyPoison.SignBit);
+  }
 }
 
 TEST_F(ValueTrackingTest, isNonZeroRecurrence) {
-- 
GitLab


From 6acc50ae5b5971af563a0f40b5f0279870171541 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Mon, 15 Apr 2024 11:50:11 +0100
Subject: [PATCH 227/464] Fix typo in
 computeKnownBits_AVGFLOORU_AVGFLOORS_AVGCEILU_AVGCEILS.

ISD::AVGFLOORU -> ISD::AVGFLOORS
---
 llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp b/llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp
index 27bcad7c24c4..e498a2f60bad 100644
--- a/llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp
+++ b/llvm/unittests/CodeGen/AArch64SelectionDAGTest.cpp
@@ -826,7 +826,7 @@ TEST_F(AArch64SelectionDAGTest,
   EXPECT_EQ(KnownAVGFLOORU.One, Ones);
 
   SDValue AVGFLOORS =
-      DAG->getNode(ISD::AVGFLOORU, Loc, Int16Vec8VT, ZextOp0, ZextOp1);
+      DAG->getNode(ISD::AVGFLOORS, Loc, Int16Vec8VT, ZextOp0, ZextOp1);
   KnownBits KnownAVGFLOORS = DAG->computeKnownBits(AVGFLOORS);
   EXPECT_EQ(KnownAVGFLOORS.Zero, Zeroes);
   EXPECT_EQ(KnownAVGFLOORS.One, Ones);
-- 
GitLab


From 14774ad59da4aaa8eb3da21c8c53b59834a1aeb7 Mon Sep 17 00:00:00 2001
From: Jie Fu <jiefu@tencent.com>
Date: Mon, 15 Apr 2024 19:01:28 +0800
Subject: [PATCH 228/464] [X86] Remove unused function (NFC)

llvm-project/llvm/lib/Target/X86/X86ISelLowering.cpp:3537:13:
error: unused function 'isAnyZeroOrUndef' [-Werror,-Wunused-function]
static bool isAnyZeroOrUndef(ArrayRef<int> Mask) {
            ^
1 error generated.
---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 97a79d4b2bcb..0a91e3f554b3 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -3532,14 +3532,6 @@ static bool isAnyZero(ArrayRef<int> Mask) {
   return llvm::any_of(Mask, [](int M) { return M == SM_SentinelZero; });
 }
 
-/// Return true if the value of any element in Mask is the zero or undef
-/// sentinel values.
-static bool isAnyZeroOrUndef(ArrayRef<int> Mask) {
-  return llvm::any_of(Mask, [](int M) {
-    return M == SM_SentinelZero || M == SM_SentinelUndef;
-  });
-}
-
 /// Return true if Val is undef or if its value falls within the
 /// specified range (L, H].
 static bool isUndefOrInRange(int Val, int Low, int Hi) {
-- 
GitLab


From 9da0ef165efd91b862b8c3cff1604472476839d6 Mon Sep 17 00:00:00 2001
From: Johannes Reifferscheid <jreiffers@google.com>
Date: Mon, 15 Apr 2024 13:13:01 +0200
Subject: [PATCH 229/464] Fix complex tanh overflows. (#88708)

This ports the XLA lowering and was verified using XLA's
exhaustive_unary_test_complex test.
---
 .../ComplexToStandard/ComplexToStandard.cpp   |  90 ++++++++++++----
 .../convert-to-standard.mlir                  | 102 +++++++++++++++---
 2 files changed, 157 insertions(+), 35 deletions(-)

diff --git a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
index 3ebee9baff31..03e578136e59 100644
--- a/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
+++ b/mlir/lib/Conversion/ComplexToStandard/ComplexToStandard.cpp
@@ -978,30 +978,84 @@ struct TanhOpConversion : public OpConversionPattern<complex::TanhOp> {
   LogicalResult
   matchAndRewrite(complex::TanhOp op, OpAdaptor adaptor,
                   ConversionPatternRewriter &rewriter) const override {
+    ImplicitLocOpBuilder b(op.getLoc(), rewriter);
     auto loc = op.getLoc();
     auto type = cast<ComplexType>(adaptor.getComplex().getType());
     auto elementType = cast<FloatType>(type.getElementType());
-    arith::FastMathFlagsAttr fmf = op.getFastMathFlagsAttr();
+    arith::FastMathFlags fmf = op.getFastMathFlagsAttr().getValue();
+    const auto &floatSemantics = elementType.getFloatSemantics();
 
-    // The hyperbolic tangent for complex number can be calculated as follows.
-    // tanh(x + i * y) = (tanh(x) + i * tan(y)) / (1 + tanh(x) * tan(y))
-    // See: https://proofwiki.org/wiki/Hyperbolic_Tangent_of_Complex_Number
     Value real =
-        rewriter.create<complex::ReOp>(loc, elementType, adaptor.getComplex());
+        b.create<complex::ReOp>(loc, elementType, adaptor.getComplex());
     Value imag =
-        rewriter.create<complex::ImOp>(loc, elementType, adaptor.getComplex());
-    Value tanhA = rewriter.create<math::TanhOp>(loc, real, fmf);
-    Value cosB = rewriter.create<math::CosOp>(loc, imag, fmf);
-    Value sinB = rewriter.create<math::SinOp>(loc, imag, fmf);
-    Value tanB = rewriter.create<arith::DivFOp>(loc, sinB, cosB, fmf);
-    Value numerator =
-        rewriter.create<complex::CreateOp>(loc, type, tanhA, tanB);
-    Value one = rewriter.create<arith::ConstantOp>(
-        loc, elementType, rewriter.getFloatAttr(elementType, 1));
-    Value mul = rewriter.create<arith::MulFOp>(loc, tanhA, tanB, fmf);
-    Value denominator = rewriter.create<complex::CreateOp>(loc, type, one, mul);
-    rewriter.replaceOpWithNewOp<complex::DivOp>(op, numerator, denominator,
-                                                fmf);
+        b.create<complex::ImOp>(loc, elementType, adaptor.getComplex());
+
+    auto cst = [&](APFloat v) {
+      return b.create<arith::ConstantOp>(elementType,
+                                         b.getFloatAttr(elementType, v));
+    };
+    Value inf = cst(APFloat::getInf(floatSemantics));
+    Value negOne = b.create<arith::ConstantOp>(
+        elementType, b.getFloatAttr(elementType, -1.0));
+    Value four = b.create<arith::ConstantOp>(elementType,
+                                             b.getFloatAttr(elementType, 4.0));
+    Value twoReal = b.create<arith::AddFOp>(real, real, fmf);
+    Value negTwoReal = b.create<arith::MulFOp>(negOne, twoReal, fmf);
+
+    Value expTwoRealMinusOne = b.create<math::ExpM1Op>(twoReal, fmf);
+    Value expNegTwoRealMinusOne = b.create<math::ExpM1Op>(negTwoReal, fmf);
+    Value realNum =
+        b.create<arith::SubFOp>(expTwoRealMinusOne, expNegTwoRealMinusOne, fmf);
+
+    Value cosImag = b.create<math::CosOp>(imag, fmf);
+    Value cosImagSq = b.create<arith::MulFOp>(cosImag, cosImag, fmf);
+    Value twoCosTwoImagPlusOne = b.create<arith::MulFOp>(cosImagSq, four, fmf);
+    Value sinImag = b.create<math::SinOp>(imag, fmf);
+
+    Value imagNum = b.create<arith::MulFOp>(
+        four, b.create<arith::MulFOp>(cosImag, sinImag, fmf), fmf);
+
+    Value expSumMinusTwo =
+        b.create<arith::AddFOp>(expTwoRealMinusOne, expNegTwoRealMinusOne, fmf);
+    Value denom =
+        b.create<arith::AddFOp>(expSumMinusTwo, twoCosTwoImagPlusOne, fmf);
+
+    Value isInf = b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ,
+                                          expSumMinusTwo, inf, fmf);
+    Value realLimit = b.create<math::CopySignOp>(negOne, real, fmf);
+
+    Value resultReal = b.create<arith::SelectOp>(
+        isInf, realLimit, b.create<arith::DivFOp>(realNum, denom, fmf));
+    Value resultImag = b.create<arith::DivFOp>(imagNum, denom, fmf);
+
+    if (!arith::bitEnumContainsAll(fmf, arith::FastMathFlags::nnan |
+                                            arith::FastMathFlags::ninf)) {
+      Value absReal = b.create<math::AbsFOp>(real, fmf);
+      Value zero = b.create<arith::ConstantOp>(
+          elementType, b.getFloatAttr(elementType, 0.0));
+      Value nan = cst(APFloat::getNaN(floatSemantics));
+
+      Value absRealIsInf =
+          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, absReal, inf, fmf);
+      Value imagIsZero =
+          b.create<arith::CmpFOp>(arith::CmpFPredicate::OEQ, imag, zero, fmf);
+      Value absRealIsNotInf = b.create<arith::XOrIOp>(
+          absRealIsInf, b.create<arith::ConstantIntOp>(true, /*width=*/1));
+
+      Value imagNumIsNaN = b.create<arith::CmpFOp>(arith::CmpFPredicate::UNO,
+                                                   imagNum, imagNum, fmf);
+      Value resultRealIsNaN =
+          b.create<arith::AndIOp>(imagNumIsNaN, absRealIsNotInf);
+      Value resultImagIsZero = b.create<arith::OrIOp>(
+          imagIsZero, b.create<arith::AndIOp>(absRealIsInf, imagNumIsNaN));
+
+      resultReal = b.create<arith::SelectOp>(resultRealIsNaN, nan, resultReal);
+      resultImag =
+          b.create<arith::SelectOp>(resultImagIsZero, zero, resultImag);
+    }
+
+    rewriter.replaceOpWithNewOp<complex::CreateOp>(op, type, resultReal,
+                                                   resultImag);
     return success();
   }
 };
diff --git a/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir b/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir
index 8b4ea9777f79..fa1d564d6ad3 100644
--- a/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir
+++ b/mlir/test/Conversion/ComplexToStandard/convert-to-standard.mlir
@@ -679,14 +679,42 @@ func.func @complex_tanh(%arg: complex<f32>) -> complex<f32> {
 }
 // CHECK: %[[REAL:.*]] = complex.re %[[ARG]] : complex<f32>
 // CHECK: %[[IMAG:.*]] = complex.im %[[ARG]] : complex<f32>
-// CHECK: %[[TANH_A:.*]] = math.tanh %[[REAL]] : f32
-// CHECK: %[[COS_B:.*]] = math.cos %[[IMAG]] : f32
-// CHECK: %[[SIN_B:.*]] = math.sin %[[IMAG]] : f32
-// CHECK: %[[TAN_B:.*]] = arith.divf %[[SIN_B]], %[[COS_B]] : f32
-// CHECK: %[[NUM:.*]] = complex.create %[[TANH_A]], %[[TAN_B]] : complex<f32>
-// CHECK: %[[ONE:.*]] = arith.constant 1.000000e+00 : f32
-// CHECK: %[[MUL:.*]] = arith.mulf %[[TANH_A]], %[[TAN_B]] : f32
-// CHECK: %[[DENOM:.*]] = complex.create %[[ONE]], %[[MUL]] : complex<f32>
+// CHECK: %[[INF:.*]] = arith.constant 0x7F800000 : f32
+// CHECK: %[[NEG_ONE:.*]] = arith.constant -1.000000e+00 : f32
+// CHECK: %[[FOUR:.*]] = arith.constant 4.000000e+00 : f32
+// CHECK: %[[TWO_REAL:.*]] = arith.addf %[[REAL]], %[[REAL]] : f32
+// CHECK: %[[NEG_TWO_REAL:.*]] = arith.mulf %[[NEG_ONE]], %[[TWO_REAL]] : f32
+// CHECK: %[[EXPM1:.*]] = math.expm1 %[[TWO_REAL]] : f32
+// CHECK: %[[EXPM1_2:.*]] = math.expm1 %[[NEG_TWO_REAL]] : f32
+// CHECK: %[[REAL_NUM:.*]] = arith.subf %[[EXPM1]], %[[EXPM1_2]] : f32
+// CHECK: %[[COS:.*]] = math.cos %[[IMAG]] : f32
+// CHECK: %[[COS_SQ:.*]] = arith.mulf %[[COS]], %[[COS]] : f32
+// CHECK: %[[FOUR_COS_SQ:.*]] = arith.mulf %[[COS_SQ]], %[[FOUR]] : f32
+// CHECK: %[[SIN:.*]] = math.sin %[[IMAG]] : f32
+// CHECK: %[[MUL:.*]] = arith.mulf %[[COS]], %[[SIN]] : f32
+// CHECK: %[[IMAG_NUM:.*]] = arith.mulf %[[FOUR]], %[[MUL]] : f32
+// CHECK: %[[ADD:.*]] = arith.addf %[[EXPM1]], %[[EXPM1_2]] : f32
+// CHECK: %[[DENOM:.*]] = arith.addf %[[ADD]], %[[FOUR_COS_SQ]] : f32
+// CHECK: %[[IS_INF:.*]] = arith.cmpf oeq, %[[ADD]], %[[INF]] : f32
+// CHECK: %[[LIMIT:.*]] = math.copysign %[[NEG_ONE]], %[[REAL]] : f32
+// CHECK: %[[RESULT_REAL:.*]] = arith.divf %[[REAL_NUM]], %[[DENOM]] : f32
+// CHECK: %[[RESULT_REAL2:.*]] = arith.select %[[IS_INF]], %[[LIMIT]], %[[RESULT_REAL]] : f32
+// CHECK: %[[RESULT_IMAG:.*]] = arith.divf %[[IMAG_NUM]], %[[DENOM]] : f32
+// CHECK: %[[ABS_REAL:.*]] = math.absf %[[REAL]] : f32
+// CHECK: %[[ZERO:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK: %[[NAN:.*]] = arith.constant 0x7FC00000 : f32
+// CHECK: %[[ABS_REAL_INF:.*]] = arith.cmpf oeq, %[[ABS_REAL]], %[[INF]] : f32
+// CHECK: %[[IMAG_ZERO:.*]] = arith.cmpf oeq, %[[IMAG]], %[[ZERO]] : f32
+// CHECK: %true = arith.constant true
+// CHECK: %[[ABS_REAL_NOT_INF:.*]] = arith.xori %[[ABS_REAL_INF]], %true : i1
+// CHECK: %[[IMAG_IS_NAN:.*]] = arith.cmpf uno, %[[IMAG_NUM]], %[[IMAG_NUM]] : f32
+// CHECK: %[[REAL_IS_NAN:.*]] = arith.andi %[[IMAG_IS_NAN]], %[[ABS_REAL_NOT_INF]] : i1
+// CHECK: %[[AND:.*]] = arith.andi %[[ABS_REAL_INF]], %[[IMAG_IS_NAN]] : i1
+// CHECK: %[[IMAG_IS_NAN2:.*]] = arith.ori %[[IMAG_ZERO]], %[[AND]] : i1
+// CHECK: %[[RESULT_REAL3:.*]] = arith.select %[[REAL_IS_NAN]], %[[NAN]], %[[RESULT_REAL2]] : f32
+// CHECK: %[[RESULT_IMAG2:.*]] = arith.select %[[IMAG_IS_NAN2]], %[[ZERO]], %[[RESULT_IMAG]] : f32
+// CHECK: %[[RESULT:.*]] = complex.create %[[RESULT_REAL3]], %[[RESULT_IMAG2]] : complex<f32>
+// CHECK: return %[[RESULT]] : complex<f32>
 
 // -----
 
@@ -2100,7 +2128,6 @@ func.func @complex_tan_with_fmf(%arg: complex<f32>) -> complex<f32> {
 // CHECK: %[[RESULT:.*]] = complex.create %[[RESULT_REAL_WITH_SPECIAL_CASES]], %[[RESULT_IMAG_WITH_SPECIAL_CASES]] : complex<f32>
 // CHECK: return %[[RESULT]] : complex<f32>
 
-
 // -----
 
 // CHECK-LABEL: func @complex_tanh_with_fmf
@@ -2109,13 +2136,54 @@ func.func @complex_tanh_with_fmf(%arg: complex<f32>) -> complex<f32> {
   %tanh = complex.tanh %arg fastmath<nnan,contract> : complex<f32>
   return %tanh : complex<f32>
 }
+
 // CHECK: %[[REAL:.*]] = complex.re %[[ARG]] : complex<f32>
 // CHECK: %[[IMAG:.*]] = complex.im %[[ARG]] : complex<f32>
-// CHECK: %[[TANH_A:.*]] = math.tanh %[[REAL]] fastmath<nnan,contract> : f32
-// CHECK: %[[COS_B:.*]] = math.cos %[[IMAG]] fastmath<nnan,contract> : f32
-// CHECK: %[[SIN_B:.*]] = math.sin %[[IMAG]] fastmath<nnan,contract> : f32
-// CHECK: %[[TAN_B:.*]] = arith.divf %[[SIN_B]], %[[COS_B]] fastmath<nnan,contract> : f32
-// CHECK: %[[NUM:.*]] = complex.create %[[TANH_A]], %[[TAN_B]] : complex<f32>
-// CHECK: %[[ONE:.*]] = arith.constant 1.000000e+00 : f32
-// CHECK: %[[MUL:.*]] = arith.mulf %[[TANH_A]], %[[TAN_B]] fastmath<nnan,contract> : f32
-// CHECK: %[[DENOM:.*]] = complex.create %[[ONE]], %[[MUL]] : complex<f32>
+// CHECK: %[[INF:.*]] = arith.constant 0x7F800000 : f32
+// CHECK: %[[NEG_ONE:.*]] = arith.constant -1.000000e+00 : f32
+// CHECK: %[[FOUR:.*]] = arith.constant 4.000000e+00 : f32
+// CHECK: %[[TWO_REAL:.*]] = arith.addf %[[REAL]], %[[REAL]] fastmath<nnan,contract> : f32
+// CHECK: %[[NEG_TWO_REAL:.*]] = arith.mulf %[[NEG_ONE]], %[[TWO_REAL]] fastmath<nnan,contract> : f32
+// CHECK: %[[EXPM1:.*]] = math.expm1 %[[TWO_REAL]] fastmath<nnan,contract> : f32
+// CHECK: %[[EXPM1_2:.*]] = math.expm1 %[[NEG_TWO_REAL]] fastmath<nnan,contract> : f32
+// CHECK: %[[REAL_NUM:.*]] = arith.subf %[[EXPM1]], %[[EXPM1_2]] fastmath<nnan,contract> : f32
+// CHECK: %[[COS:.*]] = math.cos %[[IMAG]] fastmath<nnan,contract> : f32
+// CHECK: %[[COS_SQ:.*]] = arith.mulf %[[COS]], %[[COS]] fastmath<nnan,contract> : f32
+// CHECK: %[[FOUR_COS_SQ:.*]] = arith.mulf %[[COS_SQ]], %[[FOUR]] fastmath<nnan,contract> : f32
+// CHECK: %[[SIN:.*]] = math.sin %[[IMAG]] fastmath<nnan,contract> : f32
+// CHECK: %[[MUL:.*]] = arith.mulf %[[COS]], %[[SIN]] fastmath<nnan,contract> : f32
+// CHECK: %[[IMAG_NUM:.*]] = arith.mulf %[[FOUR]], %[[MUL]] fastmath<nnan,contract> : f32
+// CHECK: %[[ADD:.*]] = arith.addf %[[EXPM1]], %[[EXPM1_2]] fastmath<nnan,contract> : f32
+// CHECK: %[[DENOM:.*]] = arith.addf %[[ADD]], %[[FOUR_COS_SQ]] fastmath<nnan,contract> : f32
+// CHECK: %[[IS_INF:.*]] = arith.cmpf oeq, %[[ADD]], %[[INF]] fastmath<nnan,contract> : f32
+// CHECK: %[[LIMIT:.*]] = math.copysign %[[NEG_ONE]], %[[REAL]] fastmath<nnan,contract> : f32
+// CHECK: %[[RESULT_REAL:.*]] = arith.divf %[[REAL_NUM]], %[[DENOM]] fastmath<nnan,contract> : f32
+// CHECK: %[[RESULT_REAL2:.*]] = arith.select %[[IS_INF]], %[[LIMIT]], %[[RESULT_REAL]] : f32
+// CHECK: %[[RESULT_IMAG:.*]] = arith.divf %[[IMAG_NUM]], %[[DENOM]] fastmath<nnan,contract> : f32
+// CHECK: %[[ABS_REAL:.*]] = math.absf %[[REAL]] fastmath<nnan,contract> : f32
+// CHECK: %[[ZERO:.*]] = arith.constant 0.000000e+00 : f32
+// CHECK: %[[NAN:.*]] = arith.constant 0x7FC00000 : f32
+// CHECK: %[[ABS_REAL_INF:.*]] = arith.cmpf oeq, %[[ABS_REAL]], %[[INF]] fastmath<nnan,contract> : f32
+// CHECK: %[[IMAG_ZERO:.*]] = arith.cmpf oeq, %[[IMAG]], %[[ZERO]] fastmath<nnan,contract> : f32
+// CHECK: %true = arith.constant true
+// CHECK: %[[ABS_REAL_NOT_INF:.*]] = arith.xori %[[ABS_REAL_INF]], %true : i1
+// CHECK: %[[IMAG_IS_NAN:.*]] = arith.cmpf uno, %[[IMAG_NUM]], %[[IMAG_NUM]] fastmath<nnan,contract> : f32
+// CHECK: %[[REAL_IS_NAN:.*]] = arith.andi %[[IMAG_IS_NAN]], %[[ABS_REAL_NOT_INF]] : i1
+// CHECK: %[[AND:.*]] = arith.andi %[[ABS_REAL_INF]], %[[IMAG_IS_NAN]] : i1
+// CHECK: %[[IMAG_IS_NAN2:.*]] = arith.ori %[[IMAG_ZERO]], %[[AND]] : i1
+// CHECK: %[[RESULT_REAL3:.*]] = arith.select %[[REAL_IS_NAN]], %[[NAN]], %[[RESULT_REAL2]] : f32
+// CHECK: %[[RESULT_IMAG2:.*]] = arith.select %[[IMAG_IS_NAN2]], %[[ZERO]], %[[RESULT_IMAG]] : f32
+// CHECK: %[[RESULT:.*]] = complex.create %[[RESULT_REAL3]], %[[RESULT_IMAG2]] : complex<f32>
+// CHECK: return %[[RESULT]] : complex<f32>
+
+// -----
+
+// CHECK-LABEL: func @complex_tanh_nnan_ninf
+// CHECK-SAME: %[[ARG:.*]]: complex<f32>
+func.func @complex_tanh_nnan_ninf(%arg: complex<f32>) -> complex<f32> {
+  %tanh = complex.tanh %arg fastmath<nnan,ninf> : complex<f32>
+  return %tanh : complex<f32>
+}
+
+// CHECK-COUNT-1: arith.select
+// CHECK-NOT: arith.select
-- 
GitLab


From d48ebac0389ba75defce5bd83d8539d51008b28d Mon Sep 17 00:00:00 2001
From: Paul Walker <paul.walker@arm.com>
Date: Mon, 15 Apr 2024 12:24:42 +0100
Subject: [PATCH 230/464] [LLVM][SVE][CodeGen] Fix incorrect isel for signed
 saturating instructions. (#88136)

The immediate forms of SQADD and SQSUB interpret their immediate
operand as unsigned and thus effectively only support positive
immediate operands.

The original code is only wrong for the i8 cases because they
previously accepted all values, however, the new patterns enable more
uses and this I've added tests for the larger element types as well.
---
 .../Target/AArch64/AArch64ISelDAGToDAG.cpp    |  57 ++++++++++
 .../lib/Target/AArch64/AArch64SVEInstrInfo.td |   4 +-
 llvm/lib/Target/AArch64/SVEInstrFormats.td    |  28 +++++
 .../AArch64/sve-intrinsics-int-arith-imm.ll   | 104 ++++++++++++++++++
 4 files changed, 191 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index 51bec3604026..80272213dd38 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -258,6 +258,11 @@ public:
     return SelectSVEAddSubImm(N, VT, Imm, Shift);
   }
 
+  template <MVT::SimpleValueType VT, bool Negate>
+  bool SelectSVEAddSubSSatImm(SDValue N, SDValue &Imm, SDValue &Shift) {
+    return SelectSVEAddSubSSatImm(N, VT, Imm, Shift, Negate);
+  }
+
   template <MVT::SimpleValueType VT>
   bool SelectSVECpyDupImm(SDValue N, SDValue &Imm, SDValue &Shift) {
     return SelectSVECpyDupImm(N, VT, Imm, Shift);
@@ -484,6 +489,8 @@ private:
   bool SelectCMP_SWAP(SDNode *N);
 
   bool SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift);
+  bool SelectSVEAddSubSSatImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift,
+                              bool Negate);
   bool SelectSVECpyDupImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift);
   bool SelectSVELogicalImm(SDValue N, MVT VT, SDValue &Imm, bool Invert);
 
@@ -4014,6 +4021,56 @@ bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm,
   return false;
 }
 
+bool AArch64DAGToDAGISel::SelectSVEAddSubSSatImm(SDValue N, MVT VT,
+                                                 SDValue &Imm, SDValue &Shift,
+                                                 bool Negate) {
+  if (!isa<ConstantSDNode>(N))
+    return false;
+
+  SDLoc DL(N);
+  int64_t Val = cast<ConstantSDNode>(N)
+                    ->getAPIntValue()
+                    .trunc(VT.getFixedSizeInBits())
+                    .getSExtValue();
+
+  if (Negate)
+    Val = -Val;
+
+  // Signed saturating instructions treat their immediate operand as unsigned,
+  // whereas the related intrinsics define their operands to be signed. This
+  // means we can only use the immediate form when the operand is non-negative.
+  if (Val < 0)
+    return false;
+
+  switch (VT.SimpleTy) {
+  case MVT::i8:
+    // All positive immediates are supported.
+    Shift = CurDAG->getTargetConstant(0, DL, MVT::i32);
+    Imm = CurDAG->getTargetConstant(Val, DL, MVT::i32);
+    return true;
+  case MVT::i16:
+  case MVT::i32:
+  case MVT::i64:
+    // Support 8bit positive immediates.
+    if (Val <= 255) {
+      Shift = CurDAG->getTargetConstant(0, DL, MVT::i32);
+      Imm = CurDAG->getTargetConstant(Val, DL, MVT::i32);
+      return true;
+    }
+    // Support 16bit positive immediates that are a multiple of 256.
+    if (Val <= 65280 && Val % 256 == 0) {
+      Shift = CurDAG->getTargetConstant(8, DL, MVT::i32);
+      Imm = CurDAG->getTargetConstant(Val >> 8, DL, MVT::i32);
+      return true;
+    }
+    break;
+  default:
+    break;
+  }
+
+  return false;
+}
+
 bool AArch64DAGToDAGISel::SelectSVECpyDupImm(SDValue N, MVT VT, SDValue &Imm,
                                              SDValue &Shift) {
   if (!isa<ConstantSDNode>(N))
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 9c747198c12d..6972acd985cb 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -554,9 +554,9 @@ let Predicates = [HasSVEorSME] in {
   defm ADD_ZI   : sve_int_arith_imm0<0b000, "add", add>;
   defm SUB_ZI   : sve_int_arith_imm0<0b001, "sub", sub>;
   defm SUBR_ZI  : sve_int_arith_imm0<0b011, "subr", AArch64subr>;
-  defm SQADD_ZI : sve_int_arith_imm0<0b100, "sqadd", saddsat>;
+  defm SQADD_ZI : sve_int_arith_imm0_ssat<0b100, "sqadd", saddsat, ssubsat>;
   defm UQADD_ZI : sve_int_arith_imm0<0b101, "uqadd", uaddsat>;
-  defm SQSUB_ZI : sve_int_arith_imm0<0b110, "sqsub", ssubsat>;
+  defm SQSUB_ZI : sve_int_arith_imm0_ssat<0b110, "sqsub", ssubsat, saddsat>;
   defm UQSUB_ZI : sve_int_arith_imm0<0b111, "uqsub", usubsat>;
 
   defm MAD_ZPmZZ : sve_int_mladdsub_vvv_pred<0b0, "mad", AArch64mad_m1, "MLA_ZPmZZ", /*isReverseInstr*/ 1>;
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index fb0c6188edb3..3317cf84aa6f 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -249,6 +249,16 @@ def SVEAddSubImm16Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i16>", [
 def SVEAddSubImm32Pat : ComplexPattern<i32, 2, "SelectSVEAddSubImm<MVT::i32>", []>;
 def SVEAddSubImm64Pat : ComplexPattern<i64, 2, "SelectSVEAddSubImm<MVT::i64>", []>;
 
+def SVEAddSubSSatNegImm8Pat  : ComplexPattern<i32, 2, "SelectSVEAddSubSSatImm<MVT::i8, true>", []>;
+def SVEAddSubSSatNegImm16Pat : ComplexPattern<i32, 2, "SelectSVEAddSubSSatImm<MVT::i16, true>", []>;
+def SVEAddSubSSatNegImm32Pat : ComplexPattern<i32, 2, "SelectSVEAddSubSSatImm<MVT::i32, true>", []>;
+def SVEAddSubSSatNegImm64Pat : ComplexPattern<i64, 2, "SelectSVEAddSubSSatImm<MVT::i64, true>", []>;
+
+def SVEAddSubSSatPosImm8Pat  : ComplexPattern<i32, 2, "SelectSVEAddSubSSatImm<MVT::i8, false>", []>;
+def SVEAddSubSSatPosImm16Pat : ComplexPattern<i32, 2, "SelectSVEAddSubSSatImm<MVT::i16, false>", []>;
+def SVEAddSubSSatPosImm32Pat : ComplexPattern<i32, 2, "SelectSVEAddSubSSatImm<MVT::i32, false>", []>;
+def SVEAddSubSSatPosImm64Pat : ComplexPattern<i64, 2, "SelectSVEAddSubSSatImm<MVT::i64, false>", []>;
+
 def SVECpyDupImm8Pat  : ComplexPattern<i32, 2, "SelectSVECpyDupImm<MVT::i8>", []>;
 def SVECpyDupImm16Pat : ComplexPattern<i32, 2, "SelectSVECpyDupImm<MVT::i16>", []>;
 def SVECpyDupImm32Pat : ComplexPattern<i32, 2, "SelectSVECpyDupImm<MVT::i32>", []>;
@@ -4775,6 +4785,24 @@ multiclass sve_int_arith_imm0<bits<3> opc, string asm, SDPatternOperator op> {
   def : SVE_1_Op_Imm_OptLsl_Pat<nxv2i64, op, ZPR64, i64, SVEAddSubImm64Pat, !cast<Instruction>(NAME # _D)>;
 }
 
+multiclass sve_int_arith_imm0_ssat<bits<3> opc, string asm, SDPatternOperator op,
+                                   SDPatternOperator inv_op> {
+  def _B : sve_int_arith_imm0<0b00, opc, asm, ZPR8,  addsub_imm8_opt_lsl_i8>;
+  def _H : sve_int_arith_imm0<0b01, opc, asm, ZPR16, addsub_imm8_opt_lsl_i16>;
+  def _S : sve_int_arith_imm0<0b10, opc, asm, ZPR32, addsub_imm8_opt_lsl_i32>;
+  def _D : sve_int_arith_imm0<0b11, opc, asm, ZPR64, addsub_imm8_opt_lsl_i64>;
+
+  def : SVE_1_Op_Imm_OptLsl_Pat<nxv16i8, op, ZPR8,  i32, SVEAddSubSSatPosImm8Pat,  !cast<Instruction>(NAME # _B)>;
+  def : SVE_1_Op_Imm_OptLsl_Pat<nxv8i16, op, ZPR16, i32, SVEAddSubSSatPosImm16Pat, !cast<Instruction>(NAME # _H)>;
+  def : SVE_1_Op_Imm_OptLsl_Pat<nxv4i32, op, ZPR32, i32, SVEAddSubSSatPosImm32Pat, !cast<Instruction>(NAME # _S)>;
+  def : SVE_1_Op_Imm_OptLsl_Pat<nxv2i64, op, ZPR64, i64, SVEAddSubSSatPosImm64Pat, !cast<Instruction>(NAME # _D)>;
+
+  def : SVE_1_Op_Imm_OptLsl_Pat<nxv16i8, inv_op, ZPR8,  i32, SVEAddSubSSatNegImm8Pat,  !cast<Instruction>(NAME # _B)>;
+  def : SVE_1_Op_Imm_OptLsl_Pat<nxv8i16, inv_op, ZPR16, i32, SVEAddSubSSatNegImm16Pat, !cast<Instruction>(NAME # _H)>;
+  def : SVE_1_Op_Imm_OptLsl_Pat<nxv4i32, inv_op, ZPR32, i32, SVEAddSubSSatNegImm32Pat, !cast<Instruction>(NAME # _S)>;
+  def : SVE_1_Op_Imm_OptLsl_Pat<nxv2i64, inv_op, ZPR64, i64, SVEAddSubSSatNegImm64Pat, !cast<Instruction>(NAME # _D)>;
+}
+
 class sve_int_arith_imm<bits<2> sz8_64, bits<6> opc, string asm,
                         ZPRRegOp zprty, Operand immtype>
 : I<(outs zprty:$Zdn), (ins zprty:$_Zdn, immtype:$imm),
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-imm.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-imm.ll
index c70006d988c1..3e453a6b7817 100644
--- a/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-imm.ll
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-int-arith-imm.ll
@@ -1059,6 +1059,19 @@ define <vscale x 16 x i8> @sqadd_b_lowimm(<vscale x 16 x i8> %a) {
   ret <vscale x 16 x i8> %out
 }
 
+; Immediate instruction form only supports positive values.
+define <vscale x 16 x i8> @sqadd_b_negimm(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: sqadd_b_negimm:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sqsub z0.b, z0.b, #128 // =0x80
+; CHECK-NEXT:    ret
+  %elt = insertelement <vscale x 16 x i8> undef, i8 -128, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.sqadd.x.nxv16i8(<vscale x 16 x i8> %a,
+                                                                   <vscale x 16 x i8> %splat)
+  ret <vscale x 16 x i8> %out
+}
+
 define <vscale x 8 x i16> @sqadd_h_lowimm(<vscale x 8 x i16> %a) {
 ; CHECK-LABEL: sqadd_h_lowimm:
 ; CHECK:       // %bb.0:
@@ -1083,6 +1096,19 @@ define <vscale x 8 x i16> @sqadd_h_highimm(<vscale x 8 x i16> %a) {
   ret <vscale x 8 x i16> %out
 }
 
+; Immediate instruction form only supports positive values.
+define <vscale x 8 x i16> @sqadd_h_negimm(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: sqadd_h_negimm:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sqsub z0.h, z0.h, #1 // =0x1
+; CHECK-NEXT:    ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 -1, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sqadd.x.nxv8i16(<vscale x 8 x i16> %a,
+                                                                   <vscale x 8 x i16> %splat)
+  ret <vscale x 8 x i16> %out
+}
+
 define <vscale x 4 x i32> @sqadd_s_lowimm(<vscale x 4 x i32> %a) {
 ; CHECK-LABEL: sqadd_s_lowimm:
 ; CHECK:       // %bb.0:
@@ -1107,6 +1133,19 @@ define <vscale x 4 x i32> @sqadd_s_highimm(<vscale x 4 x i32> %a) {
   ret <vscale x 4 x i32> %out
 }
 
+; Immediate instruction form only supports positive values.
+define <vscale x 4 x i32> @sqadd_s_negimm(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: sqadd_s_negimm:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sqsub z0.s, z0.s, #65280 // =0xff00
+; CHECK-NEXT:    ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 -65280, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sqadd.x.nxv4i32(<vscale x 4 x i32> %a,
+                                                                   <vscale x 4 x i32> %splat)
+  ret <vscale x 4 x i32> %out
+}
+
 define <vscale x 2 x i64> @sqadd_d_lowimm(<vscale x 2 x i64> %a) {
 ; CHECK-LABEL: sqadd_d_lowimm:
 ; CHECK:       // %bb.0:
@@ -1131,6 +1170,19 @@ define <vscale x 2 x i64> @sqadd_d_highimm(<vscale x 2 x i64> %a) {
   ret <vscale x 2 x i64> %out
 }
 
+; Immediate instruction form only supports positive values.
+define <vscale x 2 x i64> @sqadd_d_negimm(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: sqadd_d_negimm:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sqsub z0.d, z0.d, #3840 // =0xf00
+; CHECK-NEXT:    ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 -3840, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sqadd.x.nxv2i64(<vscale x 2 x i64> %a,
+                                                                   <vscale x 2 x i64> %splat)
+  ret <vscale x 2 x i64> %out
+}
+
 ; SQSUB
 
 define <vscale x 16 x i8> @sqsub_b_lowimm(<vscale x 16 x i8> %a) {
@@ -1145,6 +1197,19 @@ define <vscale x 16 x i8> @sqsub_b_lowimm(<vscale x 16 x i8> %a) {
   ret <vscale x 16 x i8> %out
 }
 
+; Immediate instruction form only supports positive values.
+define <vscale x 16 x i8> @sqsub_b_negimm(<vscale x 16 x i8> %a) {
+; CHECK-LABEL: sqsub_b_negimm:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sqadd z0.b, z0.b, #1 // =0x1
+; CHECK-NEXT:    ret
+  %elt = insertelement <vscale x 16 x i8> undef, i8 -1, i32 0
+  %splat = shufflevector <vscale x 16 x i8> %elt, <vscale x 16 x i8> undef, <vscale x 16 x i32> zeroinitializer
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.sqsub.x.nxv16i8(<vscale x 16 x i8> %a,
+                                                                   <vscale x 16 x i8> %splat)
+  ret <vscale x 16 x i8> %out
+}
+
 define <vscale x 8 x i16> @sqsub_h_lowimm(<vscale x 8 x i16> %a) {
 ; CHECK-LABEL: sqsub_h_lowimm:
 ; CHECK:       // %bb.0:
@@ -1169,6 +1234,19 @@ define <vscale x 8 x i16> @sqsub_h_highimm(<vscale x 8 x i16> %a) {
   ret <vscale x 8 x i16> %out
 }
 
+; Immediate instruction form only supports positive values.
+define <vscale x 8 x i16> @sqsub_h_negimm(<vscale x 8 x i16> %a) {
+; CHECK-LABEL: sqsub_h_negimm:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sqadd z0.h, z0.h, #128 // =0x80
+; CHECK-NEXT:    ret
+  %elt = insertelement <vscale x 8 x i16> undef, i16 -128, i32 0
+  %splat = shufflevector <vscale x 8 x i16> %elt, <vscale x 8 x i16> undef, <vscale x 8 x i32> zeroinitializer
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.sqsub.x.nxv8i16(<vscale x 8 x i16> %a,
+                                                                   <vscale x 8 x i16> %splat)
+  ret <vscale x 8 x i16> %out
+}
+
 define <vscale x 4 x i32> @sqsub_s_lowimm(<vscale x 4 x i32> %a) {
 ; CHECK-LABEL: sqsub_s_lowimm:
 ; CHECK:       // %bb.0:
@@ -1193,6 +1271,19 @@ define <vscale x 4 x i32> @sqsub_s_highimm(<vscale x 4 x i32> %a) {
   ret <vscale x 4 x i32> %out
 }
 
+; Immediate instruction form only supports positive values.
+define <vscale x 4 x i32> @sqsub_s_negimm(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: sqsub_s_negimm:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sqadd z0.s, z0.s, #32768 // =0x8000
+; CHECK-NEXT:    ret
+  %elt = insertelement <vscale x 4 x i32> undef, i32 -32768, i32 0
+  %splat = shufflevector <vscale x 4 x i32> %elt, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.sqsub.x.nxv4i32(<vscale x 4 x i32> %a,
+                                                                   <vscale x 4 x i32> %splat)
+  ret <vscale x 4 x i32> %out
+}
+
 define <vscale x 2 x i64> @sqsub_d_lowimm(<vscale x 2 x i64> %a) {
 ; CHECK-LABEL: sqsub_d_lowimm:
 ; CHECK:       // %bb.0:
@@ -1217,6 +1308,19 @@ define <vscale x 2 x i64> @sqsub_d_highimm(<vscale x 2 x i64> %a) {
   ret <vscale x 2 x i64> %out
 }
 
+; Immediate instruction form only supports positive values.
+define <vscale x 2 x i64> @sqsub_d_negimm(<vscale x 2 x i64> %a) {
+; CHECK-LABEL: sqsub_d_negimm:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sqadd z0.d, z0.d, #57344 // =0xe000
+; CHECK-NEXT:    ret
+  %elt = insertelement <vscale x 2 x i64> undef, i64 -57344, i32 0
+  %splat = shufflevector <vscale x 2 x i64> %elt, <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.sqsub.x.nxv2i64(<vscale x 2 x i64> %a,
+                                                                   <vscale x 2 x i64> %splat)
+  ret <vscale x 2 x i64> %out
+}
+
 ; UQADD
 
 define <vscale x 16 x i8> @uqadd_b_lowimm(<vscale x 16 x i8> %a) {
-- 
GitLab


From 6254b6dd892728306ddfe657f10eb43a3799864d Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Mon, 15 Apr 2024 12:27:45 +0100
Subject: [PATCH 231/464] [VPlan] Version VPValue names in VPSlotTracker. 
 (#81411)

This patch restructures the way names for printing VPValues are handled.
It moves the logic to generate names for printing to VPSlotTracker.

VPSlotTracker will now version names of the same underlying value if it
is used by multiple VPValues, by adding a .V suffix to the name.

This fixes cases where at the moment the same name is printed for
different VPValues.

PR: https://github.com/llvm/llvm-project/pull/81411
---
 llvm/lib/Transforms/Vectorize/VPlan.cpp       | 93 ++++++++++++++-----
 llvm/lib/Transforms/Vectorize/VPlanValue.h    | 34 ++++---
 .../vplan-sink-scalars-and-merge.ll           |  8 +-
 3 files changed, 94 insertions(+), 41 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 3e1069d82dda..999236ae8489 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1300,18 +1300,7 @@ void VPValue::replaceUsesWithIf(
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void VPValue::printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const {
-  if (const Value *UV = getUnderlyingValue()) {
-    OS << "ir<";
-    UV->printAsOperand(OS, false);
-    OS << ">";
-    return;
-  }
-
-  unsigned Slot = Tracker.getSlot(this);
-  if (Slot == unsigned(-1))
-    OS << "<badref>";
-  else
-    OS << "vp<%" << Tracker.getSlot(this) << ">";
+  OS << Tracker.getOrCreateName(this);
 }
 
 void VPUser::printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const {
@@ -1373,32 +1362,88 @@ VPInterleavedAccessInfo::VPInterleavedAccessInfo(VPlan &Plan,
   visitRegion(Plan.getVectorLoopRegion(), Old2New, IAI);
 }
 
-void VPSlotTracker::assignSlot(const VPValue *V) {
-  if (V->getUnderlyingValue())
+void VPSlotTracker::assignName(const VPValue *V) {
+  assert(!VPValue2Name.contains(V) && "VPValue already has a name!");
+  auto *UV = V->getUnderlyingValue();
+  if (!UV) {
+    VPValue2Name[V] = (Twine("vp<%") + Twine(NextSlot) + ">").str();
+    NextSlot++;
     return;
-  assert(!Slots.contains(V) && "VPValue already has a slot!");
-  Slots[V] = NextSlot++;
+  }
+
+  // Use the name of the underlying Value, wrapped in "ir<>", and versioned by
+  // appending ".Number" to the name if there are multiple uses.
+  std::string Name;
+  raw_string_ostream S(Name);
+  UV->printAsOperand(S, false);
+  assert(!Name.empty() && "Name cannot be empty.");
+  std::string BaseName = (Twine("ir<") + Name + Twine(">")).str();
+
+  // First assign the base name for V.
+  const auto &[A, _] = VPValue2Name.insert({V, BaseName});
+  // Integer or FP constants with different types will result in he same string
+  // due to stripping types.
+  if (V->isLiveIn() && isa<ConstantInt, ConstantFP>(UV))
+    return;
+
+  // If it is already used by C > 0 other VPValues, increase the version counter
+  // C and use it for V.
+  const auto &[C, UseInserted] = BaseName2Version.insert({BaseName, 0});
+  if (!UseInserted) {
+    C->second++;
+    A->second = (BaseName + Twine(".") + Twine(C->second)).str();
+  }
 }
 
-void VPSlotTracker::assignSlots(const VPlan &Plan) {
+void VPSlotTracker::assignNames(const VPlan &Plan) {
   if (Plan.VFxUF.getNumUsers() > 0)
-    assignSlot(&Plan.VFxUF);
-  assignSlot(&Plan.VectorTripCount);
+    assignName(&Plan.VFxUF);
+  assignName(&Plan.VectorTripCount);
   if (Plan.BackedgeTakenCount)
-    assignSlot(Plan.BackedgeTakenCount);
-  assignSlots(Plan.getPreheader());
+    assignName(Plan.BackedgeTakenCount);
+  for (VPValue *LI : Plan.VPLiveInsToFree)
+    assignName(LI);
+  assignNames(Plan.getPreheader());
 
   ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<const VPBlockBase *>>
       RPOT(VPBlockDeepTraversalWrapper<const VPBlockBase *>(Plan.getEntry()));
   for (const VPBasicBlock *VPBB :
        VPBlockUtils::blocksOnly<const VPBasicBlock>(RPOT))
-    assignSlots(VPBB);
+    assignNames(VPBB);
 }
 
-void VPSlotTracker::assignSlots(const VPBasicBlock *VPBB) {
+void VPSlotTracker::assignNames(const VPBasicBlock *VPBB) {
   for (const VPRecipeBase &Recipe : *VPBB)
     for (VPValue *Def : Recipe.definedValues())
-      assignSlot(Def);
+      assignName(Def);
+}
+
+std::string VPSlotTracker::getOrCreateName(const VPValue *V) const {
+  std::string Name = VPValue2Name.lookup(V);
+  if (!Name.empty())
+    return Name;
+
+  // If no name was assigned, no VPlan was provided when creating the slot
+  // tracker or it is not reachable from the provided VPlan. This can happen,
+  // e.g. when trying to print a recipe that has not been inserted into a VPlan
+  // in a debugger.
+  // TODO: Update VPSlotTracker constructor to assign names to recipes &
+  // VPValues not associated with a VPlan, instead of constructing names ad-hoc
+  // here.
+  const VPRecipeBase *DefR = V->getDefiningRecipe();
+  (void)DefR;
+  assert((!DefR || !DefR->getParent() || !DefR->getParent()->getPlan()) &&
+         "VPValue defined by a recipe in a VPlan?");
+
+  // Use the underlying value's name, if there is one.
+  if (auto *UV = V->getUnderlyingValue()) {
+    std::string Name;
+    raw_string_ostream S(Name);
+    UV->printAsOperand(S, false);
+    return (Twine("ir<") + Name + ">").str();
+  }
+
+  return "<badref>";
 }
 
 bool vputils::onlyFirstLaneUsed(const VPValue *Def) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h
index 8b221d30e525..da3a768552fc 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanValue.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h
@@ -23,6 +23,7 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/TinyPtrVector.h"
 #include "llvm/ADT/iterator_range.h"
 
@@ -443,29 +444,36 @@ public:
 class VPlan;
 class VPBasicBlock;
 
-/// This class can be used to assign consecutive numbers to all VPValues in a
-/// VPlan and allows querying the numbering for printing, similar to the
+/// This class can be used to assign names to VPValues. For VPValues without
+/// underlying value, assign consecutive numbers and use those as names (wrapped
+/// in vp<>). Otherwise, use the name from the underlying value (wrapped in
+/// ir<>), appending a .V version number if there are multiple uses of the same
+/// name. Allows querying names for VPValues for printing, similar to the
 /// ModuleSlotTracker for IR values.
 class VPSlotTracker {
-  DenseMap<const VPValue *, unsigned> Slots;
+  /// Keep track of versioned names assigned to VPValues with underlying IR
+  /// values.
+  DenseMap<const VPValue *, std::string> VPValue2Name;
+  /// Keep track of the next number to use to version the base name.
+  StringMap<unsigned> BaseName2Version;
+
+  /// Number to assign to the next VPValue without underlying value.
   unsigned NextSlot = 0;
 
-  void assignSlot(const VPValue *V);
-  void assignSlots(const VPlan &Plan);
-  void assignSlots(const VPBasicBlock *VPBB);
+  void assignName(const VPValue *V);
+  void assignNames(const VPlan &Plan);
+  void assignNames(const VPBasicBlock *VPBB);
 
 public:
   VPSlotTracker(const VPlan *Plan = nullptr) {
     if (Plan)
-      assignSlots(*Plan);
+      assignNames(*Plan);
   }
 
-  unsigned getSlot(const VPValue *V) const {
-    auto I = Slots.find(V);
-    if (I == Slots.end())
-      return -1;
-    return I->second;
-  }
+  /// Returns the name assigned to \p V, if there is one, otherwise try to
+  /// construct one from the underlying value, if there's one; else return
+  /// <badref>.
+  std::string getOrCreateName(const VPValue *V) const;
 };
 
 } // namespace llvm
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
index 0cacb02dc489..108b78a70fa1 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll
@@ -986,8 +986,8 @@ define void @sinking_requires_duplication(ptr %addr) {
 ; CHECK-NEXT:   Successor(s): pred.store.if, pred.store.continue
 ; CHECK-EMPTY:
 ; CHECK-NEXT:   pred.store.if:
-; CHECK-NEXT:     REPLICATE ir<%gep> = getelementptr ir<%addr>, vp<[[STEPS]]>
-; CHECK-NEXT:     REPLICATE store ir<1.000000e+01>, ir<%gep>
+; CHECK-NEXT:     REPLICATE ir<%gep>.1 = getelementptr ir<%addr>, vp<[[STEPS]]>
+; CHECK-NEXT:     REPLICATE store ir<1.000000e+01>, ir<%gep>.1
 ; CHECK-NEXT:   Successor(s): pred.store.continue
 ; CHECK-EMPTY:
 ; CHECK-NEXT:   pred.store.continue:
@@ -1129,8 +1129,8 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) {
 ; CHECK-NEXT:     Successor(s): pred.store.if, pred.store.continue
 ; CHECK-EMPTY:
 ; CHECK-NEXT:     pred.store.if:
-; CHECK-NEXT:       REPLICATE ir<%ptr.iv.next> = getelementptr inbounds vp<[[PTR_IV]]>, ir<-1>
-; CHECK-NEXT:       REPLICATE store ir<95>, ir<%ptr.iv.next>
+; CHECK-NEXT:       REPLICATE ir<%ptr.iv.next>.1 = getelementptr inbounds vp<[[PTR_IV]]>, ir<-1>
+; CHECK-NEXT:       REPLICATE store ir<95>, ir<%ptr.iv.next>.1
 ; CHECK-NEXT:     Successor(s): pred.store.continue
 ; CHECK-EMPTY:
 ; CHECK-NEXT:     pred.store.continue:
-- 
GitLab


From 8095b9ce6bf5831a14c72028920708f38d13d0c3 Mon Sep 17 00:00:00 2001
From: Congcong Cai <congcongcai0907@163.com>
Date: Mon, 15 Apr 2024 19:38:50 +0800
Subject: [PATCH 232/464] [clang analysis] ExprMutationAnalyzer avoid infinite
 recursion for recursive forwarding reference (#87954)

---
 clang-tools-extra/docs/ReleaseNotes.rst       |  4 +++
 .../misc/const-correctness-templates.cpp      | 15 ++++++++++
 .../Analysis/Analyses/ExprMutationAnalyzer.h  | 28 ++++++++++++-----
 clang/lib/Analysis/ExprMutationAnalyzer.cpp   | 22 +++++++++-----
 .../Analysis/ExprMutationAnalyzerTest.cpp     | 30 +++++++++++++++++++
 5 files changed, 84 insertions(+), 15 deletions(-)

diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 1405fb0df1f8..de0a7b79bb2c 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -221,6 +221,10 @@ Changes in existing checks
   <clang-tidy/checks/llvm/header-guard>` check by replacing the local
   option `HeaderFileExtensions` by the global option of the same name.
 
+- Improved :doc:`misc-const-correctness
+  <clang-tidy/checks/misc/const-correctness>` check by avoiding infinite recursion
+  for recursive forwarding reference.
+
 - Improved :doc:`misc-definitions-in-headers
   <clang-tidy/checks/misc/definitions-in-headers>` check by replacing the local
   option `HeaderFileExtensions` by the global option of the same name.
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/const-correctness-templates.cpp b/clang-tools-extra/test/clang-tidy/checkers/misc/const-correctness-templates.cpp
index 9da468128743..248374a71dd4 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/misc/const-correctness-templates.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/const-correctness-templates.cpp
@@ -58,3 +58,18 @@ void concatenate3(Args... args)
     (..., (stream << args));
 }
 } // namespace gh70323
+
+namespace gh60895 {
+
+template <class T> void f1(T &&a);
+template <class T> void f2(T &&a);
+template <class T> void f1(T &&a) { f2<T>(a); }
+template <class T> void f2(T &&a) { f1<T>(a); }
+void f() {
+  int x = 0;
+  // CHECK-MESSAGES:[[@LINE-1]]:3: warning: variable 'x' of type 'int' can be declared 'const'
+  // CHECK-FIXES: int const x = 0;
+  f1(x);
+}
+
+} // namespace gh60895
diff --git a/clang/include/clang/Analysis/Analyses/ExprMutationAnalyzer.h b/clang/include/clang/Analysis/Analyses/ExprMutationAnalyzer.h
index 1ceef944fbc3..c4e5d0badb8e 100644
--- a/clang/include/clang/Analysis/Analyses/ExprMutationAnalyzer.h
+++ b/clang/include/clang/Analysis/Analyses/ExprMutationAnalyzer.h
@@ -8,11 +8,10 @@
 #ifndef LLVM_CLANG_ANALYSIS_ANALYSES_EXPRMUTATIONANALYZER_H
 #define LLVM_CLANG_ANALYSIS_ANALYSES_EXPRMUTATIONANALYZER_H
 
-#include <type_traits>
-
 #include "clang/AST/AST.h"
 #include "clang/ASTMatchers/ASTMatchers.h"
 #include "llvm/ADT/DenseMap.h"
+#include <variant>
 
 namespace clang {
 
@@ -22,8 +21,15 @@ class FunctionParmMutationAnalyzer;
 /// a given statement.
 class ExprMutationAnalyzer {
 public:
+  friend class FunctionParmMutationAnalyzer;
+  struct Cache {
+    llvm::SmallDenseMap<const FunctionDecl *,
+                        std::unique_ptr<FunctionParmMutationAnalyzer>>
+        FuncParmAnalyzer;
+  };
+
   ExprMutationAnalyzer(const Stmt &Stm, ASTContext &Context)
-      : Stm(Stm), Context(Context) {}
+      : ExprMutationAnalyzer(Stm, Context, std::make_shared<Cache>()) {}
 
   bool isMutated(const Expr *Exp) { return findMutation(Exp) != nullptr; }
   bool isMutated(const Decl *Dec) { return findMutation(Dec) != nullptr; }
@@ -45,6 +51,11 @@ private:
   using MutationFinder = const Stmt *(ExprMutationAnalyzer::*)(const Expr *);
   using ResultMap = llvm::DenseMap<const Expr *, const Stmt *>;
 
+  ExprMutationAnalyzer(const Stmt &Stm, ASTContext &Context,
+                       std::shared_ptr<Cache> CrossAnalysisCache)
+      : Stm(Stm), Context(Context),
+        CrossAnalysisCache(std::move(CrossAnalysisCache)) {}
+
   const Stmt *findMutationMemoized(const Expr *Exp,
                                    llvm::ArrayRef<MutationFinder> Finders,
                                    ResultMap &MemoizedResults);
@@ -69,9 +80,7 @@ private:
 
   const Stmt &Stm;
   ASTContext &Context;
-  llvm::DenseMap<const FunctionDecl *,
-                 std::unique_ptr<FunctionParmMutationAnalyzer>>
-      FuncParmAnalyzer;
+  std::shared_ptr<Cache> CrossAnalysisCache;
   ResultMap Results;
   ResultMap PointeeResults;
 };
@@ -80,7 +89,12 @@ private:
 // params.
 class FunctionParmMutationAnalyzer {
 public:
-  FunctionParmMutationAnalyzer(const FunctionDecl &Func, ASTContext &Context);
+  FunctionParmMutationAnalyzer(const FunctionDecl &Func, ASTContext &Context)
+      : FunctionParmMutationAnalyzer(
+            Func, Context, std::make_shared<ExprMutationAnalyzer::Cache>()) {}
+  FunctionParmMutationAnalyzer(
+      const FunctionDecl &Func, ASTContext &Context,
+      std::shared_ptr<ExprMutationAnalyzer::Cache> CrossAnalysisCache);
 
   bool isMutated(const ParmVarDecl *Parm) {
     return findMutation(Parm) != nullptr;
diff --git a/clang/lib/Analysis/ExprMutationAnalyzer.cpp b/clang/lib/Analysis/ExprMutationAnalyzer.cpp
index bb042760d297..503ca4bea99e 100644
--- a/clang/lib/Analysis/ExprMutationAnalyzer.cpp
+++ b/clang/lib/Analysis/ExprMutationAnalyzer.cpp
@@ -638,9 +638,10 @@ const Stmt *ExprMutationAnalyzer::findFunctionArgMutation(const Expr *Exp) {
       if (!RefType->getPointeeType().getQualifiers() &&
           RefType->getPointeeType()->getAs<TemplateTypeParmType>()) {
         std::unique_ptr<FunctionParmMutationAnalyzer> &Analyzer =
-            FuncParmAnalyzer[Func];
+            CrossAnalysisCache->FuncParmAnalyzer[Func];
         if (!Analyzer)
-          Analyzer.reset(new FunctionParmMutationAnalyzer(*Func, Context));
+          Analyzer.reset(new FunctionParmMutationAnalyzer(*Func, Context,
+                                                          CrossAnalysisCache));
         if (Analyzer->findMutation(Parm))
           return Exp;
         continue;
@@ -653,13 +654,15 @@ const Stmt *ExprMutationAnalyzer::findFunctionArgMutation(const Expr *Exp) {
 }
 
 FunctionParmMutationAnalyzer::FunctionParmMutationAnalyzer(
-    const FunctionDecl &Func, ASTContext &Context)
-    : BodyAnalyzer(*Func.getBody(), Context) {
+    const FunctionDecl &Func, ASTContext &Context,
+    std::shared_ptr<ExprMutationAnalyzer::Cache> CrossAnalysisCache)
+    : BodyAnalyzer(*Func.getBody(), Context, CrossAnalysisCache) {
   if (const auto *Ctor = dyn_cast<CXXConstructorDecl>(&Func)) {
     // CXXCtorInitializer might also mutate Param but they're not part of
     // function body, check them eagerly here since they're typically trivial.
     for (const CXXCtorInitializer *Init : Ctor->inits()) {
-      ExprMutationAnalyzer InitAnalyzer(*Init->getInit(), Context);
+      ExprMutationAnalyzer InitAnalyzer(*Init->getInit(), Context,
+                                        CrossAnalysisCache);
       for (const ParmVarDecl *Parm : Ctor->parameters()) {
         if (Results.contains(Parm))
           continue;
@@ -675,11 +678,14 @@ FunctionParmMutationAnalyzer::findMutation(const ParmVarDecl *Parm) {
   const auto Memoized = Results.find(Parm);
   if (Memoized != Results.end())
     return Memoized->second;
-
+  // To handle call A -> call B -> call A. Assume parameters of A is not mutated
+  // before analyzing parameters of A. Then when analyzing the second "call A",
+  // FunctionParmMutationAnalyzer can use this memoized value to avoid infinite
+  // recursion.
+  Results[Parm] = nullptr;
   if (const Stmt *S = BodyAnalyzer.findMutation(Parm))
     return Results[Parm] = S;
-
-  return Results[Parm] = nullptr;
+  return Results[Parm];
 }
 
 } // namespace clang
diff --git a/clang/unittests/Analysis/ExprMutationAnalyzerTest.cpp b/clang/unittests/Analysis/ExprMutationAnalyzerTest.cpp
index f58ce4aebcbf..9c1dc1a76db6 100644
--- a/clang/unittests/Analysis/ExprMutationAnalyzerTest.cpp
+++ b/clang/unittests/Analysis/ExprMutationAnalyzerTest.cpp
@@ -977,6 +977,36 @@ TEST(ExprMutationAnalyzerTest, FollowFuncArgModified) {
                          "void f() { int x; g(x); }");
   Results = match(withEnclosingCompound(declRefTo("x")), AST->getASTContext());
   EXPECT_THAT(mutatedBy(Results, AST.get()), ElementsAre("g(x)"));
+
+  AST = buildASTFromCode(
+      StdRemoveReference + StdForward +
+      "template <class T> void f1(T &&a);"
+      "template <class T> void f2(T &&a);"
+      "template <class T> void f1(T &&a) { f2<T>(std::forward<T>(a)); }"
+      "template <class T> void f2(T &&a) { f1<T>(std::forward<T>(a)); }"
+      "void f() { int x; f1(x); }");
+  Results = match(withEnclosingCompound(declRefTo("x")), AST->getASTContext());
+  EXPECT_FALSE(isMutated(Results, AST.get()));
+
+  AST = buildASTFromCode(
+      StdRemoveReference + StdForward +
+      "template <class T> void f1(T &&a);"
+      "template <class T> void f2(T &&a);"
+      "template <class T> void f1(T &&a) { f2<T>(std::forward<T>(a)); }"
+      "template <class T> void f2(T &&a) { f1<T>(std::forward<T>(a)); a++; }"
+      "void f() { int x; f1(x); }");
+  Results = match(withEnclosingCompound(declRefTo("x")), AST->getASTContext());
+  EXPECT_THAT(mutatedBy(Results, AST.get()), ElementsAre("f1(x)"));
+
+  AST = buildASTFromCode(
+      StdRemoveReference + StdForward +
+      "template <class T> void f1(T &&a);"
+      "template <class T> void f2(T &&a);"
+      "template <class T> void f1(T &&a) { f2<T>(std::forward<T>(a)); a++; }"
+      "template <class T> void f2(T &&a) { f1<T>(std::forward<T>(a)); }"
+      "void f() { int x; f1(x); }");
+  Results = match(withEnclosingCompound(declRefTo("x")), AST->getASTContext());
+  EXPECT_THAT(mutatedBy(Results, AST.get()), ElementsAre("f1(x)"));
 }
 
 TEST(ExprMutationAnalyzerTest, FollowFuncArgNotModified) {
-- 
GitLab


From 673da8c83413c4d1e7e76e1b52a2924e837e7221 Mon Sep 17 00:00:00 2001
From: Paul Walker <paul.walker@arm.com>
Date: Mon, 15 Apr 2024 12:51:06 +0100
Subject: [PATCH 233/464] [LLVM][CodeGen][AArch64] Remove bogus lowering of
 sve_while{gt/ge/hi/hs} intrinsics. (#88126)

When fed constant operands we try to lower WHILE intrinsics to PTRUE
using a fixed length pattern. This is not valid for the decrementing
variants of WHILE because they construct their result predicate vector
by traversing from high->low lanes whereas the incrementing variants and
PTRUE traverse from low->high lanes.

Whilst we can still utilise PTRUE by reversing its result I figure
replacing a single WHILE with multiple SVE instructions is likely
counterproductive.
---
 .../Target/AArch64/AArch64ISelLowering.cpp    |  39 +---
 .../CodeGen/AArch64/sve2-intrinsics-while.ll  | 212 ++----------------
 2 files changed, 28 insertions(+), 223 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 80181a77c9d2..27dc79f3f8d0 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -5035,8 +5035,8 @@ static inline SDValue getPTrue(SelectionDAG &DAG, SDLoc DL, EVT VT,
                      DAG.getTargetConstant(Pattern, DL, MVT::i32));
 }
 
-static SDValue optimizeWhile(SDValue Op, SelectionDAG &DAG, bool IsSigned,
-                             bool IsLess, bool IsEqual) {
+static SDValue optimizeIncrementingWhile(SDValue Op, SelectionDAG &DAG,
+                                         bool IsSigned, bool IsEqual) {
   if (!isa<ConstantSDNode>(Op.getOperand(1)) ||
       !isa<ConstantSDNode>(Op.getOperand(2)))
     return SDValue();
@@ -5044,12 +5044,9 @@ static SDValue optimizeWhile(SDValue Op, SelectionDAG &DAG, bool IsSigned,
   SDLoc dl(Op);
   APInt X = Op.getConstantOperandAPInt(1);
   APInt Y = Op.getConstantOperandAPInt(2);
-  APInt NumActiveElems;
   bool Overflow;
-  if (IsLess)
-    NumActiveElems = IsSigned ? Y.ssub_ov(X, Overflow) : Y.usub_ov(X, Overflow);
-  else
-    NumActiveElems = IsSigned ? X.ssub_ov(Y, Overflow) : X.usub_ov(Y, Overflow);
+  APInt NumActiveElems =
+      IsSigned ? Y.ssub_ov(X, Overflow) : Y.usub_ov(X, Overflow);
 
   if (Overflow)
     return SDValue();
@@ -5396,29 +5393,17 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
     return SDValue();
   }
   case Intrinsic::aarch64_sve_whilelo:
-    return optimizeWhile(Op, DAG, /*IsSigned=*/false, /*IsLess=*/true,
-                         /*IsEqual=*/false);
+    return optimizeIncrementingWhile(Op, DAG, /*IsSigned=*/false,
+                                     /*IsEqual=*/false);
   case Intrinsic::aarch64_sve_whilelt:
-    return optimizeWhile(Op, DAG, /*IsSigned=*/true, /*IsLess=*/true,
-                         /*IsEqual=*/false);
+    return optimizeIncrementingWhile(Op, DAG, /*IsSigned=*/true,
+                                     /*IsEqual=*/false);
   case Intrinsic::aarch64_sve_whilels:
-    return optimizeWhile(Op, DAG, /*IsSigned=*/false, /*IsLess=*/true,
-                         /*IsEqual=*/true);
+    return optimizeIncrementingWhile(Op, DAG, /*IsSigned=*/false,
+                                     /*IsEqual=*/true);
   case Intrinsic::aarch64_sve_whilele:
-    return optimizeWhile(Op, DAG, /*IsSigned=*/true, /*IsLess=*/true,
-                         /*IsEqual=*/true);
-  case Intrinsic::aarch64_sve_whilege:
-    return optimizeWhile(Op, DAG, /*IsSigned=*/true, /*IsLess=*/false,
-                         /*IsEqual=*/true);
-  case Intrinsic::aarch64_sve_whilegt:
-    return optimizeWhile(Op, DAG, /*IsSigned=*/true, /*IsLess=*/false,
-                         /*IsEqual=*/false);
-  case Intrinsic::aarch64_sve_whilehs:
-    return optimizeWhile(Op, DAG, /*IsSigned=*/false, /*IsLess=*/false,
-                         /*IsEqual=*/true);
-  case Intrinsic::aarch64_sve_whilehi:
-    return optimizeWhile(Op, DAG, /*IsSigned=*/false, /*IsLess=*/false,
-                         /*IsEqual=*/false);
+    return optimizeIncrementingWhile(Op, DAG, /*IsSigned=*/true,
+                                     /*IsEqual=*/true);
   case Intrinsic::aarch64_sve_sunpkhi:
     return DAG.getNode(AArch64ISD::SUNPKHI, dl, Op.getValueType(),
                        Op.getOperand(1));
diff --git a/llvm/test/CodeGen/AArch64/sve2-intrinsics-while.ll b/llvm/test/CodeGen/AArch64/sve2-intrinsics-while.ll
index 9b82d79cfa96..2052ebf4c8bb 100644
--- a/llvm/test/CodeGen/AArch64/sve2-intrinsics-while.ll
+++ b/llvm/test/CodeGen/AArch64/sve2-intrinsics-while.ll
@@ -78,68 +78,16 @@ define <vscale x 2 x i1> @whilege_d_xx(i64 %a, i64 %b) {
   ret <vscale x 2 x i1> %out
 }
 
-define <vscale x 2 x i1> @whilege_d_ii_dont_fold_to_ptrue_larger_than_minvec() {
-; CHECK-LABEL: whilege_d_ii_dont_fold_to_ptrue_larger_than_minvec:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #3
-; CHECK-NEXT:    whilege p0.d, x8, xzr
-; CHECK-NEXT:    ret
-  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.whilege.nxv2i1.i64(i64 3, i64 0)
-  ret <vscale x 2 x i1> %out
-}
-
+; Ensure we don't convert constant decrementing while instructions to ptrue.
 define <vscale x 16 x i1> @whilege_b_ii() {
 ; CHECK-LABEL: whilege_b_ii:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ptrue p0.b, vl6
-; CHECK-NEXT:    ret
-entry:
-  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilege.nxv16i1.i32(i32 3, i32 -2)
-  ret <vscale x 16 x i1> %out
-}
-
-define <vscale x 16 x i1> @whilege_b_ii_dont_fold_to_ptrue_nonexistent_vl9() {
-; CHECK-LABEL: whilege_b_ii_dont_fold_to_ptrue_nonexistent_vl9:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov w8, #9
-; CHECK-NEXT:    whilege p0.b, x8, xzr
-; CHECK-NEXT:    ret
-entry:
-  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilege.nxv16i1.i64(i64 9, i64 0)
-  ret <vscale x 16 x i1> %out
-}
-
-define <vscale x 16 x i1> @whilege_b_ii_vl_maximum() vscale_range(16, 16) {
-; CHECK-LABEL: whilege_b_ii_vl_maximum:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p0.b, vl256
-; CHECK-NEXT:    ret
-  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilege.nxv16i1.i64(i64 255, i64 0)
-  ret <vscale x 16 x i1> %out
-}
-
-define <vscale x 16 x i1> @whilege_b_ii_dont_fold_to_ptrue_overflow() {
-; CHECK-LABEL: whilege_b_ii_dont_fold_to_ptrue_overflow:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov w8, #2
-; CHECK-NEXT:    mov w9, #2147483647
-; CHECK-NEXT:    movk w8, #32768, lsl #16
+; CHECK-NEXT:    mov w8, #-2 // =0xfffffffe
+; CHECK-NEXT:    mov w9, #3 // =0x3
 ; CHECK-NEXT:    whilege p0.b, w9, w8
 ; CHECK-NEXT:    ret
 entry:
-  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilege.nxv16i1.i32(i32 2147483647, i32 -2147483646)
-  ret <vscale x 16 x i1> %out
-}
-
-define <vscale x 16 x i1> @whilege_b_ii_dont_fold_to_ptrue_increment_overflow() {
-; CHECK-LABEL: whilege_b_ii_dont_fold_to_ptrue_increment_overflow:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov w8, #2147483647
-; CHECK-NEXT:    mov w9, #-2147483641
-; CHECK-NEXT:    whilege p0.b, w9, w8
-; CHECK-NEXT:    ret
-entry:
-  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilege.nxv16i1.i32(i32 -2147483641, i32 2147483647)
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilege.nxv16i1.i32(i32 3, i32 -2)
   ret <vscale x 16 x i1> %out
 }
 
@@ -219,69 +167,19 @@ define <vscale x 2 x i1> @whilehs_d_xx(i64 %a, i64 %b) {
   ret <vscale x 2 x i1> %out
 }
 
-define <vscale x 2 x i1> @whilehs_d_ii_dont_fold_to_ptrue_larger_than_minvec() {
-; CHECK-LABEL: whilehs_d_ii_dont_fold_to_ptrue_larger_than_minvec:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #3
-; CHECK-NEXT:    whilehs p0.d, x8, xzr
-; CHECK-NEXT:    ret
-  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.whilehs.nxv2i1.i64(i64 3, i64 0)
-  ret <vscale x 2 x i1> %out
-}
-
+; Ensure we don't convert constant decrementing while instructions to ptrue.
 define <vscale x 16 x i1> @whilehs_b_ii() {
 ; CHECK-LABEL: whilehs_b_ii:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ptrue p0.b, vl7
+; CHECK-NEXT:    mov w8, #2 // =0x2
+; CHECK-NEXT:    mov w9, #8 // =0x8
+; CHECK-NEXT:    whilehs p0.b, x9, x8
 ; CHECK-NEXT:    ret
 entry:
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilehs.nxv16i1.i64(i64 8, i64 2)
   ret <vscale x 16 x i1> %out
 }
 
-define <vscale x 16 x i1> @whilehs_b_ii_dont_fold_to_ptrue_nonexistent_vl9() {
-; CHECK-LABEL: whilehs_b_ii_dont_fold_to_ptrue_nonexistent_vl9:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov w8, #9
-; CHECK-NEXT:    whilehs p0.b, x8, xzr
-; CHECK-NEXT:    ret
-entry:
-  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilehs.nxv16i1.i64(i64 9, i64 0)
-  ret <vscale x 16 x i1> %out
-}
-
-define <vscale x 16 x i1> @whilehs_b_ii_vl_maximum() vscale_range(16, 16) {
-; CHECK-LABEL: whilehs_b_ii_vl_maximum:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p0.b, vl256
-; CHECK-NEXT:    ret
-  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilehs.nxv16i1.i64(i64 255, i64 0)
-  ret <vscale x 16 x i1> %out
-}
-
-define <vscale x 16 x i1> @whilehs_b_ii_dont_fold_to_ptrue_overflow() {
-; CHECK-LABEL: whilehs_b_ii_dont_fold_to_ptrue_overflow:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov w8, #-1
-; CHECK-NEXT:    mov w9, #6
-; CHECK-NEXT:    whilehs p0.b, w9, w8
-; CHECK-NEXT:    ret
-entry:
-  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilehs.nxv16i1.i32(i32 6, i32 4294967295)
-  ret <vscale x 16 x i1> %out
-}
-
-define <vscale x 16 x i1> @whilehs_b_ii_dont_fold_to_ptrue_increment_overflow() {
-; CHECK-LABEL: whilehs_b_ii_dont_fold_to_ptrue_increment_overflow:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov w8, #-1
-; CHECK-NEXT:    whilehs p0.b, w8, wzr
-; CHECK-NEXT:    ret
-entry:
-  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilehs.nxv16i1.i32(i32 4294967295, i32 0)
-  ret <vscale x 16 x i1> %out
-}
-
 ;
 ; WHILEGT
 ;
@@ -358,55 +256,16 @@ define <vscale x 2 x i1> @whilegt_d_xx(i64 %a, i64 %b) {
   ret <vscale x 2 x i1> %out
 }
 
-define <vscale x 2 x i1> @whilegt_d_ii_dont_fold_to_ptrue_larger_than_minvec() {
-; CHECK-LABEL: whilegt_d_ii_dont_fold_to_ptrue_larger_than_minvec:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #3
-; CHECK-NEXT:    whilegt p0.d, x8, xzr
-; CHECK-NEXT:    ret
-  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.whilegt.nxv2i1.i64(i64 3, i64 0)
-  ret <vscale x 2 x i1> %out
-}
-
+; Ensure we don't convert constant decrementing while instructions to ptrue.
 define <vscale x 16 x i1> @whilegt_b_ii() {
 ; CHECK-LABEL: whilegt_b_ii:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ptrue p0.b, vl5
-; CHECK-NEXT:    ret
-entry:
-  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilegt.nxv16i1.i32(i32 3, i32 -2)
-  ret <vscale x 16 x i1> %out
-}
-
-define <vscale x 16 x i1> @whilegt_b_ii_fold_to_ptrue_nonexistent_vl9() {
-; CHECK-LABEL: whilegt_b_ii_fold_to_ptrue_nonexistent_vl9:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov w8, #9
-; CHECK-NEXT:    whilegt p0.b, x8, xzr
-; CHECK-NEXT:    ret
-entry:
-  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilegt.nxv16i1.i64(i64 9, i64 0)
-  ret <vscale x 16 x i1> %out
-}
-
-define <vscale x 16 x i1> @whilegt_b_ii_vl_maximum() vscale_range(16, 16) {
-; CHECK-LABEL: whilegt_b_ii_vl_maximum:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p0.b, vl256
-; CHECK-NEXT:    ret
-  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilegt.nxv16i1.i64(i64 256, i64 0)
-  ret <vscale x 16 x i1> %out
-}
-
-define <vscale x 16 x i1> @whilegt_b_ii_dont_fold_to_ptrue_overflow() {
-; CHECK-LABEL: whilegt_b_ii_dont_fold_to_ptrue_overflow:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov w8, #2147483647
-; CHECK-NEXT:    mov w9, #-2147483641
+; CHECK-NEXT:    mov w8, #-2 // =0xfffffffe
+; CHECK-NEXT:    mov w9, #3 // =0x3
 ; CHECK-NEXT:    whilegt p0.b, w9, w8
 ; CHECK-NEXT:    ret
 entry:
-  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilegt.nxv16i1.i32(i32 -2147483641, i32 2147483647)
+  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilegt.nxv16i1.i32(i32 3, i32 -2)
   ret <vscale x 16 x i1> %out
 }
 
@@ -486,58 +345,19 @@ define <vscale x 2 x i1> @whilehi_d_xx(i64 %a, i64 %b) {
   ret <vscale x 2 x i1> %out
 }
 
-define <vscale x 2 x i1> @whilehi_d_ii_dont_fold_to_ptrue_larger_than_minvec() {
-; CHECK-LABEL: whilehi_d_ii_dont_fold_to_ptrue_larger_than_minvec:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #3
-; CHECK-NEXT:    whilehi p0.d, x8, xzr
-; CHECK-NEXT:    ret
-  %out = call <vscale x 2 x i1> @llvm.aarch64.sve.whilehi.nxv2i1.i64(i64 3, i64 0)
-  ret <vscale x 2 x i1> %out
-}
-
+; Ensure we don't convert constant decrementing while instructions to ptrue.
 define <vscale x 16 x i1> @whilehi_b_ii() {
 ; CHECK-LABEL: whilehi_b_ii:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ptrue p0.b, vl6
+; CHECK-NEXT:    mov w8, #2 // =0x2
+; CHECK-NEXT:    mov w9, #8 // =0x8
+; CHECK-NEXT:    whilehi p0.b, x9, x8
 ; CHECK-NEXT:    ret
 entry:
   %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilehi.nxv16i1.i64(i64 8, i64 2)
   ret <vscale x 16 x i1> %out
 }
 
-define <vscale x 16 x i1> @whilehi_b_ii_dont_fold_to_ptrue_nonexistent_vl9() {
-; CHECK-LABEL: whilehi_b_ii_dont_fold_to_ptrue_nonexistent_vl9:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov w8, #9
-; CHECK-NEXT:    whilehi p0.b, x8, xzr
-; CHECK-NEXT:    ret
-entry:
-  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilehi.nxv16i1.i64(i64 9, i64 0)
-  ret <vscale x 16 x i1> %out
-}
-
-define <vscale x 16 x i1> @whilehi_b_ii_vl_maximum() vscale_range(16, 16) {
-; CHECK-LABEL: whilehi_b_ii_vl_maximum:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ptrue p0.b, vl256
-; CHECK-NEXT:    ret
-  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilehi.nxv16i1.i64(i64 256, i64 0)
-  ret <vscale x 16 x i1> %out
-}
-
-define <vscale x 16 x i1> @whilelhi_b_ii_dont_fold_to_ptrue_overflow() {
-; CHECK-LABEL: whilelhi_b_ii_dont_fold_to_ptrue_overflow:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    mov w8, #-1
-; CHECK-NEXT:    mov w9, #7
-; CHECK-NEXT:    whilehi p0.b, w9, w8
-; CHECK-NEXT:    ret
-entry:
-  %out = call <vscale x 16 x i1> @llvm.aarch64.sve.whilehi.nxv16i1.i32(i32 7, i32 4294967295)
-  ret <vscale x 16 x i1> %out
-}
-
 declare <vscale x 16 x i1> @llvm.aarch64.sve.whilege.nxv16i1.i32(i32, i32)
 declare <vscale x 16 x i1> @llvm.aarch64.sve.whilege.nxv16i1.i64(i64, i64)
 declare <vscale x 8 x i1> @llvm.aarch64.sve.whilege.nxv8i1.i32(i32, i32)
-- 
GitLab


From 7b039c0cf0045d81da7e1575232a46266cff1e88 Mon Sep 17 00:00:00 2001
From: Stephen Tozer <stephen.tozer@sony.com>
Date: Mon, 15 Apr 2024 13:15:27 +0100
Subject: [PATCH 234/464] [DebugInfo][RemoveDIs] Fix error from not tracking
 seen debug labels (#88718)

Fixes the reported errors on:
https://github.com/llvm/llvm-project/pull/87379

A previous patch updated the bitcode reading for debug
intrinsics/records to not perform the expensive debug info format
conversion from records to intrinsics in cases where no records were
present, but the patch did not actually track when debug labels had been
seen, resulting in errors when parsing bitcode where functions contained
debug label records but no other debug records. This patch fixes that
case and adds a test for it.
---
 llvm/lib/Bitcode/Reader/BitcodeReader.cpp |  1 +
 llvm/test/Bitcode/dbg-label-record-bc.ll  | 50 +++++++++++++++++++++++
 2 files changed, 51 insertions(+)
 create mode 100644 llvm/test/Bitcode/dbg-label-record-bc.ll

diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index fe4f0d6dca6c..0b7fcd884188 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -6454,6 +6454,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
     case bitc::FUNC_CODE_DEBUG_RECORD_LABEL: {
       // DbgLabelRecords are placed after the Instructions that they are
       // attached to.
+      SeenDebugRecord = true;
       Instruction *Inst = getLastInstruction();
       if (!Inst)
         return error("Invalid dbg record: missing instruction");
diff --git a/llvm/test/Bitcode/dbg-label-record-bc.ll b/llvm/test/Bitcode/dbg-label-record-bc.ll
new file mode 100644
index 000000000000..e151f7f6cc15
--- /dev/null
+++ b/llvm/test/Bitcode/dbg-label-record-bc.ll
@@ -0,0 +1,50 @@
+;; Tests that we can parse and print a function containing a debug label record
+;; and no other debug record kinds.
+
+; RUN: llvm-as --write-experimental-debuginfo-iterators-to-bitcode=true %s -o - \
+; RUN: | opt -S | FileCheck %s --check-prefixes=CHECK,INTRINSIC
+
+; RUN: llvm-as --write-experimental-debuginfo-iterators-to-bitcode=true %s -o - \
+; RUN: | opt -S --preserve-input-debuginfo-format=true \
+; RUN: | FileCheck %s --check-prefixes=CHECK,RECORD
+
+source_filename = "bbi-94196.c"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK-LABEL: void @foo()
+; CHECK: bar:
+; INTRINSIC-NEXT: call void @llvm.dbg.label(metadata ![[LABEL:[0-9]+]]), !dbg ![[LOC:[0-9]+]]
+; RECORD-NEXT: #dbg_label(![[LABEL:[0-9]+]], ![[LOC:[0-9]+]])
+
+; CHECK-DAG: ![[LABEL]] = !DILabel({{.*}}name: "bar"
+; CHECK-DAG: ![[LOC]] = !DILocation(line: 5, column: 1
+
+define dso_local void @foo() !dbg !5 {
+entry:
+  br label %bar, !dbg !9
+
+bar:                                              ; preds = %entry
+  tail call void @llvm.dbg.label(metadata !10), !dbg !11
+  ret void, !dbg !12
+}
+
+declare void @llvm.dbg.label(metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3}
+!llvm.ident = !{!4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang version 19.0.0git", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "<stdin>", directory: "/home/gbtozers/dev/llvm-project-ddd-textual-ir")
+!2 = !{i32 2, !"Debug Info Version", i32 3}
+!3 = !{i32 1, !"wchar_size", i32 4}
+!4 = !{!"clang version 19.0.0git"}
+!5 = distinct !DISubprogram(name: "foo", scope: !6, file: !6, line: 1, type: !7, scopeLine: 2, spFlags: DISPFlagDefinition, unit: !0)
+!6 = !DIFile(filename: "bbi-94196.c", directory: "/home/gbtozers/dev/llvm-project-ddd-textual-ir")
+!7 = !DISubroutineType(types: !8)
+!8 = !{null}
+!9 = !DILocation(line: 3, column: 3, scope: !5)
+!10 = !DILabel(scope: !5, name: "bar", file: !6, line: 5)
+!11 = !DILocation(line: 5, column: 1, scope: !5)
+!12 = !DILocation(line: 6, column: 3, scope: !5)
-- 
GitLab


From 58d4470fa49443da1477a7d2e43685e91bbd6630 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Mon, 15 Apr 2024 13:02:18 +0100
Subject: [PATCH 235/464] [X86] Remove scaleVectorShuffleBlendMask and use
 APIntOps::ScaleBitMask directly.

---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 18 +++++-------------
 1 file changed, 5 insertions(+), 13 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 0a91e3f554b3..f16a751a166d 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -10589,15 +10589,6 @@ static bool matchShuffleAsBlend(MVT VT, SDValue V1, SDValue V2,
   return true;
 }
 
-static uint64_t scaleVectorShuffleBlendMask(uint64_t BlendMask, int Size,
-                                            int Scale) {
-  uint64_t ScaledMask = 0;
-  for (int i = 0; i != Size; ++i)
-    if (BlendMask & (1ull << i))
-      ScaledMask |= ((1ull << Scale) - 1) << (i * Scale);
-  return ScaledMask;
-}
-
 /// Try to emit a blend instruction for a shuffle.
 ///
 /// This doesn't do any checks for the availability of instructions for blending
@@ -40539,14 +40530,15 @@ static SDValue combineTargetShuffle(SDValue N, const SDLoc &DL,
       MVT SrcVT = N0.getOperand(0).getSimpleValueType();
       if ((VT.getScalarSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
           SrcVT.getScalarSizeInBits() >= 32) {
-        unsigned BlendMask = N.getConstantOperandVal(2);
         unsigned Size = VT.getVectorNumElements();
-        unsigned Scale = VT.getScalarSizeInBits() / SrcVT.getScalarSizeInBits();
-        BlendMask = scaleVectorShuffleBlendMask(BlendMask, Size, Scale);
+        unsigned NewSize = SrcVT.getVectorNumElements();
+        APInt BlendMask = N.getConstantOperandAPInt(2).zextOrTrunc(Size);
+        APInt NewBlendMask = APIntOps::ScaleBitMask(BlendMask, NewSize);
         return DAG.getBitcast(
             VT, DAG.getNode(X86ISD::BLENDI, DL, SrcVT, N0.getOperand(0),
                             N1.getOperand(0),
-                            DAG.getTargetConstant(BlendMask, DL, MVT::i8)));
+                            DAG.getTargetConstant(NewBlendMask.getZExtValue(),
+                                                  DL, MVT::i8)));
       }
     }
     return SDValue();
-- 
GitLab


From 7089c359a3845323f6f30c44a47dd901f2edfe63 Mon Sep 17 00:00:00 2001
From: Paul Walker <paul.walker@arm.com>
Date: Mon, 15 Apr 2024 13:50:09 +0100
Subject: [PATCH 236/464] [LLVM][SelectionDAG] Allow verification of target ISD
 nodes. (#88121)

Patch includes an initial implementation for AArch64 that covers a
handful of nodes where I've observed bogus nodes within the DAG.
---
 llvm/include/llvm/CodeGen/TargetLowering.h    |  5 +++
 .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp |  6 ++-
 .../Target/AArch64/AArch64ISelLowering.cpp    | 43 +++++++++++++++++++
 llvm/lib/Target/AArch64/AArch64ISelLowering.h |  4 ++
 4 files changed, 56 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 980d75ad91a0..e0ade0295902 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -4873,6 +4873,11 @@ public:
   bool verifyReturnAddressArgumentIsConstant(SDValue Op,
                                              SelectionDAG &DAG) const;
 
+#ifndef NDEBUG
+  /// Check the given SDNode.  Aborts if it is invalid.
+  virtual void verifyTargetSDNode(const SDNode *N) const {};
+#endif
+
   //===--------------------------------------------------------------------===//
   // Inline Asm Support hooks
   //
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index fc972664e5f0..ca0a95750ba8 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -1111,9 +1111,11 @@ void SelectionDAG::DeallocateNode(SDNode *N) {
 
 #ifndef NDEBUG
 /// VerifySDNode - Check the given SDNode.  Aborts if it is invalid.
-static void VerifySDNode(SDNode *N) {
+static void VerifySDNode(SDNode *N, const TargetLowering *TLI) {
   switch (N->getOpcode()) {
   default:
+    if (N->getOpcode() > ISD::BUILTIN_OP_END)
+      TLI->verifyTargetSDNode(N);
     break;
   case ISD::BUILD_PAIR: {
     EVT VT = N->getValueType(0);
@@ -1157,7 +1159,7 @@ void SelectionDAG::InsertNode(SDNode *N) {
   AllNodes.push_back(N);
 #ifndef NDEBUG
   N->PersistentId = NextPersistentId++;
-  VerifySDNode(N);
+  VerifySDNode(N, TLI);
 #endif
   for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
     DUL->NodeInserted(N);
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 27dc79f3f8d0..eee67a0f823c 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -27868,3 +27868,46 @@ bool AArch64TargetLowering::hasInlineStackProbe(
   return !Subtarget->isTargetWindows() &&
          MF.getInfo<AArch64FunctionInfo>()->hasStackProbing();
 }
+
+#ifndef NDEBUG
+void AArch64TargetLowering::verifyTargetSDNode(const SDNode *N) const {
+  switch (N->getOpcode()) {
+  default:
+    break;
+  case AArch64ISD::SUNPKLO:
+  case AArch64ISD::SUNPKHI:
+  case AArch64ISD::UUNPKLO:
+  case AArch64ISD::UUNPKHI: {
+    assert(N->getNumValues() == 1 && "Expected one result!");
+    assert(N->getNumOperands() == 1 && "Expected one operand!");
+    EVT VT = N->getValueType(0);
+    EVT OpVT = N->getOperand(0).getValueType();
+    assert(OpVT.isVector() && VT.isVector() && OpVT.isInteger() &&
+           VT.isInteger() && "Expected integer vectors!");
+    assert(OpVT.getSizeInBits() == VT.getSizeInBits() &&
+           "Expected vectors of equal size!");
+    // TODO: Enable assert once bogus creations have been fixed.
+    // assert(OpVT.getVectorElementCount() == VT.getVectorElementCount()*2 &&
+    //       "Expected result vector with half the lanes of its input!");
+    break;
+  }
+  case AArch64ISD::TRN1:
+  case AArch64ISD::TRN2:
+  case AArch64ISD::UZP1:
+  case AArch64ISD::UZP2:
+  case AArch64ISD::ZIP1:
+  case AArch64ISD::ZIP2: {
+    assert(N->getNumValues() == 1 && "Expected one result!");
+    assert(N->getNumOperands() == 2 && "Expected two operands!");
+    EVT VT = N->getValueType(0);
+    EVT Op0VT = N->getOperand(0).getValueType();
+    EVT Op1VT = N->getOperand(1).getValueType();
+    assert(VT.isVector() && Op0VT.isVector() && Op1VT.isVector() &&
+           "Expected vectors!");
+    // TODO: Enable assert once bogus creations have been fixed.
+    // assert(VT == Op0VT && VT == Op1VT && "Expected matching vectors!");
+    break;
+  }
+  }
+}
+#endif
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 18439dc7f010..db6e8a00d2fb 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -998,6 +998,10 @@ public:
   /// True if stack clash protection is enabled for this functions.
   bool hasInlineStackProbe(const MachineFunction &MF) const override;
 
+#ifndef NDEBUG
+  void verifyTargetSDNode(const SDNode *N) const override;
+#endif
+
 private:
   /// Keep a pointer to the AArch64Subtarget around so that we can
   /// make the right decision when generating code for different targets.
-- 
GitLab


From 2e0e3b0f10b5c433504e31ca73e55f86e0b27660 Mon Sep 17 00:00:00 2001
From: Michael Maitland <michaeltmaitland@gmail.com>
Date: Fri, 12 Apr 2024 07:50:59 -0700
Subject: [PATCH 237/464] [RISCV] Split single width convert to FP pseudos by
 SEW

---
 .../Target/RISCV/RISCVInstrInfoVPseudos.td    | 61 +++++++++++--------
 .../Target/RISCV/RISCVInstrInfoVSDPatterns.td |  2 +-
 .../Target/RISCV/RISCVInstrInfoVVLPatterns.td |  4 +-
 llvm/lib/Target/RISCV/RISCVSchedSiFive7.td    |  4 +-
 llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td | 26 ++------
 llvm/lib/Target/RISCV/RISCVScheduleV.td       |  8 +--
 6 files changed, 51 insertions(+), 54 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index a5f56a1c209c..e9cb02759270 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -3593,12 +3593,15 @@ multiclass VPseudoConversionRoundingMode<VReg RetClass,
                              VReg Op1Class,
                              LMULInfo MInfo,
                              string Constraint = "",
+                             int sew = 0,
                              int TargetConstraintType = 1> {
   let VLMul = MInfo.value in {
-    def "_" # MInfo.MX : VPseudoUnaryNoMaskRoundingMode<RetClass, Op1Class, Constraint, TargetConstraintType>;
-    def "_" # MInfo.MX # "_MASK" : VPseudoUnaryMaskRoundingMode<RetClass, Op1Class,
-                                                                Constraint, TargetConstraintType>,
-                                   RISCVMaskedPseudo<MaskIdx=2>;
+    defvar suffix = !if(sew, "_" # MInfo.MX # "_E" # sew, "_" # MInfo.MX);
+    def suffix : VPseudoUnaryNoMaskRoundingMode<RetClass, Op1Class, Constraint, TargetConstraintType>;
+    def suffix # "_MASK" : VPseudoUnaryMaskRoundingMode<RetClass, Op1Class,
+                                                        Constraint,
+                                                        TargetConstraintType>,
+                           RISCVMaskedPseudo<MaskIdx=2>;
   }
 }
 
@@ -3607,13 +3610,15 @@ multiclass VPseudoConversionRM<VReg RetClass,
                                VReg Op1Class,
                                LMULInfo MInfo,
                                string Constraint = "",
+                               int sew = 0,
                                int TargetConstraintType = 1> {
   let VLMul = MInfo.value in {
-    def "_" # MInfo.MX : VPseudoUnaryNoMask_FRM<RetClass, Op1Class,
-                                                        Constraint, TargetConstraintType>;
-    def "_" # MInfo.MX # "_MASK" : VPseudoUnaryMask_FRM<RetClass, Op1Class,
-                                                        Constraint, TargetConstraintType>,
-                                   RISCVMaskedPseudo<MaskIdx=2>;
+    defvar suffix = !if(sew, "_" # MInfo.MX # "_E" # sew, "_" # MInfo.MX);
+    def suffix : VPseudoUnaryNoMask_FRM<RetClass, Op1Class,
+                                        Constraint, TargetConstraintType>;
+    def suffix # "_MASK" : VPseudoUnaryMask_FRM<RetClass, Op1Class,
+                                                Constraint, TargetConstraintType>,
+                           RISCVMaskedPseudo<MaskIdx=2>;
   }
 }
 
@@ -3660,17 +3665,19 @@ multiclass VPseudoVFROUND_NOEXCEPT_V {
 
 multiclass VPseudoVCVTF_V_RM {
   foreach m = MxListF in {
-    defm _V : VPseudoConversionRoundingMode<m.vrclass, m.vrclass, m>,
-              SchedUnary<"WriteVFCvtIToFV", "ReadVFCvtIToFV", m.MX,
-                         forceMergeOpRead=true>;
+    foreach e = SchedSEWSet<m.MX, isF=1>.val in
+      defm _V : VPseudoConversionRoundingMode<m.vrclass, m.vrclass, m, sew=e>,
+                SchedUnary<"WriteVFCvtIToFV", "ReadVFCvtIToFV", m.MX, e,
+                           forceMergeOpRead=true>;
   }
 }
 
 multiclass VPseudoVCVTF_RM_V {
   foreach m = MxListF in {
-    defm _V : VPseudoConversionRM<m.vrclass, m.vrclass, m>,
-              SchedUnary<"WriteVFCvtIToFV", "ReadVFCvtIToFV", m.MX,
-                         forceMergeOpRead=true>;
+    foreach e = SchedSEWSet<m.MX, isF=1>.val in
+      defm _V : VPseudoConversionRM<m.vrclass, m.vrclass, m, sew=e>,
+                SchedUnary<"WriteVFCvtIToFV", "ReadVFCvtIToFV", m.MX, e,
+                           forceMergeOpRead=true>;
   }
 }
 
@@ -4905,14 +4912,17 @@ multiclass VPatConversionTARoundingMode<string intrinsic,
                                         ValueType result_type,
                                         ValueType op1_type,
                                         ValueType mask_type,
-                                        int sew,
+                                        int log2sew,
                                         LMULInfo vlmul,
                                         VReg result_reg_class,
-                                        VReg op1_reg_class> {
+                                        VReg op1_reg_class,
+                                        bit isSEWAware = 0> {
   def : VPatUnaryNoMaskRoundingMode<intrinsic, inst, kind, result_type, op1_type,
-                                    sew, vlmul, result_reg_class, op1_reg_class>;
+                                    log2sew, vlmul, result_reg_class,
+                                    op1_reg_class, isSEWAware>;
   def : VPatUnaryMaskRoundingMode<intrinsic, inst, kind, result_type, op1_type,
-                                  mask_type, sew, vlmul, result_reg_class, op1_reg_class>;
+                                  mask_type, log2sew, vlmul, result_reg_class,
+                                  op1_reg_class, isSEWAware>;
 }
 
 multiclass VPatBinaryV_VV<string intrinsic, string instruction,
@@ -5905,15 +5915,16 @@ multiclass VPatConversionVI_VF_RM<string intrinsic,
   }
 }
 
-multiclass VPatConversionVF_VI_RM<string intrinsic,
-                                  string instruction> {
+multiclass VPatConversionVF_VI_RM<string intrinsic, string instruction,
+                                  bit isSEWAware = 0> {
   foreach fvti = AllFloatVectors in {
     defvar ivti = GetIntVTypeInfo<fvti>.Vti;
     let Predicates = !listconcat(GetVTypePredicates<fvti>.Predicates,
                                  GetVTypePredicates<ivti>.Predicates) in
     defm : VPatConversionTARoundingMode<intrinsic, instruction, "V",
                                         fvti.Vector, ivti.Vector, fvti.Mask, ivti.Log2SEW,
-                                        ivti.LMul, fvti.RegClass, ivti.RegClass>;
+                                        ivti.LMul, fvti.RegClass, ivti.RegClass,
+                                        isSEWAware>;
   }
 }
 
@@ -7269,8 +7280,10 @@ defm : VPatConversionVI_VF_RM<"int_riscv_vfcvt_x_f_v", "PseudoVFCVT_X_F">;
 defm : VPatConversionVI_VF_RM<"int_riscv_vfcvt_xu_f_v", "PseudoVFCVT_XU_F">;
 defm : VPatConversionVI_VF<"int_riscv_vfcvt_rtz_xu_f_v", "PseudoVFCVT_RTZ_XU_F">;
 defm : VPatConversionVI_VF<"int_riscv_vfcvt_rtz_x_f_v", "PseudoVFCVT_RTZ_X_F">;
-defm : VPatConversionVF_VI_RM<"int_riscv_vfcvt_f_x_v", "PseudoVFCVT_F_X">;
-defm : VPatConversionVF_VI_RM<"int_riscv_vfcvt_f_xu_v", "PseudoVFCVT_F_XU">;
+defm : VPatConversionVF_VI_RM<"int_riscv_vfcvt_f_x_v", "PseudoVFCVT_F_X",
+                              isSEWAware=1>;
+defm : VPatConversionVF_VI_RM<"int_riscv_vfcvt_f_xu_v", "PseudoVFCVT_F_XU",
+                              isSEWAware=1>;
 
 //===----------------------------------------------------------------------===//
 // 13.18. Widening Floating-Point/Integer Type-Convert Instructions
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
index 7c77449b4f6e..788db6f447a1 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
@@ -410,7 +410,7 @@ multiclass VPatConvertI2FPSDNode_V_RM<SDPatternOperator vop,
     let Predicates = !listconcat(GetVTypePredicates<fvti>.Predicates,
                                  GetVTypePredicates<ivti>.Predicates) in
     def : Pat<(fvti.Vector (vop (ivti.Vector ivti.RegClass:$rs1))),
-              (!cast<Instruction>(instruction_name#"_"#fvti.LMul.MX)
+              (!cast<Instruction>(instruction_name#"_"#fvti.LMul.MX#"_E"#fvti.SEW)
                   (fvti.Vector (IMPLICIT_DEF)),
                   ivti.RegClass:$rs1,
                   // Value to indicate no rounding mode change in
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
index 322c055306e8..2b56eea29cfc 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
@@ -1229,7 +1229,7 @@ multiclass VPatConvertI2FPVL_V_RM<SDPatternOperator vop, string instruction_name
     def : Pat<(fvti.Vector (vop (ivti.Vector ivti.RegClass:$rs1),
                                 (ivti.Mask V0),
                                 VLOpFrag)),
-              (!cast<Instruction>(instruction_name#"_"#fvti.LMul.MX#"_MASK")
+              (!cast<Instruction>(instruction_name#"_"#fvti.LMul.MX#"_E"#fvti.SEW#"_MASK")
                   (fvti.Vector (IMPLICIT_DEF)), ivti.RegClass:$rs1,
                   (ivti.Mask V0),
                   // Value to indicate no rounding mode change in
@@ -1247,7 +1247,7 @@ multiclass VPatConvertI2FP_RM_VL_V<SDNode vop, string instruction_name> {
     def : Pat<(fvti.Vector (vop (ivti.Vector ivti.RegClass:$rs1),
                                 (ivti.Mask V0), (XLenVT timm:$frm),
                                 VLOpFrag)),
-              (!cast<Instruction>(instruction_name#"_"#fvti.LMul.MX#"_MASK")
+              (!cast<Instruction>(instruction_name#"_"#fvti.LMul.MX#"_E"#fvti.SEW#"_MASK")
                   (fvti.Vector (IMPLICIT_DEF)), ivti.RegClass:$rs1,
                   (ivti.Mask V0), timm:$frm, GPR:$vl, fvti.Log2SEW, TA_MA)>;
   }
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
index 2a13cb4beac1..354fe7c27e68 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
@@ -744,6 +744,7 @@ foreach mx = SchedMxListF in {
       defm "" : LMULSEWWriteResMXSEW<"WriteVFMulAddV", [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
       defm "" : LMULSEWWriteResMXSEW<"WriteVFMulAddF", [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
       defm "" : LMULSEWWriteResMXSEW<"WriteVFRecpV",   [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFCvtIToFV", [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
     }
   }
 }
@@ -751,7 +752,6 @@ foreach mx = SchedMxList in {
   defvar Cycles = SiFive7GetCyclesDefault<mx>.c;
   defvar IsWorstCase = SiFive7IsWorstCaseMX<mx, SchedMxList>.c;
   let Latency = 8, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in {
-    defm "" : LMULWriteResMX<"WriteVFCvtIToFV",  [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFCvtFToIV",  [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
   }
   let Latency = 4, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in {
@@ -1179,7 +1179,7 @@ defm "" : LMULReadAdvance<"ReadVFClassV", 0>;
 defm "" : LMULReadAdvance<"ReadVFMergeV", 0>;
 defm "" : LMULReadAdvance<"ReadVFMergeF", 0>;
 defm "" : LMULReadAdvance<"ReadVFMovF", 0>;
-defm "" : LMULReadAdvance<"ReadVFCvtIToFV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFCvtIToFV", 0>;
 defm "" : LMULReadAdvance<"ReadVFCvtFToIV", 0>;
 defm "" : LMULReadAdvanceW<"ReadVFWCvtIToFV", 0>;
 defm "" : LMULReadAdvanceFW<"ReadVFWCvtFToIV", 0>;
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
index 80090a002714..863e245c3225 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
@@ -495,36 +495,20 @@ foreach mx = SchedMxListF in {
       defm "" : LMULSEWWriteResMXSEW<"WriteVFALUF",  [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
       defm "" : LMULSEWWriteResMXSEW<"WriteVFMulV",  [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
       defm "" : LMULSEWWriteResMXSEW<"WriteVFMulF",  [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
-
-    }
-  }
-}
-
-foreach mx = SchedMxListF in {
-  foreach sew = SchedSEWSet<mx, isF=1>.val in {
-    defvar LMulLat = SiFiveP600GetLMulCycles<mx>.c;
-    defvar IsWorstCase = SiFiveP600IsWorstCaseMXSEW<mx, sew, SchedMxListF, isF=1>.c;
-    let Latency = 6, ReleaseAtCycles = [LMulLat] in {
       defm "" : LMULSEWWriteResMXSEW<"WriteVFMulAddV", [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
       defm "" : LMULSEWWriteResMXSEW<"WriteVFMulAddF", [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
     }
-  }
-}
-foreach mx = SchedMxListF in {
-  foreach sew = SchedSEWSet<mx, isF=1>.val in {
-    defvar LMulLat = SiFiveP600GetLMulCycles<mx>.c;
-    defvar IsWorstCase = SiFiveP600IsWorstCaseMXSEW<mx, sew, SchedMxListF, isF=1>.c;
     let Latency = 2, ReleaseAtCycles = [LMulLat] in
     defm "" : LMULSEWWriteResMXSEW<"WriteVFRecpV", [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
+    let Latency = 3, ReleaseAtCycles = [LMulLat] in
+    defm "" : LMULSEWWriteResMXSEW<"WriteVFCvtIToFV", [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
   }
 }
 foreach mx = SchedMxList in {
   defvar LMulLat = SiFiveP600GetLMulCycles<mx>.c;
   defvar IsWorstCase = SiFiveP600IsWorstCaseMX<mx, SchedMxList>.c;
-  let Latency = 3, ReleaseAtCycles = [LMulLat] in {
-    defm "" : LMULWriteResMX<"WriteVFCvtIToFV", [SiFiveP600VectorArith], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVFCvtFToIV", [SiFiveP600VectorArith], mx, IsWorstCase>;
-  }
+  let Latency = 3, ReleaseAtCycles = [LMulLat] in
+  defm "" : LMULWriteResMX<"WriteVFCvtFToIV", [SiFiveP600VectorArith], mx, IsWorstCase>;
   let Latency = 2, ReleaseAtCycles = [LMulLat] in {
     defm "" : LMULWriteResMX<"WriteVFCmpV",  [SiFiveP600VectorArith], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFCmpF",  [SiFiveP600VectorArith], mx, IsWorstCase>;
@@ -976,7 +960,7 @@ defm "" : LMULReadAdvance<"ReadVFClassV", 0>;
 defm "" : LMULReadAdvance<"ReadVFMergeV", 0>;
 defm "" : LMULReadAdvance<"ReadVFMergeF", 0>;
 defm "" : LMULReadAdvance<"ReadVFMovF", 0>;
-defm "" : LMULReadAdvance<"ReadVFCvtIToFV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFCvtIToFV", 0>;
 defm "" : LMULReadAdvance<"ReadVFCvtFToIV", 0>;
 defm "" : LMULReadAdvanceW<"ReadVFWCvtIToFV", 0>;
 defm "" : LMULReadAdvanceFW<"ReadVFWCvtFToIV", 0>;
diff --git a/llvm/lib/Target/RISCV/RISCVScheduleV.td b/llvm/lib/Target/RISCV/RISCVScheduleV.td
index 6070482ea175..e7acc0c87d70 100644
--- a/llvm/lib/Target/RISCV/RISCVScheduleV.td
+++ b/llvm/lib/Target/RISCV/RISCVScheduleV.td
@@ -449,7 +449,7 @@ defm "" : LMULSchedWrites<"WriteVFMergeV">;
 // 13.16. Vector Floating-Point Move Instruction
 defm "" : LMULSchedWrites<"WriteVFMovV">;
 // 13.17. Single-Width Floating-Point/Integer Type-Convert Instructions
-defm "" : LMULSchedWrites<"WriteVFCvtIToFV">;
+defm "" : LMULSEWSchedWritesF<"WriteVFCvtIToFV">;
 defm "" : LMULSchedWrites<"WriteVFCvtFToIV">;
 // 13.18. Widening Floating-Point/Integer Type-Convert Instructions
 defm "" : LMULSchedWritesW<"WriteVFWCvtIToFV">;
@@ -675,7 +675,7 @@ defm "" : LMULSchedReads<"ReadVFMergeF">;
 // 13.16. Vector Floating-Point Move Instruction
 defm "" : LMULSchedReads<"ReadVFMovF">;
 // 13.17. Single-Width Floating-Point/Integer Type-Convert Instructions
-defm "" : LMULSchedReads<"ReadVFCvtIToFV">;
+defm "" : LMULSEWSchedReadsF<"ReadVFCvtIToFV">;
 defm "" : LMULSchedReads<"ReadVFCvtFToIV">;
 // 13.18. Widening Floating-Point/Integer Type-Convert Instructions
 defm "" : LMULSchedReadsW<"ReadVFWCvtIToFV">;
@@ -905,7 +905,7 @@ defm "" : LMULWriteRes<"WriteVFCmpF", []>;
 defm "" : LMULWriteRes<"WriteVFClassV", []>;
 defm "" : LMULWriteRes<"WriteVFMergeV", []>;
 defm "" : LMULWriteRes<"WriteVFMovV", []>;
-defm "" : LMULWriteRes<"WriteVFCvtIToFV", []>;
+defm "" : LMULSEWWriteResF<"WriteVFCvtIToFV", []>;
 defm "" : LMULWriteRes<"WriteVFCvtFToIV", []>;
 defm "" : LMULWriteResW<"WriteVFWCvtIToFV", []>;
 defm "" : LMULWriteResFW<"WriteVFWCvtFToIV", []>;
@@ -1062,7 +1062,7 @@ defm "" : LMULReadAdvance<"ReadVFClassV", 0>;
 defm "" : LMULReadAdvance<"ReadVFMergeV", 0>;
 defm "" : LMULReadAdvance<"ReadVFMergeF", 0>;
 defm "" : LMULReadAdvance<"ReadVFMovF", 0>;
-defm "" : LMULReadAdvance<"ReadVFCvtIToFV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFCvtIToFV", 0>;
 defm "" : LMULReadAdvance<"ReadVFCvtFToIV", 0>;
 defm "" : LMULReadAdvanceW<"ReadVFWCvtIToFV", 0>;
 defm "" : LMULReadAdvanceFW<"ReadVFWCvtFToIV", 0>;
-- 
GitLab


From 60a1158f31bb99a5d049d5cac5a704d906a1277e Mon Sep 17 00:00:00 2001
From: Michael Maitland <michaeltmaitland@gmail.com>
Date: Fri, 12 Apr 2024 12:16:20 -0700
Subject: [PATCH 238/464] [RISCV] Split Widening convert to FP pseudos by SEW

---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   | 100 +++++++++++++++---
 .../Target/RISCV/RISCVInstrInfoVPseudos.td    |  63 ++++++-----
 .../Target/RISCV/RISCVInstrInfoVSDPatterns.td |   2 +-
 .../Target/RISCV/RISCVInstrInfoVVLPatterns.td |   4 +-
 llvm/lib/Target/RISCV/RISCVSchedSiFive7.td    |  22 ++--
 llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td |  20 ++--
 llvm/lib/Target/RISCV/RISCVScheduleV.td       |  28 +++--
 7 files changed, 168 insertions(+), 71 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 5a572002091f..27387595164a 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -17642,8 +17642,7 @@ static MachineBasicBlock *emitSelectPseudo(MachineInstr &MI,
 
 static MachineBasicBlock *emitVFROUND_NOEXCEPT_MASK(MachineInstr &MI,
                                                     MachineBasicBlock *BB,
-                                                    unsigned CVTXOpc,
-                                                    unsigned CVTFOpc) {
+                                                    unsigned CVTXOpc) {
   DebugLoc DL = MI.getDebugLoc();
 
   const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
@@ -17674,6 +17673,85 @@ static MachineBasicBlock *emitVFROUND_NOEXCEPT_MASK(MachineInstr &MI,
                                      /*IsImp*/ true));
 
   // Emit a VFCVT_F_X
+  RISCVII::VLMUL LMul = RISCVII::getLMul(MI.getDesc().TSFlags);
+  unsigned Log2SEW = MI.getOperand(RISCVII::getSEWOpNum(MI.getDesc())).getImm();
+  // There is no E8 variant for VFCVT_F_X.
+  assert(Log2SEW >= 4);
+  // Since MI (VFROUND) isn't SEW specific, we cannot use a macro to make
+  // handling of different (LMUL, SEW) pairs easier because we need to pull the
+  // SEW immediate from MI, and that information is not avaliable during macro
+  // expansion.
+  unsigned CVTFOpc;
+  if (Log2SEW == 4) {
+    switch (LMul) {
+    case RISCVII::LMUL_1:
+      CVTFOpc = RISCV::PseudoVFCVT_F_X_V_M1_E16_MASK;
+      break;
+    case RISCVII::LMUL_2:
+      CVTFOpc = RISCV::PseudoVFCVT_F_X_V_M2_E16_MASK;
+      break;
+    case RISCVII::LMUL_4:
+      CVTFOpc = RISCV::PseudoVFCVT_F_X_V_M4_E16_MASK;
+      break;
+    case RISCVII::LMUL_8:
+      CVTFOpc = RISCV::PseudoVFCVT_F_X_V_M8_E16_MASK;
+      break;
+    case RISCVII::LMUL_F2:
+      CVTFOpc = RISCV::PseudoVFCVT_F_X_V_MF2_E16_MASK;
+      break;
+    case RISCVII::LMUL_F4:
+      CVTFOpc = RISCV::PseudoVFCVT_F_X_V_MF4_E16_MASK;
+      break;
+    case RISCVII::LMUL_F8:
+    case RISCVII::LMUL_RESERVED:
+      llvm_unreachable("Unexpected LMUL and SEW combination value for MI.");
+    }
+  } else if (Log2SEW == 5) {
+    switch (LMul) {
+    case RISCVII::LMUL_1:
+      CVTFOpc = RISCV::PseudoVFCVT_F_X_V_M1_E32_MASK;
+      break;
+    case RISCVII::LMUL_2:
+      CVTFOpc = RISCV::PseudoVFCVT_F_X_V_M2_E32_MASK;
+      break;
+    case RISCVII::LMUL_4:
+      CVTFOpc = RISCV::PseudoVFCVT_F_X_V_M4_E32_MASK;
+      break;
+    case RISCVII::LMUL_8:
+      CVTFOpc = RISCV::PseudoVFCVT_F_X_V_M8_E32_MASK;
+      break;
+    case RISCVII::LMUL_F2:
+      CVTFOpc = RISCV::PseudoVFCVT_F_X_V_MF2_E32_MASK;
+      break;
+    case RISCVII::LMUL_F4:
+    case RISCVII::LMUL_F8:
+    case RISCVII::LMUL_RESERVED:
+      llvm_unreachable("Unexpected LMUL and SEW combination value for MI.");
+    }
+  } else if (Log2SEW == 6) {
+    switch (LMul) {
+    case RISCVII::LMUL_1:
+      CVTFOpc = RISCV::PseudoVFCVT_F_X_V_M1_E64_MASK;
+      break;
+    case RISCVII::LMUL_2:
+      CVTFOpc = RISCV::PseudoVFCVT_F_X_V_M2_E64_MASK;
+      break;
+    case RISCVII::LMUL_4:
+      CVTFOpc = RISCV::PseudoVFCVT_F_X_V_M4_E64_MASK;
+      break;
+    case RISCVII::LMUL_8:
+      CVTFOpc = RISCV::PseudoVFCVT_F_X_V_M8_E64_MASK;
+      break;
+    case RISCVII::LMUL_F2:
+    case RISCVII::LMUL_F4:
+    case RISCVII::LMUL_F8:
+    case RISCVII::LMUL_RESERVED:
+      llvm_unreachable("Unexpected LMUL and SEW combination value for MI.");
+    }
+  } else {
+    llvm_unreachable("Unexpected LMUL and SEW combination value for MI.");
+  }
+
   BuildMI(*BB, MI, DL, TII.get(CVTFOpc))
       .add(MI.getOperand(0))
       .add(MI.getOperand(1))
@@ -17883,23 +17961,17 @@ RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
                          Subtarget);
 
   case RISCV::PseudoVFROUND_NOEXCEPT_V_M1_MASK:
-    return emitVFROUND_NOEXCEPT_MASK(MI, BB, RISCV::PseudoVFCVT_X_F_V_M1_MASK,
-                                     RISCV::PseudoVFCVT_F_X_V_M1_MASK);
+    return emitVFROUND_NOEXCEPT_MASK(MI, BB, RISCV::PseudoVFCVT_X_F_V_M1_MASK);
   case RISCV::PseudoVFROUND_NOEXCEPT_V_M2_MASK:
-    return emitVFROUND_NOEXCEPT_MASK(MI, BB, RISCV::PseudoVFCVT_X_F_V_M2_MASK,
-                                     RISCV::PseudoVFCVT_F_X_V_M2_MASK);
+    return emitVFROUND_NOEXCEPT_MASK(MI, BB, RISCV::PseudoVFCVT_X_F_V_M2_MASK);
   case RISCV::PseudoVFROUND_NOEXCEPT_V_M4_MASK:
-    return emitVFROUND_NOEXCEPT_MASK(MI, BB, RISCV::PseudoVFCVT_X_F_V_M4_MASK,
-                                     RISCV::PseudoVFCVT_F_X_V_M4_MASK);
+    return emitVFROUND_NOEXCEPT_MASK(MI, BB, RISCV::PseudoVFCVT_X_F_V_M4_MASK);
   case RISCV::PseudoVFROUND_NOEXCEPT_V_M8_MASK:
-    return emitVFROUND_NOEXCEPT_MASK(MI, BB, RISCV::PseudoVFCVT_X_F_V_M8_MASK,
-                                     RISCV::PseudoVFCVT_F_X_V_M8_MASK);
+    return emitVFROUND_NOEXCEPT_MASK(MI, BB, RISCV::PseudoVFCVT_X_F_V_M8_MASK);
   case RISCV::PseudoVFROUND_NOEXCEPT_V_MF2_MASK:
-    return emitVFROUND_NOEXCEPT_MASK(MI, BB, RISCV::PseudoVFCVT_X_F_V_MF2_MASK,
-                                     RISCV::PseudoVFCVT_F_X_V_MF2_MASK);
+    return emitVFROUND_NOEXCEPT_MASK(MI, BB, RISCV::PseudoVFCVT_X_F_V_MF2_MASK);
   case RISCV::PseudoVFROUND_NOEXCEPT_V_MF4_MASK:
-    return emitVFROUND_NOEXCEPT_MASK(MI, BB, RISCV::PseudoVFCVT_X_F_V_MF4_MASK,
-                                     RISCV::PseudoVFCVT_F_X_V_MF4_MASK);
+    return emitVFROUND_NOEXCEPT_MASK(MI, BB, RISCV::PseudoVFCVT_X_F_V_MF4_MASK);
   case RISCV::PseudoFROUND_H:
   case RISCV::PseudoFROUND_H_INX:
   case RISCV::PseudoFROUND_S:
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index e9cb02759270..9906049e9aaf 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -3580,12 +3580,14 @@ multiclass VPseudoConversion<VReg RetClass,
                              VReg Op1Class,
                              LMULInfo MInfo,
                              string Constraint = "",
+                             int sew = 0,
                              int TargetConstraintType = 1> {
+  defvar suffix = !if(sew, "_" # MInfo.MX # "_E" # sew, "_" # MInfo.MX);
   let VLMul = MInfo.value in {
-    def "_" # MInfo.MX : VPseudoUnaryNoMask<RetClass, Op1Class, Constraint, TargetConstraintType>;
-    def "_" # MInfo.MX # "_MASK" : VPseudoUnaryMask<RetClass, Op1Class,
-                                                    Constraint, TargetConstraintType>,
-                                   RISCVMaskedPseudo<MaskIdx=2>;
+    def suffix : VPseudoUnaryNoMask<RetClass, Op1Class, Constraint, TargetConstraintType>;
+    def suffix # "_MASK" : VPseudoUnaryMask<RetClass, Op1Class,
+                                            Constraint, TargetConstraintType>,
+                           RISCVMaskedPseudo<MaskIdx=2>;
   }
 }
 
@@ -3711,18 +3713,22 @@ multiclass VPseudoVWCVTI_RM_V {
 multiclass VPseudoVWCVTF_V {
   defvar constraint = "@earlyclobber $rd";
   foreach m = MxListW in {
-    defm _V : VPseudoConversion<m.wvrclass, m.vrclass, m, constraint, TargetConstraintType=3>,
-              SchedUnary<"WriteVFWCvtIToFV", "ReadVFWCvtIToFV", m.MX,
-                         forceMergeOpRead=true>;
+    foreach e = SchedSEWSet<m.MX, isF=0, isWidening=1>.val in
+      defm _V : VPseudoConversion<m.wvrclass, m.vrclass, m, constraint, sew=e,
+                                  TargetConstraintType=3>,
+                SchedUnary<"WriteVFWCvtIToFV", "ReadVFWCvtIToFV", m.MX, e,
+                           forceMergeOpRead=true>;
   }
 }
 
 multiclass VPseudoVWCVTD_V {
   defvar constraint = "@earlyclobber $rd";
   foreach m = MxListFW in {
-    defm _V : VPseudoConversion<m.wvrclass, m.vrclass, m, constraint, TargetConstraintType=3>,
-              SchedUnary<"WriteVFWCvtFToFV", "ReadVFWCvtFToFV", m.MX,
-                         forceMergeOpRead=true>;
+    foreach e = SchedSEWSet<m.MX, isF=1, isWidening=1>.val in
+      defm _V : VPseudoConversion<m.wvrclass, m.vrclass, m, constraint, sew=e,
+                                  TargetConstraintType=3>,
+                SchedUnary<"WriteVFWCvtFToFV", "ReadVFWCvtFToFV", m.MX, e,
+                           forceMergeOpRead=true>;
   }
 }
 
@@ -4896,14 +4902,17 @@ multiclass VPatConversionTA<string intrinsic,
                             ValueType result_type,
                             ValueType op1_type,
                             ValueType mask_type,
-                            int sew,
+                            int log2sew,
                             LMULInfo vlmul,
                             VReg result_reg_class,
-                            VReg op1_reg_class> {
+                            VReg op1_reg_class,
+                            bit isSEWAware = 0> {
   def : VPatUnaryNoMask<intrinsic, inst, kind, result_type, op1_type,
-                        sew, vlmul, result_reg_class, op1_reg_class>;
+                        log2sew, vlmul, result_reg_class, op1_reg_class,
+                        isSEWAware>;
   def : VPatUnaryMask<intrinsic, inst, kind, result_type, op1_type,
-                      mask_type, sew, vlmul, result_reg_class, op1_reg_class>;
+                      mask_type, log2sew, vlmul, result_reg_class, op1_reg_class,
+                      isSEWAware>;
 }
 
 multiclass VPatConversionTARoundingMode<string intrinsic,
@@ -5952,7 +5961,8 @@ multiclass VPatConversionWI_VF_RM<string intrinsic, string instruction> {
   }
 }
 
-multiclass VPatConversionWF_VI<string intrinsic, string instruction> {
+multiclass VPatConversionWF_VI<string intrinsic, string instruction,
+                               bit isSEWAware = 0> {
   foreach vtiToWti = AllWidenableIntToFloatVectors in {
     defvar vti = vtiToWti.Vti;
     defvar fwti = vtiToWti.Wti;
@@ -5960,11 +5970,12 @@ multiclass VPatConversionWF_VI<string intrinsic, string instruction> {
                                  GetVTypePredicates<fwti>.Predicates) in
     defm : VPatConversionTA<intrinsic, instruction, "V",
                             fwti.Vector, vti.Vector, fwti.Mask, vti.Log2SEW,
-                            vti.LMul, fwti.RegClass, vti.RegClass>;
+                            vti.LMul, fwti.RegClass, vti.RegClass, isSEWAware>;
   }
 }
 
-multiclass VPatConversionWF_VF<string intrinsic, string instruction> {
+multiclass VPatConversionWF_VF<string intrinsic, string instruction,
+                               bit isSEWAware = 0> {
   foreach fvtiToFWti = AllWidenableFloatVectors in {
     defvar fvti = fvtiToFWti.Vti;
     defvar fwti = fvtiToFWti.Wti;
@@ -5974,11 +5985,12 @@ multiclass VPatConversionWF_VF<string intrinsic, string instruction> {
                                      GetVTypePredicates<fwti>.Predicates)) in
       defm : VPatConversionTA<intrinsic, instruction, "V",
                               fwti.Vector, fvti.Vector, fwti.Mask, fvti.Log2SEW,
-                              fvti.LMul, fwti.RegClass, fvti.RegClass>;
+                              fvti.LMul, fwti.RegClass, fvti.RegClass, isSEWAware>;
   }
 }
 
-multiclass VPatConversionWF_VF_BF <string intrinsic, string instruction> {
+multiclass VPatConversionWF_VF_BF <string intrinsic, string instruction,
+                                   bit isSEWAware = 0> {
   foreach fvtiToFWti = AllWidenableBFloatToFloatVectors in
   {
     defvar fvti = fvtiToFWti.Vti;
@@ -5987,7 +5999,7 @@ multiclass VPatConversionWF_VF_BF <string intrinsic, string instruction> {
                                  GetVTypePredicates<fwti>.Predicates) in
     defm : VPatConversionTA<intrinsic, instruction, "V",
                             fwti.Vector, fvti.Vector, fwti.Mask, fvti.Log2SEW,
-                            fvti.LMul, fwti.RegClass, fvti.RegClass>;
+                            fvti.LMul, fwti.RegClass, fvti.RegClass, isSEWAware>;
   }
 }
 
@@ -7292,11 +7304,14 @@ defm : VPatConversionWI_VF_RM<"int_riscv_vfwcvt_xu_f_v", "PseudoVFWCVT_XU_F">;
 defm : VPatConversionWI_VF_RM<"int_riscv_vfwcvt_x_f_v", "PseudoVFWCVT_X_F">;
 defm : VPatConversionWI_VF<"int_riscv_vfwcvt_rtz_xu_f_v", "PseudoVFWCVT_RTZ_XU_F">;
 defm : VPatConversionWI_VF<"int_riscv_vfwcvt_rtz_x_f_v", "PseudoVFWCVT_RTZ_X_F">;
-defm : VPatConversionWF_VI<"int_riscv_vfwcvt_f_xu_v", "PseudoVFWCVT_F_XU">;
-defm : VPatConversionWF_VI<"int_riscv_vfwcvt_f_x_v", "PseudoVFWCVT_F_X">;
-defm : VPatConversionWF_VF<"int_riscv_vfwcvt_f_f_v", "PseudoVFWCVT_F_F">;
+defm : VPatConversionWF_VI<"int_riscv_vfwcvt_f_xu_v", "PseudoVFWCVT_F_XU",
+                           isSEWAware=1>;
+defm : VPatConversionWF_VI<"int_riscv_vfwcvt_f_x_v", "PseudoVFWCVT_F_X",
+                           isSEWAware=1>;
+defm : VPatConversionWF_VF<"int_riscv_vfwcvt_f_f_v", "PseudoVFWCVT_F_F",
+                           isSEWAware=1>;
 defm : VPatConversionWF_VF_BF<"int_riscv_vfwcvtbf16_f_f_v",
-                              "PseudoVFWCVTBF16_F_F">;
+                              "PseudoVFWCVTBF16_F_F", isSEWAware=1>;
 
 //===----------------------------------------------------------------------===//
 // 13.19. Narrowing Floating-Point/Integer Type-Convert Instructions
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
index 788db6f447a1..23852414cf56 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
@@ -441,7 +441,7 @@ multiclass VPatWConvertI2FPSDNode_V<SDPatternOperator vop,
     let Predicates = !listconcat(GetVTypePredicates<ivti>.Predicates,
                                  GetVTypePredicates<fwti>.Predicates) in
     def : Pat<(fwti.Vector (vop (ivti.Vector ivti.RegClass:$rs1))),
-              (!cast<Instruction>(instruction_name#"_"#ivti.LMul.MX)
+              (!cast<Instruction>(instruction_name#"_"#ivti.LMul.MX#"_E"#ivti.SEW)
                   (fwti.Vector (IMPLICIT_DEF)),
                   ivti.RegClass:$rs1,
                   ivti.AVL, ivti.Log2SEW, TA_MA)>;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
index 2b56eea29cfc..eb46d47079b2 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
@@ -1315,7 +1315,7 @@ multiclass VPatWConvertI2FPVL_V<SDPatternOperator vop,
     def : Pat<(fwti.Vector (vop (ivti.Vector ivti.RegClass:$rs1),
                                 (ivti.Mask V0),
                                 VLOpFrag)),
-              (!cast<Instruction>(instruction_name#"_"#ivti.LMul.MX#"_MASK")
+              (!cast<Instruction>(instruction_name#"_"#ivti.LMul.MX#"_E"#ivti.SEW#"_MASK")
                   (fwti.Vector (IMPLICIT_DEF)), ivti.RegClass:$rs1,
                   (ivti.Mask V0),
                   GPR:$vl, ivti.Log2SEW, TA_MA)>;
@@ -2672,7 +2672,7 @@ foreach fvtiToFWti = AllWidenableFloatVectors in {
                              (fvti.Vector fvti.RegClass:$rs1),
                              (fvti.Mask V0),
                              VLOpFrag)),
-            (!cast<Instruction>("PseudoVFWCVT_F_F_V_"#fvti.LMul.MX#"_MASK")
+            (!cast<Instruction>("PseudoVFWCVT_F_F_V_"#fvti.LMul.MX#"_E"#fvti.SEW#"_MASK")
                 (fwti.Vector (IMPLICIT_DEF)), fvti.RegClass:$rs1,
                 (fvti.Mask V0),
                 GPR:$vl, fvti.Log2SEW, TA_MA)>;
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
index 354fe7c27e68..f1246749ac43 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
@@ -784,10 +784,11 @@ foreach mx = SchedMxListF in {
 
 // Widening
 foreach mx = SchedMxListW in {
-  defvar Cycles = SiFive7GetCyclesDefault<mx>.c;
-  defvar IsWorstCase = SiFive7IsWorstCaseMX<mx, SchedMxListW>.c;
-  let Latency = 8, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in {
-    defm "" : LMULWriteResMX<"WriteVFWCvtIToFV", [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+  foreach sew = SchedSEWSet<mx, isF=0, isWidening=1>.val in {
+    defvar Cycles = SiFive7GetCyclesDefault<mx>.c;
+    defvar IsWorstCase = SiFive7IsWorstCaseMXSEW<mx, sew, SchedMxListW>.c;
+    let Latency = 8, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in
+    defm "" : LMULSEWWriteResMXSEW<"WriteVFWCvtIToFV", [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
   }
 }
 foreach mx = SchedMxListFW in {
@@ -801,16 +802,13 @@ foreach mx = SchedMxListFW in {
       defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulF", [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
       defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulAddV", [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
       defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulAddF", [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFWCvtFToFV", [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
     }
   }
-}
-foreach mx = SchedMxListFW in {
   defvar Cycles = SiFive7GetCyclesDefault<mx>.c;
   defvar IsWorstCase = SiFive7IsWorstCaseMX<mx, SchedMxListFW>.c;
-  let Latency = 8, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in {
-    defm "" : LMULWriteResMX<"WriteVFWCvtFToIV", [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVFWCvtFToFV", [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
-  }
+  let Latency = 8, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in
+  defm "" : LMULWriteResMX<"WriteVFWCvtFToIV", [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
 }
 // Narrowing
 foreach mx = SchedMxListW in {
@@ -1181,9 +1179,9 @@ defm "" : LMULReadAdvance<"ReadVFMergeF", 0>;
 defm "" : LMULReadAdvance<"ReadVFMovF", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFCvtIToFV", 0>;
 defm "" : LMULReadAdvance<"ReadVFCvtFToIV", 0>;
-defm "" : LMULReadAdvanceW<"ReadVFWCvtIToFV", 0>;
+defm "" : LMULSEWReadAdvanceW<"ReadVFWCvtIToFV", 0>;
 defm "" : LMULReadAdvanceFW<"ReadVFWCvtFToIV", 0>;
-defm "" : LMULReadAdvanceFW<"ReadVFWCvtFToFV", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWCvtFToFV", 0>;
 defm "" : LMULReadAdvanceFW<"ReadVFNCvtIToFV", 0>;
 defm "" : LMULReadAdvanceW<"ReadVFNCvtFToIV", 0>;
 defm "" : LMULReadAdvanceFW<"ReadVFNCvtFToFV", 0>;
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
index 863e245c3225..0884cea39693 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
@@ -526,19 +526,18 @@ foreach mx = SchedMxList in {
 
 // Widening
 foreach mx = SchedMxListW in {
-  defvar LMulLat = SiFiveP600GetLMulCycles<mx>.c;
-  defvar IsWorstCase = SiFiveP600IsWorstCaseMX<mx, SchedMxListW>.c;
-  let Latency = 3, ReleaseAtCycles = [LMulLat] in {
-    defm "" : LMULWriteResMX<"WriteVFWCvtIToFV", [SiFiveP600VectorArith], mx, IsWorstCase>;
+  foreach sew = SchedSEWSet<mx, isF=0, isWidening=1>.val in {
+    defvar LMulLat = SiFiveP600GetLMulCycles<mx>.c;
+    defvar IsWorstCase = SiFiveP600IsWorstCaseMXSEW<mx, sew, SchedMxListW>.c;
+    let Latency = 3, ReleaseAtCycles = [LMulLat] in
+    defm "" : LMULSEWWriteResMXSEW<"WriteVFWCvtIToFV", [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
   }
 }
 foreach mx = SchedMxListFW in {
   defvar LMulLat = SiFiveP600GetLMulCycles<mx>.c;
   defvar IsWorstCase = SiFiveP600IsWorstCaseMX<mx, SchedMxListFW>.c;
-  let Latency = 6, ReleaseAtCycles = [LMulLat] in {
-    defm "" : LMULWriteResMX<"WriteVFWCvtFToIV", [SiFiveP600VectorArith], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVFWCvtFToFV", [SiFiveP600VectorArith], mx, IsWorstCase>;
-  }
+  let Latency = 6, ReleaseAtCycles = [LMulLat] in
+  defm "" : LMULWriteResMX<"WriteVFWCvtFToIV", [SiFiveP600VectorArith], mx, IsWorstCase>;
 }
 foreach mx = SchedMxListFW in {
   foreach sew = SchedSEWSet<mx, isF=1, isWidening=1>.val in {
@@ -551,6 +550,7 @@ foreach mx = SchedMxListFW in {
       defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulF", [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
       defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulAddV", [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
       defm "" : LMULSEWWriteResMXSEW<"WriteVFWMulAddF", [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFWCvtFToFV", [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
     }
   }
 }
@@ -962,9 +962,9 @@ defm "" : LMULReadAdvance<"ReadVFMergeF", 0>;
 defm "" : LMULReadAdvance<"ReadVFMovF", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFCvtIToFV", 0>;
 defm "" : LMULReadAdvance<"ReadVFCvtFToIV", 0>;
-defm "" : LMULReadAdvanceW<"ReadVFWCvtIToFV", 0>;
+defm "" : LMULSEWReadAdvanceW<"ReadVFWCvtIToFV", 0>;
 defm "" : LMULReadAdvanceFW<"ReadVFWCvtFToIV", 0>;
-defm "" : LMULReadAdvanceFW<"ReadVFWCvtFToFV", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWCvtFToFV", 0>;
 defm "" : LMULReadAdvanceFW<"ReadVFNCvtIToFV", 0>;
 defm "" : LMULReadAdvanceW<"ReadVFNCvtFToIV", 0>;
 defm "" : LMULReadAdvanceFW<"ReadVFNCvtFToFV", 0>;
diff --git a/llvm/lib/Target/RISCV/RISCVScheduleV.td b/llvm/lib/Target/RISCV/RISCVScheduleV.td
index e7acc0c87d70..42c6dde89eac 100644
--- a/llvm/lib/Target/RISCV/RISCVScheduleV.td
+++ b/llvm/lib/Target/RISCV/RISCVScheduleV.td
@@ -253,6 +253,18 @@ multiclass LMULReadAdvanceFW<string name, int val, list<SchedWrite> writes = []>
   : LMULReadAdvanceImpl<name, val, writes>;
 class LMULSchedWriteListFW<list<string> names> : LMULSchedWriteListImpl<names, SchedMxListFW>;
 
+multiclass LMULSEWSchedWritesW<string name>
+    : LMULSEWSchedWritesImpl<name, SchedMxListW, isF = 0, isWidening = 1>;
+multiclass LMULSEWSchedReadsW<string name>
+    : LMULSEWSchedReadsImpl<name, SchedMxListW, isF = 0, isWidening = 1>;
+multiclass LMULSEWWriteResW<string name, list<ProcResourceKind> resources>
+    : LMULSEWWriteResImpl<name, resources, SchedMxListW, isF = 0,
+                          isWidening = 1>;
+multiclass
+    LMULSEWReadAdvanceW<string name, int val, list<SchedWrite> writes = []>
+    : LMULSEWReadAdvanceImpl<name, val, writes, SchedMxListW, isF = 0,
+                             isWidening = 1>;
+
 multiclass LMULSEWSchedWritesFW<string name>
     : LMULSEWSchedWritesImpl<name, SchedMxListFW, isF = 1, isWidening = 1>;
 multiclass LMULSEWSchedReadsFW<string name>
@@ -452,9 +464,9 @@ defm "" : LMULSchedWrites<"WriteVFMovV">;
 defm "" : LMULSEWSchedWritesF<"WriteVFCvtIToFV">;
 defm "" : LMULSchedWrites<"WriteVFCvtFToIV">;
 // 13.18. Widening Floating-Point/Integer Type-Convert Instructions
-defm "" : LMULSchedWritesW<"WriteVFWCvtIToFV">;
+defm "" : LMULSEWSchedWritesW<"WriteVFWCvtIToFV">;
 defm "" : LMULSchedWritesFW<"WriteVFWCvtFToIV">;
-defm "" : LMULSchedWritesFW<"WriteVFWCvtFToFV">;
+defm "" : LMULSEWSchedWritesFW<"WriteVFWCvtFToFV">;
 // 13.19. Narrowing Floating-Point/Integer Type-Convert Instructions
 defm "" : LMULSchedWritesFW<"WriteVFNCvtIToFV">;
 defm "" : LMULSchedWritesW<"WriteVFNCvtFToIV">;
@@ -678,9 +690,9 @@ defm "" : LMULSchedReads<"ReadVFMovF">;
 defm "" : LMULSEWSchedReadsF<"ReadVFCvtIToFV">;
 defm "" : LMULSchedReads<"ReadVFCvtFToIV">;
 // 13.18. Widening Floating-Point/Integer Type-Convert Instructions
-defm "" : LMULSchedReadsW<"ReadVFWCvtIToFV">;
+defm "" : LMULSEWSchedReadsW<"ReadVFWCvtIToFV">;
 defm "" : LMULSchedReadsFW<"ReadVFWCvtFToIV">;
-defm "" : LMULSchedReadsFW<"ReadVFWCvtFToFV">;
+defm "" : LMULSEWSchedReadsFW<"ReadVFWCvtFToFV">;
 // 13.19. Narrowing Floating-Point/Integer Type-Convert Instructions
 defm "" : LMULSchedReadsFW<"ReadVFNCvtIToFV">;
 defm "" : LMULSchedReadsW<"ReadVFNCvtFToIV">;
@@ -907,9 +919,9 @@ defm "" : LMULWriteRes<"WriteVFMergeV", []>;
 defm "" : LMULWriteRes<"WriteVFMovV", []>;
 defm "" : LMULSEWWriteResF<"WriteVFCvtIToFV", []>;
 defm "" : LMULWriteRes<"WriteVFCvtFToIV", []>;
-defm "" : LMULWriteResW<"WriteVFWCvtIToFV", []>;
+defm "" : LMULSEWWriteResW<"WriteVFWCvtIToFV", []>;
 defm "" : LMULWriteResFW<"WriteVFWCvtFToIV", []>;
-defm "" : LMULWriteResFW<"WriteVFWCvtFToFV", []>;
+defm "" : LMULSEWWriteResFW<"WriteVFWCvtFToFV", []>;
 defm "" : LMULWriteResFW<"WriteVFNCvtIToFV", []>;
 defm "" : LMULWriteResW<"WriteVFNCvtFToIV", []>;
 defm "" : LMULWriteResFW<"WriteVFNCvtFToFV", []>;
@@ -1064,9 +1076,9 @@ defm "" : LMULReadAdvance<"ReadVFMergeF", 0>;
 defm "" : LMULReadAdvance<"ReadVFMovF", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFCvtIToFV", 0>;
 defm "" : LMULReadAdvance<"ReadVFCvtFToIV", 0>;
-defm "" : LMULReadAdvanceW<"ReadVFWCvtIToFV", 0>;
+defm "" : LMULSEWReadAdvanceW<"ReadVFWCvtIToFV", 0>;
 defm "" : LMULReadAdvanceFW<"ReadVFWCvtFToIV", 0>;
-defm "" : LMULReadAdvanceFW<"ReadVFWCvtFToFV", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFWCvtFToFV", 0>;
 defm "" : LMULReadAdvanceFW<"ReadVFNCvtIToFV", 0>;
 defm "" : LMULReadAdvanceW<"ReadVFNCvtFToIV", 0>;
 defm "" : LMULReadAdvanceFW<"ReadVFNCvtFToFV", 0>;
-- 
GitLab


From 469493f55639a5cce2fc69300222be609fcc5238 Mon Sep 17 00:00:00 2001
From: Michael Maitland <michaeltmaitland@gmail.com>
Date: Fri, 12 Apr 2024 12:43:41 -0700
Subject: [PATCH 239/464] [RISCV] Split narrowing convert to FP pseudos by SEW

---
 .../Target/RISCV/RISCVInstrInfoVPseudos.td    | 73 ++++++++++++-------
 .../Target/RISCV/RISCVInstrInfoVSDPatterns.td |  4 +-
 .../Target/RISCV/RISCVInstrInfoVVLPatterns.td |  8 +-
 llvm/lib/Target/RISCV/RISCVSchedSiFive7.td    | 16 ++--
 llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td | 16 ++--
 llvm/lib/Target/RISCV/RISCVScheduleV.td       | 16 ++--
 6 files changed, 78 insertions(+), 55 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index 9906049e9aaf..5d8b091392be 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -3762,36 +3762,45 @@ multiclass VPseudoVNCVTI_RM_W {
 multiclass VPseudoVNCVTF_W_RM {
   defvar constraint = "@earlyclobber $rd";
   foreach m = MxListFW in {
-    defm _W : VPseudoConversionRoundingMode<m.vrclass, m.wvrclass, m, constraint, TargetConstraintType=2>,
-              SchedUnary<"WriteVFNCvtIToFV", "ReadVFNCvtIToFV", m.MX,
-                         forceMergeOpRead=true>;
+    foreach e = SchedSEWSet<m.MX, isF=1, isWidening=1>.val in
+      defm _W : VPseudoConversionRoundingMode<m.vrclass, m.wvrclass, m,
+                                              constraint, sew=e,
+                                              TargetConstraintType=2>,
+                SchedUnary<"WriteVFNCvtIToFV", "ReadVFNCvtIToFV", m.MX, e,
+                           forceMergeOpRead=true>;
   }
 }
 
 multiclass VPseudoVNCVTF_RM_W {
   defvar constraint = "@earlyclobber $rd";
   foreach m = MxListFW in {
-    defm _W : VPseudoConversionRM<m.vrclass, m.wvrclass, m, constraint>,
-              SchedUnary<"WriteVFNCvtIToFV", "ReadVFNCvtIToFV", m.MX,
-                         forceMergeOpRead=true>;
+    foreach e = SchedSEWSet<m.MX, isF=1, isWidening=1>.val in
+      defm _W : VPseudoConversionRM<m.vrclass, m.wvrclass, m, constraint, sew=e>,
+                SchedUnary<"WriteVFNCvtIToFV", "ReadVFNCvtIToFV", m.MX, e,
+                           forceMergeOpRead=true>;
   }
 }
 
 multiclass VPseudoVNCVTD_W {
   defvar constraint = "@earlyclobber $rd";
   foreach m = MxListFW in {
-    defm _W : VPseudoConversion<m.vrclass, m.wvrclass, m, constraint, TargetConstraintType=2>,
-              SchedUnary<"WriteVFNCvtFToFV", "ReadVFNCvtFToFV", m.MX,
-                         forceMergeOpRead=true>;
+    foreach e = SchedSEWSet<m.MX, isF=1, isWidening=1>.val in
+      defm _W : VPseudoConversion<m.vrclass, m.wvrclass, m, constraint, sew=e,
+                                  TargetConstraintType=2>,
+                SchedUnary<"WriteVFNCvtFToFV", "ReadVFNCvtFToFV", m.MX, e,
+                           forceMergeOpRead=true>;
   }
 }
 
 multiclass VPseudoVNCVTD_W_RM {
   defvar constraint = "@earlyclobber $rd";
   foreach m = MxListFW in {
-    defm _W : VPseudoConversionRoundingMode<m.vrclass, m.wvrclass, m, constraint, TargetConstraintType=2>,
-              SchedUnary<"WriteVFNCvtFToFV", "ReadVFNCvtFToFV", m.MX,
-                         forceMergeOpRead=true>;
+    foreach e = SchedSEWSet<m.MX, isF=1, isWidening=1>.val in
+      defm _W : VPseudoConversionRoundingMode<m.vrclass, m.wvrclass, m,
+                                              constraint, sew=e,
+                                              TargetConstraintType=2>,
+                SchedUnary<"WriteVFNCvtFToFV", "ReadVFNCvtFToFV", m.MX, e,
+                           forceMergeOpRead=true>;
   }
 }
 
@@ -6027,7 +6036,8 @@ multiclass VPatConversionVI_WF_RM <string intrinsic, string instruction> {
   }
 }
 
-multiclass VPatConversionVF_WI_RM <string intrinsic, string instruction> {
+multiclass VPatConversionVF_WI_RM <string intrinsic, string instruction,
+                                   bit isSEWAware = 0> {
   foreach fvtiToFWti = AllWidenableFloatVectors in {
     defvar fvti = fvtiToFWti.Vti;
     defvar iwti = GetIntVTypeInfo<fvtiToFWti.Wti>.Vti;
@@ -6035,11 +6045,13 @@ multiclass VPatConversionVF_WI_RM <string intrinsic, string instruction> {
                                  GetVTypePredicates<iwti>.Predicates) in
     defm : VPatConversionTARoundingMode<intrinsic, instruction, "W",
                                         fvti.Vector, iwti.Vector, fvti.Mask, fvti.Log2SEW,
-                                        fvti.LMul, fvti.RegClass, iwti.RegClass>;
+                                        fvti.LMul, fvti.RegClass, iwti.RegClass,
+                                        isSEWAware>;
   }
 }
 
-multiclass VPatConversionVF_WF <string intrinsic, string instruction> {
+multiclass VPatConversionVF_WF<string intrinsic, string instruction,
+                               bit isSEWAware = 0> {
   foreach fvtiToFWti = AllWidenableFloatVectors in {
     defvar fvti = fvtiToFWti.Vti;
     defvar fwti = fvtiToFWti.Wti;
@@ -6047,12 +6059,13 @@ multiclass VPatConversionVF_WF <string intrinsic, string instruction> {
                                  GetVTypePredicates<fwti>.Predicates) in
     defm : VPatConversionTA<intrinsic, instruction, "W",
                             fvti.Vector, fwti.Vector, fvti.Mask, fvti.Log2SEW,
-                            fvti.LMul, fvti.RegClass, fwti.RegClass>;
+                            fvti.LMul, fvti.RegClass, fwti.RegClass, isSEWAware>;
   }
 }
 
-multiclass VPatConversionVF_WF_RM <string intrinsic, string instruction,
-                                   list<VTypeInfoToWide> wlist = AllWidenableFloatVectors> {
+multiclass VPatConversionVF_WF_RM<string intrinsic, string instruction,
+                                   list<VTypeInfoToWide> wlist = AllWidenableFloatVectors,
+                                   bit isSEWAware = 0> {
   foreach fvtiToFWti = wlist in {
     defvar fvti = fvtiToFWti.Vti;
     defvar fwti = fvtiToFWti.Wti;
@@ -6060,11 +6073,13 @@ multiclass VPatConversionVF_WF_RM <string intrinsic, string instruction,
                                  GetVTypePredicates<fwti>.Predicates) in
     defm : VPatConversionTARoundingMode<intrinsic, instruction, "W",
                                         fvti.Vector, fwti.Vector, fvti.Mask, fvti.Log2SEW,
-                                        fvti.LMul, fvti.RegClass, fwti.RegClass>;
+                                        fvti.LMul, fvti.RegClass, fwti.RegClass,
+                                        isSEWAware>;
   }
 }
 
-multiclass VPatConversionVF_WF_BF_RM <string intrinsic, string instruction> {
+multiclass VPatConversionVF_WF_BF_RM <string intrinsic, string instruction,
+                                      bit isSEWAware = 0> {
   foreach fvtiToFWti = AllWidenableBFloatToFloatVectors in {
     defvar fvti = fvtiToFWti.Vti;
     defvar fwti = fvtiToFWti.Wti;
@@ -6072,7 +6087,8 @@ multiclass VPatConversionVF_WF_BF_RM <string intrinsic, string instruction> {
                                  GetVTypePredicates<fwti>.Predicates) in
     defm : VPatConversionTARoundingMode<intrinsic, instruction, "W",
                                         fvti.Vector, fwti.Vector, fvti.Mask, fvti.Log2SEW,
-                                        fvti.LMul, fvti.RegClass, fwti.RegClass>;
+                                        fvti.LMul, fvti.RegClass, fwti.RegClass,
+                                        isSEWAware>;
   }
 }
 
@@ -7320,21 +7336,24 @@ defm : VPatConversionVI_WF_RM<"int_riscv_vfncvt_xu_f_w", "PseudoVFNCVT_XU_F">;
 defm : VPatConversionVI_WF_RM<"int_riscv_vfncvt_x_f_w", "PseudoVFNCVT_X_F">;
 defm : VPatConversionVI_WF<"int_riscv_vfncvt_rtz_xu_f_w", "PseudoVFNCVT_RTZ_XU_F">;
 defm : VPatConversionVI_WF<"int_riscv_vfncvt_rtz_x_f_w", "PseudoVFNCVT_RTZ_X_F">;
-defm : VPatConversionVF_WI_RM <"int_riscv_vfncvt_f_xu_w", "PseudoVFNCVT_F_XU">;
-defm : VPatConversionVF_WI_RM <"int_riscv_vfncvt_f_x_w", "PseudoVFNCVT_F_X">;
+defm : VPatConversionVF_WI_RM<"int_riscv_vfncvt_f_xu_w", "PseudoVFNCVT_F_XU",
+                              isSEWAware=1>;
+defm : VPatConversionVF_WI_RM<"int_riscv_vfncvt_f_x_w", "PseudoVFNCVT_F_X",
+                              isSEWAware=1>;
 defvar WidenableFloatVectorsExceptF16 = !filter(fvtiToFWti, AllWidenableFloatVectors,
                                                 !ne(fvtiToFWti.Vti.Scalar, f16));
 defm : VPatConversionVF_WF_RM<"int_riscv_vfncvt_f_f_w", "PseudoVFNCVT_F_F",
-                           WidenableFloatVectorsExceptF16>;
+                           WidenableFloatVectorsExceptF16, isSEWAware=1>;
 // Define vfncvt.f.f.w for f16 when Zvfhmin is enable.
 defvar F16WidenableFloatVectors = !filter(fvtiToFWti, AllWidenableFloatVectors,
                                           !eq(fvtiToFWti.Vti.Scalar, f16));
 let Predicates = [HasVInstructionsF16Minimal] in
 defm : VPatConversionVF_WF_RM<"int_riscv_vfncvt_f_f_w", "PseudoVFNCVT_F_F",
-                           F16WidenableFloatVectors>;
+                           F16WidenableFloatVectors, isSEWAware=1>;
 defm : VPatConversionVF_WF_BF_RM<"int_riscv_vfncvtbf16_f_f_w", 
-                                 "PseudoVFNCVTBF16_F_F">;
-defm : VPatConversionVF_WF<"int_riscv_vfncvt_rod_f_f_w", "PseudoVFNCVT_ROD_F_F">;
+                                 "PseudoVFNCVTBF16_F_F", isSEWAware=1>;
+defm : VPatConversionVF_WF<"int_riscv_vfncvt_rod_f_f_w", "PseudoVFNCVT_ROD_F_F",
+                           isSEWAware=1>;
 
 //===----------------------------------------------------------------------===//
 // 14. Vector Reduction Operations
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
index 23852414cf56..9b324c40cc5b 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
@@ -470,7 +470,7 @@ multiclass VPatNConvertI2FPSDNode_W_RM<SDPatternOperator vop,
     let Predicates = !listconcat(GetVTypePredicates<fvti>.Predicates,
                                  GetVTypePredicates<iwti>.Predicates) in
     def : Pat<(fvti.Vector (vop (iwti.Vector iwti.RegClass:$rs1))),
-              (!cast<Instruction>(instruction_name#"_"#fvti.LMul.MX)
+              (!cast<Instruction>(instruction_name#"_"#fvti.LMul.MX#"_E"#fvti.SEW)
                   (fvti.Vector (IMPLICIT_DEF)),
                   iwti.RegClass:$rs1,
                   // Value to indicate no rounding mode change in
@@ -1445,7 +1445,7 @@ foreach fvtiToFWti = AllWidenableFloatVectors in {
                        !listconcat(GetVTypePredicates<fvti>.Predicates,
                                    GetVTypePredicates<fwti>.Predicates)) in
   def : Pat<(fvti.Vector (fpround (fwti.Vector fwti.RegClass:$rs1))),
-            (!cast<Instruction>("PseudoVFNCVT_F_F_W_"#fvti.LMul.MX)
+            (!cast<Instruction>("PseudoVFNCVT_F_F_W_"#fvti.LMul.MX#"_E"#fvti.SEW)
                 (fvti.Vector (IMPLICIT_DEF)),
                 fwti.RegClass:$rs1,
                 // Value to indicate no rounding mode change in
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
index eb46d47079b2..8e143b74a666 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
@@ -1389,7 +1389,7 @@ multiclass VPatNConvertI2FPVL_W_RM<SDPatternOperator vop,
     def : Pat<(fvti.Vector (vop (iwti.Vector iwti.RegClass:$rs1),
                                 (iwti.Mask V0),
                                 VLOpFrag)),
-              (!cast<Instruction>(instruction_name#"_"#fvti.LMul.MX#"_MASK")
+              (!cast<Instruction>(instruction_name#"_"#fvti.LMul.MX#"_E"#fvti.SEW#"_MASK") 
                   (fvti.Vector (IMPLICIT_DEF)), iwti.RegClass:$rs1,
                   (iwti.Mask V0),
                   // Value to indicate no rounding mode change in
@@ -1408,7 +1408,7 @@ multiclass VPatNConvertI2FP_RM_VL_W<SDNode vop, string instruction_name> {
     def : Pat<(fvti.Vector (vop (iwti.Vector iwti.RegClass:$rs1),
                                 (iwti.Mask V0),  (XLenVT timm:$frm),
                                 VLOpFrag)),
-              (!cast<Instruction>(instruction_name#"_"#fvti.LMul.MX#"_MASK")
+              (!cast<Instruction>(instruction_name#"_"#fvti.LMul.MX#"_E"#fvti.SEW#"_MASK")
                   (fvti.Vector (IMPLICIT_DEF)), iwti.RegClass:$rs1,
                   (iwti.Mask V0), timm:$frm, GPR:$vl, fvti.Log2SEW, TA_MA)>;
   }
@@ -2703,7 +2703,7 @@ foreach fvtiToFWti = AllWidenableFloatVectors in {
     def : Pat<(fvti.Vector (any_riscv_fpround_vl
                                (fwti.Vector fwti.RegClass:$rs1),
                                (fwti.Mask V0), VLOpFrag)),
-              (!cast<Instruction>("PseudoVFNCVT_F_F_W_"#fvti.LMul.MX#"_MASK")
+              (!cast<Instruction>("PseudoVFNCVT_F_F_W_"#fvti.LMul.MX#"_E"#fvti.SEW#"_MASK")
                   (fvti.Vector (IMPLICIT_DEF)), fwti.RegClass:$rs1,
                   (fwti.Mask V0),
                   // Value to indicate no rounding mode change in
@@ -2716,7 +2716,7 @@ foreach fvtiToFWti = AllWidenableFloatVectors in {
     def : Pat<(fvti.Vector (any_riscv_fncvt_rod_vl
                                (fwti.Vector fwti.RegClass:$rs1),
                                (fwti.Mask V0), VLOpFrag)),
-              (!cast<Instruction>("PseudoVFNCVT_ROD_F_F_W_"#fvti.LMul.MX#"_MASK")
+              (!cast<Instruction>("PseudoVFNCVT_ROD_F_F_W_"#fvti.LMul.MX#"_E"#fvti.SEW#"_MASK")
                   (fvti.Vector (IMPLICIT_DEF)), fwti.RegClass:$rs1,
                   (fwti.Mask V0), GPR:$vl, fvti.Log2SEW, TA_MA)>;
   }
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
index f1246749ac43..feeefa7c42c4 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
@@ -819,11 +819,13 @@ foreach mx = SchedMxListW in {
   }
 }
 foreach mx = SchedMxListFW in {
-  defvar Cycles = SiFive7GetCyclesNarrowing<mx>.c;
-  defvar IsWorstCase = SiFive7IsWorstCaseMX<mx, SchedMxListFW>.c;
-  let Latency = 8, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in {
-    defm "" : LMULWriteResMX<"WriteVFNCvtIToFV", [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVFNCvtFToFV", [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+  foreach sew = SchedSEWSet<mx, isF=1, isWidening=1>.val in {
+    defvar Cycles = SiFive7GetCyclesNarrowing<mx>.c;
+    defvar IsWorstCase = SiFive7IsWorstCaseMXSEW<mx, sew, SchedMxListFW, isF=1>.c;
+    let Latency = 8, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in {
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFNCvtIToFV", [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFNCvtFToFV", [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
+    }
   }
 }
 
@@ -1182,9 +1184,9 @@ defm "" : LMULReadAdvance<"ReadVFCvtFToIV", 0>;
 defm "" : LMULSEWReadAdvanceW<"ReadVFWCvtIToFV", 0>;
 defm "" : LMULReadAdvanceFW<"ReadVFWCvtFToIV", 0>;
 defm "" : LMULSEWReadAdvanceFW<"ReadVFWCvtFToFV", 0>;
-defm "" : LMULReadAdvanceFW<"ReadVFNCvtIToFV", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFNCvtIToFV", 0>;
 defm "" : LMULReadAdvanceW<"ReadVFNCvtFToIV", 0>;
-defm "" : LMULReadAdvanceFW<"ReadVFNCvtFToFV", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFNCvtFToFV", 0>;
 
 // 15. Vector Reduction Operations
 def : ReadAdvance<ReadVIRedV, 0>;
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
index 0884cea39693..9dda9c500bd0 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
@@ -563,11 +563,13 @@ foreach mx = SchedMxListW in {
   }
 }
 foreach mx = SchedMxListFW in {
-  defvar LMulLat = SiFiveP600GetLMulCycles<mx>.c;
-  defvar IsWorstCase = SiFiveP600IsWorstCaseMX<mx, SchedMxListFW>.c;
-  let Latency = 3, ReleaseAtCycles = [LMulLat] in {
-    defm "" : LMULWriteResMX<"WriteVFNCvtIToFV", [SiFiveP600VectorArith], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVFNCvtFToFV", [SiFiveP600VectorArith], mx, IsWorstCase>;
+  foreach sew = SchedSEWSet<mx, isF=1, isWidening=1>.val in {
+    defvar LMulLat = SiFiveP600GetLMulCycles<mx>.c;
+    defvar IsWorstCase = SiFiveP600IsWorstCaseMXSEW<mx, sew, SchedMxListFW, isF=1>.c;
+    let Latency = 3, ReleaseAtCycles = [LMulLat] in {
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFNCvtIToFV", [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFNCvtFToFV", [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
+    }
   }
 }
 
@@ -965,9 +967,9 @@ defm "" : LMULReadAdvance<"ReadVFCvtFToIV", 0>;
 defm "" : LMULSEWReadAdvanceW<"ReadVFWCvtIToFV", 0>;
 defm "" : LMULReadAdvanceFW<"ReadVFWCvtFToIV", 0>;
 defm "" : LMULSEWReadAdvanceFW<"ReadVFWCvtFToFV", 0>;
-defm "" : LMULReadAdvanceFW<"ReadVFNCvtIToFV", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFNCvtIToFV", 0>;
 defm "" : LMULReadAdvanceW<"ReadVFNCvtFToIV", 0>;
-defm "" : LMULReadAdvanceFW<"ReadVFNCvtFToFV", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFNCvtFToFV", 0>;
 
 // 15. Vector Reduction Operations
 def : ReadAdvance<ReadVIRedV, 0>;
diff --git a/llvm/lib/Target/RISCV/RISCVScheduleV.td b/llvm/lib/Target/RISCV/RISCVScheduleV.td
index 42c6dde89eac..14a4e4cbc59f 100644
--- a/llvm/lib/Target/RISCV/RISCVScheduleV.td
+++ b/llvm/lib/Target/RISCV/RISCVScheduleV.td
@@ -468,9 +468,9 @@ defm "" : LMULSEWSchedWritesW<"WriteVFWCvtIToFV">;
 defm "" : LMULSchedWritesFW<"WriteVFWCvtFToIV">;
 defm "" : LMULSEWSchedWritesFW<"WriteVFWCvtFToFV">;
 // 13.19. Narrowing Floating-Point/Integer Type-Convert Instructions
-defm "" : LMULSchedWritesFW<"WriteVFNCvtIToFV">;
+defm "" : LMULSEWSchedWritesFW<"WriteVFNCvtIToFV">;
 defm "" : LMULSchedWritesW<"WriteVFNCvtFToIV">;
-defm "" : LMULSchedWritesFW<"WriteVFNCvtFToFV">;
+defm "" : LMULSEWSchedWritesFW<"WriteVFNCvtFToFV">;
 
 // 14. Vector Reduction Operations
 // The latency of reduction is determined by the size of the read resource.
@@ -694,9 +694,9 @@ defm "" : LMULSEWSchedReadsW<"ReadVFWCvtIToFV">;
 defm "" : LMULSchedReadsFW<"ReadVFWCvtFToIV">;
 defm "" : LMULSEWSchedReadsFW<"ReadVFWCvtFToFV">;
 // 13.19. Narrowing Floating-Point/Integer Type-Convert Instructions
-defm "" : LMULSchedReadsFW<"ReadVFNCvtIToFV">;
+defm "" : LMULSEWSchedReadsFW<"ReadVFNCvtIToFV">;
 defm "" : LMULSchedReadsW<"ReadVFNCvtFToIV">;
-defm "" : LMULSchedReadsFW<"ReadVFNCvtFToFV">;
+defm "" : LMULSEWSchedReadsFW<"ReadVFNCvtFToFV">;
 
 // 14. Vector Reduction Operations
 // 14.1. Vector Single-Width Integer Reduction Instructions
@@ -922,9 +922,9 @@ defm "" : LMULWriteRes<"WriteVFCvtFToIV", []>;
 defm "" : LMULSEWWriteResW<"WriteVFWCvtIToFV", []>;
 defm "" : LMULWriteResFW<"WriteVFWCvtFToIV", []>;
 defm "" : LMULSEWWriteResFW<"WriteVFWCvtFToFV", []>;
-defm "" : LMULWriteResFW<"WriteVFNCvtIToFV", []>;
+defm "" : LMULSEWWriteResFW<"WriteVFNCvtIToFV", []>;
 defm "" : LMULWriteResW<"WriteVFNCvtFToIV", []>;
-defm "" : LMULWriteResFW<"WriteVFNCvtFToFV", []>;
+defm "" : LMULSEWWriteResFW<"WriteVFNCvtFToFV", []>;
 
 // 14. Vector Reduction Operations
 defm "" : LMULSEWWriteRes<"WriteVIRedV_From", []>;
@@ -1079,9 +1079,9 @@ defm "" : LMULReadAdvance<"ReadVFCvtFToIV", 0>;
 defm "" : LMULSEWReadAdvanceW<"ReadVFWCvtIToFV", 0>;
 defm "" : LMULReadAdvanceFW<"ReadVFWCvtFToIV", 0>;
 defm "" : LMULSEWReadAdvanceFW<"ReadVFWCvtFToFV", 0>;
-defm "" : LMULReadAdvanceFW<"ReadVFNCvtIToFV", 0>;
+defm "" : LMULSEWReadAdvanceFW<"SEWReadVFNCvtIToFV", 0>;
 defm "" : LMULReadAdvanceW<"ReadVFNCvtFToIV", 0>;
-defm "" : LMULReadAdvanceFW<"ReadVFNCvtFToFV", 0>;
+defm "" : LMULSEWReadAdvanceFW<"ReadVFNCvtFToFV", 0>;
 
 // 14. Vector Reduction Operations
 def : ReadAdvance<ReadVIRedV, 0>;
-- 
GitLab


From 1b310c45e914f0a76fa594bebdff44e7a386d4f9 Mon Sep 17 00:00:00 2001
From: Michael Maitland <michaeltmaitland@gmail.com>
Date: Wed, 10 Apr 2024 12:38:06 -0700
Subject: [PATCH 240/464] [RISCV] Split PseudoVFMIN, PseudoVFMAX PseudoVFSGNJ,
 PseudoVFSGNJN, and PseudoVFSGNJX by SEW

---
 .../Target/RISCV/RISCVInstrInfoVPseudos.td    | 37 +++++++++++--------
 .../Target/RISCV/RISCVInstrInfoVSDPatterns.td | 16 ++++----
 .../Target/RISCV/RISCVInstrInfoVVLPatterns.td | 14 +++----
 llvm/lib/Target/RISCV/RISCVSchedSiFive7.td    | 18 +++++----
 llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td | 24 ++++++++----
 llvm/lib/Target/RISCV/RISCVScheduleV.td       | 32 ++++++++--------
 6 files changed, 79 insertions(+), 62 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index 5d8b091392be..ad1821d57256 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -2894,32 +2894,34 @@ multiclass VPseudoVALU_VV_VX {
 
 multiclass VPseudoVSGNJ_VV_VF {
   foreach m = MxListF in {
-    defm "" : VPseudoBinaryFV_VV<m>,
+    foreach e = SchedSEWSet<m.MX, isF=1>.val in
+    defm "" : VPseudoBinaryFV_VV<m, sew=e>,
               SchedBinary<"WriteVFSgnjV", "ReadVFSgnjV", "ReadVFSgnjV", m.MX,
-                          forceMergeOpRead=true>;
+                          e, forceMergeOpRead=true>;
   }
 
   foreach f = FPList in {
     foreach m = f.MxList in {
-      defm "" : VPseudoBinaryV_VF<m, f>,
+      defm "" : VPseudoBinaryV_VF<m, f, sew=f.SEW>,
                 SchedBinary<"WriteVFSgnjF", "ReadVFSgnjV", "ReadVFSgnjF", m.MX,
-                            forceMergeOpRead=true>;
+                            f.SEW, forceMergeOpRead=true>;
     }
   }
 }
 
 multiclass VPseudoVMAX_VV_VF {
   foreach m = MxListF in {
-    defm "" : VPseudoBinaryFV_VV<m>,
-              SchedBinary<"WriteVFMinMaxV", "ReadVFMinMaxV", "ReadVFMinMaxV", m.MX,
-                          forceMergeOpRead=true>;
+    foreach e = SchedSEWSet<m.MX, isF=1>.val in
+      defm "" : VPseudoBinaryFV_VV<m, sew=e>,
+                SchedBinary<"WriteVFMinMaxV", "ReadVFMinMaxV", "ReadVFMinMaxV",
+                            m.MX, e, forceMergeOpRead=true>;
   }
 
   foreach f = FPList in {
     foreach m = f.MxList in {
-      defm "" : VPseudoBinaryV_VF<m, f>,
-                SchedBinary<"WriteVFMinMaxF", "ReadVFMinMaxV", "ReadVFMinMaxF", m.MX,
-                            forceMergeOpRead=true>;
+      defm "" : VPseudoBinaryV_VF<m, f, sew=f.SEW>,
+                SchedBinary<"WriteVFMinMaxF", "ReadVFMinMaxV", "ReadVFMinMaxF",
+                            m.MX, f.SEW, forceMergeOpRead=true>;
     }
   }
 }
@@ -7236,15 +7238,20 @@ defm : VPatUnaryV_V_RM<"int_riscv_vfrec7", "PseudoVFREC7", AllFloatVectors, isSE
 //===----------------------------------------------------------------------===//
 // 13.11. Vector Floating-Point Min/Max Instructions
 //===----------------------------------------------------------------------===//
-defm : VPatBinaryV_VV_VX<"int_riscv_vfmin", "PseudoVFMIN", AllFloatVectors>;
-defm : VPatBinaryV_VV_VX<"int_riscv_vfmax", "PseudoVFMAX", AllFloatVectors>;
+defm : VPatBinaryV_VV_VX<"int_riscv_vfmin", "PseudoVFMIN", AllFloatVectors,
+                         isSEWAware=1>;
+defm : VPatBinaryV_VV_VX<"int_riscv_vfmax", "PseudoVFMAX", AllFloatVectors,
+                         isSEWAware=1>;
 
 //===----------------------------------------------------------------------===//
 // 13.12. Vector Floating-Point Sign-Injection Instructions
 //===----------------------------------------------------------------------===//
-defm : VPatBinaryV_VV_VX<"int_riscv_vfsgnj", "PseudoVFSGNJ", AllFloatVectors>;
-defm : VPatBinaryV_VV_VX<"int_riscv_vfsgnjn", "PseudoVFSGNJN", AllFloatVectors>;
-defm : VPatBinaryV_VV_VX<"int_riscv_vfsgnjx", "PseudoVFSGNJX", AllFloatVectors>;
+defm : VPatBinaryV_VV_VX<"int_riscv_vfsgnj", "PseudoVFSGNJ", AllFloatVectors,
+                         isSEWAware=1>;
+defm : VPatBinaryV_VV_VX<"int_riscv_vfsgnjn", "PseudoVFSGNJN", AllFloatVectors,
+                         isSEWAware=1>;
+defm : VPatBinaryV_VV_VX<"int_riscv_vfsgnjx", "PseudoVFSGNJX", AllFloatVectors,
+                         isSEWAware=1>;
 
 //===----------------------------------------------------------------------===//
 // 13.13. Vector Floating-Point Compare Instructions
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
index 9b324c40cc5b..82ee4b0cbce9 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVSDPatterns.td
@@ -1339,42 +1339,42 @@ foreach vti = AllFloatVectors in {
 
     // 13.12. Vector Floating-Point Sign-Injection Instructions
     def : Pat<(fabs (vti.Vector vti.RegClass:$rs)),
-              (!cast<Instruction>("PseudoVFSGNJX_VV_"# vti.LMul.MX)
+              (!cast<Instruction>("PseudoVFSGNJX_VV_"# vti.LMul.MX#"_E"#vti.SEW)
                    (vti.Vector (IMPLICIT_DEF)),
                    vti.RegClass:$rs, vti.RegClass:$rs, vti.AVL, vti.Log2SEW, TA_MA)>;
     // Handle fneg with VFSGNJN using the same input for both operands.
     def : Pat<(fneg (vti.Vector vti.RegClass:$rs)),
-              (!cast<Instruction>("PseudoVFSGNJN_VV_"# vti.LMul.MX)
+              (!cast<Instruction>("PseudoVFSGNJN_VV_"# vti.LMul.MX#"_E"#vti.SEW)
                    (vti.Vector (IMPLICIT_DEF)),
                    vti.RegClass:$rs, vti.RegClass:$rs, vti.AVL, vti.Log2SEW, TA_MA)>;
 
     def : Pat<(vti.Vector (fcopysign (vti.Vector vti.RegClass:$rs1),
                                      (vti.Vector vti.RegClass:$rs2))),
-              (!cast<Instruction>("PseudoVFSGNJ_VV_"# vti.LMul.MX)
+              (!cast<Instruction>("PseudoVFSGNJ_VV_"# vti.LMul.MX#"_E"#vti.SEW)
                    (vti.Vector (IMPLICIT_DEF)),
                    vti.RegClass:$rs1, vti.RegClass:$rs2, vti.AVL, vti.Log2SEW, TA_MA)>;
     def : Pat<(vti.Vector (fcopysign (vti.Vector vti.RegClass:$rs1),
                                      (vti.Vector (SplatFPOp vti.ScalarRegClass:$rs2)))),
-              (!cast<Instruction>("PseudoVFSGNJ_V"#vti.ScalarSuffix#"_"#vti.LMul.MX)
+              (!cast<Instruction>("PseudoVFSGNJ_V"#vti.ScalarSuffix#"_"#vti.LMul.MX#"_E"#vti.SEW)
                    (vti.Vector (IMPLICIT_DEF)),
                    vti.RegClass:$rs1, vti.ScalarRegClass:$rs2, vti.AVL, vti.Log2SEW, TA_MA)>;
 
     def : Pat<(vti.Vector (fcopysign (vti.Vector vti.RegClass:$rs1),
                                      (vti.Vector (fneg vti.RegClass:$rs2)))),
-              (!cast<Instruction>("PseudoVFSGNJN_VV_"# vti.LMul.MX)
+              (!cast<Instruction>("PseudoVFSGNJN_VV_"# vti.LMul.MX#"_E"#vti.SEW)
                    (vti.Vector (IMPLICIT_DEF)),
                    vti.RegClass:$rs1, vti.RegClass:$rs2, vti.AVL, vti.Log2SEW, TA_MA)>;
     def : Pat<(vti.Vector (fcopysign (vti.Vector vti.RegClass:$rs1),
                                      (vti.Vector (fneg (SplatFPOp vti.ScalarRegClass:$rs2))))),
-              (!cast<Instruction>("PseudoVFSGNJN_V"#vti.ScalarSuffix#"_"#vti.LMul.MX)
+              (!cast<Instruction>("PseudoVFSGNJN_V"#vti.ScalarSuffix#"_"#vti.LMul.MX#"_E"#vti.SEW)
                    (vti.Vector (IMPLICIT_DEF)),
                    vti.RegClass:$rs1, vti.ScalarRegClass:$rs2, vti.AVL, vti.Log2SEW, TA_MA)>;
   }
 }
 
 // 13.11. Vector Floating-Point MIN/MAX Instructions
-defm : VPatBinaryFPSDNode_VV_VF<fminnum, "PseudoVFMIN">;
-defm : VPatBinaryFPSDNode_VV_VF<fmaxnum, "PseudoVFMAX">;
+defm : VPatBinaryFPSDNode_VV_VF<fminnum, "PseudoVFMIN", isSEWAware=1>;
+defm : VPatBinaryFPSDNode_VV_VF<fmaxnum, "PseudoVFMAX", isSEWAware=1>;
 
 // 13.13. Vector Floating-Point Compare Instructions
 defm : VPatFPSetCCSDNode_VV_VF_FV<SETEQ,  "PseudoVMFEQ", "PseudoVMFEQ">;
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
index 8e143b74a666..f2d97ba14725 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVVLPatterns.td
@@ -2468,8 +2468,8 @@ defm : VPatWidenFPMulAccVL_VV_VF_RM<riscv_vfwmsub_vl, "PseudoVFWMSAC">;
 defm : VPatWidenFPMulAccVL_VV_VF_RM<riscv_vfwnmsub_vl, "PseudoVFWNMSAC">;
 
 // 13.11. Vector Floating-Point MIN/MAX Instructions
-defm : VPatBinaryFPVL_VV_VF<riscv_vfmin_vl, "PseudoVFMIN">;
-defm : VPatBinaryFPVL_VV_VF<riscv_vfmax_vl, "PseudoVFMAX">;
+defm : VPatBinaryFPVL_VV_VF<riscv_vfmin_vl, "PseudoVFMIN", isSEWAware=1>;
+defm : VPatBinaryFPVL_VV_VF<riscv_vfmax_vl, "PseudoVFMAX", isSEWAware=1>;
 
 // 13.13. Vector Floating-Point Compare Instructions
 defm : VPatFPSetCCVL_VV_VF_FV<any_riscv_fsetcc_vl, SETEQ,
@@ -2505,14 +2505,14 @@ foreach vti = AllFloatVectors in {
     // 13.12. Vector Floating-Point Sign-Injection Instructions
     def : Pat<(riscv_fabs_vl (vti.Vector vti.RegClass:$rs), (vti.Mask V0),
                              VLOpFrag),
-              (!cast<Instruction>("PseudoVFSGNJX_VV_"# vti.LMul.MX #"_MASK")
+              (!cast<Instruction>("PseudoVFSGNJX_VV_"# vti.LMul.MX #"_E"#vti.SEW#"_MASK")
                    (vti.Vector (IMPLICIT_DEF)), vti.RegClass:$rs,
                    vti.RegClass:$rs, (vti.Mask V0), GPR:$vl, vti.Log2SEW,
                    TA_MA)>;
     // Handle fneg with VFSGNJN using the same input for both operands.
     def : Pat<(riscv_fneg_vl (vti.Vector vti.RegClass:$rs), (vti.Mask V0),
                              VLOpFrag),
-              (!cast<Instruction>("PseudoVFSGNJN_VV_"# vti.LMul.MX #"_MASK")
+              (!cast<Instruction>("PseudoVFSGNJN_VV_"# vti.LMul.MX#"_E"#vti.SEW #"_MASK")
                    (vti.Vector (IMPLICIT_DEF)), vti.RegClass:$rs,
                    vti.RegClass:$rs, (vti.Mask V0), GPR:$vl, vti.Log2SEW,
                    TA_MA)>;
@@ -2522,7 +2522,7 @@ foreach vti = AllFloatVectors in {
                                   vti.RegClass:$merge,
                                   (vti.Mask V0),
                                   VLOpFrag),
-              (!cast<Instruction>("PseudoVFSGNJ_VV_"# vti.LMul.MX#"_MASK")
+              (!cast<Instruction>("PseudoVFSGNJ_VV_"# vti.LMul.MX#"_E"#vti.SEW#"_MASK")
                    vti.RegClass:$merge, vti.RegClass:$rs1,
                    vti.RegClass:$rs2, (vti.Mask V0), GPR:$vl, vti.Log2SEW,
                    TAIL_AGNOSTIC)>;
@@ -2534,7 +2534,7 @@ foreach vti = AllFloatVectors in {
                                   srcvalue,
                                   (vti.Mask true_mask),
                                   VLOpFrag),
-              (!cast<Instruction>("PseudoVFSGNJN_VV_"# vti.LMul.MX)
+              (!cast<Instruction>("PseudoVFSGNJN_VV_"# vti.LMul.MX#"_E"#vti.SEW)
         (vti.Vector (IMPLICIT_DEF)),
                    vti.RegClass:$rs1, vti.RegClass:$rs2, GPR:$vl, vti.Log2SEW, TA_MA)>;
 
@@ -2543,7 +2543,7 @@ foreach vti = AllFloatVectors in {
                                   vti.RegClass:$merge,
                                   (vti.Mask V0),
                                   VLOpFrag),
-              (!cast<Instruction>("PseudoVFSGNJ_V"#vti.ScalarSuffix#"_"# vti.LMul.MX#"_MASK")
+              (!cast<Instruction>("PseudoVFSGNJ_V"#vti.ScalarSuffix#"_"# vti.LMul.MX#"_E"#vti.SEW#"_MASK")
                    vti.RegClass:$merge, vti.RegClass:$rs1,
                    vti.ScalarRegClass:$rs2, (vti.Mask V0), GPR:$vl, vti.Log2SEW,
                    TAIL_AGNOSTIC)>;
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
index feeefa7c42c4..4dcec96df9d5 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
@@ -746,6 +746,12 @@ foreach mx = SchedMxListF in {
       defm "" : LMULSEWWriteResMXSEW<"WriteVFRecpV",   [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
       defm "" : LMULSEWWriteResMXSEW<"WriteVFCvtIToFV", [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
     }
+    let Latency = 4, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in {
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFMinMaxV", [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFMinMaxF", [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFSgnjV",   [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFSgnjF",   [SiFive7VCQ, SiFive7VA], mx, sew, IsWorstCase>;
+    }
   }
 }
 foreach mx = SchedMxList in {
@@ -755,10 +761,6 @@ foreach mx = SchedMxList in {
     defm "" : LMULWriteResMX<"WriteVFCvtFToIV",  [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
   }
   let Latency = 4, AcquireAtCycles = [0, 1], ReleaseAtCycles = [1, !add(1, Cycles)] in {
-    defm "" : LMULWriteResMX<"WriteVFSgnjV",     [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVFSgnjF",     [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVFMinMaxV",   [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVFMinMaxF",   [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFClassV",    [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFMergeV",    [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFMovV",      [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
@@ -1169,10 +1171,10 @@ defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulAddV", 0>;
 defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulAddF", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFSqrtV", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFRecpV", 0>;
-defm "" : LMULReadAdvance<"ReadVFMinMaxV", 0>;
-defm "" : LMULReadAdvance<"ReadVFMinMaxF", 0>;
-defm "" : LMULReadAdvance<"ReadVFSgnjV", 0>;
-defm "" : LMULReadAdvance<"ReadVFSgnjF", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFMinMaxV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFMinMaxF", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFSgnjV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFSgnjF", 0>;
 defm "" : LMULReadAdvance<"ReadVFCmpV", 0>;
 defm "" : LMULReadAdvance<"ReadVFCmpF", 0>;
 defm "" : LMULReadAdvance<"ReadVFClassV", 0>;
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
index 9dda9c500bd0..0ced2efa3f7a 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
@@ -504,6 +504,18 @@ foreach mx = SchedMxListF in {
     defm "" : LMULSEWWriteResMXSEW<"WriteVFCvtIToFV", [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
   }
 }
+foreach mx = SchedMxListF in {
+  foreach sew = SchedSEWSet<mx, isF=1>.val in {
+    defvar LMulLat = SiFiveP600GetLMulCycles<mx>.c;
+    defvar IsWorstCase = SiFiveP600IsWorstCaseMXSEW<mx, sew, SchedMxList, isF=1>.c;
+    let Latency = 1, ReleaseAtCycles = [LMulLat] in {
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFMinMaxV", [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFMinMaxF", [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFSgnjV",   [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
+      defm "" : LMULSEWWriteResMXSEW<"WriteVFSgnjF",   [SiFiveP600VectorArith], mx, sew, IsWorstCase>;
+    }
+  }
+}
 foreach mx = SchedMxList in {
   defvar LMulLat = SiFiveP600GetLMulCycles<mx>.c;
   defvar IsWorstCase = SiFiveP600IsWorstCaseMX<mx, SchedMxList>.c;
@@ -514,10 +526,6 @@ foreach mx = SchedMxList in {
     defm "" : LMULWriteResMX<"WriteVFCmpF",  [SiFiveP600VectorArith], mx, IsWorstCase>;
   }
   let Latency = 1, ReleaseAtCycles = [LMulLat] in {
-    defm "" : LMULWriteResMX<"WriteVFSgnjV",   [SiFiveP600VectorArith], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVFSgnjF",   [SiFiveP600VectorArith], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVFMinMaxV", [SiFiveP600VectorArith], mx, IsWorstCase>;
-    defm "" : LMULWriteResMX<"WriteVFMinMaxF", [SiFiveP600VectorArith], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFClassV",  [SiFiveP600VectorArith], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFMergeV",  [SiFiveP600VectorArith], mx, IsWorstCase>;
     defm "" : LMULWriteResMX<"WriteVFMovV",    [SiFiveP600VectorArith], mx, IsWorstCase>;
@@ -954,10 +962,10 @@ defm "" : LMULSEWReadAdvanceF<"ReadVFSqrtV", 0>;
 defm "" : LMULSEWReadAdvance<"ReadVFRecpV", 0>;
 defm "" : LMULReadAdvance<"ReadVFCmpV", 0>;
 defm "" : LMULReadAdvance<"ReadVFCmpF", 0>;
-defm "" : LMULReadAdvance<"ReadVFMinMaxV", 0>;
-defm "" : LMULReadAdvance<"ReadVFMinMaxF", 0>;
-defm "" : LMULReadAdvance<"ReadVFSgnjV", 0>;
-defm "" : LMULReadAdvance<"ReadVFSgnjF", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFMinMaxV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFMinMaxF", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFSgnjV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFSgnjF", 0>;
 defm "" : LMULReadAdvance<"ReadVFClassV", 0>;
 defm "" : LMULReadAdvance<"ReadVFMergeV", 0>;
 defm "" : LMULReadAdvance<"ReadVFMergeF", 0>;
diff --git a/llvm/lib/Target/RISCV/RISCVScheduleV.td b/llvm/lib/Target/RISCV/RISCVScheduleV.td
index 14a4e4cbc59f..5993884bc2c1 100644
--- a/llvm/lib/Target/RISCV/RISCVScheduleV.td
+++ b/llvm/lib/Target/RISCV/RISCVScheduleV.td
@@ -446,11 +446,11 @@ defm "" : LMULSEWSchedWritesF<"WriteVFSqrtV">;
 // 13.10. Vector Floating-Point Reciprocal Estimate Instruction
 defm "" : LMULSEWSchedWritesF<"WriteVFRecpV">;
 // 13.11. Vector Floating-Point MIN/MAX Instructions
-defm "" : LMULSchedWrites<"WriteVFMinMaxV">;
-defm "" : LMULSchedWrites<"WriteVFMinMaxF">;
+defm "" : LMULSEWSchedWritesF<"WriteVFMinMaxV">;
+defm "" : LMULSEWSchedWritesF<"WriteVFMinMaxF">;
 // 13.12. Vector Floating-Point Sign-Injection Instructions
-defm "" : LMULSchedWrites<"WriteVFSgnjV">;
-defm "" : LMULSchedWrites<"WriteVFSgnjF">;
+defm "" : LMULSEWSchedWritesF<"WriteVFSgnjV">;
+defm "" : LMULSEWSchedWritesF<"WriteVFSgnjF">;
 // 13.13. Vector Floating-Point Compare Instructions
 defm "" : LMULSchedWrites<"WriteVFCmpV">;
 defm "" : LMULSchedWrites<"WriteVFCmpF">;
@@ -671,11 +671,11 @@ defm "" : LMULSEWSchedReadsF<"ReadVFSqrtV">;
 // 13.10. Vector Floating-Point Reciprocal Estimate Instruction
 defm "" : LMULSEWSchedReadsF<"ReadVFRecpV">;
 // 13.11. Vector Floating-Point MIN/MAX Instructions
-defm "" : LMULSchedReads<"ReadVFMinMaxV">;
-defm "" : LMULSchedReads<"ReadVFMinMaxF">;
+defm "" : LMULSEWSchedReadsF<"ReadVFMinMaxV">;
+defm "" : LMULSEWSchedReadsF<"ReadVFMinMaxF">;
 // 13.12. Vector Floating-Point Sign-Injection Instructions
-defm "" : LMULSchedReads<"ReadVFSgnjV">;
-defm "" : LMULSchedReads<"ReadVFSgnjF">;
+defm "" : LMULSEWSchedReadsF<"ReadVFSgnjV">;
+defm "" : LMULSEWSchedReadsF<"ReadVFSgnjF">;
 // 13.13. Vector Floating-Point Compare Instructions
 defm "" : LMULSchedReads<"ReadVFCmpV">;
 defm "" : LMULSchedReads<"ReadVFCmpF">;
@@ -908,10 +908,10 @@ defm "" : LMULSEWWriteResFW<"WriteVFWMulAddV", []>;
 defm "" : LMULSEWWriteResFW<"WriteVFWMulAddF", []>;
 defm "" : LMULSEWWriteResF<"WriteVFSqrtV", []>;
 defm "" : LMULSEWWriteResF<"WriteVFRecpV", []>;
-defm "" : LMULWriteRes<"WriteVFMinMaxV", []>;
-defm "" : LMULWriteRes<"WriteVFMinMaxF", []>;
-defm "" : LMULWriteRes<"WriteVFSgnjV", []>;
-defm "" : LMULWriteRes<"WriteVFSgnjF", []>;
+defm "" : LMULSEWWriteResF<"WriteVFMinMaxV", []>;
+defm "" : LMULSEWWriteResF<"WriteVFMinMaxF", []>;
+defm "" : LMULSEWWriteResF<"WriteVFSgnjV", []>;
+defm "" : LMULSEWWriteResF<"WriteVFSgnjF", []>;
 defm "" : LMULWriteRes<"WriteVFCmpV", []>;
 defm "" : LMULWriteRes<"WriteVFCmpF", []>;
 defm "" : LMULWriteRes<"WriteVFClassV", []>;
@@ -1064,10 +1064,10 @@ defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulAddV", 0>;
 defm "" : LMULSEWReadAdvanceFW<"ReadVFWMulAddF", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFSqrtV", 0>;
 defm "" : LMULSEWReadAdvanceF<"ReadVFRecpV", 0>;
-defm "" : LMULReadAdvance<"ReadVFMinMaxV", 0>;
-defm "" : LMULReadAdvance<"ReadVFMinMaxF", 0>;
-defm "" : LMULReadAdvance<"ReadVFSgnjV", 0>;
-defm "" : LMULReadAdvance<"ReadVFSgnjF", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFMinMaxV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFMinMaxF", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFSgnjV", 0>;
+defm "" : LMULSEWReadAdvanceF<"ReadVFSgnjF", 0>;
 defm "" : LMULReadAdvance<"ReadVFCmpV", 0>;
 defm "" : LMULReadAdvance<"ReadVFCmpF", 0>;
 defm "" : LMULReadAdvance<"ReadVFClassV", 0>;
-- 
GitLab


From 17cb8a537b688ac6e4187f28f9e784491bb8abc5 Mon Sep 17 00:00:00 2001
From: Kiran Chandramohan <kiran.chandramohan@arm.com>
Date: Mon, 15 Apr 2024 14:24:42 +0100
Subject: [PATCH 241/464] [Flang][OpenMP] Accept the reduction modifier
 (#86492)

Accept the reduction modifier in the Flang parser. Issue a TODO message
during lowering.

OpenMP 5.0 introduced the reduction modifier. Details can be seen in
2.19.5.4 reductionClause.
OpenMP 5.2 relevant section is 5.5.8reductionClause.

This will help remove some of the parser errors highlighted in the
following post and also bring it to a well defined behaviour (produce
TODO errors for unsupported features, do not crash).
https://discourse.llvm.org/t/proposal-rename-flang-new-to-flang/69462/60
---
 flang/include/flang/Parser/dump-parse-tree.h  |  1 +
 flang/include/flang/Parser/parse-tree.h       |  5 +++-
 flang/lib/Lower/OpenMP/Clauses.cpp            | 26 +++++++++++++++----
 flang/lib/Lower/OpenMP/ReductionProcessor.cpp |  5 ++++
 flang/lib/Parser/openmp-parsers.cpp           |  5 ++++
 flang/lib/Parser/unparse.cpp                  |  4 +++
 flang/lib/Semantics/check-omp-structure.cpp   |  2 +-
 .../Lower/OpenMP/Todo/reduction-modifiers.f90 | 13 ++++++++++
 .../test/Parser/OpenMP/reduction-modifier.f90 | 20 ++++++++++++++
 llvm/include/llvm/Frontend/OpenMP/ClauseT.h   |  2 +-
 10 files changed, 75 insertions(+), 8 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/Todo/reduction-modifiers.f90
 create mode 100644 flang/test/Parser/OpenMP/reduction-modifier.f90

diff --git a/flang/include/flang/Parser/dump-parse-tree.h b/flang/include/flang/Parser/dump-parse-tree.h
index 06c168a5de61..477d391277ee 100644
--- a/flang/include/flang/Parser/dump-parse-tree.h
+++ b/flang/include/flang/Parser/dump-parse-tree.h
@@ -542,6 +542,7 @@ public:
   NODE_ENUM(OmpOrderModifier, Kind)
   NODE(parser, OmpProcBindClause)
   NODE_ENUM(OmpProcBindClause, Type)
+  NODE_ENUM(OmpReductionClause, ReductionModifier)
   NODE(parser, OmpReductionClause)
   NODE(parser, OmpInReductionClause)
   NODE(parser, OmpReductionCombiner)
diff --git a/flang/include/flang/Parser/parse-tree.h b/flang/include/flang/Parser/parse-tree.h
index 574a95cf22af..d7c23755c57b 100644
--- a/flang/include/flang/Parser/parse-tree.h
+++ b/flang/include/flang/Parser/parse-tree.h
@@ -3552,7 +3552,10 @@ struct OmpReductionOperator {
 //                                         variable-name-list)
 struct OmpReductionClause {
   TUPLE_CLASS_BOILERPLATE(OmpReductionClause);
-  std::tuple<OmpReductionOperator, OmpObjectList> t;
+  ENUM_CLASS(ReductionModifier, Inscan, Task, Default)
+  std::tuple<std::optional<ReductionModifier>, OmpReductionOperator,
+      OmpObjectList>
+      t;
 };
 
 // OMP 5.0 2.19.5.6 in_reduction-clause -> IN_REDUCTION (reduction-identifier:
diff --git a/flang/lib/Lower/OpenMP/Clauses.cpp b/flang/lib/Lower/OpenMP/Clauses.cpp
index 40da71c8b55f..97337cfc08c7 100644
--- a/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -1005,12 +1005,28 @@ ProcBind make(const parser::OmpClause::ProcBind &inp,
 Reduction make(const parser::OmpClause::Reduction &inp,
                semantics::SemanticsContext &semaCtx) {
   // inp.v -> parser::OmpReductionClause
-  auto &t0 = std::get<parser::OmpReductionOperator>(inp.v.t);
-  auto &t1 = std::get<parser::OmpObjectList>(inp.v.t);
+  using wrapped = parser::OmpReductionClause;
+
+  CLAUSET_ENUM_CONVERT( //
+      convert, wrapped::ReductionModifier, Reduction::ReductionModifier,
+      // clang-format off
+      MS(Inscan,  Inscan)
+      MS(Task,    Task)
+      MS(Default, Default)
+      // clang-format on
+  );
+
+  auto &t0 =
+      std::get<std::optional<parser::OmpReductionClause::ReductionModifier>>(
+          inp.v.t);
+  auto &t1 = std::get<parser::OmpReductionOperator>(inp.v.t);
+  auto &t2 = std::get<parser::OmpObjectList>(inp.v.t);
   return Reduction{
-      {/*ReductionIdentifiers=*/{makeReductionOperator(t0, semaCtx)},
-       /*ReductionModifier=*/std::nullopt,
-       /*List=*/makeObjects(t1, semaCtx)}};
+      {/*ReductionModifier=*/t0
+           ? std::make_optional<Reduction::ReductionModifier>(convert(*t0))
+           : std::nullopt,
+       /*ReductionIdentifiers=*/{makeReductionOperator(t1, semaCtx)},
+       /*List=*/makeObjects(t2, semaCtx)}};
 }
 
 // Relaxed: empty
diff --git a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
index 918edf27baf6..f42386fe2736 100644
--- a/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ReductionProcessor.cpp
@@ -574,6 +574,11 @@ void ReductionProcessor::addDeclareReduction(
     llvm::SmallVectorImpl<const Fortran::semantics::Symbol *>
         *reductionSymbols) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+
+  if (std::get<std::optional<omp::clause::Reduction::ReductionModifier>>(
+          reduction.t))
+    TODO(currentLocation, "Reduction modifiers are not supported");
+
   mlir::omp::DeclareReductionOp decl;
   const auto &redOperatorList{
       std::get<omp::clause::Reduction::ReductionIdentifiers>(reduction.t)};
diff --git a/flang/lib/Parser/openmp-parsers.cpp b/flang/lib/Parser/openmp-parsers.cpp
index bba1be27158c..eae478416914 100644
--- a/flang/lib/Parser/openmp-parsers.cpp
+++ b/flang/lib/Parser/openmp-parsers.cpp
@@ -136,6 +136,11 @@ TYPE_PARSER(construct<OmpReductionOperator>(Parser<DefinedOperator>{}) ||
     construct<OmpReductionOperator>(Parser<ProcedureDesignator>{}))
 
 TYPE_PARSER(construct<OmpReductionClause>(
+    maybe(
+        ("INSCAN" >> pure(OmpReductionClause::ReductionModifier::Inscan) ||
+            "TASK" >> pure(OmpReductionClause::ReductionModifier::Task) ||
+            "DEFAULT" >> pure(OmpReductionClause::ReductionModifier::Default)) /
+        ","),
     Parser<OmpReductionOperator>{} / ":", Parser<OmpObjectList>{}))
 
 // OMP 5.0 2.19.5.6 IN_REDUCTION (reduction-identifier: variable-name-list)
diff --git a/flang/lib/Parser/unparse.cpp b/flang/lib/Parser/unparse.cpp
index c06458833f07..3398b395f198 100644
--- a/flang/lib/Parser/unparse.cpp
+++ b/flang/lib/Parser/unparse.cpp
@@ -2090,6 +2090,8 @@ public:
     Walk(":", x.step);
   }
   void Unparse(const OmpReductionClause &x) {
+    Walk(std::get<std::optional<OmpReductionClause::ReductionModifier>>(x.t),
+        ",");
     Walk(std::get<OmpReductionOperator>(x.t));
     Put(":");
     Walk(std::get<OmpObjectList>(x.t));
@@ -2727,6 +2729,8 @@ public:
   WALK_NESTED_ENUM(OmpScheduleClause, ScheduleType) // OMP schedule-type
   WALK_NESTED_ENUM(OmpDeviceClause, DeviceModifier) // OMP device modifier
   WALK_NESTED_ENUM(OmpDeviceTypeClause, Type) // OMP DEVICE_TYPE
+  WALK_NESTED_ENUM(
+      OmpReductionClause, ReductionModifier) // OMP reduction-modifier
   WALK_NESTED_ENUM(OmpIfClause, DirectiveNameModifier) // OMP directive-modifier
   WALK_NESTED_ENUM(OmpCancelType, Type) // OMP cancel-type
   WALK_NESTED_ENUM(OmpOrderClause, Type) // OMP order-type
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index bf4debee1df3..e85d8d1f7ab5 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -2289,7 +2289,7 @@ void OmpStructureChecker::Enter(const parser::OmpClause::Reduction &x) {
 bool OmpStructureChecker::CheckReductionOperators(
     const parser::OmpClause::Reduction &x) {
 
-  const auto &definedOp{std::get<0>(x.v.t)};
+  const auto &definedOp{std::get<parser::OmpReductionOperator>(x.v.t)};
   bool ok = false;
   common::visit(
       common::visitors{
diff --git a/flang/test/Lower/OpenMP/Todo/reduction-modifiers.f90 b/flang/test/Lower/OpenMP/Todo/reduction-modifiers.f90
new file mode 100644
index 000000000000..5e566466492c
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/reduction-modifiers.f90
@@ -0,0 +1,13 @@
+! RUN: %not_todo_cmd bbc -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
+! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
+
+! CHECK: not yet implemented: Reduction modifiers are not supported
+
+subroutine foo()
+  integer :: i, j
+  j = 0
+  !$omp do reduction (inscan, *: j)
+  do i = 1, 10
+    j = j + 1
+  end do
+end subroutine
diff --git a/flang/test/Parser/OpenMP/reduction-modifier.f90 b/flang/test/Parser/OpenMP/reduction-modifier.f90
new file mode 100644
index 000000000000..d46aa7095959
--- /dev/null
+++ b/flang/test/Parser/OpenMP/reduction-modifier.f90
@@ -0,0 +1,20 @@
+! RUN: %flang_fc1 -fdebug-unparse-no-sema -fopenmp %s | FileCheck --ignore-case %s
+! RUN: %flang_fc1 -fdebug-dump-parse-tree-no-sema -fopenmp %s | FileCheck --check-prefix="PARSE-TREE" %s
+
+subroutine foo()
+  integer :: i, j
+  j = 0
+! CHECK: !$OMP DO  REDUCTION(TASK,*:j)
+! PARSE-TREE: | | ExecutionPartConstruct -> ExecutableConstruct -> OpenMPConstruct -> OpenMPLoopConstruct
+! PARSE-TREE: | | | OmpBeginLoopDirective
+! PARSE-TREE: | | | | OmpLoopDirective -> llvm::omp::Directive = do
+! PARSE-TREE: | | | | OmpClauseList -> OmpClause -> Reduction -> OmpReductionClause
+! PARSE-TREE: | | | | | ReductionModifier = Task
+! PARSE-TREE: | | | | | OmpReductionOperator -> DefinedOperator -> IntrinsicOperator = Multiply
+! PARSE-TREE: | | | | | OmpObjectList -> OmpObject -> Designator -> DataRef -> Name = 'j
+  !$omp do reduction (task, *: j)
+  do i = 1, 10
+    j = j + 1
+  end do
+  !$omp end do
+end
diff --git a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
index 6ce972adcf0f..daef02bcfc9a 100644
--- a/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
+++ b/llvm/include/llvm/Frontend/OpenMP/ClauseT.h
@@ -949,7 +949,7 @@ struct ReductionT {
   using ReductionIdentifiers = ListT<type::ReductionIdentifierT<I, E>>;
   ENUM(ReductionModifier, Default, Inscan, Task);
   using TupleTrait = std::true_type;
-  std::tuple<ReductionIdentifiers, OPT(ReductionModifier), List> t;
+  std::tuple<OPT(ReductionModifier), ReductionIdentifiers, List> t;
 };
 
 // V5.2: [15.8.1] `memory-order` clauses
-- 
GitLab


From 67e733d36ef0c5ec9fab899ecf9f191d83c7d418 Mon Sep 17 00:00:00 2001
From: Nathan Sidwell <nathan@acm.org>
Date: Mon, 15 Apr 2024 09:46:58 -0400
Subject: [PATCH 242/464] [BOLT][NFC] Refactor relocation printing (#88180)

Some refactoring.

* Make the arrays themselves const -- not just the strings being pointed to.
* Move each relocation array to the case using it.
* Add a suitable assert, to make sure there's no out-of-bounds indexing.
---
 bolt/lib/Core/Relocation.cpp | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/bolt/lib/Core/Relocation.cpp b/bolt/lib/Core/Relocation.cpp
index d16b7a94787c..4e888a5b147a 100644
--- a/bolt/lib/Core/Relocation.cpp
+++ b/bolt/lib/Core/Relocation.cpp
@@ -1064,21 +1064,19 @@ MCBinaryExpr::Opcode Relocation::getComposeOpcodeFor(uint64_t Type) {
   }
 }
 
-#define ELF_RELOC(name, value) #name,
-
 void Relocation::print(raw_ostream &OS) const {
-  static const char *X86RelocNames[] = {
-#include "llvm/BinaryFormat/ELFRelocs/x86_64.def"
-  };
-  static const char *AArch64RelocNames[] = {
-#include "llvm/BinaryFormat/ELFRelocs/AArch64.def"
-  };
   switch (Arch) {
   default:
     OS << "RType:" << Twine::utohexstr(Type);
     break;
 
   case Triple::aarch64:
+    static const char *const AArch64RelocNames[] = {
+#define ELF_RELOC(name, value) #name,
+#include "llvm/BinaryFormat/ELFRelocs/AArch64.def"
+#undef ELF_RELOC
+    };
+    assert(Type < ArrayRef(AArch64RelocNames).size());
     OS << AArch64RelocNames[Type];
     break;
 
@@ -1088,16 +1086,22 @@ void Relocation::print(raw_ostream &OS) const {
     switch (Type) {
     default:
       llvm_unreachable("illegal RISC-V relocation");
-#undef ELF_RELOC
 #define ELF_RELOC(name, value)                                                 \
   case value:                                                                  \
     OS << #name;                                                               \
     break;
 #include "llvm/BinaryFormat/ELFRelocs/RISCV.def"
+#undef ELF_RELOC
     }
     break;
 
   case Triple::x86_64:
+    static const char *const X86RelocNames[] = {
+#define ELF_RELOC(name, value) #name,
+#include "llvm/BinaryFormat/ELFRelocs/x86_64.def"
+#undef ELF_RELOC
+    };
+    assert(Type < ArrayRef(X86RelocNames).size());
     OS << X86RelocNames[Type];
     break;
   }
-- 
GitLab


From 105dcc882cf0152baeaa02ac0e50e2527b6940db Mon Sep 17 00:00:00 2001
From: Farzon Lotfi <1802579+farzonl@users.noreply.github.com>
Date: Mon, 15 Apr 2024 06:52:47 -0700
Subject: [PATCH 243/464] [HLSL][SPIRV] Add any intrinsic lowering (#88325)

- `CGBuiltin.cpp` - Switch to using
`CGM.getHLSLRuntime().get##NAME##Intrinsic()`
- `CGHLSLRuntime.h` - Add any to backend intrinsic abstraction
-  `IntrinsicsSPIRV.td` - Add any intrinsic to SPIR-V.
- `SPIRVInstructionSelector.cpp` - Add means of selecting any intrinsic.
Any and All share the same behavior up to the opCode. They are only
different in vector cases.

Completes #88045
---
 clang/lib/CodeGen/CGBuiltin.cpp               |   3 +-
 clang/lib/CodeGen/CGHLSLRuntime.h             |   1 +
 clang/test/CodeGenHLSL/builtins/any.hlsl      | 370 ++++++++++++------
 llvm/include/llvm/IR/IntrinsicsSPIRV.td       |   1 +
 .../Target/SPIRV/SPIRVInstructionSelector.cpp |  29 +-
 .../test/CodeGen/SPIRV/hlsl-intrinsics/any.ll | 187 +++++++++
 6 files changed, 460 insertions(+), 131 deletions(-)
 create mode 100644 llvm/test/CodeGen/SPIRV/hlsl-intrinsics/any.ll

diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index c7b219dcfcec..9f95697f284c 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -18194,7 +18194,8 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
     Value *Op0 = EmitScalarExpr(E->getArg(0));
     return Builder.CreateIntrinsic(
         /*ReturnType=*/llvm::Type::getInt1Ty(getLLVMContext()),
-        Intrinsic::dx_any, ArrayRef<Value *>{Op0}, nullptr, "dx.any");
+        CGM.getHLSLRuntime().getAnyIntrinsic(), ArrayRef<Value *>{Op0}, nullptr,
+        "hlsl.any");
   }
   case Builtin::BI__builtin_hlsl_elementwise_clamp: {
     Value *OpX = EmitScalarExpr(E->getArg(0));
diff --git a/clang/lib/CodeGen/CGHLSLRuntime.h b/clang/lib/CodeGen/CGHLSLRuntime.h
index 2b8073aef973..506b364f5b2e 100644
--- a/clang/lib/CodeGen/CGHLSLRuntime.h
+++ b/clang/lib/CodeGen/CGHLSLRuntime.h
@@ -73,6 +73,7 @@ public:
   //===----------------------------------------------------------------------===//
 
   GENERATE_HLSL_INTRINSIC_FUNCTION(All, all)
+  GENERATE_HLSL_INTRINSIC_FUNCTION(Any, any)
   GENERATE_HLSL_INTRINSIC_FUNCTION(ThreadId, thread_id)
 
   //===----------------------------------------------------------------------===//
diff --git a/clang/test/CodeGenHLSL/builtins/any.hlsl b/clang/test/CodeGenHLSL/builtins/any.hlsl
index ae348fec756b..84584281a3b7 100644
--- a/clang/test/CodeGenHLSL/builtins/any.hlsl
+++ b/clang/test/CodeGenHLSL/builtins/any.hlsl
@@ -1,186 +1,304 @@
 // RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN:   spirv-unknown-vulkan-compute %s -fnative-half-type \
+// RUN:   -emit-llvm -disable-llvm-passes -o - | FileCheck %s \ 
+// RUN:   --check-prefixes=CHECK,NATIVE_HALF,SPIR_NATIVE_HALF,SPIR_CHECK
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN:   spirv-unknown-vulkan-compute %s -emit-llvm -disable-llvm-passes \
+// RUN:   -o - | FileCheck %s --check-prefixes=CHECK,SPIR_NO_HALF,SPIR_CHECK
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
 // RUN:   dxil-pc-shadermodel6.3-library %s -fnative-half-type \
 // RUN:   -emit-llvm -disable-llvm-passes -o - | FileCheck %s \ 
-// RUN:   --check-prefixes=CHECK,NATIVE_HALF
+// RUN:   --check-prefixes=CHECK,NATIVE_HALF,DXIL_NATIVE_HALF,DXIL_CHECK
 // RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
 // RUN:   dxil-pc-shadermodel6.3-library %s -emit-llvm -disable-llvm-passes \
-// RUN:   -o - | FileCheck %s --check-prefixes=CHECK,NO_HALF
+// RUN:   -o - | FileCheck %s --check-prefixes=CHECK,DXIL_NO_HALF,DXIL_CHECK
 
 #ifdef __HLSL_ENABLE_16_BIT
-// NATIVE_HALF: define noundef i1 @
-// NATIVE_HALF: %dx.any = call i1 @llvm.dx.any.i16
-// NATIVE_HALF: ret i1 %dx.any
+// DXIL_NATIVE_HALF: define noundef i1 @
+// SPIR_NATIVE_HALF: define spir_func noundef i1 @
+// DXIL_NATIVE_HALF: %hlsl.any = call i1 @llvm.dx.any.i16
+// SPIR_NATIVE_HALF: %hlsl.any = call i1 @llvm.spv.any.i16
+// NATIVE_HALF: ret i1 %hlsl.any
 bool test_any_int16_t(int16_t p0) { return any(p0); }
-// NATIVE_HALF: define noundef i1 @
-// NATIVE_HALF: %dx.any = call i1 @llvm.dx.any.v2i16
-// NATIVE_HALF: ret i1 %dx.any
+
+// DXIL_NATIVE_HALF: define noundef i1 @
+// SPIR_NATIVE_HALF: define spir_func noundef i1 @
+// DXIL_NATIVE_HALF: %hlsl.any = call i1 @llvm.dx.any.v2i16
+// SPIR_NATIVE_HALF: %hlsl.any = call i1 @llvm.spv.any.v2i16
+// NATIVE_HALF: ret i1 %hlsl.any
 bool test_any_int16_t2(int16_t2 p0) { return any(p0); }
-// NATIVE_HALF: define noundef i1 @
-// NATIVE_HALF: %dx.any = call i1 @llvm.dx.any.v3i16
-// NATIVE_HALF: ret i1 %dx.any
+
+// DXIL_NATIVE_HALF: define noundef i1 @
+// SPIR_NATIVE_HALF: define spir_func noundef i1 @
+// DXIL_NATIVE_HALF: %hlsl.any = call i1 @llvm.dx.any.v3i16
+// SPIR_NATIVE_HALF: %hlsl.any = call i1 @llvm.spv.any.v3i16
+// NATIVE_HALF: ret i1 %hlsl.any
 bool test_any_int16_t3(int16_t3 p0) { return any(p0); }
-// NATIVE_HALF: define noundef i1 @
-// NATIVE_HALF: %dx.any = call i1 @llvm.dx.any.v4i16
-// NATIVE_HALF: ret i1 %dx.any
+
+// DXIL_NATIVE_HALF: define noundef i1 @
+// SPIR_NATIVE_HALF: define spir_func noundef i1 @
+// DXIL_NATIVE_HALF: %hlsl.any = call i1 @llvm.dx.any.v4i16
+// SPIR_NATIVE_HALF: %hlsl.any = call i1 @llvm.spv.any.v4i16
+// NATIVE_HALF: ret i1 %hlsl.any
 bool test_any_int16_t4(int16_t4 p0) { return any(p0); }
 
-// NATIVE_HALF: define noundef i1 @
-// NATIVE_HALF: %dx.any = call i1 @llvm.dx.any.i16
-// NATIVE_HALF: ret i1 %dx.any
+// DXIL_NATIVE_HALF: define noundef i1 @
+// SPIR_NATIVE_HALF: define spir_func noundef i1 @
+// DXIL_NATIVE_HALF: %hlsl.any = call i1 @llvm.dx.any.i16
+// SPIR_NATIVE_HALF: %hlsl.any = call i1 @llvm.spv.any.i16
+// NATIVE_HALF: ret i1 %hlsl.any
 bool test_any_uint16_t(uint16_t p0) { return any(p0); }
-// NATIVE_HALF: define noundef i1 @
-// NATIVE_HALF: %dx.any = call i1 @llvm.dx.any.v2i16
-// NATIVE_HALF: ret i1 %dx.any
+
+// DXIL_NATIVE_HALF: define noundef i1 @
+// SPIR_NATIVE_HALF: define spir_func noundef i1 @
+// DXIL_NATIVE_HALF: %hlsl.any = call i1 @llvm.dx.any.v2i16
+// SPIR_NATIVE_HALF: %hlsl.any = call i1 @llvm.spv.any.v2i16
+// NATIVE_HALF: ret i1 %hlsl.any
 bool test_any_uint16_t2(uint16_t2 p0) { return any(p0); }
-// NATIVE_HALF: define noundef i1 @
-// NATIVE_HALF: %dx.any = call i1 @llvm.dx.any.v3i16
-// NATIVE_HALF: ret i1 %dx.any
+
+// DXIL_NATIVE_HALF: define noundef i1 @
+// SPIR_NATIVE_HALF: define spir_func noundef i1 @
+// DXIL_NATIVE_HALF: %hlsl.any = call i1 @llvm.dx.any.v3i16
+// SPIR_NATIVE_HALF: %hlsl.any = call i1 @llvm.spv.any.v3i16
+// NATIVE_HALF: ret i1 %hlsl.any
 bool test_any_uint16_t3(uint16_t3 p0) { return any(p0); }
-// NATIVE_HALF: define noundef i1 @
-// NATIVE_HALF: %dx.any = call i1 @llvm.dx.any.v4i16
-// NATIVE_HALF: ret i1 %dx.any
+
+// DXIL_NATIVE_HALF: define noundef i1 @
+// SPIR_NATIVE_HALF: define spir_func noundef i1 @
+// DXIL_NATIVE_HALF: %hlsl.any = call i1 @llvm.dx.any.v4i16
+// SPIR_NATIVE_HALF: %hlsl.any = call i1 @llvm.spv.any.v4i16
+// NATIVE_HALF: ret i1 %hlsl.any
 bool test_any_uint16_t4(uint16_t4 p0) { return any(p0); }
 #endif // __HLSL_ENABLE_16_BIT
 
-// CHECK: define noundef i1 @
-// NATIVE_HALF: %dx.any = call i1 @llvm.dx.any.f16
-// NO_HALF: %dx.any = call i1 @llvm.dx.any.f32
-// CHECK: ret i1 %dx.any
+// DXIL_CHECK: define noundef i1 @
+// SPIR_CHECK: define spir_func noundef i1 @
+// DXIL_NATIVE_HALF: %hlsl.any = call i1 @llvm.dx.any.f16
+// SPIR_NATIVE_HALF: %hlsl.any = call i1 @llvm.spv.any.f16
+// DXIL_NO_HALF: %hlsl.any = call i1 @llvm.dx.any.f32
+// SPIR_NO_HALF: %hlsl.any = call i1 @llvm.spv.any.f32
+// CHECK: ret i1 %hlsl.any
 bool test_any_half(half p0) { return any(p0); }
 
-// CHECK: define noundef i1 @
-// NATIVE_HALF: %dx.any = call i1 @llvm.dx.any.v2f16
-// NO_HALF: %dx.any = call i1 @llvm.dx.any.v2f32
-// CHECK: ret i1 %dx.any
+// DXIL_CHECK: define noundef i1 @
+// SPIR_CHECK: define spir_func noundef i1 @
+// DXIL_NATIVE_HALF: %hlsl.any = call i1 @llvm.dx.any.v2f16
+// SPIR_NATIVE_HALF: %hlsl.any = call i1 @llvm.spv.any.v2f16
+// DXIL_NO_HALF: %hlsl.any = call i1 @llvm.dx.any.v2f32
+// SPIR_NO_HALF: %hlsl.any = call i1 @llvm.spv.any.v2f32
+// CHECK: ret i1 %hlsl.any
 bool test_any_half2(half2 p0) { return any(p0); }
 
-// CHECK: define noundef i1 @
-// NATIVE_HALF: %dx.any = call i1 @llvm.dx.any.v3f16
-// NO_HALF: %dx.any = call i1 @llvm.dx.any.v3f32
-// CHECK: ret i1 %dx.any
+// DXIL_CHECK: define noundef i1 @
+// SPIR_CHECK: define spir_func noundef i1 @
+// DXIL_NATIVE_HALF: %hlsl.any = call i1 @llvm.dx.any.v3f16
+// SPIR_NATIVE_HALF: %hlsl.any = call i1 @llvm.spv.any.v3f16
+// DXIL_NO_HALF: %hlsl.any = call i1 @llvm.dx.any.v3f32
+// SPIR_NO_HALF: %hlsl.any = call i1 @llvm.spv.any.v3f32
+// CHECK: ret i1 %hlsl.any
 bool test_any_half3(half3 p0) { return any(p0); }
 
-// CHECK: define noundef i1 @
-// NATIVE_HALF: %dx.any = call i1 @llvm.dx.any.v4f16
-// NO_HALF: %dx.any = call i1 @llvm.dx.any.v4f32
-// CHECK: ret i1 %dx.any
+// DXIL_CHECK: define noundef i1 @
+// SPIR_CHECK: define spir_func noundef i1 @
+// DXIL_NATIVE_HALF: %hlsl.any = call i1 @llvm.dx.any.v4f16
+// SPIR_NATIVE_HALF: %hlsl.any = call i1 @llvm.spv.any.v4f16
+// DXIL_NO_HALF: %hlsl.any = call i1 @llvm.dx.any.v4f32
+// SPIR_NO_HALF: %hlsl.any = call i1 @llvm.spv.any.v4f32
+// CHECK: ret i1 %hlsl.any
 bool test_any_half4(half4 p0) { return any(p0); }
 
-// CHECK: define noundef i1 @
-// CHECK: %dx.any = call i1 @llvm.dx.any.f32
-// CHECK: ret i1 %dx.any
+// DXIL_CHECK: define noundef i1 @
+// SPIR_CHECK: define spir_func noundef i1 @
+// DXIL_CHECK: %hlsl.any = call i1 @llvm.dx.any.f32
+// SPIR_CHECK: %hlsl.any = call i1 @llvm.spv.any.f32
+// CHECK: ret i1 %hlsl.any
 bool test_any_float(float p0) { return any(p0); }
-// CHECK: define noundef i1 @
-// CHECK: %dx.any = call i1 @llvm.dx.any.v2f32
-// CHECK: ret i1 %dx.any
+
+// DXIL_CHECK: define noundef i1 @
+// SPIR_CHECK: define spir_func noundef i1 @
+// DXIL_CHECK: %hlsl.any = call i1 @llvm.dx.any.v2f32
+// SPIR_CHECK: %hlsl.any = call i1 @llvm.spv.any.v2f32
+// CHECK: ret i1 %hlsl.any
 bool test_any_float2(float2 p0) { return any(p0); }
-// CHECK: define noundef i1 @
-// CHECK: %dx.any = call i1 @llvm.dx.any.v3f32
-// CHECK: ret i1 %dx.any
+
+// DXIL_CHECK: define noundef i1 @
+// SPIR_CHECK: define spir_func noundef i1 @
+// DXIL_CHECK: %hlsl.any = call i1 @llvm.dx.any.v3f32
+// SPIR_CHECK: %hlsl.any = call i1 @llvm.spv.any.v3f32
+// CHECK: ret i1 %hlsl.any
 bool test_any_float3(float3 p0) { return any(p0); }
-// CHECK: define noundef i1 @
-// CHECK: %dx.any = call i1 @llvm.dx.any.v4f32
-// CHECK: ret i1 %dx.any
+
+// DXIL_CHECK: define noundef i1 @
+// SPIR_CHECK: define spir_func noundef i1 @
+// DXIL_CHECK: %hlsl.any = call i1 @llvm.dx.any.v4f32
+// SPIR_CHECK: %hlsl.any = call i1 @llvm.spv.any.v4f32
+// CHECK: ret i1 %hlsl.any
 bool test_any_float4(float4 p0) { return any(p0); }
 
-// CHECK: define noundef i1 @
-// CHECK: %dx.any = call i1 @llvm.dx.any.f64
-// CHECK: ret i1 %dx.any
+// DXIL_CHECK: define noundef i1 @
+// SPIR_CHECK: define spir_func noundef i1 @
+// DXIL_CHECK: %hlsl.any = call i1 @llvm.dx.any.f64
+// SPIR_CHECK: %hlsl.any = call i1 @llvm.spv.any.f64
+// CHECK: ret i1 %hlsl.any
 bool test_any_double(double p0) { return any(p0); }
-// CHECK: define noundef i1 @
-// CHECK: %dx.any = call i1 @llvm.dx.any.v2f64
-// CHECK: ret i1 %dx.any
+
+// DXIL_CHECK: define noundef i1 @
+// SPIR_CHECK: define spir_func noundef i1 @
+// DXIL_CHECK: %hlsl.any = call i1 @llvm.dx.any.v2f64
+// SPIR_CHECK: %hlsl.any = call i1 @llvm.spv.any.v2f64
+// CHECK: ret i1 %hlsl.any
 bool test_any_double2(double2 p0) { return any(p0); }
-// CHECK: define noundef i1 @
-// CHECK: %dx.any = call i1 @llvm.dx.any.v3f64
-// CHECK: ret i1 %dx.any
+
+// DXIL_CHECK: define noundef i1 @
+// SPIR_CHECK: define spir_func noundef i1 @
+// DXIL_CHECK: %hlsl.any = call i1 @llvm.dx.any.v3f64
+// SPIR_CHECK: %hlsl.any = call i1 @llvm.spv.any.v3f64
+// CHECK: ret i1 %hlsl.any
 bool test_any_double3(double3 p0) { return any(p0); }
-// CHECK: define noundef i1 @
-// CHECK: %dx.any = call i1 @llvm.dx.any.v4f64
-// CHECK: ret i1 %dx.any
+
+// DXIL_CHECK: define noundef i1 @
+// SPIR_CHECK: define spir_func noundef i1 @
+// DXIL_CHECK: %hlsl.any = call i1 @llvm.dx.any.v4f64
+// SPIR_CHECK: %hlsl.any = call i1 @llvm.spv.any.v4f64
+// CHECK: ret i1 %hlsl.any
 bool test_any_double4(double4 p0) { return any(p0); }
 
-// CHECK: define noundef i1 @
-// CHECK: %dx.any = call i1 @llvm.dx.any.i32
-// CHECK: ret i1 %dx.any
+// DXIL_CHECK: define noundef i1 @
+// SPIR_CHECK: define spir_func noundef i1 @
+// DXIL_CHECK: %hlsl.any = call i1 @llvm.dx.any.i32
+// SPIR_CHECK: %hlsl.any = call i1 @llvm.spv.any.i32
+// CHECK: ret i1 %hlsl.any
 bool test_any_int(int p0) { return any(p0); }
-// CHECK: define noundef i1 @
-// CHECK: %dx.any = call i1 @llvm.dx.any.v2i32
-// CHECK: ret i1 %dx.any
+
+// DXIL_CHECK: define noundef i1 @
+// SPIR_CHECK: define spir_func noundef i1 @
+// DXIL_CHECK: %hlsl.any = call i1 @llvm.dx.any.v2i32
+// SPIR_CHECK: %hlsl.any = call i1 @llvm.spv.any.v2i32
+// CHECK: ret i1 %hlsl.any
 bool test_any_int2(int2 p0) { return any(p0); }
-// CHECK: define noundef i1 @
-// CHECK: %dx.any = call i1 @llvm.dx.any.v3i32
-// CHECK: ret i1 %dx.any
+
+// DXIL_CHECK: define noundef i1 @
+// SPIR_CHECK: define spir_func noundef i1 @
+// DXIL_CHECK: %hlsl.any = call i1 @llvm.dx.any.v3i32
+// SPIR_CHECK: %hlsl.any = call i1 @llvm.spv.any.v3i32
+// CHECK: ret i1 %hlsl.any
 bool test_any_int3(int3 p0) { return any(p0); }
-// CHECK: define noundef i1 @
-// CHECK: %dx.any = call i1 @llvm.dx.any.v4i32
-// CHECK: ret i1 %dx.any
+
+// DXIL_CHECK: define noundef i1 @
+// SPIR_CHECK: define spir_func noundef i1 @
+// DXIL_CHECK: %hlsl.any = call i1 @llvm.dx.any.v4i32
+// SPIR_CHECK: %hlsl.any = call i1 @llvm.spv.any.v4i32
+// CHECK: ret i1 %hlsl.any
 bool test_any_int4(int4 p0) { return any(p0); }
 
-// CHECK: define noundef i1 @
-// CHECK: %dx.any = call i1 @llvm.dx.any.i32
-// CHECK: ret i1 %dx.any
+// DXIL_CHECK: define noundef i1 @
+// SPIR_CHECK: define spir_func noundef i1 @
+// DXIL_CHECK: %hlsl.any = call i1 @llvm.dx.any.i32
+// SPIR_CHECK: %hlsl.any = call i1 @llvm.spv.any.i32
+// CHECK: ret i1 %hlsl.any
 bool test_any_uint(uint p0) { return any(p0); }
-// CHECK: define noundef i1 @
-// CHECK: %dx.any = call i1 @llvm.dx.any.v2i32
-// CHECK: ret i1 %dx.any
+
+// DXIL_CHECK: define noundef i1 @
+// SPIR_CHECK: define spir_func noundef i1 @
+// DXIL_CHECK: %hlsl.any = call i1 @llvm.dx.any.v2i32
+// SPIR_CHECK: %hlsl.any = call i1 @llvm.spv.any.v2i32
+// CHECK: ret i1 %hlsl.any
 bool test_any_uint2(uint2 p0) { return any(p0); }
-// CHECK: define noundef i1 @
-// CHECK: %dx.any = call i1 @llvm.dx.any.v3i32
-// CHECK: ret i1 %dx.any
+
+// DXIL_CHECK: define noundef i1 @
+// SPIR_CHECK: define spir_func noundef i1 @
+// DXIL_CHECK: %hlsl.any = call i1 @llvm.dx.any.v3i32
+// SPIR_CHECK: %hlsl.any = call i1 @llvm.spv.any.v3i32
+// CHECK: ret i1 %hlsl.any
 bool test_any_uint3(uint3 p0) { return any(p0); }
-// CHECK: define noundef i1 @
-// CHECK: %dx.any = call i1 @llvm.dx.any.v4i32
-// CHECK: ret i1 %dx.any
+
+// DXIL_CHECK: define noundef i1 @
+// SPIR_CHECK: define spir_func noundef i1 @
+// DXIL_CHECK: %hlsl.any = call i1 @llvm.dx.any.v4i32
+// SPIR_CHECK: %hlsl.any = call i1 @llvm.spv.any.v4i32
+// CHECK: ret i1 %hlsl.any
 bool test_any_uint4(uint4 p0) { return any(p0); }
 
-// CHECK: define noundef i1 @
-// CHECK: %dx.any = call i1 @llvm.dx.any.i64
-// CHECK: ret i1 %dx.any
+// DXIL_CHECK: define noundef i1 @
+// SPIR_CHECK: define spir_func noundef i1 @
+// DXIL_CHECK: %hlsl.any = call i1 @llvm.dx.any.i64
+// SPIR_CHECK: %hlsl.any = call i1 @llvm.spv.any.i64
+// CHECK: ret i1 %hlsl.any
 bool test_any_int64_t(int64_t p0) { return any(p0); }
-// CHECK: define noundef i1 @
-// CHECK: %dx.any = call i1 @llvm.dx.any.v2i64
-// CHECK: ret i1 %dx.any
+
+// DXIL_CHECK: define noundef i1 @
+// SPIR_CHECK: define spir_func noundef i1 @
+// DXIL_CHECK: %hlsl.any = call i1 @llvm.dx.any.v2i64
+// SPIR_CHECK: %hlsl.any = call i1 @llvm.spv.any.v2i64
+// CHECK: ret i1 %hlsl.any
 bool test_any_int64_t2(int64_t2 p0) { return any(p0); }
-// CHECK: define noundef i1 @
-// CHECK: %dx.any = call i1 @llvm.dx.any.v3i64
-// CHECK: ret i1 %dx.any
+
+// DXIL_CHECK: define noundef i1 @
+// SPIR_CHECK: define spir_func noundef i1 @
+// DXIL_CHECK: %hlsl.any = call i1 @llvm.dx.any.v3i64
+// SPIR_CHECK: %hlsl.any = call i1 @llvm.spv.any.v3i64
+// CHECK: ret i1 %hlsl.any
 bool test_any_int64_t3(int64_t3 p0) { return any(p0); }
-// CHECK: define noundef i1 @
-// CHECK: %dx.any = call i1 @llvm.dx.any.v4i64
-// CHECK: ret i1 %dx.any
+
+// DXIL_CHECK: define noundef i1 @
+// SPIR_CHECK: define spir_func noundef i1 @
+// DXIL_CHECK: %hlsl.any = call i1 @llvm.dx.any.v4i64
+// SPIR_CHECK: %hlsl.any = call i1 @llvm.spv.any.v4i64
+// CHECK: ret i1 %hlsl.any
 bool test_any_int64_t4(int64_t4 p0) { return any(p0); }
 
-// CHECK: define noundef i1 @
-// CHECK: %dx.any = call i1 @llvm.dx.any.i64
-// CHECK: ret i1 %dx.any
+// DXIL_CHECK: define noundef i1 @
+// SPIR_CHECK: define spir_func noundef i1 @
+// DXIL_CHECK: %hlsl.any = call i1 @llvm.dx.any.i64
+// SPIR_CHECK: %hlsl.any = call i1 @llvm.spv.any.i64
+// CHECK: ret i1 %hlsl.any
 bool test_any_uint64_t(uint64_t p0) { return any(p0); }
-// CHECK: define noundef i1 @
-// CHECK: %dx.any = call i1 @llvm.dx.any.v2i64
-// CHECK: ret i1 %dx.any
+
+// DXIL_CHECK: define noundef i1 @
+// SPIR_CHECK: define spir_func noundef i1 @
+// DXIL_CHECK: %hlsl.any = call i1 @llvm.dx.any.v2i64
+// SPIR_CHECK: %hlsl.any = call i1 @llvm.spv.any.v2i64
+// CHECK: ret i1 %hlsl.any
 bool test_any_uint64_t2(uint64_t2 p0) { return any(p0); }
-// CHECK: define noundef i1 @
-// CHECK: %dx.any = call i1 @llvm.dx.any.v3i64
-// CHECK: ret i1 %dx.any
+
+// DXIL_CHECK: define noundef i1 @
+// SPIR_CHECK: define spir_func noundef i1 @
+// DXIL_CHECK: %hlsl.any = call i1 @llvm.dx.any.v3i64
+// SPIR_CHECK: %hlsl.any = call i1 @llvm.spv.any.v3i64
+// CHECK: ret i1 %hlsl.any
 bool test_any_uint64_t3(uint64_t3 p0) { return any(p0); }
-// CHECK: define noundef i1 @
-// CHECK: %dx.any = call i1 @llvm.dx.any.v4i64
-// CHECK: ret i1 %dx.any
+
+// DXIL_CHECK: define noundef i1 @
+// SPIR_CHECK: define spir_func noundef i1 @
+// DXIL_CHECK: %hlsl.any = call i1 @llvm.dx.any.v4i64
+// SPIR_CHECK: %hlsl.any = call i1 @llvm.spv.any.v4i64
+// CHECK: ret i1 %hlsl.any
 bool test_any_uint64_t4(uint64_t4 p0) { return any(p0); }
 
-// CHECK: define noundef i1 @
-// CHECK: %dx.any = call i1 @llvm.dx.any.i1
-// CHECK: ret i1 %dx.any
+// DXIL_CHECK: define noundef i1 @
+// SPIR_CHECK: define spir_func noundef i1 @
+// DXIL_CHECK: %hlsl.any = call i1 @llvm.dx.any.i1
+// SPIR_CHECK: %hlsl.any = call i1 @llvm.spv.any.i1
+// CHECK: ret i1 %hlsl.any
 bool test_any_bool(bool p0) { return any(p0); }
-// CHECK: define noundef i1 @
-// CHECK: %dx.any = call i1 @llvm.dx.any.v2i1
-// CHECK: ret i1 %dx.any
+
+// DXIL_CHECK: define noundef i1 @
+// SPIR_CHECK: define spir_func noundef i1 @
+// DXIL_CHECK: %hlsl.any = call i1 @llvm.dx.any.v2i1
+// SPIR_CHECK: %hlsl.any = call i1 @llvm.spv.any.v2i1
+// CHECK: ret i1 %hlsl.any
 bool test_any_bool2(bool2 p0) { return any(p0); }
-// CHECK: define noundef i1 @
-// CHECK: %dx.any = call i1 @llvm.dx.any.v3i1
-// CHECK: ret i1 %dx.any
+
+// DXIL_CHECK: define noundef i1 @
+// SPIR_CHECK: define spir_func noundef i1 @
+// DXIL_CHECK: %hlsl.any = call i1 @llvm.dx.any.v3i1
+// SPIR_CHECK: %hlsl.any = call i1 @llvm.spv.any.v3i1
+// CHECK: ret i1 %hlsl.any
 bool test_any_bool3(bool3 p0) { return any(p0); }
-// CHECK: define noundef i1 @
-// CHECK: %dx.any = call i1 @llvm.dx.any.v4i1
-// CHECK: ret i1 %dx.any
+
+// DXIL_CHECK: define noundef i1 @
+// SPIR_CHECK: define spir_func noundef i1 @
+// DXIL_CHECK: %hlsl.any = call i1 @llvm.dx.any.v4i1
+// SPIR_CHECK: %hlsl.any = call i1 @llvm.spv.any.v4i1
+// CHECK: ret i1 %hlsl.any
 bool test_any_bool4(bool4 p0) { return any(p0); }
diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
index f843383f0b00..b6618baceb56 100644
--- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td
+++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td
@@ -57,4 +57,5 @@ let TargetPrefix = "spv" in {
   def int_spv_create_handle : ClangBuiltin<"__builtin_hlsl_create_handle">,
       Intrinsic<[ llvm_ptr_ty ], [llvm_i8_ty], [IntrWillReturn]>;
   def int_spv_all : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_any_ty]>;
+  def int_spv_any : DefaultAttrsIntrinsic<[llvm_i1_ty], [llvm_any_ty]>;
 }
diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
index c1c0fc4b7dd4..200fe38298c0 100644
--- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp
@@ -145,9 +145,15 @@ private:
   bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType,
                            MachineInstr &I) const;
 
+  bool selectAnyOrAll(Register ResVReg, const SPIRVType *ResType,
+                      MachineInstr &I, unsigned OpType) const;
+
   bool selectAll(Register ResVReg, const SPIRVType *ResType,
                  MachineInstr &I) const;
 
+  bool selectAny(Register ResVReg, const SPIRVType *ResType,
+                 MachineInstr &I) const;
+
   bool selectBitreverse(Register ResVReg, const SPIRVType *ResType,
                         MachineInstr &I) const;
 
@@ -1160,9 +1166,10 @@ static unsigned getBoolCmpOpcode(unsigned PredNum) {
   }
 }
 
-bool SPIRVInstructionSelector::selectAll(Register ResVReg,
-                                         const SPIRVType *ResType,
-                                         MachineInstr &I) const {
+bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg,
+                                              const SPIRVType *ResType,
+                                              MachineInstr &I,
+                                              unsigned OpAnyOrAll) const {
   assert(I.getNumOperands() == 3);
   assert(I.getOperand(2).isReg());
   MachineBasicBlock &BB = *I.getParent();
@@ -1212,13 +1219,25 @@ bool SPIRVInstructionSelector::selectAll(Register ResVReg,
   if (!IsVectorTy)
     return true;
 
-  return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAll))
+  return BuildMI(BB, I, I.getDebugLoc(), TII.get(OpAnyOrAll))
       .addDef(ResVReg)
       .addUse(GR.getSPIRVTypeID(SpvBoolScalarTy))
       .addUse(NotEqualReg)
       .constrainAllUses(TII, TRI, RBI);
 }
 
+bool SPIRVInstructionSelector::selectAll(Register ResVReg,
+                                         const SPIRVType *ResType,
+                                         MachineInstr &I) const {
+  return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAll);
+}
+
+bool SPIRVInstructionSelector::selectAny(Register ResVReg,
+                                         const SPIRVType *ResType,
+                                         MachineInstr &I) const {
+  return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAny);
+}
+
 bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg,
                                                 const SPIRVType *ResType,
                                                 MachineInstr &I) const {
@@ -1877,6 +1896,8 @@ bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
     return selectSpvThreadId(ResVReg, ResType, I);
   case Intrinsic::spv_all:
     return selectAll(ResVReg, ResType, I);
+  case Intrinsic::spv_any:
+    return selectAny(ResVReg, ResType, I);
   case Intrinsic::spv_lifetime_start:
   case Intrinsic::spv_lifetime_end: {
     unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart
diff --git a/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/any.ll b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/any.ll
new file mode 100644
index 000000000000..b1dd388f5c6e
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/hlsl-intrinsics/any.ll
@@ -0,0 +1,187 @@
+; RUN: llc -O0 -mtriple=spirv-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-HLSL
+; RUN: llc -O0 -mtriple=spirv32-unknown-unknown %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-OCL
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv32-unknown-unknown %s -o - -filetype=obj | spirv-val %}
+; Make sure spirv operation function calls for any are generated.
+
+; CHECK-HLSL-DAG: OpMemoryModel Logical GLSL450
+; CHECK-OCL-DAG: OpMemoryModel Physical32 OpenCL
+; CHECK-DAG: OpName %[[#any_bool_arg:]] "a"
+; CHECK-DAG: %[[#int_64:]] = OpTypeInt 64 0
+; CHECK-DAG: %[[#bool:]] = OpTypeBool
+; CHECK-DAG: %[[#int_32:]] = OpTypeInt 32 0
+; CHECK-DAG: %[[#int_16:]] = OpTypeInt 16 0
+; CHECK-DAG: %[[#float_64:]] = OpTypeFloat 64
+; CHECK-DAG: %[[#float_32:]] = OpTypeFloat 32
+; CHECK-DAG: %[[#float_16:]] = OpTypeFloat 16
+; CHECK-DAG: %[[#vec4_bool:]] = OpTypeVector %[[#bool]] 4
+; CHECK-DAG: %[[#vec4_16:]] = OpTypeVector %[[#int_16]] 4
+; CHECK-DAG: %[[#vec4_32:]] = OpTypeVector %[[#int_32]] 4
+; CHECK-DAG: %[[#vec4_64:]] = OpTypeVector %[[#int_64]] 4
+; CHECK-DAG: %[[#vec4_float_16:]] = OpTypeVector %[[#float_16]] 4
+; CHECK-DAG: %[[#vec4_float_32:]] = OpTypeVector %[[#float_32]] 4
+; CHECK-DAG: %[[#vec4_float_64:]] = OpTypeVector %[[#float_64]] 4
+
+; CHECK-HLSL-DAG: %[[#const_i64_0:]] = OpConstant %[[#int_64]] 0
+; CHECK-HLSL-DAG: %[[#const_i32_0:]] = OpConstant %[[#int_32]] 0
+; CHECK-HLSL-DAG: %[[#const_i16_0:]] = OpConstant %[[#int_16]] 0
+; CHECK-HLSL-DAG: %[[#const_f64_0:]] = OpConstant %[[#float_64]] 0
+; CHECK-HLSL-DAG: %[[#const_f32_0:]] = OpConstant %[[#float_32:]] 0
+; CHECK-HLSL-DAG: %[[#const_f16_0:]] = OpConstant %[[#float_16:]] 0
+; CHECK-HLSL-DAG: %[[#vec4_const_zeros_i16:]] = OpConstantComposite %[[#vec4_16:]] %[[#const_i16_0:]] %[[#const_i16_0:]] %[[#const_i16_0:]] %[[#const_i16_0:]]
+; CHECK-HLSL-DAG: %[[#vec4_const_zeros_i32:]] = OpConstantComposite %[[#vec4_32:]] %[[#const_i32_0:]] %[[#const_i32_0:]] %[[#const_i32_0:]] %[[#const_i32_0:]]
+; CHECK-HLSL-DAG: %[[#vec4_const_zeros_i64:]] = OpConstantComposite %[[#vec4_64:]] %[[#const_i64_0:]] %[[#const_i64_0:]] %[[#const_i64_0:]] %[[#const_i64_0:]]
+; CHECK-HLSL-DAG: %[[#vec4_const_zeros_f16:]] = OpConstantComposite %[[#vec4_float_16:]] %[[#const_f16_0:]] %[[#const_f16_0:]] %[[#const_f16_0:]] %[[#const_f16_0:]]
+; CHECK-HLSL-DAG: %[[#vec4_const_zeros_f32:]] = OpConstantComposite %[[#vec4_float_32:]] %[[#const_f32_0:]] %[[#const_f32_0:]] %[[#const_f32_0:]] %[[#const_f32_0:]]
+; CHECK-HLSL-DAG: %[[#vec4_const_zeros_f64:]] = OpConstantComposite %[[#vec4_float_64:]] %[[#const_f64_0:]] %[[#const_f64_0:]] %[[#const_f64_0:]] %[[#const_f64_0:]]
+
+; CHECK-OCL-DAG: %[[#const_i64_0:]] = OpConstantNull %[[#int_64]]
+; CHECK-OCL-DAG: %[[#const_i32_0:]] = OpConstantNull %[[#int_32]]
+; CHECK-OCL-DAG: %[[#const_i16_0:]] = OpConstantNull %[[#int_16]]
+; CHECK-OCL-DAG: %[[#const_f64_0:]] = OpConstantNull %[[#float_64]] 
+; CHECK-OCL-DAG: %[[#const_f32_0:]] = OpConstantNull %[[#float_32:]]
+; CHECK-OCL-DAG: %[[#const_f16_0:]] = OpConstantNull %[[#float_16:]]
+; CHECK-OCL-DAG: %[[#vec4_const_zeros_i16:]] = OpConstantNull %[[#vec4_16:]]
+; CHECK-OCL-DAG: %[[#vec4_const_zeros_i32:]] = OpConstantNull %[[#vec4_32:]]
+; CHECK-OCL-DAG: %[[#vec4_const_zeros_i64:]] = OpConstantNull %[[#vec4_64:]]
+; CHECK-OCL-DAG: %[[#vec4_const_zeros_f16:]] = OpConstantNull %[[#vec4_float_16:]]
+; CHECK-OCL-DAG: %[[#vec4_const_zeros_f32:]] = OpConstantNull %[[#vec4_float_32:]]
+; CHECK-OCL-DAG: %[[#vec4_const_zeros_f64:]] = OpConstantNull %[[#vec4_float_64:]]
+
+define noundef i1 @any_int64_t(i64 noundef %p0) {
+entry:
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#]]
+  ; CHECK: %[[#]] = OpINotEqual %[[#bool:]] %[[#arg0:]] %[[#const_i64_0:]]
+  %hlsl.any = call i1 @llvm.spv.any.i64(i64 %p0)
+  ret i1 %hlsl.any
+}
+
+
+define noundef i1 @any_int(i32 noundef %p0) {
+entry:
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#]]
+  ; CHECK: %[[#]] = OpINotEqual %[[#bool:]] %[[#arg0:]] %[[#const_i32_0:]]
+  %hlsl.any = call i1 @llvm.spv.any.i32(i32 %p0)
+  ret i1 %hlsl.any
+}
+
+
+define noundef i1 @any_int16_t(i16 noundef %p0) {
+entry:
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#]]
+  ; CHECK: %[[#]] = OpINotEqual %[[#bool:]] %[[#arg0:]] %[[#const_i16_0:]]
+  %hlsl.any = call i1 @llvm.spv.any.i16(i16 %p0)
+  ret i1 %hlsl.any
+}
+
+define noundef i1 @any_double(double noundef %p0) {
+entry:
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#]]
+  ; CHECK: %[[#]] = OpFOrdNotEqual %[[#bool:]] %[[#arg0:]] %[[#const_f64_0:]]
+  %hlsl.any = call i1 @llvm.spv.any.f64(double %p0)
+  ret i1 %hlsl.any
+}
+
+
+define noundef i1 @any_float(float noundef %p0) {
+entry:
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#]]
+  ; CHECK: %[[#]] = OpFOrdNotEqual %[[#bool:]] %[[#arg0:]] %[[#const_f32_0:]]
+  %hlsl.any = call i1 @llvm.spv.any.f32(float %p0)
+  ret i1 %hlsl.any
+}
+
+
+define noundef i1 @any_half(half noundef %p0) {
+entry:
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#]]
+  ; CHECK: %[[#]] = OpFOrdNotEqual %[[#bool:]] %[[#arg0:]] %[[#const_f16_0:]]
+  %hlsl.any = call i1 @llvm.spv.any.f16(half %p0)
+  ret i1 %hlsl.any
+}
+
+
+define noundef i1 @any_bool4(<4 x i1> noundef %p0) {
+entry:
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#]]
+  ; CHECK: %[[#]] = OpAny %[[#vec4_bool:]] %[[#arg0:]]
+  %hlsl.any = call i1 @llvm.spv.any.v4i1(<4 x i1> %p0)
+  ret i1 %hlsl.any
+}
+
+define noundef i1 @any_short4(<4 x i16> noundef %p0) {
+entry:
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#]]
+  ; CHECK: %[[#shortVecNotEq:]] = OpINotEqual %[[#vec4_bool:]] %[[#arg0:]] %[[#vec4_const_zeros_i16:]]
+  ; CHECK: %[[#]] = OpAny %[[#bool:]] %[[#shortVecNotEq:]]
+  %hlsl.any = call i1 @llvm.spv.any.v4i16(<4 x i16> %p0)
+  ret i1 %hlsl.any
+}
+
+define noundef i1 @any_int4(<4 x i32> noundef %p0) {
+entry:
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#]]
+  ; CHECK: %[[#i32VecNotEq:]] = OpINotEqual %[[#vec4_bool:]] %[[#arg0:]] %[[#vec4_const_zeros_i32:]]
+  ; CHECK: %[[#]] = OpAny %[[#bool:]] %[[#i32VecNotEq:]]
+  %hlsl.any = call i1 @llvm.spv.any.v4i32(<4 x i32> %p0)
+  ret i1 %hlsl.any
+}
+
+define noundef i1 @any_int64_t4(<4 x i64> noundef %p0) {
+entry:
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#]]
+  ; CHECK: %[[#i64VecNotEq:]] = OpINotEqual %[[#vec4_bool:]] %[[#arg0:]] %[[#vec4_const_zeros_i64:]]
+  ; CHECK: %[[#]] = OpAny %[[#bool:]] %[[#i64VecNotEq]]
+  %hlsl.any = call i1 @llvm.spv.any.v4i64(<4 x i64> %p0)
+  ret i1 %hlsl.any
+}
+
+define noundef i1 @any_half4(<4 x half> noundef %p0) {
+entry:
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#]]
+  ; CHECK: %[[#f16VecNotEq:]] = OpFOrdNotEqual %[[#vec4_bool:]] %[[#arg0:]] %[[#vec4_const_zeros_f16:]]
+  ; CHECK: %[[#]] = OpAny %[[#bool]] %[[#f16VecNotEq:]]
+  %hlsl.any = call i1 @llvm.spv.any.v4f16(<4 x half> %p0)
+  ret i1 %hlsl.any
+}
+
+define noundef i1 @any_float4(<4 x float> noundef %p0) {
+entry:
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#]]
+  ; CHECK: %[[#f32VecNotEq:]] = OpFOrdNotEqual %[[#vec4_bool:]] %[[#arg0:]] %[[#vec4_const_zeros_f32:]]
+  ; CHECK: %[[#]] = OpAny %[[#bool:]] %[[#f32VecNotEq:]]
+  %hlsl.any = call i1 @llvm.spv.any.v4f32(<4 x float> %p0)
+  ret i1 %hlsl.any
+}
+
+define noundef i1 @any_double4(<4 x double> noundef %p0) {
+entry:
+  ; CHECK: %[[#arg0:]] = OpFunctionParameter %[[#]]
+  ; CHECK: %[[#f64VecNotEq:]] = OpFOrdNotEqual %[[#vec4_bool:]] %[[#arg0:]] %[[#vec4_const_zeros_f64:]]
+  ; CHECK: %[[#]] = OpAny %[[#bool:]] %[[#f64VecNotEq:]]
+  %hlsl.any = call i1 @llvm.spv.any.v4f64(<4 x double> %p0)
+  ret i1 %hlsl.any
+}
+
+define noundef i1 @any_bool(i1 noundef %a) {
+entry:
+  ; CHECK: %[[#any_bool_arg:]] = OpFunctionParameter %[[#bool:]]
+  ; CHECK: OpReturnValue %[[#any_bool_arg:]]
+  %hlsl.any = call i1 @llvm.spv.any.i1(i1 %a)
+  ret i1 %hlsl.any
+}
+
+declare i1 @llvm.spv.any.v4f16(<4 x half>)
+declare i1 @llvm.spv.any.v4f32(<4 x float>)
+declare i1 @llvm.spv.any.v4f64(<4 x double>)
+declare i1 @llvm.spv.any.v4i1(<4 x i1>)
+declare i1 @llvm.spv.any.v4i16(<4 x i16>)
+declare i1 @llvm.spv.any.v4i32(<4 x i32>)
+declare i1 @llvm.spv.any.v4i64(<4 x i64>)
+declare i1 @llvm.spv.any.i1(i1)
+declare i1 @llvm.spv.any.i16(i16)
+declare i1 @llvm.spv.any.i32(i32)
+declare i1 @llvm.spv.any.i64(i64)
+declare i1 @llvm.spv.any.f16(half)
+declare i1 @llvm.spv.any.f32(float)
+declare i1 @llvm.spv.any.f64(double)
-- 
GitLab


From 32cb3c55080eec70abf282fbe75e6a58cf76f92a Mon Sep 17 00:00:00 2001
From: Piyou Chen <piyou.chen@sifive.com>
Date: Mon, 15 Apr 2024 21:58:57 +0800
Subject: [PATCH 244/464] [NFC][LLVM][CodeGen] Move LiveDebugVariables.h into
 llvm/include/llvm/CodeGen (#88374)

This patch make `LiveDebugVariables` can be used by passes outside of
`lib/CodeGen`.

If we run a pass that occurs between the split register allocation pass
without preserving this pass, it will be freed and recomputed until it
encounters the next pass that needs LiveDebugVariables.

However, `LiveDebugVariables` will raise an assertion due to the pass
being freed without emitting a debug value.

This is reason we need `LiveDebugVariables` to be available for passes
outside of lib/Codegen.
---
 llvm/{lib => include/llvm}/CodeGen/LiveDebugVariables.h | 8 ++++----
 llvm/lib/CodeGen/LiveDebugVariables.cpp                 | 2 +-
 llvm/lib/CodeGen/RegAllocBasic.cpp                      | 2 +-
 llvm/lib/CodeGen/RegAllocGreedy.cpp                     | 2 +-
 llvm/lib/CodeGen/VirtRegMap.cpp                         | 2 +-
 5 files changed, 8 insertions(+), 8 deletions(-)
 rename llvm/{lib => include/llvm}/CodeGen/LiveDebugVariables.h (90%)

diff --git a/llvm/lib/CodeGen/LiveDebugVariables.h b/llvm/include/llvm/CodeGen/LiveDebugVariables.h
similarity index 90%
rename from llvm/lib/CodeGen/LiveDebugVariables.h
rename to llvm/include/llvm/CodeGen/LiveDebugVariables.h
index 9998ce9e8dad..a4b5a87fd388 100644
--- a/llvm/lib/CodeGen/LiveDebugVariables.h
+++ b/llvm/include/llvm/CodeGen/LiveDebugVariables.h
@@ -17,8 +17,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_LIB_CODEGEN_LIVEDEBUGVARIABLES_H
-#define LLVM_LIB_CODEGEN_LIVEDEBUGVARIABLES_H
+#ifndef LLVM_CODEGEN_LIVEDEBUGVARIABLES_H
+#define LLVM_CODEGEN_LIVEDEBUGVARIABLES_H
 
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/Support/Compiler.h"
@@ -29,7 +29,7 @@ template <typename T> class ArrayRef;
 class LiveIntervals;
 class VirtRegMap;
 
-class LLVM_LIBRARY_VISIBILITY LiveDebugVariables : public MachineFunctionPass {
+class LiveDebugVariables : public MachineFunctionPass {
   void *pImpl = nullptr;
 
 public:
@@ -65,4 +65,4 @@ private:
 
 } // end namespace llvm
 
-#endif // LLVM_LIB_CODEGEN_LIVEDEBUGVARIABLES_H
+#endif // LLVM_CODEGEN_LIVEDEBUGVARIABLES_H
diff --git a/llvm/lib/CodeGen/LiveDebugVariables.cpp b/llvm/lib/CodeGen/LiveDebugVariables.cpp
index 7cb90af5ff17..3a59ae7ab066 100644
--- a/llvm/lib/CodeGen/LiveDebugVariables.cpp
+++ b/llvm/lib/CodeGen/LiveDebugVariables.cpp
@@ -18,7 +18,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "LiveDebugVariables.h"
+#include "llvm/CodeGen/LiveDebugVariables.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/IntervalMap.h"
diff --git a/llvm/lib/CodeGen/RegAllocBasic.cpp b/llvm/lib/CodeGen/RegAllocBasic.cpp
index 666199139630..5bd3b126aa16 100644
--- a/llvm/lib/CodeGen/RegAllocBasic.cpp
+++ b/llvm/lib/CodeGen/RegAllocBasic.cpp
@@ -12,10 +12,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "AllocationOrder.h"
-#include "LiveDebugVariables.h"
 #include "RegAllocBase.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/CodeGen/CalcSpillWeights.h"
+#include "llvm/CodeGen/LiveDebugVariables.h"
 #include "llvm/CodeGen/LiveIntervals.h"
 #include "llvm/CodeGen/LiveRangeEdit.h"
 #include "llvm/CodeGen/LiveRegMatrix.h"
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp
index a208bf89fadf..348277224c7a 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.cpp
+++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp
@@ -14,7 +14,6 @@
 #include "RegAllocGreedy.h"
 #include "AllocationOrder.h"
 #include "InterferenceCache.h"
-#include "LiveDebugVariables.h"
 #include "RegAllocBase.h"
 #include "RegAllocEvictionAdvisor.h"
 #include "RegAllocPriorityAdvisor.h"
@@ -31,6 +30,7 @@
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/CodeGen/CalcSpillWeights.h"
 #include "llvm/CodeGen/EdgeBundles.h"
+#include "llvm/CodeGen/LiveDebugVariables.h"
 #include "llvm/CodeGen/LiveInterval.h"
 #include "llvm/CodeGen/LiveIntervalUnion.h"
 #include "llvm/CodeGen/LiveIntervals.h"
diff --git a/llvm/lib/CodeGen/VirtRegMap.cpp b/llvm/lib/CodeGen/VirtRegMap.cpp
index 48f4ee29fbe9..2c778980f5d1 100644
--- a/llvm/lib/CodeGen/VirtRegMap.cpp
+++ b/llvm/lib/CodeGen/VirtRegMap.cpp
@@ -16,9 +16,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/CodeGen/VirtRegMap.h"
-#include "LiveDebugVariables.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/CodeGen/LiveDebugVariables.h"
 #include "llvm/CodeGen/LiveInterval.h"
 #include "llvm/CodeGen/LiveIntervals.h"
 #include "llvm/CodeGen/LiveStacks.h"
-- 
GitLab


From c3d58867bd3da85958dbbb46c4a22c0d3c60af66 Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen@arm.com>
Date: Mon, 15 Apr 2024 15:07:20 +0100
Subject: [PATCH 245/464] [AArch64][SME] Create new pass to remove
 COALESCER_BARRIER early. (#85386)

The purpose of the COALESCER_BARRIER pseudo node is to prevent the
register coalescer from coalescing certain COPY instructions around
smstart/smstop instructions, so that we spill only the (required) FPR
register rather than the encompassing ZPR register.

The pseudos are removed in the AArch64ExpandPseudo pass. However,
because the node itself is a _use_ of a register, this occassionally
leads to redundant spills/fills, because the register allocator thinks
the virtual register is actually used before an smstart/smstop
instruction, causing it to be filled, at which points it requires
immediate spilling again to ensure it stays live over the smstart/smstop
instruction.

We can avoid that by removing the pseudo nodes right after coalescing,
but before register allocation.
---
 llvm/lib/Target/AArch64/AArch64.h             |   2 +
 .../AArch64/AArch64PostCoalescerPass.cpp      | 101 ++++++++++++++++++
 .../Target/AArch64/AArch64TargetMachine.cpp   |   7 ++
 llvm/lib/Target/AArch64/CMakeLists.txt        |   1 +
 llvm/test/CodeGen/AArch64/O3-pipeline.ll      |   1 +
 .../sme-avoid-coalescing-locally-streaming.ll |  10 +-
 ...compatible-to-normal-fn-wihout-sme-attr.ll |   2 -
 .../AArch64/sme-disable-gisel-fisel.ll        |  10 +-
 ...ate-sm-changing-call-disable-coalescing.ll |   8 --
 ...ing-body-streaming-compatible-interface.ll |  26 ++---
 .../CodeGen/AArch64/sme-streaming-body.ll     |   4 -
 .../sme-streaming-compatible-interface.ll     |   4 -
 .../AArch64/sme-streaming-interface.ll        |   8 +-
 ...nging-call-disable-stackslot-scavenging.ll |   2 +-
 14 files changed, 134 insertions(+), 52 deletions(-)
 create mode 100644 llvm/lib/Target/AArch64/AArch64PostCoalescerPass.cpp

diff --git a/llvm/lib/Target/AArch64/AArch64.h b/llvm/lib/Target/AArch64/AArch64.h
index f7d81f42ef5d..b70fbe42fe5f 100644
--- a/llvm/lib/Target/AArch64/AArch64.h
+++ b/llvm/lib/Target/AArch64/AArch64.h
@@ -54,6 +54,7 @@ FunctionPass *createFalkorMarkStridedAccessesPass();
 FunctionPass *createAArch64PointerAuthPass();
 FunctionPass *createAArch64BranchTargetsPass();
 FunctionPass *createAArch64MIPeepholeOptPass();
+FunctionPass *createAArch64PostCoalescerPass();
 
 FunctionPass *createAArch64CleanupLocalDynamicTLSPass();
 
@@ -93,6 +94,7 @@ void initializeAArch64LoopIdiomTransformLegacyPassPass(PassRegistry &);
 void initializeAArch64LowerHomogeneousPrologEpilogPass(PassRegistry &);
 void initializeAArch64MIPeepholeOptPass(PassRegistry &);
 void initializeAArch64O0PreLegalizerCombinerPass(PassRegistry &);
+void initializeAArch64PostCoalescerPass(PassRegistry &);
 void initializeAArch64PostLegalizerCombinerPass(PassRegistry &);
 void initializeAArch64PostLegalizerLoweringPass(PassRegistry &);
 void initializeAArch64PostSelectOptimizePass(PassRegistry &);
diff --git a/llvm/lib/Target/AArch64/AArch64PostCoalescerPass.cpp b/llvm/lib/Target/AArch64/AArch64PostCoalescerPass.cpp
new file mode 100644
index 000000000000..dd5234c4504d
--- /dev/null
+++ b/llvm/lib/Target/AArch64/AArch64PostCoalescerPass.cpp
@@ -0,0 +1,101 @@
+//===- AArch64PostCoalescerPass.cpp - AArch64 Post Coalescer pass ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//===----------------------------------------------------------------------===//
+
+#include "AArch64InstrInfo.h"
+#include "AArch64MachineFunctionInfo.h"
+#include "llvm/CodeGen/LiveIntervals.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/InitializePasses.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "aarch64-post-coalescer-pass"
+
+namespace {
+
+struct AArch64PostCoalescer : public MachineFunctionPass {
+  static char ID;
+
+  AArch64PostCoalescer() : MachineFunctionPass(ID) {
+    initializeAArch64PostCoalescerPass(*PassRegistry::getPassRegistry());
+  }
+
+  LiveIntervals *LIS;
+  MachineRegisterInfo *MRI;
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  StringRef getPassName() const override {
+    return "AArch64 Post Coalescer pass";
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesAll();
+    AU.addRequired<LiveIntervals>();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+};
+
+char AArch64PostCoalescer::ID = 0;
+
+} // end anonymous namespace
+
+INITIALIZE_PASS_BEGIN(AArch64PostCoalescer, "aarch64-post-coalescer-pass",
+                      "AArch64 Post Coalescer Pass", false, false)
+INITIALIZE_PASS_DEPENDENCY(LiveIntervals)
+INITIALIZE_PASS_END(AArch64PostCoalescer, "aarch64-post-coalescer-pass",
+                    "AArch64 Post Coalescer Pass", false, false)
+
+bool AArch64PostCoalescer::runOnMachineFunction(MachineFunction &MF) {
+  if (skipFunction(MF.getFunction()))
+    return false;
+
+  AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
+  if (!FuncInfo->hasStreamingModeChanges())
+    return false;
+
+  MRI = &MF.getRegInfo();
+  LIS = &getAnalysis<LiveIntervals>();
+  bool Changed = false;
+
+  for (MachineBasicBlock &MBB : MF) {
+    for (MachineInstr &MI : make_early_inc_range(MBB)) {
+      switch (MI.getOpcode()) {
+      default:
+        break;
+      case AArch64::COALESCER_BARRIER_FPR16:
+      case AArch64::COALESCER_BARRIER_FPR32:
+      case AArch64::COALESCER_BARRIER_FPR64:
+      case AArch64::COALESCER_BARRIER_FPR128: {
+        Register Src = MI.getOperand(1).getReg();
+        Register Dst = MI.getOperand(0).getReg();
+        if (Src != Dst)
+          MRI->replaceRegWith(Dst, Src);
+
+        // MI must be erased from the basic block before recalculating the live
+        // interval.
+        LIS->RemoveMachineInstrFromMaps(MI);
+        MI.eraseFromParent();
+
+        LIS->removeInterval(Src);
+        LIS->createAndComputeVirtRegInterval(Src);
+
+        Changed = true;
+        break;
+      }
+      }
+    }
+  }
+
+  return Changed;
+}
+
+FunctionPass *llvm::createAArch64PostCoalescerPass() {
+  return new AArch64PostCoalescer();
+}
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
index 7d4a57d792a1..7ef78cbba352 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
@@ -235,6 +235,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64Target() {
   initializeAArch64O0PreLegalizerCombinerPass(*PR);
   initializeAArch64PreLegalizerCombinerPass(*PR);
   initializeAArch64PointerAuthPass(*PR);
+  initializeAArch64PostCoalescerPass(*PR);
   initializeAArch64PostLegalizerCombinerPass(*PR);
   initializeAArch64PostLegalizerLoweringPass(*PR);
   initializeAArch64PostSelectOptimizePass(*PR);
@@ -539,6 +540,7 @@ public:
   void addPreEmitPass() override;
   void addPostBBSections() override;
   void addPreEmitPass2() override;
+  bool addRegAssignAndRewriteOptimized() override;
 
   std::unique_ptr<CSEConfigBase> getCSEConfig() const override;
 };
@@ -876,6 +878,11 @@ void AArch64PassConfig::addPreEmitPass2() {
   addPass(createUnpackMachineBundles(nullptr));
 }
 
+bool AArch64PassConfig::addRegAssignAndRewriteOptimized() {
+  addPass(createAArch64PostCoalescerPass());
+  return TargetPassConfig::addRegAssignAndRewriteOptimized();
+}
+
 MachineFunctionInfo *AArch64TargetMachine::createMachineFunctionInfo(
     BumpPtrAllocator &Allocator, const Function &F,
     const TargetSubtargetInfo *STI) const {
diff --git a/llvm/lib/Target/AArch64/CMakeLists.txt b/llvm/lib/Target/AArch64/CMakeLists.txt
index 95b228f29320..8e76f6c9279e 100644
--- a/llvm/lib/Target/AArch64/CMakeLists.txt
+++ b/llvm/lib/Target/AArch64/CMakeLists.txt
@@ -73,6 +73,7 @@ add_llvm_target(AArch64CodeGen
   AArch64MIPeepholeOpt.cpp
   AArch64MCInstLower.cpp
   AArch64PointerAuth.cpp
+  AArch64PostCoalescerPass.cpp
   AArch64PromoteConstant.cpp
   AArch64PBQPRegAlloc.cpp
   AArch64RegisterInfo.cpp
diff --git a/llvm/test/CodeGen/AArch64/O3-pipeline.ll b/llvm/test/CodeGen/AArch64/O3-pipeline.ll
index eee9a27c90c1..d3c8e3b7e805 100644
--- a/llvm/test/CodeGen/AArch64/O3-pipeline.ll
+++ b/llvm/test/CodeGen/AArch64/O3-pipeline.ll
@@ -167,6 +167,7 @@
 ; CHECK-NEXT:       Register Coalescer
 ; CHECK-NEXT:       Rename Disconnected Subregister Components
 ; CHECK-NEXT:       Machine Instruction Scheduler
+; CHECK-NEXT:       AArch64 Post Coalescer pass
 ; CHECK-NEXT:       Machine Block Frequency Analysis
 ; CHECK-NEXT:       Debug Variable Analysis
 ; CHECK-NEXT:       Live Stack Slot Analysis
diff --git a/llvm/test/CodeGen/AArch64/sme-avoid-coalescing-locally-streaming.ll b/llvm/test/CodeGen/AArch64/sme-avoid-coalescing-locally-streaming.ll
index cd5046a9a647..8e3866fcec89 100644
--- a/llvm/test/CodeGen/AArch64/sme-avoid-coalescing-locally-streaming.ll
+++ b/llvm/test/CodeGen/AArch64/sme-avoid-coalescing-locally-streaming.ll
@@ -25,8 +25,7 @@ define void @dont_coalesce_args(<2 x i64> %a) "aarch64_pstate_sm_body" nounwind
   ; CHECK-REGALLOC: bb.0 (%ir-block.0):
   ; CHECK-REGALLOC-NEXT:   liveins: $q0
   ; CHECK-REGALLOC-NEXT: {{  $}}
-  ; CHECK-REGALLOC-NEXT:   renamable $q0 = COALESCER_BARRIER_FPR128 killed renamable $q0
-  ; CHECK-REGALLOC-NEXT:   STRQui killed renamable $q0, %stack.0, 0 :: (store (s128) into %stack.0)
+  ; CHECK-REGALLOC-NEXT:   STRQui $q0, %stack.0, 0 :: (store (s128) into %stack.0)
   ; CHECK-REGALLOC-NEXT:   MSRpstatesvcrImm1 1, 1, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit $vg, implicit-def $vg
   ; CHECK-REGALLOC-NEXT:   renamable $q0 = LDRQui %stack.0, 0 :: (load (s128) from %stack.0)
   ; CHECK-REGALLOC-NEXT:   renamable $q0 = KILL killed renamable $q0, implicit-def $z0
@@ -61,7 +60,6 @@ define <2 x i64> @dont_coalesce_res() "aarch64_pstate_sm_body" nounwind {
   ; CHECK-REGALLOC-NEXT:   BL @scalable_res, csr_aarch64_sve_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def $z0
   ; CHECK-REGALLOC-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
   ; CHECK-REGALLOC-NEXT:   renamable $q0 = KILL renamable $q0, implicit killed $z0
-  ; CHECK-REGALLOC-NEXT:   renamable $q0 = COALESCER_BARRIER_FPR128 killed renamable $q0
   ; CHECK-REGALLOC-NEXT:   STRQui killed renamable $q0, %stack.0, 0 :: (store (s128) into %stack.0)
   ; CHECK-REGALLOC-NEXT:   MSRpstatesvcrImm1 1, 0, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit-def dead $q0, implicit $vg, implicit-def $vg
   ; CHECK-REGALLOC-NEXT:   $q0 = LDRQui %stack.0, 0 :: (load (s128) from %stack.0)
@@ -94,17 +92,13 @@ define <2 x i64> @dont_coalesce_arg_that_is_also_res(<2 x i64> %a) "aarch64_psta
   ; CHECK-REGALLOC: bb.0 (%ir-block.0):
   ; CHECK-REGALLOC-NEXT:   liveins: $q0
   ; CHECK-REGALLOC-NEXT: {{  $}}
-  ; CHECK-REGALLOC-NEXT:   renamable $q0 = COALESCER_BARRIER_FPR128 killed renamable $q0
-  ; CHECK-REGALLOC-NEXT:   STRQui killed renamable $q0, %stack.0, 0 :: (store (s128) into %stack.0)
+  ; CHECK-REGALLOC-NEXT:   STRQui $q0, %stack.0, 0 :: (store (s128) into %stack.0)
   ; CHECK-REGALLOC-NEXT:   MSRpstatesvcrImm1 1, 1, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit $vg, implicit-def $vg
   ; CHECK-REGALLOC-NEXT:   renamable $q0 = LDRQui %stack.0, 0 :: (load (s128) from %stack.0)
   ; CHECK-REGALLOC-NEXT:   renamable $q0 = KILL killed renamable $q0, implicit-def $z0
   ; CHECK-REGALLOC-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
   ; CHECK-REGALLOC-NEXT:   BL @scalable_args, csr_aarch64_sve_aapcs, implicit-def dead $lr, implicit $sp, implicit $z0, implicit-def $sp
   ; CHECK-REGALLOC-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
-  ; CHECK-REGALLOC-NEXT:   renamable $q0 = LDRQui %stack.0, 0 :: (load (s128) from %stack.0)
-  ; CHECK-REGALLOC-NEXT:   renamable $q0 = COALESCER_BARRIER_FPR128 killed renamable $q0
-  ; CHECK-REGALLOC-NEXT:   STRQui killed renamable $q0, %stack.0, 0 :: (store (s128) into %stack.0)
   ; CHECK-REGALLOC-NEXT:   MSRpstatesvcrImm1 1, 0, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit-def dead $q0, implicit $vg, implicit-def $vg
   ; CHECK-REGALLOC-NEXT:   $q0 = LDRQui %stack.0, 0 :: (load (s128) from %stack.0)
   ; CHECK-REGALLOC-NEXT:   RET_ReallyLR implicit $q0
diff --git a/llvm/test/CodeGen/AArch64/sme-call-streaming-compatible-to-normal-fn-wihout-sme-attr.ll b/llvm/test/CodeGen/AArch64/sme-call-streaming-compatible-to-normal-fn-wihout-sme-attr.ll
index dba3227459b9..07377195d62a 100644
--- a/llvm/test/CodeGen/AArch64/sme-call-streaming-compatible-to-normal-fn-wihout-sme-attr.ll
+++ b/llvm/test/CodeGen/AArch64/sme-call-streaming-compatible-to-normal-fn-wihout-sme-attr.ll
@@ -52,9 +52,7 @@ define void @streaming_compatible_arg(float %f) #0 {
 ; CHECK-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    str s0, [sp, #12] // 4-byte Folded Spill
 ; CHECK-NEXT:    bl __arm_sme_state
-; CHECK-NEXT:    ldr s0, [sp, #12] // 4-byte Folded Reload
 ; CHECK-NEXT:    and x19, x0, #0x1
-; CHECK-NEXT:    str s0, [sp, #12] // 4-byte Folded Spill
 ; CHECK-NEXT:    tbz w19, #0, .LBB1_2
 ; CHECK-NEXT:  // %bb.1:
 ; CHECK-NEXT:    smstop sm
diff --git a/llvm/test/CodeGen/AArch64/sme-disable-gisel-fisel.ll b/llvm/test/CodeGen/AArch64/sme-disable-gisel-fisel.ll
index cd348be5d771..254e37e836cb 100644
--- a/llvm/test/CodeGen/AArch64/sme-disable-gisel-fisel.ll
+++ b/llvm/test/CodeGen/AArch64/sme-disable-gisel-fisel.ll
@@ -331,9 +331,9 @@ define fp128 @f128_call_sm(fp128 %a, fp128 %b) "aarch64_pstate_sm_enabled" nounw
 ; CHECK-COMMON-NEXT:    stp d11, d10, [sp, #64] // 16-byte Folded Spill
 ; CHECK-COMMON-NEXT:    stp d9, d8, [sp, #80] // 16-byte Folded Spill
 ; CHECK-COMMON-NEXT:    str x30, [sp, #96] // 8-byte Folded Spill
-; CHECK-COMMON-NEXT:    stp q1, q0, [sp] // 32-byte Folded Spill
+; CHECK-COMMON-NEXT:    stp q0, q1, [sp] // 32-byte Folded Spill
 ; CHECK-COMMON-NEXT:    smstop sm
-; CHECK-COMMON-NEXT:    ldp q1, q0, [sp] // 32-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldp q0, q1, [sp] // 32-byte Folded Reload
 ; CHECK-COMMON-NEXT:    bl __addtf3
 ; CHECK-COMMON-NEXT:    str q0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-COMMON-NEXT:    smstart sm
@@ -392,9 +392,9 @@ define float @frem_call_sm(float %a, float %b) "aarch64_pstate_sm_enabled" nounw
 ; CHECK-COMMON-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
 ; CHECK-COMMON-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
 ; CHECK-COMMON-NEXT:    str x30, [sp, #80] // 8-byte Folded Spill
-; CHECK-COMMON-NEXT:    stp s1, s0, [sp, #8] // 8-byte Folded Spill
+; CHECK-COMMON-NEXT:    stp s0, s1, [sp, #8] // 8-byte Folded Spill
 ; CHECK-COMMON-NEXT:    smstop sm
-; CHECK-COMMON-NEXT:    ldp s1, s0, [sp, #8] // 8-byte Folded Reload
+; CHECK-COMMON-NEXT:    ldp s0, s1, [sp, #8] // 8-byte Folded Reload
 ; CHECK-COMMON-NEXT:    bl fmodf
 ; CHECK-COMMON-NEXT:    str s0, [sp, #12] // 4-byte Folded Spill
 ; CHECK-COMMON-NEXT:    smstart sm
@@ -422,9 +422,7 @@ define float @frem_call_sm_compat(float %a, float %b) "aarch64_pstate_sm_compati
 ; CHECK-COMMON-NEXT:    stp x30, x19, [sp, #80] // 16-byte Folded Spill
 ; CHECK-COMMON-NEXT:    stp s0, s1, [sp, #8] // 8-byte Folded Spill
 ; CHECK-COMMON-NEXT:    bl __arm_sme_state
-; CHECK-COMMON-NEXT:    ldp s2, s0, [sp, #8] // 8-byte Folded Reload
 ; CHECK-COMMON-NEXT:    and x19, x0, #0x1
-; CHECK-COMMON-NEXT:    stp s2, s0, [sp, #8] // 8-byte Folded Spill
 ; CHECK-COMMON-NEXT:    tbz w19, #0, .LBB12_2
 ; CHECK-COMMON-NEXT:  // %bb.1:
 ; CHECK-COMMON-NEXT:    smstop sm
diff --git a/llvm/test/CodeGen/AArch64/sme-pstate-sm-changing-call-disable-coalescing.ll b/llvm/test/CodeGen/AArch64/sme-pstate-sm-changing-call-disable-coalescing.ll
index d5bea725b6d1..0c674c5685e0 100644
--- a/llvm/test/CodeGen/AArch64/sme-pstate-sm-changing-call-disable-coalescing.ll
+++ b/llvm/test/CodeGen/AArch64/sme-pstate-sm-changing-call-disable-coalescing.ll
@@ -1085,7 +1085,6 @@ define void @dont_coalesce_res_f32(ptr %ptr) #0 {
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    ldr s0, [sp, #12] // 4-byte Folded Reload
-; CHECK-NEXT:    // kill: def $s0 killed $s0 def $z0
 ; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
@@ -1116,7 +1115,6 @@ define void @dont_coalesce_res_f64(ptr %ptr) #0 {
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    ldr d0, [sp, #8] // 8-byte Folded Reload
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
@@ -1151,7 +1149,6 @@ define void @dont_coalesce_res_v1i8(ptr %ptr) #0 {
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    ptrue p0.b
 ; CHECK-NEXT:    ldr d0, [sp, #8] // 8-byte Folded Reload
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
@@ -1183,7 +1180,6 @@ define void @dont_coalesce_res_v1i16(ptr %ptr) #0 {
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    ptrue p0.h
 ; CHECK-NEXT:    ldr d0, [sp, #8] // 8-byte Folded Reload
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
@@ -1215,7 +1211,6 @@ define void @dont_coalesce_res_v1i32(ptr %ptr) #0 {
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    ldr d0, [sp, #8] // 8-byte Folded Reload
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
@@ -1247,7 +1242,6 @@ define void @dont_coalesce_res_v1i64(ptr %ptr) #0 {
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    ldr d0, [sp, #8] // 8-byte Folded Reload
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
@@ -1311,7 +1305,6 @@ define void @dont_coalesce_res_v1f32(ptr %ptr) #0 {
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    ptrue p0.s
 ; CHECK-NEXT:    ldr d0, [sp, #8] // 8-byte Folded Reload
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
@@ -1343,7 +1336,6 @@ define void @dont_coalesce_res_v1f64(ptr %ptr) #0 {
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    ptrue p0.d
 ; CHECK-NEXT:    ldr d0, [sp, #8] // 8-byte Folded Reload
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
 ; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
diff --git a/llvm/test/CodeGen/AArch64/sme-streaming-body-streaming-compatible-interface.ll b/llvm/test/CodeGen/AArch64/sme-streaming-body-streaming-compatible-interface.ll
index 6e262cc0786e..d67573384ca9 100644
--- a/llvm/test/CodeGen/AArch64/sme-streaming-body-streaming-compatible-interface.ll
+++ b/llvm/test/CodeGen/AArch64/sme-streaming-body-streaming-compatible-interface.ll
@@ -8,31 +8,27 @@ declare void @streaming_compatible_callee() "aarch64_pstate_sm_compatible";
 define float @sm_body_sm_compatible_simple() "aarch64_pstate_sm_compatible" "aarch64_pstate_sm_body" nounwind {
 ; CHECK-LABEL: sm_body_sm_compatible_simple:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    sub sp, sp, #96
-; CHECK-NEXT:    stp d15, d14, [sp, #16] // 16-byte Folded Spill
-; CHECK-NEXT:    stp d13, d12, [sp, #32] // 16-byte Folded Spill
-; CHECK-NEXT:    stp d11, d10, [sp, #48] // 16-byte Folded Spill
-; CHECK-NEXT:    stp d9, d8, [sp, #64] // 16-byte Folded Spill
-; CHECK-NEXT:    str x30, [sp, #80] // 8-byte Folded Spill
+; CHECK-NEXT:    stp d15, d14, [sp, #-80]! // 16-byte Folded Spill
+; CHECK-NEXT:    stp d13, d12, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d11, d10, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d9, d8, [sp, #48] // 16-byte Folded Spill
+; CHECK-NEXT:    str x30, [sp, #64] // 8-byte Folded Spill
 ; CHECK-NEXT:    bl __arm_sme_state
 ; CHECK-NEXT:    and x8, x0, #0x1
 ; CHECK-NEXT:    tbnz w8, #0, .LBB0_2
 ; CHECK-NEXT:  // %bb.1:
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:  .LBB0_2:
-; CHECK-NEXT:    fmov s0, wzr
-; CHECK-NEXT:    str s0, [sp, #12] // 4-byte Folded Spill
 ; CHECK-NEXT:    tbnz w8, #0, .LBB0_4
 ; CHECK-NEXT:  // %bb.3:
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:  .LBB0_4:
-; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr s0, [sp, #12] // 4-byte Folded Reload
-; CHECK-NEXT:    ldp d11, d10, [sp, #48] // 16-byte Folded Reload
-; CHECK-NEXT:    ldr x30, [sp, #80] // 8-byte Folded Reload
-; CHECK-NEXT:    ldp d13, d12, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT:    ldp d15, d14, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    add sp, sp, #96
+; CHECK-NEXT:    ldp d9, d8, [sp, #48] // 16-byte Folded Reload
+; CHECK-NEXT:    fmov s0, wzr
+; CHECK-NEXT:    ldp d11, d10, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x30, [sp, #64] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp d13, d12, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp d15, d14, [sp], #80 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   ret float zeroinitializer
 }
diff --git a/llvm/test/CodeGen/AArch64/sme-streaming-body.ll b/llvm/test/CodeGen/AArch64/sme-streaming-body.ll
index 08dec228d2f7..cd6d45f54eb9 100644
--- a/llvm/test/CodeGen/AArch64/sme-streaming-body.ll
+++ b/llvm/test/CodeGen/AArch64/sme-streaming-body.ll
@@ -247,15 +247,11 @@ define double @call_to_intrinsic_without_chain(double %x) nounwind "aarch64_psta
 ; CHECK-NEXT:    str x30, [sp, #80] // 8-byte Folded Spill
 ; CHECK-NEXT:    str d0, [sp, #8] // 8-byte Folded Spill
 ; CHECK-NEXT:    smstart sm
-; CHECK-NEXT:    ldr d0, [sp, #8] // 8-byte Folded Reload
-; CHECK-NEXT:    str d0, [sp, #8] // 8-byte Folded Spill
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    ldr d0, [sp, #8] // 8-byte Folded Reload
 ; CHECK-NEXT:    bl cos
 ; CHECK-NEXT:    str d0, [sp, #8] // 8-byte Folded Spill
 ; CHECK-NEXT:    smstart sm
-; CHECK-NEXT:    ldr d0, [sp, #8] // 8-byte Folded Reload
-; CHECK-NEXT:    str d0, [sp, #8] // 8-byte Folded Spill
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    ldp d9, d8, [sp, #64] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr d0, [sp, #8] // 8-byte Folded Reload
diff --git a/llvm/test/CodeGen/AArch64/sme-streaming-compatible-interface.ll b/llvm/test/CodeGen/AArch64/sme-streaming-compatible-interface.ll
index 6d2abf7e1841..47b24290d3c8 100644
--- a/llvm/test/CodeGen/AArch64/sme-streaming-compatible-interface.ll
+++ b/llvm/test/CodeGen/AArch64/sme-streaming-compatible-interface.ll
@@ -469,11 +469,7 @@ define void @call_to_non_streaming_pass_args(ptr nocapture noundef readnone %ptr
 ; CHECK-NEXT:    mov x9, x0
 ; CHECK-NEXT:    stp s0, s1, [sp, #8] // 8-byte Folded Spill
 ; CHECK-NEXT:    bl __arm_sme_state
-; CHECK-NEXT:    ldp s4, s0, [sp, #8] // 8-byte Folded Reload
 ; CHECK-NEXT:    and x19, x0, #0x1
-; CHECK-NEXT:    stp s4, s0, [sp, #8] // 8-byte Folded Spill
-; CHECK-NEXT:    ldp d4, d0, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT:    stp d4, d0, [sp, #16] // 16-byte Folded Spill
 ; CHECK-NEXT:    tbz w19, #0, .LBB10_2
 ; CHECK-NEXT:  // %bb.1: // %entry
 ; CHECK-NEXT:    smstop sm
diff --git a/llvm/test/CodeGen/AArch64/sme-streaming-interface.ll b/llvm/test/CodeGen/AArch64/sme-streaming-interface.ll
index de676ac5e0d2..465fb4667af6 100644
--- a/llvm/test/CodeGen/AArch64/sme-streaming-interface.ll
+++ b/llvm/test/CodeGen/AArch64/sme-streaming-interface.ll
@@ -405,11 +405,11 @@ define void @call_to_non_streaming_pass_args(ptr nocapture noundef readnone %ptr
 ; CHECK-NEXT:    stp d11, d10, [sp, #64] // 16-byte Folded Spill
 ; CHECK-NEXT:    stp d9, d8, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    str x30, [sp, #96] // 8-byte Folded Spill
-; CHECK-NEXT:    stp s1, s0, [sp, #24] // 8-byte Folded Spill
-; CHECK-NEXT:    stp d3, d2, [sp, #8] // 16-byte Folded Spill
+; CHECK-NEXT:    stp d2, d3, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    stp s0, s1, [sp, #8] // 8-byte Folded Spill
 ; CHECK-NEXT:    smstop sm
-; CHECK-NEXT:    ldp s1, s0, [sp, #24] // 8-byte Folded Reload
-; CHECK-NEXT:    ldp d3, d2, [sp, #8] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp s0, s1, [sp, #8] // 8-byte Folded Reload
+; CHECK-NEXT:    ldp d2, d3, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    bl bar
 ; CHECK-NEXT:    smstart sm
 ; CHECK-NEXT:    ldp d9, d8, [sp, #80] // 16-byte Folded Reload
diff --git a/llvm/test/CodeGen/AArch64/sme-streaming-mode-changing-call-disable-stackslot-scavenging.ll b/llvm/test/CodeGen/AArch64/sme-streaming-mode-changing-call-disable-stackslot-scavenging.ll
index cf171f8ef5ed..45ca7844b065 100644
--- a/llvm/test/CodeGen/AArch64/sme-streaming-mode-changing-call-disable-stackslot-scavenging.ll
+++ b/llvm/test/CodeGen/AArch64/sme-streaming-mode-changing-call-disable-stackslot-scavenging.ll
@@ -22,9 +22,9 @@ define void @test_no_stackslot_scavenging(float %f) #0 {
 ; CHECK-NEXT:    stp x30, x24, [sp, #80] // 16-byte Folded Spill
 ; CHECK-NEXT:    sub sp, sp, #16
 ; CHECK-NEXT:    addvl sp, sp, #-1
+; CHECK-NEXT:    str s0, [sp, #12] // 4-byte Folded Spill
 ; CHECK-NEXT:    //APP
 ; CHECK-NEXT:    //NO_APP
-; CHECK-NEXT:    str s0, [sp, #12] // 4-byte Folded Spill
 ; CHECK-NEXT:    smstop sm
 ; CHECK-NEXT:    ldr s0, [sp, #12] // 4-byte Folded Reload
 ; CHECK-NEXT:    bl use_f
-- 
GitLab


From b68ff060ab70c44fb617e34451003f34ff062d2b Mon Sep 17 00:00:00 2001
From: Amir Ayupov <aaupov@fb.com>
Date: Mon, 15 Apr 2024 16:16:12 +0200
Subject: [PATCH 246/464] [BOLT][NFC] Remove unused BinaryData::hasNameRegex
 (#88618)

---
 bolt/include/bolt/Core/BinaryData.h | 1 -
 bolt/lib/Core/BinaryData.cpp        | 8 --------
 2 files changed, 9 deletions(-)

diff --git a/bolt/include/bolt/Core/BinaryData.h b/bolt/include/bolt/Core/BinaryData.h
index 495163f1b61a..8a67b3e73b80 100644
--- a/bolt/include/bolt/Core/BinaryData.h
+++ b/bolt/include/bolt/Core/BinaryData.h
@@ -107,7 +107,6 @@ public:
   std::vector<MCSymbol *> &getSymbols() { return Symbols; }
 
   bool hasName(StringRef Name) const;
-  bool hasNameRegex(StringRef Name) const;
   bool nameStartsWith(StringRef Prefix) const;
 
   bool hasSymbol(const MCSymbol *Symbol) const {
diff --git a/bolt/lib/Core/BinaryData.cpp b/bolt/lib/Core/BinaryData.cpp
index 0068a9358004..e9ddf08d8695 100644
--- a/bolt/lib/Core/BinaryData.cpp
+++ b/bolt/lib/Core/BinaryData.cpp
@@ -55,14 +55,6 @@ bool BinaryData::hasName(StringRef Name) const {
   return false;
 }
 
-bool BinaryData::hasNameRegex(StringRef NameRegex) const {
-  Regex MatchName(NameRegex);
-  for (const MCSymbol *Symbol : Symbols)
-    if (MatchName.match(Symbol->getName()))
-      return true;
-  return false;
-}
-
 bool BinaryData::nameStartsWith(StringRef Prefix) const {
   for (const MCSymbol *Symbol : Symbols)
     if (Symbol->getName().starts_with(Prefix))
-- 
GitLab


From a3ce29f7bb5510131971ac5ccc63132dd48c8dd2 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Mon, 15 Apr 2024 10:30:00 -0400
Subject: [PATCH 247/464] [libc++][PSTL] Introduce cpu traits (#88134)

Currently, CPU backends in the PSTL are created by defining functions
in the __par_backend namespace. Then, the PSTL includes the CPU backend
that gets configured via CMake and gets those definitions.

This prevents CPU backends from easily co-existing and is a bit
confusing.
To solve this problem, this patch introduces the notion of __cpu_traits,
which is a cheap encapsulation of the basis operations required to
implement a CPU-based PSTL. Different backends can now define their own
tag and coexist, and the CPU-based PSTL will simply use __cpu_traits to
dispatch to the right implementation of e.g. __for_each.

Note that this patch doesn't change the actual implementation of the
backends in any way, it only modifies how that implementation is
accessed
to implement PSTL algorithms.

This patch is a step towards #88131.
---
 libcxx/include/CMakeLists.txt                 |   1 +
 .../__algorithm/pstl_backends/cpu_backend.h   |  47 +-
 .../pstl_backends/cpu_backends/any_of.h       |   9 +-
 .../pstl_backends/cpu_backends/backend.h      |  14 +-
 .../pstl_backends/cpu_backends/fill.h         |   3 +-
 .../pstl_backends/cpu_backends/find_if.h      |  22 +-
 .../pstl_backends/cpu_backends/for_each.h     |   3 +-
 .../pstl_backends/cpu_backends/libdispatch.h  | 457 +++++++++---------
 .../pstl_backends/cpu_backends/merge.h        |   3 +-
 .../pstl_backends/cpu_backends/serial.h       |  98 ++--
 .../pstl_backends/cpu_backends/stable_sort.h  |   3 +-
 .../pstl_backends/cpu_backends/thread.h       |  96 ++--
 .../pstl_backends/cpu_backends/transform.h    |   5 +-
 .../cpu_backends/transform_reduce.h           |  18 +-
 libcxx/include/__pstl/cpu_algos/cpu_traits.h  |  87 ++++
 libcxx/include/module.modulemap               |   2 +
 libcxx/src/pstl/libdispatch.cpp               |   7 +-
 ...pstl.libdispatch.chunk_partitions.pass.cpp |   6 +-
 libcxx/utils/generate_iwyu_mapping.py         |   1 +
 19 files changed, 476 insertions(+), 406 deletions(-)
 create mode 100644 libcxx/include/__pstl/cpu_algos/cpu_traits.h

diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 1b6c6de082da..148a51a216ed 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -594,6 +594,7 @@ set(files
   __numeric/transform_exclusive_scan.h
   __numeric/transform_inclusive_scan.h
   __numeric/transform_reduce.h
+  __pstl/cpu_algos/cpu_traits.h
   __random/bernoulli_distribution.h
   __random/binomial_distribution.h
   __random/cauchy_distribution.h
diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backend.h b/libcxx/include/__algorithm/pstl_backends/cpu_backend.h
index 6980ded189ea..53eae58f9609 100644
--- a/libcxx/include/__algorithm/pstl_backends/cpu_backend.h
+++ b/libcxx/include/__algorithm/pstl_backends/cpu_backend.h
@@ -9,52 +9,6 @@
 #ifndef _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_H
 #define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_H
 
-#include <__config>
-
-/*
-
-  // _Functor takes a subrange for [__first, __last) that should be executed in serial
-  template <class _RandomAccessIterator, class _Functor>
-  optional<__empty> __parallel_for(_RandomAccessIterator __first, _RandomAccessIterator __last, _Functor __func);
-
-  template <class _Iterator, class _UnaryOp, class _Tp, class _BinaryOp, class _Reduction>
-  optional<_Tp>
-  __parallel_transform_reduce(_Iterator __first, _Iterator __last, _UnaryOp, _Tp __init, _BinaryOp, _Reduction);
-
-  // Cancel the execution of other jobs - they aren't needed anymore
-  void __cancel_execution();
-
-  template <class _RandomAccessIterator1,
-            class _RandomAccessIterator2,
-            class _RandomAccessIterator3,
-            class _Compare,
-            class _LeafMerge>
-  optional<void> __parallel_merge(
-      _RandomAccessIterator1 __first1,
-      _RandomAccessIterator1 __last1,
-      _RandomAccessIterator2 __first2,
-      _RandomAccessIterator2 __last2,
-      _RandomAccessIterator3 __outit,
-      _Compare __comp,
-      _LeafMerge __leaf_merge);
-
-  template <class _RandomAccessIterator, class _Comp, class _LeafSort>
-  void __parallel_stable_sort(_RandomAccessIterator __first,
-                              _RandomAccessIterator __last,
-                              _Comp __comp,
-                              _LeafSort __leaf_sort);
-
-  TODO: Document the parallel backend
-
-Exception handling
-==================
-
-CPU backends are expected to report errors (i.e. failure to allocate) by returning a disengaged `optional` from their
-implementation. Exceptions shouldn't be used to report an internal failure-to-allocate, since all exceptions are turned
-into a program termination at the front-end level. When a backend returns a disengaged `optional` to the frontend, the
-frontend will turn that into a call to `std::__throw_bad_alloc();` to report the internal failure to the user.
-*/
-
 #include <__algorithm/pstl_backends/cpu_backends/any_of.h>
 #include <__algorithm/pstl_backends/cpu_backends/backend.h>
 #include <__algorithm/pstl_backends/cpu_backends/fill.h>
@@ -64,5 +18,6 @@ frontend will turn that into a call to `std::__throw_bad_alloc();` to report the
 #include <__algorithm/pstl_backends/cpu_backends/stable_sort.h>
 #include <__algorithm/pstl_backends/cpu_backends/transform.h>
 #include <__algorithm/pstl_backends/cpu_backends/transform_reduce.h>
+#include <__config>
 
 #endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_H
diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/any_of.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/any_of.h
index 13dff80086e7..be5e54f3fa5c 100644
--- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/any_of.h
+++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/any_of.h
@@ -17,6 +17,7 @@
 #include <__config>
 #include <__functional/operations.h>
 #include <__iterator/concepts.h>
+#include <__pstl/cpu_algos/cpu_traits.h>
 #include <__type_traits/is_execution_policy.h>
 #include <__utility/move.h>
 #include <__utility/pair.h>
@@ -30,13 +31,13 @@ _LIBCPP_PUSH_MACROS
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Index, class _Brick>
+template <class _Backend, class _Index, class _Brick>
 _LIBCPP_HIDE_FROM_ABI optional<bool> __parallel_or(_Index __first, _Index __last, _Brick __f) {
   std::atomic<bool> __found(false);
-  auto __ret = __par_backend::__parallel_for(__first, __last, [__f, &__found](_Index __i, _Index __j) {
+  auto __ret = __pstl::__cpu_traits<_Backend>::__parallel_for(__first, __last, [__f, &__found](_Index __i, _Index __j) {
     if (!__found.load(std::memory_order_relaxed) && __f(__i, __j)) {
       __found.store(true, std::memory_order_relaxed);
-      __par_backend::__cancel_execution();
+      __pstl::__cpu_traits<_Backend>::__cancel_execution();
     }
   });
   if (!__ret)
@@ -74,7 +75,7 @@ _LIBCPP_HIDE_FROM_ABI optional<bool>
 __pstl_any_of(__cpu_backend_tag, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) {
   if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> &&
                 __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) {
-    return std::__parallel_or(
+    return std::__parallel_or<__cpu_backend_tag>(
         __first, __last, [&__pred](_ForwardIterator __brick_first, _ForwardIterator __brick_last) {
           auto __res = std::__pstl_any_of<__remove_parallel_policy_t<_ExecutionPolicy>>(
               __cpu_backend_tag{}, __brick_first, __brick_last, __pred);
diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/backend.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/backend.h
index ea2210a4a7ad..cb9425862a2b 100644
--- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/backend.h
+++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/backend.h
@@ -26,16 +26,20 @@
 #  pragma GCC system_header
 #endif
 
-#if _LIBCPP_STD_VER >= 17
+#if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-struct __cpu_backend_tag {};
-
-inline constexpr size_t __lane_size = 64;
+#  if defined(_LIBCPP_PSTL_CPU_BACKEND_SERIAL)
+using __cpu_backend_tag = __pstl::__serial_backend_tag;
+#  elif defined(_LIBCPP_PSTL_CPU_BACKEND_THREAD)
+using __cpu_backend_tag = __pstl::__std_thread_backend_tag;
+#  elif defined(_LIBCPP_PSTL_CPU_BACKEND_LIBDISPATCH)
+using __cpu_backend_tag = __pstl::__libdispatch_backend_tag;
+#  endif
 
 _LIBCPP_END_NAMESPACE_STD
 
-#endif // _LIBCPP_STD_VER >= 17
+#endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && && _LIBCPP_STD_VER >= 17
 
 #endif // _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKEND_BACKEND_H
diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/fill.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/fill.h
index 64babe9fd2bd..49a32f6c5ce5 100644
--- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/fill.h
+++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/fill.h
@@ -13,6 +13,7 @@
 #include <__algorithm/pstl_backends/cpu_backends/backend.h>
 #include <__config>
 #include <__iterator/concepts.h>
+#include <__pstl/cpu_algos/cpu_traits.h>
 #include <__type_traits/is_execution_policy.h>
 #include <__utility/empty.h>
 #include <optional>
@@ -39,7 +40,7 @@ _LIBCPP_HIDE_FROM_ABI optional<__empty>
 __pstl_fill(__cpu_backend_tag, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) {
   if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> &&
                 __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) {
-    return __par_backend::__parallel_for(
+    return __pstl::__cpu_traits<__cpu_backend_tag>::__parallel_for(
         __first, __last, [&__value](_ForwardIterator __brick_first, _ForwardIterator __brick_last) {
           [[maybe_unused]] auto __res = std::__pstl_fill<__remove_parallel_policy_t<_ExecutionPolicy>>(
               __cpu_backend_tag{}, __brick_first, __brick_last, __value);
diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/find_if.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/find_if.h
index 170470e4fb7e..11a5668bf25a 100644
--- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/find_if.h
+++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/find_if.h
@@ -16,6 +16,7 @@
 #include <__functional/operations.h>
 #include <__iterator/concepts.h>
 #include <__iterator/iterator_traits.h>
+#include <__pstl/cpu_algos/cpu_traits.h>
 #include <__type_traits/is_execution_policy.h>
 #include <__utility/move.h>
 #include <__utility/pair.h>
@@ -33,7 +34,7 @@ _LIBCPP_PUSH_MACROS
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Index, class _Brick, class _Compare>
+template <class _Backend, class _Index, class _Brick, class _Compare>
 _LIBCPP_HIDE_FROM_ABI optional<_Index>
 __parallel_find(_Index __first, _Index __last, _Brick __f, _Compare __comp, bool __b_first) {
   typedef typename std::iterator_traits<_Index>::difference_type _DifferenceType;
@@ -41,8 +42,8 @@ __parallel_find(_Index __first, _Index __last, _Brick __f, _Compare __comp, bool
   _DifferenceType __initial_dist = __b_first ? __n : -1;
   std::atomic<_DifferenceType> __extremum(__initial_dist);
   // TODO: find out what is better here: parallel_for or parallel_reduce
-  auto __res =
-      __par_backend::__parallel_for(__first, __last, [__comp, __f, __first, &__extremum](_Index __i, _Index __j) {
+  auto __res = __pstl::__cpu_traits<_Backend>::__parallel_for(
+      __first, __last, [__comp, __f, __first, &__extremum](_Index __i, _Index __j) {
         // See "Reducing Contention Through Priority Updates", PPoPP '13, for discussion of
         // why using a shared variable scales fairly well in this situation.
         if (__comp(__i - __first, __extremum)) {
@@ -61,12 +62,12 @@ __parallel_find(_Index __first, _Index __last, _Brick __f, _Compare __comp, bool
   return __extremum.load() != __initial_dist ? __first + __extremum.load() : __last;
 }
 
-template <class _Index, class _DifferenceType, class _Compare>
+template <class _Backend, class _Index, class _DifferenceType, class _Compare>
 _LIBCPP_HIDE_FROM_ABI _Index
 __simd_first(_Index __first, _DifferenceType __begin, _DifferenceType __end, _Compare __comp) noexcept {
   // Experiments show good block sizes like this
-  const _DifferenceType __block_size                        = 8;
-  alignas(__lane_size) _DifferenceType __lane[__block_size] = {0};
+  const _DifferenceType __block_size                                                        = 8;
+  alignas(__pstl::__cpu_traits<_Backend>::__lane_size) _DifferenceType __lane[__block_size] = {0};
   while (__end - __begin >= __block_size) {
     _DifferenceType __found = 0;
     _PSTL_PRAGMA_SIMD_REDUCTION(| : __found) for (_DifferenceType __i = __begin; __i < __begin + __block_size; ++__i) {
@@ -102,7 +103,7 @@ _LIBCPP_HIDE_FROM_ABI optional<_ForwardIterator>
 __pstl_find_if(__cpu_backend_tag, _ForwardIterator __first, _ForwardIterator __last, _Predicate __pred) {
   if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> &&
                 __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) {
-    return std::__parallel_find(
+    return std::__parallel_find<__cpu_backend_tag>(
         __first,
         __last,
         [&__pred](_ForwardIterator __brick_first, _ForwardIterator __brick_last) {
@@ -116,9 +117,10 @@ __pstl_find_if(__cpu_backend_tag, _ForwardIterator __first, _ForwardIterator __l
   } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> &&
                        __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) {
     using __diff_t = __iter_diff_t<_ForwardIterator>;
-    return std::__simd_first(__first, __diff_t(0), __last - __first, [&__pred](_ForwardIterator __iter, __diff_t __i) {
-      return __pred(__iter[__i]);
-    });
+    return std::__simd_first<__cpu_backend_tag>(
+        __first, __diff_t(0), __last - __first, [&__pred](_ForwardIterator __iter, __diff_t __i) {
+          return __pred(__iter[__i]);
+        });
   } else {
     return std::find_if(__first, __last, __pred);
   }
diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/for_each.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/for_each.h
index 81fd4526b8db..1667ec0f0c4f 100644
--- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/for_each.h
+++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/for_each.h
@@ -13,6 +13,7 @@
 #include <__algorithm/pstl_backends/cpu_backends/backend.h>
 #include <__config>
 #include <__iterator/concepts.h>
+#include <__pstl/cpu_algos/cpu_traits.h>
 #include <__type_traits/is_execution_policy.h>
 #include <__utility/empty.h>
 #include <optional>
@@ -39,7 +40,7 @@ _LIBCPP_HIDE_FROM_ABI optional<__empty>
 __pstl_for_each(__cpu_backend_tag, _ForwardIterator __first, _ForwardIterator __last, _Functor __func) {
   if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> &&
                 __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) {
-    return std::__par_backend::__parallel_for(
+    return __pstl::__cpu_traits<__cpu_backend_tag>::__parallel_for(
         __first, __last, [__func](_ForwardIterator __brick_first, _ForwardIterator __brick_last) {
           [[maybe_unused]] auto __res = std::__pstl_for_each<__remove_parallel_policy_t<_ExecutionPolicy>>(
               __cpu_backend_tag{}, __brick_first, __brick_last, __func);
diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/libdispatch.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/libdispatch.h
index e885e7f22517..8757f2496803 100644
--- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/libdispatch.h
+++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/libdispatch.h
@@ -23,6 +23,7 @@
 #include <__memory/construct_at.h>
 #include <__memory/unique_ptr.h>
 #include <__numeric/reduce.h>
+#include <__pstl/cpu_algos/cpu_traits.h>
 #include <__utility/empty.h>
 #include <__utility/exception_guard.h>
 #include <__utility/move.h>
@@ -37,10 +38,11 @@ _LIBCPP_PUSH_MACROS
 #if !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17
 
 _LIBCPP_BEGIN_NAMESPACE_STD
+namespace __pstl {
 
-namespace __par_backend {
-inline namespace __libdispatch {
+struct __libdispatch_backend_tag {};
 
+namespace __libdispatch {
 // ::dispatch_apply is marked as __attribute__((nothrow)) because it doesn't let exceptions propagate, and neither do
 // we.
 // TODO: Do we want to add [[_Clang::__callback__(__func, __context, __)]]?
@@ -77,267 +79,270 @@ __dispatch_parallel_for(__chunk_partitions __partitions, _RandomAccessIterator _
 
   return __empty{};
 }
+} // namespace __libdispatch
 
-template <class _RandomAccessIterator, class _Functor>
-_LIBCPP_HIDE_FROM_ABI optional<__empty>
-__parallel_for(_RandomAccessIterator __first, _RandomAccessIterator __last, _Functor __func) {
-  return __libdispatch::__dispatch_parallel_for(
-      __libdispatch::__partition_chunks(__last - __first), std::move(__first), std::move(__func));
-}
-
-template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIteratorOut>
-struct __merge_range {
-  __merge_range(_RandomAccessIterator1 __mid1, _RandomAccessIterator2 __mid2, _RandomAccessIteratorOut __result)
-      : __mid1_(__mid1), __mid2_(__mid2), __result_(__result) {}
+template <>
+struct __cpu_traits<__libdispatch_backend_tag> {
+  template <class _RandomAccessIterator, class _Functor>
+  _LIBCPP_HIDE_FROM_ABI static optional<__empty>
+  __parallel_for(_RandomAccessIterator __first, _RandomAccessIterator __last, _Functor __func) {
+    return __libdispatch::__dispatch_parallel_for(
+        __libdispatch::__partition_chunks(__last - __first), std::move(__first), std::move(__func));
+  }
 
-  _RandomAccessIterator1 __mid1_;
-  _RandomAccessIterator2 __mid2_;
-  _RandomAccessIteratorOut __result_;
-};
+  template <class _RandomAccessIterator1, class _RandomAccessIterator2, class _RandomAccessIteratorOut>
+  struct __merge_range {
+    __merge_range(_RandomAccessIterator1 __mid1, _RandomAccessIterator2 __mid2, _RandomAccessIteratorOut __result)
+        : __mid1_(__mid1), __mid2_(__mid2), __result_(__result) {}
 
-template <typename _RandomAccessIterator1,
-          typename _RandomAccessIterator2,
-          typename _RandomAccessIterator3,
-          typename _Compare,
-          typename _LeafMerge>
-_LIBCPP_HIDE_FROM_ABI optional<__empty> __parallel_merge(
-    _RandomAccessIterator1 __first1,
-    _RandomAccessIterator1 __last1,
-    _RandomAccessIterator2 __first2,
-    _RandomAccessIterator2 __last2,
-    _RandomAccessIterator3 __result,
-    _Compare __comp,
-    _LeafMerge __leaf_merge) noexcept {
-  __chunk_partitions __partitions =
-      __libdispatch::__partition_chunks(std::max<ptrdiff_t>(__last1 - __first1, __last2 - __first2));
-
-  if (__partitions.__chunk_count_ == 0)
-    return __empty{};
+    _RandomAccessIterator1 __mid1_;
+    _RandomAccessIterator2 __mid2_;
+    _RandomAccessIteratorOut __result_;
+  };
 
-  if (__partitions.__chunk_count_ == 1) {
-    __leaf_merge(__first1, __last1, __first2, __last2, __result, __comp);
-    return __empty{};
-  }
+  template <typename _RandomAccessIterator1,
+            typename _RandomAccessIterator2,
+            typename _RandomAccessIterator3,
+            typename _Compare,
+            typename _LeafMerge>
+  _LIBCPP_HIDE_FROM_ABI static optional<__empty> __parallel_merge(
+      _RandomAccessIterator1 __first1,
+      _RandomAccessIterator1 __last1,
+      _RandomAccessIterator2 __first2,
+      _RandomAccessIterator2 __last2,
+      _RandomAccessIterator3 __result,
+      _Compare __comp,
+      _LeafMerge __leaf_merge) noexcept {
+    __libdispatch::__chunk_partitions __partitions =
+        __libdispatch::__partition_chunks(std::max<ptrdiff_t>(__last1 - __first1, __last2 - __first2));
+
+    if (__partitions.__chunk_count_ == 0)
+      return __empty{};
+
+    if (__partitions.__chunk_count_ == 1) {
+      __leaf_merge(__first1, __last1, __first2, __last2, __result, __comp);
+      return __empty{};
+    }
 
-  using __merge_range_t = __merge_range<_RandomAccessIterator1, _RandomAccessIterator2, _RandomAccessIterator3>;
-  auto const __n_ranges = __partitions.__chunk_count_ + 1;
+    using __merge_range_t = __merge_range<_RandomAccessIterator1, _RandomAccessIterator2, _RandomAccessIterator3>;
+    auto const __n_ranges = __partitions.__chunk_count_ + 1;
 
-  // TODO: use __uninitialized_buffer
-  auto __destroy = [=](__merge_range_t* __ptr) {
-    std::destroy_n(__ptr, __n_ranges);
-    std::allocator<__merge_range_t>().deallocate(__ptr, __n_ranges);
-  };
+    // TODO: use __uninitialized_buffer
+    auto __destroy = [=](__merge_range_t* __ptr) {
+      std::destroy_n(__ptr, __n_ranges);
+      std::allocator<__merge_range_t>().deallocate(__ptr, __n_ranges);
+    };
 
-  unique_ptr<__merge_range_t[], decltype(__destroy)> __ranges(
-      [&]() -> __merge_range_t* {
+    unique_ptr<__merge_range_t[], decltype(__destroy)> __ranges(
+        [&]() -> __merge_range_t* {
 #  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
-        try {
+          try {
 #  endif
-          return std::allocator<__merge_range_t>().allocate(__n_ranges);
+            return std::allocator<__merge_range_t>().allocate(__n_ranges);
 #  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
-        } catch (const std::bad_alloc&) {
-          return nullptr;
-        }
+          } catch (const std::bad_alloc&) {
+            return nullptr;
+          }
 #  endif
-      }(),
-      __destroy);
-
-  if (!__ranges)
-    return nullopt;
+        }(),
+        __destroy);
+
+    if (!__ranges)
+      return nullopt;
+
+    // TODO: Improve the case where the smaller range is merged into just a few (or even one) chunks of the larger case
+    __merge_range_t* __r = __ranges.get();
+    std::__construct_at(__r++, __first1, __first2, __result);
+
+    bool __iterate_first_range = __last1 - __first1 > __last2 - __first2;
+
+    auto __compute_chunk = [&](size_t __chunk_size) -> __merge_range_t {
+      auto [__mid1, __mid2] = [&] {
+        if (__iterate_first_range) {
+          auto __m1 = __first1 + __chunk_size;
+          auto __m2 = std::lower_bound(__first2, __last2, __m1[-1], __comp);
+          return std::make_pair(__m1, __m2);
+        } else {
+          auto __m2 = __first2 + __chunk_size;
+          auto __m1 = std::lower_bound(__first1, __last1, __m2[-1], __comp);
+          return std::make_pair(__m1, __m2);
+        }
+      }();
 
-  // TODO: Improve the case where the smaller range is merged into just a few (or even one) chunks of the larger case
-  __merge_range_t* __r = __ranges.get();
-  std::__construct_at(__r++, __first1, __first2, __result);
+      __result += (__mid1 - __first1) + (__mid2 - __first2);
+      __first1 = __mid1;
+      __first2 = __mid2;
+      return {std::move(__mid1), std::move(__mid2), __result};
+    };
 
-  bool __iterate_first_range = __last1 - __first1 > __last2 - __first2;
+    // handle first chunk
+    std::__construct_at(__r++, __compute_chunk(__partitions.__first_chunk_size_));
 
-  auto __compute_chunk = [&](size_t __chunk_size) -> __merge_range_t {
-    auto [__mid1, __mid2] = [&] {
-      if (__iterate_first_range) {
-        auto __m1 = __first1 + __chunk_size;
-        auto __m2 = std::lower_bound(__first2, __last2, __m1[-1], __comp);
-        return std::make_pair(__m1, __m2);
-      } else {
-        auto __m2 = __first2 + __chunk_size;
-        auto __m1 = std::lower_bound(__first1, __last1, __m2[-1], __comp);
-        return std::make_pair(__m1, __m2);
-      }
-    }();
+    // handle 2 -> N - 1 chunks
+    for (ptrdiff_t __i = 0; __i != __partitions.__chunk_count_ - 2; ++__i)
+      std::__construct_at(__r++, __compute_chunk(__partitions.__chunk_size_));
 
-    __result += (__mid1 - __first1) + (__mid2 - __first2);
-    __first1 = __mid1;
-    __first2 = __mid2;
-    return {std::move(__mid1), std::move(__mid2), __result};
-  };
+    // handle last chunk
+    std::__construct_at(__r, __last1, __last2, __result);
 
-  // handle first chunk
-  std::__construct_at(__r++, __compute_chunk(__partitions.__first_chunk_size_));
-
-  // handle 2 -> N - 1 chunks
-  for (ptrdiff_t __i = 0; __i != __partitions.__chunk_count_ - 2; ++__i)
-    std::__construct_at(__r++, __compute_chunk(__partitions.__chunk_size_));
-
-  // handle last chunk
-  std::__construct_at(__r, __last1, __last2, __result);
-
-  __libdispatch::__dispatch_apply(__partitions.__chunk_count_, [&](size_t __index) {
-    auto __first_iters = __ranges[__index];
-    auto __last_iters  = __ranges[__index + 1];
-    __leaf_merge(
-        __first_iters.__mid1_,
-        __last_iters.__mid1_,
-        __first_iters.__mid2_,
-        __last_iters.__mid2_,
-        __first_iters.__result_,
-        __comp);
-  });
+    __libdispatch::__dispatch_apply(__partitions.__chunk_count_, [&](size_t __index) {
+      auto __first_iters = __ranges[__index];
+      auto __last_iters  = __ranges[__index + 1];
+      __leaf_merge(
+          __first_iters.__mid1_,
+          __last_iters.__mid1_,
+          __first_iters.__mid2_,
+          __last_iters.__mid2_,
+          __first_iters.__result_,
+          __comp);
+    });
 
-  return __empty{};
-}
+    return __empty{};
+  }
 
-template <class _RandomAccessIterator, class _Transform, class _Value, class _Combiner, class _Reduction>
-_LIBCPP_HIDE_FROM_ABI optional<_Value> __parallel_transform_reduce(
-    _RandomAccessIterator __first,
-    _RandomAccessIterator __last,
-    _Transform __transform,
-    _Value __init,
-    _Combiner __combiner,
-    _Reduction __reduction) {
-  if (__first == __last)
-    return __init;
-
-  auto __partitions = __libdispatch::__partition_chunks(__last - __first);
-
-  auto __destroy = [__count = __partitions.__chunk_count_](_Value* __ptr) {
-    std::destroy_n(__ptr, __count);
-    std::allocator<_Value>().deallocate(__ptr, __count);
-  };
+  template <class _RandomAccessIterator, class _Transform, class _Value, class _Combiner, class _Reduction>
+  _LIBCPP_HIDE_FROM_ABI static optional<_Value> __parallel_transform_reduce(
+      _RandomAccessIterator __first,
+      _RandomAccessIterator __last,
+      _Transform __transform,
+      _Value __init,
+      _Combiner __combiner,
+      _Reduction __reduction) {
+    if (__first == __last)
+      return __init;
+
+    auto __partitions = __libdispatch::__partition_chunks(__last - __first);
+
+    auto __destroy = [__count = __partitions.__chunk_count_](_Value* __ptr) {
+      std::destroy_n(__ptr, __count);
+      std::allocator<_Value>().deallocate(__ptr, __count);
+    };
 
-  // TODO: use __uninitialized_buffer
-  // TODO: allocate one element per worker instead of one element per chunk
-  unique_ptr<_Value[], decltype(__destroy)> __values(
-      std::allocator<_Value>().allocate(__partitions.__chunk_count_), __destroy);
+    // TODO: use __uninitialized_buffer
+    // TODO: allocate one element per worker instead of one element per chunk
+    unique_ptr<_Value[], decltype(__destroy)> __values(
+        std::allocator<_Value>().allocate(__partitions.__chunk_count_), __destroy);
+
+    // __dispatch_apply is noexcept
+    __libdispatch::__dispatch_apply(__partitions.__chunk_count_, [&](size_t __chunk) {
+      auto __this_chunk_size = __chunk == 0 ? __partitions.__first_chunk_size_ : __partitions.__chunk_size_;
+      auto __index           = __chunk == 0 ? 0
+                                            : (__chunk * __partitions.__chunk_size_) +
+                                        (__partitions.__first_chunk_size_ - __partitions.__chunk_size_);
+      if (__this_chunk_size != 1) {
+        std::__construct_at(
+            __values.get() + __chunk,
+            __reduction(__first + __index + 2,
+                        __first + __index + __this_chunk_size,
+                        __combiner(__transform(__first + __index), __transform(__first + __index + 1))));
+      } else {
+        std::__construct_at(__values.get() + __chunk, __transform(__first + __index));
+      }
+    });
 
-  // __dispatch_apply is noexcept
-  __libdispatch::__dispatch_apply(__partitions.__chunk_count_, [&](size_t __chunk) {
-    auto __this_chunk_size = __chunk == 0 ? __partitions.__first_chunk_size_ : __partitions.__chunk_size_;
-    auto __index =
-        __chunk == 0
-            ? 0
-            : (__chunk * __partitions.__chunk_size_) + (__partitions.__first_chunk_size_ - __partitions.__chunk_size_);
-    if (__this_chunk_size != 1) {
-      std::__construct_at(
-          __values.get() + __chunk,
-          __reduction(__first + __index + 2,
-                      __first + __index + __this_chunk_size,
-                      __combiner(__transform(__first + __index), __transform(__first + __index + 1))));
-    } else {
-      std::__construct_at(__values.get() + __chunk, __transform(__first + __index));
-    }
-  });
+    return std::reduce(
+        std::make_move_iterator(__values.get()),
+        std::make_move_iterator(__values.get() + __partitions.__chunk_count_),
+        std::move(__init),
+        __combiner);
+  }
 
-  return std::reduce(
-      std::make_move_iterator(__values.get()),
-      std::make_move_iterator(__values.get() + __partitions.__chunk_count_),
-      std::move(__init),
-      __combiner);
-}
+  template <class _RandomAccessIterator, class _Comp, class _LeafSort>
+  _LIBCPP_HIDE_FROM_ABI static optional<__empty> __parallel_stable_sort(
+      _RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp, _LeafSort __leaf_sort) {
+    const auto __size = __last - __first;
+    auto __partitions = __libdispatch::__partition_chunks(__size);
 
-template <class _RandomAccessIterator, class _Comp, class _LeafSort>
-_LIBCPP_HIDE_FROM_ABI optional<__empty> __parallel_stable_sort(
-    _RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp, _LeafSort __leaf_sort) {
-  const auto __size = __last - __first;
-  auto __partitions = __libdispatch::__partition_chunks(__size);
+    if (__partitions.__chunk_count_ == 0)
+      return __empty{};
 
-  if (__partitions.__chunk_count_ == 0)
-    return __empty{};
+    if (__partitions.__chunk_count_ == 1) {
+      __leaf_sort(__first, __last, __comp);
+      return __empty{};
+    }
 
-  if (__partitions.__chunk_count_ == 1) {
-    __leaf_sort(__first, __last, __comp);
-    return __empty{};
-  }
+    using _Value = __iter_value_type<_RandomAccessIterator>;
 
-  using _Value = __iter_value_type<_RandomAccessIterator>;
+    auto __destroy = [__size](_Value* __ptr) {
+      std::destroy_n(__ptr, __size);
+      std::allocator<_Value>().deallocate(__ptr, __size);
+    };
 
-  auto __destroy = [__size](_Value* __ptr) {
-    std::destroy_n(__ptr, __size);
-    std::allocator<_Value>().deallocate(__ptr, __size);
-  };
+    // TODO: use __uninitialized_buffer
+    unique_ptr<_Value[], decltype(__destroy)> __values(std::allocator<_Value>().allocate(__size), __destroy);
 
-  // TODO: use __uninitialized_buffer
-  unique_ptr<_Value[], decltype(__destroy)> __values(std::allocator<_Value>().allocate(__size), __destroy);
+    // Initialize all elements to a moved-from state
+    // TODO: Don't do this - this can be done in the first merge - see https://llvm.org/PR63928
+    std::__construct_at(__values.get(), std::move(*__first));
+    for (__iter_diff_t<_RandomAccessIterator> __i = 1; __i != __size; ++__i) {
+      std::__construct_at(__values.get() + __i, std::move(__values.get()[__i - 1]));
+    }
+    *__first = std::move(__values.get()[__size - 1]);
+
+    __libdispatch::__dispatch_parallel_for(
+        __partitions,
+        __first,
+        [&__leaf_sort, &__comp](_RandomAccessIterator __chunk_first, _RandomAccessIterator __chunk_last) {
+          __leaf_sort(std::move(__chunk_first), std::move(__chunk_last), __comp);
+        });
+
+    bool __objects_are_in_buffer = false;
+    do {
+      const auto __old_chunk_size = __partitions.__chunk_size_;
+      if (__partitions.__chunk_count_ % 2 == 1) {
+        auto __inplace_merge_chunks = [&__comp, &__partitions](auto __first_chunk_begin) {
+          std::inplace_merge(
+              __first_chunk_begin,
+              __first_chunk_begin + __partitions.__first_chunk_size_,
+              __first_chunk_begin + __partitions.__first_chunk_size_ + __partitions.__chunk_size_,
+              __comp);
+        };
+        if (__objects_are_in_buffer)
+          __inplace_merge_chunks(__values.get());
+        else
+          __inplace_merge_chunks(__first);
+        __partitions.__first_chunk_size_ += 2 * __partitions.__chunk_size_;
+      } else {
+        __partitions.__first_chunk_size_ += __partitions.__chunk_size_;
+      }
 
-  // Initialize all elements to a moved-from state
-  // TODO: Don't do this - this can be done in the first merge - see https://llvm.org/PR63928
-  std::__construct_at(__values.get(), std::move(*__first));
-  for (__iter_diff_t<_RandomAccessIterator> __i = 1; __i != __size; ++__i) {
-    std::__construct_at(__values.get() + __i, std::move(__values.get()[__i - 1]));
-  }
-  *__first = std::move(__values.get()[__size - 1]);
-
-  __libdispatch::__dispatch_parallel_for(
-      __partitions,
-      __first,
-      [&__leaf_sort, &__comp](_RandomAccessIterator __chunk_first, _RandomAccessIterator __chunk_last) {
-        __leaf_sort(std::move(__chunk_first), std::move(__chunk_last), __comp);
-      });
-
-  bool __objects_are_in_buffer = false;
-  do {
-    const auto __old_chunk_size = __partitions.__chunk_size_;
-    if (__partitions.__chunk_count_ % 2 == 1) {
-      auto __inplace_merge_chunks = [&__comp, &__partitions](auto __first_chunk_begin) {
-        std::inplace_merge(
-            __first_chunk_begin,
-            __first_chunk_begin + __partitions.__first_chunk_size_,
-            __first_chunk_begin + __partitions.__first_chunk_size_ + __partitions.__chunk_size_,
-            __comp);
+      __partitions.__chunk_size_ *= 2;
+      __partitions.__chunk_count_ /= 2;
+
+      auto __merge_chunks = [__partitions, __old_chunk_size, &__comp](auto __from_first, auto __to_first) {
+        __libdispatch::__dispatch_parallel_for(
+            __partitions,
+            __from_first,
+            [__old_chunk_size, &__from_first, &__to_first, &__comp](auto __chunk_first, auto __chunk_last) {
+              std::merge(std::make_move_iterator(__chunk_first),
+                         std::make_move_iterator(__chunk_last - __old_chunk_size),
+                         std::make_move_iterator(__chunk_last - __old_chunk_size),
+                         std::make_move_iterator(__chunk_last),
+                         __to_first + (__chunk_first - __from_first),
+                         __comp);
+            });
       };
+
       if (__objects_are_in_buffer)
-        __inplace_merge_chunks(__values.get());
+        __merge_chunks(__values.get(), __first);
       else
-        __inplace_merge_chunks(__first);
-      __partitions.__first_chunk_size_ += 2 * __partitions.__chunk_size_;
-    } else {
-      __partitions.__first_chunk_size_ += __partitions.__chunk_size_;
-    }
-
-    __partitions.__chunk_size_ *= 2;
-    __partitions.__chunk_count_ /= 2;
-
-    auto __merge_chunks = [__partitions, __old_chunk_size, &__comp](auto __from_first, auto __to_first) {
-      __libdispatch::__dispatch_parallel_for(
-          __partitions,
-          __from_first,
-          [__old_chunk_size, &__from_first, &__to_first, &__comp](auto __chunk_first, auto __chunk_last) {
-            std::merge(std::make_move_iterator(__chunk_first),
-                       std::make_move_iterator(__chunk_last - __old_chunk_size),
-                       std::make_move_iterator(__chunk_last - __old_chunk_size),
-                       std::make_move_iterator(__chunk_last),
-                       __to_first + (__chunk_first - __from_first),
-                       __comp);
-          });
-    };
+        __merge_chunks(__first, __values.get());
+      __objects_are_in_buffer = !__objects_are_in_buffer;
+    } while (__partitions.__chunk_count_ > 1);
 
-    if (__objects_are_in_buffer)
-      __merge_chunks(__values.get(), __first);
-    else
-      __merge_chunks(__first, __values.get());
-    __objects_are_in_buffer = !__objects_are_in_buffer;
-  } while (__partitions.__chunk_count_ > 1);
+    if (__objects_are_in_buffer) {
+      std::move(__values.get(), __values.get() + __size, __first);
+    }
 
-  if (__objects_are_in_buffer) {
-    std::move(__values.get(), __values.get() + __size, __first);
+    return __empty{};
   }
 
-  return __empty{};
-}
+  _LIBCPP_HIDE_FROM_ABI static void __cancel_execution() {}
 
-_LIBCPP_HIDE_FROM_ABI inline void __cancel_execution() {}
-
-} // namespace __libdispatch
-} // namespace __par_backend
+  static constexpr size_t __lane_size = 64;
+};
 
+} // namespace __pstl
 _LIBCPP_END_NAMESPACE_STD
 
 #endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && _LIBCPP_STD_VER >= 17
diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/merge.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/merge.h
index b0db70f58b2e..d03444790487 100644
--- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/merge.h
+++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/merge.h
@@ -13,6 +13,7 @@
 #include <__algorithm/pstl_backends/cpu_backends/backend.h>
 #include <__config>
 #include <__iterator/concepts.h>
+#include <__pstl/cpu_algos/cpu_traits.h>
 #include <__type_traits/is_execution_policy.h>
 #include <__utility/move.h>
 #include <optional>
@@ -45,7 +46,7 @@ _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator> __pstl_merge(
                 __has_random_access_iterator_category_or_concept<_ForwardIterator1>::value &&
                 __has_random_access_iterator_category_or_concept<_ForwardIterator2>::value &&
                 __has_random_access_iterator_category_or_concept<_ForwardOutIterator>::value) {
-    auto __res = __par_backend::__parallel_merge(
+    auto __res = __pstl::__cpu_traits<__cpu_backend_tag>::__parallel_merge(
         __first1,
         __last1,
         __first2,
diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/serial.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/serial.h
index afcc7ffb2661..c3d2905daed1 100644
--- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/serial.h
+++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/serial.h
@@ -11,6 +11,7 @@
 #define _LIBCPP___ALGORITHM_PSTL_BACKENDS_CPU_BACKENDS_SERIAL_H
 
 #include <__config>
+#include <__pstl/cpu_algos/cpu_traits.h>
 #include <__utility/empty.h>
 #include <__utility/move.h>
 #include <cstddef>
@@ -26,54 +27,55 @@ _LIBCPP_PUSH_MACROS
 #  include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
-
-namespace __par_backend {
-inline namespace __serial_cpu_backend {
-
-template <class _RandomAccessIterator, class _Fp>
-_LIBCPP_HIDE_FROM_ABI optional<__empty>
-__parallel_for(_RandomAccessIterator __first, _RandomAccessIterator __last, _Fp __f) {
-  __f(__first, __last);
-  return __empty{};
-}
-
-template <class _Index, class _UnaryOp, class _Tp, class _BinaryOp, class _Reduce>
-_LIBCPP_HIDE_FROM_ABI optional<_Tp>
-__parallel_transform_reduce(_Index __first, _Index __last, _UnaryOp, _Tp __init, _BinaryOp, _Reduce __reduce) {
-  return __reduce(std::move(__first), std::move(__last), std::move(__init));
-}
-
-template <class _RandomAccessIterator, class _Compare, class _LeafSort>
-_LIBCPP_HIDE_FROM_ABI optional<__empty> __parallel_stable_sort(
-    _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp, _LeafSort __leaf_sort) {
-  __leaf_sort(__first, __last, __comp);
-  return __empty{};
-}
-
-_LIBCPP_HIDE_FROM_ABI inline void __cancel_execution() {}
-
-template <class _RandomAccessIterator1,
-          class _RandomAccessIterator2,
-          class _RandomAccessIterator3,
-          class _Compare,
-          class _LeafMerge>
-_LIBCPP_HIDE_FROM_ABI optional<__empty> __parallel_merge(
-    _RandomAccessIterator1 __first1,
-    _RandomAccessIterator1 __last1,
-    _RandomAccessIterator2 __first2,
-    _RandomAccessIterator2 __last2,
-    _RandomAccessIterator3 __outit,
-    _Compare __comp,
-    _LeafMerge __leaf_merge) {
-  __leaf_merge(__first1, __last1, __first2, __last2, __outit, __comp);
-  return __empty{};
-}
-
-// TODO: Complete this list
-
-} // namespace __serial_cpu_backend
-} // namespace __par_backend
-
+namespace __pstl {
+
+struct __serial_backend_tag {};
+
+template <>
+struct __cpu_traits<__serial_backend_tag> {
+  template <class _RandomAccessIterator, class _Fp>
+  _LIBCPP_HIDE_FROM_ABI static optional<__empty>
+  __parallel_for(_RandomAccessIterator __first, _RandomAccessIterator __last, _Fp __f) {
+    __f(__first, __last);
+    return __empty{};
+  }
+
+  template <class _Index, class _UnaryOp, class _Tp, class _BinaryOp, class _Reduce>
+  _LIBCPP_HIDE_FROM_ABI static optional<_Tp>
+  __parallel_transform_reduce(_Index __first, _Index __last, _UnaryOp, _Tp __init, _BinaryOp, _Reduce __reduce) {
+    return __reduce(std::move(__first), std::move(__last), std::move(__init));
+  }
+
+  template <class _RandomAccessIterator, class _Compare, class _LeafSort>
+  _LIBCPP_HIDE_FROM_ABI static optional<__empty> __parallel_stable_sort(
+      _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp, _LeafSort __leaf_sort) {
+    __leaf_sort(__first, __last, __comp);
+    return __empty{};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static void __cancel_execution() {}
+
+  template <class _RandomAccessIterator1,
+            class _RandomAccessIterator2,
+            class _RandomAccessIterator3,
+            class _Compare,
+            class _LeafMerge>
+  _LIBCPP_HIDE_FROM_ABI static optional<__empty> __parallel_merge(
+      _RandomAccessIterator1 __first1,
+      _RandomAccessIterator1 __last1,
+      _RandomAccessIterator2 __first2,
+      _RandomAccessIterator2 __last2,
+      _RandomAccessIterator3 __outit,
+      _Compare __comp,
+      _LeafMerge __leaf_merge) {
+    __leaf_merge(__first1, __last1, __first2, __last2, __outit, __comp);
+    return __empty{};
+  }
+
+  static constexpr size_t __lane_size = 64;
+};
+
+} // namespace __pstl
 _LIBCPP_END_NAMESPACE_STD
 
 _LIBCPP_POP_MACROS
diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/stable_sort.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/stable_sort.h
index 34c423586c4b..ebfa0fc69147 100644
--- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/stable_sort.h
+++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/stable_sort.h
@@ -12,6 +12,7 @@
 #include <__algorithm/pstl_backends/cpu_backends/backend.h>
 #include <__algorithm/stable_sort.h>
 #include <__config>
+#include <__pstl/cpu_algos/cpu_traits.h>
 #include <__type_traits/is_execution_policy.h>
 #include <__utility/empty.h>
 #include <optional>
@@ -28,7 +29,7 @@ template <class _ExecutionPolicy, class _RandomAccessIterator, class _Comp>
 _LIBCPP_HIDE_FROM_ABI optional<__empty>
 __pstl_stable_sort(__cpu_backend_tag, _RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp) {
   if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy>) {
-    return __par_backend::__parallel_stable_sort(
+    return __pstl::__cpu_traits<__cpu_backend_tag>::__parallel_stable_sort(
         __first, __last, __comp, [](_RandomAccessIterator __g_first, _RandomAccessIterator __g_last, _Comp __g_comp) {
           std::stable_sort(__g_first, __g_last, __g_comp);
         });
diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/thread.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/thread.h
index eb11a961b760..8d1cb221c3d8 100644
--- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/thread.h
+++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/thread.h
@@ -11,6 +11,7 @@
 
 #include <__assert>
 #include <__config>
+#include <__pstl/cpu_algos/cpu_traits.h>
 #include <__utility/empty.h>
 #include <__utility/move.h>
 #include <cstddef>
@@ -29,52 +30,55 @@ _LIBCPP_PUSH_MACROS
 // by a proper implementation once the PSTL implementation is somewhat stable.
 
 _LIBCPP_BEGIN_NAMESPACE_STD
-
-namespace __par_backend {
-inline namespace __thread_cpu_backend {
-
-template <class _RandomAccessIterator, class _Fp>
-_LIBCPP_HIDE_FROM_ABI optional<__empty>
-__parallel_for(_RandomAccessIterator __first, _RandomAccessIterator __last, _Fp __f) {
-  __f(__first, __last);
-  return __empty{};
-}
-
-template <class _Index, class _UnaryOp, class _Tp, class _BinaryOp, class _Reduce>
-_LIBCPP_HIDE_FROM_ABI optional<_Tp>
-__parallel_transform_reduce(_Index __first, _Index __last, _UnaryOp, _Tp __init, _BinaryOp, _Reduce __reduce) {
-  return __reduce(std::move(__first), std::move(__last), std::move(__init));
-}
-
-template <class _RandomAccessIterator, class _Compare, class _LeafSort>
-_LIBCPP_HIDE_FROM_ABI optional<__empty> __parallel_stable_sort(
-    _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp, _LeafSort __leaf_sort) {
-  __leaf_sort(__first, __last, __comp);
-  return __empty{};
-}
-
-_LIBCPP_HIDE_FROM_ABI inline void __cancel_execution() {}
-
-template <class _RandomAccessIterator1,
-          class _RandomAccessIterator2,
-          class _RandomAccessIterator3,
-          class _Compare,
-          class _LeafMerge>
-_LIBCPP_HIDE_FROM_ABI optional<__empty> __parallel_merge(
-    _RandomAccessIterator1 __first1,
-    _RandomAccessIterator1 __last1,
-    _RandomAccessIterator2 __first2,
-    _RandomAccessIterator2 __last2,
-    _RandomAccessIterator3 __outit,
-    _Compare __comp,
-    _LeafMerge __leaf_merge) {
-  __leaf_merge(__first1, __last1, __first2, __last2, __outit, __comp);
-  return __empty{};
-}
-
-} // namespace __thread_cpu_backend
-} // namespace __par_backend
-
+namespace __pstl {
+
+struct __std_thread_backend_tag {};
+
+template <>
+struct __cpu_traits<__std_thread_backend_tag> {
+  template <class _RandomAccessIterator, class _Fp>
+  _LIBCPP_HIDE_FROM_ABI static optional<__empty>
+  __parallel_for(_RandomAccessIterator __first, _RandomAccessIterator __last, _Fp __f) {
+    __f(__first, __last);
+    return __empty{};
+  }
+
+  template <class _Index, class _UnaryOp, class _Tp, class _BinaryOp, class _Reduce>
+  _LIBCPP_HIDE_FROM_ABI static optional<_Tp>
+  __parallel_transform_reduce(_Index __first, _Index __last, _UnaryOp, _Tp __init, _BinaryOp, _Reduce __reduce) {
+    return __reduce(std::move(__first), std::move(__last), std::move(__init));
+  }
+
+  template <class _RandomAccessIterator, class _Compare, class _LeafSort>
+  _LIBCPP_HIDE_FROM_ABI static optional<__empty> __parallel_stable_sort(
+      _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp, _LeafSort __leaf_sort) {
+    __leaf_sort(__first, __last, __comp);
+    return __empty{};
+  }
+
+  _LIBCPP_HIDE_FROM_ABI static void __cancel_execution() {}
+
+  template <class _RandomAccessIterator1,
+            class _RandomAccessIterator2,
+            class _RandomAccessIterator3,
+            class _Compare,
+            class _LeafMerge>
+  _LIBCPP_HIDE_FROM_ABI static optional<__empty> __parallel_merge(
+      _RandomAccessIterator1 __first1,
+      _RandomAccessIterator1 __last1,
+      _RandomAccessIterator2 __first2,
+      _RandomAccessIterator2 __last2,
+      _RandomAccessIterator3 __outit,
+      _Compare __comp,
+      _LeafMerge __leaf_merge) {
+    __leaf_merge(__first1, __last1, __first2, __last2, __outit, __comp);
+    return __empty{};
+  }
+
+  static constexpr size_t __lane_size = 64;
+};
+
+} // namespace __pstl
 _LIBCPP_END_NAMESPACE_STD
 
 #endif // !defined(_LIBCPP_HAS_NO_INCOMPLETE_PSTL) && && _LIBCPP_STD_VER >= 17
diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform.h
index fdf1a2e78dad..d4c383997a67 100644
--- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform.h
+++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform.h
@@ -14,6 +14,7 @@
 #include <__config>
 #include <__iterator/concepts.h>
 #include <__iterator/iterator_traits.h>
+#include <__pstl/cpu_algos/cpu_traits.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/is_execution_policy.h>
 #include <__type_traits/remove_cvref.h>
@@ -49,7 +50,7 @@ _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator> __pstl_transform(
   if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> &&
                 __has_random_access_iterator_category_or_concept<_ForwardIterator>::value &&
                 __has_random_access_iterator_category_or_concept<_ForwardOutIterator>::value) {
-    std::__par_backend::__parallel_for(
+    __pstl::__cpu_traits<__cpu_backend_tag>::__parallel_for(
         __first, __last, [__op, __first, __result](_ForwardIterator __brick_first, _ForwardIterator __brick_last) {
           auto __res = std::__pstl_transform<__remove_parallel_policy_t<_ExecutionPolicy>>(
               __cpu_backend_tag{}, __brick_first, __brick_last, __result + (__brick_first - __first), __op);
@@ -97,7 +98,7 @@ _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator> __pstl_transform(
                 __has_random_access_iterator_category_or_concept<_ForwardIterator1>::value &&
                 __has_random_access_iterator_category_or_concept<_ForwardIterator2>::value &&
                 __has_random_access_iterator_category_or_concept<_ForwardOutIterator>::value) {
-    auto __res = std::__par_backend::__parallel_for(
+    auto __res = __pstl::__cpu_traits<__cpu_backend_tag>::__parallel_for(
         __first1,
         __last1,
         [__op, __first1, __first2, __result](_ForwardIterator1 __brick_first, _ForwardIterator1 __brick_last) {
diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform_reduce.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform_reduce.h
index 376abd39fa36..956c7d6a88ce 100644
--- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform_reduce.h
+++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform_reduce.h
@@ -14,6 +14,7 @@
 #include <__iterator/concepts.h>
 #include <__iterator/iterator_traits.h>
 #include <__numeric/transform_reduce.h>
+#include <__pstl/cpu_algos/cpu_traits.h>
 #include <__type_traits/desugars_to.h>
 #include <__type_traits/is_arithmetic.h>
 #include <__type_traits/is_execution_policy.h>
@@ -32,7 +33,8 @@ _LIBCPP_PUSH_MACROS
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <typename _DifferenceType,
+template <typename _Backend,
+          typename _DifferenceType,
           typename _Tp,
           typename _BinaryOperation,
           typename _UnaryOperation,
@@ -48,7 +50,8 @@ __simd_transform_reduce(_DifferenceType __n, _Tp __init, _BinaryOperation, _Unar
   return __init;
 }
 
-template <typename _Size,
+template <typename _Backend,
+          typename _Size,
           typename _Tp,
           typename _BinaryOperation,
           typename _UnaryOperation,
@@ -58,7 +61,8 @@ template <typename _Size,
                         int>    = 0>
 _LIBCPP_HIDE_FROM_ABI _Tp
 __simd_transform_reduce(_Size __n, _Tp __init, _BinaryOperation __binary_op, _UnaryOperation __f) noexcept {
-  const _Size __block_size = __lane_size / sizeof(_Tp);
+  constexpr size_t __lane_size = __pstl::__cpu_traits<_Backend>::__lane_size;
+  const _Size __block_size     = __lane_size / sizeof(_Tp);
   if (__n > 2 * __block_size && __block_size > 1) {
     alignas(__lane_size) char __lane_buffer[__lane_size];
     _Tp* __lane = reinterpret_cast<_Tp*>(__lane_buffer);
@@ -116,7 +120,7 @@ _LIBCPP_HIDE_FROM_ABI optional<_Tp> __pstl_transform_reduce(
   if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> &&
                 __has_random_access_iterator_category_or_concept<_ForwardIterator1>::value &&
                 __has_random_access_iterator_category_or_concept<_ForwardIterator2>::value) {
-    return __par_backend::__parallel_transform_reduce(
+    return __pstl::__cpu_traits<__cpu_backend_tag>::__parallel_transform_reduce(
         __first1,
         std::move(__last1),
         [__first1, __first2, __transform](_ForwardIterator1 __iter) {
@@ -138,7 +142,7 @@ _LIBCPP_HIDE_FROM_ABI optional<_Tp> __pstl_transform_reduce(
   } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> &&
                        __has_random_access_iterator_category_or_concept<_ForwardIterator1>::value &&
                        __has_random_access_iterator_category_or_concept<_ForwardIterator2>::value) {
-    return std::__simd_transform_reduce(
+    return std::__simd_transform_reduce<__cpu_backend_tag>(
         __last1 - __first1, std::move(__init), std::move(__reduce), [&](__iter_diff_t<_ForwardIterator1> __i) {
           return __transform(__first1[__i], __first2[__i]);
         });
@@ -163,7 +167,7 @@ _LIBCPP_HIDE_FROM_ABI optional<_Tp> __pstl_transform_reduce(
     _UnaryOperation __transform) {
   if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> &&
                 __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) {
-    return __par_backend::__parallel_transform_reduce(
+    return __pstl::__cpu_traits<__cpu_backend_tag>::__parallel_transform_reduce(
         std::move(__first),
         std::move(__last),
         [__transform](_ForwardIterator __iter) { return __transform(*__iter); },
@@ -182,7 +186,7 @@ _LIBCPP_HIDE_FROM_ABI optional<_Tp> __pstl_transform_reduce(
         });
   } else if constexpr (__is_unsequenced_execution_policy_v<_ExecutionPolicy> &&
                        __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) {
-    return std::__simd_transform_reduce(
+    return std::__simd_transform_reduce<__cpu_backend_tag>(
         __last - __first,
         std::move(__init),
         std::move(__reduce),
diff --git a/libcxx/include/__pstl/cpu_algos/cpu_traits.h b/libcxx/include/__pstl/cpu_algos/cpu_traits.h
new file mode 100644
index 000000000000..2f0db46e9be8
--- /dev/null
+++ b/libcxx/include/__pstl/cpu_algos/cpu_traits.h
@@ -0,0 +1,87 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___PSTL_CPU_ALGOS_CPU_TRAITS_H
+#define _LIBCPP___PSTL_CPU_ALGOS_CPU_TRAITS_H
+
+#include <__config>
+#include <cstddef>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+namespace __pstl {
+
+// __cpu_traits
+//
+// This traits class encapsulates the basis operations for a CPU-based implementation of the PSTL.
+// All the operations in the PSTL can be implemented from these basis operations, so a pure CPU backend
+// only needs to customize these traits in order to get an implementation of the whole PSTL.
+//
+// Basis operations
+// ================
+//
+//  template <class _RandomAccessIterator, class _Functor>
+//  optional<__empty> __parallel_for(_RandomAccessIterator __first, _RandomAccessIterator __last, _Functor __func);
+//    - __func must take a subrange of [__first, __last) that should be executed in serial
+//
+//  template <class _Iterator, class _UnaryOp, class _Tp, class _BinaryOp, class _Reduction>
+//  optional<_Tp> __parallel_transform_reduce(_Iterator __first, _Iterator __last, _UnaryOp, _Tp __init, _BinaryOp,
+//  _Reduction);
+//
+//  template <class _RandomAccessIterator1,
+//            class _RandomAccessIterator2,
+//            class _RandomAccessIterator3,
+//            class _Compare,
+//            class _LeafMerge>
+//  optional<_RandomAccessIterator3> __parallel_merge(_RandomAccessIterator1 __first1,
+//                                                    _RandomAccessIterator1 __last1,
+//                                                    _RandomAccessIterator2 __first2,
+//                                                    _RandomAccessIterator2 __last2,
+//                                                    _RandomAccessIterator3 __outit,
+//                                                    _Compare __comp,
+//                                                    _LeafMerge __leaf_merge);
+//
+//  template <class _RandomAccessIterator, class _Comp, class _LeafSort>
+//  optional<__empty> __parallel_stable_sort(_RandomAccessIterator __first,
+//                                           _RandomAccessIterator __last,
+//                                           _Comp __comp,
+//                                           _LeafSort __leaf_sort);
+//
+//   void __cancel_execution();
+//      Cancel the execution of other jobs - they aren't needed anymore. This is not a binding request,
+//      some backends may not actually be able to cancel jobs.
+//
+//   constexpr size_t __lane_size;
+//      Size of SIMD lanes.
+//      TODO: Merge this with __native_vector_size from __algorithm/simd_utils.h
+//
+//
+// Exception handling
+// ==================
+//
+// CPU backends are expected to report errors (i.e. failure to allocate) by returning a disengaged `optional` from their
+// implementation. Exceptions shouldn't be used to report an internal failure-to-allocate, since all exceptions are
+// turned into a program termination at the front-end level. When a backend returns a disengaged `optional` to the
+// frontend, the frontend will turn that into a call to `std::__throw_bad_alloc();` to report the internal failure to
+// the user.
+
+template <class _Backend>
+struct __cpu_traits;
+
+} // namespace __pstl
+_LIBCPP_END_NAMESPACE_STD
+
+_LIBCPP_POP_MACROS
+
+#endif // _LIBCPP___PSTL_CPU_ALGOS_CPU_TRAITS_H
diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap
index ccee7029824e..7312a744ef67 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap
@@ -1613,6 +1613,8 @@ module std_private_numeric_transform_exclusive_scan [system] { header "__numeric
 module std_private_numeric_transform_inclusive_scan [system] { header "__numeric/transform_inclusive_scan.h" }
 module std_private_numeric_transform_reduce         [system] { header "__numeric/transform_reduce.h" }
 
+module std_private_pstl_cpu_algos_cpu_traits [system] { header "__pstl/cpu_algos/cpu_traits.h" }
+
 module std_private_queue_fwd [system] { header "__fwd/queue.h" }
 
 module std_private_random_bernoulli_distribution          [system] { header "__random/bernoulli_distribution.h" }
diff --git a/libcxx/src/pstl/libdispatch.cpp b/libcxx/src/pstl/libdispatch.cpp
index 52d4afbcce6e..d997a9c73463 100644
--- a/libcxx/src/pstl/libdispatch.cpp
+++ b/libcxx/src/pstl/libdispatch.cpp
@@ -12,8 +12,7 @@
 #include <dispatch/dispatch.h>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
-
-namespace __par_backend::inline __libdispatch {
+namespace __pstl::__libdispatch {
 
 void __dispatch_apply(size_t chunk_count, void* context, void (*func)(void* context, size_t chunk)) noexcept {
   ::dispatch_apply_f(chunk_count, DISPATCH_APPLY_AUTO, context, func);
@@ -29,7 +28,5 @@ __chunk_partitions __partition_chunks(ptrdiff_t element_count) noexcept {
   return partitions;
 }
 
-// NOLINTNEXTLINE(llvm-namespace-comment) // This is https://llvm.org/PR56804
-} // namespace __par_backend::inline __libdispatch
-
+} // namespace __pstl::__libdispatch
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/test/libcxx/algorithms/pstl.libdispatch.chunk_partitions.pass.cpp b/libcxx/test/libcxx/algorithms/pstl.libdispatch.chunk_partitions.pass.cpp
index 554924a0179d..8c7016a80b81 100644
--- a/libcxx/test/libcxx/algorithms/pstl.libdispatch.chunk_partitions.pass.cpp
+++ b/libcxx/test/libcxx/algorithms/pstl.libdispatch.chunk_partitions.pass.cpp
@@ -18,21 +18,21 @@
 
 int main(int, char**) {
   {
-    auto chunks = std::__par_backend::__libdispatch::__partition_chunks(0);
+    auto chunks = std::__pstl::__libdispatch::__partition_chunks(0);
     assert(chunks.__chunk_count_ == 1);
     assert(chunks.__first_chunk_size_ == 0);
     assert(chunks.__chunk_size_ == 0);
   }
 
   {
-    auto chunks = std::__par_backend::__libdispatch::__partition_chunks(1);
+    auto chunks = std::__pstl::__libdispatch::__partition_chunks(1);
     assert(chunks.__chunk_count_ == 1);
     assert(chunks.__first_chunk_size_ == 1);
     assert(chunks.__chunk_size_ == 1);
   }
 
   for (std::ptrdiff_t i = 2; i != 2ll << 20; ++i) {
-    auto chunks = std::__par_backend::__libdispatch::__partition_chunks(i);
+    auto chunks = std::__pstl::__libdispatch::__partition_chunks(i);
     assert(chunks.__chunk_count_ >= 1);
     assert(chunks.__chunk_count_ <= i);
     assert((chunks.__chunk_count_ - 1) * chunks.__chunk_size_ + chunks.__first_chunk_size_ == i);
diff --git a/libcxx/utils/generate_iwyu_mapping.py b/libcxx/utils/generate_iwyu_mapping.py
index 2265438ab49c..33607f199684 100644
--- a/libcxx/utils/generate_iwyu_mapping.py
+++ b/libcxx/utils/generate_iwyu_mapping.py
@@ -10,6 +10,7 @@ def IWYU_mapping(header: str) -> typing.Optional[typing.List[str]]:
     ignore = [
         "__debug_utils/.+",
         "__fwd/get[.]h",
+        "__pstl/.+",
         "__support/.+",
         "__utility/private_constructor_tag.h",
     ]
-- 
GitLab


From 7e4c6e98fa05f5c3bf14f96365ae74a8d12c6257 Mon Sep 17 00:00:00 2001
From: Jonas Paulsson <paulson1@linux.ibm.com>
Date: Mon, 15 Apr 2024 16:32:14 +0200
Subject: [PATCH 248/464] [SystemZ] Bugfix in getDemandedSrcElements().
 (#88623)

For the intrinsic s390_vperm, all of the elements are demanded, so use
an APInt with the value of '-1' for them (not '1').

Fixes https://github.com/llvm/llvm-project/issues/88397
---
 .../Target/SystemZ/SystemZISelLowering.cpp    |  2 +-
 .../SystemZ/knownbits-intrinsics-binop.ll     | 19 +++++++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 3b3057f8fe96..5c2579f3bf18 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -7818,7 +7818,7 @@ static APInt getDemandedSrcElements(SDValue Op, const APInt &DemandedElts,
       break;
     }
     case Intrinsic::s390_vperm:
-      SrcDemE = APInt(NumElts, 1);
+      SrcDemE = APInt(NumElts, -1);
       break;
     default:
       llvm_unreachable("Unhandled intrinsic.");
diff --git a/llvm/test/CodeGen/SystemZ/knownbits-intrinsics-binop.ll b/llvm/test/CodeGen/SystemZ/knownbits-intrinsics-binop.ll
index 3bcbbb45581f..b855d0193478 100644
--- a/llvm/test/CodeGen/SystemZ/knownbits-intrinsics-binop.ll
+++ b/llvm/test/CodeGen/SystemZ/knownbits-intrinsics-binop.ll
@@ -458,3 +458,22 @@ define <16 x i8> @f30() {
                                i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
   ret <16 x i8> %res
 }
+
+; Test VPERM with various constant operands.
+define i32 @f31() {
+; CHECK-LABEL: f31:
+; CHECK-LABEL: # %bb.0:
+; CHECK-NEXT: larl %r1, .LCPI31_0
+; CHECK-NEXT: vl %v0, 0(%r1), 3
+; CHECK-NEXT: larl %r1, .LCPI31_1
+; CHECK-NEXT: vl %v1, 0(%r1), 3
+; CHECK-NEXT: vperm %v0, %v1, %v1, %v0
+; CHECK-NEXT: vlgvb %r2, %v0, 0
+; CHECK-NEXT: nilf %r2, 7
+; CHECK-NEXT:                           # kill: def $r2l killed $r2l killed $r2d
+; CHECK-NEXT: br %r14
+  %P = tail call <16 x i8> @llvm.s390.vperm(<16 x i8> <i8 0, i8 1, i8 1, i8 2, i8 1, i8 2, i8 2, i8 3, i8 1, i8 2, i8 2, i8 3, i8 2, i8 3, i8 3, i8 4>, <16 x i8> <i8 0, i8 1, i8 1, i8 2, i8 1, i8 2, i8 2, i8 3, i8 1, i8 2, i8 2, i8 3, i8 2, i8 3, i8 3, i8 4>, <16 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 0>)
+  %E = extractelement <16 x i8> %P, i64 0
+  %res = zext i8 %E to i32
+  ret i32 %res
+}
-- 
GitLab


From a2691959cd23de9505243e9396897a3415606689 Mon Sep 17 00:00:00 2001
From: whisperity <whisperity@gmail.com>
Date: Mon, 15 Apr 2024 16:32:52 +0200
Subject: [PATCH 249/464] [clang-tidy][NFC] Fix `linuxkernel-must-check-errs`
 documentation file name (#88655)

The check was originally introduced in commit
fc8c65b2e11d21b1c0ce070cc0a4ee031a542dae with conflicting names, likely as a
result of a partial rename, with the name in the documentation
(`linuxkernel-must-use-errs`) not matching the name of the check as configured
by users.
---
 .../clang-tidy/linuxkernel/MustCheckErrsCheck.h          | 9 +--------
 clang-tools-extra/docs/ReleaseNotes.rst                  | 4 ++++
 .../{must-use-errs.rst => must-check-errs.rst}           | 6 +++---
 clang-tools-extra/docs/clang-tidy/checks/list.rst        | 2 +-
 4 files changed, 9 insertions(+), 12 deletions(-)
 rename clang-tools-extra/docs/clang-tidy/checks/linuxkernel/{must-use-errs.rst => must-check-errs.rst} (88%)

diff --git a/clang-tools-extra/clang-tidy/linuxkernel/MustCheckErrsCheck.h b/clang-tools-extra/clang-tidy/linuxkernel/MustCheckErrsCheck.h
index f08fed479839..7406aaead836 100644
--- a/clang-tools-extra/clang-tidy/linuxkernel/MustCheckErrsCheck.h
+++ b/clang-tools-extra/clang-tidy/linuxkernel/MustCheckErrsCheck.h
@@ -17,15 +17,8 @@ namespace clang::tidy::linuxkernel {
 /// linux/err.h. Also checks to see if code uses the results from functions that
 /// directly return a value from one of these error functions.
 ///
-/// This is important in the Linux kernel because ERR_PTR, PTR_ERR, IS_ERR,
-/// IS_ERR_OR_NULL, ERR_CAST, and PTR_ERR_OR_ZERO return values must be checked,
-/// since positive pointers and negative error codes are being used in the same
-/// context. These functions are marked with
-/// __attribute__((warn_unused_result)), but some kernel versions do not have
-/// this warning enabled for clang.
-///
 /// For the user-facing documentation see:
-/// http://clang.llvm.org/extra/clang-tidy/checks/linuxkernel/must-use-errs.html
+/// http://clang.llvm.org/extra/clang-tidy/checks/linuxkernel/must-check-errs.html
 class MustCheckErrsCheck : public ClangTidyCheck {
 public:
   MustCheckErrsCheck(StringRef Name, ClangTidyContext *Context)
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index de0a7b79bb2c..7095c564444f 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -303,6 +303,10 @@ Miscellaneous
   ``--format`` option is specified. Now :program:`clang-apply-replacements`
   applies formatting only with the option.
 
+- Fixed the :doc:`linuxkernel-must-check-errs
+  <clang-tidy/checks/linuxkernel/must-check-errs>` documentation to consistently
+  use the check's proper name.
+
 Improvements to include-fixer
 -----------------------------
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/linuxkernel/must-use-errs.rst b/clang-tools-extra/docs/clang-tidy/checks/linuxkernel/must-check-errs.rst
similarity index 88%
rename from clang-tools-extra/docs/clang-tidy/checks/linuxkernel/must-use-errs.rst
rename to clang-tools-extra/docs/clang-tidy/checks/linuxkernel/must-check-errs.rst
index 8a8542688098..cef5a70db309 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/linuxkernel/must-use-errs.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/linuxkernel/must-check-errs.rst
@@ -1,7 +1,7 @@
-.. title:: clang-tidy - linuxkernel-must-use-errs
+.. title:: clang-tidy - linuxkernel-must-check-errs
 
-linuxkernel-must-use-errs
-=========================
+linuxkernel-must-check-errs
+===========================
 
 Checks Linux kernel code to see if it uses the results from the functions in
 ``linux/err.h``. Also checks to see if code uses the results from functions that
diff --git a/clang-tools-extra/docs/clang-tidy/checks/list.rst b/clang-tools-extra/docs/clang-tidy/checks/list.rst
index 188a42bfddd3..8bc46acad56c 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/list.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/list.rst
@@ -233,7 +233,7 @@ Clang-Tidy Checks
    :doc:`hicpp-multiway-paths-covered <hicpp/multiway-paths-covered>`,
    :doc:`hicpp-no-assembler <hicpp/no-assembler>`,
    :doc:`hicpp-signed-bitwise <hicpp/signed-bitwise>`,
-   :doc:`linuxkernel-must-use-errs <linuxkernel/must-use-errs>`,
+   :doc:`linuxkernel-must-check-errs <linuxkernel/must-check-errs>`,
    :doc:`llvm-header-guard <llvm/header-guard>`,
    :doc:`llvm-include-order <llvm/include-order>`, "Yes"
    :doc:`llvm-namespace-comment <llvm/namespace-comment>`,
-- 
GitLab


From a06073f91e7bbbb532e68bbf6b903c2f5051f4c2 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Mon, 15 Apr 2024 10:45:26 -0400
Subject: [PATCH 250/464] [libc++] Add a utility to check whether a range is
 valid (#87665)

In the future, this utility could be made to also work with iterators,
including bounded iterators. We could also query the ASAN runtime for
this information when it's around.
---
 libcxx/include/CMakeLists.txt                 |  1 +
 .../include/__utility/is_pointer_in_range.h   |  6 +-
 libcxx/include/__utility/is_valid_range.h     | 37 ++++++++++
 libcxx/include/libcxx.imp                     |  1 +
 libcxx/include/module.modulemap               |  1 +
 .../libcxx/utilities/is_valid_range.pass.cpp  | 68 +++++++++++++++++++
 6 files changed, 111 insertions(+), 3 deletions(-)
 create mode 100644 libcxx/include/__utility/is_valid_range.h
 create mode 100644 libcxx/test/libcxx/utilities/is_valid_range.pass.cpp

diff --git a/libcxx/include/CMakeLists.txt b/libcxx/include/CMakeLists.txt
index 148a51a216ed..a2af1d9915be 100644
--- a/libcxx/include/CMakeLists.txt
+++ b/libcxx/include/CMakeLists.txt
@@ -860,6 +860,7 @@ set(files
   __utility/in_place.h
   __utility/integer_sequence.h
   __utility/is_pointer_in_range.h
+  __utility/is_valid_range.h
   __utility/move.h
   __utility/no_destroy.h
   __utility/pair.h
diff --git a/libcxx/include/__utility/is_pointer_in_range.h b/libcxx/include/__utility/is_pointer_in_range.h
index 68cdfea6f945..9eee8bf811c6 100644
--- a/libcxx/include/__utility/is_pointer_in_range.h
+++ b/libcxx/include/__utility/is_pointer_in_range.h
@@ -17,6 +17,7 @@
 #include <__type_traits/is_constant_evaluated.h>
 #include <__type_traits/void_t.h>
 #include <__utility/declval.h>
+#include <__utility/is_valid_range.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
@@ -34,16 +35,15 @@ struct __is_less_than_comparable<_Tp, _Up, __void_t<decltype(std::declval<_Tp>()
 template <class _Tp, class _Up, __enable_if_t<__is_less_than_comparable<const _Tp*, const _Up*>::value, int> = 0>
 _LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_SANITIZE("address") bool __is_pointer_in_range(
     const _Tp* __begin, const _Tp* __end, const _Up* __ptr) {
-  if (__libcpp_is_constant_evaluated()) {
-    _LIBCPP_ASSERT_VALID_INPUT_RANGE(__builtin_constant_p(__begin <= __end), "__begin and __end do not form a range");
+  _LIBCPP_ASSERT_VALID_INPUT_RANGE(std::__is_valid_range(__begin, __end), "[__begin, __end) is not a valid range");
 
+  if (__libcpp_is_constant_evaluated()) {
     // If this is not a constant during constant evaluation we know that __ptr is not part of the allocation where
     // [__begin, __end) is.
     if (!__builtin_constant_p(__begin <= __ptr && __ptr < __end))
       return false;
   }
 
-  // Checking this for unrelated pointers is technically UB, but no compiler optimizes based on it (currently).
   return !__less<>()(__ptr, __begin) && __less<>()(__ptr, __end);
 }
 
diff --git a/libcxx/include/__utility/is_valid_range.h b/libcxx/include/__utility/is_valid_range.h
new file mode 100644
index 000000000000..7286662dbf30
--- /dev/null
+++ b/libcxx/include/__utility/is_valid_range.h
@@ -0,0 +1,37 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP___UTILITY_IS_VALID_RANGE_H
+#define _LIBCPP___UTILITY_IS_VALID_RANGE_H
+
+#include <__algorithm/comp.h>
+#include <__config>
+#include <__type_traits/is_constant_evaluated.h>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+#  pragma GCC system_header
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+template <class _Tp>
+_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_SANITIZE("address") bool
+__is_valid_range(const _Tp* __first, const _Tp* __last) {
+  if (__libcpp_is_constant_evaluated()) {
+    // If this is not a constant during constant evaluation, that is because __first and __last are not
+    // part of the same allocation. If they are part of the same allocation, we must still make sure they
+    // are ordered properly.
+    return __builtin_constant_p(__first <= __last) && __first <= __last;
+  }
+
+  return !__less<>()(__last, __first);
+}
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP___UTILITY_IS_VALID_RANGE_H
diff --git a/libcxx/include/libcxx.imp b/libcxx/include/libcxx.imp
index 6c77ba8343c6..8820fb8c0936 100644
--- a/libcxx/include/libcxx.imp
+++ b/libcxx/include/libcxx.imp
@@ -853,6 +853,7 @@
   { include: [ "<__utility/in_place.h>", "private", "<utility>", "public" ] },
   { include: [ "<__utility/integer_sequence.h>", "private", "<utility>", "public" ] },
   { include: [ "<__utility/is_pointer_in_range.h>", "private", "<utility>", "public" ] },
+  { include: [ "<__utility/is_valid_range.h>", "private", "<utility>", "public" ] },
   { include: [ "<__utility/move.h>", "private", "<utility>", "public" ] },
   { include: [ "<__utility/no_destroy.h>", "private", "<utility>", "public" ] },
   { include: [ "<__utility/pair.h>", "private", "<utility>", "public" ] },
diff --git a/libcxx/include/module.modulemap b/libcxx/include/module.modulemap
index 7312a744ef67..ce133e471deb 100644
--- a/libcxx/include/module.modulemap
+++ b/libcxx/include/module.modulemap
@@ -2075,6 +2075,7 @@ module std_private_utility_forward_like           [system] { header "__utility/f
 module std_private_utility_in_place               [system] { header "__utility/in_place.h" }
 module std_private_utility_integer_sequence       [system] { header "__utility/integer_sequence.h" }
 module std_private_utility_is_pointer_in_range    [system] { header "__utility/is_pointer_in_range.h" }
+module std_private_utility_is_valid_range         [system] { header "__utility/is_valid_range.h" }
 module std_private_utility_move                   [system] {
   header "__utility/move.h"
   export std_private_type_traits_is_copy_constructible
diff --git a/libcxx/test/libcxx/utilities/is_valid_range.pass.cpp b/libcxx/test/libcxx/utilities/is_valid_range.pass.cpp
new file mode 100644
index 000000000000..345e2feeda81
--- /dev/null
+++ b/libcxx/test/libcxx/utilities/is_valid_range.pass.cpp
@@ -0,0 +1,68 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <__utility/is_valid_range.h>
+#include <cassert>
+
+#include "test_macros.h"
+
+template <class T, class TQualified>
+TEST_CONSTEXPR_CXX14 void check_type() {
+  {
+    // We need to ensure that the addresses of i and j are ordered as &i < &j for
+    // the test below to work portably, so we define them in a struct.
+    struct {
+      T i = 0;
+      T j = 0;
+    } storage;
+    assert(std::__is_valid_range(static_cast<TQualified*>(&storage.i), static_cast<TQualified*>(&storage.i)));
+    assert(std::__is_valid_range(static_cast<TQualified*>(&storage.i), static_cast<TQualified*>(&storage.i + 1)));
+
+    assert(!std::__is_valid_range(static_cast<TQualified*>(&storage.j), static_cast<TQualified*>(&storage.i)));
+    assert(!std::__is_valid_range(static_cast<TQualified*>(&storage.i + 1), static_cast<TQualified*>(&storage.i)));
+
+    // We detect this as being a valid range even though it is not really valid.
+    assert(std::__is_valid_range(static_cast<TQualified*>(&storage.i), static_cast<TQualified*>(&storage.j)));
+  }
+
+  {
+    T arr[3] = {1, 2, 3};
+    assert(std::__is_valid_range(static_cast<TQualified*>(&arr[0]), static_cast<TQualified*>(&arr[0])));
+    assert(std::__is_valid_range(static_cast<TQualified*>(&arr[0]), static_cast<TQualified*>(&arr[1])));
+    assert(std::__is_valid_range(static_cast<TQualified*>(&arr[0]), static_cast<TQualified*>(&arr[2])));
+
+    assert(!std::__is_valid_range(static_cast<TQualified*>(&arr[1]), static_cast<TQualified*>(&arr[0])));
+    assert(!std::__is_valid_range(static_cast<TQualified*>(&arr[2]), static_cast<TQualified*>(&arr[0])));
+  }
+
+#if TEST_STD_VER >= 20
+  {
+    T* arr = new int[4]{1, 2, 3, 4};
+    assert(std::__is_valid_range(static_cast<TQualified*>(arr), static_cast<TQualified*>(arr + 4)));
+    delete[] arr;
+  }
+#endif
+}
+
+TEST_CONSTEXPR_CXX14 bool test() {
+  check_type<int, int>();
+  check_type<int, int const>();
+  check_type<int, int volatile>();
+  check_type<int, int const volatile>();
+
+  return true;
+}
+
+int main(int, char**) {
+  test();
+#if TEST_STD_VER >= 14
+  static_assert(test(), "");
+#endif
+
+  return 0;
+}
-- 
GitLab


From 7b9b28dbd2938877a94737e28b7eb9e3cdd82755 Mon Sep 17 00:00:00 2001
From: yronglin <yronglin777@gmail.com>
Date: Mon, 15 Apr 2024 23:52:26 +0800
Subject: [PATCH 251/464] [Clang] Refactor implementation of "Lifetime
 extension in range-based for loops" (#87930)

This PR remove `InMaterializeTemporaryObjectContext` , because it's
redundant, materialize non-cv void prvalue temporaries in discarded
expressions can only appear under lifetime-extension context.

Signed-off-by: yronglin <yronglin777@gmail.com>
---
 clang/include/clang/Sema/Sema.h               | 47 -------------------
 clang/lib/Parse/ParseDecl.cpp                 |  4 --
 clang/lib/Sema/SemaExpr.cpp                   |  7 ++-
 clang/lib/Sema/SemaExprCXX.cpp                |  2 +-
 .../lib/Sema/SemaTemplateInstantiateDecl.cpp  |  1 -
 clang/lib/Sema/TreeTransform.h                |  5 --
 6 files changed, 4 insertions(+), 62 deletions(-)

diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index c6035445e901..d93ac7863b72 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -5099,34 +5099,6 @@ public:
     /// example, in a for-range initializer).
     bool InLifetimeExtendingContext = false;
 
-    /// Whether we are currently in a context in which all temporaries must be
-    /// materialized.
-    ///
-    /// [class.temporary]/p2:
-    /// The materialization of a temporary object is generally delayed as long
-    /// as possible in order to avoid creating unnecessary temporary objects.
-    ///
-    /// Temporary objects are materialized:
-    ///   (2.1) when binding a reference to a prvalue ([dcl.init.ref],
-    ///   [expr.type.conv], [expr.dynamic.cast], [expr.static.cast],
-    ///   [expr.const.cast], [expr.cast]),
-    ///
-    ///   (2.2) when performing member access on a class prvalue ([expr.ref],
-    ///   [expr.mptr.oper]),
-    ///
-    ///   (2.3) when performing an array-to-pointer conversion or subscripting
-    ///   on an array prvalue ([conv.array], [expr.sub]),
-    ///
-    ///   (2.4) when initializing an object of type
-    ///   std​::​initializer_list<T> from a braced-init-list
-    ///   ([dcl.init.list]),
-    ///
-    ///   (2.5) for certain unevaluated operands ([expr.typeid], [expr.sizeof])
-    ///
-    ///   (2.6) when a prvalue that has type other than cv void appears as a
-    ///   discarded-value expression ([expr.context]).
-    bool InMaterializeTemporaryObjectContext = false;
-
     // When evaluating immediate functions in the initializer of a default
     // argument or default member initializer, this is the declaration whose
     // default initializer is being evaluated and the location of the call
@@ -6398,19 +6370,6 @@ public:
     }
   }
 
-  /// keepInMaterializeTemporaryObjectContext - Pull down
-  /// InMaterializeTemporaryObjectContext flag from previous context.
-  void keepInMaterializeTemporaryObjectContext() {
-    if (ExprEvalContexts.size() > 2 &&
-        ExprEvalContexts[ExprEvalContexts.size() - 2]
-            .InMaterializeTemporaryObjectContext) {
-      auto &LastRecord = ExprEvalContexts.back();
-      auto &PrevRecord = ExprEvalContexts[ExprEvalContexts.size() - 2];
-      LastRecord.InMaterializeTemporaryObjectContext =
-          PrevRecord.InMaterializeTemporaryObjectContext;
-    }
-  }
-
   DefaultedComparisonKind getDefaultedComparisonKind(const FunctionDecl *FD) {
     return getDefaultedFunctionKind(FD).asComparison();
   }
@@ -6554,12 +6513,6 @@ public:
   /// used in initializer of the field.
   llvm::MapVector<FieldDecl *, DeleteLocs> DeleteExprs;
 
-  bool isInMaterializeTemporaryObjectContext() const {
-    assert(!ExprEvalContexts.empty() &&
-           "Must be in an expression evaluation context");
-    return ExprEvalContexts.back().InMaterializeTemporaryObjectContext;
-  }
-
   ParsedType getInheritingConstructorName(CXXScopeSpec &SS,
                                           SourceLocation NameLoc,
                                           const IdentifierInfo &Name);
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index a990ef398c82..2b934234b7cf 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -2380,10 +2380,6 @@ Parser::DeclGroupPtrTy Parser::ParseDeclGroup(ParsingDeclSpec &DS,
       if (getLangOpts().CPlusPlus23) {
         auto &LastRecord = Actions.ExprEvalContexts.back();
         LastRecord.InLifetimeExtendingContext = true;
-
-        // Materialize non-`cv void` prvalue temporaries in discarded
-        // expressions. These materialized temporaries may be lifetime-extented.
-        LastRecord.InMaterializeTemporaryObjectContext = true;
       }
 
       if (getLangOpts().OpenMP)
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index c5395fb2068a..24f354f1c724 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -6314,7 +6314,6 @@ ExprResult Sema::BuildCXXDefaultArgExpr(SourceLocation CallLoc,
       // Pass down lifetime extending flag, and collect temporaries in
       // CreateMaterializeTemporaryExpr when we rewrite the call argument.
       keepInLifetimeExtendingContext();
-      keepInMaterializeTemporaryObjectContext();
       EnsureImmediateInvocationInDefaultArgs Immediate(*this);
       ExprResult Res;
       runWithSufficientStackSpace(CallLoc, [&] {
@@ -18679,9 +18678,9 @@ void Sema::PopExpressionEvaluationContext() {
   // Append the collected materialized temporaries into previous context before
   // exit if the previous also is a lifetime extending context.
   auto &PrevRecord = ExprEvalContexts[ExprEvalContexts.size() - 2];
-  if (getLangOpts().CPlusPlus23 && isInLifetimeExtendingContext() &&
-      PrevRecord.InLifetimeExtendingContext && !ExprEvalContexts.empty()) {
-    auto &PrevRecord = ExprEvalContexts[ExprEvalContexts.size() - 2];
+  if (getLangOpts().CPlusPlus23 && Rec.InLifetimeExtendingContext &&
+      PrevRecord.InLifetimeExtendingContext &&
+      !Rec.ForRangeLifetimeExtendTemps.empty()) {
     PrevRecord.ForRangeLifetimeExtendTemps.append(
         Rec.ForRangeLifetimeExtendTemps);
   }
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index fa7b92a43346..74ed3fe7bd52 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -8429,7 +8429,7 @@ ExprResult Sema::IgnoredValueConversions(Expr *E) {
     // unnecessary temporary objects. If we skip this step, IR generation is
     // able to synthesize the storage for itself in the aggregate case, and
     // adding the extra node to the AST is just clutter.
-    if (isInMaterializeTemporaryObjectContext() && getLangOpts().CPlusPlus17 &&
+    if (isInLifetimeExtendingContext() && getLangOpts().CPlusPlus17 &&
         E->isPRValue() && !E->getType()->isVoidType()) {
       ExprResult Res = TemporaryMaterializationConversion(E);
       if (Res.isInvalid())
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index 02d0b35ce940..c45a8d1408ff 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -5483,7 +5483,6 @@ void Sema::InstantiateVariableInitializer(
         *this, Sema::ExpressionEvaluationContext::PotentiallyEvaluated, Var);
 
     keepInLifetimeExtendingContext();
-    keepInMaterializeTemporaryObjectContext();
     // Instantiate the initializer.
     ExprResult Init;
 
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index fdf84c512c49..8c96134af7c8 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -4177,7 +4177,6 @@ ExprResult TreeTransform<Derived>::TransformInitializer(Expr *Init,
       getSema(), EnterExpressionEvaluationContext::InitList,
       Construct->isListInitialization());
 
-  getSema().keepInLifetimeExtendingContext();
   getSema().keepInLifetimeExtendingContext();
   SmallVector<Expr*, 8> NewArgs;
   bool ArgChanged = false;
@@ -8752,10 +8751,6 @@ TreeTransform<Derived>::TransformCXXForRangeStmt(CXXForRangeStmt *S) {
   if (getSema().getLangOpts().CPlusPlus23) {
     auto &LastRecord = getSema().ExprEvalContexts.back();
     LastRecord.InLifetimeExtendingContext = true;
-
-    // Materialize non-`cv void` prvalue temporaries in discarded
-    // expressions. These materialized temporaries may be lifetime-extented.
-    LastRecord.InMaterializeTemporaryObjectContext = true;
   }
   StmtResult Init =
       S->getInit() ? getDerived().TransformStmt(S->getInit()) : StmtResult();
-- 
GitLab


From 8927ac865755d7e98411500a7a2da70e8a185cc2 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Mon, 15 Apr 2024 08:56:57 -0700
Subject: [PATCH 252/464] [BOLT] Fix a warning

This patch fixes:

  bolt/lib/Profile/BoltAddressTranslation.cpp:380:37: error: operator
  '<<' has lower precedence than '+'; '+' will be evaluated first
  [-Werror,-Wshift-op-parentheses]
---
 bolt/lib/Profile/BoltAddressTranslation.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bolt/lib/Profile/BoltAddressTranslation.cpp b/bolt/lib/Profile/BoltAddressTranslation.cpp
index ac6e3d01f271..7cfb9c132c2c 100644
--- a/bolt/lib/Profile/BoltAddressTranslation.cpp
+++ b/bolt/lib/Profile/BoltAddressTranslation.cpp
@@ -377,7 +377,7 @@ void BoltAddressTranslation::parseMaps(std::vector<uint64_t> &HotFuncs,
       PrevAddress = OutputAddress;
       int64_t InputDelta = 0;
       if (J < EqualElems) {
-        InputOffset = (OutputOffset + ColdInputSkew << 1) | BEBitMask[J];
+        InputOffset = ((OutputOffset + ColdInputSkew) << 1) | BEBitMask[J];
       } else {
         InputDelta = DE.getSLEB128(&Offset, &Err);
         InputOffset += InputDelta;
-- 
GitLab


From b4776b8d8ea742a46039002fac4c280e619ac48d Mon Sep 17 00:00:00 2001
From: Mark de Wever <koraq@xs4all.nl>
Date: Mon, 15 Apr 2024 17:58:37 +0200
Subject: [PATCH 253/464] [libc++][CI] Tests LLDB libc++ data formatters.
 (#88312)

This enables testing of the LLDB libc++ specific data formatters.
This is enabled in the bootstrap build since building LLDB requires
Clang and this
is quite expensive. Adding this test changes the build time from 31 to
34 minutes.
---
 libcxx/utils/ci/run-buildbot                    | 10 +++++++---
 lldb/packages/Python/lldbsuite/test/lldbtest.py |  2 ++
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/libcxx/utils/ci/run-buildbot b/libcxx/utils/ci/run-buildbot
index cc72f4639b1a..23a2a1bbbc63 100755
--- a/libcxx/utils/ci/run-buildbot
+++ b/libcxx/utils/ci/run-buildbot
@@ -368,18 +368,22 @@ bootstrapping-build)
           -DCMAKE_CXX_COMPILER_LAUNCHER="ccache" \
           -DCMAKE_BUILD_TYPE=Release \
           -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \
-          -DLLVM_ENABLE_PROJECTS="clang" \
+          -DLLVM_ENABLE_PROJECTS="clang;lldb" \
           -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;libunwind" \
           -DLLVM_RUNTIME_TARGETS="$(${CXX} --print-target-triple)" \
+          -DLLVM_HOST_TRIPLE="$(${CXX} --print-target-triple)" \
           -DLLVM_TARGETS_TO_BUILD="host" \
           -DRUNTIMES_BUILD_ALLOW_DARWIN=ON \
           -DLLVM_ENABLE_ASSERTIONS=ON \
           -DLLVM_LIT_ARGS="-sv --xunit-xml-output test-results.xml --timeout=1500 --time-tests"
 
-    echo "+++ Running the libc++ and libc++abi tests"
+    echo "+++ Running the LLDB libc++ data formatter tests"
+    ${NINJA} -vC "${BUILD_DIR}" check-lldb-api-functionalities-data-formatter-data-formatter-stl-libcxx
+
+    echo "--- Running the libc++ and libc++abi tests"
     ${NINJA} -vC "${BUILD_DIR}" check-runtimes
 
-    echo "--- Installing libc++ and libc++abi to a fake location"
+    echo "+++ Installing libc++ and libc++abi to a fake location"
     ${NINJA} -vC "${BUILD_DIR}" install-runtimes
 
     ccache -s
diff --git a/lldb/packages/Python/lldbsuite/test/lldbtest.py b/lldb/packages/Python/lldbsuite/test/lldbtest.py
index c28a78a2c4a2..7a7afec73457 100644
--- a/lldb/packages/Python/lldbsuite/test/lldbtest.py
+++ b/lldb/packages/Python/lldbsuite/test/lldbtest.py
@@ -751,6 +751,8 @@ class Base(unittest.TestCase):
             "settings set symbols.enable-external-lookup false",
             # Inherit the TCC permissions from the inferior's parent.
             "settings set target.inherit-tcc true",
+            # Based on https://discourse.llvm.org/t/running-lldb-in-a-container/76801/4
+            "settings set target.disable-aslr false",
             # Kill rather than detach from the inferior if something goes wrong.
             "settings set target.detach-on-error false",
             # Disable fix-its by default so that incorrect expressions in tests don't
-- 
GitLab


From 1c63a3e0cdb3f02098f8ec525ca31a3b44d9d31b Mon Sep 17 00:00:00 2001
From: mmoadeli <mahmoud.moadeli@codeplay.com>
Date: Mon, 15 Apr 2024 17:05:40 +0100
Subject: [PATCH 254/464] Resolve static analyser report on pointer
 dereferencing after null check (#88278)

- Resolve Static Analyzer Check Failure: Pointer Dereferencing After
Null Check.
- Minor naming and style improvement
---
 .../AMDGPU/AMDGPULowerModuleLDSPass.cpp       | 33 +++++++++----------
 1 file changed, 15 insertions(+), 18 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp b/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
index 595f09664c55..c8bf9dd39e38 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULowerModuleLDSPass.cpp
@@ -1052,21 +1052,18 @@ public:
   void removeNoLdsKernelIdFromReachable(CallGraph &CG, Function *KernelRoot) {
     KernelRoot->removeFnAttr("amdgpu-no-lds-kernel-id");
 
-    SmallVector<Function *> Tmp({CG[KernelRoot]->getFunction()});
-    if (!Tmp.back())
-      return;
-
+    SmallVector<Function *> WorkList({CG[KernelRoot]->getFunction()});
     SmallPtrSet<Function *, 8> Visited;
     bool SeenUnknownCall = false;
 
-    do {
-      Function *F = Tmp.pop_back_val();
+    while (!WorkList.empty()) {
+      Function *F = WorkList.pop_back_val();
 
-      for (auto &N : *CG[F]) {
-        if (!N.second)
+      for (auto &CallRecord : *CG[F]) {
+        if (!CallRecord.second)
           continue;
 
-        Function *Callee = N.second->getFunction();
+        Function *Callee = CallRecord.second->getFunction();
         if (!Callee) {
           if (!SeenUnknownCall) {
             SeenUnknownCall = true;
@@ -1074,21 +1071,21 @@ public:
             // If we see any indirect calls, assume nothing about potential
             // targets.
             // TODO: This could be refined to possible LDS global users.
-            for (auto &N : *CG.getExternalCallingNode()) {
-              Function *PotentialCallee = N.second->getFunction();
+            for (auto &ExternalCallRecord : *CG.getExternalCallingNode()) {
+              Function *PotentialCallee =
+                  ExternalCallRecord.second->getFunction();
+              assert(PotentialCallee);
               if (!isKernelLDS(PotentialCallee))
                 PotentialCallee->removeFnAttr("amdgpu-no-lds-kernel-id");
             }
-
-            continue;
           }
+        } else {
+          Callee->removeFnAttr("amdgpu-no-lds-kernel-id");
+          if (Visited.insert(Callee).second)
+            WorkList.push_back(Callee);
         }
-
-        Callee->removeFnAttr("amdgpu-no-lds-kernel-id");
-        if (Visited.insert(Callee).second)
-          Tmp.push_back(Callee);
       }
-    } while (!Tmp.empty());
+    }
   }
 
   DenseMap<Function *, GlobalVariable *> lowerDynamicLDSVariables(
-- 
GitLab


From 9abb1ffc5cb75465340cb604988d1e386415bd72 Mon Sep 17 00:00:00 2001
From: Valery Dmitriev <valery.n.dmitriev@intel.com>
Date: Mon, 15 Apr 2024 09:09:39 -0700
Subject: [PATCH 255/464] [SLP][NFC] Add option to bypass early profitability
 check. (#88594)

The option intended primarily for LIT tests to suppress heuristic based
profitability check and proceed vectorization of a seemingly
unprofitable alternate operation pattern. This allows the vectorizer to
execute path that was the original intent of a test.
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    |  7 +++++-
 .../X86/ext-int-reduced-not-operand.ll        | 19 +++++++++++++++
 .../X86/gather-move-out-of-loop.ll            | 16 +++++++++++++
 ...gathered-delayed-nodes-with-reused-user.ll | 23 +++++++++++++++++++
 .../non-scheduled-inst-reused-as-last-inst.ll | 22 ++++++++++++++++++
 5 files changed, 86 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index b031b40a978f..9a0eaca2f598 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -118,6 +118,11 @@ static cl::opt<int>
                      cl::desc("Only vectorize if you gain more than this "
                               "number "));
 
+static cl::opt<bool> SLPSkipEarlyProfitabilityCheck(
+    "slp-skip-early-profitability-check", cl::init(false), cl::Hidden,
+    cl::desc("When true, SLP vectorizer bypasses profitability checks based on "
+             "heuristics and makes vectorization decision via cost modeling."));
+
 static cl::opt<bool>
 ShouldVectorizeHor("slp-vectorize-hor", cl::init(true), cl::Hidden,
                    cl::desc("Attempt to vectorize horizontal reductions"));
@@ -6250,7 +6255,7 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
       LLVM_DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n");
       return TreeEntry::NeedToGather;
     }
-    if (!areAltOperandsProfitable(S, VL)) {
+    if (!SLPSkipEarlyProfitabilityCheck && !areAltOperandsProfitable(S, VL)) {
       LLVM_DEBUG(
           dbgs()
           << "SLP: ShuffleVector not vectorized, operands are buildvector and "
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/ext-int-reduced-not-operand.ll b/llvm/test/Transforms/SLPVectorizer/X86/ext-int-reduced-not-operand.ll
index b76e26e0fd57..2ff6785c0da6 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/ext-int-reduced-not-operand.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/ext-int-reduced-not-operand.ll
@@ -1,7 +1,26 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
 ; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-99999 < %s | FileCheck %s
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-99999\
+; RUN: -slp-skip-early-profitability-check < %s | FileCheck %s --check-prefixes=FORCED
 
 define i64 @wombat() {
+; FORCED-LABEL: define i64 @wombat() {
+; FORCED-NEXT:  bb:
+; FORCED-NEXT:    br label [[BB2:%.*]]
+; FORCED:       bb1:
+; FORCED-NEXT:    br label [[BB2]]
+; FORCED:       bb2:
+; FORCED-NEXT:    [[PHI:%.*]] = phi i32 [ 0, [[BB:%.*]] ], [ 0, [[BB1:%.*]] ]
+; FORCED-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[PHI]], i32 0
+; FORCED-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <2 x i32> zeroinitializer
+; FORCED-NEXT:    [[TMP2:%.*]] = trunc <2 x i32> [[TMP1]] to <2 x i1>
+; FORCED-NEXT:    [[TMP3:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0
+; FORCED-NEXT:    [[TMP4:%.*]] = zext i1 [[TMP3]] to i64
+; FORCED-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP2]], i32 1
+; FORCED-NEXT:    [[TMP6:%.*]] = zext i1 [[TMP5]] to i64
+; FORCED-NEXT:    [[OR:%.*]] = or i64 [[TMP4]], [[TMP6]]
+; FORCED-NEXT:    ret i64 [[OR]]
+;
 ; CHECK-LABEL: define i64 @wombat() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    br label [[BB2:%.*]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/gather-move-out-of-loop.ll b/llvm/test/Transforms/SLPVectorizer/X86/gather-move-out-of-loop.ll
index 3c3dea3f1ea8..f2ea2df7cc98 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/gather-move-out-of-loop.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/gather-move-out-of-loop.ll
@@ -1,7 +1,23 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -passes=slp-vectorizer -slp-threshold=-100 -mtriple=x86_64-w64-windows-gnu < %s | FileCheck %s
+; RUN: opt -S -passes=slp-vectorizer -slp-threshold=-100 -mtriple=x86_64-w64-windows-gnu\
+; RUN: -slp-skip-early-profitability-check < %s | FileCheck %s --check-prefixes=FORCED
 
 define void @test(i16 %0) {
+; FORCED-LABEL: @test(
+; FORCED-NEXT:  for.body92.preheader:
+; FORCED-NEXT:    [[TMP1:%.*]] = insertelement <2 x i16> <i16 0, i16 poison>, i16 [[TMP0:%.*]], i32 1
+; FORCED-NEXT:    [[TMP2:%.*]] = sext <2 x i16> [[TMP1]] to <2 x i32>
+; FORCED-NEXT:    [[TMP3:%.*]] = zext <2 x i16> [[TMP1]] to <2 x i32>
+; FORCED-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> <i32 0, i32 3>
+; FORCED-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <4 x i32> <i32 0, i32 poison, i32 1, i32 poison>
+; FORCED-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> <i32 poison, i32 0, i32 poison, i32 0>, <4 x i32> [[TMP5]], <4 x i32> <i32 4, i32 1, i32 6, i32 3>
+; FORCED-NEXT:    br label [[FOR_BODY92:%.*]]
+; FORCED:       for.body92:
+; FORCED-NEXT:    [[TMP7:%.*]] = add nsw <4 x i32> zeroinitializer, [[TMP6]]
+; FORCED-NEXT:    store <4 x i32> [[TMP7]], ptr undef, align 8
+; FORCED-NEXT:    br label [[FOR_BODY92]]
+;
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:  for.body92.preheader:
 ; CHECK-NEXT:    br label [[FOR_BODY92:%.*]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/gathered-delayed-nodes-with-reused-user.ll b/llvm/test/Transforms/SLPVectorizer/X86/gathered-delayed-nodes-with-reused-user.ll
index 19a8aa9b6181..d7144d750321 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/gathered-delayed-nodes-with-reused-user.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/gathered-delayed-nodes-with-reused-user.ll
@@ -1,7 +1,30 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
 ; RUN: opt -passes=slp-vectorizer -S -slp-threshold=-9999 < %s | FileCheck %s
+; RUN: opt -passes=slp-vectorizer -S -slp-threshold=-9999\
+; RUN: -slp-skip-early-profitability-check < %s | FileCheck %s --check-prefixes=FORCED
 
 define i64 @foo() {
+; FORCED-LABEL: define i64 @foo() {
+; FORCED-NEXT:  bb:
+; FORCED-NEXT:    br label [[BB3:%.*]]
+; FORCED:       bb1:
+; FORCED-NEXT:    [[TMP0:%.*]] = phi <2 x i64> [ [[TMP5:%.*]], [[BB3]] ]
+; FORCED-NEXT:    ret i64 0
+; FORCED:       bb3:
+; FORCED-NEXT:    [[PHI5:%.*]] = phi i64 [ 0, [[BB:%.*]] ], [ 0, [[BB3]] ]
+; FORCED-NEXT:    [[TMP1:%.*]] = phi <2 x i64> [ zeroinitializer, [[BB]] ], [ [[TMP7:%.*]], [[BB3]] ]
+; FORCED-NEXT:    [[TMP2:%.*]] = insertelement <2 x i64> <i64 poison, i64 0>, i64 [[PHI5]], i32 0
+; FORCED-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[TMP1]], [[TMP2]]
+; FORCED-NEXT:    [[TMP4:%.*]] = or <2 x i64> [[TMP1]], [[TMP2]]
+; FORCED-NEXT:    [[TMP5]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <2 x i32> <i32 0, i32 3>
+; FORCED-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> <i64 poison, i64 0>, <2 x i32> <i32 0, i32 3>
+; FORCED-NEXT:    [[TMP7]] = add <2 x i64> [[TMP6]], [[TMP2]]
+; FORCED-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[TMP7]], i32 1
+; FORCED-NEXT:    [[GETELEMENTPTR:%.*]] = getelementptr i64, ptr addrspace(1) null, i64 [[TMP8]]
+; FORCED-NEXT:    [[TMP9:%.*]] = extractelement <2 x i64> [[TMP5]], i32 1
+; FORCED-NEXT:    [[ICMP:%.*]] = icmp ult i64 [[TMP9]], 0
+; FORCED-NEXT:    br i1 false, label [[BB3]], label [[BB1:%.*]]
+;
 ; CHECK-LABEL: define i64 @foo() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    br label [[BB3:%.*]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/non-scheduled-inst-reused-as-last-inst.ll b/llvm/test/Transforms/SLPVectorizer/X86/non-scheduled-inst-reused-as-last-inst.ll
index 59cd1c0ccddf..1163c8219dab 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/non-scheduled-inst-reused-as-last-inst.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/non-scheduled-inst-reused-as-last-inst.ll
@@ -1,7 +1,29 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
 ; RUN: opt -S -passes=slp-vectorizer -slp-threshold=-9999 -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+; RUN: opt -S -passes=slp-vectorizer -slp-threshold=-9999 -mtriple=x86_64-unknown-linux-gnu\
+; RUN: -slp-skip-early-profitability-check < %s | FileCheck %s --check-prefixes=FORCED
 
 define void @foo() {
+; FORCED-LABEL: define void @foo() {
+; FORCED-NEXT:  bb:
+; FORCED-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> <i32 poison, i32 0>, i32 0, i32 0
+; FORCED-NEXT:    br label [[BB1:%.*]]
+; FORCED:       bb1:
+; FORCED-NEXT:    [[TMP1:%.*]] = phi <2 x i32> [ zeroinitializer, [[BB:%.*]] ], [ [[TMP6:%.*]], [[BB4:%.*]] ]
+; FORCED-NEXT:    [[TMP2:%.*]] = shl <2 x i32> [[TMP1]], [[TMP0]]
+; FORCED-NEXT:    [[TMP3:%.*]] = or <2 x i32> [[TMP1]], [[TMP0]]
+; FORCED-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> <i32 0, i32 3>
+; FORCED-NEXT:    [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> [[TMP1]], <2 x i32> <i32 0, i32 3>
+; FORCED-NEXT:    [[TMP6]] = or <2 x i32> [[TMP5]], zeroinitializer
+; FORCED-NEXT:    [[TMP7:%.*]] = extractelement <2 x i32> [[TMP6]], i32 0
+; FORCED-NEXT:    [[CALL:%.*]] = call i64 null(i32 [[TMP7]])
+; FORCED-NEXT:    br label [[BB4]]
+; FORCED:       bb4:
+; FORCED-NEXT:    br i1 false, label [[BB5:%.*]], label [[BB1]]
+; FORCED:       bb5:
+; FORCED-NEXT:    [[TMP8:%.*]] = phi <2 x i32> [ [[TMP4]], [[BB4]] ]
+; FORCED-NEXT:    ret void
+;
 ; CHECK-LABEL: define void @foo() {
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    br label [[BB1:%.*]]
-- 
GitLab


From 55b6f17071d25b77fcdc910ca9b15f89305137e0 Mon Sep 17 00:00:00 2001
From: Jeremy Kun <jkun@google.com>
Date: Mon, 15 Apr 2024 09:12:00 -0700
Subject: [PATCH 256/464] Add a polynomial dialect shell, attributes, and types
 (#72081)

RFC:
https://discourse.llvm.org/t/rfc-a-poly-dialect-for-polynomial-arithmetic/73891

This PR implements the minimal work needed to represent the polynomial
type such that it can be tested with `lit`.

In this PR:

- Dialect shell
- `Polynomial` data structure needed for folding
- Polynomial attributes (`PolynomialAttr` and `RingAttr` which store a polynomial)
- `polynomial.polynomial` type
- Basic lit tests

---------

Co-authored-by: Jeremy Kun <j2kun@users.noreply.github.com>
---
 mlir/include/mlir/Dialect/CMakeLists.txt      |   1 +
 .../mlir/Dialect/Polynomial/CMakeLists.txt    |   1 +
 .../mlir/Dialect/Polynomial/IR/CMakeLists.txt |  10 +
 .../mlir/Dialect/Polynomial/IR/Polynomial.h   | 130 +++++++++++
 .../mlir/Dialect/Polynomial/IR/Polynomial.td  | 153 +++++++++++++
 .../Polynomial/IR/PolynomialAttributes.h      |  17 ++
 .../Dialect/Polynomial/IR/PolynomialDialect.h |  19 ++
 .../Dialect/Polynomial/IR/PolynomialOps.h     |  21 ++
 .../Dialect/Polynomial/IR/PolynomialTypes.h   |  17 ++
 mlir/include/mlir/InitAllDialects.h           |   2 +
 mlir/lib/Dialect/CMakeLists.txt               |   1 +
 mlir/lib/Dialect/Polynomial/CMakeLists.txt    |   1 +
 mlir/lib/Dialect/Polynomial/IR/CMakeLists.txt |  19 ++
 mlir/lib/Dialect/Polynomial/IR/Polynomial.cpp |  96 ++++++++
 .../Polynomial/IR/PolynomialAttributes.cpp    | 213 ++++++++++++++++++
 .../Polynomial/IR/PolynomialDialect.cpp       |  41 ++++
 .../Dialect/Polynomial/IR/PolynomialOps.cpp   |  15 ++
 mlir/test/Dialect/Polynomial/attributes.mlir  |  45 ++++
 mlir/test/Dialect/Polynomial/types.mlir       |  42 ++++
 19 files changed, 844 insertions(+)
 create mode 100644 mlir/include/mlir/Dialect/Polynomial/CMakeLists.txt
 create mode 100644 mlir/include/mlir/Dialect/Polynomial/IR/CMakeLists.txt
 create mode 100644 mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.h
 create mode 100644 mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.td
 create mode 100644 mlir/include/mlir/Dialect/Polynomial/IR/PolynomialAttributes.h
 create mode 100644 mlir/include/mlir/Dialect/Polynomial/IR/PolynomialDialect.h
 create mode 100644 mlir/include/mlir/Dialect/Polynomial/IR/PolynomialOps.h
 create mode 100644 mlir/include/mlir/Dialect/Polynomial/IR/PolynomialTypes.h
 create mode 100644 mlir/lib/Dialect/Polynomial/CMakeLists.txt
 create mode 100644 mlir/lib/Dialect/Polynomial/IR/CMakeLists.txt
 create mode 100644 mlir/lib/Dialect/Polynomial/IR/Polynomial.cpp
 create mode 100644 mlir/lib/Dialect/Polynomial/IR/PolynomialAttributes.cpp
 create mode 100644 mlir/lib/Dialect/Polynomial/IR/PolynomialDialect.cpp
 create mode 100644 mlir/lib/Dialect/Polynomial/IR/PolynomialOps.cpp
 create mode 100644 mlir/test/Dialect/Polynomial/attributes.mlir
 create mode 100644 mlir/test/Dialect/Polynomial/types.mlir

diff --git a/mlir/include/mlir/Dialect/CMakeLists.txt b/mlir/include/mlir/Dialect/CMakeLists.txt
index 2da79011fa26..4bd7f12fabf7 100644
--- a/mlir/include/mlir/Dialect/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/CMakeLists.txt
@@ -28,6 +28,7 @@ add_subdirectory(OpenACCMPCommon)
 add_subdirectory(OpenMP)
 add_subdirectory(PDL)
 add_subdirectory(PDLInterp)
+add_subdirectory(Polynomial)
 add_subdirectory(Quant)
 add_subdirectory(SCF)
 add_subdirectory(Shape)
diff --git a/mlir/include/mlir/Dialect/Polynomial/CMakeLists.txt b/mlir/include/mlir/Dialect/Polynomial/CMakeLists.txt
new file mode 100644
index 000000000000..f33061b2d87c
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Polynomial/CMakeLists.txt
@@ -0,0 +1 @@
+add_subdirectory(IR)
diff --git a/mlir/include/mlir/Dialect/Polynomial/IR/CMakeLists.txt b/mlir/include/mlir/Dialect/Polynomial/IR/CMakeLists.txt
new file mode 100644
index 000000000000..d8039deb5ee2
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Polynomial/IR/CMakeLists.txt
@@ -0,0 +1,10 @@
+add_mlir_dialect(Polynomial polynomial)
+add_mlir_doc(PolynomialDialect PolynomialDialect Polynomial/ -gen-dialect-doc)
+add_mlir_doc(PolynomialOps PolynomialOps Polynomial/ -gen-op-doc)
+add_mlir_doc(PolynomialAttributes PolynomialAttributes Dialects/ -gen-attrdef-doc)
+add_mlir_doc(PolynomialTypes PolynomialTypes Dialects/ -gen-typedef-doc)
+
+set(LLVM_TARGET_DEFINITIONS Polynomial.td)
+mlir_tablegen(PolynomialAttributes.cpp.inc -gen-attrdef-defs -attrdefs-dialect=polynomial)
+mlir_tablegen(PolynomialAttributes.h.inc -gen-attrdef-decls -attrdefs-dialect=polynomial)
+add_public_tablegen_target(MLIRPolynomialAttributesIncGen)
diff --git a/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.h b/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.h
new file mode 100644
index 000000000000..39b05b9d3ad1
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.h
@@ -0,0 +1,130 @@
+//===- Polynomial.h - A data class for polynomials --------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_POLYNOMIAL_IR_POLYNOMIAL_H_
+#define MLIR_DIALECT_POLYNOMIAL_IR_POLYNOMIAL_H_
+
+#include "mlir/Support/LLVM.h"
+#include "mlir/Support/LogicalResult.h"
+#include "llvm/ADT/APInt.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/Hashing.h"
+#include "llvm/ADT/SmallVector.h"
+
+namespace mlir {
+
+class MLIRContext;
+
+namespace polynomial {
+
+/// This restricts statically defined polynomials to have at most 64-bit
+/// coefficients. This may be relaxed in the future, but it seems unlikely one
+/// would want to specify 128-bit polynomials statically in the source code.
+constexpr unsigned apintBitWidth = 64;
+
+/// A class representing a monomial of a single-variable polynomial with integer
+/// coefficients.
+class Monomial {
+public:
+  Monomial(int64_t coeff, uint64_t expo)
+      : coefficient(apintBitWidth, coeff), exponent(apintBitWidth, expo) {}
+
+  Monomial(const APInt &coeff, const APInt &expo)
+      : coefficient(coeff), exponent(expo) {}
+
+  Monomial() : coefficient(apintBitWidth, 0), exponent(apintBitWidth, 0) {}
+
+  bool operator==(const Monomial &other) const {
+    return other.coefficient == coefficient && other.exponent == exponent;
+  }
+  bool operator!=(const Monomial &other) const {
+    return other.coefficient != coefficient || other.exponent != exponent;
+  }
+
+  /// Monomials are ordered by exponent.
+  bool operator<(const Monomial &other) const {
+    return (exponent.ult(other.exponent));
+  }
+
+  // Prints polynomial to 'os'.
+  void print(raw_ostream &os) const;
+
+  friend ::llvm::hash_code hash_value(const Monomial &arg);
+
+public:
+  APInt coefficient;
+
+  // Always unsigned
+  APInt exponent;
+};
+
+/// A single-variable polynomial with integer coefficients.
+///
+/// Eg: x^1024 + x + 1
+///
+/// The symbols used as the polynomial's indeterminate don't matter, so long as
+/// it is used consistently throughout the polynomial.
+class Polynomial {
+public:
+  Polynomial() = delete;
+
+  explicit Polynomial(ArrayRef<Monomial> terms) : terms(terms){};
+
+  // Returns a Polynomial from a list of monomials.
+  // Fails if two monomials have the same exponent.
+  static FailureOr<Polynomial> fromMonomials(ArrayRef<Monomial> monomials);
+
+  /// Returns a polynomial with coefficients given by `coeffs`. The value
+  /// coeffs[i] is converted to a monomial with exponent i.
+  static Polynomial fromCoefficients(ArrayRef<int64_t> coeffs);
+
+  explicit operator bool() const { return !terms.empty(); }
+  bool operator==(const Polynomial &other) const {
+    return other.terms == terms;
+  }
+  bool operator!=(const Polynomial &other) const {
+    return !(other.terms == terms);
+  }
+
+  // Prints polynomial to 'os'.
+  void print(raw_ostream &os) const;
+  void print(raw_ostream &os, ::llvm::StringRef separator,
+             ::llvm::StringRef exponentiation) const;
+  void dump() const;
+
+  // Prints polynomial so that it can be used as a valid identifier
+  std::string toIdentifier() const;
+
+  unsigned getDegree() const;
+
+  friend ::llvm::hash_code hash_value(const Polynomial &arg);
+
+private:
+  // The monomial terms for this polynomial.
+  SmallVector<Monomial> terms;
+};
+
+// Make Polynomial hashable.
+inline ::llvm::hash_code hash_value(const Polynomial &arg) {
+  return ::llvm::hash_combine_range(arg.terms.begin(), arg.terms.end());
+}
+
+inline ::llvm::hash_code hash_value(const Monomial &arg) {
+  return llvm::hash_combine(::llvm::hash_value(arg.coefficient),
+                            ::llvm::hash_value(arg.exponent));
+}
+
+inline raw_ostream &operator<<(raw_ostream &os, const Polynomial &polynomial) {
+  polynomial.print(os);
+  return os;
+}
+
+} // namespace polynomial
+} // namespace mlir
+
+#endif // MLIR_DIALECT_POLYNOMIAL_IR_POLYNOMIAL_H_
diff --git a/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.td b/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.td
new file mode 100644
index 000000000000..5d8da8399b01
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Polynomial/IR/Polynomial.td
@@ -0,0 +1,153 @@
+//===- PolynomialOps.td - Polynomial dialect ---------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef POLYNOMIAL_OPS
+#define POLYNOMIAL_OPS
+
+include "mlir/IR/BuiltinAttributes.td"
+include "mlir/IR/OpBase.td"
+include "mlir/Interfaces/InferTypeOpInterface.td"
+include "mlir/Interfaces/SideEffectInterfaces.td"
+
+def Polynomial_Dialect : Dialect {
+  let name = "polynomial";
+  let cppNamespace = "::mlir::polynomial";
+  let description = [{
+    The Polynomial dialect defines single-variable polynomial types and
+    operations.
+
+    The simplest use of `polynomial` is to represent mathematical operations in
+    a polynomial ring `R[x]`, where `R` is another MLIR type like `i32`.
+
+    More generally, this dialect supports representing polynomial operations in a
+    quotient ring `R[X]/(f(x))` for some statically fixed polynomial `f(x)`.
+    Two polyomials `p(x), q(x)` are considered equal in this ring if they have the
+    same remainder when dividing by `f(x)`. When a modulus is given, ring operations
+    are performed with reductions modulo `f(x)` and relative to the coefficient ring
+    `R`.
+
+    Examples:
+
+    ```mlir
+    // A constant polynomial in a ring with i32 coefficients and no polynomial modulus
+    #ring = #polynomial.ring<ctype=i32>
+    %a = polynomial.constant <1 + x**2 - 3x**3> : polynomial.polynomial<#ring>
+
+    // A constant polynomial in a ring with i32 coefficients, modulo (x^1024 + 1)
+    #modulus = #polynomial.polynomial<1 + x**1024>
+    #ring = #polynomial.ring<ctype=i32, ideal=#modulus>
+    %a = polynomial.constant <1 + x**2 - 3x**3> : polynomial.polynomial<#ring>
+
+    // A constant polynomial in a ring with i32 coefficients, with a polynomial
+    // modulus of (x^1024 + 1) and a coefficient modulus of 17.
+    #modulus = #polynomial.polynomial<1 + x**1024>
+    #ring = #polynomial.ring<ctype=i32, cmod=17, ideal=#modulus>
+    %a = polynomial.constant <1 + x**2 - 3x**3> : polynomial.polynomial<#ring>
+    ```
+  }];
+
+  let useDefaultTypePrinterParser = 1;
+  let useDefaultAttributePrinterParser = 1;
+}
+
+class Polynomial_Attr<string name, string attrMnemonic, list<Trait> traits = []>
+    : AttrDef<Polynomial_Dialect, name, traits> {
+  let mnemonic = attrMnemonic;
+}
+
+def Polynomial_PolynomialAttr : Polynomial_Attr<"Polynomial", "polynomial"> {
+  let summary = "An attribute containing a single-variable polynomial.";
+  let description = [{
+     #poly = #polynomial.poly<x**1024 + 1>
+  }];
+  let parameters = (ins "Polynomial":$polynomial);
+  let hasCustomAssemblyFormat = 1;
+}
+
+def Polynomial_RingAttr : Polynomial_Attr<"Ring", "ring"> {
+  let summary = "An attribute specifying a polynomial ring.";
+  let description = [{
+    A ring describes the domain in which polynomial arithmetic occurs. The ring
+    attribute in `polynomial` represents the more specific case of polynomials
+    with a single indeterminate; whose coefficients can be represented by
+    another MLIR type (`coefficientType`); and, if the coefficient type is
+    integral, whose coefficients are taken modulo some statically known modulus
+    (`coefficientModulus`).
+
+    Additionally, a polynomial ring can specify an _ideal_, which converts
+    polynomial arithmetic to the analogue of modular integer arithmetic, where
+    each polynomial is represented as its remainder when dividing by the
+    modulus. For single-variable polynomials, an "ideal" is always specificed
+    via a single polynomial, which we call `polynomialModulus`.
+
+    An expressive example is polynomials with i32 coefficients, whose
+    coefficients are taken modulo `2**32 - 5`, with a polynomial modulus of
+    `x**1024 - 1`.
+
+    ```mlir
+    #poly_mod = #polynomial.polynomial<-1 + x**1024>
+    #ring = #polynomial.ring<coefficientType=i32,
+                             coefficientModulus=4294967291,
+                             polynomialModulus=#poly_mod>
+
+    %0 = ... : polynomial.polynomial<#ring>
+    ```
+
+    In this case, the value of a polynomial is always "converted" to a
+    canonical form by applying repeated reductions by setting `x**1024 = 1`
+    and simplifying.
+
+    The coefficient and polynomial modulus parameters are optional, and the
+    coefficient modulus is only allowed if the coefficient type is integral.
+  }];
+
+  let parameters = (ins
+    "Type": $coefficientType,
+    OptionalParameter<"IntegerAttr">: $coefficientModulus,
+    OptionalParameter<"PolynomialAttr">: $polynomialModulus
+  );
+
+  let hasCustomAssemblyFormat = 1;
+}
+
+class Polynomial_Type<string name, string typeMnemonic>
+    : TypeDef<Polynomial_Dialect, name> {
+  let mnemonic = typeMnemonic;
+}
+
+def Polynomial_PolynomialType : Polynomial_Type<"Polynomial", "polynomial"> {
+  let summary = "An element of a polynomial ring.";
+
+  let description = [{
+    A type for polynomials in a polynomial quotient ring.
+  }];
+
+  let parameters = (ins Polynomial_RingAttr:$ring);
+  let assemblyFormat = "`<` $ring `>`";
+}
+
+class Polynomial_Op<string mnemonic, list<Trait> traits = []> :
+    Op<Polynomial_Dialect, mnemonic, traits # [Pure]>;
+
+class Polynomial_UnaryOp<string mnemonic, list<Trait> traits = []> :
+    Polynomial_Op<mnemonic, traits # [SameOperandsAndResultType]> {
+  let arguments = (ins Polynomial_PolynomialType:$operand);
+  let results = (outs Polynomial_PolynomialType:$result);
+
+  let assemblyFormat = "$operand attr-dict `:` qualified(type($result))";
+}
+
+class Polynomial_BinaryOp<string mnemonic, list<Trait> traits = []> :
+    Polynomial_Op<mnemonic, traits # [SameOperandsAndResultType]> {
+  let arguments = (ins Polynomial_PolynomialType:$lhs, Polynomial_PolynomialType:$rhs);
+  let results = (outs Polynomial_PolynomialType:$result);
+
+  let assemblyFormat = "$lhs `,` $rhs attr-dict `:` qualified(type($result))";
+}
+
+#endif // POLYNOMIAL_OPS
diff --git a/mlir/include/mlir/Dialect/Polynomial/IR/PolynomialAttributes.h b/mlir/include/mlir/Dialect/Polynomial/IR/PolynomialAttributes.h
new file mode 100644
index 000000000000..b37d17bb89fb
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Polynomial/IR/PolynomialAttributes.h
@@ -0,0 +1,17 @@
+//===- PolynomialAttributes.h - polynomial dialect attributes ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef MLIR_INCLUDE_MLIR_DIALECT_POLYNOMIAL_IR_POLYNOMIALATTRIBUTES_H_
+#define MLIR_INCLUDE_MLIR_DIALECT_POLYNOMIAL_IR_POLYNOMIALATTRIBUTES_H_
+
+#include "Polynomial.h"
+#include "PolynomialDialect.h"
+
+#define GET_ATTRDEF_CLASSES
+#include "mlir/Dialect/Polynomial/IR/PolynomialAttributes.h.inc"
+
+#endif // MLIR_INCLUDE_MLIR_DIALECT_POLYNOMIAL_IR_POLYNOMIALATTRIBUTES_H_
diff --git a/mlir/include/mlir/Dialect/Polynomial/IR/PolynomialDialect.h b/mlir/include/mlir/Dialect/Polynomial/IR/PolynomialDialect.h
new file mode 100644
index 000000000000..7b7acebe7a93
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Polynomial/IR/PolynomialDialect.h
@@ -0,0 +1,19 @@
+//===- PolynomialDialect.h - The Polynomial dialect -------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef MLIR_INCLUDE_MLIR_DIALECT_POLYNOMIAL_IR_POLYNOMIALDIALECT_H_
+#define MLIR_INCLUDE_MLIR_DIALECT_POLYNOMIAL_IR_POLYNOMIALDIALECT_H_
+
+#include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/Dialect.h"
+#include "mlir/IR/DialectImplementation.h"
+
+// Generated headers (block clang-format from messing up order)
+#include "mlir/Dialect/Polynomial/IR/PolynomialDialect.h.inc"
+
+#endif // MLIR_INCLUDE_MLIR_DIALECT_POLYNOMIAL_IR_POLYNOMIALDIALECT_H_
diff --git a/mlir/include/mlir/Dialect/Polynomial/IR/PolynomialOps.h b/mlir/include/mlir/Dialect/Polynomial/IR/PolynomialOps.h
new file mode 100644
index 000000000000..bacaad81ce8e
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Polynomial/IR/PolynomialOps.h
@@ -0,0 +1,21 @@
+//===- PolynomialOps.h - Ops for the Polynomial dialect ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef MLIR_INCLUDE_MLIR_DIALECT_POLYNOMIAL_IR_POLYNOMIALOPS_H_
+#define MLIR_INCLUDE_MLIR_DIALECT_POLYNOMIAL_IR_POLYNOMIALOPS_H_
+
+#include "PolynomialDialect.h"
+#include "PolynomialTypes.h"
+#include "mlir/IR/BuiltinOps.h"
+#include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/Dialect.h"
+#include "mlir/Interfaces/InferTypeOpInterface.h"
+
+#define GET_OP_CLASSES
+#include "mlir/Dialect/Polynomial/IR/Polynomial.h.inc"
+
+#endif // MLIR_INCLUDE_MLIR_DIALECT_POLYNOMIAL_IR_POLYNOMIALOPS_H_
diff --git a/mlir/include/mlir/Dialect/Polynomial/IR/PolynomialTypes.h b/mlir/include/mlir/Dialect/Polynomial/IR/PolynomialTypes.h
new file mode 100644
index 000000000000..2fc687745254
--- /dev/null
+++ b/mlir/include/mlir/Dialect/Polynomial/IR/PolynomialTypes.h
@@ -0,0 +1,17 @@
+//===- PolynomialTypes.h - Types for the Polynomial dialect -----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef MLIR_INCLUDE_MLIR_DIALECT_POLYNOMIAL_IR_POLYNOMIALTYPES_H_
+#define MLIR_INCLUDE_MLIR_DIALECT_POLYNOMIAL_IR_POLYNOMIALTYPES_H_
+
+#include "mlir/Dialect/Polynomial/IR/PolynomialAttributes.h"
+#include "mlir/Dialect/Polynomial/IR/PolynomialDialect.h"
+
+#define GET_TYPEDEF_CLASSES
+#include "mlir/Dialect/Polynomial/IR/PolynomialTypes.h.inc"
+
+#endif // MLIR_INCLUDE_MLIR_DIALECT_POLYNOMIAL_IR_POLYNOMIALTYPES_H_
diff --git a/mlir/include/mlir/InitAllDialects.h b/mlir/include/mlir/InitAllDialects.h
index c558dc53cc7f..c4d788cf8ed3 100644
--- a/mlir/include/mlir/InitAllDialects.h
+++ b/mlir/include/mlir/InitAllDialects.h
@@ -61,6 +61,7 @@
 #include "mlir/Dialect/OpenMP/OpenMPDialect.h"
 #include "mlir/Dialect/PDL/IR/PDL.h"
 #include "mlir/Dialect/PDLInterp/IR/PDLInterp.h"
+#include "mlir/Dialect/Polynomial/IR/PolynomialDialect.h"
 #include "mlir/Dialect/Quant/QuantOps.h"
 #include "mlir/Dialect/SCF/IR/SCF.h"
 #include "mlir/Dialect/SCF/IR/ValueBoundsOpInterfaceImpl.h"
@@ -131,6 +132,7 @@ inline void registerAllDialects(DialectRegistry &registry) {
                   omp::OpenMPDialect,
                   pdl::PDLDialect,
                   pdl_interp::PDLInterpDialect,
+                  polynomial::PolynomialDialect,
                   quant::QuantizationDialect,
                   ROCDL::ROCDLDialect,
                   scf::SCFDialect,
diff --git a/mlir/lib/Dialect/CMakeLists.txt b/mlir/lib/Dialect/CMakeLists.txt
index b1ba5a3bc881..a324ce7f9b19 100644
--- a/mlir/lib/Dialect/CMakeLists.txt
+++ b/mlir/lib/Dialect/CMakeLists.txt
@@ -28,6 +28,7 @@ add_subdirectory(OpenACCMPCommon)
 add_subdirectory(OpenMP)
 add_subdirectory(PDL)
 add_subdirectory(PDLInterp)
+add_subdirectory(Polynomial)
 add_subdirectory(Quant)
 add_subdirectory(SCF)
 add_subdirectory(Shape)
diff --git a/mlir/lib/Dialect/Polynomial/CMakeLists.txt b/mlir/lib/Dialect/Polynomial/CMakeLists.txt
new file mode 100644
index 000000000000..f33061b2d87c
--- /dev/null
+++ b/mlir/lib/Dialect/Polynomial/CMakeLists.txt
@@ -0,0 +1 @@
+add_subdirectory(IR)
diff --git a/mlir/lib/Dialect/Polynomial/IR/CMakeLists.txt b/mlir/lib/Dialect/Polynomial/IR/CMakeLists.txt
new file mode 100644
index 000000000000..7f5b3255d5d9
--- /dev/null
+++ b/mlir/lib/Dialect/Polynomial/IR/CMakeLists.txt
@@ -0,0 +1,19 @@
+add_mlir_dialect_library(MLIRPolynomialDialect
+  Polynomial.cpp
+  PolynomialAttributes.cpp
+  PolynomialDialect.cpp
+  PolynomialOps.cpp
+
+  ADDITIONAL_HEADER_DIRS
+  ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/Polynomial
+
+  DEPENDS
+  MLIRPolynomialIncGen
+  MLIRPolynomialAttributesIncGen
+  MLIRBuiltinAttributesIncGen
+
+  LINK_LIBS PUBLIC
+  MLIRSupport
+  MLIRDialect
+  MLIRIR
+  )
diff --git a/mlir/lib/Dialect/Polynomial/IR/Polynomial.cpp b/mlir/lib/Dialect/Polynomial/IR/Polynomial.cpp
new file mode 100644
index 000000000000..5916ffba78e2
--- /dev/null
+++ b/mlir/lib/Dialect/Polynomial/IR/Polynomial.cpp
@@ -0,0 +1,96 @@
+//===- Polynomial.cpp - MLIR storage type for static Polynomial -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Polynomial/IR/Polynomial.h"
+
+#include "mlir/Support/LogicalResult.h"
+#include "llvm/ADT/APInt.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/Support/raw_ostream.h"
+
+namespace mlir {
+namespace polynomial {
+
+FailureOr<Polynomial> Polynomial::fromMonomials(ArrayRef<Monomial> monomials) {
+  // A polynomial's terms are canonically stored in order of increasing degree.
+  auto monomialsCopy = llvm::SmallVector<Monomial>(monomials);
+  std::sort(monomialsCopy.begin(), monomialsCopy.end());
+
+  // Ensure non-unique exponents are not present. Since we sorted the list by
+  // exponent, a linear scan of adjancent monomials suffices.
+  if (std::adjacent_find(monomialsCopy.begin(), monomialsCopy.end(),
+                         [](const Monomial &lhs, const Monomial &rhs) {
+                           return lhs.exponent == rhs.exponent;
+                         }) != monomialsCopy.end()) {
+    return failure();
+  }
+
+  return Polynomial(monomialsCopy);
+}
+
+Polynomial Polynomial::fromCoefficients(ArrayRef<int64_t> coeffs) {
+  llvm::SmallVector<Monomial> monomials;
+  auto size = coeffs.size();
+  monomials.reserve(size);
+  for (size_t i = 0; i < size; i++) {
+    monomials.emplace_back(coeffs[i], i);
+  }
+  auto result = Polynomial::fromMonomials(monomials);
+  // Construction guarantees unique exponents, so the failure mode of
+  // fromMonomials can be bypassed.
+  assert(succeeded(result));
+  return result.value();
+}
+
+void Polynomial::print(raw_ostream &os, ::llvm::StringRef separator,
+                       ::llvm::StringRef exponentiation) const {
+  bool first = true;
+  for (const Monomial &term : terms) {
+    if (first) {
+      first = false;
+    } else {
+      os << separator;
+    }
+    std::string coeffToPrint;
+    if (term.coefficient == 1 && term.exponent.uge(1)) {
+      coeffToPrint = "";
+    } else {
+      llvm::SmallString<16> coeffString;
+      term.coefficient.toStringSigned(coeffString);
+      coeffToPrint = coeffString.str();
+    }
+
+    if (term.exponent == 0) {
+      os << coeffToPrint;
+    } else if (term.exponent == 1) {
+      os << coeffToPrint << "x";
+    } else {
+      llvm::SmallString<16> expString;
+      term.exponent.toStringSigned(expString);
+      os << coeffToPrint << "x" << exponentiation << expString;
+    }
+  }
+}
+
+void Polynomial::print(raw_ostream &os) const { print(os, " + ", "**"); }
+
+std::string Polynomial::toIdentifier() const {
+  std::string result;
+  llvm::raw_string_ostream os(result);
+  print(os, "_", "");
+  return os.str();
+}
+
+unsigned Polynomial::getDegree() const {
+  return terms.back().exponent.getZExtValue();
+}
+
+} // namespace polynomial
+} // namespace mlir
diff --git a/mlir/lib/Dialect/Polynomial/IR/PolynomialAttributes.cpp b/mlir/lib/Dialect/Polynomial/IR/PolynomialAttributes.cpp
new file mode 100644
index 000000000000..ee09c73bb3c4
--- /dev/null
+++ b/mlir/lib/Dialect/Polynomial/IR/PolynomialAttributes.cpp
@@ -0,0 +1,213 @@
+//===- PolynomialAttributes.cpp - Polynomial dialect attrs ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "mlir/Dialect/Polynomial/IR/PolynomialAttributes.h"
+
+#include "mlir/Dialect/Polynomial/IR/Polynomial.h"
+#include "mlir/Support/LLVM.h"
+#include "mlir/Support/LogicalResult.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSet.h"
+
+namespace mlir {
+namespace polynomial {
+
+void PolynomialAttr::print(AsmPrinter &p) const {
+  p << '<';
+  p << getPolynomial();
+  p << '>';
+}
+
+/// Try to parse a monomial. If successful, populate the fields of the outparam
+/// `monomial` with the results, and the `variable` outparam with the parsed
+/// variable name. Sets shouldParseMore to true if the monomial is followed by
+/// a '+'.
+ParseResult parseMonomial(AsmParser &parser, Monomial &monomial,
+                          llvm::StringRef &variable, bool &isConstantTerm,
+                          bool &shouldParseMore) {
+  APInt parsedCoeff(apintBitWidth, 1);
+  auto parsedCoeffResult = parser.parseOptionalInteger(parsedCoeff);
+  monomial.coefficient = parsedCoeff;
+
+  isConstantTerm = false;
+  shouldParseMore = false;
+
+  // A + indicates it's a constant term with more to go, as in `1 + x`.
+  if (succeeded(parser.parseOptionalPlus())) {
+    // If no coefficient was parsed, and there's a +, then it's effectively
+    // parsing an empty string.
+    if (!parsedCoeffResult.has_value()) {
+      return failure();
+    }
+    monomial.exponent = APInt(apintBitWidth, 0);
+    isConstantTerm = true;
+    shouldParseMore = true;
+    return success();
+  }
+
+  // A monomial can be a trailing constant term, as in `x + 1`.
+  if (failed(parser.parseOptionalKeyword(&variable))) {
+    // If neither a coefficient nor a variable was found, then it's effectively
+    // parsing an empty string.
+    if (!parsedCoeffResult.has_value()) {
+      return failure();
+    }
+
+    monomial.exponent = APInt(apintBitWidth, 0);
+    isConstantTerm = true;
+    return success();
+  }
+
+  // Parse exponentiation symbol as `**`. We can't use caret because it's
+  // reserved for basic block identifiers If no star is present, it's treated
+  // as a polynomial with exponent 1.
+  if (succeeded(parser.parseOptionalStar())) {
+    // If there's one * there must be two.
+    if (failed(parser.parseStar())) {
+      return failure();
+    }
+
+    // If there's a **, then the integer exponent is required.
+    APInt parsedExponent(apintBitWidth, 0);
+    if (failed(parser.parseInteger(parsedExponent))) {
+      parser.emitError(parser.getCurrentLocation(),
+                       "found invalid integer exponent");
+      return failure();
+    }
+
+    monomial.exponent = parsedExponent;
+  } else {
+    monomial.exponent = APInt(apintBitWidth, 1);
+  }
+
+  if (succeeded(parser.parseOptionalPlus())) {
+    shouldParseMore = true;
+  }
+  return success();
+}
+
+Attribute PolynomialAttr::parse(AsmParser &parser, Type type) {
+  if (failed(parser.parseLess()))
+    return {};
+
+  llvm::SmallVector<Monomial> monomials;
+  llvm::StringSet<> variables;
+
+  while (true) {
+    Monomial parsedMonomial;
+    llvm::StringRef parsedVariableRef;
+    bool isConstantTerm;
+    bool shouldParseMore;
+    if (failed(parseMonomial(parser, parsedMonomial, parsedVariableRef,
+                             isConstantTerm, shouldParseMore))) {
+      parser.emitError(parser.getCurrentLocation(), "expected a monomial");
+      return {};
+    }
+
+    if (!isConstantTerm) {
+      std::string parsedVariable = parsedVariableRef.str();
+      variables.insert(parsedVariable);
+    }
+    monomials.push_back(parsedMonomial);
+
+    if (shouldParseMore)
+      continue;
+
+    if (succeeded(parser.parseOptionalGreater())) {
+      break;
+    }
+    parser.emitError(
+        parser.getCurrentLocation(),
+        "expected + and more monomials, or > to end polynomial attribute");
+    return {};
+  }
+
+  if (variables.size() > 1) {
+    std::string vars = llvm::join(variables.keys(), ", ");
+    parser.emitError(
+        parser.getCurrentLocation(),
+        "polynomials must have one indeterminate, but there were multiple: " +
+            vars);
+  }
+
+  auto result = Polynomial::fromMonomials(monomials);
+  if (failed(result)) {
+    parser.emitError(parser.getCurrentLocation())
+        << "parsed polynomial must have unique exponents among monomials";
+    return {};
+  }
+  return PolynomialAttr::get(parser.getContext(), result.value());
+}
+
+void RingAttr::print(AsmPrinter &p) const {
+  p << "#polynomial.ring<coefficientType=" << getCoefficientType()
+    << ", coefficientModulus=" << getCoefficientModulus()
+    << ", polynomialModulus=" << getPolynomialModulus() << '>';
+}
+
+Attribute RingAttr::parse(AsmParser &parser, Type type) {
+  if (failed(parser.parseLess()))
+    return {};
+
+  if (failed(parser.parseKeyword("coefficientType")))
+    return {};
+
+  if (failed(parser.parseEqual()))
+    return {};
+
+  Type ty;
+  if (failed(parser.parseType(ty)))
+    return {};
+
+  if (failed(parser.parseComma()))
+    return {};
+
+  IntegerAttr coefficientModulusAttr = nullptr;
+  if (succeeded(parser.parseKeyword("coefficientModulus"))) {
+    if (failed(parser.parseEqual()))
+      return {};
+
+    IntegerType iType = ty.dyn_cast<IntegerType>();
+    if (!iType) {
+      parser.emitError(parser.getCurrentLocation(),
+                       "coefficientType must specify an integer type");
+      return {};
+    }
+    APInt coefficientModulus(iType.getWidth(), 0);
+    auto result = parser.parseInteger(coefficientModulus);
+    if (failed(result)) {
+      parser.emitError(parser.getCurrentLocation(),
+                       "invalid coefficient modulus");
+      return {};
+    }
+    coefficientModulusAttr = IntegerAttr::get(iType, coefficientModulus);
+
+    if (failed(parser.parseComma()))
+      return {};
+  }
+
+  PolynomialAttr polyAttr = nullptr;
+  if (succeeded(parser.parseKeyword("polynomialModulus"))) {
+    if (failed(parser.parseEqual()))
+      return {};
+
+    PolynomialAttr attr;
+    if (failed(parser.parseAttribute<PolynomialAttr>(attr)))
+      return {};
+    polyAttr = attr;
+  }
+
+  if (failed(parser.parseGreater()))
+    return {};
+
+  return RingAttr::get(parser.getContext(), ty, coefficientModulusAttr,
+                       polyAttr);
+}
+
+} // namespace polynomial
+} // namespace mlir
diff --git a/mlir/lib/Dialect/Polynomial/IR/PolynomialDialect.cpp b/mlir/lib/Dialect/Polynomial/IR/PolynomialDialect.cpp
new file mode 100644
index 000000000000..a672a59b8a46
--- /dev/null
+++ b/mlir/lib/Dialect/Polynomial/IR/PolynomialDialect.cpp
@@ -0,0 +1,41 @@
+//===- PolynomialDialect.cpp - Polynomial dialect ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Polynomial/IR/Polynomial.h"
+
+#include "mlir/Dialect/Polynomial/IR/PolynomialAttributes.h"
+#include "mlir/Dialect/Polynomial/IR/PolynomialOps.h"
+#include "mlir/Dialect/Polynomial/IR/PolynomialTypes.h"
+#include "llvm/ADT/TypeSwitch.h"
+
+using namespace mlir;
+using namespace mlir::polynomial;
+
+#include "mlir/Dialect/Polynomial/IR/PolynomialDialect.cpp.inc"
+
+#define GET_ATTRDEF_CLASSES
+#include "mlir/Dialect/Polynomial/IR/PolynomialAttributes.cpp.inc"
+#define GET_TYPEDEF_CLASSES
+#include "mlir/Dialect/Polynomial/IR/PolynomialTypes.cpp.inc"
+#define GET_OP_CLASSES
+#include "mlir/Dialect/Polynomial/IR/Polynomial.cpp.inc"
+
+void PolynomialDialect::initialize() {
+  addAttributes<
+#define GET_ATTRDEF_LIST
+#include "mlir/Dialect/Polynomial/IR/PolynomialAttributes.cpp.inc"
+      >();
+  addTypes<
+#define GET_TYPEDEF_LIST
+#include "mlir/Dialect/Polynomial/IR/PolynomialTypes.cpp.inc"
+      >();
+  addOperations<
+#define GET_OP_LIST
+#include "mlir/Dialect/Polynomial/IR/Polynomial.cpp.inc"
+      >();
+}
diff --git a/mlir/lib/Dialect/Polynomial/IR/PolynomialOps.cpp b/mlir/lib/Dialect/Polynomial/IR/PolynomialOps.cpp
new file mode 100644
index 000000000000..96c59a28b8fd
--- /dev/null
+++ b/mlir/lib/Dialect/Polynomial/IR/PolynomialOps.cpp
@@ -0,0 +1,15 @@
+//===- PolynomialOps.cpp - Polynomial dialect ops ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Polynomial/IR/Polynomial.h"
+
+using namespace mlir;
+using namespace mlir::polynomial;
+
+#define GET_OP_CLASSES
+#include "mlir/Dialect/Polynomial/IR/Polynomial.cpp.inc"
diff --git a/mlir/test/Dialect/Polynomial/attributes.mlir b/mlir/test/Dialect/Polynomial/attributes.mlir
new file mode 100644
index 000000000000..3973ae394433
--- /dev/null
+++ b/mlir/test/Dialect/Polynomial/attributes.mlir
@@ -0,0 +1,45 @@
+// RUN: mlir-opt %s --split-input-file --verify-diagnostics
+
+#my_poly = #polynomial.polynomial<y + x**1024>
+// expected-error@below {{polynomials must have one indeterminate, but there were multiple: x, y}}
+#ring1 = #polynomial.ring<coefficientType=i32, coefficientModulus=2837465, polynomialModulus=#my_poly>
+
+// -----
+
+// expected-error@below {{expected integer value}}
+// expected-error@below {{expected a monomial}}
+// expected-error@below {{found invalid integer exponent}}
+#my_poly = #polynomial.polynomial<5 + x**f>
+#ring1 = #polynomial.ring<coefficientType=i32, coefficientModulus=2837465, polynomialModulus=#my_poly>
+
+// -----
+
+#my_poly = #polynomial.polynomial<5 + x**2 + 3x**2>
+// expected-error@below {{parsed polynomial must have unique exponents among monomials}}
+#ring1 = #polynomial.ring<coefficientType=i32, coefficientModulus=2837465, polynomialModulus=#my_poly>
+
+// -----
+
+// expected-error@below {{expected + and more monomials, or > to end polynomial attribute}}
+#my_poly = #polynomial.polynomial<5 + x**2 7>
+#ring1 = #polynomial.ring<coefficientType=i32, coefficientModulus=2837465, polynomialModulus=#my_poly>
+
+// -----
+
+// expected-error@below {{expected a monomial}}
+#my_poly = #polynomial.polynomial<5 + x**2 +>
+#ring1 = #polynomial.ring<coefficientType=i32, coefficientModulus=2837465, polynomialModulus=#my_poly>
+
+
+// -----
+
+#my_poly = #polynomial.polynomial<5 + x**2>
+// expected-error@below {{coefficientType must specify an integer type}}
+#ring1 = #polynomial.ring<coefficientType=f64, coefficientModulus=2837465, polynomialModulus=#my_poly>
+
+// -----
+
+#my_poly = #polynomial.polynomial<5 + x**2>
+// expected-error@below {{expected integer value}}
+// expected-error@below {{invalid coefficient modulus}}
+#ring1 = #polynomial.ring<coefficientType=i32, coefficientModulus=x, polynomialModulus=#my_poly>
diff --git a/mlir/test/Dialect/Polynomial/types.mlir b/mlir/test/Dialect/Polynomial/types.mlir
new file mode 100644
index 000000000000..64b74d9d36bb
--- /dev/null
+++ b/mlir/test/Dialect/Polynomial/types.mlir
@@ -0,0 +1,42 @@
+// RUN: mlir-opt %s | FileCheck %s
+
+// CHECK-LABEL: func @test_types
+// CHECK-SAME:  !polynomial.polynomial<
+// CHECK-SAME:    #polynomial.ring<
+// CHECK-SAME:       coefficientType=i32,
+// CHECK-SAME:       coefficientModulus=2837465 : i32,
+// CHECK-SAME:       polynomialModulus=#polynomial.polynomial<1 + x**1024>>>
+#my_poly = #polynomial.polynomial<1 + x**1024>
+#ring1 = #polynomial.ring<coefficientType=i32, coefficientModulus=2837465, polynomialModulus=#my_poly>
+!ty = !polynomial.polynomial<#ring1>
+func.func @test_types(%0: !ty) -> !ty {
+  return %0 : !ty
+}
+
+
+// CHECK-LABEL: func @test_non_x_variable_64_bit
+// CHECK-SAME:  !polynomial.polynomial<
+// CHECK-SAME:    #polynomial.ring<
+// CHECK-SAME:       coefficientType=i64,
+// CHECK-SAME:       coefficientModulus=2837465 : i64,
+// CHECK-SAME:       polynomialModulus=#polynomial.polynomial<2 + 4x + x**3>>>
+#my_poly_2 = #polynomial.polynomial<t**3 + 4t + 2>
+#ring2 = #polynomial.ring<coefficientType=i64, coefficientModulus=2837465, polynomialModulus=#my_poly_2>
+!ty2 = !polynomial.polynomial<#ring2>
+func.func @test_non_x_variable_64_bit(%0: !ty2) -> !ty2 {
+  return %0 : !ty2
+}
+
+
+// CHECK-LABEL: func @test_linear_poly
+// CHECK-SAME:  !polynomial.polynomial<
+// CHECK-SAME:    #polynomial.ring<
+// CHECK-SAME:       coefficientType=i32,
+// CHECK-SAME:       coefficientModulus=12 : i32,
+// CHECK-SAME:       polynomialModulus=#polynomial.polynomial<4x>>
+#my_poly_3 = #polynomial.polynomial<4x>
+#ring3 = #polynomial.ring<coefficientType=i32, coefficientModulus=12, polynomialModulus=#my_poly_3>
+!ty3 = !polynomial.polynomial<#ring3>
+func.func @test_linear_poly(%0: !ty3) -> !ty3 {
+  return %0 : !ty3
+}
-- 
GitLab


From f8d314f0ee5e750e1e47b47fb55d14f6d1e991e1 Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Mon, 15 Apr 2024 18:14:18 +0200
Subject: [PATCH 257/464] [mlir][Interfaces][NFC] Add TableGen test op for
 value bounds tests (#88717)

This commit is a code cleanup. It defines the test ops the are used for
the `ValueBoundsOpInterface` tests in TableGen, along with proper
verifiers.

---------

Co-authored-by: Benjamin Maxwell <benjamin.maxwell@arm.com>
---
 .../value-bounds-op-interface-impl.mlir       |   2 +-
 .../Affine/value-bounds-reification.mlir      |   6 +-
 .../value-bounds-op-interface-impl.mlir       |   4 +-
 .../Dialect/Vector/test-scalable-bounds.mlir  |  18 +-
 mlir/test/lib/Dialect/Affine/CMakeLists.txt   |   8 +
 .../Dialect/Affine/TestReifyValueBounds.cpp   | 305 ++++++------------
 mlir/test/lib/Dialect/Test/CMakeLists.txt     |   1 +
 mlir/test/lib/Dialect/Test/TestDialect.cpp    |  48 +++
 mlir/test/lib/Dialect/Test/TestDialect.h      |   1 +
 mlir/test/lib/Dialect/Test/TestOps.td         |  47 +++
 .../mlir/test/BUILD.bazel                     |   2 +
 11 files changed, 226 insertions(+), 216 deletions(-)

diff --git a/mlir/test/Dialect/Affine/value-bounds-op-interface-impl.mlir b/mlir/test/Dialect/Affine/value-bounds-op-interface-impl.mlir
index 10da91870f49..23c6872dcebe 100644
--- a/mlir/test/Dialect/Affine/value-bounds-op-interface-impl.mlir
+++ b/mlir/test/Dialect/Affine/value-bounds-op-interface-impl.mlir
@@ -74,7 +74,7 @@ func.func @composed_affine_apply(%i1 : index) -> (index) {
   %i2 = affine.apply affine_map<(d0) -> ((d0 floordiv 32) * 16)>(%i1)
   %i3 = affine.apply affine_map<(d0) -> ((d0 floordiv 32) * 16 + 8)>(%i1)
   %s = affine.apply affine_map<()[s0, s1] -> (s0 - s1)>()[%i2, %i3]
-  %reified = "test.reify_constant_bound"(%s) {type = "EQ"} : (index) -> (index)
+  %reified = "test.reify_bound"(%s) {type = "EQ", constant} : (index) -> (index)
   return %reified : index
 }
 
diff --git a/mlir/test/Dialect/Affine/value-bounds-reification.mlir b/mlir/test/Dialect/Affine/value-bounds-reification.mlir
index 909c9098c516..75622f59af83 100644
--- a/mlir/test/Dialect/Affine/value-bounds-reification.mlir
+++ b/mlir/test/Dialect/Affine/value-bounds-reification.mlir
@@ -47,7 +47,7 @@ func.func @reify_slice_bound(%t: tensor<?x?xi32>, %idx: index, %ub: index, %f: f
     %bound = "test.reify_bound"(%filled) {dim = 1, type = "UB"} : (tensor<1x?xi32>) -> (index)
     "test.some_use"(%bound) : (index) -> ()
 
-    %bound_const = "test.reify_constant_bound"(%filled) {dim = 1, type = "UB"} : (tensor<1x?xi32>) -> (index)
+    %bound_const = "test.reify_bound"(%filled) {dim = 1, type = "UB", constant} : (tensor<1x?xi32>) -> (index)
     "test.some_use"(%bound_const) : (index) -> ()
   }
   return
@@ -93,7 +93,7 @@ func.func @reify_slice_bound2(%lb0: index, %ub0: index, %step0: index,
 
     // CHECK: %[[c129:.*]] = arith.constant 129 : index
     // CHECK: "test.some_use"(%[[c129]])
-    %lb1_ub_const = "test.reify_constant_bound"(%lb1) {type = "UB"} : (index) -> (index)
+    %lb1_ub_const = "test.reify_bound"(%lb1) {type = "UB", constant} : (index) -> (index)
     "test.some_use"(%lb1_ub_const) : (index) -> ()
 
     scf.for %iv1 = %lb1 to %ub1 step %c32 {
@@ -116,7 +116,7 @@ func.func @reify_slice_bound2(%lb0: index, %ub0: index, %step0: index,
 
         // CHECK: %[[c32:.*]] = arith.constant 32 : index
         // CHECK: "test.some_use"(%[[c32]])
-        %matmul_ub_const = "test.reify_constant_bound"(%matmul) {dim = 1, type = "UB"} : (tensor<1x?xi32>) -> (index)
+        %matmul_ub_const = "test.reify_bound"(%matmul) {dim = 1, type = "UB", constant} : (tensor<1x?xi32>) -> (index)
         "test.some_use"(%matmul_ub_const) : (index) -> ()
       }
     }
diff --git a/mlir/test/Dialect/Tensor/value-bounds-op-interface-impl.mlir b/mlir/test/Dialect/Tensor/value-bounds-op-interface-impl.mlir
index 0c90bcdb4202..0ba9983723a0 100644
--- a/mlir/test/Dialect/Tensor/value-bounds-op-interface-impl.mlir
+++ b/mlir/test/Dialect/Tensor/value-bounds-op-interface-impl.mlir
@@ -83,7 +83,7 @@ func.func @extract_slice_static(%t: tensor<?xf32>) -> index {
 func.func @extract_slice_dynamic_constant(%t: tensor<?xf32>, %sz: index) -> index {
   %0 = tensor.extract_slice %t[2][%sz][1] : tensor<?xf32> to tensor<?xf32>
   // expected-error @below{{could not reify bound}}
-  %1 = "test.reify_constant_bound"(%0) {dim = 0} : (tensor<?xf32>) -> (index)
+  %1 = "test.reify_bound"(%0) {dim = 0, constant} : (tensor<?xf32>) -> (index)
   return %1 : index
 }
 
@@ -95,7 +95,7 @@ func.func @extract_slice_dynamic_constant(%t: tensor<?xf32>, %sz: index) -> inde
 //       CHECK:   return %[[c5]]
 func.func @extract_slice_static_constant(%t: tensor<?xf32>) -> index {
   %0 = tensor.extract_slice %t[2][5][1] : tensor<?xf32> to tensor<5xf32>
-  %1 = "test.reify_constant_bound"(%0) {dim = 0} : (tensor<5xf32>) -> (index)
+  %1 = "test.reify_bound"(%0) {dim = 0, constant} : (tensor<5xf32>) -> (index)
   return %1 : index
 }
 
diff --git a/mlir/test/Dialect/Vector/test-scalable-bounds.mlir b/mlir/test/Dialect/Vector/test-scalable-bounds.mlir
index 245a6f5c13ac..d549c5bd1c37 100644
--- a/mlir/test/Dialect/Vector/test-scalable-bounds.mlir
+++ b/mlir/test/Dialect/Vector/test-scalable-bounds.mlir
@@ -26,8 +26,8 @@ func.func @fixed_size_loop_nest() {
     %min_i = affine.min #map_dim_i(%i)[%c4_vscale]
     scf.for %j = %c0 to %c16 step %c4_vscale {
       %min_j = affine.min #map_dim_j(%j)[%c4_vscale]
-      %bound_i = "test.reify_scalable_bound"(%min_i) {type = "UB", vscale_min = 1, vscale_max = 16} : (index) -> index
-      %bound_j = "test.reify_scalable_bound"(%min_j) {type = "UB", vscale_min = 1, vscale_max = 16} : (index) -> index
+      %bound_i = "test.reify_bound"(%min_i) {type = "UB", vscale_min = 1, vscale_max = 16, scalable} : (index) -> index
+      %bound_j = "test.reify_bound"(%min_j) {type = "UB", vscale_min = 1, vscale_max = 16, scalable} : (index) -> index
       "test.some_use"(%bound_i, %bound_j) : (index, index) -> ()
     }
   }
@@ -58,8 +58,8 @@ func.func @dynamic_size_loop_nest(%dim0: index, %dim1: index) {
     %min_i = affine.min #map_dynamic_dim(%i)[%c4_vscale, %dim0]
     scf.for %j = %c0 to %dim1 step %c4_vscale {
       %min_j = affine.min #map_dynamic_dim(%j)[%c4_vscale, %dim1]
-      %bound_i = "test.reify_scalable_bound"(%min_i) {type = "UB", vscale_min = 1, vscale_max = 16} : (index) -> index
-      %bound_j = "test.reify_scalable_bound"(%min_j) {type = "UB", vscale_min = 1, vscale_max = 16} : (index) -> index
+      %bound_i = "test.reify_bound"(%min_i) {type = "UB", vscale_min = 1, vscale_max = 16, scalable} : (index) -> index
+      %bound_j = "test.reify_bound"(%min_j) {type = "UB", vscale_min = 1, vscale_max = 16, scalable} : (index) -> index
       "test.some_use"(%bound_i, %bound_j) : (index, index) -> ()
     }
   }
@@ -80,7 +80,7 @@ func.func @add_to_vscale() {
   %vscale = vector.vscale
   %c8 = arith.constant 8 : index
   %vscale_plus_c8 = arith.addi %vscale, %c8 : index
-  %bound = "test.reify_scalable_bound"(%vscale_plus_c8) {type = "EQ", vscale_min = 1, vscale_max = 16} : (index) -> index
+  %bound = "test.reify_bound"(%vscale_plus_c8) {type = "EQ", vscale_min = 1, vscale_max = 16, scalable} : (index) -> index
   "test.some_use"(%bound) : (index) -> ()
   return
 }
@@ -94,7 +94,7 @@ func.func @add_to_vscale() {
 //       CHECK:   "test.some_use"(%[[C2]]) : (index) -> ()
 func.func @vscale_fixed_size() {
   %vscale = vector.vscale
-  %bound = "test.reify_scalable_bound"(%vscale) {type = "EQ", vscale_min = 2, vscale_max = 2} : (index) -> index
+  %bound = "test.reify_bound"(%vscale) {type = "EQ", vscale_min = 2, vscale_max = 2, scalable} : (index) -> index
   "test.some_use"(%bound) : (index) -> ()
   return
 }
@@ -107,7 +107,7 @@ func.func @unknown_bound(%a: index) {
   %vscale = vector.vscale
   %vscale_plus_a = arith.muli %vscale, %a : index
   // expected-error @below{{could not reify bound}}
-  %bound = "test.reify_scalable_bound"(%vscale_plus_a) {type = "UB", vscale_min = 1, vscale_max = 16} : (index) -> index
+  %bound = "test.reify_bound"(%vscale_plus_a) {type = "UB", vscale_min = 1, vscale_max = 16, scalable} : (index) -> index
   "test.some_use"(%bound) : (index) -> ()
   return
 }
@@ -134,7 +134,7 @@ func.func @duplicate_vscale_values() {
   %c2_vscale = arith.muli %vscale_1, %c2 : index
   %add = arith.addi %c2_vscale, %c4_vscale : index
 
-  %bound = "test.reify_scalable_bound"(%add) {type = "EQ", vscale_min = 1, vscale_max = 16} : (index) -> index
+  %bound = "test.reify_bound"(%add) {type = "EQ", vscale_min = 1, vscale_max = 16, scalable} : (index) -> index
   "test.some_use"(%bound) : (index) -> ()
   return
 }
@@ -154,7 +154,7 @@ func.func @non_scalable_code() {
   %c0 = arith.constant 0 : index
   scf.for %i = %c0 to %c1024 step %c4 {
     %min_i = affine.min #map_dim_i(%i)[%c4]
-    %bound_i = "test.reify_scalable_bound"(%min_i) {type = "UB", vscale_min = 1, vscale_max = 16} : (index) -> index
+    %bound_i = "test.reify_bound"(%min_i) {type = "UB", vscale_min = 1, vscale_max = 16, scalable} : (index) -> index
     "test.some_use"(%bound_i) : (index) -> ()
   }
   return
diff --git a/mlir/test/lib/Dialect/Affine/CMakeLists.txt b/mlir/test/lib/Dialect/Affine/CMakeLists.txt
index 14960a45d39b..33cefab9fa2e 100644
--- a/mlir/test/lib/Dialect/Affine/CMakeLists.txt
+++ b/mlir/test/lib/Dialect/Affine/CMakeLists.txt
@@ -30,5 +30,13 @@ add_mlir_library(MLIRAffineTransformsTestPasses
   MLIRSupport
   MLIRMemRefDialect
   MLIRTensorDialect
+  MLIRTestDialect
   MLIRVectorUtils
   )
+
+target_include_directories(MLIRAffineTransformsTestPasses
+  PRIVATE
+  ${CMAKE_CURRENT_SOURCE_DIR}/../Test
+  ${CMAKE_CURRENT_BINARY_DIR}/../Test
+  )
+
diff --git a/mlir/test/lib/Dialect/Affine/TestReifyValueBounds.cpp b/mlir/test/lib/Dialect/Affine/TestReifyValueBounds.cpp
index f38631054fb3..6730f9b292ad 100644
--- a/mlir/test/lib/Dialect/Affine/TestReifyValueBounds.cpp
+++ b/mlir/test/lib/Dialect/Affine/TestReifyValueBounds.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "TestDialect.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Affine/IR/ValueBoundsOpInterfaceImpl.h"
 #include "mlir/Dialect/Affine/Transforms/Transforms.h"
@@ -57,31 +58,6 @@ private:
 
 } // namespace
 
-static FailureOr<BoundType> parseBoundType(const std::string &type) {
-  if (type == "EQ")
-    return BoundType::EQ;
-  if (type == "LB")
-    return BoundType::LB;
-  if (type == "UB")
-    return BoundType::UB;
-  return failure();
-}
-
-static FailureOr<ValueBoundsConstraintSet::ComparisonOperator>
-parseComparisonOperator(const std::string &type) {
-  if (type == "EQ")
-    return ValueBoundsConstraintSet::ComparisonOperator::EQ;
-  if (type == "LT")
-    return ValueBoundsConstraintSet::ComparisonOperator::LT;
-  if (type == "LE")
-    return ValueBoundsConstraintSet::ComparisonOperator::LE;
-  if (type == "GT")
-    return ValueBoundsConstraintSet::ComparisonOperator::GT;
-  if (type == "GE")
-    return ValueBoundsConstraintSet::ComparisonOperator::GE;
-  return failure();
-}
-
 static ValueBoundsConstraintSet::ComparisonOperator
 invertComparisonOperator(ValueBoundsConstraintSet::ComparisonOperator cmp) {
   if (cmp == ValueBoundsConstraintSet::ComparisonOperator::LT)
@@ -101,144 +77,89 @@ static LogicalResult testReifyValueBounds(func::FuncOp funcOp,
                                           bool reifyToFuncArgs,
                                           bool useArithOps) {
   IRRewriter rewriter(funcOp.getContext());
-  WalkResult result = funcOp.walk([&](Operation *op) {
-    // Look for test.reify_bound ops.
-    if (op->getName().getStringRef() == "test.reify_bound" ||
-        op->getName().getStringRef() == "test.reify_constant_bound" ||
-        op->getName().getStringRef() == "test.reify_scalable_bound") {
-      if (op->getNumOperands() != 1 || op->getNumResults() != 1 ||
-          !op->getResultTypes()[0].isIndex()) {
-        op->emitOpError("invalid op");
-        return WalkResult::skip();
-      }
-      Value value = op->getOperand(0);
-      if (isa<IndexType>(value.getType()) !=
-          !op->hasAttrOfType<IntegerAttr>("dim")) {
-        // Op should have "dim" attribute if and only if the operand is an
-        // index-typed value.
-        op->emitOpError("invalid op");
-        return WalkResult::skip();
-      }
-
-      // Get bound type.
-      std::string boundTypeStr = "EQ";
-      if (auto boundTypeAttr = op->getAttrOfType<StringAttr>("type"))
-        boundTypeStr = boundTypeAttr.str();
-      auto boundType = parseBoundType(boundTypeStr);
-      if (failed(boundType)) {
-        op->emitOpError("invalid op");
-        return WalkResult::interrupt();
-      }
-
-      // Get shape dimension (if any).
-      auto dim = value.getType().isIndex()
-                     ? std::nullopt
-                     : std::make_optional<int64_t>(
-                           op->getAttrOfType<IntegerAttr>("dim").getInt());
-
-      // Check if a constant was requested.
-      bool constant =
-          op->getName().getStringRef() == "test.reify_constant_bound";
-
-      bool scalable = !constant && op->getName().getStringRef() ==
-                                       "test.reify_scalable_bound";
-
-      // Prepare stop condition. By default, reify in terms of the op's
-      // operands. No stop condition is used when a constant was requested.
-      std::function<bool(Value, std::optional<int64_t>,
-                         ValueBoundsConstraintSet & cstr)>
-          stopCondition = [&](Value v, std::optional<int64_t> d,
-                              ValueBoundsConstraintSet &cstr) {
-            // Reify in terms of SSA values that are different from `value`.
-            return v != value;
-          };
-      if (reifyToFuncArgs) {
-        // Reify in terms of function block arguments.
-        stopCondition = [](Value v, std::optional<int64_t> d,
-                           ValueBoundsConstraintSet &cstr) {
-          auto bbArg = dyn_cast<BlockArgument>(v);
-          if (!bbArg)
-            return false;
-          return isa<FunctionOpInterface>(
-              bbArg.getParentBlock()->getParentOp());
+  WalkResult result = funcOp.walk([&](test::ReifyBoundOp op) {
+    auto boundType = op.getBoundType();
+    Value value = op.getVar();
+    std::optional<int64_t> dim = op.getDim();
+    bool constant = op.getConstant();
+    bool scalable = op.getScalable();
+
+    // Prepare stop condition. By default, reify in terms of the op's
+    // operands. No stop condition is used when a constant was requested.
+    std::function<bool(Value, std::optional<int64_t>,
+                       ValueBoundsConstraintSet & cstr)>
+        stopCondition = [&](Value v, std::optional<int64_t> d,
+                            ValueBoundsConstraintSet &cstr) {
+          // Reify in terms of SSA values that are different from `value`.
+          return v != value;
         };
-      }
+    if (reifyToFuncArgs) {
+      // Reify in terms of function block arguments.
+      stopCondition = [](Value v, std::optional<int64_t> d,
+                         ValueBoundsConstraintSet &cstr) {
+        auto bbArg = dyn_cast<BlockArgument>(v);
+        if (!bbArg)
+          return false;
+        return isa<FunctionOpInterface>(bbArg.getParentBlock()->getParentOp());
+      };
+    }
 
-      // Reify value bound
-      rewriter.setInsertionPointAfter(op);
-      FailureOr<OpFoldResult> reified = failure();
-      if (constant) {
-        auto reifiedConst = ValueBoundsConstraintSet::computeConstantBound(
-            *boundType, value, dim, /*stopCondition=*/nullptr);
-        if (succeeded(reifiedConst))
-          reified =
-              FailureOr<OpFoldResult>(rewriter.getIndexAttr(*reifiedConst));
-      } else if (scalable) {
-        unsigned vscaleMin = 0;
-        unsigned vscaleMax = 0;
-        if (auto attr = "vscale_min"; op->hasAttrOfType<IntegerAttr>(attr)) {
-          vscaleMin = unsigned(op->getAttrOfType<IntegerAttr>(attr).getInt());
-        } else {
-          op->emitOpError("expected `vscale_min` to be provided");
-          return WalkResult::skip();
+    // Reify value bound
+    rewriter.setInsertionPointAfter(op);
+    FailureOr<OpFoldResult> reified = failure();
+    if (constant) {
+      auto reifiedConst = ValueBoundsConstraintSet::computeConstantBound(
+          boundType, value, dim, /*stopCondition=*/nullptr);
+      if (succeeded(reifiedConst))
+        reified = FailureOr<OpFoldResult>(rewriter.getIndexAttr(*reifiedConst));
+    } else if (scalable) {
+      auto loc = op->getLoc();
+      auto reifiedScalable =
+          vector::ScalableValueBoundsConstraintSet::computeScalableBound(
+              value, dim, *op.getVscaleMin(), *op.getVscaleMax(), boundType);
+      if (succeeded(reifiedScalable)) {
+        SmallVector<std::pair<Value, std::optional<int64_t>>, 1> vscaleOperand;
+        if (reifiedScalable->map.getNumInputs() == 1) {
+          // The only possible input to the bound is vscale.
+          vscaleOperand.push_back(std::make_pair(
+              rewriter.create<vector::VectorScaleOp>(loc), std::nullopt));
         }
-        if (auto attr = "vscale_max"; op->hasAttrOfType<IntegerAttr>(attr)) {
-          vscaleMax = unsigned(op->getAttrOfType<IntegerAttr>(attr).getInt());
+        reified = affine::materializeComputedBound(
+            rewriter, loc, reifiedScalable->map, vscaleOperand);
+      }
+    } else {
+      if (dim) {
+        if (useArithOps) {
+          reified = arith::reifyShapedValueDimBound(
+              rewriter, op->getLoc(), boundType, value, *dim, stopCondition);
         } else {
-          op->emitOpError("expected `vscale_max` to be provided");
-          return WalkResult::skip();
-        }
-
-        auto loc = op->getLoc();
-        auto reifiedScalable =
-            vector::ScalableValueBoundsConstraintSet::computeScalableBound(
-                value, dim, vscaleMin, vscaleMax, *boundType);
-        if (succeeded(reifiedScalable)) {
-          SmallVector<std::pair<Value, std::optional<int64_t>>, 1>
-              vscaleOperand;
-          if (reifiedScalable->map.getNumInputs() == 1) {
-            // The only possible input to the bound is vscale.
-            vscaleOperand.push_back(std::make_pair(
-                rewriter.create<vector::VectorScaleOp>(loc), std::nullopt));
-          }
-          reified = affine::materializeComputedBound(
-              rewriter, loc, reifiedScalable->map, vscaleOperand);
+          reified = reifyShapedValueDimBound(rewriter, op->getLoc(), boundType,
+                                             value, *dim, stopCondition);
         }
       } else {
-        if (dim) {
-          if (useArithOps) {
-            reified = arith::reifyShapedValueDimBound(
-                rewriter, op->getLoc(), *boundType, value, *dim, stopCondition);
-          } else {
-            reified = reifyShapedValueDimBound(
-                rewriter, op->getLoc(), *boundType, value, *dim, stopCondition);
-          }
+        if (useArithOps) {
+          reified = arith::reifyIndexValueBound(
+              rewriter, op->getLoc(), boundType, value, stopCondition);
         } else {
-          if (useArithOps) {
-            reified = arith::reifyIndexValueBound(
-                rewriter, op->getLoc(), *boundType, value, stopCondition);
-          } else {
-            reified = reifyIndexValueBound(rewriter, op->getLoc(), *boundType,
-                                           value, stopCondition);
-          }
+          reified = reifyIndexValueBound(rewriter, op->getLoc(), boundType,
+                                         value, stopCondition);
         }
       }
-      if (failed(reified)) {
-        op->emitOpError("could not reify bound");
-        return WalkResult::interrupt();
-      }
+    }
+    if (failed(reified)) {
+      op->emitOpError("could not reify bound");
+      return WalkResult::interrupt();
+    }
 
-      // Replace the op with the reified bound.
-      if (auto val = llvm::dyn_cast_if_present<Value>(*reified)) {
-        rewriter.replaceOp(op, val);
-        return WalkResult::skip();
-      }
-      Value constOp = rewriter.create<arith::ConstantIndexOp>(
-          op->getLoc(), cast<IntegerAttr>(reified->get<Attribute>()).getInt());
-      rewriter.replaceOp(op, constOp);
+    // Replace the op with the reified bound.
+    if (auto val = llvm::dyn_cast_if_present<Value>(*reified)) {
+      rewriter.replaceOp(op, val);
       return WalkResult::skip();
     }
-    return WalkResult::advance();
+    Value constOp = rewriter.create<arith::ConstantIndexOp>(
+        op->getLoc(), cast<IntegerAttr>(reified->get<Attribute>()).getInt());
+    rewriter.replaceOp(op, constOp);
+    return WalkResult::skip();
   });
   return failure(result.wasInterrupted());
 }
@@ -246,60 +167,42 @@ static LogicalResult testReifyValueBounds(func::FuncOp funcOp,
 /// Look for "test.compare" ops and emit errors/remarks.
 static LogicalResult testEquality(func::FuncOp funcOp) {
   IRRewriter rewriter(funcOp.getContext());
-  WalkResult result = funcOp.walk([&](Operation *op) {
-    // Look for test.compare ops.
-    if (op->getName().getStringRef() == "test.compare") {
-      if (op->getNumOperands() != 2 || !op->getOperand(0).getType().isIndex() ||
-          !op->getOperand(1).getType().isIndex()) {
-        op->emitOpError("invalid op");
-        return WalkResult::skip();
-      }
-
-      // Get comparison operator.
-      std::string cmpStr = "EQ";
-      if (auto cmpAttr = op->getAttrOfType<StringAttr>("cmp"))
-        cmpStr = cmpAttr.str();
-      auto cmpType = parseComparisonOperator(cmpStr);
-      if (failed(cmpType)) {
-        op->emitOpError("invalid comparison operator");
+  WalkResult result = funcOp.walk([&](test::CompareOp op) {
+    auto cmpType = op.getComparisonOperator();
+    if (op.getCompose()) {
+      if (cmpType != ValueBoundsConstraintSet::EQ) {
+        op->emitOpError(
+            "comparison operator must be EQ when 'composed' is specified");
         return WalkResult::interrupt();
       }
-
-      if (op->hasAttr("compose")) {
-        if (cmpType != ValueBoundsConstraintSet::EQ) {
-          op->emitOpError(
-              "comparison operator must be EQ when 'composed' is specified");
-          return WalkResult::interrupt();
-        }
-        FailureOr<int64_t> delta = affine::fullyComposeAndComputeConstantDelta(
-            op->getOperand(0), op->getOperand(1));
-        if (failed(delta)) {
-          op->emitError("could not determine equality");
-        } else if (*delta == 0) {
-          op->emitRemark("equal");
-        } else {
-          op->emitRemark("different");
-        }
-        return WalkResult::advance();
-      }
-
-      auto compare = [&](ValueBoundsConstraintSet::ComparisonOperator cmp) {
-        return ValueBoundsConstraintSet::compare(
-            /*lhs=*/op->getOperand(0), /*lhsDim=*/std::nullopt, cmp,
-            /*rhs=*/op->getOperand(1), /*rhsDim=*/std::nullopt);
-      };
-      if (compare(*cmpType)) {
-        op->emitRemark("true");
-      } else if (*cmpType != ValueBoundsConstraintSet::EQ &&
-                 compare(invertComparisonOperator(*cmpType))) {
-        op->emitRemark("false");
-      } else if (*cmpType == ValueBoundsConstraintSet::EQ &&
-                 (compare(ValueBoundsConstraintSet::ComparisonOperator::LT) ||
-                  compare(ValueBoundsConstraintSet::ComparisonOperator::GT))) {
-        op->emitRemark("false");
+      FailureOr<int64_t> delta = affine::fullyComposeAndComputeConstantDelta(
+          op->getOperand(0), op->getOperand(1));
+      if (failed(delta)) {
+        op->emitError("could not determine equality");
+      } else if (*delta == 0) {
+        op->emitRemark("equal");
       } else {
-        op->emitError("unknown");
+        op->emitRemark("different");
       }
+      return WalkResult::advance();
+    }
+
+    auto compare = [&](ValueBoundsConstraintSet::ComparisonOperator cmp) {
+      return ValueBoundsConstraintSet::compare(
+          /*lhs=*/op.getLhs(), /*lhsDim=*/std::nullopt, cmp,
+          /*rhs=*/op.getRhs(), /*rhsDim=*/std::nullopt);
+    };
+    if (compare(cmpType)) {
+      op->emitRemark("true");
+    } else if (cmpType != ValueBoundsConstraintSet::EQ &&
+               compare(invertComparisonOperator(cmpType))) {
+      op->emitRemark("false");
+    } else if (cmpType == ValueBoundsConstraintSet::EQ &&
+               (compare(ValueBoundsConstraintSet::ComparisonOperator::LT) ||
+                compare(ValueBoundsConstraintSet::ComparisonOperator::GT))) {
+      op->emitRemark("false");
+    } else {
+      op->emitError("unknown");
     }
     return WalkResult::advance();
   });
diff --git a/mlir/test/lib/Dialect/Test/CMakeLists.txt b/mlir/test/lib/Dialect/Test/CMakeLists.txt
index 47ddcf652474..d246c0492a3b 100644
--- a/mlir/test/lib/Dialect/Test/CMakeLists.txt
+++ b/mlir/test/lib/Dialect/Test/CMakeLists.txt
@@ -85,6 +85,7 @@ add_mlir_library(MLIRTestDialect
   MLIRTensorDialect
   MLIRTransformUtils
   MLIRTransforms
+  MLIRValueBoundsOpInterface
 )
 
 add_mlir_translation_library(MLIRTestFromLLVMIRTranslation
diff --git a/mlir/test/lib/Dialect/Test/TestDialect.cpp b/mlir/test/lib/Dialect/Test/TestDialect.cpp
index 74378633032c..25c5190ca0ef 100644
--- a/mlir/test/lib/Dialect/Test/TestDialect.cpp
+++ b/mlir/test/lib/Dialect/Test/TestDialect.cpp
@@ -516,6 +516,54 @@ static void printOptionalCustomParser(AsmPrinter &p, Operation *,
   p.printAttribute(result);
 }
 
+//===----------------------------------------------------------------------===//
+// ReifyBoundOp
+//===----------------------------------------------------------------------===//
+
+::mlir::presburger::BoundType ReifyBoundOp::getBoundType() {
+  if (getType() == "EQ")
+    return ::mlir::presburger::BoundType::EQ;
+  if (getType() == "LB")
+    return ::mlir::presburger::BoundType::LB;
+  if (getType() == "UB")
+    return ::mlir::presburger::BoundType::UB;
+  llvm_unreachable("invalid bound type");
+}
+
+LogicalResult ReifyBoundOp::verify() {
+  if (isa<ShapedType>(getVar().getType())) {
+    if (!getDim().has_value())
+      return emitOpError("expected 'dim' attribute for shaped type variable");
+  } else if (getVar().getType().isIndex()) {
+    if (getDim().has_value())
+      return emitOpError("unexpected 'dim' attribute for index variable");
+  } else {
+    return emitOpError("expected index-typed variable or shape type variable");
+  }
+  if (getConstant() && getScalable())
+    return emitOpError("'scalable' and 'constant' are mutually exlusive");
+  if (getScalable() != getVscaleMin().has_value())
+    return emitOpError("expected 'vscale_min' if and only if 'scalable'");
+  if (getScalable() != getVscaleMax().has_value())
+    return emitOpError("expected 'vscale_min' if and only if 'scalable'");
+  return success();
+}
+
+::mlir::ValueBoundsConstraintSet::ComparisonOperator
+CompareOp::getComparisonOperator() {
+  if (getCmp() == "EQ")
+    return ValueBoundsConstraintSet::ComparisonOperator::EQ;
+  if (getCmp() == "LT")
+    return ValueBoundsConstraintSet::ComparisonOperator::LT;
+  if (getCmp() == "LE")
+    return ValueBoundsConstraintSet::ComparisonOperator::LE;
+  if (getCmp() == "GT")
+    return ValueBoundsConstraintSet::ComparisonOperator::GT;
+  if (getCmp() == "GE")
+    return ValueBoundsConstraintSet::ComparisonOperator::GE;
+  llvm_unreachable("invalid comparison operator");
+}
+
 //===----------------------------------------------------------------------===//
 // Test removing op with inner ops.
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/lib/Dialect/Test/TestDialect.h b/mlir/test/lib/Dialect/Test/TestDialect.h
index 4ba28c47ed1c..d5b2fbeafc41 100644
--- a/mlir/test/lib/Dialect/Test/TestDialect.h
+++ b/mlir/test/lib/Dialect/Test/TestDialect.h
@@ -41,6 +41,7 @@
 #include "mlir/Interfaces/InferTypeOpInterface.h"
 #include "mlir/Interfaces/LoopLikeInterface.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
+#include "mlir/Interfaces/ValueBoundsOpInterface.h"
 #include "mlir/Interfaces/ViewLikeInterface.h"
 
 #include <memory>
diff --git a/mlir/test/lib/Dialect/Test/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td
index c88f85b8b6b3..ebf158b8bb82 100644
--- a/mlir/test/lib/Dialect/Test/TestOps.td
+++ b/mlir/test/lib/Dialect/Test/TestOps.td
@@ -2184,6 +2184,53 @@ def ForwardBufferOp : TEST_Op<"forward_buffer", [Pure]> {
   let results = (outs AnyRankedOrUnrankedMemRef:$result);
 }
 
+//===----------------------------------------------------------------------===//
+// Test ValueBoundsOpInterface
+//===----------------------------------------------------------------------===//
+
+def ReifyBoundOp : TEST_Op<"reify_bound", [Pure]> {
+  let description = [{
+    Reify a bound for the given index-typed value or dimension size of a shaped
+    value. "LB", "EQ" and "UB" bounds are supported. If `scalable` is set, 
+    `vscale_min` and `vscale_max` must be provided, which allows computing
+    a bound in terms of "vector.vscale" for a given range of vscale.
+  }];
+
+  let arguments = (ins AnyType:$var,
+                       OptionalAttr<I64Attr>:$dim,
+                       DefaultValuedAttr<StrAttr, "\"EQ\"">:$type,
+                       UnitAttr:$constant,
+                       UnitAttr:$scalable,
+                       OptionalAttr<I64Attr>:$vscale_min,
+                       OptionalAttr<I64Attr>:$vscale_max);
+  let results = (outs Index:$result);
+
+  let extraClassDeclaration = [{
+    ::mlir::presburger::BoundType getBoundType();
+  }];
+
+  let hasVerifier = 1;
+}
+
+def CompareOp : TEST_Op<"compare"> {
+  let description = [{
+    Compare `lhs` and `rhs`. A remark is emitted which indicates whether the
+    specified comparison operator was proven to hold. The remark also indicates
+    whether the opposite comparison operator was proven to hold.
+  }];
+
+  let arguments = (ins Index:$lhs,
+                       Index:$rhs,
+                       DefaultValuedAttr<StrAttr, "\"EQ\"">:$cmp,
+                       UnitAttr:$compose);
+  let results = (outs);
+
+  let extraClassDeclaration = [{
+    ::mlir::ValueBoundsConstraintSet::ComparisonOperator
+        getComparisonOperator();
+  }];
+}
+
 //===----------------------------------------------------------------------===//
 // Test RegionBranchOpInterface
 //===----------------------------------------------------------------------===//
diff --git a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
index 684b59e7f62f..dc5f4047c286 100644
--- a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
@@ -421,6 +421,7 @@ cc_library(
         "//mlir:TranslateLib",
         "//mlir:TransformUtils",
         "//mlir:Transforms",
+        "//mlir:ValueBoundsOpInterface",
         "//mlir:ViewLikeInterface",
     ],
 )
@@ -644,6 +645,7 @@ cc_library(
         "lib/Dialect/Affine/*.cpp",
     ]),
     deps = [
+        ":TestDialect",
         "//llvm:Support",
         "//mlir:AffineAnalysis",
         "//mlir:AffineDialect",
-- 
GitLab


From a34887550b7f2926ea335d4aedf8b72811f9c945 Mon Sep 17 00:00:00 2001
From: Mark de Wever <koraq@xs4all.nl>
Date: Mon, 15 Apr 2024 18:46:58 +0200
Subject: [PATCH 258/464] [LLDB][libc++] Adds valarray proxy data formatters.
 (#88613)

These proxies are returned by operator[](...). These proxies all
"behave" the same. They store a pointer to the data of the valarray they
are a proxy for and they have an internal array of indices. This
internal array is considered its contents.
---
 .../Plugins/Language/CPlusPlus/CMakeLists.txt |   1 +
 .../Language/CPlusPlus/CPlusPlusLanguage.cpp  |  11 +
 .../Plugins/Language/CPlusPlus/LibCxx.h       |   4 +
 .../Language/CPlusPlus/LibCxxProxyArray.cpp   | 194 ++++++++++++++++++
 .../TestDataFormatterLibcxxValarray.py        |  88 +++++++-
 .../libcxx/valarray/main.cpp                  |   6 +-
 6 files changed, 295 insertions(+), 9 deletions(-)
 create mode 100644 lldb/source/Plugins/Language/CPlusPlus/LibCxxProxyArray.cpp

diff --git a/lldb/source/Plugins/Language/CPlusPlus/CMakeLists.txt b/lldb/source/Plugins/Language/CPlusPlus/CMakeLists.txt
index 0c6fdb2b9573..f59032c42388 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/CMakeLists.txt
+++ b/lldb/source/Plugins/Language/CPlusPlus/CMakeLists.txt
@@ -14,6 +14,7 @@ add_lldb_library(lldbPluginCPlusPlusLanguage PLUGIN
   LibCxxQueue.cpp
   LibCxxRangesRefView.cpp
   LibCxxSliceArray.cpp
+  LibCxxProxyArray.cpp
   LibCxxSpan.cpp
   LibCxxTuple.cpp
   LibCxxUnorderedMap.cpp
diff --git a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
index afb683f7d846..5f0684163328 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
+++ b/lldb/source/Plugins/Language/CPlusPlus/CPlusPlusLanguage.cpp
@@ -760,6 +760,12 @@ static void LoadLibCxxFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
       lldb_private::formatters::LibcxxStdSliceArraySyntheticFrontEndCreator,
       "libc++ std::slice_array synthetic children",
       "^std::__[[:alnum:]]+::slice_array<.+>$", stl_deref_flags, true);
+  AddCXXSynthetic(
+      cpp_category_sp,
+      lldb_private::formatters::LibcxxStdProxyArraySyntheticFrontEndCreator,
+      "libc++ synthetic children for the valarray proxy arrays",
+      "^std::__[[:alnum:]]+::(gslice|mask|indirect)_array<.+>$",
+      stl_deref_flags, true);
   AddCXXSynthetic(
       cpp_category_sp,
       lldb_private::formatters::LibcxxStdForwardListSyntheticFrontEndCreator,
@@ -890,6 +896,11 @@ static void LoadLibCxxFormatters(lldb::TypeCategoryImplSP cpp_category_sp) {
                 "libc++ std::slice_array summary provider",
                 "^std::__[[:alnum:]]+::slice_array<.+>$", stl_summary_flags,
                 true);
+  AddCXXSummary(cpp_category_sp,
+                lldb_private::formatters::LibcxxContainerSummaryProvider,
+                "libc++ summary provider for the valarray proxy arrays",
+                "^std::__[[:alnum:]]+::(gslice|mask|indirect)_array<.+>$",
+                stl_summary_flags, true);
   AddCXXSummary(
       cpp_category_sp, lldb_private::formatters::LibcxxContainerSummaryProvider,
       "libc++ std::list summary provider",
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
index 8e97174dc307..7fe15d1bf3f7 100644
--- a/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxx.h
@@ -231,6 +231,10 @@ SyntheticChildrenFrontEnd *
 LibcxxStdSliceArraySyntheticFrontEndCreator(CXXSyntheticChildren *,
                                             lldb::ValueObjectSP);
 
+SyntheticChildrenFrontEnd *
+LibcxxStdProxyArraySyntheticFrontEndCreator(CXXSyntheticChildren *,
+                                            lldb::ValueObjectSP);
+
 SyntheticChildrenFrontEnd *
 LibcxxStdListSyntheticFrontEndCreator(CXXSyntheticChildren *,
                                       lldb::ValueObjectSP);
diff --git a/lldb/source/Plugins/Language/CPlusPlus/LibCxxProxyArray.cpp b/lldb/source/Plugins/Language/CPlusPlus/LibCxxProxyArray.cpp
new file mode 100644
index 000000000000..726f06523b97
--- /dev/null
+++ b/lldb/source/Plugins/Language/CPlusPlus/LibCxxProxyArray.cpp
@@ -0,0 +1,194 @@
+//===-- LibCxxProxyArray.cpp-----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "LibCxx.h"
+
+#include "lldb/Core/ValueObject.h"
+#include "lldb/DataFormatters/FormattersHelpers.h"
+#include <optional>
+
+using namespace lldb;
+using namespace lldb_private;
+using namespace lldb_private::formatters;
+
+namespace lldb_private {
+namespace formatters {
+
+/// Data formatter for libc++'s std::"proxy_array".
+///
+/// A proxy_array's are created by using:
+///   std::gslice_array   operator[](const std::gslice& gslicearr);
+///   std::mask_array     operator[](const std::valarray<bool>& boolarr);
+///   std::indirect_array operator[](const std::valarray<std::size_t>& indarr);
+///
+/// These arrays have the following members:
+/// - __vp_ points to std::valarray::__begin_
+/// - __1d_ an array of offsets of the elements from @a __vp_
+class LibcxxStdProxyArraySyntheticFrontEnd : public SyntheticChildrenFrontEnd {
+public:
+  LibcxxStdProxyArraySyntheticFrontEnd(lldb::ValueObjectSP valobj_sp);
+
+  ~LibcxxStdProxyArraySyntheticFrontEnd() override;
+
+  llvm::Expected<uint32_t> CalculateNumChildren() override;
+
+  lldb::ValueObjectSP GetChildAtIndex(uint32_t idx) override;
+
+  lldb::ChildCacheState Update() override;
+
+  bool MightHaveChildren() override;
+
+  size_t GetIndexOfChildWithName(ConstString name) override;
+
+private:
+  /// A non-owning pointer to the array's __vp_.
+  ValueObject *m_base = nullptr;
+  /// The type of the array's template argument T.
+  CompilerType m_element_type;
+  /// The sizeof the array's template argument T.
+  uint32_t m_element_size = 0;
+
+  /// A non-owning pointer to the array's __1d_.__begin_.
+  ValueObject *m_start = nullptr;
+  /// A non-owning pointer to the array's __1d_.__end_.
+  ValueObject *m_finish = nullptr;
+  /// The type of the __1d_ array's template argument T (size_t).
+  CompilerType m_element_type_size_t;
+  /// The sizeof the __1d_ array's template argument T (size_t)
+  uint32_t m_element_size_size_t = 0;
+};
+
+} // namespace formatters
+} // namespace lldb_private
+
+lldb_private::formatters::LibcxxStdProxyArraySyntheticFrontEnd::
+    LibcxxStdProxyArraySyntheticFrontEnd(lldb::ValueObjectSP valobj_sp)
+    : SyntheticChildrenFrontEnd(*valobj_sp), m_element_type() {
+  if (valobj_sp)
+    Update();
+}
+
+lldb_private::formatters::LibcxxStdProxyArraySyntheticFrontEnd::
+    ~LibcxxStdProxyArraySyntheticFrontEnd() {
+  // these need to stay around because they are child objects who will follow
+  // their parent's life cycle
+  // delete m_base;
+}
+
+llvm::Expected<uint32_t> lldb_private::formatters::
+    LibcxxStdProxyArraySyntheticFrontEnd::CalculateNumChildren() {
+
+  if (!m_start || !m_finish)
+    return 0;
+  uint64_t start_val = m_start->GetValueAsUnsigned(0);
+  uint64_t finish_val = m_finish->GetValueAsUnsigned(0);
+
+  if (start_val == 0 || finish_val == 0)
+    return 0;
+
+  if (start_val >= finish_val)
+    return 0;
+
+  size_t num_children = (finish_val - start_val);
+  if (num_children % m_element_size_size_t)
+    return 0;
+  return num_children / m_element_size_size_t;
+}
+
+lldb::ValueObjectSP
+lldb_private::formatters::LibcxxStdProxyArraySyntheticFrontEnd::GetChildAtIndex(
+    uint32_t idx) {
+  if (!m_base)
+    return lldb::ValueObjectSP();
+
+  uint64_t offset = idx * m_element_size_size_t;
+  offset = offset + m_start->GetValueAsUnsigned(0);
+
+  lldb::ValueObjectSP indirect = CreateValueObjectFromAddress(
+      "", offset, m_backend.GetExecutionContextRef(), m_element_type_size_t);
+  if (!indirect)
+    return lldb::ValueObjectSP();
+
+  const size_t value = indirect->GetValueAsUnsigned(0);
+  if (!value)
+    return lldb::ValueObjectSP();
+
+  offset = value * m_element_size;
+  offset = offset + m_base->GetValueAsUnsigned(0);
+
+  StreamString name;
+  name.Printf("[%" PRIu64 "] -> [%zu]", (uint64_t)idx, value);
+  return CreateValueObjectFromAddress(name.GetString(), offset,
+                                      m_backend.GetExecutionContextRef(),
+                                      m_element_type);
+}
+
+lldb::ChildCacheState
+lldb_private::formatters::LibcxxStdProxyArraySyntheticFrontEnd::Update() {
+  m_base = nullptr;
+  m_start = nullptr;
+  m_finish = nullptr;
+
+  CompilerType type = m_backend.GetCompilerType();
+  if (type.GetNumTemplateArguments() == 0)
+    return ChildCacheState::eRefetch;
+
+  m_element_type = type.GetTypeTemplateArgument(0);
+  if (std::optional<uint64_t> size = m_element_type.GetByteSize(nullptr))
+    m_element_size = *size;
+
+  if (m_element_size == 0)
+    return ChildCacheState::eRefetch;
+
+  ValueObjectSP vector = m_backend.GetChildMemberWithName("__1d_");
+  if (!vector)
+    return ChildCacheState::eRefetch;
+
+  type = vector->GetCompilerType();
+  if (type.GetNumTemplateArguments() == 0)
+    return ChildCacheState::eRefetch;
+
+  m_element_type_size_t = type.GetTypeTemplateArgument(0);
+  if (std::optional<uint64_t> size = m_element_type_size_t.GetByteSize(nullptr))
+    m_element_size_size_t = *size;
+
+  if (m_element_size_size_t == 0)
+    return ChildCacheState::eRefetch;
+
+  ValueObjectSP base = m_backend.GetChildMemberWithName("__vp_");
+  ValueObjectSP start = vector->GetChildMemberWithName("__begin_");
+  ValueObjectSP finish = vector->GetChildMemberWithName("__end_");
+  if (!base || !start || !finish)
+    return ChildCacheState::eRefetch;
+
+  m_base = base.get();
+  m_start = start.get();
+  m_finish = finish.get();
+
+  return ChildCacheState::eRefetch;
+}
+
+bool lldb_private::formatters::LibcxxStdProxyArraySyntheticFrontEnd::
+    MightHaveChildren() {
+  return true;
+}
+
+size_t lldb_private::formatters::LibcxxStdProxyArraySyntheticFrontEnd::
+    GetIndexOfChildWithName(ConstString name) {
+  if (!m_base)
+    return std::numeric_limits<size_t>::max();
+  return ExtractIndexFromString(name.GetCString());
+}
+
+lldb_private::SyntheticChildrenFrontEnd *
+lldb_private::formatters::LibcxxStdProxyArraySyntheticFrontEndCreator(
+    CXXSyntheticChildren *, lldb::ValueObjectSP valobj_sp) {
+  if (!valobj_sp)
+    return nullptr;
+  return new LibcxxStdProxyArraySyntheticFrontEnd(valobj_sp);
+}
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/valarray/TestDataFormatterLibcxxValarray.py b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/valarray/TestDataFormatterLibcxxValarray.py
index b59b770ed679..613546b50a77 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/valarray/TestDataFormatterLibcxxValarray.py
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/valarray/TestDataFormatterLibcxxValarray.py
@@ -89,21 +89,93 @@ class LibcxxChronoDataFormatterTestCase(TestBase):
             "frame variable sa",
             substrs=[
                 "sa = stride=2 size=4",
-                "[0] = 1",
-                "[1] = 3",
-                "[2] = 5",
-                "[3] = 7",
+                "[0] = 11",
+                "[1] = 13",
+                "[2] = 15",
+                "[3] = 17",
                 "}",
             ],
         )
 
         # check access-by-index
-        self.expect("frame variable sa[0]", substrs=["1"])
-        self.expect("frame variable sa[1]", substrs=["3"])
-        self.expect("frame variable sa[2]", substrs=["5"])
-        self.expect("frame variable sa[3]", substrs=["7"])
+        self.expect("frame variable sa[0]", substrs=["11"])
+        self.expect("frame variable sa[1]", substrs=["13"])
+        self.expect("frame variable sa[2]", substrs=["15"])
+        self.expect("frame variable sa[3]", substrs=["17"])
         self.expect(
             "frame variable sa[4]",
             error=True,
             substrs=['array index 4 is not valid for "(slice_array<int>) sa"'],
         )
+
+        #
+        # std::gslice_array
+        #
+
+        self.expect(
+            "frame variable ga",
+            substrs=[
+                "ga = size=3",
+                "[0] -> [3] = 13",
+                "[1] -> [4] = 14",
+                "[2] -> [5] = 15",
+                "}",
+            ],
+        )
+
+        # check access-by-index
+        self.expect("frame variable ga[0]", substrs=["13"])
+        self.expect("frame variable ga[1]", substrs=["14"])
+        self.expect("frame variable ga[2]", substrs=["15"])
+        self.expect(
+            "frame variable ga[3]",
+            error=True,
+            substrs=['array index 3 is not valid for "(gslice_array<int>) ga"'],
+        )
+        #
+        # std::mask_array
+        #
+
+        self.expect(
+            "frame variable ma",
+            substrs=[
+                "ma = size=2",
+                "[0] -> [1] = 11",
+                "[1] -> [2] = 12",
+                "}",
+            ],
+        )
+
+        # check access-by-index
+        self.expect("frame variable ma[0]", substrs=["11"])
+        self.expect("frame variable ma[1]", substrs=["12"])
+        self.expect(
+            "frame variable ma[2]",
+            error=True,
+            substrs=['array index 2 is not valid for "(mask_array<int>) ma"'],
+        )
+
+        #
+        # std::indirect_array
+        #
+
+        self.expect(
+            "frame variable ia",
+            substrs=[
+                "ia = size=3",
+                "[0] -> [3] = 13",
+                "[1] -> [6] = 16",
+                "[2] -> [9] = 19",
+                "}",
+            ],
+        )
+
+        # check access-by-index
+        self.expect("frame variable ia[0]", substrs=["13"])
+        self.expect("frame variable ia[1]", substrs=["16"])
+        self.expect("frame variable ia[2]", substrs=["19"])
+        self.expect(
+            "frame variable ia[3]",
+            error=True,
+            substrs=['array index 3 is not valid for "(indirect_array<int>) ia"'],
+        )
diff --git a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/valarray/main.cpp b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/valarray/main.cpp
index 1481d8b40329..d31951c755ea 100644
--- a/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/valarray/main.cpp
+++ b/lldb/test/API/functionalities/data-formatter/data-formatter-stl/libcxx/valarray/main.cpp
@@ -13,8 +13,12 @@ int main() {
 
   std::valarray<double> va_double({1.0, 0.5, 0.25, 0.125});
 
-  std::valarray<int> va({0, 1, 2, 3, 4, 5, 6, 7, 8, 9});
+  std::valarray<int> va({10, 11, 12, 13, 14, 15, 16, 17, 18, 19});
   std::slice_array<int> sa = va[std::slice(1, 4, 2)];
+  std::gslice_array<int> ga = va[std::gslice(
+      3, std::valarray<std::size_t>(3, 1), std::valarray<std::size_t>(1, 1))];
+  std::mask_array<int> ma = va[std::valarray<bool>{false, true, true}];
+  std::indirect_array<int> ia = va[std::valarray<size_t>{3, 6, 9}];
 
   std::cout << "break here\n";
 }
-- 
GitLab


From 9dbb6e1978d3d2d61ef65c2dac1fd8add5a4c7a2 Mon Sep 17 00:00:00 2001
From: Hsiangkai Wang <hsiangkai.wang@arm.com>
Date: Mon, 15 Apr 2024 17:59:17 +0100
Subject: [PATCH 259/464] [mlir][spirv] Add target width to SPIR-V ABI (#88555)

There are execution modes need target width as their extra operands.
SignedZeroInfNanPreserve is one of them. This patch adds `target width`
as one of SPIR-V ABI attributes.
---
 .../mlir/Dialect/SPIRV/IR/SPIRVAttributes.td  |  4 +++-
 .../mlir/Dialect/SPIRV/IR/TargetAndABI.h      |  7 ++++++-
 mlir/lib/Dialect/SPIRV/IR/TargetAndABI.cpp    | 11 +++++-----
 .../Transforms/LowerABIAttributesPass.cpp     | 20 ++++++++++++++++---
 .../Conversion/GPUToSPIRV/entry-point.mlir    | 12 +++++++++++
 .../lib/Dialect/SPIRV/TestEntryPointAbi.cpp   | 17 +++++++++++++++-
 6 files changed, 59 insertions(+), 12 deletions(-)

diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVAttributes.td b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVAttributes.td
index 74d36445e311..3a11284da051 100644
--- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVAttributes.td
+++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVAttributes.td
@@ -32,10 +32,12 @@ class SPIRV_Attr<string attrName, string attrMnemonic>
 // points in the generated SPIR-V module:
 // 1) [optional] Requested workgroup size.
 // 2) [optional] Requested subgroup size.
+// 3) [optional] Requested target width.
 def SPIRV_EntryPointABIAttr : SPIRV_Attr<"EntryPointABI", "entry_point_abi"> {
   let parameters = (ins
     OptionalParameter<"DenseI32ArrayAttr">:$workgroup_size,
-    OptionalParameter<"std::optional<int>">:$subgroup_size
+    OptionalParameter<"std::optional<int>">:$subgroup_size,
+    OptionalParameter<"std::optional<int>">:$target_width
   );
   let assemblyFormat = "`<` struct(params) `>`";
 }
diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/TargetAndABI.h b/mlir/include/mlir/Dialect/SPIRV/IR/TargetAndABI.h
index c35a8c26c2bc..24574bfaf619 100644
--- a/mlir/include/mlir/Dialect/SPIRV/IR/TargetAndABI.h
+++ b/mlir/include/mlir/Dialect/SPIRV/IR/TargetAndABI.h
@@ -87,9 +87,14 @@ bool needsInterfaceVarABIAttrs(TargetEnvAttr targetAttr);
 StringRef getEntryPointABIAttrName();
 
 /// Gets the EntryPointABIAttr given its fields.
+/// targetWidth is used by several execution modes. It is the element width
+/// of floating-point operations.
+/// Refer to Execution Mode in SPIR-V specification.
+/// https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#_execution_mode
 EntryPointABIAttr getEntryPointABIAttr(MLIRContext *context,
                                        ArrayRef<int32_t> workgroupSize = {},
-                                       std::optional<int> subgroupSize = {});
+                                       std::optional<int> subgroupSize = {},
+                                       std::optional<int> targetWidth = {});
 
 /// Queries the entry point ABI on the nearest function-like op containing the
 /// given `op`. Returns null attribute if not found.
diff --git a/mlir/lib/Dialect/SPIRV/IR/TargetAndABI.cpp b/mlir/lib/Dialect/SPIRV/IR/TargetAndABI.cpp
index 5b7c0a59ba42..bbc318e17300 100644
--- a/mlir/lib/Dialect/SPIRV/IR/TargetAndABI.cpp
+++ b/mlir/lib/Dialect/SPIRV/IR/TargetAndABI.cpp
@@ -120,17 +120,16 @@ bool spirv::needsInterfaceVarABIAttrs(spirv::TargetEnvAttr targetAttr) {
 
 StringRef spirv::getEntryPointABIAttrName() { return "spirv.entry_point_abi"; }
 
-spirv::EntryPointABIAttr
-spirv::getEntryPointABIAttr(MLIRContext *context,
-                            ArrayRef<int32_t> workgroupSize,
-                            std::optional<int> subgroupSize) {
+spirv::EntryPointABIAttr spirv::getEntryPointABIAttr(
+    MLIRContext *context, ArrayRef<int32_t> workgroupSize,
+    std::optional<int> subgroupSize, std::optional<int> targetWidth) {
   DenseI32ArrayAttr workgroupSizeAttr;
   if (!workgroupSize.empty()) {
     assert(workgroupSize.size() == 3);
     workgroupSizeAttr = DenseI32ArrayAttr::get(context, workgroupSize);
   }
-  return spirv::EntryPointABIAttr::get(context, workgroupSizeAttr,
-                                       subgroupSize);
+  return spirv::EntryPointABIAttr::get(context, workgroupSizeAttr, subgroupSize,
+                                       targetWidth);
 }
 
 spirv::EntryPointABIAttr spirv::lookupEntryPointABI(Operation *op) {
diff --git a/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp b/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
index 6150b5ee1785..2024a2e5279f 100644
--- a/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
+++ b/mlir/lib/Dialect/SPIRV/Transforms/LowerABIAttributesPass.cpp
@@ -157,7 +157,7 @@ static LogicalResult lowerEntryPointABIAttr(spirv::FuncOp funcOp,
       // Erase workgroup size.
       entryPointAttr = spirv::EntryPointABIAttr::get(
           entryPointAttr.getContext(), DenseI32ArrayAttr(),
-          entryPointAttr.getSubgroupSize());
+          entryPointAttr.getSubgroupSize(), entryPointAttr.getTargetWidth());
     }
   }
   if (std::optional<int> subgroupSize = entryPointAttr.getSubgroupSize()) {
@@ -170,10 +170,24 @@ static LogicalResult lowerEntryPointABIAttr(spirv::FuncOp funcOp,
       // Erase subgroup size.
       entryPointAttr = spirv::EntryPointABIAttr::get(
           entryPointAttr.getContext(), entryPointAttr.getWorkgroupSize(),
-          std::nullopt);
+          std::nullopt, entryPointAttr.getTargetWidth());
     }
   }
-  if (entryPointAttr.getWorkgroupSize() || entryPointAttr.getSubgroupSize())
+  if (std::optional<int> targetWidth = entryPointAttr.getTargetWidth()) {
+    std::optional<ArrayRef<spirv::Capability>> caps =
+        spirv::getCapabilities(spirv::ExecutionMode::SignedZeroInfNanPreserve);
+    if (!caps || targetEnv.allows(*caps)) {
+      builder.create<spirv::ExecutionModeOp>(
+          funcOp.getLoc(), funcOp,
+          spirv::ExecutionMode::SignedZeroInfNanPreserve, *targetWidth);
+      // Erase target width.
+      entryPointAttr = spirv::EntryPointABIAttr::get(
+          entryPointAttr.getContext(), entryPointAttr.getWorkgroupSize(),
+          entryPointAttr.getSubgroupSize(), std::nullopt);
+    }
+  }
+  if (entryPointAttr.getWorkgroupSize() || entryPointAttr.getSubgroupSize() ||
+      entryPointAttr.getTargetWidth())
     funcOp->setAttr(entryPointAttrName, entryPointAttr);
   else
     funcOp->removeAttr(entryPointAttrName);
diff --git a/mlir/test/Conversion/GPUToSPIRV/entry-point.mlir b/mlir/test/Conversion/GPUToSPIRV/entry-point.mlir
index 99369d11a4ba..8b14d6569f48 100644
--- a/mlir/test/Conversion/GPUToSPIRV/entry-point.mlir
+++ b/mlir/test/Conversion/GPUToSPIRV/entry-point.mlir
@@ -1,5 +1,8 @@
 // RUN: mlir-opt -test-spirv-entry-point-abi %s | FileCheck %s -check-prefix=DEFAULT
 // RUN: mlir-opt -test-spirv-entry-point-abi="workgroup-size=32" %s | FileCheck %s -check-prefix=WG32
+// RUN: mlir-opt -test-spirv-entry-point-abi="subgroup-size=4" %s | FileCheck %s -check-prefix=SG4
+// RUN: mlir-opt -test-spirv-entry-point-abi="target-width=32" %s | FileCheck %s -check-prefix=TW32
+// RUN: mlir-opt -test-spirv-entry-point-abi="workgroup-size=32,8 subgroup-size=4 target-width=32" %s | FileCheck %s -check-prefix=WG32_8-SG4-TW32
 
 //      DEFAULT: gpu.func @foo()
 // DEFAULT-SAME: spirv.entry_point_abi = #spirv.entry_point_abi<workgroup_size = [1, 1, 1]>
@@ -7,6 +10,15 @@
 //      WG32: gpu.func @foo()
 // WG32-SAME:  spirv.entry_point_abi = #spirv.entry_point_abi<workgroup_size = [32, 1, 1]>
 
+//      SG4: gpu.func @foo()
+// SG4-SAME:  spirv.entry_point_abi = #spirv.entry_point_abi<workgroup_size = [1, 1, 1], subgroup_size = 4>
+
+//      TW32: gpu.func @foo()
+// TW32-SAME:  spirv.entry_point_abi = #spirv.entry_point_abi<workgroup_size = [1, 1, 1], target_width = 32>
+
+//      WG32_8-SG4-TW32: gpu.func @foo()
+// WG32_8-SG4-TW32-SAME:  spirv.entry_point_abi = #spirv.entry_point_abi<workgroup_size = [32, 8, 1], subgroup_size = 4, target_width = 32>
+
 gpu.module @kernels {
   gpu.func @foo() kernel {
     gpu.return
diff --git a/mlir/test/lib/Dialect/SPIRV/TestEntryPointAbi.cpp b/mlir/test/lib/Dialect/SPIRV/TestEntryPointAbi.cpp
index 47b06e453115..0e8dfb8e3ee9 100644
--- a/mlir/test/lib/Dialect/SPIRV/TestEntryPointAbi.cpp
+++ b/mlir/test/lib/Dialect/SPIRV/TestEntryPointAbi.cpp
@@ -45,6 +45,16 @@ private:
           "Workgroup size to use for all gpu.func kernels in the module, "
           "specified with x-dimension first, y-dimension next and z-dimension "
           "last. Unspecified dimensions will be set to 1")};
+  Pass::Option<int> subgroupSize{
+      *this, "subgroup-size",
+      llvm::cl::desc(
+          "Subgroup size to use for all gpu.func kernels in the module"),
+      llvm::cl::init(0)};
+  Pass::Option<int> targetWidth{
+      *this, "target-width",
+      llvm::cl::desc(
+          "Specify the component width of floating-point instructions"),
+      llvm::cl::init(0)};
 };
 } // namespace
 
@@ -60,7 +70,12 @@ void TestSpirvEntryPointABIPass::runOnOperation() {
                                              workgroupSize.end());
     workgroupSizeVec.resize(3, 1);
     gpuFunc->setAttr(attrName,
-                     spirv::getEntryPointABIAttr(context, workgroupSizeVec));
+                     spirv::getEntryPointABIAttr(
+                         context, workgroupSizeVec,
+                         (subgroupSize == 0) ? std::nullopt
+                                             : std::optional<int>(subgroupSize),
+                         (targetWidth == 0) ? std::nullopt
+                                            : std::optional<int>(targetWidth)));
   }
 }
 
-- 
GitLab


From e09245b3b16fe78e30e79e92f8ce16d1edf7787b Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Mon, 15 Apr 2024 10:00:56 -0700
Subject: [PATCH 260/464] [memprof] Fix typos in serializedSizeV0 and
 serializedSizeV2 (#88629)

The first field to serialize is the size of
IndexedMemProfRecord::AllocSites.  It has nothing to do with
GlobalValue::GUID.  This happens to work because of:

  using GUID = uint64_t;
---
 llvm/lib/ProfileData/MemProf.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/ProfileData/MemProf.cpp b/llvm/lib/ProfileData/MemProf.cpp
index 96aeedf2e691..97414505f1c1 100644
--- a/llvm/lib/ProfileData/MemProf.cpp
+++ b/llvm/lib/ProfileData/MemProf.cpp
@@ -42,7 +42,8 @@ size_t IndexedAllocationInfo::serializedSize(IndexedVersion Version) const {
 }
 
 static size_t serializedSizeV0(const IndexedMemProfRecord &Record) {
-  size_t Result = sizeof(GlobalValue::GUID);
+  // The number of alloc sites to serialize.
+  size_t Result = sizeof(uint64_t);
   for (const IndexedAllocationInfo &N : Record.AllocSites)
     Result += N.serializedSize(Version0);
 
@@ -57,7 +58,8 @@ static size_t serializedSizeV0(const IndexedMemProfRecord &Record) {
 }
 
 static size_t serializedSizeV2(const IndexedMemProfRecord &Record) {
-  size_t Result = sizeof(GlobalValue::GUID);
+  // The number of alloc sites to serialize.
+  size_t Result = sizeof(uint64_t);
   for (const IndexedAllocationInfo &N : Record.AllocSites)
     Result += N.serializedSize(Version2);
 
-- 
GitLab


From 51d85b7b107e9508b90cec274f3f122444fe7ee6 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Mon, 15 Apr 2024 10:02:23 -0700
Subject: [PATCH 261/464] [Support] Add llvm::support::endian::writeNext
 (#88685)

`writeNext` overloads increase the pointer argument like `readNext`.
Code like the following
```
endian::write32<ELFT::Endianness>(p, 42);
p += 4;
endian::write32<ELFT::Endianness>(p, 43);
p += 4;
```

can be simplified to:
```
endian::writeNext<uint32_t, ELFT::Endianness>(p, 42);
endian::writeNext<uint32_t, ELFT::Endianness>(p, 43);
```
---
 llvm/include/llvm/Support/Endian.h    | 15 +++++++++++++++
 llvm/unittests/Support/EndianTest.cpp | 23 +++++++++++++++++++++++
 2 files changed, 38 insertions(+)

diff --git a/llvm/include/llvm/Support/Endian.h b/llvm/include/llvm/Support/Endian.h
index 4c0405cf1e2f..1cdb5ca0d5ea 100644
--- a/llvm/include/llvm/Support/Endian.h
+++ b/llvm/include/llvm/Support/Endian.h
@@ -102,6 +102,21 @@ inline void write(void *memory, value_type value) {
   write<value_type, alignment>(memory, value, endian);
 }
 
+/// Write a value of a particular endianness, and increment the buffer past that
+/// value.
+template <typename value_type, std::size_t alignment = unaligned,
+          typename CharT>
+inline void writeNext(CharT *&memory, value_type value, endianness endian) {
+  write(memory, value, endian);
+  memory += sizeof(value_type);
+}
+
+template <typename value_type, endianness endian,
+          std::size_t alignment = unaligned, typename CharT>
+inline void writeNext(CharT *&memory, value_type value) {
+  writeNext<value_type, alignment, CharT>(memory, value, endian);
+}
+
 template <typename value_type>
 using make_unsigned_t = std::make_unsigned_t<value_type>;
 
diff --git a/llvm/unittests/Support/EndianTest.cpp b/llvm/unittests/Support/EndianTest.cpp
index ab7dfc380069..bba1a56168f7 100644
--- a/llvm/unittests/Support/EndianTest.cpp
+++ b/llvm/unittests/Support/EndianTest.cpp
@@ -36,6 +36,29 @@ TEST(Endian, Read) {
                                                                   1)));
 }
 
+TEST(Endian, WriteNext) {
+  unsigned char bigval[] = {0x00, 0x00}, *p = bigval;
+  endian::writeNext<int16_t, llvm::endianness::big>(p, short(0xaabb));
+  EXPECT_EQ(bigval[0], 0xaa);
+  EXPECT_EQ(bigval[1], 0xbb);
+  EXPECT_EQ(p, bigval + 2);
+
+  char littleval[8] = {}, *q = littleval;
+  endian::writeNext<uint32_t, llvm::endianness::little>(q, 0x44556677);
+  EXPECT_EQ(littleval[0], 0x77);
+  EXPECT_EQ(littleval[1], 0x66);
+  EXPECT_EQ(littleval[2], 0x55);
+  EXPECT_EQ(littleval[3], 0x44);
+  EXPECT_EQ(q, littleval + 4);
+
+  endian::writeNext<uint32_t>(q, 0x11223344, llvm::endianness::little);
+  EXPECT_EQ(littleval[4], 0x44);
+  EXPECT_EQ(littleval[5], 0x33);
+  EXPECT_EQ(littleval[6], 0x22);
+  EXPECT_EQ(littleval[7], 0x11);
+  EXPECT_EQ(q, littleval + 8);
+}
+
 TEST(Endian, ReadBitAligned) {
   // Simple test to make sure we properly pull out the 0x0 word.
   unsigned char littleval[] = {0x3f, 0x00, 0x00, 0x00, 0xc0, 0xff, 0xff, 0xff};
-- 
GitLab


From 603fa4c6b9a540206ec6f6b2115274878b65bfbb Mon Sep 17 00:00:00 2001
From: Nathan Sidwell <nathan@acm.org>
Date: Mon, 15 Apr 2024 13:11:29 -0400
Subject: [PATCH 262/464] [BOLT][NFC] Be more obvious about selecting X86
 (#88527)

Use `isX86()` rather than `!isAArch64() && !isRISCV()`, and similar.
---
 bolt/lib/Core/BinaryContext.cpp      | 2 +-
 bolt/lib/Core/BinaryEmitter.cpp      | 2 +-
 bolt/lib/Rewrite/RewriteInstance.cpp | 6 ++++--
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp
index 7c2d8c52287b..ad2eb18caf10 100644
--- a/bolt/lib/Core/BinaryContext.cpp
+++ b/bolt/lib/Core/BinaryContext.cpp
@@ -1880,7 +1880,7 @@ MarkerSymType BinaryContext::getMarkerType(const SymbolRef &Symbol) const {
   // For aarch64 and riscv, the ABI defines mapping symbols so we identify data
   // in the code section (see IHI0056B). $x identifies a symbol starting code or
   // the end of a data chunk inside code, $d identifies start of data.
-  if ((!isAArch64() && !isRISCV()) || ELFSymbolRef(Symbol).getSize())
+  if (isX86() || ELFSymbolRef(Symbol).getSize())
     return MarkerSymType::NONE;
 
   Expected<StringRef> NameOrError = Symbol.getName();
diff --git a/bolt/lib/Core/BinaryEmitter.cpp b/bolt/lib/Core/BinaryEmitter.cpp
index 97d19b75200f..6f86ddc77454 100644
--- a/bolt/lib/Core/BinaryEmitter.cpp
+++ b/bolt/lib/Core/BinaryEmitter.cpp
@@ -512,7 +512,7 @@ void BinaryEmitter::emitFunctionBody(BinaryFunction &BF, FunctionFragment &FF,
 
       // Emit sized NOPs via MCAsmBackend::writeNopData() interface on x86.
       // This is a workaround for invalid NOPs handling by asm/disasm layer.
-      if (BC.MIB->isNoop(Instr) && BC.isX86()) {
+      if (BC.isX86() && BC.MIB->isNoop(Instr)) {
         if (std::optional<uint32_t> Size = BC.MIB->getSize(Instr)) {
           SmallString<15> Code;
           raw_svector_ostream VecOS(Code);
diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index f22bede002da..bb76abd2f843 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -1670,7 +1670,9 @@ void RewriteInstance::disassemblePLT() {
       return disassemblePLTSectionAArch64(Section);
     if (BC->isRISCV())
       return disassemblePLTSectionRISCV(Section);
-    return disassemblePLTSectionX86(Section, EntrySize);
+    if (BC->isX86())
+      return disassemblePLTSectionX86(Section, EntrySize);
+    llvm_unreachable("Unmplemented PLT");
   };
 
   for (BinarySection &Section : BC->allocatableSections()) {
@@ -2605,7 +2607,7 @@ void RewriteInstance::handleRelocation(const SectionRef &RelocatedSection,
   const bool IsToCode = ReferencedSection && ReferencedSection->isText();
 
   // Special handling of PC-relative relocations.
-  if (!IsAArch64 && !BC->isRISCV() && Relocation::isPCRelative(RType)) {
+  if (BC->isX86() && Relocation::isPCRelative(RType)) {
     if (!IsFromCode && IsToCode) {
       // PC-relative relocations from data to code are tricky since the
       // original information is typically lost after linking, even with
-- 
GitLab


From bd7b170e97e314cb786dd33a750ef79412fc9682 Mon Sep 17 00:00:00 2001
From: Nathan Sidwell <nathan@acm.org>
Date: Mon, 15 Apr 2024 13:12:53 -0400
Subject: [PATCH 263/464] [BOLT][NFC] Remove extraneous braces (#88620)

A small cleanup -- no braces needed here.
---
 bolt/lib/Rewrite/RewriteInstance.cpp | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index bb76abd2f843..fd2477231142 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -2861,15 +2861,14 @@ void RewriteInstance::handleRelocation(const SectionRef &RelocatedSection,
       BC->isRISCV())
     ForceRelocation = true;
 
-  if (IsFromCode) {
+  if (IsFromCode)
     ContainingBF->addRelocation(Rel.getOffset(), ReferencedSymbol, RType,
                                 Addend, ExtractedValue);
-  } else if (IsToCode || ForceRelocation) {
+  else if (IsToCode || ForceRelocation)
     BC->addRelocation(Rel.getOffset(), ReferencedSymbol, RType, Addend,
                       ExtractedValue);
-  } else {
+  else
     LLVM_DEBUG(dbgs() << "BOLT-DEBUG: ignoring relocation from data to data\n");
-  }
 }
 
 void RewriteInstance::selectFunctionsToProcess() {
-- 
GitLab


From 9c970d5ecd6a85188cd2b0a941fcd4d60063ef81 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martin=20Storsj=C3=B6?= <martin@martin.st>
Date: Mon, 15 Apr 2024 20:14:07 +0300
Subject: [PATCH 264/464] [LLD] [COFF] Don't add pseudo relocs for dangling
 references (#88487)

When doing GC, we normally won't have dangling references, because such
a reference would keep the other section alive, keeping it from being
eliminated.

However, references within DWARF sections are ignored for the purposes
of GC (because otherwise, they would essentially keep everything alive,
defeating the point of the GC), see
c579a5b1d92a9bc2046d00ee2d427832e0f5ddec for more context.

Therefore, dangling relocations against discarded symbols are ignored
within DWARF sections (see maybeReportRelocationToDiscarded in
Chunks.cpp). Consequently, we also shouldn't create any pseudo
relocations for these cases, as we run into a null pointer dereference
when trying to generate the pseudo relocation info for it.

This fixes the downstream bug
https://github.com/mstorsjo/llvm-mingw/issues/418, fixing crashes on
combinations with -ffunction-sections, -fdata-sections,
-Wl,--gc-sections and debug info.
---
 lld/COFF/Chunks.cpp           |  7 ++++++
 lld/test/COFF/autoimport-gc.s | 41 +++++++++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+)
 create mode 100644 lld/test/COFF/autoimport-gc.s

diff --git a/lld/COFF/Chunks.cpp b/lld/COFF/Chunks.cpp
index 004d71097387..0cae52785a00 100644
--- a/lld/COFF/Chunks.cpp
+++ b/lld/COFF/Chunks.cpp
@@ -648,6 +648,13 @@ void SectionChunk::getRuntimePseudoRelocs(
         dyn_cast_or_null<Defined>(file->getSymbol(rel.SymbolTableIndex));
     if (!target || !target->isRuntimePseudoReloc)
       continue;
+    // If the target doesn't have a chunk allocated, it may be a
+    // DefinedImportData symbol which ended up unnecessary after GC.
+    // Normally we wouldn't eliminate section chunks that are referenced, but
+    // references within DWARF sections don't count for keeping section chunks
+    // alive. Thus such dangling references in DWARF sections are expected.
+    if (!target->getChunk())
+      continue;
     int sizeInBits =
         getRuntimePseudoRelocSize(rel.Type, file->ctx.config.machine);
     if (sizeInBits == 0) {
diff --git a/lld/test/COFF/autoimport-gc.s b/lld/test/COFF/autoimport-gc.s
new file mode 100644
index 000000000000..fef6c02eba82
--- /dev/null
+++ b/lld/test/COFF/autoimport-gc.s
@@ -0,0 +1,41 @@
+# REQUIRES: x86
+# RUN: split-file %s %t.dir
+
+# RUN: llvm-mc -triple=x86_64-windows-gnu %t.dir/lib.s -filetype=obj -o %t.dir/lib.obj
+# RUN: lld-link -out:%t.dir/lib.dll -dll -entry:DllMainCRTStartup %t.dir/lib.obj -lldmingw -implib:%t.dir/lib.lib
+
+# RUN: llvm-mc -triple=x86_64-windows-gnu %t.dir/main.s -filetype=obj -o %t.dir/main.obj
+# RUN: lld-link -lldmingw -out:%t.dir/main.exe -entry:main %t.dir/main.obj %t.dir/lib.lib -opt:ref -debug:dwarf
+
+#--- main.s
+    .global main
+    .section .text$main,"xr",one_only,main
+main:
+    ret
+
+    .global other
+    .section .text$other,"xr",one_only,other
+other:
+    movq .refptr.variable(%rip), %rax
+    movl (%rax), %eax
+    ret
+
+    .section .rdata$.refptr.variable,"dr",discard,.refptr.variable
+    .global .refptr.variable
+.refptr.variable:
+    .quad   variable
+
+    .section .debug_info
+    .long 1
+    .quad variable
+    .long 2
+
+#--- lib.s
+    .global variable
+    .global DllMainCRTStartup
+    .text
+DllMainCRTStartup:
+    ret
+    .data
+variable:
+    .long 42
-- 
GitLab


From 64dc55835a1552dbe74eef79460094014edf659e Mon Sep 17 00:00:00 2001
From: David Tenty <daltenty@ibm.com>
Date: Mon, 15 Apr 2024 13:18:01 -0400
Subject: [PATCH 265/464] [NFC][libunwind][AIX] Add diagnostic push and TODO

clang treats the cast here as an error since 999d4f840777bf8de26d45947192aa0728edc0fb landed
with -Xextra -Werror. We believe this cast to be safe for the reasons noted in comment, but
we should do some further cleanup at some point.
---
 libcxxabi/src/aix_state_tab_eh.inc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/libcxxabi/src/aix_state_tab_eh.inc b/libcxxabi/src/aix_state_tab_eh.inc
index 0cd94834b37a..9f46001b0209 100644
--- a/libcxxabi/src/aix_state_tab_eh.inc
+++ b/libcxxabi/src/aix_state_tab_eh.inc
@@ -195,8 +195,13 @@ static void invoke_destructor(FSMEntry* fsmEntry, void* addr) {
       _LIBCXXABI_TRACE_STATETAB0("returned from scalar destructor\n");
     } else {
       _LIBCXXABI_TRACE_STATETAB0("calling vector destructor\n");
+      // TODO: in the legacy ABI, destructors had a second argument. We don't expect to encounter
+      // destructors of this type in the itanium-based ABI, so this should be safe, but this could use some cleanup.
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wcast-function-type"
       __cxa_vec_cleanup(addr, reinterpret_cast<size_t>(fsmEntry->elementCount), fsmEntry->elemSize,
                         reinterpret_cast<destruct_f>(fsmEntry->destructor));
+#pragma GCC diagnostic pop
       _LIBCXXABI_TRACE_STATETAB0("returned from vector destructor\n");
     }
   } catch (...) {
-- 
GitLab


From 4d28d3f93bfee3cecbeb234d1ce306250e12e912 Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333@gmail.com>
Date: Tue, 16 Apr 2024 01:28:28 +0800
Subject: [PATCH 266/464] [SDAG] Turn umin into smin if the saturation pattern
 is broken (#88505)

As we canonicalizes smin with non-negative operands into umin in the
middle-end, the saturation pattern will be broken.
This patch reverts the transform in DAGCombine to fix the regression on
ARM.

Fixes https://github.com/llvm/llvm-project/issues/85706.
---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 11 ++-
 llvm/test/CodeGen/AMDGPU/umed3.ll             |  3 +-
 llvm/test/CodeGen/ARM/usat.ll                 | 80 ++++++++-----------
 3 files changed, 43 insertions(+), 51 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index dbbe8d9193c2..cbba3a294b3d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -5577,9 +5577,12 @@ SDValue DAGCombiner::visitIMINMAX(SDNode *N) {
     return RMINMAX;
 
   // Is sign bits are zero, flip between UMIN/UMAX and SMIN/SMAX.
-  // Only do this if the current op isn't legal and the flipped is.
-  if (!TLI.isOperationLegal(Opcode, VT) &&
-      (N0.isUndef() || DAG.SignBitIsZero(N0)) &&
+  // Only do this if:
+  // 1. The current op isn't legal and the flipped is.
+  // 2. The saturation pattern is broken by canonicalization in InstCombine.
+  bool IsOpIllegal = !TLI.isOperationLegal(Opcode, VT);
+  bool IsSatBroken = Opcode == ISD::UMIN && N0.getOpcode() == ISD::SMAX;
+  if ((IsSatBroken || IsOpIllegal) && (N0.isUndef() || DAG.SignBitIsZero(N0)) &&
       (N1.isUndef() || DAG.SignBitIsZero(N1))) {
     unsigned AltOpcode;
     switch (Opcode) {
@@ -5589,7 +5592,7 @@ SDValue DAGCombiner::visitIMINMAX(SDNode *N) {
     case ISD::UMAX: AltOpcode = ISD::SMAX; break;
     default: llvm_unreachable("Unknown MINMAX opcode");
     }
-    if (TLI.isOperationLegal(AltOpcode, VT))
+    if ((IsSatBroken && IsOpIllegal) || TLI.isOperationLegal(AltOpcode, VT))
       return DAG.getNode(AltOpcode, DL, VT, N0, N1);
   }
 
diff --git a/llvm/test/CodeGen/AMDGPU/umed3.ll b/llvm/test/CodeGen/AMDGPU/umed3.ll
index a2d99f1f8c2c..557d023c45f9 100644
--- a/llvm/test/CodeGen/AMDGPU/umed3.ll
+++ b/llvm/test/CodeGen/AMDGPU/umed3.ll
@@ -43,8 +43,7 @@ define amdgpu_kernel void @v_test_umed3_multi_use_r_i_i_i32(ptr addrspace(1) %ou
 }
 
 ; GCN-LABEL: {{^}}v_test_umed3_r_i_i_sign_mismatch_i32:
-; GCN: v_max_i32_e32 v{{[0-9]+}}, 12, v{{[0-9]+}}
-; GCN: v_min_u32_e32 v{{[0-9]+}}, 17, v{{[0-9]+}}
+; GCN: v_med3_i32 v{{[0-9]+}}, v{{[0-9]+}}, 12, 17
 define amdgpu_kernel void @v_test_umed3_r_i_i_sign_mismatch_i32(ptr addrspace(1) %out, ptr addrspace(1) %aptr) #1 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr i32, ptr addrspace(1) %aptr, i32 %tid
diff --git a/llvm/test/CodeGen/ARM/usat.ll b/llvm/test/CodeGen/ARM/usat.ll
index 024a98dd2934..d01aa1520b32 100644
--- a/llvm/test/CodeGen/ARM/usat.ll
+++ b/llvm/test/CodeGen/ARM/usat.ll
@@ -756,7 +756,7 @@ define i32 @mm_unsigned_sat_upper_lower_1(i32 %x) {
 ; V4T-NEXT:    bic r1, r0, r0, asr #31
 ; V4T-NEXT:    ldr r0, .LCPI20_0
 ; V4T-NEXT:    cmp r1, r0
-; V4T-NEXT:    movlo r0, r1
+; V4T-NEXT:    movlt r0, r1
 ; V4T-NEXT:    bx lr
 ; V4T-NEXT:    .p2align 2
 ; V4T-NEXT:  @ %bb.1:
@@ -765,23 +765,12 @@ define i32 @mm_unsigned_sat_upper_lower_1(i32 %x) {
 ;
 ; V6-LABEL: mm_unsigned_sat_upper_lower_1:
 ; V6:       @ %bb.0: @ %entry
-; V6-NEXT:    bic r1, r0, r0, asr #31
-; V6-NEXT:    ldr r0, .LCPI20_0
-; V6-NEXT:    cmp r1, r0
-; V6-NEXT:    movlo r0, r1
+; V6-NEXT:    usat r0, #23, r0
 ; V6-NEXT:    bx lr
-; V6-NEXT:    .p2align 2
-; V6-NEXT:  @ %bb.1:
-; V6-NEXT:  .LCPI20_0:
-; V6-NEXT:    .long 8388607 @ 0x7fffff
 ;
 ; V6T2-LABEL: mm_unsigned_sat_upper_lower_1:
 ; V6T2:       @ %bb.0: @ %entry
-; V6T2-NEXT:    bic r1, r0, r0, asr #31
-; V6T2-NEXT:    movw r0, #65535
-; V6T2-NEXT:    movt r0, #127
-; V6T2-NEXT:    cmp r1, r0
-; V6T2-NEXT:    movlo r0, r1
+; V6T2-NEXT:    usat r0, #23, r0
 ; V6T2-NEXT:    bx lr
 entry:
   %0 = call i32 @llvm.smax.i32(i32 %x, i32 0)
@@ -795,7 +784,7 @@ define i32 @mm_unsigned_sat_upper_lower_2(i32 %x) {
 ; V4T-NEXT:    bic r1, r0, r0, asr #31
 ; V4T-NEXT:    ldr r0, .LCPI21_0
 ; V4T-NEXT:    cmp r1, r0
-; V4T-NEXT:    movlo r0, r1
+; V4T-NEXT:    movlt r0, r1
 ; V4T-NEXT:    bx lr
 ; V4T-NEXT:    .p2align 2
 ; V4T-NEXT:  @ %bb.1:
@@ -804,23 +793,12 @@ define i32 @mm_unsigned_sat_upper_lower_2(i32 %x) {
 ;
 ; V6-LABEL: mm_unsigned_sat_upper_lower_2:
 ; V6:       @ %bb.0: @ %entry
-; V6-NEXT:    bic r1, r0, r0, asr #31
-; V6-NEXT:    ldr r0, .LCPI21_0
-; V6-NEXT:    cmp r1, r0
-; V6-NEXT:    movlo r0, r1
+; V6-NEXT:    usat r0, #23, r0
 ; V6-NEXT:    bx lr
-; V6-NEXT:    .p2align 2
-; V6-NEXT:  @ %bb.1:
-; V6-NEXT:  .LCPI21_0:
-; V6-NEXT:    .long 8388607 @ 0x7fffff
 ;
 ; V6T2-LABEL: mm_unsigned_sat_upper_lower_2:
 ; V6T2:       @ %bb.0: @ %entry
-; V6T2-NEXT:    bic r1, r0, r0, asr #31
-; V6T2-NEXT:    movw r0, #65535
-; V6T2-NEXT:    movt r0, #127
-; V6T2-NEXT:    cmp r1, r0
-; V6T2-NEXT:    movlo r0, r1
+; V6T2-NEXT:    usat r0, #23, r0
 ; V6T2-NEXT:    bx lr
 entry:
   %0 = call i32 @llvm.smax.i32(i32 %x, i32 0)
@@ -834,7 +812,7 @@ define i32 @mm_unsigned_sat_upper_lower_3(i32 %x) {
 ; V4T-NEXT:    bic r1, r0, r0, asr #31
 ; V4T-NEXT:    ldr r0, .LCPI22_0
 ; V4T-NEXT:    cmp r1, r0
-; V4T-NEXT:    movlo r0, r1
+; V4T-NEXT:    movlt r0, r1
 ; V4T-NEXT:    bx lr
 ; V4T-NEXT:    .p2align 2
 ; V4T-NEXT:  @ %bb.1:
@@ -843,23 +821,12 @@ define i32 @mm_unsigned_sat_upper_lower_3(i32 %x) {
 ;
 ; V6-LABEL: mm_unsigned_sat_upper_lower_3:
 ; V6:       @ %bb.0: @ %entry
-; V6-NEXT:    bic r1, r0, r0, asr #31
-; V6-NEXT:    ldr r0, .LCPI22_0
-; V6-NEXT:    cmp r1, r0
-; V6-NEXT:    movlo r0, r1
+; V6-NEXT:    usat r0, #23, r0
 ; V6-NEXT:    bx lr
-; V6-NEXT:    .p2align 2
-; V6-NEXT:  @ %bb.1:
-; V6-NEXT:  .LCPI22_0:
-; V6-NEXT:    .long 8388607 @ 0x7fffff
 ;
 ; V6T2-LABEL: mm_unsigned_sat_upper_lower_3:
 ; V6T2:       @ %bb.0: @ %entry
-; V6T2-NEXT:    bic r1, r0, r0, asr #31
-; V6T2-NEXT:    movw r0, #65535
-; V6T2-NEXT:    movt r0, #127
-; V6T2-NEXT:    cmp r1, r0
-; V6T2-NEXT:    movlo r0, r1
+; V6T2-NEXT:    usat r0, #23, r0
 ; V6T2-NEXT:    bx lr
 entry:
   %0 = call i32 @llvm.smax.i32(i32 %x, i32 0)
@@ -913,7 +880,7 @@ define i32 @mm_no_unsigned_sat_incorrect_constant2(i32 %x) {
 ; V4T-NEXT:    mov r0, #1
 ; V4T-NEXT:    orr r0, r0, #8388608
 ; V4T-NEXT:    cmp r1, #8388608
-; V4T-NEXT:    movls r0, r1
+; V4T-NEXT:    movle r0, r1
 ; V4T-NEXT:    bx lr
 ;
 ; V6-LABEL: mm_no_unsigned_sat_incorrect_constant2:
@@ -922,7 +889,7 @@ define i32 @mm_no_unsigned_sat_incorrect_constant2(i32 %x) {
 ; V6-NEXT:    mov r0, #1
 ; V6-NEXT:    orr r0, r0, #8388608
 ; V6-NEXT:    cmp r1, #8388608
-; V6-NEXT:    movls r0, r1
+; V6-NEXT:    movle r0, r1
 ; V6-NEXT:    bx lr
 ;
 ; V6T2-LABEL: mm_no_unsigned_sat_incorrect_constant2:
@@ -931,7 +898,7 @@ define i32 @mm_no_unsigned_sat_incorrect_constant2(i32 %x) {
 ; V6T2-NEXT:    movw r0, #1
 ; V6T2-NEXT:    movt r0, #128
 ; V6T2-NEXT:    cmp r1, #8388608
-; V6T2-NEXT:    movls r0, r1
+; V6T2-NEXT:    movle r0, r1
 ; V6T2-NEXT:    bx lr
 entry:
   %0 = call i32 @llvm.smax.i32(i32 %x, i32 0)
@@ -981,6 +948,29 @@ entry:
   ret i32 %1
 }
 
+define i32 @test_umin_smax_usat(i32 %x) {
+; V4T-LABEL: test_umin_smax_usat:
+; V4T:       @ %bb.0: @ %entry
+; V4T-NEXT:    bic r0, r0, r0, asr #31
+; V4T-NEXT:    cmp r0, #255
+; V4T-NEXT:    movge r0, #255
+; V4T-NEXT:    bx lr
+;
+; V6-LABEL: test_umin_smax_usat:
+; V6:       @ %bb.0: @ %entry
+; V6-NEXT:    usat r0, #8, r0
+; V6-NEXT:    bx lr
+;
+; V6T2-LABEL: test_umin_smax_usat:
+; V6T2:       @ %bb.0: @ %entry
+; V6T2-NEXT:    usat r0, #8, r0
+; V6T2-NEXT:    bx lr
+entry:
+  %v1 = tail call i32 @llvm.smax.i32(i32 %x, i32 0)
+  %v2 = tail call i32 @llvm.umin.i32(i32 %v1, i32 255)
+  ret i32 %v2
+}
+
 declare i32 @llvm.smin.i32(i32, i32)
 declare i32 @llvm.smax.i32(i32, i32)
 declare i16 @llvm.smin.i16(i16, i16)
-- 
GitLab


From b2f07a91192596a3584040208b7fac4f5ef0dd9c Mon Sep 17 00:00:00 2001
From: Florian Mayer <fmayer@google.com>
Date: Mon, 15 Apr 2024 10:47:13 -0700
Subject: [PATCH 267/464] Revert "[clang analysis] ExprMutationAnalyzer avoid
 infinite recursion for recursive forwarding reference" (#88765)

Reverts llvm/llvm-project#87954

Broke sanitizer bots, e.g.
https://lab.llvm.org/buildbot/#/builders/239/builds/6587/steps/10/logs/stdio
---
 clang-tools-extra/docs/ReleaseNotes.rst       |  4 ---
 .../misc/const-correctness-templates.cpp      | 15 ----------
 .../Analysis/Analyses/ExprMutationAnalyzer.h  | 28 +++++------------
 clang/lib/Analysis/ExprMutationAnalyzer.cpp   | 22 +++++---------
 .../Analysis/ExprMutationAnalyzerTest.cpp     | 30 -------------------
 5 files changed, 15 insertions(+), 84 deletions(-)

diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 7095c564444f..4dfbd8ca49ab 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -221,10 +221,6 @@ Changes in existing checks
   <clang-tidy/checks/llvm/header-guard>` check by replacing the local
   option `HeaderFileExtensions` by the global option of the same name.
 
-- Improved :doc:`misc-const-correctness
-  <clang-tidy/checks/misc/const-correctness>` check by avoiding infinite recursion
-  for recursive forwarding reference.
-
 - Improved :doc:`misc-definitions-in-headers
   <clang-tidy/checks/misc/definitions-in-headers>` check by replacing the local
   option `HeaderFileExtensions` by the global option of the same name.
diff --git a/clang-tools-extra/test/clang-tidy/checkers/misc/const-correctness-templates.cpp b/clang-tools-extra/test/clang-tidy/checkers/misc/const-correctness-templates.cpp
index 248374a71dd4..9da468128743 100644
--- a/clang-tools-extra/test/clang-tidy/checkers/misc/const-correctness-templates.cpp
+++ b/clang-tools-extra/test/clang-tidy/checkers/misc/const-correctness-templates.cpp
@@ -58,18 +58,3 @@ void concatenate3(Args... args)
     (..., (stream << args));
 }
 } // namespace gh70323
-
-namespace gh60895 {
-
-template <class T> void f1(T &&a);
-template <class T> void f2(T &&a);
-template <class T> void f1(T &&a) { f2<T>(a); }
-template <class T> void f2(T &&a) { f1<T>(a); }
-void f() {
-  int x = 0;
-  // CHECK-MESSAGES:[[@LINE-1]]:3: warning: variable 'x' of type 'int' can be declared 'const'
-  // CHECK-FIXES: int const x = 0;
-  f1(x);
-}
-
-} // namespace gh60895
diff --git a/clang/include/clang/Analysis/Analyses/ExprMutationAnalyzer.h b/clang/include/clang/Analysis/Analyses/ExprMutationAnalyzer.h
index c4e5d0badb8e..1ceef944fbc3 100644
--- a/clang/include/clang/Analysis/Analyses/ExprMutationAnalyzer.h
+++ b/clang/include/clang/Analysis/Analyses/ExprMutationAnalyzer.h
@@ -8,10 +8,11 @@
 #ifndef LLVM_CLANG_ANALYSIS_ANALYSES_EXPRMUTATIONANALYZER_H
 #define LLVM_CLANG_ANALYSIS_ANALYSES_EXPRMUTATIONANALYZER_H
 
+#include <type_traits>
+
 #include "clang/AST/AST.h"
 #include "clang/ASTMatchers/ASTMatchers.h"
 #include "llvm/ADT/DenseMap.h"
-#include <variant>
 
 namespace clang {
 
@@ -21,15 +22,8 @@ class FunctionParmMutationAnalyzer;
 /// a given statement.
 class ExprMutationAnalyzer {
 public:
-  friend class FunctionParmMutationAnalyzer;
-  struct Cache {
-    llvm::SmallDenseMap<const FunctionDecl *,
-                        std::unique_ptr<FunctionParmMutationAnalyzer>>
-        FuncParmAnalyzer;
-  };
-
   ExprMutationAnalyzer(const Stmt &Stm, ASTContext &Context)
-      : ExprMutationAnalyzer(Stm, Context, std::make_shared<Cache>()) {}
+      : Stm(Stm), Context(Context) {}
 
   bool isMutated(const Expr *Exp) { return findMutation(Exp) != nullptr; }
   bool isMutated(const Decl *Dec) { return findMutation(Dec) != nullptr; }
@@ -51,11 +45,6 @@ private:
   using MutationFinder = const Stmt *(ExprMutationAnalyzer::*)(const Expr *);
   using ResultMap = llvm::DenseMap<const Expr *, const Stmt *>;
 
-  ExprMutationAnalyzer(const Stmt &Stm, ASTContext &Context,
-                       std::shared_ptr<Cache> CrossAnalysisCache)
-      : Stm(Stm), Context(Context),
-        CrossAnalysisCache(std::move(CrossAnalysisCache)) {}
-
   const Stmt *findMutationMemoized(const Expr *Exp,
                                    llvm::ArrayRef<MutationFinder> Finders,
                                    ResultMap &MemoizedResults);
@@ -80,7 +69,9 @@ private:
 
   const Stmt &Stm;
   ASTContext &Context;
-  std::shared_ptr<Cache> CrossAnalysisCache;
+  llvm::DenseMap<const FunctionDecl *,
+                 std::unique_ptr<FunctionParmMutationAnalyzer>>
+      FuncParmAnalyzer;
   ResultMap Results;
   ResultMap PointeeResults;
 };
@@ -89,12 +80,7 @@ private:
 // params.
 class FunctionParmMutationAnalyzer {
 public:
-  FunctionParmMutationAnalyzer(const FunctionDecl &Func, ASTContext &Context)
-      : FunctionParmMutationAnalyzer(
-            Func, Context, std::make_shared<ExprMutationAnalyzer::Cache>()) {}
-  FunctionParmMutationAnalyzer(
-      const FunctionDecl &Func, ASTContext &Context,
-      std::shared_ptr<ExprMutationAnalyzer::Cache> CrossAnalysisCache);
+  FunctionParmMutationAnalyzer(const FunctionDecl &Func, ASTContext &Context);
 
   bool isMutated(const ParmVarDecl *Parm) {
     return findMutation(Parm) != nullptr;
diff --git a/clang/lib/Analysis/ExprMutationAnalyzer.cpp b/clang/lib/Analysis/ExprMutationAnalyzer.cpp
index 503ca4bea99e..bb042760d297 100644
--- a/clang/lib/Analysis/ExprMutationAnalyzer.cpp
+++ b/clang/lib/Analysis/ExprMutationAnalyzer.cpp
@@ -638,10 +638,9 @@ const Stmt *ExprMutationAnalyzer::findFunctionArgMutation(const Expr *Exp) {
       if (!RefType->getPointeeType().getQualifiers() &&
           RefType->getPointeeType()->getAs<TemplateTypeParmType>()) {
         std::unique_ptr<FunctionParmMutationAnalyzer> &Analyzer =
-            CrossAnalysisCache->FuncParmAnalyzer[Func];
+            FuncParmAnalyzer[Func];
         if (!Analyzer)
-          Analyzer.reset(new FunctionParmMutationAnalyzer(*Func, Context,
-                                                          CrossAnalysisCache));
+          Analyzer.reset(new FunctionParmMutationAnalyzer(*Func, Context));
         if (Analyzer->findMutation(Parm))
           return Exp;
         continue;
@@ -654,15 +653,13 @@ const Stmt *ExprMutationAnalyzer::findFunctionArgMutation(const Expr *Exp) {
 }
 
 FunctionParmMutationAnalyzer::FunctionParmMutationAnalyzer(
-    const FunctionDecl &Func, ASTContext &Context,
-    std::shared_ptr<ExprMutationAnalyzer::Cache> CrossAnalysisCache)
-    : BodyAnalyzer(*Func.getBody(), Context, CrossAnalysisCache) {
+    const FunctionDecl &Func, ASTContext &Context)
+    : BodyAnalyzer(*Func.getBody(), Context) {
   if (const auto *Ctor = dyn_cast<CXXConstructorDecl>(&Func)) {
     // CXXCtorInitializer might also mutate Param but they're not part of
     // function body, check them eagerly here since they're typically trivial.
     for (const CXXCtorInitializer *Init : Ctor->inits()) {
-      ExprMutationAnalyzer InitAnalyzer(*Init->getInit(), Context,
-                                        CrossAnalysisCache);
+      ExprMutationAnalyzer InitAnalyzer(*Init->getInit(), Context);
       for (const ParmVarDecl *Parm : Ctor->parameters()) {
         if (Results.contains(Parm))
           continue;
@@ -678,14 +675,11 @@ FunctionParmMutationAnalyzer::findMutation(const ParmVarDecl *Parm) {
   const auto Memoized = Results.find(Parm);
   if (Memoized != Results.end())
     return Memoized->second;
-  // To handle call A -> call B -> call A. Assume parameters of A is not mutated
-  // before analyzing parameters of A. Then when analyzing the second "call A",
-  // FunctionParmMutationAnalyzer can use this memoized value to avoid infinite
-  // recursion.
-  Results[Parm] = nullptr;
+
   if (const Stmt *S = BodyAnalyzer.findMutation(Parm))
     return Results[Parm] = S;
-  return Results[Parm];
+
+  return Results[Parm] = nullptr;
 }
 
 } // namespace clang
diff --git a/clang/unittests/Analysis/ExprMutationAnalyzerTest.cpp b/clang/unittests/Analysis/ExprMutationAnalyzerTest.cpp
index 9c1dc1a76db6..f58ce4aebcbf 100644
--- a/clang/unittests/Analysis/ExprMutationAnalyzerTest.cpp
+++ b/clang/unittests/Analysis/ExprMutationAnalyzerTest.cpp
@@ -977,36 +977,6 @@ TEST(ExprMutationAnalyzerTest, FollowFuncArgModified) {
                          "void f() { int x; g(x); }");
   Results = match(withEnclosingCompound(declRefTo("x")), AST->getASTContext());
   EXPECT_THAT(mutatedBy(Results, AST.get()), ElementsAre("g(x)"));
-
-  AST = buildASTFromCode(
-      StdRemoveReference + StdForward +
-      "template <class T> void f1(T &&a);"
-      "template <class T> void f2(T &&a);"
-      "template <class T> void f1(T &&a) { f2<T>(std::forward<T>(a)); }"
-      "template <class T> void f2(T &&a) { f1<T>(std::forward<T>(a)); }"
-      "void f() { int x; f1(x); }");
-  Results = match(withEnclosingCompound(declRefTo("x")), AST->getASTContext());
-  EXPECT_FALSE(isMutated(Results, AST.get()));
-
-  AST = buildASTFromCode(
-      StdRemoveReference + StdForward +
-      "template <class T> void f1(T &&a);"
-      "template <class T> void f2(T &&a);"
-      "template <class T> void f1(T &&a) { f2<T>(std::forward<T>(a)); }"
-      "template <class T> void f2(T &&a) { f1<T>(std::forward<T>(a)); a++; }"
-      "void f() { int x; f1(x); }");
-  Results = match(withEnclosingCompound(declRefTo("x")), AST->getASTContext());
-  EXPECT_THAT(mutatedBy(Results, AST.get()), ElementsAre("f1(x)"));
-
-  AST = buildASTFromCode(
-      StdRemoveReference + StdForward +
-      "template <class T> void f1(T &&a);"
-      "template <class T> void f2(T &&a);"
-      "template <class T> void f1(T &&a) { f2<T>(std::forward<T>(a)); a++; }"
-      "template <class T> void f2(T &&a) { f1<T>(std::forward<T>(a)); }"
-      "void f() { int x; f1(x); }");
-  Results = match(withEnclosingCompound(declRefTo("x")), AST->getASTContext());
-  EXPECT_THAT(mutatedBy(Results, AST.get()), ElementsAre("f1(x)"));
 }
 
 TEST(ExprMutationAnalyzerTest, FollowFuncArgNotModified) {
-- 
GitLab


From b5b17bf613010b5ea900d2981365b9d2c846a20f Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333@gmail.com>
Date: Tue, 16 Apr 2024 01:50:22 +0800
Subject: [PATCH 268/464] [RISCV] Fix assertion failure in `genShXAddAddShift`
 (#88757)

Fix assertion failure in our downstream CI
https://github.com/dtcxzyw/llvm-codegen-benchmark/issues/1.
---
 llvm/lib/Target/RISCV/RISCVInstrInfo.cpp |  2 +-
 llvm/test/CodeGen/RISCV/rv64zba.ll       | 23 +++++++++++++++++++++++
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index 508f607fab20..b0fda040519a 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -1986,7 +1986,7 @@ genShXAddAddShift(MachineInstr &Root, unsigned AddOpIdx,
       MRI.getUniqueVRegDef(AddMI->getOperand(AddOpIdx).getReg());
 
   unsigned InnerShiftAmt = ShiftMI->getOperand(2).getImm();
-  assert(InnerShiftAmt > OuterShiftAmt && "Unexpected shift amount");
+  assert(InnerShiftAmt >= OuterShiftAmt && "Unexpected shift amount");
 
   unsigned InnerOpc;
   switch (InnerShiftAmt - OuterShiftAmt) {
diff --git a/llvm/test/CodeGen/RISCV/rv64zba.ll b/llvm/test/CodeGen/RISCV/rv64zba.ll
index 067addc819f7..bb4be323ecb2 100644
--- a/llvm/test/CodeGen/RISCV/rv64zba.ll
+++ b/llvm/test/CodeGen/RISCV/rv64zba.ll
@@ -2389,3 +2389,26 @@ define i64 @array_index_sh4_sh3(ptr %p, i64 %idx1, i64 %idx2) {
   %b = load i64, ptr %a, align 8
   ret i64 %b
 }
+
+define ptr @test_gep_gep_dont_crash(ptr %p, i64 %a1, i64 %a2) {
+; RV64I-LABEL: test_gep_gep_dont_crash:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    srliw a2, a2, 6
+; RV64I-NEXT:    slli a2, a2, 3
+; RV64I-NEXT:    slli a1, a1, 3
+; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    add a0, a0, a2
+; RV64I-NEXT:    ret
+;
+; RV64ZBA-LABEL: test_gep_gep_dont_crash:
+; RV64ZBA:       # %bb.0:
+; RV64ZBA-NEXT:    srliw a2, a2, 6
+; RV64ZBA-NEXT:    add a1, a2, a1
+; RV64ZBA-NEXT:    sh3add a0, a1, a0
+; RV64ZBA-NEXT:    ret
+  %lshr = lshr i64 %a2, 6
+  %and = and i64 %lshr, 67108863
+  %gep1 = getelementptr i64, ptr %p, i64 %and
+  %gep2 = getelementptr i64, ptr %gep1, i64 %a1
+  ret ptr %gep2
+}
-- 
GitLab


From 09c3bfe9b3eb47a2af0c10531b25f90cfb5fa9f4 Mon Sep 17 00:00:00 2001
From: Leonard Chan <leonardchan@google.com>
Date: Thu, 11 Apr 2024 14:51:58 -0700
Subject: [PATCH 269/464] Reapply "[llvm] Teach whole program devirtualization
 about relative vtables"

This reverts commit 474f5efebed24547e76d022f0c5ffcc9db97ce6f.
---
 .../include/llvm/Analysis/TypeMetadataUtils.h |   2 +-
 llvm/lib/Analysis/ModuleSummaryAnalysis.cpp   |  37 ++++-
 llvm/lib/Analysis/TypeMetadataUtils.cpp       |  16 +-
 llvm/test/ThinLTO/X86/devirt.ll               |  95 ++++++++++-
 .../WholeProgramDevirt/Inputs/export.yaml     |   8 +
 .../WholeProgramDevirt/branch-funnel.ll       | 157 ++++++++++++++++++
 .../WholeProgramDevirt/constant-arg.ll        |  45 +++++
 .../devirt-single-impl-check.ll               |  25 +++
 .../devirt-single-impl-multiple-assumes.ll    |  20 +++
 .../WholeProgramDevirt/devirt-single-impl.ll  |  47 +++++-
 .../WholeProgramDevirt/pointer-vtable.ll      |  16 ++
 11 files changed, 458 insertions(+), 10 deletions(-)

diff --git a/llvm/include/llvm/Analysis/TypeMetadataUtils.h b/llvm/include/llvm/Analysis/TypeMetadataUtils.h
index 8894945c28d9..2c7ce0858b9d 100644
--- a/llvm/include/llvm/Analysis/TypeMetadataUtils.h
+++ b/llvm/include/llvm/Analysis/TypeMetadataUtils.h
@@ -66,7 +66,7 @@ void findDevirtualizableCallsForTypeCheckedLoad(
 /// Used for example from GlobalDCE to find an entry in a C++ vtable that
 /// matches a vcall offset.
 ///
-/// To support Swift vtables, getPointerAtOffset can see through "relative
+/// To support relative vtables, getPointerAtOffset can see through "relative
 /// pointers", i.e. (sub-)expressions of the form of:
 ///
 /// @symbol = ... {
diff --git a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
index deda1eebb3b5..c3d15afe6d9c 100644
--- a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
+++ b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
@@ -22,6 +22,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Analysis/BlockFrequencyInfo.h"
 #include "llvm/Analysis/BranchProbabilityInfo.h"
+#include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/IndirectCallPromotionAnalysis.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/MemoryProfileInfo.h"
@@ -668,7 +669,8 @@ static void computeFunctionSummary(
 /// within the initializer.
 static void findFuncPointers(const Constant *I, uint64_t StartingOffset,
                              const Module &M, ModuleSummaryIndex &Index,
-                             VTableFuncList &VTableFuncs) {
+                             VTableFuncList &VTableFuncs,
+                             const GlobalVariable &OrigGV) {
   // First check if this is a function pointer.
   if (I->getType()->isPointerTy()) {
     auto C = I->stripPointerCasts();
@@ -696,7 +698,7 @@ static void findFuncPointers(const Constant *I, uint64_t StartingOffset,
       auto Offset = SL->getElementOffset(EI.index());
       unsigned Op = SL->getElementContainingOffset(Offset);
       findFuncPointers(cast<Constant>(I->getOperand(Op)),
-                       StartingOffset + Offset, M, Index, VTableFuncs);
+                       StartingOffset + Offset, M, Index, VTableFuncs, OrigGV);
     }
   } else if (auto *C = dyn_cast<ConstantArray>(I)) {
     ArrayType *ATy = C->getType();
@@ -704,7 +706,34 @@ static void findFuncPointers(const Constant *I, uint64_t StartingOffset,
     uint64_t EltSize = DL.getTypeAllocSize(EltTy);
     for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i) {
       findFuncPointers(cast<Constant>(I->getOperand(i)),
-                       StartingOffset + i * EltSize, M, Index, VTableFuncs);
+                       StartingOffset + i * EltSize, M, Index, VTableFuncs,
+                       OrigGV);
+    }
+  } else if (const auto *CE = dyn_cast<ConstantExpr>(I)) {
+    // For relative vtables, the next sub-component should be a trunc.
+    if (CE->getOpcode() != Instruction::Trunc ||
+        !(CE = dyn_cast<ConstantExpr>(CE->getOperand(0))))
+      return;
+
+    // If this constant can be reduced to the offset between a function and a
+    // global, then we know this is a valid virtual function if the RHS is the
+    // original vtable we're scanning through.
+    if (CE->getOpcode() == Instruction::Sub) {
+      GlobalValue *LHS, *RHS;
+      APSInt LHSOffset, RHSOffset;
+      if (IsConstantOffsetFromGlobal(CE->getOperand(0), LHS, LHSOffset, DL) &&
+          IsConstantOffsetFromGlobal(CE->getOperand(1), RHS, RHSOffset, DL) &&
+          RHS == &OrigGV &&
+
+          // For relative vtables, this component should point to the callable
+          // function without any offsets.
+          LHSOffset == 0 &&
+
+          // Also, the RHS should always point to somewhere within the vtable.
+          RHSOffset <=
+              static_cast<uint64_t>(DL.getTypeAllocSize(OrigGV.getInitializer()->getType()))) {
+        findFuncPointers(LHS, StartingOffset, M, Index, VTableFuncs, OrigGV);
+      }
     }
   }
 }
@@ -717,7 +746,7 @@ static void computeVTableFuncs(ModuleSummaryIndex &Index,
     return;
 
   findFuncPointers(V.getInitializer(), /*StartingOffset=*/0, M, Index,
-                   VTableFuncs);
+                   VTableFuncs, V);
 
 #ifndef NDEBUG
   // Validate that the VTableFuncs list is ordered by offset.
diff --git a/llvm/lib/Analysis/TypeMetadataUtils.cpp b/llvm/lib/Analysis/TypeMetadataUtils.cpp
index b8dcc39e9223..ee400175ef11 100644
--- a/llvm/lib/Analysis/TypeMetadataUtils.cpp
+++ b/llvm/lib/Analysis/TypeMetadataUtils.cpp
@@ -67,6 +67,14 @@ static void findLoadCallsAtConstantOffset(
         findLoadCallsAtConstantOffset(M, DevirtCalls, User, Offset + GEPOffset,
                                       CI, DT);
       }
+    } else if (auto *Call = dyn_cast<CallInst>(User)) {
+      if (Call->getIntrinsicID() == llvm::Intrinsic::load_relative) {
+        if (auto *LoadOffset = dyn_cast<ConstantInt>(Call->getOperand(1))) {
+          findCallsAtConstantOffset(DevirtCalls, nullptr, User,
+                                    Offset + LoadOffset->getSExtValue(), CI,
+                                    DT);
+        }
+      }
     }
   }
 }
@@ -131,6 +139,12 @@ void llvm::findDevirtualizableCallsForTypeCheckedLoad(
 
 Constant *llvm::getPointerAtOffset(Constant *I, uint64_t Offset, Module &M,
                                    Constant *TopLevelGlobal) {
+  // TODO: Ideally it would be the caller who knows if it's appropriate to strip
+  // the DSOLocalEquicalent. More generally, it would feel more appropriate to
+  // have two functions that handle absolute and relative pointers separately.
+  if (auto *Equiv = dyn_cast<DSOLocalEquivalent>(I))
+    I = Equiv->getGlobalValue();
+
   if (I->getType()->isPointerTy()) {
     if (Offset == 0)
       return I;
@@ -161,7 +175,7 @@ Constant *llvm::getPointerAtOffset(Constant *I, uint64_t Offset, Module &M,
                               Offset % ElemSize, M, TopLevelGlobal);
   }
 
-  // (Swift-specific) relative-pointer support starts here.
+  // Relative-pointer support starts here.
   if (auto *CI = dyn_cast<ConstantInt>(I)) {
     if (Offset == 0 && CI->isZero()) {
       return I;
diff --git a/llvm/test/ThinLTO/X86/devirt.ll b/llvm/test/ThinLTO/X86/devirt.ll
index 472e43d77680..a40f89e644f0 100644
--- a/llvm/test/ThinLTO/X86/devirt.ll
+++ b/llvm/test/ThinLTO/X86/devirt.ll
@@ -27,24 +27,36 @@
 ; NOENABLESPLITFLAG-DAG: [[B:\^[0-9]+]] = gv: (name: "_ZTV1B", {{.*}} vTableFuncs: ((virtFunc: [[Bf]], offset: 16), (virtFunc: [[An]], offset: 24)), refs: ([[Bf]], [[An]])
 ; NOENABLESPLITFLAG-DAG: [[C:\^[0-9]+]] = gv: (name: "_ZTV1C", {{.*}} vTableFuncs: ((virtFunc: [[Cf]], offset: 16), (virtFunc: [[An]], offset: 24)), refs: ([[An]], [[Cf]])
 ; NOENABLESPLITFLAG-DAG: [[D:\^[0-9]+]] = gv: (name: "_ZTV1D", {{.*}} vTableFuncs: ((virtFunc: [[Dm]], offset: 16)), refs: ([[Dm]])
+; NOENABLESPLITFLAG-DAG: [[B_RV:\^[0-9]+]] = gv: (name: "_ZTV1B_RV", {{.*}} vTableFuncs: ((virtFunc: [[Bf]], offset: 8), (virtFunc: [[An]], offset: 12)), refs: ([[B_RV]], [[Bf]], [[An]])
+; NOENABLESPLITFLAG-DAG: [[C_RV:\^[0-9]+]] = gv: (name: "_ZTV1C_RV", {{.*}} vTableFuncs: ((virtFunc: [[Cf]], offset: 8), (virtFunc: [[An]], offset: 12)), refs: ([[C_RV]], [[An]], [[Cf]])
+; NOENABLESPLITFLAG-DAG: [[D_RV:\^[0-9]+]] = gv: (name: "_ZTV1D_RV", {{.*}} vTableFuncs: ((virtFunc: [[Dm]], offset: 8)), refs: ([[D_RV]], [[Dm]])
 ; NOENABLESPLITFLAG-DAG: typeidCompatibleVTable: (name: "_ZTS1A", summary: ((offset: 16, [[B]]), (offset: 16, [[C]])))
 ; NOENABLESPLITFLAG-DAG: typeidCompatibleVTable: (name: "_ZTS1B", summary: ((offset: 16, [[B]])))
 ; NOENABLESPLITFLAG-DAG: typeidCompatibleVTable: (name: "_ZTS1C", summary: ((offset: 16, [[C]])))
+; NOENABLESPLITFLAG-DAG: typeidCompatibleVTable: (name: "_ZTS1A_RV", summary: ((offset: 8, [[B_RV]]), (offset: 8, [[C_RV]])))
+; NOENABLESPLITFLAG-DAG: typeidCompatibleVTable: (name: "_ZTS1B_RV", summary: ((offset: 8, [[B_RV]])))
+; NOENABLESPLITFLAG-DAG: typeidCompatibleVTable: (name: "_ZTS1C_RV", summary: ((offset: 8, [[C_RV]])))
 ; Type Id on _ZTV1D should have been promoted
 ; NOENABLESPLITFLAG-DAG: typeidCompatibleVTable: (name: "1.{{.*}}", summary: ((offset: 16, [[D]])))
+; NOENABLESPLITFLAG-DAG: typeidCompatibleVTable: (name: "2.{{.*}}", summary: ((offset: 8, [[D_RV]])))
 
 ; Index based WPD
 ; RUN: llvm-lto2 run %t2.o -save-temps -pass-remarks=. \
 ; RUN:   -whole-program-visibility \
 ; RUN:   -o %t3 \
 ; RUN:   -r=%t2.o,test,px \
+; RUN:   -r=%t2.o,test_rv,px \
 ; RUN:   -r=%t2.o,_ZN1A1nEi,p \
 ; RUN:   -r=%t2.o,_ZN1B1fEi,p \
 ; RUN:   -r=%t2.o,_ZN1C1fEi,p \
 ; RUN:   -r=%t2.o,_ZN1D1mEi,p \
 ; RUN:   -r=%t2.o,_ZTV1B,px \
 ; RUN:   -r=%t2.o,_ZTV1C,px \
-; RUN:   -r=%t2.o,_ZTV1D,px 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN:   -r=%t2.o,_ZTV1D,px \
+; RUN:   -r=%t2.o,_ZTV1B_RV,px \
+; RUN:   -r=%t2.o,_ZTV1C_RV,px \
+; RUN:   -r=%t2.o,_ZTV1D_RV,px \
+; RUN:   2>&1 | FileCheck %s --check-prefix=REMARK
 ; RUN: llvm-dis %t3.1.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR
 
 ; Check that we're able to prevent specific function from being
@@ -54,18 +66,24 @@
 ; RUN:   -wholeprogramdevirt-skip=_ZN1A1nEi \
 ; RUN:   -o %t3 \
 ; RUN:   -r=%t2.o,test,px \
+; RUN:   -r=%t2.o,test_rv,px \
 ; RUN:   -r=%t2.o,_ZN1A1nEi,p \
 ; RUN:   -r=%t2.o,_ZN1B1fEi,p \
 ; RUN:   -r=%t2.o,_ZN1C1fEi,p \
 ; RUN:   -r=%t2.o,_ZN1D1mEi,p \
 ; RUN:   -r=%t2.o,_ZTV1B,px \
 ; RUN:   -r=%t2.o,_ZTV1C,px \
-; RUN:   -r=%t2.o,_ZTV1D,px 2>&1 | FileCheck %s --check-prefix=SKIP
+; RUN:   -r=%t2.o,_ZTV1D,px \
+; RUN:   -r=%t2.o,_ZTV1B_RV,px \
+; RUN:   -r=%t2.o,_ZTV1C_RV,px \
+; RUN:   -r=%t2.o,_ZTV1D_RV,px \
+; RUN:   2>&1 | FileCheck %s --check-prefix=SKIP
 
 ; RUN: llvm-lto2 run %t.o -save-temps -pass-remarks=. \
 ; RUN:   -whole-program-visibility \
 ; RUN:   -o %t3 \
 ; RUN:   -r=%t.o,test,px \
+; RUN:   -r=%t.o,test_rv,px \
 ; RUN:   -r=%t.o,_ZN1A1nEi,p \
 ; RUN:   -r=%t.o,_ZN1B1fEi,p \
 ; RUN:   -r=%t.o,_ZN1C1fEi,p \
@@ -73,15 +91,24 @@
 ; RUN:   -r=%t.o,_ZTV1B, \
 ; RUN:   -r=%t.o,_ZTV1C, \
 ; RUN:   -r=%t.o,_ZTV1D, \
+; RUN:   -r=%t.o,_ZTV1B_RV, \
+; RUN:   -r=%t.o,_ZTV1C_RV, \
+; RUN:   -r=%t.o,_ZTV1D_RV, \
 ; RUN:   -r=%t.o,_ZN1A1nEi, \
 ; RUN:   -r=%t.o,_ZN1B1fEi, \
 ; RUN:   -r=%t.o,_ZN1C1fEi, \
 ; RUN:   -r=%t.o,_ZN1D1mEi, \
 ; RUN:   -r=%t.o,_ZTV1B,px \
 ; RUN:   -r=%t.o,_ZTV1C,px \
-; RUN:   -r=%t.o,_ZTV1D,px 2>&1 | FileCheck %s --check-prefix=REMARK --dump-input=fail
+; RUN:   -r=%t.o,_ZTV1D,px \
+; RUN:   -r=%t.o,_ZTV1B_RV,px \
+; RUN:   -r=%t.o,_ZTV1C_RV,px \
+; RUN:   -r=%t.o,_ZTV1D_RV,px \
+; RUN:   2>&1 | FileCheck %s --check-prefix=REMARK --dump-input=fail
 ; RUN: llvm-dis %t3.1.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR
 
+; REMARK-DAG: single-impl: devirtualized a call to _ZN1A1nEi
+; REMARK-DAG: single-impl: devirtualized a call to _ZN1D1mEi
 ; REMARK-DAG: single-impl: devirtualized a call to _ZN1A1nEi
 ; REMARK-DAG: single-impl: devirtualized a call to _ZN1D1mEi
 
@@ -99,6 +126,25 @@ target triple = "x86_64-grtev4-linux-gnu"
 @_ZTV1C = constant { [4 x ptr] } { [4 x ptr] [ptr null, ptr undef, ptr @_ZN1C1fEi, ptr @_ZN1A1nEi] }, !type !0, !type !2
 @_ZTV1D = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr undef, ptr @_ZN1D1mEi] }, !type !3
 
+@_ZTV1B_RV = constant { [4 x i32] } { [4 x i32] [
+  i32 0,
+  i32 undef,
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @_ZN1B1fEi to i64), i64 ptrtoint (ptr getelementptr inbounds ({ [4 x i32] }, ptr @_ZTV1B_RV, i32 0, i32 0, i32 2) to i64)) to i32),
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @_ZN1A1nEi to i64), i64 ptrtoint (ptr getelementptr inbounds ({ [4 x i32] }, ptr @_ZTV1B_RV, i32 0, i32 0, i32 3) to i64)) to i32)
+] }, !type !7, !type !8
+
+@_ZTV1C_RV = constant { [4 x i32] } { [4 x i32] [
+  i32 0,
+  i32 undef,
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @_ZN1C1fEi to i64), i64 ptrtoint (ptr getelementptr inbounds ({ [4 x i32] }, ptr @_ZTV1C_RV, i32 0, i32 0, i32 2) to i64)) to i32),
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @_ZN1A1nEi to i64), i64 ptrtoint (ptr getelementptr inbounds ({ [4 x i32] }, ptr @_ZTV1C_RV, i32 0, i32 0, i32 3) to i64)) to i32)
+] }, !type !7, !type !9
+
+@_ZTV1D_RV = constant { [3 x i32] } { [3 x i32] [
+  i32 0,
+  i32 undef,
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @_ZN1D1mEi to i64), i64 ptrtoint (ptr getelementptr inbounds ({ [3 x i32] }, ptr @_ZTV1D_RV, i32 0, i32 0, i32 2) to i64)) to i32)
+] }, !type !10
 
 ; CHECK-IR-LABEL: define {{(noundef )?}}i32 @test
 define i32 @test(ptr %obj, ptr %obj2, i32 %a) {
@@ -136,6 +182,43 @@ entry:
 ; CHECK-IR-LABEL: ret i32
 ; CHECK-IR-LABEL: }
 
+declare ptr @llvm.load.relative.i32(ptr, i32)
+
+; CHECK-IR-LABEL: define i32 @test_rv
+define i32 @test_rv(ptr %obj, ptr %obj2, i32 %a) {
+entry:
+  %vtable = load ptr, ptr %obj
+  %p = call i1 @llvm.type.test(ptr %vtable, metadata !"_ZTS1A_RV")
+  call void @llvm.assume(i1 %p)
+  %fptr1_rv = call ptr @llvm.load.relative.i32(ptr %vtable, i32 4)
+
+  ; Check that the call was devirtualized.
+  ; CHECK-IR: %call = tail call i32 @_ZN1A1nEi
+  ; Ensure !prof and !callees metadata for indirect call promotion removed.
+  ; CHECK-IR-NOT: prof
+  ; CHECK-IR-NOT: callees
+  %call = tail call i32 %fptr1_rv(ptr nonnull %obj, i32 %a), !prof !5, !callees !6
+
+  %fptr22_rv = call ptr @llvm.load.relative.i32(ptr %vtable, i32 0)
+
+  ; We still have to call it as virtual.
+  ; CHECK-IR: %call3 = tail call i32 %fptr22
+  %call3 = tail call i32 %fptr22_rv(ptr nonnull %obj, i32 %call)
+
+  %vtable2 = load ptr, ptr %obj2
+  %p2 = call i1 @llvm.type.test(ptr %vtable2, metadata !11)
+  call void @llvm.assume(i1 %p2)
+
+  %fptr33_rv = call ptr @llvm.load.relative.i32(ptr %vtable2, i32 0)
+
+  ; Check that the call was devirtualized.
+  ; CHECK-IR: %call4 = tail call i32 @_ZN1D1mEi
+  %call4 = tail call i32 %fptr33_rv(ptr nonnull %obj2, i32 %call3)
+  ret i32 %call4
+}
+; CHECK-IR-LABEL: ret i32
+; CHECK-IR-LABEL: }
+
 declare i1 @llvm.type.test(ptr, metadata)
 declare void @llvm.assume(i1)
 
@@ -165,3 +248,9 @@ attributes #0 = { noinline optnone }
 !4 = distinct !{}
 !5 = !{!"VP", i32 0, i64 1, i64 1621563287929432257, i64 1}
 !6 = !{ptr @_ZN1A1nEi}
+
+!7 = !{i64 8, !"_ZTS1A_RV"}
+!8 = !{i64 8, !"_ZTS1B_RV"}
+!9 = !{i64 8, !"_ZTS1C_RV"}
+!10 = !{i64 8, !11}
+!11 = distinct !{}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/Inputs/export.yaml b/llvm/test/Transforms/WholeProgramDevirt/Inputs/export.yaml
index 71cf38b216c7..dd0c90d7ae8c 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/Inputs/export.yaml
+++ b/llvm/test/Transforms/WholeProgramDevirt/Inputs/export.yaml
@@ -5,14 +5,22 @@ GlobalValueMap:
       TypeTestAssumeVCalls:
         - GUID: 14276520915468743435  # typeid1
           Offset: 0
+        - GUID: 271751036925422857  # typeid1_rv
+          Offset: 0
       TypeCheckedLoadVCalls:
         - GUID: 15427464259790519041  # typeid2
           Offset: 0
+        - GUID: 1146149264729288256  # typeid2_rv
+          Offset: 0
       TypeTestAssumeConstVCalls:
         - VFunc:
             GUID: 3515965990081467659  # typeid3
             Offset: 0
           Args: [12, 24]
+        - VFunc:
+            GUID: 2777626534618191571  # typeid3_rv
+            Offset: 0
+          Args: [12, 24]
       TypeCheckedLoadConstVCalls:
         - VFunc:
             GUID: 17525413373118030901  # typeid4
diff --git a/llvm/test/Transforms/WholeProgramDevirt/branch-funnel.ll b/llvm/test/Transforms/WholeProgramDevirt/branch-funnel.ll
index b55713fe4b29..0b1023eee273 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/branch-funnel.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/branch-funnel.ll
@@ -8,6 +8,45 @@
 ; RUN: FileCheck --check-prefix=SUMMARY %s < %t
 
 ; SUMMARY:      TypeIdMap:       
+; SUMMARY-NEXT:  typeid1_rv:
+; SUMMARY-NEXT:    TTRes:
+; SUMMARY-NEXT:      Kind:            Unknown
+; SUMMARY-NEXT:      SizeM1BitWidth:  0
+; SUMMARY-NEXT:      AlignLog2:       0
+; SUMMARY-NEXT:      SizeM1:          0
+; SUMMARY-NEXT:      BitMask:         0
+; SUMMARY-NEXT:      InlineBits:      0
+; SUMMARY-NEXT:    WPDRes:
+; SUMMARY-NEXT:      0:
+; SUMMARY-NEXT:        Kind:            BranchFunnel
+; SUMMARY-NEXT:        SingleImplName:  ''
+; SUMMARY-NEXT:        ResByArg:
+; SUMMARY-NEXT:  typeid2_rv:
+; SUMMARY-NEXT:    TTRes:
+; SUMMARY-NEXT:      Kind:            Unknown
+; SUMMARY-NEXT:      SizeM1BitWidth:  0
+; SUMMARY-NEXT:      AlignLog2:       0
+; SUMMARY-NEXT:      SizeM1:          0
+; SUMMARY-NEXT:      BitMask:         0
+; SUMMARY-NEXT:      InlineBits:      0
+; SUMMARY-NEXT:    WPDRes:
+; SUMMARY-NEXT:      0:
+; SUMMARY-NEXT:        Kind:            Indir
+; SUMMARY-NEXT:        SingleImplName:  ''
+; SUMMARY-NEXT:        ResByArg:
+; SUMMARY-NEXT:   typeid3_rv:
+; SUMMARY-NEXT:     TTRes:           
+; SUMMARY-NEXT:       Kind:            Unknown
+; SUMMARY-NEXT:       SizeM1BitWidth:  0
+; SUMMARY-NEXT:       AlignLog2:       0
+; SUMMARY-NEXT:       SizeM1:          0
+; SUMMARY-NEXT:       BitMask:         0
+; SUMMARY-NEXT:       InlineBits:      0
+; SUMMARY-NEXT:     WPDRes:          
+; SUMMARY-NEXT:       0:               
+; SUMMARY-NEXT:         Kind:            BranchFunnel
+; SUMMARY-NEXT:         SingleImplName:  ''
+; SUMMARY-NEXT:         ResByArg:        
 ; SUMMARY-NEXT:   typeid3:
 ; SUMMARY-NEXT:     TTRes:           
 ; SUMMARY-NEXT:       Kind:            Unknown
@@ -93,6 +132,29 @@ declare i32 @vf3_2(ptr %this, i32 %arg)
 declare i32 @vf4_1(ptr %this, i32 %arg)
 declare i32 @vf4_2(ptr %this, i32 %arg)
 
+declare ptr @llvm.load.relative.i32(ptr, i32)
+
+;; These are relative vtables equivalent to the ones above.
+@vt1_1_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf1_1 to i64), i64 ptrtoint (ptr @vt1_1_rv to i64)) to i32)], !type !5
+@vt1_2_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf1_2 to i64), i64 ptrtoint (ptr @vt1_2_rv to i64)) to i32)], !type !5
+
+@vt2_1_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_1 to i64), i64 ptrtoint (ptr @vt2_1_rv to i64)) to i32)], !type !6
+@vt2_2_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_2 to i64), i64 ptrtoint (ptr @vt2_2_rv to i64)) to i32)], !type !6
+@vt2_3_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_3 to i64), i64 ptrtoint (ptr @vt2_3_rv to i64)) to i32)], !type !6
+@vt2_4_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_4 to i64), i64 ptrtoint (ptr @vt2_4_rv to i64)) to i32)], !type !6
+@vt2_5_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_5 to i64), i64 ptrtoint (ptr @vt2_5_rv to i64)) to i32)], !type !6
+@vt2_6_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_6 to i64), i64 ptrtoint (ptr @vt2_6_rv to i64)) to i32)], !type !6
+@vt2_7_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_7 to i64), i64 ptrtoint (ptr @vt2_7_rv to i64)) to i32)], !type !6
+@vt2_8_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_8 to i64), i64 ptrtoint (ptr @vt2_8_rv to i64)) to i32)], !type !6
+@vt2_9_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_9 to i64), i64 ptrtoint (ptr @vt2_9_rv to i64)) to i32)], !type !6
+@vt2_10_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_10 to i64), i64 ptrtoint (ptr @vt2_10_rv to i64)) to i32)], !type !6
+@vt2_11_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_11 to i64), i64 ptrtoint (ptr @vt2_11_rv to i64)) to i32)], !type !6
+
+@vt3_1_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf3_1 to i64), i64 ptrtoint (ptr @vt3_1_rv to i64)) to i32)], !type !7
+@vt3_2_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf3_2 to i64), i64 ptrtoint (ptr @vt3_2_rv to i64)) to i32)], !type !7
+
+@vt4_1_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf4_1 to i64), i64 ptrtoint (ptr @vt4_1_rv to i64)) to i32)], !type !8
+@vt4_2_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf4_2 to i64), i64 ptrtoint (ptr @vt4_2_rv to i64)) to i32)], !type !8
 
 
 ; CHECK-LABEL: define i32 @fn1
@@ -108,6 +170,19 @@ define i32 @fn1(ptr %obj) #0 {
   ret i32 %result
 }
 
+; CHECK-LABEL: define i32 @fn1_rv
+; CHECK-NOT: call void (...) @llvm.icall.branch.funnel
+define i32 @fn1_rv(ptr %obj) #0 {
+  %vtable = load ptr, ptr %obj
+  %p = call i1 @llvm.type.test(ptr %vtable, metadata !"typeid1_rv")
+  call void @llvm.assume(i1 %p)
+  %fptr = call ptr @llvm.load.relative.i32(ptr %vtable, i32 0)
+  ; RETP: call i32 @__typeid_typeid1_rv_0_branch_funnel(ptr nest %vtable, ptr %obj, i32 1)
+  %result = call i32 %fptr(ptr %obj, i32 1)
+  ; NORETP: call i32 %
+  ret i32 %result
+}
+
 ; CHECK-LABEL: define i32 @fn2
 ; CHECK-NOT: call void (...) @llvm.icall.branch.funnel
 define i32 @fn2(ptr %obj) #0 {
@@ -120,6 +195,18 @@ define i32 @fn2(ptr %obj) #0 {
   ret i32 %result
 }
 
+; CHECK-LABEL: define i32 @fn2_rv
+; CHECK-NOT: call void (...) @llvm.icall.branch.funnel
+define i32 @fn2_rv(ptr %obj) #0 {
+  %vtable = load ptr, ptr %obj
+  %p = call i1 @llvm.type.test(ptr %vtable, metadata !"typeid2_rv")
+  call void @llvm.assume(i1 %p)
+  %fptr = call ptr @llvm.load.relative.i32(ptr %vtable, i32 0)
+  ; CHECK: call i32 %
+  %result = call i32 %fptr(ptr %obj, i32 1)
+  ret i32 %result
+}
+
 ; CHECK-LABEL: define i32 @fn3
 ; CHECK-NOT: call void (...) @llvm.icall.branch.funnel
 define i32 @fn3(ptr %obj) #0 {
@@ -133,10 +220,75 @@ define i32 @fn3(ptr %obj) #0 {
   ret i32 %result
 }
 
+; CHECK-LABEL: define i32 @fn3_rv
+; CHECK-NOT: call void (...) @llvm.icall.branch.funnel
+define i32 @fn3_rv(ptr %obj) #0 {
+  %vtable = load ptr, ptr %obj
+  %p = call i1 @llvm.type.test(ptr %vtable, metadata !9)
+  call void @llvm.assume(i1 %p)
+  %fptr = call ptr @llvm.load.relative.i32(ptr %vtable, i32 0)
+  ; RETP: call i32 @branch_funnel.1(ptr
+  ; NORETP: call i32 %
+  %result = call i32 %fptr(ptr %obj, i32 1)
+  ret i32 %result
+}
+
+; CHECK-LABEL: define i32 @fn4
+; CHECK-NOT: call void (...) @llvm.icall.branch.funnel
+define i32 @fn4(ptr %obj) #0 {
+  %p = call i1 @llvm.type.test(ptr @vt1_1, metadata !"typeid1")
+  call void @llvm.assume(i1 %p)
+  %fptr = load ptr, ptr @vt1_1
+  ; RETP: call i32 @__typeid_typeid1_0_branch_funnel(ptr nest @vt1_1, ptr %obj, i32 1)
+  %result = call i32 %fptr(ptr %obj, i32 1)
+  ; NORETP: call i32 %
+  ret i32 %result
+}
+
+; CHECK-LABEL: define i32 @fn4_cpy
+; CHECK-NOT: call void (...) @llvm.icall.branch.funnel
+define i32 @fn4_cpy(ptr %obj) #0 {
+  %p = call i1 @llvm.type.test(ptr @vt1_1, metadata !"typeid1")
+  call void @llvm.assume(i1 %p)
+  %fptr = load ptr, ptr @vt1_1
+  ; RETP: call i32 @__typeid_typeid1_0_branch_funnel(ptr nest @vt1_1, ptr %obj, i32 1)
+  %result = call i32 %fptr(ptr %obj, i32 1)
+  ; NORETP: call i32 %
+  ret i32 %result
+}
+
+; CHECK-LABEL: define i32 @fn4_rv
+; CHECK-NOT: call void (...) @llvm.icall.branch.funnel
+define i32 @fn4_rv(ptr %obj) #0 {
+  %p = call i1 @llvm.type.test(ptr @vt1_1_rv, metadata !"typeid1_rv")
+  call void @llvm.assume(i1 %p)
+  %fptr = call ptr @llvm.load.relative.i32(ptr @vt1_1_rv, i32 0)
+  ; RETP: call i32 @__typeid_typeid1_rv_0_branch_funnel(ptr nest @vt1_1_rv, ptr %obj, i32 1)
+  %result = call i32 %fptr(ptr %obj, i32 1)
+  ; NORETP: call i32 %
+  ret i32 %result
+}
+
+; CHECK-LABEL: define i32 @fn4_rv_cpy
+; CHECK-NOT: call void (...) @llvm.icall.branch.funnel
+define i32 @fn4_rv_cpy(ptr %obj) #0 {
+  %p = call i1 @llvm.type.test(ptr @vt1_1_rv, metadata !"typeid1_rv")
+  call void @llvm.assume(i1 %p)
+  %fptr = call ptr @llvm.load.relative.i32(ptr @vt1_1_rv, i32 0)
+  ; RETP: call i32 @__typeid_typeid1_rv_0_branch_funnel(ptr nest @vt1_1_rv, ptr %obj, i32 1)
+  %result = call i32 %fptr(ptr %obj, i32 1)
+  ; NORETP: call i32 %
+  ret i32 %result
+}
+
 ; CHECK-LABEL: define hidden void @__typeid_typeid1_0_branch_funnel(ptr nest %0, ...)
 ; CHECK-NEXT: musttail call void (...) @llvm.icall.branch.funnel(ptr %0, ptr {{(nonnull )?}}@vt1_1, ptr {{(nonnull )?}}@vf1_1, ptr {{(nonnull )?}}@vt1_2, ptr {{(nonnull )?}}@vf1_2, ...)
 
+; CHECK-LABEL: define hidden void @__typeid_typeid1_rv_0_branch_funnel(ptr nest %0, ...)
+; CHECK-NEXT: musttail call void (...) @llvm.icall.branch.funnel(ptr %0, ptr {{(nonnull )?}}@vt1_1_rv, ptr {{(nonnull )?}}@vf1_1, ptr {{(nonnull )?}}@vt1_2_rv, ptr {{(nonnull )?}}@vf1_2, ...)
+
 ; CHECK: define internal void @branch_funnel(ptr
+; CHECK: define internal void @branch_funnel.1(ptr
 
 declare i1 @llvm.type.test(ptr, metadata)
 declare void @llvm.assume(i1)
@@ -146,5 +298,10 @@ declare void @llvm.assume(i1)
 !2 = !{i32 0, !"typeid3"}
 !3 = !{i32 0, !4}
 !4 = distinct !{}
+!5 = !{i32 0, !"typeid1_rv"}
+!6 = !{i32 0, !"typeid2_rv"}
+!7 = !{i32 0, !"typeid3_rv"}
+!8 = !{i32 0, !9}
+!9 = distinct !{}
 
 attributes #0 = { "target-features"="+retpoline" }
diff --git a/llvm/test/Transforms/WholeProgramDevirt/constant-arg.ll b/llvm/test/Transforms/WholeProgramDevirt/constant-arg.ll
index 91bae4af9a21..e685a1a54c52 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/constant-arg.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/constant-arg.ll
@@ -7,6 +7,10 @@ target triple = "x86_64-unknown-linux-gnu"
 ; CHECK: private constant { [8 x i8], [1 x ptr], [0 x i8] } { [8 x i8] c"\00\00\00\00\00\00\00\02", [1 x ptr] [ptr @vf2], [0 x i8] zeroinitializer }, !type [[T8]]
 ; CHECK: private constant { [8 x i8], [1 x ptr], [0 x i8] } { [8 x i8] c"\00\00\00\00\00\00\00\01", [1 x ptr] [ptr @vf4], [0 x i8] zeroinitializer }, !type [[T8]]
 ; CHECK: private constant { [8 x i8], [1 x ptr], [0 x i8] } { [8 x i8] c"\00\00\00\00\00\00\00\02", [1 x ptr] [ptr @vf8], [0 x i8] zeroinitializer }, !type [[T8]]
+; CHECK: private constant { [4 x i8], [1 x i32], [0 x i8] } { [4 x i8] c"\00\00\00\01", [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf1 to i64), i64 ptrtoint (ptr @vt1_rv to i64)) to i32)], [0 x i8] zeroinitializer }, align 4, !type [[T4:![0-9]+]]
+; CHECK: private constant { [4 x i8], [1 x i32], [0 x i8] } { [4 x i8] c"\00\00\00\02", [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2 to i64), i64 ptrtoint (ptr @vt2_rv to i64)) to i32)], [0 x i8] zeroinitializer }, align 4, !type [[T4]]
+; CHECK: private constant { [4 x i8], [1 x i32], [0 x i8] } { [4 x i8] c"\00\00\00\01", [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf4 to i64), i64 ptrtoint (ptr @vt4_rv to i64)) to i32)], [0 x i8] zeroinitializer }, align 4, !type [[T4]]
+; CHECK: private constant { [4 x i8], [1 x i32], [0 x i8] } { [4 x i8] c"\00\00\00\02", [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf8 to i64), i64 ptrtoint (ptr @vt8_rv to i64)) to i32)], [0 x i8] zeroinitializer }, align 4, !type [[T4]]
 
 @vt1 = constant [1 x ptr] [ptr @vf1], !type !0
 @vt2 = constant [1 x ptr] [ptr @vf2], !type !0
@@ -61,8 +65,49 @@ define i1 @call2(ptr %obj) {
   ret i1 %result
 }
 
+declare ptr @llvm.load.relative.i32(ptr, i32)
+
+@vt1_rv = private unnamed_addr constant [1 x i32] [
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf1 to i64), i64 ptrtoint (ptr @vt1_rv to i64)) to i32)
+], align 4, !type !1
+@vt2_rv = private unnamed_addr constant [1 x i32] [
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2 to i64), i64 ptrtoint (ptr @vt2_rv to i64)) to i32)
+], align 4, !type !1
+@vt4_rv = private unnamed_addr constant [1 x i32] [
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf4 to i64), i64 ptrtoint (ptr @vt4_rv to i64)) to i32)
+], align 4, !type !1
+@vt8_rv = private unnamed_addr constant [1 x i32] [
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf8 to i64), i64 ptrtoint (ptr @vt8_rv to i64)) to i32)
+], align 4, !type !1
+
+; CHECK: define i1 @call3
+define i1 @call3(ptr %obj) {
+  %vtable = load ptr, ptr %obj
+  %p = call i1 @llvm.type.test(ptr %vtable, metadata !"typeid2")
+  call void @llvm.assume(i1 %p)
+  %fptr = call ptr @llvm.load.relative.i32(ptr %vtable, i32 0)
+  ; CHECK: getelementptr {{.*}} -1
+  ; CHECK: and {{.*}}, 1
+  %result = call i1 %fptr(ptr %obj, i32 5)
+  ret i1 %result
+}
+
+; CHECK: define i1 @call4
+define i1 @call4(ptr %obj) {
+  %vtable = load ptr, ptr %obj
+  %p = call i1 @llvm.type.test(ptr %vtable, metadata !"typeid2")
+  call void @llvm.assume(i1 %p)
+  %fptr = call ptr @llvm.load.relative.i32(ptr %vtable, i32 0)
+  ; CHECK: getelementptr {{.*}} -1
+  ; CHECK: and {{.*}}, 2
+  %result = call i1 %fptr(ptr %obj, i32 10)
+  ret i1 %result
+}
+
 declare i1 @llvm.type.test(ptr, metadata)
 declare void @llvm.assume(i1)
 
 ; CHECK: [[T8]] = !{i32 8, !"typeid"}
+; CHECK: [[T4]] = !{i32 4, !"typeid2"}
 !0 = !{i32 0, !"typeid"}
+!1 = !{i32 0, !"typeid2"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check.ll b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check.ll
index dc7b2024bc59..5a9e6c12c358 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check.ll
@@ -3,6 +3,7 @@
 target datalayout = "e-p:64:64"
 target triple = "x86_64-unknown-linux-gnu"
 
+; CHECK: remark: <unknown>:0:0: single-impl: devirtualized a call to vf
 ; CHECK: remark: <unknown>:0:0: single-impl: devirtualized a call to vf
 ; CHECK: remark: <unknown>:0:0: devirtualized vf
 ; CHECK-NOT: devirtualized
@@ -33,7 +34,31 @@ trap:
   unreachable
 }
 
+@vt3 = private unnamed_addr constant [1 x i32] [
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf to i64), i64 ptrtoint (ptr @vt3 to i64)) to i32)
+], align 4, !type !1
+
+; CHECK: define void @call2
+define void @call2(ptr %obj) {
+  %vtable = load ptr, ptr %obj
+  %pair = call {ptr, i1} @llvm.type.checked.load(ptr %vtable, i32 0, metadata !"typeid2")
+  %fptr = extractvalue {ptr, i1} %pair, 0
+  %p = extractvalue {ptr, i1} %pair, 1
+  ; CHECK: br i1 true,
+  br i1 %p, label %cont, label %trap
+
+cont:
+  ; CHECK: call void @vf(
+  call void %fptr(ptr %obj)
+  ret void
+
+trap:
+  call void @llvm.trap()
+  unreachable
+}
+
 declare {ptr, i1} @llvm.type.checked.load(ptr, i32, metadata)
 declare void @llvm.trap()
 
 !0 = !{i32 0, !"typeid"}
+!1 = !{i32 0, !"typeid2"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-multiple-assumes.ll b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-multiple-assumes.ll
index ed144c23302d..bfbbeaf8a065 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-multiple-assumes.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-multiple-assumes.ll
@@ -23,7 +23,27 @@ define void @call(ptr %obj) {
   ret void
 }
 
+declare ptr @llvm.load.relative.i32(ptr, i32)
+
+@vt3 = private unnamed_addr constant [1 x i32] [
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf to i64), i64 ptrtoint (ptr @vt3 to i64)) to i32)
+], align 4, !type !1
+
+; CHECK: define void @call2
+define void @call2(ptr %obj) {
+  %vtable = load ptr, ptr %obj
+  %p = call i1 @llvm.type.test(ptr %vtable, metadata !"typeid2")
+  call void @llvm.assume(i1 %p)
+  %p2 = call i1 @llvm.type.test(ptr %vtable, metadata !"typeid2")
+  call void @llvm.assume(i1 %p2)
+  %fptr = call ptr @llvm.load.relative.i32(ptr %vtable, i32 0)
+  ; CHECK: call void @vf(
+  call void %fptr(ptr %obj)
+  ret void
+}
+
 declare i1 @llvm.type.test(ptr, metadata)
 declare void @llvm.assume(i1)
 
 !0 = !{i32 0, !"typeid"}
+!1 = !{i32 0, !"typeid2"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl.ll b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl.ll
index 1ce96f70d905..d7f33df59495 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl.ll
@@ -7,6 +7,8 @@ target datalayout = "e-p:64:64"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; CHECK: remark: devirt-single.cc:30:32: single-impl: devirtualized a call to vf
+; CHECK: remark: devirt-single.cc:41:32: single-impl: devirtualized a call to vf
+; CHECK: remark: devirt-single.cc:51:32: single-impl: devirtualized a call to vf
 ; CHECK: remark: devirt-single.cc:13:0: devirtualized vf
 ; CHECK-NOT: devirtualized
 
@@ -28,6 +30,41 @@ define void @call(ptr %obj) #1 !dbg !5 {
   ret void
 }
 
+declare ptr @llvm.load.relative.i32(ptr, i32)
+
+@vt3 = private unnamed_addr constant [1 x i32] [
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf to i64), i64 ptrtoint (ptr @vt3 to i64)) to i32)
+], align 4, !type !11
+
+; CHECK: define void @call2
+define void @call2(ptr %obj) #1 !dbg !9 {
+  %vtable = load ptr, ptr %obj
+  %p = call i1 @llvm.type.test(ptr %vtable, metadata !"typeid2")
+  call void @llvm.assume(i1 %p)
+  %fptr = call ptr @llvm.load.relative.i32(ptr %vtable, i32 0)
+  ; CHECK: call void @vf(
+  call void %fptr(ptr %obj), !dbg !10
+  ret void
+}
+
+@_ZTV1A.local = private unnamed_addr constant { [3 x i32] } { [3 x i32] [
+  i32 0,  ; offset to top
+  i32 0,  ; rtti
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf to i64), i64 ptrtoint (ptr getelementptr inbounds ({ [3 x i32] }, ptr @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32)  ; vfunc offset
+] }, align 4, !type !14
+
+; CHECK: define void @call3
+define void @call3(ptr %obj) #1 !dbg !12 {
+  %vtable = load ptr, ptr %obj
+  %p = call i1 @llvm.type.test(ptr %vtable, metadata !"typeid3")
+  call void @llvm.assume(i1 %p)
+  %fptr = call ptr @llvm.load.relative.i32(ptr %vtable, i32 8)
+  ; CHECK: call void @vf(
+  call void %fptr(ptr %obj), !dbg !13
+  ret void
+}
+
+
 declare i1 @llvm.type.test(ptr, metadata)
 declare void @llvm.assume(i1)
 
@@ -45,5 +82,13 @@ declare void @llvm.assume(i1)
 !7 = distinct !DISubprogram(name: "vf", linkageName: "_ZN3vt12vfEv", scope: !1, file: !1, line: 13, isLocal: false, isDefinition: true, scopeLine: 13, flags: DIFlagPrototyped, isOptimized: false, unit: !0)
 !8 = !{i32 0, !"typeid"}
 
+!9 = distinct !DISubprogram(name: "call2", linkageName: "_Z5call2Pv", scope: !1, file: !1, line: 40, isLocal: false, isDefinition: true, scopeLine: 9, flags: DIFlagPrototyped, isOptimized: false, unit: !0)
+!10 = !DILocation(line: 41, column: 32, scope: !9)
+!11 = !{i32 0, !"typeid2"}
+
+!12 = distinct !DISubprogram(name: "call3", linkageName: "_Z5call3Pv", scope: !1, file: !1, line: 50, isLocal: false, isDefinition: true, scopeLine: 9, flags: DIFlagPrototyped, isOptimized: false, unit: !0)
+!13 = !DILocation(line: 51, column: 32, scope: !12)
+!14 = !{i32 0, !"typeid3"}
+
 ; CHECK: 1 wholeprogramdevirt - Number of whole program devirtualization targets
-; CHECK: 1 wholeprogramdevirt - Number of single implementation devirtualizations
+; CHECK: 3 wholeprogramdevirt - Number of single implementation devirtualizations
diff --git a/llvm/test/Transforms/WholeProgramDevirt/pointer-vtable.ll b/llvm/test/Transforms/WholeProgramDevirt/pointer-vtable.ll
index 3c5d9b839bc4..062cef9b88fe 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/pointer-vtable.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/pointer-vtable.ll
@@ -20,7 +20,23 @@ define void @call(ptr %obj) {
   ret void
 }
 
+@vt2 = constant i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf to i64), i64 ptrtoint (ptr @vt2 to i64)) to i32), !type !1
+
+declare ptr @llvm.load.relative.i32(ptr, i32)
+
+; CHECK: define void @call2
+define void @call2(ptr %obj) {
+  %vtable = load ptr, ptr %obj
+  %p = call i1 @llvm.type.test(ptr %vtable, metadata !"typeid2")
+  call void @llvm.assume(i1 %p)
+  %fptr = call ptr @llvm.load.relative.i32(ptr %vtable, i32 0)
+  ; CHECK: call void @vf(
+  call void %fptr(ptr %obj)
+  ret void
+}
+
 declare i1 @llvm.type.test(ptr, metadata)
 declare void @llvm.assume(i1)
 
 !0 = !{i32 0, !"typeid"}
+!1 = !{i32 0, !"typeid2"}
-- 
GitLab


From b8b2f9902cb23af0811ec218a4f43226c0691510 Mon Sep 17 00:00:00 2001
From: Leonard Chan <leonardchan@google.com>
Date: Thu, 11 Apr 2024 14:55:17 -0700
Subject: [PATCH 270/464] Reapply "[llvm] Teach GlobalDCE about
 dso_local_equivalent"

This reverts commit 1c604a9780fcfe92a99d539913553f0835b81de3.
---
 .../include/llvm/Analysis/TypeMetadataUtils.h |  4 ++--
 ...virtual-functions-relative-pointers-bad.ll | 18 ++++++++++++++
 ...virtual-functions-relative-pointers-gep.ll | 24 +++++++++++++++++++
 .../virtual-functions-relative-pointers.ll    | 24 +++++++++++++++++++
 4 files changed, 68 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/Analysis/TypeMetadataUtils.h b/llvm/include/llvm/Analysis/TypeMetadataUtils.h
index 2c7ce0858b9d..bdb477b54b53 100644
--- a/llvm/include/llvm/Analysis/TypeMetadataUtils.h
+++ b/llvm/include/llvm/Analysis/TypeMetadataUtils.h
@@ -87,8 +87,8 @@ std::pair<Function *, Constant *>
 getFunctionAtVTableOffset(GlobalVariable *GV, uint64_t Offset, Module &M);
 
 /// Finds the same "relative pointer" pattern as described above, where the
-/// target is `F`, and replaces the entire pattern with a constant zero.
-void replaceRelativePointerUsersWithZero(Function *F);
+/// target is `C`, and replaces the entire pattern with a constant zero.
+void replaceRelativePointerUsersWithZero(Constant *C);
 
 } // namespace llvm
 
diff --git a/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers-bad.ll b/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers-bad.ll
index 9d87fdb084a1..9a4bff80383c 100644
--- a/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers-bad.ll
+++ b/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers-bad.ll
@@ -16,14 +16,32 @@ declare { ptr, i1 } @llvm.type.checked.load(ptr, i32, metadata)
 
 ; CHECK:      @vtable = internal unnamed_addr constant { [3 x i32] } zeroinitializer, align 8, !type !0, !type !1, !vcall_visibility !2
 
+@vtable2 = internal unnamed_addr constant { [3 x i32] } { [3 x i32] [
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vfunc3 to i64), i64 ptrtoint (ptr @vtable2 to i64)) to i32),
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vfunc4 to i64), i64 ptrtoint (ptr @vtable2 to i64)) to i32),
+
+  ; a "bad" relative pointer because it's base is not the @vtable symbol
+  i32 trunc (i64 sub (i64 ptrtoint (ptr @weird_ref_3         to i64), i64 ptrtoint (ptr @weird_ref_4 to i64)) to i32)
+]}, align 4, !type !3, !type !4, !vcall_visibility !{i64 2}
+!3 = !{i64 0, !"vfunc3.type"}
+!4 = !{i64 4, !"vfunc4.type"}
+
+; CHECK:      @vtable2 = internal unnamed_addr constant { [3 x i32] } zeroinitializer, align 4, !type !3, !type !4, !vcall_visibility !2
+
 define internal void @vfunc1() { ret void }
 define internal void @vfunc2() { ret void }
 define internal void @weird_ref_1() { ret void }
 define internal void @weird_ref_2() { ret void }
+declare void @vfunc3()
+declare void @vfunc4()
+declare void @weird_ref_3()
+declare void @weird_ref_4()
 
 define void @main() {
   %1 = ptrtoint ptr @vtable to i64 ; to keep @vtable alive
   call void @weird_ref_2()
+  %2 = ptrtoint ptr @vtable2 to i64 ; to keep @vtable2 alive
+  call void @weird_ref_4()
   ret void
 }
 
diff --git a/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers-gep.ll b/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers-gep.ll
index fb45d37dde6e..d89b024e7a6d 100644
--- a/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers-gep.ll
+++ b/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers-gep.ll
@@ -19,6 +19,20 @@ declare { ptr, i1 } @llvm.type.checked.load(ptr, i32, metadata)
 ; CHECK-SAME:   i32 0
 ; CHECK-SAME: ] }, align 8, !type !0, !type !1, !vcall_visibility !2
 
+@vtable2 = internal unnamed_addr constant { [4 x i32] } { [4 x i32] [
+  i32 42,
+  i32 1337,
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vfunc3_live_extern to i64), i64 ptrtoint (ptr getelementptr inbounds ({ [4 x i32] }, ptr @vtable2, i32 0, i32 0, i32 2) to i64)) to i32),
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vfunc4_dead_extern to i64), i64 ptrtoint (ptr getelementptr inbounds ({ [4 x i32] }, ptr @vtable2, i32 0, i32 0, i32 2) to i64)) to i32)
+]}, align 4, !type !3, !type !4, !vcall_visibility !{i64 2}
+!3 = !{i64 8, !"vfunc3.type"}
+!4 = !{i64 12, !"vfunc4.type"}
+
+; CHECK:      @vtable2 = internal unnamed_addr constant { [4 x i32] } { [4 x i32] [
+; CHECK-SAME:   i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vfunc3_live_extern to i64), i64 ptrtoint (ptr getelementptr inbounds ({ [4 x i32] }, ptr @vtable2, i32 0, i32 0, i32 2) to i64)) to i32),
+; CHECK-SAME:   i32 0
+; CHECK-SAME: ] }, align 4, !type !3, !type !4, !vcall_visibility !2
+
 ; (1) vfunc1_live is referenced from @main, stays alive
 define internal void @vfunc1_live() {
   ; CHECK: define internal void @vfunc1_live(
@@ -31,9 +45,19 @@ define internal void @vfunc2_dead() {
   ret void
 }
 
+; (3) vfunc3_live_extern is referenced from @main, stays alive
+; CHECK: declare void @vfunc3_live_extern
+declare void @vfunc3_live_extern()
+
+; (4) vfunc4_dead_extern is never referenced, gets removed and vtable slot is null'd
+; CHECK-NOT: declare void @vfunc4_dead_extern
+declare void @vfunc4_dead_extern()
+
 define void @main() {
   %1 = ptrtoint ptr @vtable to i64 ; to keep @vtable alive
   %2 = tail call { ptr, i1 } @llvm.type.checked.load(ptr null, i32 0, metadata !"vfunc1.type")
+  %3 = ptrtoint ptr @vtable2 to i64 ; to keep @vtable2 alive
+  %4 = tail call { ptr, i1 } @llvm.type.checked.load(ptr null, i32 0, metadata !"vfunc3.type")
   ret void
 }
 
diff --git a/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers.ll b/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers.ll
index 76a617ea99e6..b652765296bc 100644
--- a/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers.ll
+++ b/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers.ll
@@ -17,6 +17,20 @@ declare { ptr, i1 } @llvm.type.checked.load(ptr, i32, metadata)
 ; CHECK-SAME:   i32 0
 ; CHECK-SAME: ] }, align 8, !type !0, !type !1, !vcall_visibility !2
 
+; Similar to above, but the vtable is more aligned to how C++ relative vtables look.
+; That is, the functions may not be dso-local.
+@vtable2 = internal unnamed_addr constant { [2 x i32] } { [2 x i32] [
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vfunc3_live_extern to i64), i64 ptrtoint (ptr @vtable2 to i64)) to i32),
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vfunc4_dead_extern to i64), i64 ptrtoint (ptr @vtable2 to i64)) to i32)
+]}, align 4, !type !3, !type !4, !vcall_visibility !{i64 2}
+!3 = !{i64 0, !"vfunc3.type"}
+!4 = !{i64 4, !"vfunc4.type"}
+
+; CHECK:      @vtable2 = internal unnamed_addr constant { [2 x i32] } { [2 x i32] [
+; CHECK-SAME:   i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vfunc3_live_extern to i64), i64 ptrtoint (ptr @vtable2 to i64)) to i32),
+; CHECK-SAME:   i32 0
+; CHECK-SAME: ] }, align 4, !type !3, !type !4, !vcall_visibility !2
+
 ; (1) vfunc1_live is referenced from @main, stays alive
 define internal void @vfunc1_live() {
   ; CHECK: define internal void @vfunc1_live(
@@ -29,9 +43,19 @@ define internal void @vfunc2_dead() {
   ret void
 }
 
+; (3) vfunc3_live_extern is referenced from @main, stays alive
+; CHECK: declare void @vfunc3_live_extern
+declare void @vfunc3_live_extern()
+
+; (4) vfunc4_dead_extern is never referenced, gets removed and vtable slot is null'd
+; CHECK-NOT: declare void @vfunc4_dead_extern
+declare void @vfunc4_dead_extern()
+
 define void @main() {
   %1 = ptrtoint ptr @vtable to i64 ; to keep @vtable alive
   %2 = tail call { ptr, i1 } @llvm.type.checked.load(ptr null, i32 0, metadata !"vfunc1.type")
+  %3 = ptrtoint ptr @vtable2 to i64 ; to keep @vtable2 alive
+  %4 = tail call { ptr, i1 } @llvm.type.checked.load(ptr null, i32 0, metadata !"vfunc3.type")
   ret void
 }
 
-- 
GitLab


From 472ccf61a39387098ad3ed66c1353f82c7245ba3 Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Mon, 15 Apr 2024 17:59:04 +0000
Subject: [PATCH 271/464] [gn build] Port 61f1f1300254

---
 llvm/utils/gn/secondary/libcxx/include/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 4270bae57ff2..2819eacaa857 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -415,6 +415,7 @@ if (current_toolchain == default_toolchain) {
       "__coroutine/noop_coroutine_handle.h",
       "__coroutine/trivial_awaitables.h",
       "__debug_utils/randomize_range.h",
+      "__debug_utils/sanitizers.h",
       "__debug_utils/strict_weak_ordering_check.h",
       "__exception/exception.h",
       "__exception/exception_ptr.h",
-- 
GitLab


From c71be83638917f089f8f154525cc6d6f61591f21 Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Mon, 15 Apr 2024 17:59:05 +0000
Subject: [PATCH 272/464] [gn build] Port a06073f91e7b

---
 llvm/utils/gn/secondary/libcxx/include/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 2819eacaa857..20116606fa64 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -928,6 +928,7 @@ if (current_toolchain == default_toolchain) {
       "__utility/in_place.h",
       "__utility/integer_sequence.h",
       "__utility/is_pointer_in_range.h",
+      "__utility/is_valid_range.h",
       "__utility/move.h",
       "__utility/no_destroy.h",
       "__utility/pair.h",
-- 
GitLab


From 8969bf3ea670dfeafbac53624883dc5c7ce23f0c Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Mon, 15 Apr 2024 17:59:06 +0000
Subject: [PATCH 273/464] [gn build] Port a11a4324bb27

---
 llvm/utils/gn/secondary/clang/unittests/Tooling/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/clang/unittests/Tooling/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/Tooling/BUILD.gn
index 43f8b886f9b9..5b9fed07186c 100644
--- a/llvm/utils/gn/secondary/clang/unittests/Tooling/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/unittests/Tooling/BUILD.gn
@@ -31,6 +31,7 @@ unittest("ToolingTests") {
     "CommentHandlerTest.cpp",
     "CompilationDatabaseTest.cpp",
     "DependencyScannerTest.cpp",
+    "DependencyScanning/DependencyScanningFilesystemTest.cpp",
     "DiagnosticsYamlTest.cpp",
     "ExecutionTest.cpp",
     "FixItTest.cpp",
-- 
GitLab


From 2f720a1c65e0b9669e67d7c319d77d6c221d423d Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Mon, 15 Apr 2024 17:59:07 +0000
Subject: [PATCH 274/464] [gn build] Port a3ce29f7bb55

---
 llvm/utils/gn/secondary/libcxx/include/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 20116606fa64..4383f1d6d18f 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -664,6 +664,7 @@ if (current_toolchain == default_toolchain) {
       "__numeric/transform_exclusive_scan.h",
       "__numeric/transform_inclusive_scan.h",
       "__numeric/transform_reduce.h",
+      "__pstl/cpu_algos/cpu_traits.h",
       "__random/bernoulli_distribution.h",
       "__random/binomial_distribution.h",
       "__random/cauchy_distribution.h",
-- 
GitLab


From 2601ce41d6139761a4267b7cc4add4fea499d172 Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Mon, 15 Apr 2024 17:59:08 +0000
Subject: [PATCH 275/464] [gn build] Port c11976f525f3

---
 llvm/utils/gn/secondary/clang/unittests/Tooling/BUILD.gn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/utils/gn/secondary/clang/unittests/Tooling/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/Tooling/BUILD.gn
index 5b9fed07186c..258fb61038df 100644
--- a/llvm/utils/gn/secondary/clang/unittests/Tooling/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/unittests/Tooling/BUILD.gn
@@ -30,7 +30,7 @@ unittest("ToolingTests") {
     "CastExprTest.cpp",
     "CommentHandlerTest.cpp",
     "CompilationDatabaseTest.cpp",
-    "DependencyScannerTest.cpp",
+    "DependencyScanning/DependencyScannerTest.cpp",
     "DependencyScanning/DependencyScanningFilesystemTest.cpp",
     "DiagnosticsYamlTest.cpp",
     "ExecutionTest.cpp",
-- 
GitLab


From 831515147c551f2c2dc7289d4bc0be2215e6b445 Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Mon, 15 Apr 2024 17:59:09 +0000
Subject: [PATCH 276/464] [gn build] Port c3d58867bd3d

---
 llvm/utils/gn/secondary/llvm/lib/Target/AArch64/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AArch64/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AArch64/BUILD.gn
index a61e6058053d..879b7f0960a9 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AArch64/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AArch64/BUILD.gn
@@ -139,6 +139,7 @@ static_library("LLVMAArch64CodeGen") {
     "AArch64MacroFusion.cpp",
     "AArch64PBQPRegAlloc.cpp",
     "AArch64PointerAuth.cpp",
+    "AArch64PostCoalescerPass.cpp",
     "AArch64PromoteConstant.cpp",
     "AArch64RedundantCopyElimination.cpp",
     "AArch64RegisterInfo.cpp",
-- 
GitLab


From b570eb5e99567c92636279d014f376938fe4e79a Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Mon, 15 Apr 2024 18:05:13 +0000
Subject: [PATCH 277/464] [gn build] Manually port 44de2bb

---
 llvm/utils/gn/secondary/clang/include/clang/Config/BUILD.gn | 1 +
 llvm/utils/gn/secondary/clang/test/BUILD.gn                 | 1 +
 2 files changed, 2 insertions(+)

diff --git a/llvm/utils/gn/secondary/clang/include/clang/Config/BUILD.gn b/llvm/utils/gn/secondary/clang/include/clang/Config/BUILD.gn
index 7273803dd516..a3fb952b1112 100644
--- a/llvm/utils/gn/secondary/clang/include/clang/Config/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/include/clang/Config/BUILD.gn
@@ -27,6 +27,7 @@ write_cmake_config("Config") {
     "ENABLE_LINKER_BUILD_ID=",
     "ENABLE_X86_RELAX_RELOCATIONS=1",
     "CLANG_ENABLE_OBJC_REWRITER=1",  # FIXME: flag?
+    "CLANG_ENABLE_CIR=",
     "CLANG_SYSTEMZ_DEFAULT_ARCH=z10",
     "PPC_LINUX_DEFAULT_IEEELONGDOUBLE=",
   ]
diff --git a/llvm/utils/gn/secondary/clang/test/BUILD.gn b/llvm/utils/gn/secondary/clang/test/BUILD.gn
index bcf7c86ab9d0..4ed9352da9c9 100644
--- a/llvm/utils/gn/secondary/clang/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/test/BUILD.gn
@@ -48,6 +48,7 @@ write_lit_config("lit_site_cfg") {
     "CLANG_BUILT_STANDALONE=0",
     "CLANG_DEFAULT_CXX_STDLIB=",  # Empty string means "default value" here.
     "CLANG_DEFAULT_PIE_ON_LINUX=0",
+    "CLANG_ENABLE_CIR=0",
     "CLANG_PLUGIN_SUPPORT=0",  # FIXME: Analysis/plugins need global -fPIC
     "CURRENT_TOOLS_DIR=" + rebase_path("$root_out_dir/bin", dir),
     "CLANG_VENDOR_UTI=org.llvm.clang",
-- 
GitLab


From fa5fdfb126f9beeaa2f719930ef951d339f41e36 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Mon, 15 Apr 2024 11:06:35 -0700
Subject: [PATCH 278/464] [CodeGen] Remove extraneous ArrayRef (NFC) (#88601)

We don't need to create these instances of ArrayRef because
StructType::create and ConstantStruct::get take ArrayRef, and ArrayRef
can be implicitly constructed from C arrays.
---
 llvm/lib/CodeGen/LowerEmuTLS.cpp | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/CodeGen/LowerEmuTLS.cpp b/llvm/lib/CodeGen/LowerEmuTLS.cpp
index af0b0a20c856..ec36b669ac01 100644
--- a/llvm/lib/CodeGen/LowerEmuTLS.cpp
+++ b/llvm/lib/CodeGen/LowerEmuTLS.cpp
@@ -139,8 +139,7 @@ bool addEmuTlsVar(Module &M, const GlobalVariable *GV) {
   IntegerType *WordType = DL.getIntPtrType(C);
   PointerType *InitPtrType = PointerType::getUnqual(C);
   Type *ElementTypes[4] = {WordType, WordType, VoidPtrType, InitPtrType};
-  ArrayRef<Type*> ElementTypeArray(ElementTypes, 4);
-  StructType *EmuTlsVarType = StructType::create(ElementTypeArray);
+  StructType *EmuTlsVarType = StructType::create(ElementTypes);
   EmuTlsVar = cast<GlobalVariable>(
       M.getOrInsertGlobal(EmuTlsVarName, EmuTlsVarType));
   copyLinkageVisibility(M, GV, EmuTlsVar);
@@ -170,9 +169,7 @@ bool addEmuTlsVar(Module &M, const GlobalVariable *GV) {
       ConstantInt::get(WordType, DL.getTypeStoreSize(GVType)),
       ConstantInt::get(WordType, GVAlignment.value()), NullPtr,
       EmuTlsTmplVar ? EmuTlsTmplVar : NullPtr};
-  ArrayRef<Constant*> ElementValueArray(ElementValues, 4);
-  EmuTlsVar->setInitializer(
-      ConstantStruct::get(EmuTlsVarType, ElementValueArray));
+  EmuTlsVar->setInitializer(ConstantStruct::get(EmuTlsVarType, ElementValues));
   Align MaxAlignment =
       std::max(DL.getABITypeAlign(WordType), DL.getABITypeAlign(VoidPtrType));
   EmuTlsVar->setAlignment(MaxAlignment);
-- 
GitLab


From e39c653dbab3b4fdc5118219c7b5179244d0e1b6 Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Mon, 15 Apr 2024 18:07:33 +0000
Subject: [PATCH 279/464] [gn build] Port a34887550b7f

---
 .../gn/secondary/lldb/source/Plugins/Language/CPlusPlus/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/lldb/source/Plugins/Language/CPlusPlus/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Plugins/Language/CPlusPlus/BUILD.gn
index 9d57f2f0877e..91e5be0c8e27 100644
--- a/llvm/utils/gn/secondary/lldb/source/Plugins/Language/CPlusPlus/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Plugins/Language/CPlusPlus/BUILD.gn
@@ -32,6 +32,7 @@ static_library("CPlusPlus") {
     "LibCxxInitializerList.cpp",
     "LibCxxList.cpp",
     "LibCxxMap.cpp",
+    "LibCxxProxyArray.cpp",
     "LibCxxQueue.cpp",
     "LibCxxRangesRefView.cpp",
     "LibCxxSliceArray.cpp",
-- 
GitLab


From 1c5d5478f7f35d7f212fcbf2af042fc3c720e9ed Mon Sep 17 00:00:00 2001
From: Leonard Chan <leonardchan@google.com>
Date: Mon, 15 Apr 2024 11:11:20 -0700
Subject: [PATCH 280/464] Revert "Reapply "[llvm] Teach GlobalDCE about
 dso_local_equivalent""

This reverts commit b8b2f9902cb23af0811ec218a4f43226c0691510.

This was failing on a local build.
---
 .../include/llvm/Analysis/TypeMetadataUtils.h |  4 ++--
 ...virtual-functions-relative-pointers-bad.ll | 18 --------------
 ...virtual-functions-relative-pointers-gep.ll | 24 -------------------
 .../virtual-functions-relative-pointers.ll    | 24 -------------------
 4 files changed, 2 insertions(+), 68 deletions(-)

diff --git a/llvm/include/llvm/Analysis/TypeMetadataUtils.h b/llvm/include/llvm/Analysis/TypeMetadataUtils.h
index bdb477b54b53..2c7ce0858b9d 100644
--- a/llvm/include/llvm/Analysis/TypeMetadataUtils.h
+++ b/llvm/include/llvm/Analysis/TypeMetadataUtils.h
@@ -87,8 +87,8 @@ std::pair<Function *, Constant *>
 getFunctionAtVTableOffset(GlobalVariable *GV, uint64_t Offset, Module &M);
 
 /// Finds the same "relative pointer" pattern as described above, where the
-/// target is `C`, and replaces the entire pattern with a constant zero.
-void replaceRelativePointerUsersWithZero(Constant *C);
+/// target is `F`, and replaces the entire pattern with a constant zero.
+void replaceRelativePointerUsersWithZero(Function *F);
 
 } // namespace llvm
 
diff --git a/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers-bad.ll b/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers-bad.ll
index 9a4bff80383c..9d87fdb084a1 100644
--- a/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers-bad.ll
+++ b/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers-bad.ll
@@ -16,32 +16,14 @@ declare { ptr, i1 } @llvm.type.checked.load(ptr, i32, metadata)
 
 ; CHECK:      @vtable = internal unnamed_addr constant { [3 x i32] } zeroinitializer, align 8, !type !0, !type !1, !vcall_visibility !2
 
-@vtable2 = internal unnamed_addr constant { [3 x i32] } { [3 x i32] [
-  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vfunc3 to i64), i64 ptrtoint (ptr @vtable2 to i64)) to i32),
-  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vfunc4 to i64), i64 ptrtoint (ptr @vtable2 to i64)) to i32),
-
-  ; a "bad" relative pointer because it's base is not the @vtable symbol
-  i32 trunc (i64 sub (i64 ptrtoint (ptr @weird_ref_3         to i64), i64 ptrtoint (ptr @weird_ref_4 to i64)) to i32)
-]}, align 4, !type !3, !type !4, !vcall_visibility !{i64 2}
-!3 = !{i64 0, !"vfunc3.type"}
-!4 = !{i64 4, !"vfunc4.type"}
-
-; CHECK:      @vtable2 = internal unnamed_addr constant { [3 x i32] } zeroinitializer, align 4, !type !3, !type !4, !vcall_visibility !2
-
 define internal void @vfunc1() { ret void }
 define internal void @vfunc2() { ret void }
 define internal void @weird_ref_1() { ret void }
 define internal void @weird_ref_2() { ret void }
-declare void @vfunc3()
-declare void @vfunc4()
-declare void @weird_ref_3()
-declare void @weird_ref_4()
 
 define void @main() {
   %1 = ptrtoint ptr @vtable to i64 ; to keep @vtable alive
   call void @weird_ref_2()
-  %2 = ptrtoint ptr @vtable2 to i64 ; to keep @vtable2 alive
-  call void @weird_ref_4()
   ret void
 }
 
diff --git a/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers-gep.ll b/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers-gep.ll
index d89b024e7a6d..fb45d37dde6e 100644
--- a/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers-gep.ll
+++ b/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers-gep.ll
@@ -19,20 +19,6 @@ declare { ptr, i1 } @llvm.type.checked.load(ptr, i32, metadata)
 ; CHECK-SAME:   i32 0
 ; CHECK-SAME: ] }, align 8, !type !0, !type !1, !vcall_visibility !2
 
-@vtable2 = internal unnamed_addr constant { [4 x i32] } { [4 x i32] [
-  i32 42,
-  i32 1337,
-  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vfunc3_live_extern to i64), i64 ptrtoint (ptr getelementptr inbounds ({ [4 x i32] }, ptr @vtable2, i32 0, i32 0, i32 2) to i64)) to i32),
-  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vfunc4_dead_extern to i64), i64 ptrtoint (ptr getelementptr inbounds ({ [4 x i32] }, ptr @vtable2, i32 0, i32 0, i32 2) to i64)) to i32)
-]}, align 4, !type !3, !type !4, !vcall_visibility !{i64 2}
-!3 = !{i64 8, !"vfunc3.type"}
-!4 = !{i64 12, !"vfunc4.type"}
-
-; CHECK:      @vtable2 = internal unnamed_addr constant { [4 x i32] } { [4 x i32] [
-; CHECK-SAME:   i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vfunc3_live_extern to i64), i64 ptrtoint (ptr getelementptr inbounds ({ [4 x i32] }, ptr @vtable2, i32 0, i32 0, i32 2) to i64)) to i32),
-; CHECK-SAME:   i32 0
-; CHECK-SAME: ] }, align 4, !type !3, !type !4, !vcall_visibility !2
-
 ; (1) vfunc1_live is referenced from @main, stays alive
 define internal void @vfunc1_live() {
   ; CHECK: define internal void @vfunc1_live(
@@ -45,19 +31,9 @@ define internal void @vfunc2_dead() {
   ret void
 }
 
-; (3) vfunc3_live_extern is referenced from @main, stays alive
-; CHECK: declare void @vfunc3_live_extern
-declare void @vfunc3_live_extern()
-
-; (4) vfunc4_dead_extern is never referenced, gets removed and vtable slot is null'd
-; CHECK-NOT: declare void @vfunc4_dead_extern
-declare void @vfunc4_dead_extern()
-
 define void @main() {
   %1 = ptrtoint ptr @vtable to i64 ; to keep @vtable alive
   %2 = tail call { ptr, i1 } @llvm.type.checked.load(ptr null, i32 0, metadata !"vfunc1.type")
-  %3 = ptrtoint ptr @vtable2 to i64 ; to keep @vtable2 alive
-  %4 = tail call { ptr, i1 } @llvm.type.checked.load(ptr null, i32 0, metadata !"vfunc3.type")
   ret void
 }
 
diff --git a/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers.ll b/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers.ll
index b652765296bc..76a617ea99e6 100644
--- a/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers.ll
+++ b/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers.ll
@@ -17,20 +17,6 @@ declare { ptr, i1 } @llvm.type.checked.load(ptr, i32, metadata)
 ; CHECK-SAME:   i32 0
 ; CHECK-SAME: ] }, align 8, !type !0, !type !1, !vcall_visibility !2
 
-; Similar to above, but the vtable is more aligned to how C++ relative vtables look.
-; That is, the functions may not be dso-local.
-@vtable2 = internal unnamed_addr constant { [2 x i32] } { [2 x i32] [
-  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vfunc3_live_extern to i64), i64 ptrtoint (ptr @vtable2 to i64)) to i32),
-  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vfunc4_dead_extern to i64), i64 ptrtoint (ptr @vtable2 to i64)) to i32)
-]}, align 4, !type !3, !type !4, !vcall_visibility !{i64 2}
-!3 = !{i64 0, !"vfunc3.type"}
-!4 = !{i64 4, !"vfunc4.type"}
-
-; CHECK:      @vtable2 = internal unnamed_addr constant { [2 x i32] } { [2 x i32] [
-; CHECK-SAME:   i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vfunc3_live_extern to i64), i64 ptrtoint (ptr @vtable2 to i64)) to i32),
-; CHECK-SAME:   i32 0
-; CHECK-SAME: ] }, align 4, !type !3, !type !4, !vcall_visibility !2
-
 ; (1) vfunc1_live is referenced from @main, stays alive
 define internal void @vfunc1_live() {
   ; CHECK: define internal void @vfunc1_live(
@@ -43,19 +29,9 @@ define internal void @vfunc2_dead() {
   ret void
 }
 
-; (3) vfunc3_live_extern is referenced from @main, stays alive
-; CHECK: declare void @vfunc3_live_extern
-declare void @vfunc3_live_extern()
-
-; (4) vfunc4_dead_extern is never referenced, gets removed and vtable slot is null'd
-; CHECK-NOT: declare void @vfunc4_dead_extern
-declare void @vfunc4_dead_extern()
-
 define void @main() {
   %1 = ptrtoint ptr @vtable to i64 ; to keep @vtable alive
   %2 = tail call { ptr, i1 } @llvm.type.checked.load(ptr null, i32 0, metadata !"vfunc1.type")
-  %3 = ptrtoint ptr @vtable2 to i64 ; to keep @vtable2 alive
-  %4 = tail call { ptr, i1 } @llvm.type.checked.load(ptr null, i32 0, metadata !"vfunc3.type")
   ret void
 }
 
-- 
GitLab


From c0b77e0a4a9bb090e5ad3a28adcd0aa98715cfe5 Mon Sep 17 00:00:00 2001
From: Leonard Chan <leonardchan@google.com>
Date: Mon, 15 Apr 2024 11:14:50 -0700
Subject: [PATCH 281/464] Revert "Reapply "[llvm] Teach whole program
 devirtualization about relative vtables""

This reverts commit 09c3bfe9b3eb47a2af0c10531b25f90cfb5fa9f4.
---
 .../include/llvm/Analysis/TypeMetadataUtils.h |   2 +-
 llvm/lib/Analysis/ModuleSummaryAnalysis.cpp   |  37 +----
 llvm/lib/Analysis/TypeMetadataUtils.cpp       |  16 +-
 llvm/test/ThinLTO/X86/devirt.ll               |  95 +----------
 .../WholeProgramDevirt/Inputs/export.yaml     |   8 -
 .../WholeProgramDevirt/branch-funnel.ll       | 157 ------------------
 .../WholeProgramDevirt/constant-arg.ll        |  45 -----
 .../devirt-single-impl-check.ll               |  25 ---
 .../devirt-single-impl-multiple-assumes.ll    |  20 ---
 .../WholeProgramDevirt/devirt-single-impl.ll  |  47 +-----
 .../WholeProgramDevirt/pointer-vtable.ll      |  16 --
 11 files changed, 10 insertions(+), 458 deletions(-)

diff --git a/llvm/include/llvm/Analysis/TypeMetadataUtils.h b/llvm/include/llvm/Analysis/TypeMetadataUtils.h
index 2c7ce0858b9d..8894945c28d9 100644
--- a/llvm/include/llvm/Analysis/TypeMetadataUtils.h
+++ b/llvm/include/llvm/Analysis/TypeMetadataUtils.h
@@ -66,7 +66,7 @@ void findDevirtualizableCallsForTypeCheckedLoad(
 /// Used for example from GlobalDCE to find an entry in a C++ vtable that
 /// matches a vcall offset.
 ///
-/// To support relative vtables, getPointerAtOffset can see through "relative
+/// To support Swift vtables, getPointerAtOffset can see through "relative
 /// pointers", i.e. (sub-)expressions of the form of:
 ///
 /// @symbol = ... {
diff --git a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
index c3d15afe6d9c..deda1eebb3b5 100644
--- a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
+++ b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
@@ -22,7 +22,6 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Analysis/BlockFrequencyInfo.h"
 #include "llvm/Analysis/BranchProbabilityInfo.h"
-#include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/IndirectCallPromotionAnalysis.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/MemoryProfileInfo.h"
@@ -669,8 +668,7 @@ static void computeFunctionSummary(
 /// within the initializer.
 static void findFuncPointers(const Constant *I, uint64_t StartingOffset,
                              const Module &M, ModuleSummaryIndex &Index,
-                             VTableFuncList &VTableFuncs,
-                             const GlobalVariable &OrigGV) {
+                             VTableFuncList &VTableFuncs) {
   // First check if this is a function pointer.
   if (I->getType()->isPointerTy()) {
     auto C = I->stripPointerCasts();
@@ -698,7 +696,7 @@ static void findFuncPointers(const Constant *I, uint64_t StartingOffset,
       auto Offset = SL->getElementOffset(EI.index());
       unsigned Op = SL->getElementContainingOffset(Offset);
       findFuncPointers(cast<Constant>(I->getOperand(Op)),
-                       StartingOffset + Offset, M, Index, VTableFuncs, OrigGV);
+                       StartingOffset + Offset, M, Index, VTableFuncs);
     }
   } else if (auto *C = dyn_cast<ConstantArray>(I)) {
     ArrayType *ATy = C->getType();
@@ -706,34 +704,7 @@ static void findFuncPointers(const Constant *I, uint64_t StartingOffset,
     uint64_t EltSize = DL.getTypeAllocSize(EltTy);
     for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i) {
       findFuncPointers(cast<Constant>(I->getOperand(i)),
-                       StartingOffset + i * EltSize, M, Index, VTableFuncs,
-                       OrigGV);
-    }
-  } else if (const auto *CE = dyn_cast<ConstantExpr>(I)) {
-    // For relative vtables, the next sub-component should be a trunc.
-    if (CE->getOpcode() != Instruction::Trunc ||
-        !(CE = dyn_cast<ConstantExpr>(CE->getOperand(0))))
-      return;
-
-    // If this constant can be reduced to the offset between a function and a
-    // global, then we know this is a valid virtual function if the RHS is the
-    // original vtable we're scanning through.
-    if (CE->getOpcode() == Instruction::Sub) {
-      GlobalValue *LHS, *RHS;
-      APSInt LHSOffset, RHSOffset;
-      if (IsConstantOffsetFromGlobal(CE->getOperand(0), LHS, LHSOffset, DL) &&
-          IsConstantOffsetFromGlobal(CE->getOperand(1), RHS, RHSOffset, DL) &&
-          RHS == &OrigGV &&
-
-          // For relative vtables, this component should point to the callable
-          // function without any offsets.
-          LHSOffset == 0 &&
-
-          // Also, the RHS should always point to somewhere within the vtable.
-          RHSOffset <=
-              static_cast<uint64_t>(DL.getTypeAllocSize(OrigGV.getInitializer()->getType()))) {
-        findFuncPointers(LHS, StartingOffset, M, Index, VTableFuncs, OrigGV);
-      }
+                       StartingOffset + i * EltSize, M, Index, VTableFuncs);
     }
   }
 }
@@ -746,7 +717,7 @@ static void computeVTableFuncs(ModuleSummaryIndex &Index,
     return;
 
   findFuncPointers(V.getInitializer(), /*StartingOffset=*/0, M, Index,
-                   VTableFuncs, V);
+                   VTableFuncs);
 
 #ifndef NDEBUG
   // Validate that the VTableFuncs list is ordered by offset.
diff --git a/llvm/lib/Analysis/TypeMetadataUtils.cpp b/llvm/lib/Analysis/TypeMetadataUtils.cpp
index ee400175ef11..b8dcc39e9223 100644
--- a/llvm/lib/Analysis/TypeMetadataUtils.cpp
+++ b/llvm/lib/Analysis/TypeMetadataUtils.cpp
@@ -67,14 +67,6 @@ static void findLoadCallsAtConstantOffset(
         findLoadCallsAtConstantOffset(M, DevirtCalls, User, Offset + GEPOffset,
                                       CI, DT);
       }
-    } else if (auto *Call = dyn_cast<CallInst>(User)) {
-      if (Call->getIntrinsicID() == llvm::Intrinsic::load_relative) {
-        if (auto *LoadOffset = dyn_cast<ConstantInt>(Call->getOperand(1))) {
-          findCallsAtConstantOffset(DevirtCalls, nullptr, User,
-                                    Offset + LoadOffset->getSExtValue(), CI,
-                                    DT);
-        }
-      }
     }
   }
 }
@@ -139,12 +131,6 @@ void llvm::findDevirtualizableCallsForTypeCheckedLoad(
 
 Constant *llvm::getPointerAtOffset(Constant *I, uint64_t Offset, Module &M,
                                    Constant *TopLevelGlobal) {
-  // TODO: Ideally it would be the caller who knows if it's appropriate to strip
-  // the DSOLocalEquicalent. More generally, it would feel more appropriate to
-  // have two functions that handle absolute and relative pointers separately.
-  if (auto *Equiv = dyn_cast<DSOLocalEquivalent>(I))
-    I = Equiv->getGlobalValue();
-
   if (I->getType()->isPointerTy()) {
     if (Offset == 0)
       return I;
@@ -175,7 +161,7 @@ Constant *llvm::getPointerAtOffset(Constant *I, uint64_t Offset, Module &M,
                               Offset % ElemSize, M, TopLevelGlobal);
   }
 
-  // Relative-pointer support starts here.
+  // (Swift-specific) relative-pointer support starts here.
   if (auto *CI = dyn_cast<ConstantInt>(I)) {
     if (Offset == 0 && CI->isZero()) {
       return I;
diff --git a/llvm/test/ThinLTO/X86/devirt.ll b/llvm/test/ThinLTO/X86/devirt.ll
index a40f89e644f0..472e43d77680 100644
--- a/llvm/test/ThinLTO/X86/devirt.ll
+++ b/llvm/test/ThinLTO/X86/devirt.ll
@@ -27,36 +27,24 @@
 ; NOENABLESPLITFLAG-DAG: [[B:\^[0-9]+]] = gv: (name: "_ZTV1B", {{.*}} vTableFuncs: ((virtFunc: [[Bf]], offset: 16), (virtFunc: [[An]], offset: 24)), refs: ([[Bf]], [[An]])
 ; NOENABLESPLITFLAG-DAG: [[C:\^[0-9]+]] = gv: (name: "_ZTV1C", {{.*}} vTableFuncs: ((virtFunc: [[Cf]], offset: 16), (virtFunc: [[An]], offset: 24)), refs: ([[An]], [[Cf]])
 ; NOENABLESPLITFLAG-DAG: [[D:\^[0-9]+]] = gv: (name: "_ZTV1D", {{.*}} vTableFuncs: ((virtFunc: [[Dm]], offset: 16)), refs: ([[Dm]])
-; NOENABLESPLITFLAG-DAG: [[B_RV:\^[0-9]+]] = gv: (name: "_ZTV1B_RV", {{.*}} vTableFuncs: ((virtFunc: [[Bf]], offset: 8), (virtFunc: [[An]], offset: 12)), refs: ([[B_RV]], [[Bf]], [[An]])
-; NOENABLESPLITFLAG-DAG: [[C_RV:\^[0-9]+]] = gv: (name: "_ZTV1C_RV", {{.*}} vTableFuncs: ((virtFunc: [[Cf]], offset: 8), (virtFunc: [[An]], offset: 12)), refs: ([[C_RV]], [[An]], [[Cf]])
-; NOENABLESPLITFLAG-DAG: [[D_RV:\^[0-9]+]] = gv: (name: "_ZTV1D_RV", {{.*}} vTableFuncs: ((virtFunc: [[Dm]], offset: 8)), refs: ([[D_RV]], [[Dm]])
 ; NOENABLESPLITFLAG-DAG: typeidCompatibleVTable: (name: "_ZTS1A", summary: ((offset: 16, [[B]]), (offset: 16, [[C]])))
 ; NOENABLESPLITFLAG-DAG: typeidCompatibleVTable: (name: "_ZTS1B", summary: ((offset: 16, [[B]])))
 ; NOENABLESPLITFLAG-DAG: typeidCompatibleVTable: (name: "_ZTS1C", summary: ((offset: 16, [[C]])))
-; NOENABLESPLITFLAG-DAG: typeidCompatibleVTable: (name: "_ZTS1A_RV", summary: ((offset: 8, [[B_RV]]), (offset: 8, [[C_RV]])))
-; NOENABLESPLITFLAG-DAG: typeidCompatibleVTable: (name: "_ZTS1B_RV", summary: ((offset: 8, [[B_RV]])))
-; NOENABLESPLITFLAG-DAG: typeidCompatibleVTable: (name: "_ZTS1C_RV", summary: ((offset: 8, [[C_RV]])))
 ; Type Id on _ZTV1D should have been promoted
 ; NOENABLESPLITFLAG-DAG: typeidCompatibleVTable: (name: "1.{{.*}}", summary: ((offset: 16, [[D]])))
-; NOENABLESPLITFLAG-DAG: typeidCompatibleVTable: (name: "2.{{.*}}", summary: ((offset: 8, [[D_RV]])))
 
 ; Index based WPD
 ; RUN: llvm-lto2 run %t2.o -save-temps -pass-remarks=. \
 ; RUN:   -whole-program-visibility \
 ; RUN:   -o %t3 \
 ; RUN:   -r=%t2.o,test,px \
-; RUN:   -r=%t2.o,test_rv,px \
 ; RUN:   -r=%t2.o,_ZN1A1nEi,p \
 ; RUN:   -r=%t2.o,_ZN1B1fEi,p \
 ; RUN:   -r=%t2.o,_ZN1C1fEi,p \
 ; RUN:   -r=%t2.o,_ZN1D1mEi,p \
 ; RUN:   -r=%t2.o,_ZTV1B,px \
 ; RUN:   -r=%t2.o,_ZTV1C,px \
-; RUN:   -r=%t2.o,_ZTV1D,px \
-; RUN:   -r=%t2.o,_ZTV1B_RV,px \
-; RUN:   -r=%t2.o,_ZTV1C_RV,px \
-; RUN:   -r=%t2.o,_ZTV1D_RV,px \
-; RUN:   2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN:   -r=%t2.o,_ZTV1D,px 2>&1 | FileCheck %s --check-prefix=REMARK
 ; RUN: llvm-dis %t3.1.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR
 
 ; Check that we're able to prevent specific function from being
@@ -66,24 +54,18 @@
 ; RUN:   -wholeprogramdevirt-skip=_ZN1A1nEi \
 ; RUN:   -o %t3 \
 ; RUN:   -r=%t2.o,test,px \
-; RUN:   -r=%t2.o,test_rv,px \
 ; RUN:   -r=%t2.o,_ZN1A1nEi,p \
 ; RUN:   -r=%t2.o,_ZN1B1fEi,p \
 ; RUN:   -r=%t2.o,_ZN1C1fEi,p \
 ; RUN:   -r=%t2.o,_ZN1D1mEi,p \
 ; RUN:   -r=%t2.o,_ZTV1B,px \
 ; RUN:   -r=%t2.o,_ZTV1C,px \
-; RUN:   -r=%t2.o,_ZTV1D,px \
-; RUN:   -r=%t2.o,_ZTV1B_RV,px \
-; RUN:   -r=%t2.o,_ZTV1C_RV,px \
-; RUN:   -r=%t2.o,_ZTV1D_RV,px \
-; RUN:   2>&1 | FileCheck %s --check-prefix=SKIP
+; RUN:   -r=%t2.o,_ZTV1D,px 2>&1 | FileCheck %s --check-prefix=SKIP
 
 ; RUN: llvm-lto2 run %t.o -save-temps -pass-remarks=. \
 ; RUN:   -whole-program-visibility \
 ; RUN:   -o %t3 \
 ; RUN:   -r=%t.o,test,px \
-; RUN:   -r=%t.o,test_rv,px \
 ; RUN:   -r=%t.o,_ZN1A1nEi,p \
 ; RUN:   -r=%t.o,_ZN1B1fEi,p \
 ; RUN:   -r=%t.o,_ZN1C1fEi,p \
@@ -91,24 +73,15 @@
 ; RUN:   -r=%t.o,_ZTV1B, \
 ; RUN:   -r=%t.o,_ZTV1C, \
 ; RUN:   -r=%t.o,_ZTV1D, \
-; RUN:   -r=%t.o,_ZTV1B_RV, \
-; RUN:   -r=%t.o,_ZTV1C_RV, \
-; RUN:   -r=%t.o,_ZTV1D_RV, \
 ; RUN:   -r=%t.o,_ZN1A1nEi, \
 ; RUN:   -r=%t.o,_ZN1B1fEi, \
 ; RUN:   -r=%t.o,_ZN1C1fEi, \
 ; RUN:   -r=%t.o,_ZN1D1mEi, \
 ; RUN:   -r=%t.o,_ZTV1B,px \
 ; RUN:   -r=%t.o,_ZTV1C,px \
-; RUN:   -r=%t.o,_ZTV1D,px \
-; RUN:   -r=%t.o,_ZTV1B_RV,px \
-; RUN:   -r=%t.o,_ZTV1C_RV,px \
-; RUN:   -r=%t.o,_ZTV1D_RV,px \
-; RUN:   2>&1 | FileCheck %s --check-prefix=REMARK --dump-input=fail
+; RUN:   -r=%t.o,_ZTV1D,px 2>&1 | FileCheck %s --check-prefix=REMARK --dump-input=fail
 ; RUN: llvm-dis %t3.1.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR
 
-; REMARK-DAG: single-impl: devirtualized a call to _ZN1A1nEi
-; REMARK-DAG: single-impl: devirtualized a call to _ZN1D1mEi
 ; REMARK-DAG: single-impl: devirtualized a call to _ZN1A1nEi
 ; REMARK-DAG: single-impl: devirtualized a call to _ZN1D1mEi
 
@@ -126,25 +99,6 @@ target triple = "x86_64-grtev4-linux-gnu"
 @_ZTV1C = constant { [4 x ptr] } { [4 x ptr] [ptr null, ptr undef, ptr @_ZN1C1fEi, ptr @_ZN1A1nEi] }, !type !0, !type !2
 @_ZTV1D = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr undef, ptr @_ZN1D1mEi] }, !type !3
 
-@_ZTV1B_RV = constant { [4 x i32] } { [4 x i32] [
-  i32 0,
-  i32 undef,
-  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @_ZN1B1fEi to i64), i64 ptrtoint (ptr getelementptr inbounds ({ [4 x i32] }, ptr @_ZTV1B_RV, i32 0, i32 0, i32 2) to i64)) to i32),
-  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @_ZN1A1nEi to i64), i64 ptrtoint (ptr getelementptr inbounds ({ [4 x i32] }, ptr @_ZTV1B_RV, i32 0, i32 0, i32 3) to i64)) to i32)
-] }, !type !7, !type !8
-
-@_ZTV1C_RV = constant { [4 x i32] } { [4 x i32] [
-  i32 0,
-  i32 undef,
-  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @_ZN1C1fEi to i64), i64 ptrtoint (ptr getelementptr inbounds ({ [4 x i32] }, ptr @_ZTV1C_RV, i32 0, i32 0, i32 2) to i64)) to i32),
-  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @_ZN1A1nEi to i64), i64 ptrtoint (ptr getelementptr inbounds ({ [4 x i32] }, ptr @_ZTV1C_RV, i32 0, i32 0, i32 3) to i64)) to i32)
-] }, !type !7, !type !9
-
-@_ZTV1D_RV = constant { [3 x i32] } { [3 x i32] [
-  i32 0,
-  i32 undef,
-  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @_ZN1D1mEi to i64), i64 ptrtoint (ptr getelementptr inbounds ({ [3 x i32] }, ptr @_ZTV1D_RV, i32 0, i32 0, i32 2) to i64)) to i32)
-] }, !type !10
 
 ; CHECK-IR-LABEL: define {{(noundef )?}}i32 @test
 define i32 @test(ptr %obj, ptr %obj2, i32 %a) {
@@ -182,43 +136,6 @@ entry:
 ; CHECK-IR-LABEL: ret i32
 ; CHECK-IR-LABEL: }
 
-declare ptr @llvm.load.relative.i32(ptr, i32)
-
-; CHECK-IR-LABEL: define i32 @test_rv
-define i32 @test_rv(ptr %obj, ptr %obj2, i32 %a) {
-entry:
-  %vtable = load ptr, ptr %obj
-  %p = call i1 @llvm.type.test(ptr %vtable, metadata !"_ZTS1A_RV")
-  call void @llvm.assume(i1 %p)
-  %fptr1_rv = call ptr @llvm.load.relative.i32(ptr %vtable, i32 4)
-
-  ; Check that the call was devirtualized.
-  ; CHECK-IR: %call = tail call i32 @_ZN1A1nEi
-  ; Ensure !prof and !callees metadata for indirect call promotion removed.
-  ; CHECK-IR-NOT: prof
-  ; CHECK-IR-NOT: callees
-  %call = tail call i32 %fptr1_rv(ptr nonnull %obj, i32 %a), !prof !5, !callees !6
-
-  %fptr22_rv = call ptr @llvm.load.relative.i32(ptr %vtable, i32 0)
-
-  ; We still have to call it as virtual.
-  ; CHECK-IR: %call3 = tail call i32 %fptr22
-  %call3 = tail call i32 %fptr22_rv(ptr nonnull %obj, i32 %call)
-
-  %vtable2 = load ptr, ptr %obj2
-  %p2 = call i1 @llvm.type.test(ptr %vtable2, metadata !11)
-  call void @llvm.assume(i1 %p2)
-
-  %fptr33_rv = call ptr @llvm.load.relative.i32(ptr %vtable2, i32 0)
-
-  ; Check that the call was devirtualized.
-  ; CHECK-IR: %call4 = tail call i32 @_ZN1D1mEi
-  %call4 = tail call i32 %fptr33_rv(ptr nonnull %obj2, i32 %call3)
-  ret i32 %call4
-}
-; CHECK-IR-LABEL: ret i32
-; CHECK-IR-LABEL: }
-
 declare i1 @llvm.type.test(ptr, metadata)
 declare void @llvm.assume(i1)
 
@@ -248,9 +165,3 @@ attributes #0 = { noinline optnone }
 !4 = distinct !{}
 !5 = !{!"VP", i32 0, i64 1, i64 1621563287929432257, i64 1}
 !6 = !{ptr @_ZN1A1nEi}
-
-!7 = !{i64 8, !"_ZTS1A_RV"}
-!8 = !{i64 8, !"_ZTS1B_RV"}
-!9 = !{i64 8, !"_ZTS1C_RV"}
-!10 = !{i64 8, !11}
-!11 = distinct !{}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/Inputs/export.yaml b/llvm/test/Transforms/WholeProgramDevirt/Inputs/export.yaml
index dd0c90d7ae8c..71cf38b216c7 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/Inputs/export.yaml
+++ b/llvm/test/Transforms/WholeProgramDevirt/Inputs/export.yaml
@@ -5,22 +5,14 @@ GlobalValueMap:
       TypeTestAssumeVCalls:
         - GUID: 14276520915468743435  # typeid1
           Offset: 0
-        - GUID: 271751036925422857  # typeid1_rv
-          Offset: 0
       TypeCheckedLoadVCalls:
         - GUID: 15427464259790519041  # typeid2
           Offset: 0
-        - GUID: 1146149264729288256  # typeid2_rv
-          Offset: 0
       TypeTestAssumeConstVCalls:
         - VFunc:
             GUID: 3515965990081467659  # typeid3
             Offset: 0
           Args: [12, 24]
-        - VFunc:
-            GUID: 2777626534618191571  # typeid3_rv
-            Offset: 0
-          Args: [12, 24]
       TypeCheckedLoadConstVCalls:
         - VFunc:
             GUID: 17525413373118030901  # typeid4
diff --git a/llvm/test/Transforms/WholeProgramDevirt/branch-funnel.ll b/llvm/test/Transforms/WholeProgramDevirt/branch-funnel.ll
index 0b1023eee273..b55713fe4b29 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/branch-funnel.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/branch-funnel.ll
@@ -8,45 +8,6 @@
 ; RUN: FileCheck --check-prefix=SUMMARY %s < %t
 
 ; SUMMARY:      TypeIdMap:       
-; SUMMARY-NEXT:  typeid1_rv:
-; SUMMARY-NEXT:    TTRes:
-; SUMMARY-NEXT:      Kind:            Unknown
-; SUMMARY-NEXT:      SizeM1BitWidth:  0
-; SUMMARY-NEXT:      AlignLog2:       0
-; SUMMARY-NEXT:      SizeM1:          0
-; SUMMARY-NEXT:      BitMask:         0
-; SUMMARY-NEXT:      InlineBits:      0
-; SUMMARY-NEXT:    WPDRes:
-; SUMMARY-NEXT:      0:
-; SUMMARY-NEXT:        Kind:            BranchFunnel
-; SUMMARY-NEXT:        SingleImplName:  ''
-; SUMMARY-NEXT:        ResByArg:
-; SUMMARY-NEXT:  typeid2_rv:
-; SUMMARY-NEXT:    TTRes:
-; SUMMARY-NEXT:      Kind:            Unknown
-; SUMMARY-NEXT:      SizeM1BitWidth:  0
-; SUMMARY-NEXT:      AlignLog2:       0
-; SUMMARY-NEXT:      SizeM1:          0
-; SUMMARY-NEXT:      BitMask:         0
-; SUMMARY-NEXT:      InlineBits:      0
-; SUMMARY-NEXT:    WPDRes:
-; SUMMARY-NEXT:      0:
-; SUMMARY-NEXT:        Kind:            Indir
-; SUMMARY-NEXT:        SingleImplName:  ''
-; SUMMARY-NEXT:        ResByArg:
-; SUMMARY-NEXT:   typeid3_rv:
-; SUMMARY-NEXT:     TTRes:           
-; SUMMARY-NEXT:       Kind:            Unknown
-; SUMMARY-NEXT:       SizeM1BitWidth:  0
-; SUMMARY-NEXT:       AlignLog2:       0
-; SUMMARY-NEXT:       SizeM1:          0
-; SUMMARY-NEXT:       BitMask:         0
-; SUMMARY-NEXT:       InlineBits:      0
-; SUMMARY-NEXT:     WPDRes:          
-; SUMMARY-NEXT:       0:               
-; SUMMARY-NEXT:         Kind:            BranchFunnel
-; SUMMARY-NEXT:         SingleImplName:  ''
-; SUMMARY-NEXT:         ResByArg:        
 ; SUMMARY-NEXT:   typeid3:
 ; SUMMARY-NEXT:     TTRes:           
 ; SUMMARY-NEXT:       Kind:            Unknown
@@ -132,29 +93,6 @@ declare i32 @vf3_2(ptr %this, i32 %arg)
 declare i32 @vf4_1(ptr %this, i32 %arg)
 declare i32 @vf4_2(ptr %this, i32 %arg)
 
-declare ptr @llvm.load.relative.i32(ptr, i32)
-
-;; These are relative vtables equivalent to the ones above.
-@vt1_1_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf1_1 to i64), i64 ptrtoint (ptr @vt1_1_rv to i64)) to i32)], !type !5
-@vt1_2_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf1_2 to i64), i64 ptrtoint (ptr @vt1_2_rv to i64)) to i32)], !type !5
-
-@vt2_1_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_1 to i64), i64 ptrtoint (ptr @vt2_1_rv to i64)) to i32)], !type !6
-@vt2_2_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_2 to i64), i64 ptrtoint (ptr @vt2_2_rv to i64)) to i32)], !type !6
-@vt2_3_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_3 to i64), i64 ptrtoint (ptr @vt2_3_rv to i64)) to i32)], !type !6
-@vt2_4_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_4 to i64), i64 ptrtoint (ptr @vt2_4_rv to i64)) to i32)], !type !6
-@vt2_5_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_5 to i64), i64 ptrtoint (ptr @vt2_5_rv to i64)) to i32)], !type !6
-@vt2_6_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_6 to i64), i64 ptrtoint (ptr @vt2_6_rv to i64)) to i32)], !type !6
-@vt2_7_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_7 to i64), i64 ptrtoint (ptr @vt2_7_rv to i64)) to i32)], !type !6
-@vt2_8_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_8 to i64), i64 ptrtoint (ptr @vt2_8_rv to i64)) to i32)], !type !6
-@vt2_9_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_9 to i64), i64 ptrtoint (ptr @vt2_9_rv to i64)) to i32)], !type !6
-@vt2_10_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_10 to i64), i64 ptrtoint (ptr @vt2_10_rv to i64)) to i32)], !type !6
-@vt2_11_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_11 to i64), i64 ptrtoint (ptr @vt2_11_rv to i64)) to i32)], !type !6
-
-@vt3_1_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf3_1 to i64), i64 ptrtoint (ptr @vt3_1_rv to i64)) to i32)], !type !7
-@vt3_2_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf3_2 to i64), i64 ptrtoint (ptr @vt3_2_rv to i64)) to i32)], !type !7
-
-@vt4_1_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf4_1 to i64), i64 ptrtoint (ptr @vt4_1_rv to i64)) to i32)], !type !8
-@vt4_2_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf4_2 to i64), i64 ptrtoint (ptr @vt4_2_rv to i64)) to i32)], !type !8
 
 
 ; CHECK-LABEL: define i32 @fn1
@@ -170,19 +108,6 @@ define i32 @fn1(ptr %obj) #0 {
   ret i32 %result
 }
 
-; CHECK-LABEL: define i32 @fn1_rv
-; CHECK-NOT: call void (...) @llvm.icall.branch.funnel
-define i32 @fn1_rv(ptr %obj) #0 {
-  %vtable = load ptr, ptr %obj
-  %p = call i1 @llvm.type.test(ptr %vtable, metadata !"typeid1_rv")
-  call void @llvm.assume(i1 %p)
-  %fptr = call ptr @llvm.load.relative.i32(ptr %vtable, i32 0)
-  ; RETP: call i32 @__typeid_typeid1_rv_0_branch_funnel(ptr nest %vtable, ptr %obj, i32 1)
-  %result = call i32 %fptr(ptr %obj, i32 1)
-  ; NORETP: call i32 %
-  ret i32 %result
-}
-
 ; CHECK-LABEL: define i32 @fn2
 ; CHECK-NOT: call void (...) @llvm.icall.branch.funnel
 define i32 @fn2(ptr %obj) #0 {
@@ -195,18 +120,6 @@ define i32 @fn2(ptr %obj) #0 {
   ret i32 %result
 }
 
-; CHECK-LABEL: define i32 @fn2_rv
-; CHECK-NOT: call void (...) @llvm.icall.branch.funnel
-define i32 @fn2_rv(ptr %obj) #0 {
-  %vtable = load ptr, ptr %obj
-  %p = call i1 @llvm.type.test(ptr %vtable, metadata !"typeid2_rv")
-  call void @llvm.assume(i1 %p)
-  %fptr = call ptr @llvm.load.relative.i32(ptr %vtable, i32 0)
-  ; CHECK: call i32 %
-  %result = call i32 %fptr(ptr %obj, i32 1)
-  ret i32 %result
-}
-
 ; CHECK-LABEL: define i32 @fn3
 ; CHECK-NOT: call void (...) @llvm.icall.branch.funnel
 define i32 @fn3(ptr %obj) #0 {
@@ -220,75 +133,10 @@ define i32 @fn3(ptr %obj) #0 {
   ret i32 %result
 }
 
-; CHECK-LABEL: define i32 @fn3_rv
-; CHECK-NOT: call void (...) @llvm.icall.branch.funnel
-define i32 @fn3_rv(ptr %obj) #0 {
-  %vtable = load ptr, ptr %obj
-  %p = call i1 @llvm.type.test(ptr %vtable, metadata !9)
-  call void @llvm.assume(i1 %p)
-  %fptr = call ptr @llvm.load.relative.i32(ptr %vtable, i32 0)
-  ; RETP: call i32 @branch_funnel.1(ptr
-  ; NORETP: call i32 %
-  %result = call i32 %fptr(ptr %obj, i32 1)
-  ret i32 %result
-}
-
-; CHECK-LABEL: define i32 @fn4
-; CHECK-NOT: call void (...) @llvm.icall.branch.funnel
-define i32 @fn4(ptr %obj) #0 {
-  %p = call i1 @llvm.type.test(ptr @vt1_1, metadata !"typeid1")
-  call void @llvm.assume(i1 %p)
-  %fptr = load ptr, ptr @vt1_1
-  ; RETP: call i32 @__typeid_typeid1_0_branch_funnel(ptr nest @vt1_1, ptr %obj, i32 1)
-  %result = call i32 %fptr(ptr %obj, i32 1)
-  ; NORETP: call i32 %
-  ret i32 %result
-}
-
-; CHECK-LABEL: define i32 @fn4_cpy
-; CHECK-NOT: call void (...) @llvm.icall.branch.funnel
-define i32 @fn4_cpy(ptr %obj) #0 {
-  %p = call i1 @llvm.type.test(ptr @vt1_1, metadata !"typeid1")
-  call void @llvm.assume(i1 %p)
-  %fptr = load ptr, ptr @vt1_1
-  ; RETP: call i32 @__typeid_typeid1_0_branch_funnel(ptr nest @vt1_1, ptr %obj, i32 1)
-  %result = call i32 %fptr(ptr %obj, i32 1)
-  ; NORETP: call i32 %
-  ret i32 %result
-}
-
-; CHECK-LABEL: define i32 @fn4_rv
-; CHECK-NOT: call void (...) @llvm.icall.branch.funnel
-define i32 @fn4_rv(ptr %obj) #0 {
-  %p = call i1 @llvm.type.test(ptr @vt1_1_rv, metadata !"typeid1_rv")
-  call void @llvm.assume(i1 %p)
-  %fptr = call ptr @llvm.load.relative.i32(ptr @vt1_1_rv, i32 0)
-  ; RETP: call i32 @__typeid_typeid1_rv_0_branch_funnel(ptr nest @vt1_1_rv, ptr %obj, i32 1)
-  %result = call i32 %fptr(ptr %obj, i32 1)
-  ; NORETP: call i32 %
-  ret i32 %result
-}
-
-; CHECK-LABEL: define i32 @fn4_rv_cpy
-; CHECK-NOT: call void (...) @llvm.icall.branch.funnel
-define i32 @fn4_rv_cpy(ptr %obj) #0 {
-  %p = call i1 @llvm.type.test(ptr @vt1_1_rv, metadata !"typeid1_rv")
-  call void @llvm.assume(i1 %p)
-  %fptr = call ptr @llvm.load.relative.i32(ptr @vt1_1_rv, i32 0)
-  ; RETP: call i32 @__typeid_typeid1_rv_0_branch_funnel(ptr nest @vt1_1_rv, ptr %obj, i32 1)
-  %result = call i32 %fptr(ptr %obj, i32 1)
-  ; NORETP: call i32 %
-  ret i32 %result
-}
-
 ; CHECK-LABEL: define hidden void @__typeid_typeid1_0_branch_funnel(ptr nest %0, ...)
 ; CHECK-NEXT: musttail call void (...) @llvm.icall.branch.funnel(ptr %0, ptr {{(nonnull )?}}@vt1_1, ptr {{(nonnull )?}}@vf1_1, ptr {{(nonnull )?}}@vt1_2, ptr {{(nonnull )?}}@vf1_2, ...)
 
-; CHECK-LABEL: define hidden void @__typeid_typeid1_rv_0_branch_funnel(ptr nest %0, ...)
-; CHECK-NEXT: musttail call void (...) @llvm.icall.branch.funnel(ptr %0, ptr {{(nonnull )?}}@vt1_1_rv, ptr {{(nonnull )?}}@vf1_1, ptr {{(nonnull )?}}@vt1_2_rv, ptr {{(nonnull )?}}@vf1_2, ...)
-
 ; CHECK: define internal void @branch_funnel(ptr
-; CHECK: define internal void @branch_funnel.1(ptr
 
 declare i1 @llvm.type.test(ptr, metadata)
 declare void @llvm.assume(i1)
@@ -298,10 +146,5 @@ declare void @llvm.assume(i1)
 !2 = !{i32 0, !"typeid3"}
 !3 = !{i32 0, !4}
 !4 = distinct !{}
-!5 = !{i32 0, !"typeid1_rv"}
-!6 = !{i32 0, !"typeid2_rv"}
-!7 = !{i32 0, !"typeid3_rv"}
-!8 = !{i32 0, !9}
-!9 = distinct !{}
 
 attributes #0 = { "target-features"="+retpoline" }
diff --git a/llvm/test/Transforms/WholeProgramDevirt/constant-arg.ll b/llvm/test/Transforms/WholeProgramDevirt/constant-arg.ll
index e685a1a54c52..91bae4af9a21 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/constant-arg.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/constant-arg.ll
@@ -7,10 +7,6 @@ target triple = "x86_64-unknown-linux-gnu"
 ; CHECK: private constant { [8 x i8], [1 x ptr], [0 x i8] } { [8 x i8] c"\00\00\00\00\00\00\00\02", [1 x ptr] [ptr @vf2], [0 x i8] zeroinitializer }, !type [[T8]]
 ; CHECK: private constant { [8 x i8], [1 x ptr], [0 x i8] } { [8 x i8] c"\00\00\00\00\00\00\00\01", [1 x ptr] [ptr @vf4], [0 x i8] zeroinitializer }, !type [[T8]]
 ; CHECK: private constant { [8 x i8], [1 x ptr], [0 x i8] } { [8 x i8] c"\00\00\00\00\00\00\00\02", [1 x ptr] [ptr @vf8], [0 x i8] zeroinitializer }, !type [[T8]]
-; CHECK: private constant { [4 x i8], [1 x i32], [0 x i8] } { [4 x i8] c"\00\00\00\01", [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf1 to i64), i64 ptrtoint (ptr @vt1_rv to i64)) to i32)], [0 x i8] zeroinitializer }, align 4, !type [[T4:![0-9]+]]
-; CHECK: private constant { [4 x i8], [1 x i32], [0 x i8] } { [4 x i8] c"\00\00\00\02", [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2 to i64), i64 ptrtoint (ptr @vt2_rv to i64)) to i32)], [0 x i8] zeroinitializer }, align 4, !type [[T4]]
-; CHECK: private constant { [4 x i8], [1 x i32], [0 x i8] } { [4 x i8] c"\00\00\00\01", [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf4 to i64), i64 ptrtoint (ptr @vt4_rv to i64)) to i32)], [0 x i8] zeroinitializer }, align 4, !type [[T4]]
-; CHECK: private constant { [4 x i8], [1 x i32], [0 x i8] } { [4 x i8] c"\00\00\00\02", [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf8 to i64), i64 ptrtoint (ptr @vt8_rv to i64)) to i32)], [0 x i8] zeroinitializer }, align 4, !type [[T4]]
 
 @vt1 = constant [1 x ptr] [ptr @vf1], !type !0
 @vt2 = constant [1 x ptr] [ptr @vf2], !type !0
@@ -65,49 +61,8 @@ define i1 @call2(ptr %obj) {
   ret i1 %result
 }
 
-declare ptr @llvm.load.relative.i32(ptr, i32)
-
-@vt1_rv = private unnamed_addr constant [1 x i32] [
-  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf1 to i64), i64 ptrtoint (ptr @vt1_rv to i64)) to i32)
-], align 4, !type !1
-@vt2_rv = private unnamed_addr constant [1 x i32] [
-  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2 to i64), i64 ptrtoint (ptr @vt2_rv to i64)) to i32)
-], align 4, !type !1
-@vt4_rv = private unnamed_addr constant [1 x i32] [
-  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf4 to i64), i64 ptrtoint (ptr @vt4_rv to i64)) to i32)
-], align 4, !type !1
-@vt8_rv = private unnamed_addr constant [1 x i32] [
-  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf8 to i64), i64 ptrtoint (ptr @vt8_rv to i64)) to i32)
-], align 4, !type !1
-
-; CHECK: define i1 @call3
-define i1 @call3(ptr %obj) {
-  %vtable = load ptr, ptr %obj
-  %p = call i1 @llvm.type.test(ptr %vtable, metadata !"typeid2")
-  call void @llvm.assume(i1 %p)
-  %fptr = call ptr @llvm.load.relative.i32(ptr %vtable, i32 0)
-  ; CHECK: getelementptr {{.*}} -1
-  ; CHECK: and {{.*}}, 1
-  %result = call i1 %fptr(ptr %obj, i32 5)
-  ret i1 %result
-}
-
-; CHECK: define i1 @call4
-define i1 @call4(ptr %obj) {
-  %vtable = load ptr, ptr %obj
-  %p = call i1 @llvm.type.test(ptr %vtable, metadata !"typeid2")
-  call void @llvm.assume(i1 %p)
-  %fptr = call ptr @llvm.load.relative.i32(ptr %vtable, i32 0)
-  ; CHECK: getelementptr {{.*}} -1
-  ; CHECK: and {{.*}}, 2
-  %result = call i1 %fptr(ptr %obj, i32 10)
-  ret i1 %result
-}
-
 declare i1 @llvm.type.test(ptr, metadata)
 declare void @llvm.assume(i1)
 
 ; CHECK: [[T8]] = !{i32 8, !"typeid"}
-; CHECK: [[T4]] = !{i32 4, !"typeid2"}
 !0 = !{i32 0, !"typeid"}
-!1 = !{i32 0, !"typeid2"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check.ll b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check.ll
index 5a9e6c12c358..dc7b2024bc59 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check.ll
@@ -3,7 +3,6 @@
 target datalayout = "e-p:64:64"
 target triple = "x86_64-unknown-linux-gnu"
 
-; CHECK: remark: <unknown>:0:0: single-impl: devirtualized a call to vf
 ; CHECK: remark: <unknown>:0:0: single-impl: devirtualized a call to vf
 ; CHECK: remark: <unknown>:0:0: devirtualized vf
 ; CHECK-NOT: devirtualized
@@ -34,31 +33,7 @@ trap:
   unreachable
 }
 
-@vt3 = private unnamed_addr constant [1 x i32] [
-  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf to i64), i64 ptrtoint (ptr @vt3 to i64)) to i32)
-], align 4, !type !1
-
-; CHECK: define void @call2
-define void @call2(ptr %obj) {
-  %vtable = load ptr, ptr %obj
-  %pair = call {ptr, i1} @llvm.type.checked.load(ptr %vtable, i32 0, metadata !"typeid2")
-  %fptr = extractvalue {ptr, i1} %pair, 0
-  %p = extractvalue {ptr, i1} %pair, 1
-  ; CHECK: br i1 true,
-  br i1 %p, label %cont, label %trap
-
-cont:
-  ; CHECK: call void @vf(
-  call void %fptr(ptr %obj)
-  ret void
-
-trap:
-  call void @llvm.trap()
-  unreachable
-}
-
 declare {ptr, i1} @llvm.type.checked.load(ptr, i32, metadata)
 declare void @llvm.trap()
 
 !0 = !{i32 0, !"typeid"}
-!1 = !{i32 0, !"typeid2"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-multiple-assumes.ll b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-multiple-assumes.ll
index bfbbeaf8a065..ed144c23302d 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-multiple-assumes.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-multiple-assumes.ll
@@ -23,27 +23,7 @@ define void @call(ptr %obj) {
   ret void
 }
 
-declare ptr @llvm.load.relative.i32(ptr, i32)
-
-@vt3 = private unnamed_addr constant [1 x i32] [
-  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf to i64), i64 ptrtoint (ptr @vt3 to i64)) to i32)
-], align 4, !type !1
-
-; CHECK: define void @call2
-define void @call2(ptr %obj) {
-  %vtable = load ptr, ptr %obj
-  %p = call i1 @llvm.type.test(ptr %vtable, metadata !"typeid2")
-  call void @llvm.assume(i1 %p)
-  %p2 = call i1 @llvm.type.test(ptr %vtable, metadata !"typeid2")
-  call void @llvm.assume(i1 %p2)
-  %fptr = call ptr @llvm.load.relative.i32(ptr %vtable, i32 0)
-  ; CHECK: call void @vf(
-  call void %fptr(ptr %obj)
-  ret void
-}
-
 declare i1 @llvm.type.test(ptr, metadata)
 declare void @llvm.assume(i1)
 
 !0 = !{i32 0, !"typeid"}
-!1 = !{i32 0, !"typeid2"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl.ll b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl.ll
index d7f33df59495..1ce96f70d905 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl.ll
@@ -7,8 +7,6 @@ target datalayout = "e-p:64:64"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; CHECK: remark: devirt-single.cc:30:32: single-impl: devirtualized a call to vf
-; CHECK: remark: devirt-single.cc:41:32: single-impl: devirtualized a call to vf
-; CHECK: remark: devirt-single.cc:51:32: single-impl: devirtualized a call to vf
 ; CHECK: remark: devirt-single.cc:13:0: devirtualized vf
 ; CHECK-NOT: devirtualized
 
@@ -30,41 +28,6 @@ define void @call(ptr %obj) #1 !dbg !5 {
   ret void
 }
 
-declare ptr @llvm.load.relative.i32(ptr, i32)
-
-@vt3 = private unnamed_addr constant [1 x i32] [
-  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf to i64), i64 ptrtoint (ptr @vt3 to i64)) to i32)
-], align 4, !type !11
-
-; CHECK: define void @call2
-define void @call2(ptr %obj) #1 !dbg !9 {
-  %vtable = load ptr, ptr %obj
-  %p = call i1 @llvm.type.test(ptr %vtable, metadata !"typeid2")
-  call void @llvm.assume(i1 %p)
-  %fptr = call ptr @llvm.load.relative.i32(ptr %vtable, i32 0)
-  ; CHECK: call void @vf(
-  call void %fptr(ptr %obj), !dbg !10
-  ret void
-}
-
-@_ZTV1A.local = private unnamed_addr constant { [3 x i32] } { [3 x i32] [
-  i32 0,  ; offset to top
-  i32 0,  ; rtti
-  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf to i64), i64 ptrtoint (ptr getelementptr inbounds ({ [3 x i32] }, ptr @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32)  ; vfunc offset
-] }, align 4, !type !14
-
-; CHECK: define void @call3
-define void @call3(ptr %obj) #1 !dbg !12 {
-  %vtable = load ptr, ptr %obj
-  %p = call i1 @llvm.type.test(ptr %vtable, metadata !"typeid3")
-  call void @llvm.assume(i1 %p)
-  %fptr = call ptr @llvm.load.relative.i32(ptr %vtable, i32 8)
-  ; CHECK: call void @vf(
-  call void %fptr(ptr %obj), !dbg !13
-  ret void
-}
-
-
 declare i1 @llvm.type.test(ptr, metadata)
 declare void @llvm.assume(i1)
 
@@ -82,13 +45,5 @@ declare void @llvm.assume(i1)
 !7 = distinct !DISubprogram(name: "vf", linkageName: "_ZN3vt12vfEv", scope: !1, file: !1, line: 13, isLocal: false, isDefinition: true, scopeLine: 13, flags: DIFlagPrototyped, isOptimized: false, unit: !0)
 !8 = !{i32 0, !"typeid"}
 
-!9 = distinct !DISubprogram(name: "call2", linkageName: "_Z5call2Pv", scope: !1, file: !1, line: 40, isLocal: false, isDefinition: true, scopeLine: 9, flags: DIFlagPrototyped, isOptimized: false, unit: !0)
-!10 = !DILocation(line: 41, column: 32, scope: !9)
-!11 = !{i32 0, !"typeid2"}
-
-!12 = distinct !DISubprogram(name: "call3", linkageName: "_Z5call3Pv", scope: !1, file: !1, line: 50, isLocal: false, isDefinition: true, scopeLine: 9, flags: DIFlagPrototyped, isOptimized: false, unit: !0)
-!13 = !DILocation(line: 51, column: 32, scope: !12)
-!14 = !{i32 0, !"typeid3"}
-
 ; CHECK: 1 wholeprogramdevirt - Number of whole program devirtualization targets
-; CHECK: 3 wholeprogramdevirt - Number of single implementation devirtualizations
+; CHECK: 1 wholeprogramdevirt - Number of single implementation devirtualizations
diff --git a/llvm/test/Transforms/WholeProgramDevirt/pointer-vtable.ll b/llvm/test/Transforms/WholeProgramDevirt/pointer-vtable.ll
index 062cef9b88fe..3c5d9b839bc4 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/pointer-vtable.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/pointer-vtable.ll
@@ -20,23 +20,7 @@ define void @call(ptr %obj) {
   ret void
 }
 
-@vt2 = constant i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf to i64), i64 ptrtoint (ptr @vt2 to i64)) to i32), !type !1
-
-declare ptr @llvm.load.relative.i32(ptr, i32)
-
-; CHECK: define void @call2
-define void @call2(ptr %obj) {
-  %vtable = load ptr, ptr %obj
-  %p = call i1 @llvm.type.test(ptr %vtable, metadata !"typeid2")
-  call void @llvm.assume(i1 %p)
-  %fptr = call ptr @llvm.load.relative.i32(ptr %vtable, i32 0)
-  ; CHECK: call void @vf(
-  call void %fptr(ptr %obj)
-  ret void
-}
-
 declare i1 @llvm.type.test(ptr, metadata)
 declare void @llvm.assume(i1)
 
 !0 = !{i32 0, !"typeid"}
-!1 = !{i32 0, !"typeid2"}
-- 
GitLab


From b7b183371b54a2c4b5d2a39c594d3967a7384cb8 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Mon, 15 Apr 2024 10:55:41 -0700
Subject: [PATCH 282/464] [SLP][NFC]Improve perf of BoUpSLP::buildExternalUses
 function, NFC.

Perform required operations, only when the result is required + check if
value was not inserted already into the list of the externally used
scalars and early exit, if it was.
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    | 20 ++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 9a0eaca2f598..c63b500f546f 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -5514,6 +5514,7 @@ void BoUpSLP::reorderBottomToTop(bool IgnoreReorder) {
 
 void BoUpSLP::buildExternalUses(
     const ExtraValueToDebugLocsMap &ExternallyUsedValues) {
+  DenseMap<Value *, unsigned> ScalarToExtUses;
   // Collect the values that we need to extract from the tree.
   for (auto &TEPtr : VectorizableTree) {
     TreeEntry *Entry = TEPtr.get();
@@ -5527,13 +5528,18 @@ void BoUpSLP::buildExternalUses(
       Value *Scalar = Entry->Scalars[Lane];
       if (!isa<Instruction>(Scalar))
         continue;
-      int FoundLane = Entry->findLaneForValue(Scalar);
+      // All uses must be replaced already? No need to do it again.
+      auto It = ScalarToExtUses.find(Scalar);
+      if (It != ScalarToExtUses.end() && !ExternalUses[It->second].User)
+        continue;
 
       // Check if the scalar is externally used as an extra arg.
       const auto *ExtI = ExternallyUsedValues.find(Scalar);
       if (ExtI != ExternallyUsedValues.end()) {
+        int FoundLane = Entry->findLaneForValue(Scalar);
         LLVM_DEBUG(dbgs() << "SLP: Need to extract: Extra arg from lane "
-                          << Lane << " from " << *Scalar << ".\n");
+                          << FoundLane << " from " << *Scalar << ".\n");
+        ScalarToExtUses.try_emplace(Scalar, ExternalUses.size());
         ExternalUses.emplace_back(Scalar, nullptr, FoundLane);
       }
       for (User *U : Scalar->users()) {
@@ -5561,12 +5567,20 @@ void BoUpSLP::buildExternalUses(
             continue;
           }
           U = nullptr;
+          if (It != ScalarToExtUses.end()) {
+            ExternalUses[It->second].User = nullptr;
+            break;
+          }
         }
 
+        int FoundLane = Entry->findLaneForValue(Scalar);
         LLVM_DEBUG(dbgs() << "SLP: Need to extract:" << *UserInst
-                          << " from lane " << Lane << " from " << *Scalar
+                          << " from lane " << FoundLane << " from " << *Scalar
                           << ".\n");
+        It = ScalarToExtUses.try_emplace(Scalar, ExternalUses.size()).first;
         ExternalUses.emplace_back(Scalar, U, FoundLane);
+        if (!U)
+          break;
       }
     }
   }
-- 
GitLab


From 3eb097339e5fb78e91182674e891445ebe5a845b Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Mon, 15 Apr 2024 14:33:53 -0400
Subject: [PATCH 283/464] [lld/mac] Fix (adorable!) typo to cycle bots

---
 lld/MachO/ObjC.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lld/MachO/ObjC.cpp b/lld/MachO/ObjC.cpp
index 5902b82d30f5..c28f2b42a72c 100644
--- a/lld/MachO/ObjC.cpp
+++ b/lld/MachO/ObjC.cpp
@@ -458,7 +458,7 @@ private:
                                const PointerListInfo &ptrList);
 
   Defined *emitCategory(const ClassExtensionInfo &extInfo);
-  Defined *emitCatListEntrySec(const std::string &forCateogryName,
+  Defined *emitCatListEntrySec(const std::string &forCategoryName,
                                const std::string &forBaseClassName,
                                ObjFile *objFile);
   Defined *emitCategoryBody(const std::string &name, const Defined *nameSym,
@@ -878,7 +878,7 @@ void ObjcCategoryMerger::emitAndLinkPointerList(
 
 // This method creates an __objc_catlist ConcatInputSection with a single slot
 Defined *
-ObjcCategoryMerger::emitCatListEntrySec(const std::string &forCateogryName,
+ObjcCategoryMerger::emitCatListEntrySec(const std::string &forCategoryName,
                                         const std::string &forBaseClassName,
                                         ObjFile *objFile) {
   uint32_t sectionSize = target->wordSize;
@@ -894,7 +894,7 @@ ObjcCategoryMerger::emitCatListEntrySec(const std::string &forCateogryName,
   newCatList->parent = infoCategoryWriter.catListInfo.outputSection;
 
   std::string catSymName = "<__objc_catlist slot for merged category ";
-  catSymName += forBaseClassName + "(" + forCateogryName + ")>";
+  catSymName += forBaseClassName + "(" + forCategoryName + ")>";
 
   Defined *catListSym = make<Defined>(
       newStringData(catSymName.c_str()), /*file=*/objFile, newCatList,
@@ -1069,7 +1069,7 @@ void ObjcCategoryMerger::collectAndValidateCategoriesData() {
          off += target->wordSize) {
       Defined *categorySym = tryGetDefinedAtIsecOffset(catListCisec, off);
       assert(categorySym &&
-             "Failed to get a valid cateogry at __objc_catlit offset");
+             "Failed to get a valid category at __objc_catlit offset");
 
       // We only support ObjC categories (no swift + @objc)
       // TODO: Support swift + @objc categories also
-- 
GitLab


From 58c5f50f4c73945fdc5440dee2fba03267a460d4 Mon Sep 17 00:00:00 2001
From: Leonard Chan <leonardchan@google.com>
Date: Thu, 11 Apr 2024 14:55:17 -0700
Subject: [PATCH 284/464] Reapply "[llvm] Teach GlobalDCE about
 dso_local_equivalent"

Also reapply "[llvm] Teach whole program devirtualization about
relative vtables"

This reverts commit 1c604a9780fcfe92a99d539913553f0835b81de3 and
474f5efebed24547e76d022f0c5ffcc9db97ce6f.
---
 .../include/llvm/Analysis/TypeMetadataUtils.h |   6 +-
 llvm/lib/Analysis/ModuleSummaryAnalysis.cpp   |  37 ++++-
 llvm/lib/Analysis/TypeMetadataUtils.cpp       |  47 ++++--
 llvm/test/ThinLTO/X86/devirt.ll               |  95 ++++++++++-
 ...virtual-functions-relative-pointers-bad.ll |  18 ++
 ...virtual-functions-relative-pointers-gep.ll |  24 +++
 .../virtual-functions-relative-pointers.ll    |  30 +++-
 .../WholeProgramDevirt/Inputs/export.yaml     |   8 +
 .../WholeProgramDevirt/branch-funnel.ll       | 157 ++++++++++++++++++
 .../WholeProgramDevirt/constant-arg.ll        |  45 +++++
 .../devirt-single-impl-check.ll               |  25 +++
 .../devirt-single-impl-multiple-assumes.ll    |  20 +++
 .../WholeProgramDevirt/devirt-single-impl.ll  |  47 +++++-
 .../WholeProgramDevirt/pointer-vtable.ll      |  16 ++
 14 files changed, 548 insertions(+), 27 deletions(-)

diff --git a/llvm/include/llvm/Analysis/TypeMetadataUtils.h b/llvm/include/llvm/Analysis/TypeMetadataUtils.h
index 8894945c28d9..bdb477b54b53 100644
--- a/llvm/include/llvm/Analysis/TypeMetadataUtils.h
+++ b/llvm/include/llvm/Analysis/TypeMetadataUtils.h
@@ -66,7 +66,7 @@ void findDevirtualizableCallsForTypeCheckedLoad(
 /// Used for example from GlobalDCE to find an entry in a C++ vtable that
 /// matches a vcall offset.
 ///
-/// To support Swift vtables, getPointerAtOffset can see through "relative
+/// To support relative vtables, getPointerAtOffset can see through "relative
 /// pointers", i.e. (sub-)expressions of the form of:
 ///
 /// @symbol = ... {
@@ -87,8 +87,8 @@ std::pair<Function *, Constant *>
 getFunctionAtVTableOffset(GlobalVariable *GV, uint64_t Offset, Module &M);
 
 /// Finds the same "relative pointer" pattern as described above, where the
-/// target is `F`, and replaces the entire pattern with a constant zero.
-void replaceRelativePointerUsersWithZero(Function *F);
+/// target is `C`, and replaces the entire pattern with a constant zero.
+void replaceRelativePointerUsersWithZero(Constant *C);
 
 } // namespace llvm
 
diff --git a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
index deda1eebb3b5..c3d15afe6d9c 100644
--- a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
+++ b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
@@ -22,6 +22,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Analysis/BlockFrequencyInfo.h"
 #include "llvm/Analysis/BranchProbabilityInfo.h"
+#include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/IndirectCallPromotionAnalysis.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/MemoryProfileInfo.h"
@@ -668,7 +669,8 @@ static void computeFunctionSummary(
 /// within the initializer.
 static void findFuncPointers(const Constant *I, uint64_t StartingOffset,
                              const Module &M, ModuleSummaryIndex &Index,
-                             VTableFuncList &VTableFuncs) {
+                             VTableFuncList &VTableFuncs,
+                             const GlobalVariable &OrigGV) {
   // First check if this is a function pointer.
   if (I->getType()->isPointerTy()) {
     auto C = I->stripPointerCasts();
@@ -696,7 +698,7 @@ static void findFuncPointers(const Constant *I, uint64_t StartingOffset,
       auto Offset = SL->getElementOffset(EI.index());
       unsigned Op = SL->getElementContainingOffset(Offset);
       findFuncPointers(cast<Constant>(I->getOperand(Op)),
-                       StartingOffset + Offset, M, Index, VTableFuncs);
+                       StartingOffset + Offset, M, Index, VTableFuncs, OrigGV);
     }
   } else if (auto *C = dyn_cast<ConstantArray>(I)) {
     ArrayType *ATy = C->getType();
@@ -704,7 +706,34 @@ static void findFuncPointers(const Constant *I, uint64_t StartingOffset,
     uint64_t EltSize = DL.getTypeAllocSize(EltTy);
     for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i) {
       findFuncPointers(cast<Constant>(I->getOperand(i)),
-                       StartingOffset + i * EltSize, M, Index, VTableFuncs);
+                       StartingOffset + i * EltSize, M, Index, VTableFuncs,
+                       OrigGV);
+    }
+  } else if (const auto *CE = dyn_cast<ConstantExpr>(I)) {
+    // For relative vtables, the next sub-component should be a trunc.
+    if (CE->getOpcode() != Instruction::Trunc ||
+        !(CE = dyn_cast<ConstantExpr>(CE->getOperand(0))))
+      return;
+
+    // If this constant can be reduced to the offset between a function and a
+    // global, then we know this is a valid virtual function if the RHS is the
+    // original vtable we're scanning through.
+    if (CE->getOpcode() == Instruction::Sub) {
+      GlobalValue *LHS, *RHS;
+      APSInt LHSOffset, RHSOffset;
+      if (IsConstantOffsetFromGlobal(CE->getOperand(0), LHS, LHSOffset, DL) &&
+          IsConstantOffsetFromGlobal(CE->getOperand(1), RHS, RHSOffset, DL) &&
+          RHS == &OrigGV &&
+
+          // For relative vtables, this component should point to the callable
+          // function without any offsets.
+          LHSOffset == 0 &&
+
+          // Also, the RHS should always point to somewhere within the vtable.
+          RHSOffset <=
+              static_cast<uint64_t>(DL.getTypeAllocSize(OrigGV.getInitializer()->getType()))) {
+        findFuncPointers(LHS, StartingOffset, M, Index, VTableFuncs, OrigGV);
+      }
     }
   }
 }
@@ -717,7 +746,7 @@ static void computeVTableFuncs(ModuleSummaryIndex &Index,
     return;
 
   findFuncPointers(V.getInitializer(), /*StartingOffset=*/0, M, Index,
-                   VTableFuncs);
+                   VTableFuncs, V);
 
 #ifndef NDEBUG
   // Validate that the VTableFuncs list is ordered by offset.
diff --git a/llvm/lib/Analysis/TypeMetadataUtils.cpp b/llvm/lib/Analysis/TypeMetadataUtils.cpp
index b8dcc39e9223..67ce1540112b 100644
--- a/llvm/lib/Analysis/TypeMetadataUtils.cpp
+++ b/llvm/lib/Analysis/TypeMetadataUtils.cpp
@@ -67,6 +67,14 @@ static void findLoadCallsAtConstantOffset(
         findLoadCallsAtConstantOffset(M, DevirtCalls, User, Offset + GEPOffset,
                                       CI, DT);
       }
+    } else if (auto *Call = dyn_cast<CallInst>(User)) {
+      if (Call->getIntrinsicID() == llvm::Intrinsic::load_relative) {
+        if (auto *LoadOffset = dyn_cast<ConstantInt>(Call->getOperand(1))) {
+          findCallsAtConstantOffset(DevirtCalls, nullptr, User,
+                                    Offset + LoadOffset->getSExtValue(), CI,
+                                    DT);
+        }
+      }
     }
   }
 }
@@ -131,6 +139,12 @@ void llvm::findDevirtualizableCallsForTypeCheckedLoad(
 
 Constant *llvm::getPointerAtOffset(Constant *I, uint64_t Offset, Module &M,
                                    Constant *TopLevelGlobal) {
+  // TODO: Ideally it would be the caller who knows if it's appropriate to strip
+  // the DSOLocalEquicalent. More generally, it would feel more appropriate to
+  // have two functions that handle absolute and relative pointers separately.
+  if (auto *Equiv = dyn_cast<DSOLocalEquivalent>(I))
+    I = Equiv->getGlobalValue();
+
   if (I->getType()->isPointerTy()) {
     if (Offset == 0)
       return I;
@@ -161,7 +175,7 @@ Constant *llvm::getPointerAtOffset(Constant *I, uint64_t Offset, Module &M,
                               Offset % ElemSize, M, TopLevelGlobal);
   }
 
-  // (Swift-specific) relative-pointer support starts here.
+  // Relative-pointer support starts here.
   if (auto *CI = dyn_cast<ConstantInt>(I)) {
     if (Offset == 0 && CI->isZero()) {
       return I;
@@ -221,19 +235,26 @@ llvm::getFunctionAtVTableOffset(GlobalVariable *GV, uint64_t Offset,
   return std::pair<Function *, Constant *>(Fn, C);
 }
 
-void llvm::replaceRelativePointerUsersWithZero(Function *F) {
-  for (auto *U : F->users()) {
-    auto *PtrExpr = dyn_cast<ConstantExpr>(U);
-    if (!PtrExpr || PtrExpr->getOpcode() != Instruction::PtrToInt)
-      continue;
+static void replaceRelativePointerUserWithZero(User *U) {
+  auto *PtrExpr = dyn_cast<ConstantExpr>(U);
+  if (!PtrExpr || PtrExpr->getOpcode() != Instruction::PtrToInt)
+    return;
 
-    for (auto *PtrToIntUser : PtrExpr->users()) {
-      auto *SubExpr = dyn_cast<ConstantExpr>(PtrToIntUser);
-      if (!SubExpr || SubExpr->getOpcode() != Instruction::Sub)
-        continue;
+  for (auto *PtrToIntUser : PtrExpr->users()) {
+    auto *SubExpr = dyn_cast<ConstantExpr>(PtrToIntUser);
+    if (!SubExpr || SubExpr->getOpcode() != Instruction::Sub)
+      return;
 
-      SubExpr->replaceNonMetadataUsesWith(
-          ConstantInt::get(SubExpr->getType(), 0));
-    }
+    SubExpr->replaceNonMetadataUsesWith(
+        ConstantInt::get(SubExpr->getType(), 0));
+  }
+}
+
+void llvm::replaceRelativePointerUsersWithZero(Constant *C) {
+  for (auto *U : C->users()) {
+    if (auto *Equiv = dyn_cast<DSOLocalEquivalent>(U))
+      replaceRelativePointerUsersWithZero(Equiv);
+    else
+      replaceRelativePointerUserWithZero(U);
   }
 }
diff --git a/llvm/test/ThinLTO/X86/devirt.ll b/llvm/test/ThinLTO/X86/devirt.ll
index 472e43d77680..c4c2f861dc1f 100644
--- a/llvm/test/ThinLTO/X86/devirt.ll
+++ b/llvm/test/ThinLTO/X86/devirt.ll
@@ -27,24 +27,36 @@
 ; NOENABLESPLITFLAG-DAG: [[B:\^[0-9]+]] = gv: (name: "_ZTV1B", {{.*}} vTableFuncs: ((virtFunc: [[Bf]], offset: 16), (virtFunc: [[An]], offset: 24)), refs: ([[Bf]], [[An]])
 ; NOENABLESPLITFLAG-DAG: [[C:\^[0-9]+]] = gv: (name: "_ZTV1C", {{.*}} vTableFuncs: ((virtFunc: [[Cf]], offset: 16), (virtFunc: [[An]], offset: 24)), refs: ([[An]], [[Cf]])
 ; NOENABLESPLITFLAG-DAG: [[D:\^[0-9]+]] = gv: (name: "_ZTV1D", {{.*}} vTableFuncs: ((virtFunc: [[Dm]], offset: 16)), refs: ([[Dm]])
+; NOENABLESPLITFLAG-DAG: [[B_RV:\^[0-9]+]] = gv: (name: "_ZTV1B_RV", {{.*}} vTableFuncs: ((virtFunc: [[Bf]], offset: 8), (virtFunc: [[An]], offset: 12)), refs: ([[B_RV]], [[Bf]], [[An]])
+; NOENABLESPLITFLAG-DAG: [[C_RV:\^[0-9]+]] = gv: (name: "_ZTV1C_RV", {{.*}} vTableFuncs: ((virtFunc: [[Cf]], offset: 8), (virtFunc: [[An]], offset: 12)), refs: ([[C_RV]], [[An]], [[Cf]])
+; NOENABLESPLITFLAG-DAG: [[D_RV:\^[0-9]+]] = gv: (name: "_ZTV1D_RV", {{.*}} vTableFuncs: ((virtFunc: [[Dm]], offset: 8)), refs: ([[D_RV]], [[Dm]])
 ; NOENABLESPLITFLAG-DAG: typeidCompatibleVTable: (name: "_ZTS1A", summary: ((offset: 16, [[B]]), (offset: 16, [[C]])))
 ; NOENABLESPLITFLAG-DAG: typeidCompatibleVTable: (name: "_ZTS1B", summary: ((offset: 16, [[B]])))
 ; NOENABLESPLITFLAG-DAG: typeidCompatibleVTable: (name: "_ZTS1C", summary: ((offset: 16, [[C]])))
+; NOENABLESPLITFLAG-DAG: typeidCompatibleVTable: (name: "_ZTS1A_RV", summary: ((offset: 8, [[B_RV]]), (offset: 8, [[C_RV]])))
+; NOENABLESPLITFLAG-DAG: typeidCompatibleVTable: (name: "_ZTS1B_RV", summary: ((offset: 8, [[B_RV]])))
+; NOENABLESPLITFLAG-DAG: typeidCompatibleVTable: (name: "_ZTS1C_RV", summary: ((offset: 8, [[C_RV]])))
 ; Type Id on _ZTV1D should have been promoted
 ; NOENABLESPLITFLAG-DAG: typeidCompatibleVTable: (name: "1.{{.*}}", summary: ((offset: 16, [[D]])))
+; NOENABLESPLITFLAG-DAG: typeidCompatibleVTable: (name: "2.{{.*}}", summary: ((offset: 8, [[D_RV]])))
 
 ; Index based WPD
 ; RUN: llvm-lto2 run %t2.o -save-temps -pass-remarks=. \
 ; RUN:   -whole-program-visibility \
 ; RUN:   -o %t3 \
 ; RUN:   -r=%t2.o,test,px \
+; RUN:   -r=%t2.o,test_rv,px \
 ; RUN:   -r=%t2.o,_ZN1A1nEi,p \
 ; RUN:   -r=%t2.o,_ZN1B1fEi,p \
 ; RUN:   -r=%t2.o,_ZN1C1fEi,p \
 ; RUN:   -r=%t2.o,_ZN1D1mEi,p \
 ; RUN:   -r=%t2.o,_ZTV1B,px \
 ; RUN:   -r=%t2.o,_ZTV1C,px \
-; RUN:   -r=%t2.o,_ZTV1D,px 2>&1 | FileCheck %s --check-prefix=REMARK
+; RUN:   -r=%t2.o,_ZTV1D,px \
+; RUN:   -r=%t2.o,_ZTV1B_RV,px \
+; RUN:   -r=%t2.o,_ZTV1C_RV,px \
+; RUN:   -r=%t2.o,_ZTV1D_RV,px \
+; RUN:   2>&1 | FileCheck %s --check-prefix=REMARK
 ; RUN: llvm-dis %t3.1.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR
 
 ; Check that we're able to prevent specific function from being
@@ -54,18 +66,24 @@
 ; RUN:   -wholeprogramdevirt-skip=_ZN1A1nEi \
 ; RUN:   -o %t3 \
 ; RUN:   -r=%t2.o,test,px \
+; RUN:   -r=%t2.o,test_rv,px \
 ; RUN:   -r=%t2.o,_ZN1A1nEi,p \
 ; RUN:   -r=%t2.o,_ZN1B1fEi,p \
 ; RUN:   -r=%t2.o,_ZN1C1fEi,p \
 ; RUN:   -r=%t2.o,_ZN1D1mEi,p \
 ; RUN:   -r=%t2.o,_ZTV1B,px \
 ; RUN:   -r=%t2.o,_ZTV1C,px \
-; RUN:   -r=%t2.o,_ZTV1D,px 2>&1 | FileCheck %s --check-prefix=SKIP
+; RUN:   -r=%t2.o,_ZTV1D,px \
+; RUN:   -r=%t2.o,_ZTV1B_RV,px \
+; RUN:   -r=%t2.o,_ZTV1C_RV,px \
+; RUN:   -r=%t2.o,_ZTV1D_RV,px \
+; RUN:   2>&1 | FileCheck %s --check-prefix=SKIP
 
 ; RUN: llvm-lto2 run %t.o -save-temps -pass-remarks=. \
 ; RUN:   -whole-program-visibility \
 ; RUN:   -o %t3 \
 ; RUN:   -r=%t.o,test,px \
+; RUN:   -r=%t.o,test_rv,px \
 ; RUN:   -r=%t.o,_ZN1A1nEi,p \
 ; RUN:   -r=%t.o,_ZN1B1fEi,p \
 ; RUN:   -r=%t.o,_ZN1C1fEi,p \
@@ -73,15 +91,24 @@
 ; RUN:   -r=%t.o,_ZTV1B, \
 ; RUN:   -r=%t.o,_ZTV1C, \
 ; RUN:   -r=%t.o,_ZTV1D, \
+; RUN:   -r=%t.o,_ZTV1B_RV, \
+; RUN:   -r=%t.o,_ZTV1C_RV, \
+; RUN:   -r=%t.o,_ZTV1D_RV, \
 ; RUN:   -r=%t.o,_ZN1A1nEi, \
 ; RUN:   -r=%t.o,_ZN1B1fEi, \
 ; RUN:   -r=%t.o,_ZN1C1fEi, \
 ; RUN:   -r=%t.o,_ZN1D1mEi, \
 ; RUN:   -r=%t.o,_ZTV1B,px \
 ; RUN:   -r=%t.o,_ZTV1C,px \
-; RUN:   -r=%t.o,_ZTV1D,px 2>&1 | FileCheck %s --check-prefix=REMARK --dump-input=fail
+; RUN:   -r=%t.o,_ZTV1D,px \
+; RUN:   -r=%t.o,_ZTV1B_RV,px \
+; RUN:   -r=%t.o,_ZTV1C_RV,px \
+; RUN:   -r=%t.o,_ZTV1D_RV,px \
+; RUN:   2>&1 | FileCheck %s --check-prefix=REMARK --dump-input=fail
 ; RUN: llvm-dis %t3.1.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR
 
+; REMARK-DAG: single-impl: devirtualized a call to _ZN1A1nEi
+; REMARK-DAG: single-impl: devirtualized a call to _ZN1D1mEi
 ; REMARK-DAG: single-impl: devirtualized a call to _ZN1A1nEi
 ; REMARK-DAG: single-impl: devirtualized a call to _ZN1D1mEi
 
@@ -99,6 +126,25 @@ target triple = "x86_64-grtev4-linux-gnu"
 @_ZTV1C = constant { [4 x ptr] } { [4 x ptr] [ptr null, ptr undef, ptr @_ZN1C1fEi, ptr @_ZN1A1nEi] }, !type !0, !type !2
 @_ZTV1D = constant { [3 x ptr] } { [3 x ptr] [ptr null, ptr undef, ptr @_ZN1D1mEi] }, !type !3
 
+@_ZTV1B_RV = constant { [4 x i32] } { [4 x i32] [
+  i32 0,
+  i32 undef,
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @_ZN1B1fEi to i64), i64 ptrtoint (ptr getelementptr inbounds ({ [4 x i32] }, ptr @_ZTV1B_RV, i32 0, i32 0, i32 2) to i64)) to i32),
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @_ZN1A1nEi to i64), i64 ptrtoint (ptr getelementptr inbounds ({ [4 x i32] }, ptr @_ZTV1B_RV, i32 0, i32 0, i32 3) to i64)) to i32)
+] }, !type !7, !type !8
+
+@_ZTV1C_RV = constant { [4 x i32] } { [4 x i32] [
+  i32 0,
+  i32 undef,
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @_ZN1C1fEi to i64), i64 ptrtoint (ptr getelementptr inbounds ({ [4 x i32] }, ptr @_ZTV1C_RV, i32 0, i32 0, i32 2) to i64)) to i32),
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @_ZN1A1nEi to i64), i64 ptrtoint (ptr getelementptr inbounds ({ [4 x i32] }, ptr @_ZTV1C_RV, i32 0, i32 0, i32 3) to i64)) to i32)
+] }, !type !7, !type !9
+
+@_ZTV1D_RV = constant { [3 x i32] } { [3 x i32] [
+  i32 0,
+  i32 undef,
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @_ZN1D1mEi to i64), i64 ptrtoint (ptr getelementptr inbounds ({ [3 x i32] }, ptr @_ZTV1D_RV, i32 0, i32 0, i32 2) to i64)) to i32)
+] }, !type !10
 
 ; CHECK-IR-LABEL: define {{(noundef )?}}i32 @test
 define i32 @test(ptr %obj, ptr %obj2, i32 %a) {
@@ -136,6 +182,43 @@ entry:
 ; CHECK-IR-LABEL: ret i32
 ; CHECK-IR-LABEL: }
 
+declare ptr @llvm.load.relative.i32(ptr, i32)
+
+; CHECK-IR-LABEL: define {{.*}}i32 @test_rv
+define i32 @test_rv(ptr %obj, ptr %obj2, i32 %a) {
+entry:
+  %vtable = load ptr, ptr %obj
+  %p = call i1 @llvm.type.test(ptr %vtable, metadata !"_ZTS1A_RV")
+  call void @llvm.assume(i1 %p)
+  %fptr1_rv = call ptr @llvm.load.relative.i32(ptr %vtable, i32 4)
+
+  ; Check that the call was devirtualized.
+  ; CHECK-IR: %call = tail call i32 @_ZN1A1nEi
+  ; Ensure !prof and !callees metadata for indirect call promotion removed.
+  ; CHECK-IR-NOT: prof
+  ; CHECK-IR-NOT: callees
+  %call = tail call i32 %fptr1_rv(ptr nonnull %obj, i32 %a), !prof !5, !callees !6
+
+  %fptr22_rv = call ptr @llvm.load.relative.i32(ptr %vtable, i32 0)
+
+  ; We still have to call it as virtual.
+  ; CHECK-IR: %call3 = tail call i32 %fptr22
+  %call3 = tail call i32 %fptr22_rv(ptr nonnull %obj, i32 %call)
+
+  %vtable2 = load ptr, ptr %obj2
+  %p2 = call i1 @llvm.type.test(ptr %vtable2, metadata !11)
+  call void @llvm.assume(i1 %p2)
+
+  %fptr33_rv = call ptr @llvm.load.relative.i32(ptr %vtable2, i32 0)
+
+  ; Check that the call was devirtualized.
+  ; CHECK-IR: %call4 = tail call i32 @_ZN1D1mEi
+  %call4 = tail call i32 %fptr33_rv(ptr nonnull %obj2, i32 %call3)
+  ret i32 %call4
+}
+; CHECK-IR-LABEL: ret i32
+; CHECK-IR-LABEL: }
+
 declare i1 @llvm.type.test(ptr, metadata)
 declare void @llvm.assume(i1)
 
@@ -165,3 +248,9 @@ attributes #0 = { noinline optnone }
 !4 = distinct !{}
 !5 = !{!"VP", i32 0, i64 1, i64 1621563287929432257, i64 1}
 !6 = !{ptr @_ZN1A1nEi}
+
+!7 = !{i64 8, !"_ZTS1A_RV"}
+!8 = !{i64 8, !"_ZTS1B_RV"}
+!9 = !{i64 8, !"_ZTS1C_RV"}
+!10 = !{i64 8, !11}
+!11 = distinct !{}
diff --git a/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers-bad.ll b/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers-bad.ll
index 9d87fdb084a1..9a4bff80383c 100644
--- a/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers-bad.ll
+++ b/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers-bad.ll
@@ -16,14 +16,32 @@ declare { ptr, i1 } @llvm.type.checked.load(ptr, i32, metadata)
 
 ; CHECK:      @vtable = internal unnamed_addr constant { [3 x i32] } zeroinitializer, align 8, !type !0, !type !1, !vcall_visibility !2
 
+@vtable2 = internal unnamed_addr constant { [3 x i32] } { [3 x i32] [
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vfunc3 to i64), i64 ptrtoint (ptr @vtable2 to i64)) to i32),
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vfunc4 to i64), i64 ptrtoint (ptr @vtable2 to i64)) to i32),
+
+  ; a "bad" relative pointer because it's base is not the @vtable symbol
+  i32 trunc (i64 sub (i64 ptrtoint (ptr @weird_ref_3         to i64), i64 ptrtoint (ptr @weird_ref_4 to i64)) to i32)
+]}, align 4, !type !3, !type !4, !vcall_visibility !{i64 2}
+!3 = !{i64 0, !"vfunc3.type"}
+!4 = !{i64 4, !"vfunc4.type"}
+
+; CHECK:      @vtable2 = internal unnamed_addr constant { [3 x i32] } zeroinitializer, align 4, !type !3, !type !4, !vcall_visibility !2
+
 define internal void @vfunc1() { ret void }
 define internal void @vfunc2() { ret void }
 define internal void @weird_ref_1() { ret void }
 define internal void @weird_ref_2() { ret void }
+declare void @vfunc3()
+declare void @vfunc4()
+declare void @weird_ref_3()
+declare void @weird_ref_4()
 
 define void @main() {
   %1 = ptrtoint ptr @vtable to i64 ; to keep @vtable alive
   call void @weird_ref_2()
+  %2 = ptrtoint ptr @vtable2 to i64 ; to keep @vtable2 alive
+  call void @weird_ref_4()
   ret void
 }
 
diff --git a/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers-gep.ll b/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers-gep.ll
index fb45d37dde6e..d89b024e7a6d 100644
--- a/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers-gep.ll
+++ b/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers-gep.ll
@@ -19,6 +19,20 @@ declare { ptr, i1 } @llvm.type.checked.load(ptr, i32, metadata)
 ; CHECK-SAME:   i32 0
 ; CHECK-SAME: ] }, align 8, !type !0, !type !1, !vcall_visibility !2
 
+@vtable2 = internal unnamed_addr constant { [4 x i32] } { [4 x i32] [
+  i32 42,
+  i32 1337,
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vfunc3_live_extern to i64), i64 ptrtoint (ptr getelementptr inbounds ({ [4 x i32] }, ptr @vtable2, i32 0, i32 0, i32 2) to i64)) to i32),
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vfunc4_dead_extern to i64), i64 ptrtoint (ptr getelementptr inbounds ({ [4 x i32] }, ptr @vtable2, i32 0, i32 0, i32 2) to i64)) to i32)
+]}, align 4, !type !3, !type !4, !vcall_visibility !{i64 2}
+!3 = !{i64 8, !"vfunc3.type"}
+!4 = !{i64 12, !"vfunc4.type"}
+
+; CHECK:      @vtable2 = internal unnamed_addr constant { [4 x i32] } { [4 x i32] [
+; CHECK-SAME:   i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vfunc3_live_extern to i64), i64 ptrtoint (ptr getelementptr inbounds ({ [4 x i32] }, ptr @vtable2, i32 0, i32 0, i32 2) to i64)) to i32),
+; CHECK-SAME:   i32 0
+; CHECK-SAME: ] }, align 4, !type !3, !type !4, !vcall_visibility !2
+
 ; (1) vfunc1_live is referenced from @main, stays alive
 define internal void @vfunc1_live() {
   ; CHECK: define internal void @vfunc1_live(
@@ -31,9 +45,19 @@ define internal void @vfunc2_dead() {
   ret void
 }
 
+; (3) vfunc3_live_extern is referenced from @main, stays alive
+; CHECK: declare void @vfunc3_live_extern
+declare void @vfunc3_live_extern()
+
+; (4) vfunc4_dead_extern is never referenced, gets removed and vtable slot is null'd
+; CHECK-NOT: declare void @vfunc4_dead_extern
+declare void @vfunc4_dead_extern()
+
 define void @main() {
   %1 = ptrtoint ptr @vtable to i64 ; to keep @vtable alive
   %2 = tail call { ptr, i1 } @llvm.type.checked.load(ptr null, i32 0, metadata !"vfunc1.type")
+  %3 = ptrtoint ptr @vtable2 to i64 ; to keep @vtable2 alive
+  %4 = tail call { ptr, i1 } @llvm.type.checked.load(ptr null, i32 0, metadata !"vfunc3.type")
   ret void
 }
 
diff --git a/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers.ll b/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers.ll
index 76a617ea99e6..0b70148aa0ee 100644
--- a/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers.ll
+++ b/llvm/test/Transforms/GlobalDCE/virtual-functions-relative-pointers.ll
@@ -6,17 +6,31 @@ declare { ptr, i1 } @llvm.type.checked.load(ptr, i32, metadata)
 
 ; A vtable with "relative pointers", slots don't contain pointers to implementations, but instead have an i32 offset from the vtable itself to the implementation.
 @vtable = internal unnamed_addr constant { [2 x i32] } { [2 x i32] [
-  i32 trunc (i64 sub (i64 ptrtoint (ptr @vfunc1_live              to i64), i64 ptrtoint (ptr @vtable to i64)) to i32),
-  i32 trunc (i64 sub (i64 ptrtoint (ptr @vfunc2_dead              to i64), i64 ptrtoint (ptr @vtable to i64)) to i32)
+  i32 trunc (i64 sub (i64 ptrtoint (ptr @vfunc1_live to i64), i64 ptrtoint (ptr @vtable to i64)) to i32),
+  i32 trunc (i64 sub (i64 ptrtoint (ptr @vfunc2_dead to i64), i64 ptrtoint (ptr @vtable to i64)) to i32)
 ]}, align 8, !type !0, !type !1, !vcall_visibility !{i64 2}
 !0 = !{i64 0, !"vfunc1.type"}
 !1 = !{i64 4, !"vfunc2.type"}
 
 ; CHECK:      @vtable = internal unnamed_addr constant { [2 x i32] } { [2 x i32] [
-; CHECK-SAME:   i32 trunc (i64 sub (i64 ptrtoint (ptr @vfunc1_live              to i64), i64 ptrtoint (ptr @vtable to i64)) to i32),
+; CHECK-SAME:   i32 trunc (i64 sub (i64 ptrtoint (ptr @vfunc1_live to i64), i64 ptrtoint (ptr @vtable to i64)) to i32),
 ; CHECK-SAME:   i32 0
 ; CHECK-SAME: ] }, align 8, !type !0, !type !1, !vcall_visibility !2
 
+; Similar to above, but the vtable is more aligned to how C++ relative vtables look.
+; That is, the functions may not be dso-local.
+@vtable2 = internal unnamed_addr constant { [2 x i32] } { [2 x i32] [
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vfunc3_live_extern to i64), i64 ptrtoint (ptr @vtable2 to i64)) to i32),
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vfunc4_dead_extern to i64), i64 ptrtoint (ptr @vtable2 to i64)) to i32)
+]}, align 4, !type !3, !type !4, !vcall_visibility !{i64 2}
+!3 = !{i64 0, !"vfunc3.type"}
+!4 = !{i64 4, !"vfunc4.type"}
+
+; CHECK:      @vtable2 = internal unnamed_addr constant { [2 x i32] } { [2 x i32] [
+; CHECK-SAME:   i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vfunc3_live_extern to i64), i64 ptrtoint (ptr @vtable2 to i64)) to i32),
+; CHECK-SAME:   i32 0
+; CHECK-SAME: ] }, align 4, !type !3, !type !4, !vcall_visibility !2
+
 ; (1) vfunc1_live is referenced from @main, stays alive
 define internal void @vfunc1_live() {
   ; CHECK: define internal void @vfunc1_live(
@@ -29,9 +43,19 @@ define internal void @vfunc2_dead() {
   ret void
 }
 
+; (3) vfunc3_live_extern is referenced from @main, stays alive
+; CHECK: declare void @vfunc3_live_extern
+declare void @vfunc3_live_extern()
+
+; (4) vfunc4_dead_extern is never referenced, gets removed and vtable slot is null'd
+; CHECK-NOT: declare void @vfunc4_dead_extern
+declare void @vfunc4_dead_extern()
+
 define void @main() {
   %1 = ptrtoint ptr @vtable to i64 ; to keep @vtable alive
   %2 = tail call { ptr, i1 } @llvm.type.checked.load(ptr null, i32 0, metadata !"vfunc1.type")
+  %3 = ptrtoint ptr @vtable2 to i64 ; to keep @vtable2 alive
+  %4 = tail call { ptr, i1 } @llvm.type.checked.load(ptr null, i32 0, metadata !"vfunc3.type")
   ret void
 }
 
diff --git a/llvm/test/Transforms/WholeProgramDevirt/Inputs/export.yaml b/llvm/test/Transforms/WholeProgramDevirt/Inputs/export.yaml
index 71cf38b216c7..dd0c90d7ae8c 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/Inputs/export.yaml
+++ b/llvm/test/Transforms/WholeProgramDevirt/Inputs/export.yaml
@@ -5,14 +5,22 @@ GlobalValueMap:
       TypeTestAssumeVCalls:
         - GUID: 14276520915468743435  # typeid1
           Offset: 0
+        - GUID: 271751036925422857  # typeid1_rv
+          Offset: 0
       TypeCheckedLoadVCalls:
         - GUID: 15427464259790519041  # typeid2
           Offset: 0
+        - GUID: 1146149264729288256  # typeid2_rv
+          Offset: 0
       TypeTestAssumeConstVCalls:
         - VFunc:
             GUID: 3515965990081467659  # typeid3
             Offset: 0
           Args: [12, 24]
+        - VFunc:
+            GUID: 2777626534618191571  # typeid3_rv
+            Offset: 0
+          Args: [12, 24]
       TypeCheckedLoadConstVCalls:
         - VFunc:
             GUID: 17525413373118030901  # typeid4
diff --git a/llvm/test/Transforms/WholeProgramDevirt/branch-funnel.ll b/llvm/test/Transforms/WholeProgramDevirt/branch-funnel.ll
index b55713fe4b29..0b1023eee273 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/branch-funnel.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/branch-funnel.ll
@@ -8,6 +8,45 @@
 ; RUN: FileCheck --check-prefix=SUMMARY %s < %t
 
 ; SUMMARY:      TypeIdMap:       
+; SUMMARY-NEXT:  typeid1_rv:
+; SUMMARY-NEXT:    TTRes:
+; SUMMARY-NEXT:      Kind:            Unknown
+; SUMMARY-NEXT:      SizeM1BitWidth:  0
+; SUMMARY-NEXT:      AlignLog2:       0
+; SUMMARY-NEXT:      SizeM1:          0
+; SUMMARY-NEXT:      BitMask:         0
+; SUMMARY-NEXT:      InlineBits:      0
+; SUMMARY-NEXT:    WPDRes:
+; SUMMARY-NEXT:      0:
+; SUMMARY-NEXT:        Kind:            BranchFunnel
+; SUMMARY-NEXT:        SingleImplName:  ''
+; SUMMARY-NEXT:        ResByArg:
+; SUMMARY-NEXT:  typeid2_rv:
+; SUMMARY-NEXT:    TTRes:
+; SUMMARY-NEXT:      Kind:            Unknown
+; SUMMARY-NEXT:      SizeM1BitWidth:  0
+; SUMMARY-NEXT:      AlignLog2:       0
+; SUMMARY-NEXT:      SizeM1:          0
+; SUMMARY-NEXT:      BitMask:         0
+; SUMMARY-NEXT:      InlineBits:      0
+; SUMMARY-NEXT:    WPDRes:
+; SUMMARY-NEXT:      0:
+; SUMMARY-NEXT:        Kind:            Indir
+; SUMMARY-NEXT:        SingleImplName:  ''
+; SUMMARY-NEXT:        ResByArg:
+; SUMMARY-NEXT:   typeid3_rv:
+; SUMMARY-NEXT:     TTRes:           
+; SUMMARY-NEXT:       Kind:            Unknown
+; SUMMARY-NEXT:       SizeM1BitWidth:  0
+; SUMMARY-NEXT:       AlignLog2:       0
+; SUMMARY-NEXT:       SizeM1:          0
+; SUMMARY-NEXT:       BitMask:         0
+; SUMMARY-NEXT:       InlineBits:      0
+; SUMMARY-NEXT:     WPDRes:          
+; SUMMARY-NEXT:       0:               
+; SUMMARY-NEXT:         Kind:            BranchFunnel
+; SUMMARY-NEXT:         SingleImplName:  ''
+; SUMMARY-NEXT:         ResByArg:        
 ; SUMMARY-NEXT:   typeid3:
 ; SUMMARY-NEXT:     TTRes:           
 ; SUMMARY-NEXT:       Kind:            Unknown
@@ -93,6 +132,29 @@ declare i32 @vf3_2(ptr %this, i32 %arg)
 declare i32 @vf4_1(ptr %this, i32 %arg)
 declare i32 @vf4_2(ptr %this, i32 %arg)
 
+declare ptr @llvm.load.relative.i32(ptr, i32)
+
+;; These are relative vtables equivalent to the ones above.
+@vt1_1_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf1_1 to i64), i64 ptrtoint (ptr @vt1_1_rv to i64)) to i32)], !type !5
+@vt1_2_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf1_2 to i64), i64 ptrtoint (ptr @vt1_2_rv to i64)) to i32)], !type !5
+
+@vt2_1_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_1 to i64), i64 ptrtoint (ptr @vt2_1_rv to i64)) to i32)], !type !6
+@vt2_2_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_2 to i64), i64 ptrtoint (ptr @vt2_2_rv to i64)) to i32)], !type !6
+@vt2_3_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_3 to i64), i64 ptrtoint (ptr @vt2_3_rv to i64)) to i32)], !type !6
+@vt2_4_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_4 to i64), i64 ptrtoint (ptr @vt2_4_rv to i64)) to i32)], !type !6
+@vt2_5_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_5 to i64), i64 ptrtoint (ptr @vt2_5_rv to i64)) to i32)], !type !6
+@vt2_6_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_6 to i64), i64 ptrtoint (ptr @vt2_6_rv to i64)) to i32)], !type !6
+@vt2_7_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_7 to i64), i64 ptrtoint (ptr @vt2_7_rv to i64)) to i32)], !type !6
+@vt2_8_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_8 to i64), i64 ptrtoint (ptr @vt2_8_rv to i64)) to i32)], !type !6
+@vt2_9_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_9 to i64), i64 ptrtoint (ptr @vt2_9_rv to i64)) to i32)], !type !6
+@vt2_10_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_10 to i64), i64 ptrtoint (ptr @vt2_10_rv to i64)) to i32)], !type !6
+@vt2_11_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2_11 to i64), i64 ptrtoint (ptr @vt2_11_rv to i64)) to i32)], !type !6
+
+@vt3_1_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf3_1 to i64), i64 ptrtoint (ptr @vt3_1_rv to i64)) to i32)], !type !7
+@vt3_2_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf3_2 to i64), i64 ptrtoint (ptr @vt3_2_rv to i64)) to i32)], !type !7
+
+@vt4_1_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf4_1 to i64), i64 ptrtoint (ptr @vt4_1_rv to i64)) to i32)], !type !8
+@vt4_2_rv = constant [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf4_2 to i64), i64 ptrtoint (ptr @vt4_2_rv to i64)) to i32)], !type !8
 
 
 ; CHECK-LABEL: define i32 @fn1
@@ -108,6 +170,19 @@ define i32 @fn1(ptr %obj) #0 {
   ret i32 %result
 }
 
+; CHECK-LABEL: define i32 @fn1_rv
+; CHECK-NOT: call void (...) @llvm.icall.branch.funnel
+define i32 @fn1_rv(ptr %obj) #0 {
+  %vtable = load ptr, ptr %obj
+  %p = call i1 @llvm.type.test(ptr %vtable, metadata !"typeid1_rv")
+  call void @llvm.assume(i1 %p)
+  %fptr = call ptr @llvm.load.relative.i32(ptr %vtable, i32 0)
+  ; RETP: call i32 @__typeid_typeid1_rv_0_branch_funnel(ptr nest %vtable, ptr %obj, i32 1)
+  %result = call i32 %fptr(ptr %obj, i32 1)
+  ; NORETP: call i32 %
+  ret i32 %result
+}
+
 ; CHECK-LABEL: define i32 @fn2
 ; CHECK-NOT: call void (...) @llvm.icall.branch.funnel
 define i32 @fn2(ptr %obj) #0 {
@@ -120,6 +195,18 @@ define i32 @fn2(ptr %obj) #0 {
   ret i32 %result
 }
 
+; CHECK-LABEL: define i32 @fn2_rv
+; CHECK-NOT: call void (...) @llvm.icall.branch.funnel
+define i32 @fn2_rv(ptr %obj) #0 {
+  %vtable = load ptr, ptr %obj
+  %p = call i1 @llvm.type.test(ptr %vtable, metadata !"typeid2_rv")
+  call void @llvm.assume(i1 %p)
+  %fptr = call ptr @llvm.load.relative.i32(ptr %vtable, i32 0)
+  ; CHECK: call i32 %
+  %result = call i32 %fptr(ptr %obj, i32 1)
+  ret i32 %result
+}
+
 ; CHECK-LABEL: define i32 @fn3
 ; CHECK-NOT: call void (...) @llvm.icall.branch.funnel
 define i32 @fn3(ptr %obj) #0 {
@@ -133,10 +220,75 @@ define i32 @fn3(ptr %obj) #0 {
   ret i32 %result
 }
 
+; CHECK-LABEL: define i32 @fn3_rv
+; CHECK-NOT: call void (...) @llvm.icall.branch.funnel
+define i32 @fn3_rv(ptr %obj) #0 {
+  %vtable = load ptr, ptr %obj
+  %p = call i1 @llvm.type.test(ptr %vtable, metadata !9)
+  call void @llvm.assume(i1 %p)
+  %fptr = call ptr @llvm.load.relative.i32(ptr %vtable, i32 0)
+  ; RETP: call i32 @branch_funnel.1(ptr
+  ; NORETP: call i32 %
+  %result = call i32 %fptr(ptr %obj, i32 1)
+  ret i32 %result
+}
+
+; CHECK-LABEL: define i32 @fn4
+; CHECK-NOT: call void (...) @llvm.icall.branch.funnel
+define i32 @fn4(ptr %obj) #0 {
+  %p = call i1 @llvm.type.test(ptr @vt1_1, metadata !"typeid1")
+  call void @llvm.assume(i1 %p)
+  %fptr = load ptr, ptr @vt1_1
+  ; RETP: call i32 @__typeid_typeid1_0_branch_funnel(ptr nest @vt1_1, ptr %obj, i32 1)
+  %result = call i32 %fptr(ptr %obj, i32 1)
+  ; NORETP: call i32 %
+  ret i32 %result
+}
+
+; CHECK-LABEL: define i32 @fn4_cpy
+; CHECK-NOT: call void (...) @llvm.icall.branch.funnel
+define i32 @fn4_cpy(ptr %obj) #0 {
+  %p = call i1 @llvm.type.test(ptr @vt1_1, metadata !"typeid1")
+  call void @llvm.assume(i1 %p)
+  %fptr = load ptr, ptr @vt1_1
+  ; RETP: call i32 @__typeid_typeid1_0_branch_funnel(ptr nest @vt1_1, ptr %obj, i32 1)
+  %result = call i32 %fptr(ptr %obj, i32 1)
+  ; NORETP: call i32 %
+  ret i32 %result
+}
+
+; CHECK-LABEL: define i32 @fn4_rv
+; CHECK-NOT: call void (...) @llvm.icall.branch.funnel
+define i32 @fn4_rv(ptr %obj) #0 {
+  %p = call i1 @llvm.type.test(ptr @vt1_1_rv, metadata !"typeid1_rv")
+  call void @llvm.assume(i1 %p)
+  %fptr = call ptr @llvm.load.relative.i32(ptr @vt1_1_rv, i32 0)
+  ; RETP: call i32 @__typeid_typeid1_rv_0_branch_funnel(ptr nest @vt1_1_rv, ptr %obj, i32 1)
+  %result = call i32 %fptr(ptr %obj, i32 1)
+  ; NORETP: call i32 %
+  ret i32 %result
+}
+
+; CHECK-LABEL: define i32 @fn4_rv_cpy
+; CHECK-NOT: call void (...) @llvm.icall.branch.funnel
+define i32 @fn4_rv_cpy(ptr %obj) #0 {
+  %p = call i1 @llvm.type.test(ptr @vt1_1_rv, metadata !"typeid1_rv")
+  call void @llvm.assume(i1 %p)
+  %fptr = call ptr @llvm.load.relative.i32(ptr @vt1_1_rv, i32 0)
+  ; RETP: call i32 @__typeid_typeid1_rv_0_branch_funnel(ptr nest @vt1_1_rv, ptr %obj, i32 1)
+  %result = call i32 %fptr(ptr %obj, i32 1)
+  ; NORETP: call i32 %
+  ret i32 %result
+}
+
 ; CHECK-LABEL: define hidden void @__typeid_typeid1_0_branch_funnel(ptr nest %0, ...)
 ; CHECK-NEXT: musttail call void (...) @llvm.icall.branch.funnel(ptr %0, ptr {{(nonnull )?}}@vt1_1, ptr {{(nonnull )?}}@vf1_1, ptr {{(nonnull )?}}@vt1_2, ptr {{(nonnull )?}}@vf1_2, ...)
 
+; CHECK-LABEL: define hidden void @__typeid_typeid1_rv_0_branch_funnel(ptr nest %0, ...)
+; CHECK-NEXT: musttail call void (...) @llvm.icall.branch.funnel(ptr %0, ptr {{(nonnull )?}}@vt1_1_rv, ptr {{(nonnull )?}}@vf1_1, ptr {{(nonnull )?}}@vt1_2_rv, ptr {{(nonnull )?}}@vf1_2, ...)
+
 ; CHECK: define internal void @branch_funnel(ptr
+; CHECK: define internal void @branch_funnel.1(ptr
 
 declare i1 @llvm.type.test(ptr, metadata)
 declare void @llvm.assume(i1)
@@ -146,5 +298,10 @@ declare void @llvm.assume(i1)
 !2 = !{i32 0, !"typeid3"}
 !3 = !{i32 0, !4}
 !4 = distinct !{}
+!5 = !{i32 0, !"typeid1_rv"}
+!6 = !{i32 0, !"typeid2_rv"}
+!7 = !{i32 0, !"typeid3_rv"}
+!8 = !{i32 0, !9}
+!9 = distinct !{}
 
 attributes #0 = { "target-features"="+retpoline" }
diff --git a/llvm/test/Transforms/WholeProgramDevirt/constant-arg.ll b/llvm/test/Transforms/WholeProgramDevirt/constant-arg.ll
index 91bae4af9a21..e685a1a54c52 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/constant-arg.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/constant-arg.ll
@@ -7,6 +7,10 @@ target triple = "x86_64-unknown-linux-gnu"
 ; CHECK: private constant { [8 x i8], [1 x ptr], [0 x i8] } { [8 x i8] c"\00\00\00\00\00\00\00\02", [1 x ptr] [ptr @vf2], [0 x i8] zeroinitializer }, !type [[T8]]
 ; CHECK: private constant { [8 x i8], [1 x ptr], [0 x i8] } { [8 x i8] c"\00\00\00\00\00\00\00\01", [1 x ptr] [ptr @vf4], [0 x i8] zeroinitializer }, !type [[T8]]
 ; CHECK: private constant { [8 x i8], [1 x ptr], [0 x i8] } { [8 x i8] c"\00\00\00\00\00\00\00\02", [1 x ptr] [ptr @vf8], [0 x i8] zeroinitializer }, !type [[T8]]
+; CHECK: private constant { [4 x i8], [1 x i32], [0 x i8] } { [4 x i8] c"\00\00\00\01", [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf1 to i64), i64 ptrtoint (ptr @vt1_rv to i64)) to i32)], [0 x i8] zeroinitializer }, align 4, !type [[T4:![0-9]+]]
+; CHECK: private constant { [4 x i8], [1 x i32], [0 x i8] } { [4 x i8] c"\00\00\00\02", [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2 to i64), i64 ptrtoint (ptr @vt2_rv to i64)) to i32)], [0 x i8] zeroinitializer }, align 4, !type [[T4]]
+; CHECK: private constant { [4 x i8], [1 x i32], [0 x i8] } { [4 x i8] c"\00\00\00\01", [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf4 to i64), i64 ptrtoint (ptr @vt4_rv to i64)) to i32)], [0 x i8] zeroinitializer }, align 4, !type [[T4]]
+; CHECK: private constant { [4 x i8], [1 x i32], [0 x i8] } { [4 x i8] c"\00\00\00\02", [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf8 to i64), i64 ptrtoint (ptr @vt8_rv to i64)) to i32)], [0 x i8] zeroinitializer }, align 4, !type [[T4]]
 
 @vt1 = constant [1 x ptr] [ptr @vf1], !type !0
 @vt2 = constant [1 x ptr] [ptr @vf2], !type !0
@@ -61,8 +65,49 @@ define i1 @call2(ptr %obj) {
   ret i1 %result
 }
 
+declare ptr @llvm.load.relative.i32(ptr, i32)
+
+@vt1_rv = private unnamed_addr constant [1 x i32] [
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf1 to i64), i64 ptrtoint (ptr @vt1_rv to i64)) to i32)
+], align 4, !type !1
+@vt2_rv = private unnamed_addr constant [1 x i32] [
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf2 to i64), i64 ptrtoint (ptr @vt2_rv to i64)) to i32)
+], align 4, !type !1
+@vt4_rv = private unnamed_addr constant [1 x i32] [
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf4 to i64), i64 ptrtoint (ptr @vt4_rv to i64)) to i32)
+], align 4, !type !1
+@vt8_rv = private unnamed_addr constant [1 x i32] [
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf8 to i64), i64 ptrtoint (ptr @vt8_rv to i64)) to i32)
+], align 4, !type !1
+
+; CHECK: define i1 @call3
+define i1 @call3(ptr %obj) {
+  %vtable = load ptr, ptr %obj
+  %p = call i1 @llvm.type.test(ptr %vtable, metadata !"typeid2")
+  call void @llvm.assume(i1 %p)
+  %fptr = call ptr @llvm.load.relative.i32(ptr %vtable, i32 0)
+  ; CHECK: getelementptr {{.*}} -1
+  ; CHECK: and {{.*}}, 1
+  %result = call i1 %fptr(ptr %obj, i32 5)
+  ret i1 %result
+}
+
+; CHECK: define i1 @call4
+define i1 @call4(ptr %obj) {
+  %vtable = load ptr, ptr %obj
+  %p = call i1 @llvm.type.test(ptr %vtable, metadata !"typeid2")
+  call void @llvm.assume(i1 %p)
+  %fptr = call ptr @llvm.load.relative.i32(ptr %vtable, i32 0)
+  ; CHECK: getelementptr {{.*}} -1
+  ; CHECK: and {{.*}}, 2
+  %result = call i1 %fptr(ptr %obj, i32 10)
+  ret i1 %result
+}
+
 declare i1 @llvm.type.test(ptr, metadata)
 declare void @llvm.assume(i1)
 
 ; CHECK: [[T8]] = !{i32 8, !"typeid"}
+; CHECK: [[T4]] = !{i32 4, !"typeid2"}
 !0 = !{i32 0, !"typeid"}
+!1 = !{i32 0, !"typeid2"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check.ll b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check.ll
index dc7b2024bc59..5a9e6c12c358 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check.ll
@@ -3,6 +3,7 @@
 target datalayout = "e-p:64:64"
 target triple = "x86_64-unknown-linux-gnu"
 
+; CHECK: remark: <unknown>:0:0: single-impl: devirtualized a call to vf
 ; CHECK: remark: <unknown>:0:0: single-impl: devirtualized a call to vf
 ; CHECK: remark: <unknown>:0:0: devirtualized vf
 ; CHECK-NOT: devirtualized
@@ -33,7 +34,31 @@ trap:
   unreachable
 }
 
+@vt3 = private unnamed_addr constant [1 x i32] [
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf to i64), i64 ptrtoint (ptr @vt3 to i64)) to i32)
+], align 4, !type !1
+
+; CHECK: define void @call2
+define void @call2(ptr %obj) {
+  %vtable = load ptr, ptr %obj
+  %pair = call {ptr, i1} @llvm.type.checked.load(ptr %vtable, i32 0, metadata !"typeid2")
+  %fptr = extractvalue {ptr, i1} %pair, 0
+  %p = extractvalue {ptr, i1} %pair, 1
+  ; CHECK: br i1 true,
+  br i1 %p, label %cont, label %trap
+
+cont:
+  ; CHECK: call void @vf(
+  call void %fptr(ptr %obj)
+  ret void
+
+trap:
+  call void @llvm.trap()
+  unreachable
+}
+
 declare {ptr, i1} @llvm.type.checked.load(ptr, i32, metadata)
 declare void @llvm.trap()
 
 !0 = !{i32 0, !"typeid"}
+!1 = !{i32 0, !"typeid2"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-multiple-assumes.ll b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-multiple-assumes.ll
index ed144c23302d..bfbbeaf8a065 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-multiple-assumes.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-multiple-assumes.ll
@@ -23,7 +23,27 @@ define void @call(ptr %obj) {
   ret void
 }
 
+declare ptr @llvm.load.relative.i32(ptr, i32)
+
+@vt3 = private unnamed_addr constant [1 x i32] [
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf to i64), i64 ptrtoint (ptr @vt3 to i64)) to i32)
+], align 4, !type !1
+
+; CHECK: define void @call2
+define void @call2(ptr %obj) {
+  %vtable = load ptr, ptr %obj
+  %p = call i1 @llvm.type.test(ptr %vtable, metadata !"typeid2")
+  call void @llvm.assume(i1 %p)
+  %p2 = call i1 @llvm.type.test(ptr %vtable, metadata !"typeid2")
+  call void @llvm.assume(i1 %p2)
+  %fptr = call ptr @llvm.load.relative.i32(ptr %vtable, i32 0)
+  ; CHECK: call void @vf(
+  call void %fptr(ptr %obj)
+  ret void
+}
+
 declare i1 @llvm.type.test(ptr, metadata)
 declare void @llvm.assume(i1)
 
 !0 = !{i32 0, !"typeid"}
+!1 = !{i32 0, !"typeid2"}
diff --git a/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl.ll b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl.ll
index 1ce96f70d905..d7f33df59495 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl.ll
@@ -7,6 +7,8 @@ target datalayout = "e-p:64:64"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; CHECK: remark: devirt-single.cc:30:32: single-impl: devirtualized a call to vf
+; CHECK: remark: devirt-single.cc:41:32: single-impl: devirtualized a call to vf
+; CHECK: remark: devirt-single.cc:51:32: single-impl: devirtualized a call to vf
 ; CHECK: remark: devirt-single.cc:13:0: devirtualized vf
 ; CHECK-NOT: devirtualized
 
@@ -28,6 +30,41 @@ define void @call(ptr %obj) #1 !dbg !5 {
   ret void
 }
 
+declare ptr @llvm.load.relative.i32(ptr, i32)
+
+@vt3 = private unnamed_addr constant [1 x i32] [
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf to i64), i64 ptrtoint (ptr @vt3 to i64)) to i32)
+], align 4, !type !11
+
+; CHECK: define void @call2
+define void @call2(ptr %obj) #1 !dbg !9 {
+  %vtable = load ptr, ptr %obj
+  %p = call i1 @llvm.type.test(ptr %vtable, metadata !"typeid2")
+  call void @llvm.assume(i1 %p)
+  %fptr = call ptr @llvm.load.relative.i32(ptr %vtable, i32 0)
+  ; CHECK: call void @vf(
+  call void %fptr(ptr %obj), !dbg !10
+  ret void
+}
+
+@_ZTV1A.local = private unnamed_addr constant { [3 x i32] } { [3 x i32] [
+  i32 0,  ; offset to top
+  i32 0,  ; rtti
+  i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf to i64), i64 ptrtoint (ptr getelementptr inbounds ({ [3 x i32] }, ptr @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32)  ; vfunc offset
+] }, align 4, !type !14
+
+; CHECK: define void @call3
+define void @call3(ptr %obj) #1 !dbg !12 {
+  %vtable = load ptr, ptr %obj
+  %p = call i1 @llvm.type.test(ptr %vtable, metadata !"typeid3")
+  call void @llvm.assume(i1 %p)
+  %fptr = call ptr @llvm.load.relative.i32(ptr %vtable, i32 8)
+  ; CHECK: call void @vf(
+  call void %fptr(ptr %obj), !dbg !13
+  ret void
+}
+
+
 declare i1 @llvm.type.test(ptr, metadata)
 declare void @llvm.assume(i1)
 
@@ -45,5 +82,13 @@ declare void @llvm.assume(i1)
 !7 = distinct !DISubprogram(name: "vf", linkageName: "_ZN3vt12vfEv", scope: !1, file: !1, line: 13, isLocal: false, isDefinition: true, scopeLine: 13, flags: DIFlagPrototyped, isOptimized: false, unit: !0)
 !8 = !{i32 0, !"typeid"}
 
+!9 = distinct !DISubprogram(name: "call2", linkageName: "_Z5call2Pv", scope: !1, file: !1, line: 40, isLocal: false, isDefinition: true, scopeLine: 9, flags: DIFlagPrototyped, isOptimized: false, unit: !0)
+!10 = !DILocation(line: 41, column: 32, scope: !9)
+!11 = !{i32 0, !"typeid2"}
+
+!12 = distinct !DISubprogram(name: "call3", linkageName: "_Z5call3Pv", scope: !1, file: !1, line: 50, isLocal: false, isDefinition: true, scopeLine: 9, flags: DIFlagPrototyped, isOptimized: false, unit: !0)
+!13 = !DILocation(line: 51, column: 32, scope: !12)
+!14 = !{i32 0, !"typeid3"}
+
 ; CHECK: 1 wholeprogramdevirt - Number of whole program devirtualization targets
-; CHECK: 1 wholeprogramdevirt - Number of single implementation devirtualizations
+; CHECK: 3 wholeprogramdevirt - Number of single implementation devirtualizations
diff --git a/llvm/test/Transforms/WholeProgramDevirt/pointer-vtable.ll b/llvm/test/Transforms/WholeProgramDevirt/pointer-vtable.ll
index 3c5d9b839bc4..062cef9b88fe 100644
--- a/llvm/test/Transforms/WholeProgramDevirt/pointer-vtable.ll
+++ b/llvm/test/Transforms/WholeProgramDevirt/pointer-vtable.ll
@@ -20,7 +20,23 @@ define void @call(ptr %obj) {
   ret void
 }
 
+@vt2 = constant i32 trunc (i64 sub (i64 ptrtoint (ptr dso_local_equivalent @vf to i64), i64 ptrtoint (ptr @vt2 to i64)) to i32), !type !1
+
+declare ptr @llvm.load.relative.i32(ptr, i32)
+
+; CHECK: define void @call2
+define void @call2(ptr %obj) {
+  %vtable = load ptr, ptr %obj
+  %p = call i1 @llvm.type.test(ptr %vtable, metadata !"typeid2")
+  call void @llvm.assume(i1 %p)
+  %fptr = call ptr @llvm.load.relative.i32(ptr %vtable, i32 0)
+  ; CHECK: call void @vf(
+  call void %fptr(ptr %obj)
+  ret void
+}
+
 declare i1 @llvm.type.test(ptr, metadata)
 declare void @llvm.assume(i1)
 
 !0 = !{i32 0, !"typeid"}
+!1 = !{i32 0, !"typeid2"}
-- 
GitLab


From 75bbf4dd7cfda744d4a2487cfd06a58194d02db3 Mon Sep 17 00:00:00 2001
From: Job Henandez Lara <hj93@protonmail.com>
Date: Mon, 15 Apr 2024 11:45:16 -0700
Subject: [PATCH 285/464] [libc] Add proxy headers for fenv types. (#88467)

Fixes #88187
---
 libc/hdr/types/CMakeLists.txt                 | 18 +++++++++++++++
 libc/hdr/types/fenv_t.h                       | 22 +++++++++++++++++++
 libc/hdr/types/fexcept_t.h                    | 22 +++++++++++++++++++
 libc/src/__support/FPUtil/CMakeLists.txt      |  2 +-
 libc/src/__support/FPUtil/aarch64/FEnvImpl.h  |  2 +-
 .../FPUtil/aarch64/fenv_darwin_impl.h         |  2 +-
 libc/src/__support/FPUtil/arm/FEnvImpl.h      |  2 +-
 libc/src/__support/FPUtil/riscv/FEnvImpl.h    |  1 +
 libc/src/__support/FPUtil/x86_64/FEnvImpl.h   |  2 +-
 libc/src/fenv/CMakeLists.txt                  |  7 +++++-
 libc/src/fenv/fegetenv.h                      |  2 +-
 libc/src/fenv/fegetexceptflag.cpp             |  3 +--
 libc/src/fenv/feholdexcept.cpp                |  2 +-
 libc/src/fenv/feholdexcept.h                  |  2 +-
 libc/src/fenv/fesetenv.h                      |  2 +-
 libc/src/fenv/fesetexceptflag.cpp             |  2 +-
 libc/src/fenv/feupdateenv.h                   |  2 +-
 libc/test/UnitTest/CMakeLists.txt             |  1 +
 libc/test/UnitTest/FPExceptMatcher.cpp        |  7 +++---
 libc/test/src/fenv/exception_flags_test.cpp   |  3 +--
 libc/test/src/fenv/feholdexcept_test.cpp      |  3 +--
 libc/test/src/fenv/feupdateenv_test.cpp       |  2 +-
 libc/test/src/fenv/getenv_and_setenv_test.cpp |  3 +--
 .../llvm-project-overlay/libc/BUILD.bazel     | 13 +++++++++++
 .../libc/test/UnitTest/BUILD.bazel            |  1 +
 .../libc/test/src/fenv/BUILD.bazel            |  4 ++++
 26 files changed, 108 insertions(+), 24 deletions(-)
 create mode 100644 libc/hdr/types/fenv_t.h
 create mode 100644 libc/hdr/types/fexcept_t.h

diff --git a/libc/hdr/types/CMakeLists.txt b/libc/hdr/types/CMakeLists.txt
index b685d82fd8cc..ecb952b60cc0 100644
--- a/libc/hdr/types/CMakeLists.txt
+++ b/libc/hdr/types/CMakeLists.txt
@@ -21,3 +21,21 @@ add_proxy_header_library(
   FULL_BUILD_DEPENDS
     libc.include.llvm-libc-types.struct_timespec
 )
+
+add_proxy_header_library(
+  fenv_t
+  HDRS
+    fenv_t.h
+  FULL_BUILD_DEPENDS
+    libc.include.llvm-libc-types.fenv_t
+    libc.incude.fenv
+)
+
+add_proxy_header_library(
+  fexcept_t
+  HDRS
+    fexcept_t.h
+  FULL_BUILD_DEPENDS
+    libc.include.llvm-libc-types.fexcept_t
+    libc.incude.fenv
+)
diff --git a/libc/hdr/types/fenv_t.h b/libc/hdr/types/fenv_t.h
new file mode 100644
index 000000000000..dc2dd0217a16
--- /dev/null
+++ b/libc/hdr/types/fenv_t.h
@@ -0,0 +1,22 @@
+//===-- Definition of macros from fenv_t.h --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_FENV_T_H
+#define LLVM_LIBC_HDR_FENV_T_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-types/fenv_t.h"
+
+#else // Overlay mode
+
+#include <fenv.h>
+
+#endif // LLVM_LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_FENV_T_H
diff --git a/libc/hdr/types/fexcept_t.h b/libc/hdr/types/fexcept_t.h
new file mode 100644
index 000000000000..7a6702516f0a
--- /dev/null
+++ b/libc/hdr/types/fexcept_t.h
@@ -0,0 +1,22 @@
+//===-- Definition of macros from fexcept_t.h -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_FEXCEPT_T_H
+#define LLVM_LIBC_HDR_FEXCEPT_T_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-types/fexcept_t.h"
+
+#else // Overlay mode
+
+#include <fenv.h>
+
+#endif // LLVM_LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_FENV_T_H
diff --git a/libc/src/__support/FPUtil/CMakeLists.txt b/libc/src/__support/FPUtil/CMakeLists.txt
index 0b5ea8368943..01ca4254c799 100644
--- a/libc/src/__support/FPUtil/CMakeLists.txt
+++ b/libc/src/__support/FPUtil/CMakeLists.txt
@@ -3,7 +3,7 @@ add_header_library(
   HDRS
     FEnvImpl.h
   DEPENDS
-    libc.include.fenv
+    libc.hdr.types.fenv_t
     libc.hdr.fenv_macros
     libc.hdr.math_macros
     libc.src.__support.macros.attributes
diff --git a/libc/src/__support/FPUtil/aarch64/FEnvImpl.h b/libc/src/__support/FPUtil/aarch64/FEnvImpl.h
index 4b593cdd8cc4..d1d92169475d 100644
--- a/libc/src/__support/FPUtil/aarch64/FEnvImpl.h
+++ b/libc/src/__support/FPUtil/aarch64/FEnvImpl.h
@@ -17,10 +17,10 @@
 #endif
 
 #include <arm_acle.h>
-#include <fenv.h>
 #include <stdint.h>
 
 #include "hdr/fenv_macros.h"
+#include "hdr/types/fenv_t.h"
 #include "src/__support/FPUtil/FPBits.h"
 
 namespace LIBC_NAMESPACE {
diff --git a/libc/src/__support/FPUtil/aarch64/fenv_darwin_impl.h b/libc/src/__support/FPUtil/aarch64/fenv_darwin_impl.h
index 773d6bfe9f89..5b59ba38d67b 100644
--- a/libc/src/__support/FPUtil/aarch64/fenv_darwin_impl.h
+++ b/libc/src/__support/FPUtil/aarch64/fenv_darwin_impl.h
@@ -17,10 +17,10 @@
 #endif
 
 #include <arm_acle.h>
-#include <fenv.h>
 #include <stdint.h>
 
 #include "hdr/fenv_macros.h"
+#include "hdr/types/fenv_t.h"
 #include "src/__support/FPUtil/FPBits.h"
 
 namespace LIBC_NAMESPACE {
diff --git a/libc/src/__support/FPUtil/arm/FEnvImpl.h b/libc/src/__support/FPUtil/arm/FEnvImpl.h
index ddb0edcf8278..78fbda4f7aff 100644
--- a/libc/src/__support/FPUtil/arm/FEnvImpl.h
+++ b/libc/src/__support/FPUtil/arm/FEnvImpl.h
@@ -10,9 +10,9 @@
 #define LLVM_LIBC_SRC___SUPPORT_FPUTIL_ARM_FENVIMPL_H
 
 #include "hdr/fenv_macros.h"
+#include "hdr/types/fenv_t.h"
 #include "src/__support/FPUtil/FPBits.h"
 #include "src/__support/macros/attributes.h" // For LIBC_INLINE
-#include <fenv.h>
 #include <stdint.h>
 
 namespace LIBC_NAMESPACE {
diff --git a/libc/src/__support/FPUtil/riscv/FEnvImpl.h b/libc/src/__support/FPUtil/riscv/FEnvImpl.h
index a5224330f339..6e940453f7a9 100644
--- a/libc/src/__support/FPUtil/riscv/FEnvImpl.h
+++ b/libc/src/__support/FPUtil/riscv/FEnvImpl.h
@@ -10,6 +10,7 @@
 #define LLVM_LIBC_SRC___SUPPORT_FPUTIL_RISCV_FENVIMPL_H
 
 #include "hdr/fenv_macros.h"
+#include "hdr/types/fenv_t.h"
 #include "src/__support/FPUtil/FPBits.h"
 #include "src/__support/macros/attributes.h" // For LIBC_INLINE_ASM
 #include "src/__support/macros/config.h"     // For LIBC_INLINE
diff --git a/libc/src/__support/FPUtil/x86_64/FEnvImpl.h b/libc/src/__support/FPUtil/x86_64/FEnvImpl.h
index b73b70e90779..0595658d7df3 100644
--- a/libc/src/__support/FPUtil/x86_64/FEnvImpl.h
+++ b/libc/src/__support/FPUtil/x86_64/FEnvImpl.h
@@ -16,9 +16,9 @@
 #error "Invalid include"
 #endif
 
-#include <fenv.h>
 #include <stdint.h>
 
+#include "hdr/types/fenv_t.h"
 #include "src/__support/macros/sanitizer.h"
 
 namespace LIBC_NAMESPACE {
diff --git a/libc/src/fenv/CMakeLists.txt b/libc/src/fenv/CMakeLists.txt
index 5dcf21de04f1..a28a7ca4c2d8 100644
--- a/libc/src/fenv/CMakeLists.txt
+++ b/libc/src/fenv/CMakeLists.txt
@@ -5,7 +5,6 @@ add_entrypoint_object(
   HDRS
     fegetround.h
   DEPENDS
-    libc.include.fenv
     libc.src.__support.FPUtil.fenv_impl
   COMPILE_OPTIONS
     -O2
@@ -71,6 +70,7 @@ add_entrypoint_object(
     fegetenv.h
   DEPENDS
     libc.hdr.fenv_macros
+    libc.hdr.types.fenv_t
     libc.src.__support.FPUtil.fenv_impl
   COMPILE_OPTIONS
     -O2
@@ -84,6 +84,7 @@ add_entrypoint_object(
     fesetenv.h
   DEPENDS
     libc.hdr.fenv_macros
+    libc.hdr.types.fenv_t
     libc.src.__support.FPUtil.fenv_impl
   COMPILE_OPTIONS
     -O2
@@ -97,6 +98,7 @@ add_entrypoint_object(
     fegetexceptflag.h
   DEPENDS
     libc.hdr.fenv_macros
+    libc.hdr.types.fexcept_t
     libc.src.__support.FPUtil.fenv_impl
   COMPILE_OPTIONS
     -O2
@@ -123,6 +125,7 @@ add_entrypoint_object(
     fesetexceptflag.h
   DEPENDS
     libc.hdr.fenv_macros
+    libc.hdr.types.fexcept_t
     libc.src.__support.FPUtil.fenv_impl
   COMPILE_OPTIONS
     -O2
@@ -136,6 +139,7 @@ add_entrypoint_object(
     feholdexcept.h
   DEPENDS
     libc.hdr.fenv_macros
+    libc.hdr.types.fenv_t
     libc.src.__support.FPUtil.fenv_impl
   COMPILE_OPTIONS
     -O2
@@ -149,6 +153,7 @@ add_entrypoint_object(
     feupdateenv.h
   DEPENDS
     libc.hdr.fenv_macros
+    libc.hdr.types.fenv_t
     libc.src.__support.FPUtil.fenv_impl
   COMPILE_OPTIONS
     -O2
diff --git a/libc/src/fenv/fegetenv.h b/libc/src/fenv/fegetenv.h
index 658316482984..8d3302964740 100644
--- a/libc/src/fenv/fegetenv.h
+++ b/libc/src/fenv/fegetenv.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_LIBC_SRC_FENV_FEGETENV_H
 #define LLVM_LIBC_SRC_FENV_FEGETENV_H
 
-#include <fenv.h>
+#include "hdr/types/fenv_t.h"
 
 namespace LIBC_NAMESPACE {
 
diff --git a/libc/src/fenv/fegetexceptflag.cpp b/libc/src/fenv/fegetexceptflag.cpp
index c6160da7afbd..72f31bf7abd5 100644
--- a/libc/src/fenv/fegetexceptflag.cpp
+++ b/libc/src/fenv/fegetexceptflag.cpp
@@ -7,11 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/fenv/fegetexceptflag.h"
+#include "hdr/types/fexcept_t.h"
 #include "src/__support/FPUtil/FEnvImpl.h"
 #include "src/__support/common.h"
 
-#include <fenv.h>
-
 namespace LIBC_NAMESPACE {
 
 LLVM_LIBC_FUNCTION(int, fegetexceptflag, (fexcept_t * flagp, int excepts)) {
diff --git a/libc/src/fenv/feholdexcept.cpp b/libc/src/fenv/feholdexcept.cpp
index f264c5ae251d..e5ca257e2be5 100644
--- a/libc/src/fenv/feholdexcept.cpp
+++ b/libc/src/fenv/feholdexcept.cpp
@@ -7,9 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/fenv/feholdexcept.h"
+#include "hdr/types/fenv_t.h"
 #include "src/__support/FPUtil/FEnvImpl.h"
 #include "src/__support/common.h"
-#include <fenv.h>
 
 namespace LIBC_NAMESPACE {
 
diff --git a/libc/src/fenv/feholdexcept.h b/libc/src/fenv/feholdexcept.h
index bbefc4ecbd41..e53619e12fea 100644
--- a/libc/src/fenv/feholdexcept.h
+++ b/libc/src/fenv/feholdexcept.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_LIBC_SRC_FENV_FEHOLDEXCEPT_H
 #define LLVM_LIBC_SRC_FENV_FEHOLDEXCEPT_H
 
-#include <fenv.h>
+#include "hdr/types/fenv_t.h"
 
 namespace LIBC_NAMESPACE {
 
diff --git a/libc/src/fenv/fesetenv.h b/libc/src/fenv/fesetenv.h
index 8b56bebc2e36..8636711cb1fe 100644
--- a/libc/src/fenv/fesetenv.h
+++ b/libc/src/fenv/fesetenv.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_LIBC_SRC_FENV_FESETENV_H
 #define LLVM_LIBC_SRC_FENV_FESETENV_H
 
-#include <fenv.h>
+#include "hdr/types/fenv_t.h"
 
 namespace LIBC_NAMESPACE {
 
diff --git a/libc/src/fenv/fesetexceptflag.cpp b/libc/src/fenv/fesetexceptflag.cpp
index 3ff8e270dc0a..628f33dcb9c4 100644
--- a/libc/src/fenv/fesetexceptflag.cpp
+++ b/libc/src/fenv/fesetexceptflag.cpp
@@ -7,9 +7,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/fenv/fesetexceptflag.h"
+#include "hdr/types/fexcept_t.h"
 #include "src/__support/FPUtil/FEnvImpl.h"
 #include "src/__support/common.h"
-#include <fenv.h>
 
 namespace LIBC_NAMESPACE {
 
diff --git a/libc/src/fenv/feupdateenv.h b/libc/src/fenv/feupdateenv.h
index 294c041ddeae..3c9c88b36e72 100644
--- a/libc/src/fenv/feupdateenv.h
+++ b/libc/src/fenv/feupdateenv.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_LIBC_SRC_FENV_FEUPDATEENV_H
 #define LLVM_LIBC_SRC_FENV_FEUPDATEENV_H
 
-#include <fenv.h>
+#include "hdr/types/fenv_t.h"
 
 namespace LIBC_NAMESPACE {
 
diff --git a/libc/test/UnitTest/CMakeLists.txt b/libc/test/UnitTest/CMakeLists.txt
index 4411170502ed..9113eca388e0 100644
--- a/libc/test/UnitTest/CMakeLists.txt
+++ b/libc/test/UnitTest/CMakeLists.txt
@@ -135,6 +135,7 @@ add_unittest_framework_library(
     LibcTest
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.fenv_impl
+    libc.hdr.types.fenv_t
 )
 
 add_unittest_framework_library(
diff --git a/libc/test/UnitTest/FPExceptMatcher.cpp b/libc/test/UnitTest/FPExceptMatcher.cpp
index 1601b7e53f2b..53ea72ad9ddd 100644
--- a/libc/test/UnitTest/FPExceptMatcher.cpp
+++ b/libc/test/UnitTest/FPExceptMatcher.cpp
@@ -8,7 +8,8 @@
 
 #include "FPExceptMatcher.h"
 
-#include <fenv.h>
+#include "hdr/types/fenv_t.h"
+#include "src/__support/FPUtil/FEnvImpl.h"
 #include <memory>
 #include <setjmp.h>
 #include <signal.h>
@@ -36,12 +37,12 @@ FPExceptMatcher::FPExceptMatcher(FunctionCaller *func) {
 
   caughtExcept = false;
   fenv_t oldEnv;
-  fegetenv(&oldEnv);
+  fputil::get_env(&oldEnv);
   if (sigsetjmp(jumpBuffer, 1) == 0)
     funcUP->call();
   // We restore the previous floating point environment after
   // the call to the function which can potentially raise SIGFPE.
-  fesetenv(&oldEnv);
+  fputil::set_env(&oldEnv);
   signal(SIGFPE, oldSIGFPEHandler);
   exceptionRaised = caughtExcept;
 }
diff --git a/libc/test/src/fenv/exception_flags_test.cpp b/libc/test/src/fenv/exception_flags_test.cpp
index 434adc06b1a3..d1d8bfcc53db 100644
--- a/libc/test/src/fenv/exception_flags_test.cpp
+++ b/libc/test/src/fenv/exception_flags_test.cpp
@@ -6,14 +6,13 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "hdr/types/fexcept_t.h"
 #include "src/fenv/fegetexceptflag.h"
 #include "src/fenv/fesetexceptflag.h"
 
 #include "src/__support/FPUtil/FEnvImpl.h"
 #include "test/UnitTest/Test.h"
 
-#include <fenv.h>
-
 TEST(LlvmLibcFenvTest, GetExceptFlagAndSetExceptFlag) {
   // We will disable all exceptions to prevent invocation of the exception
   // handler.
diff --git a/libc/test/src/fenv/feholdexcept_test.cpp b/libc/test/src/fenv/feholdexcept_test.cpp
index 735c7705ff49..0689d89ab233 100644
--- a/libc/test/src/fenv/feholdexcept_test.cpp
+++ b/libc/test/src/fenv/feholdexcept_test.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "hdr/types/fenv_t.h"
 #include "src/fenv/feholdexcept.h"
 
 #include "src/__support/FPUtil/FEnvImpl.h"
@@ -13,8 +14,6 @@
 #include "test/UnitTest/FPExceptMatcher.h"
 #include "test/UnitTest/Test.h"
 
-#include <fenv.h>
-
 TEST(LlvmLibcFEnvTest, RaiseAndCrash) {
 #if defined(LIBC_TARGET_ARCH_IS_ANY_ARM) ||                                    \
     defined(LIBC_TARGET_ARCH_IS_ANY_RISCV)
diff --git a/libc/test/src/fenv/feupdateenv_test.cpp b/libc/test/src/fenv/feupdateenv_test.cpp
index 96f253f76077..251b8566aac3 100644
--- a/libc/test/src/fenv/feupdateenv_test.cpp
+++ b/libc/test/src/fenv/feupdateenv_test.cpp
@@ -6,12 +6,12 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "hdr/types/fenv_t.h"
 #include "src/fenv/feupdateenv.h"
 
 #include "src/__support/FPUtil/FEnvImpl.h"
 #include "test/UnitTest/Test.h"
 
-#include <fenv.h>
 #include <signal.h>
 
 TEST(LlvmLibcFEnvTest, UpdateEnvTest) {
diff --git a/libc/test/src/fenv/getenv_and_setenv_test.cpp b/libc/test/src/fenv/getenv_and_setenv_test.cpp
index 8184a5c3bb99..f767e8ab9b2f 100644
--- a/libc/test/src/fenv/getenv_and_setenv_test.cpp
+++ b/libc/test/src/fenv/getenv_and_setenv_test.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "hdr/types/fenv_t.h"
 #include "src/fenv/fegetenv.h"
 #include "src/fenv/fegetround.h"
 #include "src/fenv/fesetenv.h"
@@ -14,8 +15,6 @@
 #include "src/__support/FPUtil/FEnvImpl.h"
 #include "test/UnitTest/Test.h"
 
-#include <fenv.h>
-
 TEST(LlvmLibcFenvTest, GetEnvAndSetEnv) {
   // We will disable all exceptions to prevent invocation of the exception
   // handler.
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index 9270a9d75b01..fb37f113b310 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -130,6 +130,16 @@ libc_support_library(
 
 ############################ Type Proxy Header Files ###########################
 
+libc_support_library(
+    name = "types_fenv_t",
+    hdrs = ["hdr/types/fenv_t.h"],
+)
+
+libc_support_library(
+    name = "types_fexcept_t",
+    hdrs = ["hdr/types/fexcept_t.h"],
+)
+
 libc_support_library(
     name = "types_sigset_t",
     hdrs = ["hdr/types/sigset_t.h"],
@@ -776,6 +786,7 @@ libc_support_library(
         ":errno",
         ":hdr_fenv_macros",
         ":hdr_math_macros",
+	":types_fenv_t"
     ],
 )
 
@@ -1251,6 +1262,7 @@ libc_function(
     deps = [
         ":__support_common",
         ":__support_fputil_fenv_impl",
+	":types_fexcept_t"
     ],
 )
 
@@ -1261,6 +1273,7 @@ libc_function(
     deps = [
         ":__support_common",
         ":__support_fputil_fenv_impl",
+	":types_fexcept_t",
     ],
 )
 
diff --git a/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel
index 82c015a7eeda..b1140a5a1609 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/UnitTest/BUILD.bazel
@@ -87,6 +87,7 @@ libc_support_library(
         "//libc:__support_fputil_rounding_mode",
         "//libc:hdr_math_macros",
 	"//libc:hdr_fenv_macros",
+	"//libc:types_fenv_t",
     ],
 )
 
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/fenv/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/fenv/BUILD.bazel
index bce1dd786a85..359db0723dfd 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/fenv/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/fenv/BUILD.bazel
@@ -65,6 +65,7 @@ libc_test(
         "//libc:__support_fputil_fenv_impl",
         "//libc:__support_macros_properties_architectures",
         "//libc/test/UnitTest:fp_test_helpers",
+	"//libc:types_fenv_t",
     ],
 )
 
@@ -77,6 +78,7 @@ libc_test(
     ],
     deps = [
         "//libc:__support_fputil_fenv_impl",
+	"//libc:types_fexcept_t",
     ],
 )
 
@@ -115,6 +117,7 @@ libc_test(
     ],
     deps = [
         "//libc:__support_fputil_fenv_impl",
+	"//libc:types_fenv_t",
     ],
 )
 
@@ -129,5 +132,6 @@ libc_test(
     ],
     deps = [
         "//libc:__support_fputil_fenv_impl",
+	"//libc:types_fenv_t",
     ],
 )
-- 
GitLab


From 45339568486cafaa414a1e40d73a251f7d15e284 Mon Sep 17 00:00:00 2001
From: Yijia Gu <yijiagu@google.com>
Date: Mon, 15 Apr 2024 11:46:40 -0700
Subject: [PATCH 286/464] [mlir] add polynomial dialect in bazel BUILD

---
 .../llvm-project-overlay/mlir/BUILD.bazel     | 104 ++++++++++++++++++
 1 file changed, 104 insertions(+)

diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 67052fcd3993..522e3efe4e0a 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -6602,6 +6602,109 @@ gentbl_cc_library(
     deps = [":PDLInterpOpsTdFiles"],
 )
 
+cc_library(
+    name = "PolynomialDialect",
+    srcs = glob([
+        "lib/Dialect/Polynomial/IR/*.cpp",
+    ]),
+    hdrs = glob([
+        "include/mlir/Dialect/Polynomial/IR/*.h",
+    ]),
+    includes = ["include"],
+    deps = [
+        ":IR",
+        ":InferTypeOpInterface",
+        ":PolynomialAttributesIncGen",
+        ":PolynomialIncGen",
+        ":Support",
+        "//third_party/llvm/llvm-project/llvm:Support",
+    ],
+)
+
+td_library(
+    name = "PolynomialTdFiles",
+    srcs = ["include/mlir/Dialect/Polynomial/IR/Polynomial.td"],
+    includes = ["include"],
+    deps = [
+        ":BuiltinDialectTdFiles",
+        ":InferTypeOpInterfaceTdFiles",
+        ":OpBaseTdFiles",
+        ":SideEffectInterfacesTdFiles",
+    ],
+)
+
+gentbl_cc_library(
+    name = "PolynomialIncGen",
+    tbl_outs = [
+        (
+            ["-gen-op-decls"],
+            "include/mlir/Dialect/Polynomial/IR/Polynomial.h.inc",
+        ),
+        (
+            ["-gen-op-defs"],
+            "include/mlir/Dialect/Polynomial/IR/Polynomial.cpp.inc",
+        ),
+        (
+            [
+                "-gen-dialect-decls",
+                "-dialect=polynomial",
+            ],
+            "include/mlir/Dialect/Polynomial/IR/PolynomialDialect.h.inc",
+        ),
+        (
+            [
+                "-gen-dialect-defs",
+                "-dialect=polynomial",
+            ],
+            "include/mlir/Dialect/Polynomial/IR/PolynomialDialect.cpp.inc",
+        ),
+        (
+            [
+                "--gen-typedef-decls",
+                "-typedefs-dialect=polynomial",
+            ],
+            "include/mlir/Dialect/Polynomial/IR/PolynomialTypes.h.inc",
+        ),
+        (
+            [
+                "--gen-typedef-defs",
+                "-typedefs-dialect=polynomial",
+            ],
+            "include/mlir/Dialect/Polynomial/IR/PolynomialTypes.cpp.inc",
+        ),
+        (
+            ["-gen-op-doc"],
+            "g3doc/Dialects/Polynomial/Polynomial.md",
+        ),
+    ],
+    tblgen = ":mlir-tblgen",
+    td_file = "include/mlir/Dialect/Polynomial/IR/Polynomial.td",
+    deps = [":PolynomialTdFiles"],
+)
+
+gentbl_cc_library(
+    name = "PolynomialAttributesIncGen",
+    tbl_outs = [
+        (
+            [
+                "-gen-attrdef-decls",
+                "-attrdefs-dialect=polynomial",
+            ],
+            "include/mlir/Dialect/Polynomial/IR/PolynomialAttributes.h.inc",
+        ),
+        (
+            [
+                "-gen-attrdef-defs",
+                "-attrdefs-dialect=polynomial",
+            ],
+            "include/mlir/Dialect/Polynomial/IR/PolynomialAttributes.cpp.inc",
+        ),
+    ],
+    tblgen = ":mlir-tblgen",
+    td_file = "include/mlir/Dialect/Polynomial/IR/Polynomial.td",
+    deps = [":PolynomialTdFiles"],
+)
+
 td_library(
     name = "SPIRVOpsTdFiles",
     srcs = glob(["include/mlir/Dialect/SPIRV/IR/*.td"]),
@@ -9076,6 +9179,7 @@ cc_library(
         ":PDLDialect",
         ":PDLInterpDialect",
         ":PDLToPDLInterp",
+        ":PolynomialDialect",
         ":QuantOps",
         ":ROCDLDialect",
         ":ROCDLTarget",
-- 
GitLab


From f9d761df50d12147fc7caa19927be81816d60e2d Mon Sep 17 00:00:00 2001
From: Justin Fargnoli <justinfargnoli@gmail.com>
Date: Mon, 15 Apr 2024 11:53:15 -0700
Subject: [PATCH 287/464] [tools] Fix `LLVM_USE_SANITIZE_COVERAGE` build
 (#87722)

---
 llvm/tools/llvm-dis-fuzzer/CMakeLists.txt                   | 2 ++
 llvm/tools/vfabi-demangle-fuzzer/vfabi-demangler-fuzzer.cpp | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/llvm/tools/llvm-dis-fuzzer/CMakeLists.txt b/llvm/tools/llvm-dis-fuzzer/CMakeLists.txt
index 4b0b80e2f758..b424a14374f3 100644
--- a/llvm/tools/llvm-dis-fuzzer/CMakeLists.txt
+++ b/llvm/tools/llvm-dis-fuzzer/CMakeLists.txt
@@ -1,5 +1,7 @@
 set(LLVM_LINK_COMPONENTS
   BitReader
+  Core
+  Support
 )
 add_llvm_fuzzer(llvm-dis-fuzzer
   llvm-dis-fuzzer.cpp
diff --git a/llvm/tools/vfabi-demangle-fuzzer/vfabi-demangler-fuzzer.cpp b/llvm/tools/vfabi-demangle-fuzzer/vfabi-demangler-fuzzer.cpp
index e1041bd4eef9..309fe2b9957a 100644
--- a/llvm/tools/vfabi-demangle-fuzzer/vfabi-demangler-fuzzer.cpp
+++ b/llvm/tools/vfabi-demangle-fuzzer/vfabi-demangler-fuzzer.cpp
@@ -11,7 +11,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/AsmParser/Parser.h"
+#include "llvm/IR/Module.h"
 #include "llvm/IR/VFABIDemangler.h"
+#include "llvm/Support/SourceMgr.h"
 
 using namespace llvm;
 
-- 
GitLab


From c31a8104d1173d79f8b71518829046b441d98d59 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 15 Apr 2024 12:01:41 -0700
Subject: [PATCH 288/464] [RISCV] Add -mattr=+v to intrinsic-cttz-elts.ll. NFC

---
 .../test/CodeGen/RISCV/intrinsic-cttz-elts.ll | 108 ++++++------------
 1 file changed, 38 insertions(+), 70 deletions(-)

diff --git a/llvm/test/CodeGen/RISCV/intrinsic-cttz-elts.ll b/llvm/test/CodeGen/RISCV/intrinsic-cttz-elts.ll
index 15abc9b75883..49d4760a2e9a 100644
--- a/llvm/test/CodeGen/RISCV/intrinsic-cttz-elts.ll
+++ b/llvm/test/CodeGen/RISCV/intrinsic-cttz-elts.ll
@@ -1,38 +1,22 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
-; RUN: llc -mtriple=riscv32 < %s | FileCheck %s -check-prefix=RV32
-; RUN: llc -mtriple=riscv64 < %s | FileCheck %s -check-prefix=RV64
+; RUN: llc -mtriple=riscv32 -mattr=+v < %s | FileCheck %s -check-prefix=RV32
+; RUN: llc -mtriple=riscv64 -mattr=+v < %s | FileCheck %s -check-prefix=RV64
 
 ; FIXED WIDTH
 
 define i16 @ctz_v4i32(<4 x i32> %a) {
 ; RV32-LABEL: ctz_v4i32:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    lw a3, 0(a0)
-; RV32-NEXT:    lw a1, 4(a0)
-; RV32-NEXT:    lw a2, 12(a0)
-; RV32-NEXT:    lw a4, 8(a0)
-; RV32-NEXT:    seqz a0, a3
-; RV32-NEXT:    addi a0, a0, -1
-; RV32-NEXT:    andi a0, a0, 4
-; RV32-NEXT:    seqz a3, a4
-; RV32-NEXT:    addi a3, a3, -1
-; RV32-NEXT:    andi a3, a3, 2
-; RV32-NEXT:    bltu a3, a0, .LBB0_2
-; RV32-NEXT:  # %bb.1:
-; RV32-NEXT:    mv a0, a3
-; RV32-NEXT:  .LBB0_2:
-; RV32-NEXT:    snez a2, a2
-; RV32-NEXT:    seqz a1, a1
-; RV32-NEXT:    addi a1, a1, -1
-; RV32-NEXT:    andi a1, a1, 3
-; RV32-NEXT:    bltu a2, a1, .LBB0_4
-; RV32-NEXT:  # %bb.3:
-; RV32-NEXT:    mv a1, a2
-; RV32-NEXT:  .LBB0_4:
-; RV32-NEXT:    bltu a1, a0, .LBB0_6
-; RV32-NEXT:  # %bb.5:
-; RV32-NEXT:    mv a0, a1
-; RV32-NEXT:  .LBB0_6:
+; RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV32-NEXT:    vmsne.vi v0, v8, 0
+; RV32-NEXT:    vsetvli zero, zero, e8, mf4, ta, ma
+; RV32-NEXT:    vmv.v.i v8, 0
+; RV32-NEXT:    vmerge.vim v8, v8, -1, v0
+; RV32-NEXT:    vid.v v9
+; RV32-NEXT:    vrsub.vi v9, v9, 4
+; RV32-NEXT:    vand.vv v8, v8, v9
+; RV32-NEXT:    vredmaxu.vs v8, v8, v8
+; RV32-NEXT:    vmv.x.s a0, v8
 ; RV32-NEXT:    li a1, 4
 ; RV32-NEXT:    sub a1, a1, a0
 ; RV32-NEXT:    andi a0, a1, 255
@@ -40,32 +24,16 @@ define i16 @ctz_v4i32(<4 x i32> %a) {
 ;
 ; RV64-LABEL: ctz_v4i32:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    lw a3, 0(a0)
-; RV64-NEXT:    lw a1, 8(a0)
-; RV64-NEXT:    lw a2, 24(a0)
-; RV64-NEXT:    lw a4, 16(a0)
-; RV64-NEXT:    seqz a0, a3
-; RV64-NEXT:    addi a0, a0, -1
-; RV64-NEXT:    andi a0, a0, 4
-; RV64-NEXT:    seqz a3, a4
-; RV64-NEXT:    addi a3, a3, -1
-; RV64-NEXT:    andi a3, a3, 2
-; RV64-NEXT:    bltu a3, a0, .LBB0_2
-; RV64-NEXT:  # %bb.1:
-; RV64-NEXT:    mv a0, a3
-; RV64-NEXT:  .LBB0_2:
-; RV64-NEXT:    snez a2, a2
-; RV64-NEXT:    seqz a1, a1
-; RV64-NEXT:    addi a1, a1, -1
-; RV64-NEXT:    andi a1, a1, 3
-; RV64-NEXT:    bltu a2, a1, .LBB0_4
-; RV64-NEXT:  # %bb.3:
-; RV64-NEXT:    mv a1, a2
-; RV64-NEXT:  .LBB0_4:
-; RV64-NEXT:    bltu a1, a0, .LBB0_6
-; RV64-NEXT:  # %bb.5:
-; RV64-NEXT:    mv a0, a1
-; RV64-NEXT:  .LBB0_6:
+; RV64-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; RV64-NEXT:    vmsne.vi v0, v8, 0
+; RV64-NEXT:    vsetvli zero, zero, e8, mf4, ta, ma
+; RV64-NEXT:    vmv.v.i v8, 0
+; RV64-NEXT:    vmerge.vim v8, v8, -1, v0
+; RV64-NEXT:    vid.v v9
+; RV64-NEXT:    vrsub.vi v9, v9, 4
+; RV64-NEXT:    vand.vv v8, v8, v9
+; RV64-NEXT:    vredmaxu.vs v8, v8, v8
+; RV64-NEXT:    vmv.x.s a0, v8
 ; RV64-NEXT:    li a1, 4
 ; RV64-NEXT:    subw a1, a1, a0
 ; RV64-NEXT:    andi a0, a1, 255
@@ -79,14 +47,14 @@ define i16 @ctz_v4i32(<4 x i32> %a) {
 define i32 @ctz_v2i1_poison(<2 x i1> %a) {
 ; RV32-LABEL: ctz_v2i1_poison:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    andi a1, a1, 1
-; RV32-NEXT:    slli a0, a0, 31
-; RV32-NEXT:    srai a0, a0, 31
-; RV32-NEXT:    andi a0, a0, 2
-; RV32-NEXT:    bltu a1, a0, .LBB1_2
-; RV32-NEXT:  # %bb.1:
-; RV32-NEXT:    mv a0, a1
-; RV32-NEXT:  .LBB1_2:
+; RV32-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
+; RV32-NEXT:    vmv.v.i v8, 0
+; RV32-NEXT:    vmerge.vim v8, v8, -1, v0
+; RV32-NEXT:    vid.v v9
+; RV32-NEXT:    vrsub.vi v9, v9, 2
+; RV32-NEXT:    vand.vv v8, v8, v9
+; RV32-NEXT:    vredmaxu.vs v8, v8, v8
+; RV32-NEXT:    vmv.x.s a0, v8
 ; RV32-NEXT:    li a1, 2
 ; RV32-NEXT:    sub a1, a1, a0
 ; RV32-NEXT:    andi a0, a1, 255
@@ -94,14 +62,14 @@ define i32 @ctz_v2i1_poison(<2 x i1> %a) {
 ;
 ; RV64-LABEL: ctz_v2i1_poison:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    andi a1, a1, 1
-; RV64-NEXT:    slli a0, a0, 63
-; RV64-NEXT:    srai a0, a0, 63
-; RV64-NEXT:    andi a0, a0, 2
-; RV64-NEXT:    bltu a1, a0, .LBB1_2
-; RV64-NEXT:  # %bb.1:
-; RV64-NEXT:    mv a0, a1
-; RV64-NEXT:  .LBB1_2:
+; RV64-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
+; RV64-NEXT:    vmv.v.i v8, 0
+; RV64-NEXT:    vmerge.vim v8, v8, -1, v0
+; RV64-NEXT:    vid.v v9
+; RV64-NEXT:    vrsub.vi v9, v9, 2
+; RV64-NEXT:    vand.vv v8, v8, v9
+; RV64-NEXT:    vredmaxu.vs v8, v8, v8
+; RV64-NEXT:    vmv.x.s a0, v8
 ; RV64-NEXT:    li a1, 2
 ; RV64-NEXT:    subw a1, a1, a0
 ; RV64-NEXT:    andi a0, a1, 255
-- 
GitLab


From b153c05cba9be7f009b8ad8413c5840baf7d278c Mon Sep 17 00:00:00 2001
From: Ivan Butygin <ivan.butygin@gmail.com>
Date: Mon, 15 Apr 2024 22:16:59 +0300
Subject: [PATCH 289/464] [mlir][scf] Uplift `scf.while` to `scf.for` (#76108)

Add uplifting from `scf.while` to `scf.for`.

This uplifting expects a very specific ops pattern:
* `before` block consisting of single `arith.cmp` op
* `after` block containing `arith.addi`

We also have a set of patterns to cleanup `scf.while` loops to get them
close to the desired form, they will be added in separate PRs.

This is part of upstreaming `numba-mlir` scf uplifting pipeline: `cf ->
scf.while -> scf.for -> scf.parallel`

Original code:
https://github.com/numba/numba-mlir/blob/main/mlir/lib/Transforms/PromoteToParallel.cpp
---
 .../mlir/Dialect/SCF/Transforms/Patterns.h    |   6 +
 .../mlir/Dialect/SCF/Transforms/Transforms.h  |   6 +
 .../lib/Dialect/SCF/Transforms/CMakeLists.txt |   1 +
 .../SCF/Transforms/UpliftWhileToFor.cpp       | 222 ++++++++++++++++++
 mlir/test/Dialect/SCF/uplift-while.mlir       | 157 +++++++++++++
 mlir/test/lib/Dialect/SCF/CMakeLists.txt      |   1 +
 .../lib/Dialect/SCF/TestUpliftWhileToFor.cpp  |  50 ++++
 mlir/tools/mlir-opt/mlir-opt.cpp              |   2 +
 8 files changed, 445 insertions(+)
 create mode 100644 mlir/lib/Dialect/SCF/Transforms/UpliftWhileToFor.cpp
 create mode 100644 mlir/test/Dialect/SCF/uplift-while.mlir
 create mode 100644 mlir/test/lib/Dialect/SCF/TestUpliftWhileToFor.cpp

diff --git a/mlir/include/mlir/Dialect/SCF/Transforms/Patterns.h b/mlir/include/mlir/Dialect/SCF/Transforms/Patterns.h
index 5c0d5643c019..fdf257062698 100644
--- a/mlir/include/mlir/Dialect/SCF/Transforms/Patterns.h
+++ b/mlir/include/mlir/Dialect/SCF/Transforms/Patterns.h
@@ -79,6 +79,12 @@ void populateSCFLoopPipeliningPatterns(RewritePatternSet &patterns,
 /// loop bounds and loop steps are canonicalized.
 void populateSCFForLoopCanonicalizationPatterns(RewritePatternSet &patterns);
 
+/// Populate patterns to uplift `scf.while` ops to `scf.for`.
+/// Uplifitng expects a specific ops pattern:
+///  * `before` block consisting of single arith.cmp op
+///  * `after` block containing arith.addi
+void populateUpliftWhileToForPatterns(RewritePatternSet &patterns);
+
 } // namespace scf
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Dialect/SCF/Transforms/Transforms.h b/mlir/include/mlir/Dialect/SCF/Transforms/Transforms.h
index 690cd146c606..220dcb35571d 100644
--- a/mlir/include/mlir/Dialect/SCF/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/SCF/Transforms/Transforms.h
@@ -222,6 +222,12 @@ FailureOr<WhileOp> wrapWhileLoopInZeroTripCheck(WhileOp whileOp,
                                                 RewriterBase &rewriter,
                                                 bool forceCreateCheck = false);
 
+/// Try to uplift `scf.while` op to `scf.for`.
+/// Uplifitng expects a specific ops pattern:
+///  * `before` block consisting of single arith.cmp op
+///  * `after` block containing arith.addi
+FailureOr<ForOp> upliftWhileToForLoop(RewriterBase &rewriter, WhileOp loop);
+
 } // namespace scf
 } // namespace mlir
 
diff --git a/mlir/lib/Dialect/SCF/Transforms/CMakeLists.txt b/mlir/lib/Dialect/SCF/Transforms/CMakeLists.txt
index e5494205e086..a2925aef17ca 100644
--- a/mlir/lib/Dialect/SCF/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/SCF/Transforms/CMakeLists.txt
@@ -14,6 +14,7 @@ add_mlir_dialect_library(MLIRSCFTransforms
   StructuralTypeConversions.cpp
   TileUsingInterface.cpp
   WrapInZeroTripCheck.cpp
+  UpliftWhileToFor.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/SCF
diff --git a/mlir/lib/Dialect/SCF/Transforms/UpliftWhileToFor.cpp b/mlir/lib/Dialect/SCF/Transforms/UpliftWhileToFor.cpp
new file mode 100644
index 000000000000..fea2f659535b
--- /dev/null
+++ b/mlir/lib/Dialect/SCF/Transforms/UpliftWhileToFor.cpp
@@ -0,0 +1,222 @@
+//===- UpliftWhileToFor.cpp - scf.while to scf.for loop uplifting ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Transforms SCF.WhileOp's into SCF.ForOp's.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/SCF/Transforms/Passes.h"
+
+#include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/SCF/IR/SCF.h"
+#include "mlir/Dialect/SCF/Transforms/Patterns.h"
+#include "mlir/IR/Dominance.h"
+#include "mlir/IR/PatternMatch.h"
+
+using namespace mlir;
+
+namespace {
+struct UpliftWhileOp : public OpRewritePattern<scf::WhileOp> {
+  using OpRewritePattern::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(scf::WhileOp loop,
+                                PatternRewriter &rewriter) const override {
+    return upliftWhileToForLoop(rewriter, loop);
+  }
+};
+} // namespace
+
+FailureOr<scf::ForOp> mlir::scf::upliftWhileToForLoop(RewriterBase &rewriter,
+                                                      scf::WhileOp loop) {
+  Block *beforeBody = loop.getBeforeBody();
+  if (!llvm::hasSingleElement(beforeBody->without_terminator()))
+    return rewriter.notifyMatchFailure(loop, "Loop body must have single op");
+
+  auto cmp = dyn_cast<arith::CmpIOp>(beforeBody->front());
+  if (!cmp)
+    return rewriter.notifyMatchFailure(loop,
+                                       "Loop body must have single cmp op");
+
+  scf::ConditionOp beforeTerm = loop.getConditionOp();
+  if (!cmp->hasOneUse() || beforeTerm.getCondition() != cmp.getResult())
+    return rewriter.notifyMatchFailure(loop, [&](Diagnostic &diag) {
+      diag << "Expected single condition use: " << *cmp;
+    });
+
+  // All `before` block args must be directly forwarded to ConditionOp.
+  // They will be converted to `scf.for` `iter_vars` except induction var.
+  if (ValueRange(beforeBody->getArguments()) != beforeTerm.getArgs())
+    return rewriter.notifyMatchFailure(loop, "Invalid args order");
+
+  using Pred = arith::CmpIPredicate;
+  Pred predicate = cmp.getPredicate();
+  if (predicate != Pred::slt && predicate != Pred::sgt)
+    return rewriter.notifyMatchFailure(loop, [&](Diagnostic &diag) {
+      diag << "Expected 'slt' or 'sgt' predicate: " << *cmp;
+    });
+
+  BlockArgument inductionVar;
+  Value ub;
+  DominanceInfo dom;
+
+  // Check if cmp has a suitable form. One of the arguments must be a `before`
+  // block arg, other must be defined outside `scf.while` and will be treated
+  // as upper bound.
+  for (bool reverse : {false, true}) {
+    auto expectedPred = reverse ? Pred::sgt : Pred::slt;
+    if (cmp.getPredicate() != expectedPred)
+      continue;
+
+    auto arg1 = reverse ? cmp.getRhs() : cmp.getLhs();
+    auto arg2 = reverse ? cmp.getLhs() : cmp.getRhs();
+
+    auto blockArg = dyn_cast<BlockArgument>(arg1);
+    if (!blockArg || blockArg.getOwner() != beforeBody)
+      continue;
+
+    if (!dom.properlyDominates(arg2, loop))
+      continue;
+
+    inductionVar = blockArg;
+    ub = arg2;
+    break;
+  }
+
+  if (!inductionVar)
+    return rewriter.notifyMatchFailure(loop, [&](Diagnostic &diag) {
+      diag << "Unrecognized cmp form: " << *cmp;
+    });
+
+  // inductionVar must have 2 uses: one is in `cmp` and other is `condition`
+  // arg.
+  if (!llvm::hasNItems(inductionVar.getUses(), 2))
+    return rewriter.notifyMatchFailure(loop, [&](Diagnostic &diag) {
+      diag << "Unrecognized induction var: " << inductionVar;
+    });
+
+  Block *afterBody = loop.getAfterBody();
+  scf::YieldOp afterTerm = loop.getYieldOp();
+  auto argNumber = inductionVar.getArgNumber();
+  auto afterTermIndArg = afterTerm.getResults()[argNumber];
+
+  auto inductionVarAfter = afterBody->getArgument(argNumber);
+
+  Value step;
+
+  // Find suitable `addi` op inside `after` block, one of the args must be an
+  // Induction var passed from `before` block and second arg must be defined
+  // outside of the loop and will be considered step value.
+  // TODO: Add `subi` support?
+  for (auto &use : inductionVarAfter.getUses()) {
+    auto owner = dyn_cast<arith::AddIOp>(use.getOwner());
+    if (!owner)
+      continue;
+
+    auto other =
+        (inductionVarAfter == owner.getLhs() ? owner.getRhs() : owner.getLhs());
+    if (!dom.properlyDominates(other, loop))
+      continue;
+
+    if (afterTermIndArg != owner.getResult())
+      continue;
+
+    step = other;
+    break;
+  }
+
+  if (!step)
+    return rewriter.notifyMatchFailure(loop, "Didn't found suitable 'addi' op");
+
+  auto lb = loop.getInits()[argNumber];
+
+  assert(lb.getType().isIntOrIndex());
+  assert(lb.getType() == ub.getType());
+  assert(lb.getType() == step.getType());
+
+  llvm::SmallVector<Value> newArgs;
+
+  // Populate inits for new `scf.for`, skip induction var.
+  newArgs.reserve(loop.getInits().size());
+  for (auto &&[i, init] : llvm::enumerate(loop.getInits())) {
+    if (i == argNumber)
+      continue;
+
+    newArgs.emplace_back(init);
+  }
+
+  Location loc = loop.getLoc();
+
+  // With `builder == nullptr`, ForOp::build will try to insert terminator at
+  // the end of newly created block and we don't want it. Provide empty
+  // dummy builder instead.
+  auto emptyBuilder = [](OpBuilder &, Location, Value, ValueRange) {};
+  auto newLoop =
+      rewriter.create<scf::ForOp>(loc, lb, ub, step, newArgs, emptyBuilder);
+
+  Block *newBody = newLoop.getBody();
+
+  // Populate block args for `scf.for` body, move induction var to the front.
+  newArgs.clear();
+  ValueRange newBodyArgs = newBody->getArguments();
+  for (auto i : llvm::seq<size_t>(0, newBodyArgs.size())) {
+    if (i < argNumber) {
+      newArgs.emplace_back(newBodyArgs[i + 1]);
+    } else if (i == argNumber) {
+      newArgs.emplace_back(newBodyArgs.front());
+    } else {
+      newArgs.emplace_back(newBodyArgs[i]);
+    }
+  }
+
+  rewriter.inlineBlockBefore(loop.getAfterBody(), newBody, newBody->end(),
+                             newArgs);
+
+  auto term = cast<scf::YieldOp>(newBody->getTerminator());
+
+  // Populate new yield args, skipping the induction var.
+  newArgs.clear();
+  for (auto &&[i, arg] : llvm::enumerate(term.getResults())) {
+    if (i == argNumber)
+      continue;
+
+    newArgs.emplace_back(arg);
+  }
+
+  OpBuilder::InsertionGuard g(rewriter);
+  rewriter.setInsertionPoint(term);
+  rewriter.replaceOpWithNewOp<scf::YieldOp>(term, newArgs);
+
+  // Compute induction var value after loop execution.
+  rewriter.setInsertionPointAfter(newLoop);
+  Value one;
+  if (isa<IndexType>(step.getType())) {
+    one = rewriter.create<arith::ConstantIndexOp>(loc, 1);
+  } else {
+    one = rewriter.create<arith::ConstantIntOp>(loc, 1, step.getType());
+  }
+
+  Value stepDec = rewriter.create<arith::SubIOp>(loc, step, one);
+  Value len = rewriter.create<arith::SubIOp>(loc, ub, lb);
+  len = rewriter.create<arith::AddIOp>(loc, len, stepDec);
+  len = rewriter.create<arith::DivSIOp>(loc, len, step);
+  len = rewriter.create<arith::SubIOp>(loc, len, one);
+  Value res = rewriter.create<arith::MulIOp>(loc, len, step);
+  res = rewriter.create<arith::AddIOp>(loc, lb, res);
+
+  // Reconstruct `scf.while` results, inserting final induction var value
+  // into proper place.
+  newArgs.clear();
+  llvm::append_range(newArgs, newLoop.getResults());
+  newArgs.insert(newArgs.begin() + argNumber, res);
+  rewriter.replaceOp(loop, newArgs);
+  return newLoop;
+}
+
+void mlir::scf::populateUpliftWhileToForPatterns(RewritePatternSet &patterns) {
+  patterns.add<UpliftWhileOp>(patterns.getContext());
+}
diff --git a/mlir/test/Dialect/SCF/uplift-while.mlir b/mlir/test/Dialect/SCF/uplift-while.mlir
new file mode 100644
index 000000000000..25ea6142a332
--- /dev/null
+++ b/mlir/test/Dialect/SCF/uplift-while.mlir
@@ -0,0 +1,157 @@
+// RUN: mlir-opt %s -pass-pipeline='builtin.module(func.func(test-scf-uplift-while-to-for))' -split-input-file -allow-unregistered-dialect | FileCheck %s
+
+func.func @uplift_while(%arg0: index, %arg1: index, %arg2: index) -> index {
+  %0 = scf.while (%arg3 = %arg0) : (index) -> (index) {
+    %1 = arith.cmpi slt, %arg3, %arg1 : index
+    scf.condition(%1) %arg3 : index
+  } do {
+  ^bb0(%arg3: index):
+    "test.test1"(%arg3) : (index) -> ()
+    %added = arith.addi %arg3, %arg2 : index
+    "test.test2"(%added) : (index) -> ()
+    scf.yield %added : index
+  }
+  return %0 : index
+}
+
+// CHECK-LABEL: func @uplift_while
+//  CHECK-SAME:     (%[[BEGIN:.*]]: index, %[[END:.*]]: index, %[[STEP:.*]]: index) -> index
+//       CHECK:     %[[C1:.*]] = arith.constant 1 : index
+//       CHECK:     scf.for %[[I:.*]] = %[[BEGIN]] to %[[END]] step %[[STEP]] {
+//       CHECK:     "test.test1"(%[[I]]) : (index) -> ()
+//       CHECK:     %[[INC:.*]] = arith.addi %[[I]], %[[STEP]] : index
+//       CHECK:     "test.test2"(%[[INC]]) : (index) -> ()
+//       CHECK:     %[[R1:.*]] = arith.subi %[[STEP]], %[[C1]] : index
+//       CHECK:     %[[R2:.*]] = arith.subi %[[END]], %[[BEGIN]] : index
+//       CHECK:     %[[R3:.*]] = arith.addi %[[R2]], %[[R1]] : index
+//       CHECK:     %[[R4:.*]] = arith.divsi %[[R3]], %[[STEP]] : index
+//       CHECK:     %[[R5:.*]] = arith.subi %[[R4]], %[[C1]] : index
+//       CHECK:     %[[R6:.*]] = arith.muli %[[R5]], %[[STEP]] : index
+//       CHECK:     %[[R7:.*]] = arith.addi %[[BEGIN]], %[[R6]] : index
+//       CHECK:     return %[[R7]] : index
+
+// -----
+
+func.func @uplift_while(%arg0: index, %arg1: index, %arg2: index) -> index {
+  %0 = scf.while (%arg3 = %arg0) : (index) -> (index) {
+    %1 = arith.cmpi sgt, %arg1, %arg3 : index
+    scf.condition(%1) %arg3 : index
+  } do {
+  ^bb0(%arg3: index):
+    "test.test1"(%arg3) : (index) -> ()
+    %added = arith.addi %arg3, %arg2 : index
+    "test.test2"(%added) : (index) -> ()
+    scf.yield %added : index
+  }
+  return %0 : index
+}
+
+// CHECK-LABEL: func @uplift_while
+//  CHECK-SAME:     (%[[BEGIN:.*]]: index, %[[END:.*]]: index, %[[STEP:.*]]: index) -> index
+//       CHECK:     %[[C1:.*]] = arith.constant 1 : index
+//       CHECK:     scf.for %[[I:.*]] = %[[BEGIN]] to %[[END]] step %[[STEP]] {
+//       CHECK:     "test.test1"(%[[I]]) : (index) -> ()
+//       CHECK:     %[[INC:.*]] = arith.addi %[[I]], %[[STEP]] : index
+//       CHECK:     "test.test2"(%[[INC]]) : (index) -> ()
+//       CHECK:     %[[R1:.*]] = arith.subi %[[STEP]], %[[C1]] : index
+//       CHECK:     %[[R2:.*]] = arith.subi %[[END]], %[[BEGIN]] : index
+//       CHECK:     %[[R3:.*]] = arith.addi %[[R2]], %[[R1]] : index
+//       CHECK:     %[[R4:.*]] = arith.divsi %[[R3]], %[[STEP]] : index
+//       CHECK:     %[[R5:.*]] = arith.subi %[[R4]], %[[C1]] : index
+//       CHECK:     %[[R6:.*]] = arith.muli %[[R5]], %[[STEP]] : index
+//       CHECK:     %[[R7:.*]] = arith.addi %[[BEGIN]], %[[R6]] : index
+//       CHECK:     return %[[R7]] : index
+
+// -----
+
+func.func @uplift_while(%arg0: index, %arg1: index, %arg2: index) -> index {
+  %0 = scf.while (%arg3 = %arg0) : (index) -> (index) {
+    %1 = arith.cmpi slt, %arg3, %arg1 : index
+    scf.condition(%1) %arg3 : index
+  } do {
+  ^bb0(%arg3: index):
+    "test.test1"(%arg3) : (index) -> ()
+    %added = arith.addi %arg2, %arg3 : index
+    "test.test2"(%added) : (index) -> ()
+    scf.yield %added : index
+  }
+  return %0 : index
+}
+
+// CHECK-LABEL: func @uplift_while
+//  CHECK-SAME:     (%[[BEGIN:.*]]: index, %[[END:.*]]: index, %[[STEP:.*]]: index) -> index
+//       CHECK:     %[[C1:.*]] = arith.constant 1 : index
+//       CHECK:     scf.for %[[I:.*]] = %[[BEGIN]] to %[[END]] step %[[STEP]] {
+//       CHECK:     "test.test1"(%[[I]]) : (index) -> ()
+//       CHECK:     %[[INC:.*]] = arith.addi %[[STEP]], %[[I]] : index
+//       CHECK:     "test.test2"(%[[INC]]) : (index) -> ()
+//       CHECK:     %[[R1:.*]] = arith.subi %[[STEP]], %[[C1]] : index
+//       CHECK:     %[[R2:.*]] = arith.subi %[[END]], %[[BEGIN]] : index
+//       CHECK:     %[[R3:.*]] = arith.addi %[[R2]], %[[R1]] : index
+//       CHECK:     %[[R4:.*]] = arith.divsi %[[R3]], %[[STEP]] : index
+//       CHECK:     %[[R5:.*]] = arith.subi %[[R4]], %[[C1]] : index
+//       CHECK:     %[[R6:.*]] = arith.muli %[[R5]], %[[STEP]] : index
+//       CHECK:     %[[R7:.*]] = arith.addi %[[BEGIN]], %[[R6]] : index
+//       CHECK:     return %[[R7]] : index
+
+
+// -----
+
+func.func @uplift_while(%arg0: index, %arg1: index, %arg2: index) -> (i32, f32) {
+  %c1 = arith.constant 1 : i32
+  %c2 = arith.constant 2.0 : f32
+  %0:3 = scf.while (%arg4 = %c1, %arg3 = %arg0, %arg5 = %c2) : (i32, index, f32) -> (i32, index, f32) {
+    %1 = arith.cmpi slt, %arg3, %arg1 : index
+    scf.condition(%1) %arg4, %arg3, %arg5 : i32, index, f32
+  } do {
+  ^bb0(%arg4: i32, %arg3: index, %arg5: f32):
+    %1 = "test.test1"(%arg4) : (i32) -> i32
+    %added = arith.addi %arg3, %arg2 : index
+    %2 = "test.test2"(%arg5) : (f32) -> f32
+    scf.yield %1, %added, %2 : i32, index, f32
+  }
+  return %0#0, %0#2 : i32, f32
+}
+
+// CHECK-LABEL: func @uplift_while
+//  CHECK-SAME:     (%[[BEGIN:.*]]: index, %[[END:.*]]: index, %[[STEP:.*]]: index) -> (i32, f32)
+//   CHECK-DAG:     %[[C1:.*]] = arith.constant 1 : i32
+//   CHECK-DAG:     %[[C2:.*]] = arith.constant 2.000000e+00 : f32
+//       CHECK:     %[[RES:.*]]:2 = scf.for %[[I:.*]] = %[[BEGIN]] to %[[END]] step %[[STEP]]
+//  CHECK-SAME:     iter_args(%[[ARG1:.*]] = %[[C1]], %[[ARG2:.*]] = %[[C2]]) -> (i32, f32) {
+//       CHECK:     %[[T1:.*]] = "test.test1"(%[[ARG1]]) : (i32) -> i32
+//       CHECK:     %[[T2:.*]] = "test.test2"(%[[ARG2]]) : (f32) -> f32
+//       CHECK:     scf.yield %[[T1]], %[[T2]] : i32, f32
+//       CHECK:     return %[[RES]]#0, %[[RES]]#1 : i32, f32
+
+// -----
+
+func.func @uplift_while(%arg0: i64, %arg1: i64, %arg2: i64) -> i64 {
+  %0 = scf.while (%arg3 = %arg0) : (i64) -> (i64) {
+    %1 = arith.cmpi slt, %arg3, %arg1 : i64
+    scf.condition(%1) %arg3 : i64
+  } do {
+  ^bb0(%arg3: i64):
+    "test.test1"(%arg3) : (i64) -> ()
+    %added = arith.addi %arg3, %arg2 : i64
+    "test.test2"(%added) : (i64) -> ()
+    scf.yield %added : i64
+  }
+  return %0 : i64
+}
+
+// CHECK-LABEL: func @uplift_while
+//  CHECK-SAME:     (%[[BEGIN:.*]]: i64, %[[END:.*]]: i64, %[[STEP:.*]]: i64) -> i64
+//       CHECK:     %[[C1:.*]] = arith.constant 1 : i64
+//       CHECK:     scf.for %[[I:.*]] = %[[BEGIN]] to %[[END]] step %[[STEP]] : i64 {
+//       CHECK:     "test.test1"(%[[I]]) : (i64) -> ()
+//       CHECK:     %[[INC:.*]] = arith.addi %[[I]], %[[STEP]] : i64
+//       CHECK:     "test.test2"(%[[INC]]) : (i64) -> ()
+//       CHECK:     %[[R1:.*]] = arith.subi %[[STEP]], %[[C1]] : i64
+//       CHECK:     %[[R2:.*]] = arith.subi %[[END]], %[[BEGIN]] : i64
+//       CHECK:     %[[R3:.*]] = arith.addi %[[R2]], %[[R1]] : i64
+//       CHECK:     %[[R4:.*]] = arith.divsi %[[R3]], %[[STEP]] : i64
+//       CHECK:     %[[R5:.*]] = arith.subi %[[R4]], %[[C1]] : i64
+//       CHECK:     %[[R6:.*]] = arith.muli %[[R5]], %[[STEP]] : i64
+//       CHECK:     %[[R7:.*]] = arith.addi %[[BEGIN]], %[[R6]] : i64
+//       CHECK:     return %[[R7]] : i64
diff --git a/mlir/test/lib/Dialect/SCF/CMakeLists.txt b/mlir/test/lib/Dialect/SCF/CMakeLists.txt
index d93bd5591518..792430cc84b6 100644
--- a/mlir/test/lib/Dialect/SCF/CMakeLists.txt
+++ b/mlir/test/lib/Dialect/SCF/CMakeLists.txt
@@ -4,6 +4,7 @@ add_mlir_library(MLIRSCFTestPasses
   TestLoopUnrolling.cpp
   TestSCFUtils.cpp
   TestSCFWrapInZeroTripCheck.cpp
+  TestUpliftWhileToFor.cpp
   TestWhileOpBuilder.cpp
 
   EXCLUDE_FROM_LIBMLIR
diff --git a/mlir/test/lib/Dialect/SCF/TestUpliftWhileToFor.cpp b/mlir/test/lib/Dialect/SCF/TestUpliftWhileToFor.cpp
new file mode 100644
index 000000000000..468bc0ca7848
--- /dev/null
+++ b/mlir/test/lib/Dialect/SCF/TestUpliftWhileToFor.cpp
@@ -0,0 +1,50 @@
+//===- TestUpliftWhileToFor.cpp - while to for loop uplifting test pass ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Pass to test transforms SCF.WhileOp's into SCF.ForOp's.
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/SCF/Transforms/Patterns.h"
+#include "mlir/IR/PatternMatch.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
+
+using namespace mlir;
+
+namespace {
+
+struct TestSCFUpliftWhileToFor
+    : public PassWrapper<TestSCFUpliftWhileToFor, OperationPass<void>> {
+  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestSCFUpliftWhileToFor)
+
+  StringRef getArgument() const final { return "test-scf-uplift-while-to-for"; }
+
+  StringRef getDescription() const final {
+    return "test scf while to for uplifting";
+  }
+
+  void runOnOperation() override {
+    Operation *op = getOperation();
+    MLIRContext *ctx = op->getContext();
+    RewritePatternSet patterns(ctx);
+    scf::populateUpliftWhileToForPatterns(patterns);
+    if (failed(applyPatternsAndFoldGreedily(op, std::move(patterns))))
+      signalPassFailure();
+  }
+};
+
+} // namespace
+
+namespace mlir {
+namespace test {
+void registerTestSCFUpliftWhileToFor() {
+  PassRegistration<TestSCFUpliftWhileToFor>();
+}
+} // namespace test
+} // namespace mlir
diff --git a/mlir/tools/mlir-opt/mlir-opt.cpp b/mlir/tools/mlir-opt/mlir-opt.cpp
index 6ce9f3041d6f..237ebeb166dc 100644
--- a/mlir/tools/mlir-opt/mlir-opt.cpp
+++ b/mlir/tools/mlir-opt/mlir-opt.cpp
@@ -130,6 +130,7 @@ void registerTestOneToNTypeConversionPass();
 void registerTestOpaqueLoc();
 void registerTestPadFusion();
 void registerTestRecursiveTypesPass();
+void registerTestSCFUpliftWhileToFor();
 void registerTestSCFUtilsPass();
 void registerTestSCFWhileOpBuilderPass();
 void registerTestSCFWrapInZeroTripCheckPasses();
@@ -258,6 +259,7 @@ void registerTestPasses() {
   mlir::test::registerTestOpaqueLoc();
   mlir::test::registerTestPadFusion();
   mlir::test::registerTestRecursiveTypesPass();
+  mlir::test::registerTestSCFUpliftWhileToFor();
   mlir::test::registerTestSCFUtilsPass();
   mlir::test::registerTestSCFWhileOpBuilderPass();
   mlir::test::registerTestSCFWrapInZeroTripCheckPasses();
-- 
GitLab


From 43b4e5b9bff1719a6d3628a3a805cfeb59adaf16 Mon Sep 17 00:00:00 2001
From: Utkarsh Saxena <usx@google.com>
Date: Mon, 15 Apr 2024 21:30:18 +0200
Subject: [PATCH 290/464] Use pushFullExprCleanup for deferred destroy (#88670)

Instead of directly pushing the `Destroy`, we should use
`pushFullExprCleanup` which handles conditional branches.
This fixes a backend crash due to
89ba7e183e6e2c64370ed1b963e54c06352211db.

Added minimized crash reproducer.
---
 clang/lib/CodeGen/CGDecl.cpp                  |  7 +++--
 .../CodeGenCXX/control-flow-in-stmt-expr.cpp  | 31 ++++++++++++++++++-
 2 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/clang/lib/CodeGen/CGDecl.cpp b/clang/lib/CodeGen/CGDecl.cpp
index 8bdafa7c569b..3f05ebb561da 100644
--- a/clang/lib/CodeGen/CGDecl.cpp
+++ b/clang/lib/CodeGen/CGDecl.cpp
@@ -2216,8 +2216,11 @@ void CodeGenFunction::pushDestroyAndDeferDeactivation(
 void CodeGenFunction::pushDestroyAndDeferDeactivation(
     CleanupKind cleanupKind, Address addr, QualType type, Destroyer *destroyer,
     bool useEHCleanupForArray) {
-  pushCleanupAndDeferDeactivation<DestroyObject>(
-      cleanupKind, addr, type, destroyer, useEHCleanupForArray);
+  llvm::Instruction *DominatingIP =
+      Builder.CreateFlagLoad(llvm::Constant::getNullValue(Int8PtrTy));
+  pushDestroy(cleanupKind, addr, type, destroyer, useEHCleanupForArray);
+  DeferredDeactivationCleanupStack.push_back(
+      {EHStack.stable_begin(), DominatingIP});
 }
 
 void CodeGenFunction::pushStackRestore(CleanupKind Kind, Address SPMem) {
diff --git a/clang/test/CodeGenCXX/control-flow-in-stmt-expr.cpp b/clang/test/CodeGenCXX/control-flow-in-stmt-expr.cpp
index ffde1bd6a724..95deee8bb1f1 100644
--- a/clang/test/CodeGenCXX/control-flow-in-stmt-expr.cpp
+++ b/clang/test/CodeGenCXX/control-flow-in-stmt-expr.cpp
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 --std=c++20 -triple x86_64-linux-gnu -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 --std=c++20 -fexceptions -triple x86_64-linux-gnu -emit-llvm %s -o - | FileCheck -check-prefixes=EH %s
+// RUN: %clang_cc1 --std=c++20 -triple x86_64-linux-gnu -emit-llvm %s -o - | FileCheck -check-prefixes=NOEH,CHECK %s
 
 struct Printy {
   Printy(const char *name) : name(name) {}
@@ -349,6 +350,34 @@ void NewArrayInit() {
   // CHECK-NEXT:    br label %return
 }
 
+void DestroyInConditionalCleanup() {
+  // EH-LABEL: DestroyInConditionalCleanupv()
+  // NOEH-LABEL: DestroyInConditionalCleanupv()
+  struct A {
+    A() {}
+    ~A() {}
+  };
+
+  struct Value {
+    Value(A) {}
+    ~Value() {}
+  };
+
+  struct V2 {
+    Value K;
+    Value V;
+  };
+  // Verify we use conditional cleanups.
+  (void)(foo() ? V2{A(), A()} : V2{A(), A()});
+  // NOEH:   cond.true:
+  // NOEH:      call void @_ZZ27DestroyInConditionalCleanupvEN1AC1Ev
+  // NOEH:      store ptr %{{.*}}, ptr %cond-cleanup.save
+
+  // EH:   cond.true:
+  // EH:        invoke void @_ZZ27DestroyInConditionalCleanupvEN1AC1Ev
+  // EH:        store ptr %{{.*}}, ptr %cond-cleanup.save
+}
+
 void ArrayInitWithContinue() {
   // CHECK-LABEL: @_Z21ArrayInitWithContinuev
   // Verify that we start to emit the array destructor.
-- 
GitLab


From b03dc7d16251bf249b67e4db545353551727f646 Mon Sep 17 00:00:00 2001
From: cor3ntin <corentinjabot@gmail.com>
Date: Mon, 15 Apr 2024 21:38:10 +0200
Subject: [PATCH 291/464] [Clang] Properly  set the value category of dependent
 unary operators (#88740)

This fixes an assertion in Expr::Classify when a
trying to deduce a dependent dereference operator.

Fixes #88329
---
 clang/docs/ReleaseNotes.rst                |  1 +
 clang/lib/Sema/SemaOverload.cpp            | 13 ++++++++++---
 clang/test/CXX/over/over.built/ast.cpp     |  4 ++--
 clang/test/SemaCXX/overloaded-operator.cpp | 10 ++++++++++
 4 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index de5fa99bff5d..dc108785f6cc 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -537,6 +537,7 @@ Bug Fixes to C++ Support
 - Clang now correctly tracks type dependence of by-value captures in lambdas with an explicit
   object parameter.
   Fixes (#GH70604), (#GH79754), (#GH84163), (#GH84425), (#GH86054), (#GH86398), and (#GH86399).
+- Fix a crash when deducing ``auto`` from an invalid dereference (#GH88329).
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index 675eb6ec05e7..227ef564ba3e 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -14395,9 +14395,16 @@ Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, UnaryOperatorKind Opc,
   ArrayRef<Expr *> ArgsArray(Args, NumArgs);
 
   if (Input->isTypeDependent()) {
+    ExprValueKind VK = ExprValueKind::VK_PRValue;
+    // [C++26][expr.unary.op][expr.pre.incr]
+    // The * operator yields an lvalue of type
+    // The pre/post increment operators yied an lvalue.
+    if (Opc == UO_PreDec || Opc == UO_PreInc || Opc == UO_Deref)
+      VK = VK_LValue;
+
     if (Fns.empty())
-      return UnaryOperator::Create(Context, Input, Opc, Context.DependentTy,
-                                   VK_PRValue, OK_Ordinary, OpLoc, false,
+      return UnaryOperator::Create(Context, Input, Opc, Context.DependentTy, VK,
+                                   OK_Ordinary, OpLoc, false,
                                    CurFPFeatureOverrides());
 
     CXXRecordDecl *NamingClass = nullptr; // lookup ignores member operators
@@ -14406,7 +14413,7 @@ Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, UnaryOperatorKind Opc,
     if (Fn.isInvalid())
       return ExprError();
     return CXXOperatorCallExpr::Create(Context, Op, Fn.get(), ArgsArray,
-                                       Context.DependentTy, VK_PRValue, OpLoc,
+                                       Context.DependentTy, VK, OpLoc,
                                        CurFPFeatureOverrides());
   }
 
diff --git a/clang/test/CXX/over/over.built/ast.cpp b/clang/test/CXX/over/over.built/ast.cpp
index f76606b1f986..56a63431269f 100644
--- a/clang/test/CXX/over/over.built/ast.cpp
+++ b/clang/test/CXX/over/over.built/ast.cpp
@@ -4,11 +4,11 @@ struct A{};
 
 template <typename T, typename U>
 auto Test(T* pt, U* pu) {
-  // CHECK: UnaryOperator {{.*}} '<dependent type>' prefix '*'
+  // CHECK: UnaryOperator {{.*}} '<dependent type>' lvalue prefix '*'
   // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
   (void)*pt;
 
-  // CHECK: UnaryOperator {{.*}} '<dependent type>' prefix '++'
+  // CHECK: UnaryOperator {{.*}} '<dependent type>' lvalue prefix '++'
   // CHECK-NEXT: DeclRefExpr {{.*}} 'T *' lvalue ParmVar {{.*}} 'pt' 'T *'
   (void)(++pt);
 
diff --git a/clang/test/SemaCXX/overloaded-operator.cpp b/clang/test/SemaCXX/overloaded-operator.cpp
index 49311625d7ab..cab21d67a002 100644
--- a/clang/test/SemaCXX/overloaded-operator.cpp
+++ b/clang/test/SemaCXX/overloaded-operator.cpp
@@ -682,3 +682,13 @@ namespace nw{
   }
 }
 #endif
+
+#if __cplusplus >= 201703L
+namespace GH88329 {
+
+template <auto T> struct A {};
+template <auto T> A<*T> operator *() { return {}; }
+// expected-error@-1 {{overloaded 'operator*' must have at least one parameter of class or enumeration type}}
+}
+
+#endif
-- 
GitLab


From 187c035c047616860d31fc399702538ad1f42755 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Mon, 15 Apr 2024 15:38:25 -0400
Subject: [PATCH 292/464] [gn] port dfafe3822b1

---
 llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn b/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn
index d533e79e6374..2b358f74ae6c 100644
--- a/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/compiler-rt/test/BUILD.gn
@@ -56,6 +56,7 @@ write_cmake_config("lit_common_configured") {
     "COMPILER_RT_INTERCEPT_LIBDISPATCH_PYBOOL=False",
     "COMPILER_RT_RESOLVED_LIBRARY_OUTPUT_DIR=" +
         rebase_path(crt_current_out_dir),
+    "COMPILER_RT_RESOLVED_OUTPUT_DIR=" + rebase_path(crt_current_out_dir),
     "COMPILER_RT_EMULATOR=",
     "COMPILER_RT_ASAN_SHADOW_SCALE=",
     "COMPILER_RT_MEMPROF_SHADOW_SCALE=",
-- 
GitLab


From 51bc8cdf7bbc98cd691fc480b333907e665b8eae Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 15 Apr 2024 12:45:02 -0700
Subject: [PATCH 293/464] [RISCV] Add missing space to RISCVInstrInfoZvk.td.
 NFC

---
 llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
index 575f9b41accb..aac7dc444a2d 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZvk.td
@@ -152,7 +152,7 @@ let Predicates = [HasStdExtZvknhaOrZvknhb], RVVConstraint = Sha2Constraint in {
   def VSHA2MS_VV : PALUVVNoVmTernary<0b101101, OPMVV, "vsha2ms.vv">;
 } // Predicates = [HasStdExtZvknhaOrZvknhb]
 
-let Predicates = [HasStdExtZvkned]in {
+let Predicates = [HasStdExtZvkned] in {
   defm VAESDF     : VAES_MV_V_S<0b101000, 0b101001, 0b00001, OPMVV, "vaesdf">;
   defm VAESDM     : VAES_MV_V_S<0b101000, 0b101001, 0b00000, OPMVV, "vaesdm">;
   defm VAESEF     : VAES_MV_V_S<0b101000, 0b101001, 0b00011, OPMVV, "vaesef">;
-- 
GitLab


From 52a88d3f5af6bed08a47f1c2351f320e0e1f4a88 Mon Sep 17 00:00:00 2001
From: Yijia Gu <yijiagu@google.com>
Date: Mon, 15 Apr 2024 12:47:47 -0700
Subject: [PATCH 294/464] [mlir] fix llvm:support dependency error

---
 utils/bazel/llvm-project-overlay/mlir/BUILD.bazel | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 522e3efe4e0a..653c4bd30600 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -6617,7 +6617,7 @@ cc_library(
         ":PolynomialAttributesIncGen",
         ":PolynomialIncGen",
         ":Support",
-        "//third_party/llvm/llvm-project/llvm:Support",
+        "//llvm:Support",
     ],
 )
 
-- 
GitLab


From faad4e3fa81d10b91a9128ca84abfe381ac7c2e0 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Mon, 15 Apr 2024 15:03:37 -0500
Subject: [PATCH 295/464] [Libomptarget] Split PowerPC into separate triples
 (#88773)

Summary:
The previous patch mistakenly merged these when they indeed need to be
treated like separate triples because it's what's passed to the test
suite.
---
 .../libomptarget/plugins-nextgen/host/CMakeLists.txt   | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt
index ccbf7d033fd6..c1493d293d30 100644
--- a/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt
+++ b/openmp/libomptarget/plugins-nextgen/host/CMakeLists.txt
@@ -51,8 +51,14 @@ else()
 endif()
 
 # Define the target specific triples and ELF machine values.
-if(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le$" OR
-   CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64$")
+if(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le$")
+  target_compile_definitions(omptarget.rtl.${machine} PRIVATE TARGET_ELF_ID=EM_PPC64)
+  target_compile_definitions(omptarget.rtl.${machine} PRIVATE
+      LIBOMPTARGET_NEXTGEN_GENERIC_PLUGIN_TRIPLE="powerpc64le-ibm-linux-gnu")
+  list(APPEND LIBOMPTARGET_SYSTEM_TARGETS 
+       "powerpc64le-ibm-linux-gnu" "powerpc64le-ibm-linux-gnu-LTO")
+  set(LIBOMPTARGET_SYSTEM_TARGETS "${LIBOMPTARGET_SYSTEM_TARGETS}" PARENT_SCOPE)
+elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64$")
   target_compile_definitions(omptarget.rtl.${machine} PRIVATE TARGET_ELF_ID=EM_PPC64)
   target_compile_definitions(omptarget.rtl.${machine} PRIVATE
       LIBOMPTARGET_NEXTGEN_GENERIC_PLUGIN_TRIPLE="powerpc64-ibm-linux-gnu")
-- 
GitLab


From fb9c7dd6f1f3463b539d1f62f6ec7b5c7e061cf8 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 15 Apr 2024 13:04:14 -0700
Subject: [PATCH 296/464] [RISCV] Minor cleanups to
 RISCVISAInfo::parseArchString. NFC

Remove unneeded an unneeded variable and rename another.
---
 llvm/lib/Support/RISCVISAInfo.cpp | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Support/RISCVISAInfo.cpp b/llvm/lib/Support/RISCVISAInfo.cpp
index 7a19d24d1ff4..cbdc64bc7a97 100644
--- a/llvm/lib/Support/RISCVISAInfo.cpp
+++ b/llvm/lib/Support/RISCVISAInfo.cpp
@@ -935,7 +935,6 @@ RISCVISAInfo::parseArchString(StringRef Arch, bool EnableExperimentalExtension,
 
   // The canonical order specified in ISA manual.
   // Ref: Table 22.1 in RISC-V User-Level ISA V2.2
-  StringRef StdExts = AllStdExts;
   char Baseline = Arch[4];
 
   // First letter should be 'e', 'i' or 'g'.
@@ -951,7 +950,6 @@ RISCVISAInfo::parseArchString(StringRef Arch, bool EnableExperimentalExtension,
     if (Arch.size() > 5 && isDigit(Arch[5]))
       return createStringError(errc::invalid_argument,
                                "version not supported for 'g'");
-    StdExts = StdExts.drop_front(4);
     break;
   }
 
@@ -1001,11 +999,11 @@ RISCVISAInfo::parseArchString(StringRef Arch, bool EnableExperimentalExtension,
   Exts = Exts.drop_front(ConsumeLength);
   Exts.consume_front("_");
 
-  std::vector<std::string> SplittedExts;
-  if (auto E = splitExtsByUnderscore(Exts, SplittedExts))
+  std::vector<std::string> SplitExts;
+  if (auto E = splitExtsByUnderscore(Exts, SplitExts))
     return std::move(E);
 
-  for (auto &Ext : SplittedExts) {
+  for (auto &Ext : SplitExts) {
     StringRef CurrExt = Ext;
     while (!CurrExt.empty()) {
       if (AllStdExts.contains(CurrExt.front())) {
-- 
GitLab


From 9e485f48228114ab6c34151503740ea15cddb351 Mon Sep 17 00:00:00 2001
From: cmtice <cmtice@google.com>
Date: Mon, 15 Apr 2024 13:11:24 -0700
Subject: [PATCH 297/464] [DWARF] Add accessor for NameIndex Offsets field.
 (#88783)

Add an accessor function to DWARFDebugNames::NameIndex to allow other
classes to access the Offsets field within NameIndex. This will be used
by LLD to create a merged .debug_names index
(https://github.com/llvm/llvm-project/pull/86508)
---
 llvm/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h
index 9543b78ea613..0d447a78cdc6 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h
@@ -608,6 +608,9 @@ public:
     /// Returns Hdr field
     Header getHeader() const { return Hdr; }
 
+    /// Returns Offsets field
+    DWARFDebugNamesOffsets getOffsets() const { return Offsets; }
+
     /// Reads offset of compilation unit CU. CU is 0-based.
     uint64_t getCUOffset(uint32_t CU) const;
     uint32_t getCUCount() const { return Hdr.CompUnitCount; }
-- 
GitLab


From 071ac0ae3029594167d2bb4d28859bdc36367034 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Mon, 15 Apr 2024 13:18:26 -0700
Subject: [PATCH 298/464] [lldb] Skip lldb-server unit tests when building with
 ASan

The lldb-server unit tests are timing out on GreenDragon and swift-ci.
We haven't been able to reproduce this locally or figure out why. We've
made several attempts to address potential issues, but we've reached a
point where we need to skip them to get signal out of this bot again.
---
 lldb/unittests/tools/CMakeLists.txt | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lldb/unittests/tools/CMakeLists.txt b/lldb/unittests/tools/CMakeLists.txt
index 055fc6e6f5df..42b0c25dd1fc 100644
--- a/lldb/unittests/tools/CMakeLists.txt
+++ b/lldb/unittests/tools/CMakeLists.txt
@@ -1,3 +1,5 @@
 if(LLDB_TOOL_LLDB_SERVER_BUILD)
-  add_subdirectory(lldb-server)
+  if (NOT LLVM_USE_SANITIZER MATCHES ".*Address.*")
+    add_subdirectory(lldb-server)
+  endif()
 endif()
-- 
GitLab


From 0287a5cc4e2a5ded1ae2e4079f91052e6a6b8d9b Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Mon, 15 Apr 2024 15:28:05 -0500
Subject: [PATCH 299/464] [OpenMP] Remove 'minncta' attributes from NVPTX
 kernels (#88398)

Summary:
Currently we treat this attribute as a minimum number for the amount of
blocks scheduled on the kernel. However, the doucmentation states that
this applies to CTA's mapped onto a *single* SM. Currently we just set
it to the total number of blocks, which will almost always result in a
warning that the value is out of range and will be ignored. We don't
have a good way to automatically know how many CTAs can be put on a
single SM nor if we should do this, so we should probably leave this up
to users manually adding it.


https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#performance-tuning-directives-minnctapersm
---
 clang/test/OpenMP/ompx_attributes_codegen.cpp | 3 +--
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     | 4 +---
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/clang/test/OpenMP/ompx_attributes_codegen.cpp b/clang/test/OpenMP/ompx_attributes_codegen.cpp
index 6735972c6b10..87eb2913537b 100644
--- a/clang/test/OpenMP/ompx_attributes_codegen.cpp
+++ b/clang/test/OpenMP/ompx_attributes_codegen.cpp
@@ -36,6 +36,5 @@ void func() {
 // NVIDIA: "omp_target_thread_limit"="45"
 // NVIDIA: "omp_target_thread_limit"="17"
 // NVIDIA: !{ptr @__omp_offloading[[HASH1:.*]]_l16, !"maxntidx", i32 20}
-// NVIDIA: !{ptr @__omp_offloading[[HASH2:.*]]_l18, !"minctasm", i32 90}
-// NVIDIA: !{ptr @__omp_offloading[[HASH2]]_l18, !"maxntidx", i32 45}
+// NVIDIA: !{ptr @__omp_offloading[[HASH2:.*]]_l18, !"maxntidx", i32 45}
 // NVIDIA: !{ptr @__omp_offloading[[HASH3:.*]]_l20, !"maxntidx", i32 17}
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 7fd8474c2ec8..4d2d352f7520 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -4786,11 +4786,9 @@ OpenMPIRBuilder::readTeamBoundsForKernel(const Triple &, Function &Kernel) {
 
 void OpenMPIRBuilder::writeTeamsForKernel(const Triple &T, Function &Kernel,
                                           int32_t LB, int32_t UB) {
-  if (T.isNVPTX()) {
+  if (T.isNVPTX())
     if (UB > 0)
       updateNVPTXMetadata(Kernel, "maxclusterrank", UB, true);
-    updateNVPTXMetadata(Kernel, "minctasm", LB, false);
-  }
   if (T.isAMDGPU())
     Kernel.addFnAttr("amdgpu-max-num-workgroups", llvm::utostr(LB) + ",1,1");
 
-- 
GitLab


From 7eb5022cbb11fdf5e74d707b4c93d3669a46eccf Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12@gmail.com>
Date: Mon, 15 Apr 2024 21:45:00 +0100
Subject: [PATCH 300/464] [lldb][NFC] Turn ChildCacheState into an unscoped
 enum (#88703)

Done for consistency with the rest of the enums in
`lldb-enumerations.h`. See
https://github.com/llvm/llvm-project/pull/80167#issuecomment-2043721298
---
 lldb/include/lldb/lldb-enumerations.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lldb/include/lldb/lldb-enumerations.h b/lldb/include/lldb/lldb-enumerations.h
index f3b07ea6d203..15e458571860 100644
--- a/lldb/include/lldb/lldb-enumerations.h
+++ b/lldb/include/lldb/lldb-enumerations.h
@@ -1310,7 +1310,7 @@ enum CompletionType {
 
 /// Specifies if children need to be re-computed
 /// after a call to \ref SyntheticChildrenFrontEnd::Update.
-enum class ChildCacheState {
+enum ChildCacheState {
   eRefetch = 0, ///< Children need to be recomputed dynamically.
 
   eReuse = 1, ///< Children did not change and don't need to be recomputed;
-- 
GitLab


From b8c16db938327efdabc63f21fdc770069ac4406b Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12@gmail.com>
Date: Mon, 15 Apr 2024 21:45:25 +0100
Subject: [PATCH 301/464] [lldb][TypeSystemClang][NFCI] Use
 LangOptions::setLangDefaults when creating new LangOptions (#88721)

This logic was originally copied from `CompilerInstance::parseLangArgs`.
Since then, the `CompilerInstance` uses `LangOptions::setLangDefaults`
to set up new `LangOptions` instances. In our case, we only ever passed
`Language::ObjCXX` into LLDB's `ParseLangArgs`, so most of this function
was dead code.

This patch replaces the duplicated logic with a call to
`LangOptions::setLangDefaults`.
---
 .../TypeSystem/Clang/TypeSystemClang.cpp      | 79 ++-----------------
 1 file changed, 6 insertions(+), 73 deletions(-)

diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
index 44bd02bd4b36..be0ddb06f82c 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
@@ -459,85 +459,19 @@ TypeSystemClang::ConvertAccessTypeToAccessSpecifier(AccessType access) {
   return AS_none;
 }
 
-static void ParseLangArgs(LangOptions &Opts, InputKind IK, const char *triple) {
+static void ParseLangArgs(LangOptions &Opts, ArchSpec arch) {
   // FIXME: Cleanup per-file based stuff.
 
-  // Set some properties which depend solely on the input kind; it would be
-  // nice to move these to the language standard, and have the driver resolve
-  // the input kind + language standard.
-  if (IK.getLanguage() == clang::Language::Asm) {
-    Opts.AsmPreprocessor = 1;
-  } else if (IK.isObjectiveC()) {
-    Opts.ObjC = 1;
-  }
-
-  LangStandard::Kind LangStd = LangStandard::lang_unspecified;
-
-  if (LangStd == LangStandard::lang_unspecified) {
-    // Based on the base language, pick one.
-    switch (IK.getLanguage()) {
-    case clang::Language::Unknown:
-    case clang::Language::CIR:
-    case clang::Language::LLVM_IR:
-    case clang::Language::RenderScript:
-      llvm_unreachable("Invalid input kind!");
-    case clang::Language::OpenCL:
-      LangStd = LangStandard::lang_opencl10;
-      break;
-    case clang::Language::OpenCLCXX:
-      LangStd = LangStandard::lang_openclcpp10;
-      break;
-    case clang::Language::Asm:
-    case clang::Language::C:
-    case clang::Language::ObjC:
-      LangStd = LangStandard::lang_gnu99;
-      break;
-    case clang::Language::CXX:
-    case clang::Language::ObjCXX:
-      LangStd = LangStandard::lang_gnucxx98;
-      break;
-    case clang::Language::CUDA:
-    case clang::Language::HIP:
-      LangStd = LangStandard::lang_gnucxx17;
-      break;
-    case clang::Language::HLSL:
-      LangStd = LangStandard::lang_hlsl;
-      break;
-    }
-  }
-
-  const LangStandard &Std = LangStandard::getLangStandardForKind(LangStd);
-  Opts.LineComment = Std.hasLineComments();
-  Opts.C99 = Std.isC99();
-  Opts.CPlusPlus = Std.isCPlusPlus();
-  Opts.CPlusPlus11 = Std.isCPlusPlus11();
-  Opts.CPlusPlus14 = Std.isCPlusPlus14();
-  Opts.CPlusPlus17 = Std.isCPlusPlus17();
-  Opts.CPlusPlus20 = Std.isCPlusPlus20();
-  Opts.Digraphs = Std.hasDigraphs();
-  Opts.GNUMode = Std.isGNUMode();
-  Opts.GNUInline = !Std.isC99();
-  Opts.HexFloats = Std.hasHexFloats();
-
-  Opts.WChar = true;
-
-  // OpenCL has some additional defaults.
-  if (LangStd == LangStandard::lang_opencl10) {
-    Opts.OpenCL = 1;
-    Opts.AltiVec = 1;
-    Opts.CXXOperatorNames = 1;
-    Opts.setLaxVectorConversions(LangOptions::LaxVectorConversionKind::All);
-  }
-
-  // OpenCL and C++ both have bool, true, false keywords.
-  Opts.Bool = Opts.OpenCL || Opts.CPlusPlus;
+  std::vector<std::string> Includes;
+  LangOptions::setLangDefaults(Opts, clang::Language::ObjCXX, arch.GetTriple(),
+                               Includes, clang::LangStandard::lang_gnucxx98);
 
   Opts.setValueVisibilityMode(DefaultVisibility);
 
   // Mimicing gcc's behavior, trigraphs are only enabled if -trigraphs is
   // specified, or -std is set to a conforming mode.
   Opts.Trigraphs = !Opts.GNUMode;
-  Opts.CharIsSigned = ArchSpec(triple).CharIsSignedByDefault();
+  Opts.CharIsSigned = arch.CharIsSignedByDefault();
   Opts.OptimizeSize = 0;
 
   // FIXME: Eliminate this dependency.
@@ -727,8 +661,7 @@ void TypeSystemClang::CreateASTContext() {
   m_ast_owned = true;
 
   m_language_options_up = std::make_unique<LangOptions>();
-  ParseLangArgs(*m_language_options_up, clang::Language::ObjCXX,
-                GetTargetTriple());
+  ParseLangArgs(*m_language_options_up, ArchSpec(GetTargetTriple()));
 
   m_identifier_table_up =
       std::make_unique<IdentifierTable>(*m_language_options_up, nullptr);
-- 
GitLab


From 905d2ecbb63f4087b50d633b012c9915ab3a6829 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12@gmail.com>
Date: Mon, 15 Apr 2024 21:45:41 +0100
Subject: [PATCH 302/464] [lldb][ClangExpressionDeclMap][NFC] Remove unused
 NameSearchContext::m_found_function (#88724)

This member was never actually used, ever since its introduction in
`ca4e0fd7e63b90e6f68044af47248c64f250ee8f`.
---
 .../Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp | 4 ----
 .../source/Plugins/ExpressionParser/Clang/NameSearchContext.h | 1 -
 2 files changed, 5 deletions(-)

diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp
index 2d306b42760b..31f6447d66f6 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp
@@ -1049,7 +1049,6 @@ void ClangExpressionDeclMap::LookupInModulesDeclVendor(
     context.AddNamedDecl(copied_function);
 
     context.m_found_function_with_type_info = true;
-    context.m_found_function = true;
   } else if (auto copied_var = dyn_cast<clang::VarDecl>(copied_decl)) {
     context.AddNamedDecl(copied_var);
     context.m_found_variable = true;
@@ -1299,7 +1298,6 @@ void ClangExpressionDeclMap::LookupFunction(
 
         AddOneFunction(context, sym_ctx.function, nullptr);
         context.m_found_function_with_type_info = true;
-        context.m_found_function = true;
       } else if (sym_ctx.symbol) {
         Symbol *symbol = sym_ctx.symbol;
         if (target && symbol->GetType() == eSymbolTypeReExported) {
@@ -1331,10 +1329,8 @@ void ClangExpressionDeclMap::LookupFunction(
     if (!context.m_found_function_with_type_info) {
       if (extern_symbol) {
         AddOneFunction(context, nullptr, extern_symbol);
-        context.m_found_function = true;
       } else if (non_extern_symbol) {
         AddOneFunction(context, nullptr, non_extern_symbol);
-        context.m_found_function = true;
       }
     }
   }
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/NameSearchContext.h b/lldb/source/Plugins/ExpressionParser/Clang/NameSearchContext.h
index dc8621dd6aba..9a3320636081 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/NameSearchContext.h
+++ b/lldb/source/Plugins/ExpressionParser/Clang/NameSearchContext.h
@@ -41,7 +41,6 @@ struct NameSearchContext {
 
   bool m_found_variable = false;
   bool m_found_function_with_type_info = false;
-  bool m_found_function = false;
   bool m_found_local_vars_nsp = false;
   bool m_found_type = false;
 
-- 
GitLab


From 69d861e1320119e9a02907155ca626b1db90ad93 Mon Sep 17 00:00:00 2001
From: Nathan Sidwell <nathan@acm.org>
Date: Mon, 15 Apr 2024 16:55:58 -0400
Subject: [PATCH 303/464] [clang] Move tailclipping to bitfield allocation
 (#87090)

Move bitfield access clipping to bitfield access computation.
---
 clang/lib/CodeGen/CGRecordLayoutBuilder.cpp | 84 +++++++++++----------
 clang/test/CodeGen/bitfield-access-unit.c   | 18 +++++
 2 files changed, 62 insertions(+), 40 deletions(-)

diff --git a/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp b/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp
index 634a55fec518..868b1ab98e04 100644
--- a/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp
+++ b/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp
@@ -41,10 +41,11 @@ namespace {
 ///   contains enough information to determine where the runs break.  Microsoft
 ///   and Itanium follow different rules and use different codepaths.
 /// * It is desired that, when possible, bitfields use the appropriate iN type
-///   when lowered to llvm types.  For example unsigned x : 24 gets lowered to
+///   when lowered to llvm types. For example unsigned x : 24 gets lowered to
 ///   i24.  This isn't always possible because i24 has storage size of 32 bit
-///   and if it is possible to use that extra byte of padding we must use
-///   [i8 x 3] instead of i24.  The function clipTailPadding does this.
+///   and if it is possible to use that extra byte of padding we must use [i8 x
+///   3] instead of i24. This is computed when accumulating bitfields in
+///   accumulateBitfields.
 ///   C++ examples that require clipping:
 ///   struct { int a : 24; char b; }; // a must be clipped, b goes at offset 3
 ///   struct A { int a : 24; ~A(); }; // a must be clipped because:
@@ -62,11 +63,7 @@ namespace {
 ///   that the tail padding is not used in the complete class.) However,
 ///   because LLVM reads from the complete type it can generate incorrect code
 ///   if we do not clip the tail padding off of the bitfield in the complete
-///   layout.  This introduces a somewhat awkward extra unnecessary clip stage.
-///   The location of the clip is stored internally as a sentinel of type
-///   SCISSOR.  If LLVM were updated to read base types (which it probably
-///   should because locations of things such as VBases are bogus in the llvm
-///   type anyway) then we could eliminate the SCISSOR.
+///   layout.
 /// * Itanium allows nearly empty primary virtual bases.  These bases don't get
 ///   get their own storage because they're laid out as part of another base
 ///   or at the beginning of the structure.  Determining if a VBase actually
@@ -200,9 +197,7 @@ struct CGRecordLowering {
                      const CXXRecordDecl *Query) const;
   void calculateZeroInit();
   CharUnits calculateTailClippingOffset(bool isNonVirtualBaseType) const;
-  /// Lowers bitfield storage types to I8 arrays for bitfields with tail
-  /// padding that is or can potentially be used.
-  void clipTailPadding();
+  void checkBitfieldClipping() const;
   /// Determines if we need a packed llvm struct.
   void determinePacked(bool NVBaseType);
   /// Inserts padding everywhere it's needed.
@@ -305,7 +300,7 @@ void CGRecordLowering::lower(bool NVBaseType) {
   }
   llvm::stable_sort(Members);
   Members.push_back(StorageInfo(Size, getIntNType(8)));
-  clipTailPadding();
+  checkBitfieldClipping();
   determinePacked(NVBaseType);
   insertPadding();
   Members.pop_back();
@@ -531,6 +526,7 @@ CGRecordLowering::accumulateBitFields(bool isNonVirtualBaseType,
   // available padding characters.
   RecordDecl::field_iterator BestEnd = Begin;
   CharUnits BestEndOffset;
+  bool BestClipped; // Whether the representation must be in a byte array.
 
   for (;;) {
     // AtAlignedBoundary is true iff Field is the (potential) start of a new
@@ -593,10 +589,9 @@ CGRecordLowering::accumulateBitFields(bool isNonVirtualBaseType,
         // this is the best seen so far.
         BestEnd = Field;
         BestEndOffset = BeginOffset + AccessSize;
-        if (Types.getCodeGenOpts().FineGrainedBitfieldAccesses)
-          // Fine-grained access, so no merging of spans.
-          InstallBest = true;
-        else if (!BitSizeSinceBegin)
+        // Assume clipped until proven not below.
+        BestClipped = true;
+        if (!BitSizeSinceBegin)
           // A zero-sized initial span -- this will install nothing and reset
           // for another.
           InstallBest = true;
@@ -624,6 +619,12 @@ CGRecordLowering::accumulateBitFields(bool isNonVirtualBaseType,
             // The access unit is not at a naturally aligned offset within the
             // structure.
             InstallBest = true;
+
+          if (InstallBest && BestEnd == Field)
+            // We're installing the first span, whose clipping was presumed
+            // above. Compute it correctly.
+            if (getSize(Type) == AccessSize)
+              BestClipped = false;
         }
 
         if (!InstallBest) {
@@ -656,11 +657,15 @@ CGRecordLowering::accumulateBitFields(bool isNonVirtualBaseType,
             // access unit.
             BestEndOffset = BeginOffset + TypeSize;
             BestEnd = Field;
+            BestClipped = false;
           }
 
           if (Barrier)
             // The next field is a barrier that we cannot merge across.
             InstallBest = true;
+          else if (Types.getCodeGenOpts().FineGrainedBitfieldAccesses)
+            // Fine-grained access, so no merging of spans.
+            InstallBest = true;
           else
             // Otherwise, we're not installing. Update the bit size
             // of the current span to go all the way to LimitOffset, which is
@@ -679,7 +684,17 @@ CGRecordLowering::accumulateBitFields(bool isNonVirtualBaseType,
         // Add the storage member for the access unit to the record. The
         // bitfields get the offset of their storage but come afterward and
         // remain there after a stable sort.
-        llvm::Type *Type = getIntNType(Context.toBits(AccessSize));
+        llvm::Type *Type;
+        if (BestClipped) {
+          assert(getSize(getIntNType(Context.toBits(AccessSize))) >
+                     AccessSize &&
+                 "Clipped access need not be clipped");
+          Type = getByteArrayType(AccessSize);
+        } else {
+          Type = getIntNType(Context.toBits(AccessSize));
+          assert(getSize(Type) == AccessSize &&
+                 "Unclipped access must be clipped");
+        }
         Members.push_back(StorageInfo(BeginOffset, Type));
         for (; Begin != BestEnd; ++Begin)
           if (!Begin->isZeroLengthBitField(Context))
@@ -934,32 +949,21 @@ void CGRecordLowering::calculateZeroInit() {
   }
 }
 
-void CGRecordLowering::clipTailPadding() {
-  std::vector<MemberInfo>::iterator Prior = Members.begin();
-  CharUnits Tail = getSize(Prior->Data);
-  for (std::vector<MemberInfo>::iterator Member = Prior + 1,
-                                         MemberEnd = Members.end();
-       Member != MemberEnd; ++Member) {
+// Verify accumulateBitfields computed the correct storage representations.
+void CGRecordLowering::checkBitfieldClipping() const {
+#ifndef NDEBUG
+  auto Tail = CharUnits::Zero();
+  for (const auto &M : Members) {
     // Only members with data and the scissor can cut into tail padding.
-    if (!Member->Data && Member->Kind != MemberInfo::Scissor)
+    if (!M.Data && M.Kind != MemberInfo::Scissor)
       continue;
-    if (Member->Offset < Tail) {
-      assert(Prior->Kind == MemberInfo::Field &&
-             "Only storage fields have tail padding!");
-      if (!Prior->FD || Prior->FD->isBitField())
-        Prior->Data = getByteArrayType(bitsToCharUnits(llvm::alignTo(
-            cast<llvm::IntegerType>(Prior->Data)->getIntegerBitWidth(), 8)));
-      else {
-        assert(Prior->FD->hasAttr<NoUniqueAddressAttr>() &&
-               "should not have reused this field's tail padding");
-        Prior->Data = getByteArrayType(
-            Context.getTypeInfoDataSizeInChars(Prior->FD->getType()).Width);
-      }
-    }
-    if (Member->Data)
-      Prior = Member;
-    Tail = Prior->Offset + getSize(Prior->Data);
+
+    assert(M.Offset >= Tail && "Bitfield access unit is not clipped");
+    Tail = M.Offset;
+    if (M.Data)
+      Tail += getSize(M.Data);
   }
+#endif
 }
 
 void CGRecordLowering::determinePacked(bool NVBaseType) {
diff --git a/clang/test/CodeGen/bitfield-access-unit.c b/clang/test/CodeGen/bitfield-access-unit.c
index 1aed2e7202fc..d0553c5183ee 100644
--- a/clang/test/CodeGen/bitfield-access-unit.c
+++ b/clang/test/CodeGen/bitfield-access-unit.c
@@ -222,6 +222,24 @@ struct G {
 // LAYOUT-DWN32-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:7 IsSigned:1 StorageSize:8 StorageOffset:4
 // CHECK-NEXT: ]>
 
+struct __attribute__((aligned(8))) H {
+  char a;
+  unsigned b : 24; // on expensive alignment we want this to stay 24
+  unsigned c __attribute__((aligned(8))); // Think 'long long' or lp64 ptr
+} h;
+// CHECK-LABEL: LLVMType:%struct.H =
+// LAYOUT-FLEX-SAME: type <{ i8, i32, [3 x i8], i32, [4 x i8] }>
+// LAYOUT-STRICT-SAME: type { i8, [3 x i8], [4 x i8], i32, [4 x i8] }
+// LAYOUT-DWN32-FLEX-SAME: type <{ i8, i32, [3 x i8], i32, [4 x i8] }>
+// LAYOUT-DWN32-STRICT-SAME: type { i8, [3 x i8], [4 x i8], i32, [4 x i8] }
+// CHECK: BitFields:[
+// LAYOUT-FLEX-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:24 IsSigned:0 StorageSize:32 StorageOffset:1
+// LAYOUT-STRICT-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:24 IsSigned:0 StorageSize:24 StorageOffset:1
+
+// LAYOUT-DWN32-FLEX-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:24 IsSigned:0 StorageSize:32 StorageOffset:1
+// LAYOUT-DWN32-STRICT-NEXT: <CGBitFieldInfo Offset:{{[0-9]+}} Size:24 IsSigned:0 StorageSize:24 StorageOffset:1
+// CHECK-NEXT: ]>
+
 #if _LP64
 struct A64 {
   int a : 16;
-- 
GitLab


From c303945409a740c8fdb4103a4f21df55187aa84f Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Mon, 15 Apr 2024 17:01:28 -0400
Subject: [PATCH 304/464] [gn] port e356f68 more

---
 llvm/utils/gn/secondary/libcxx/include/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 4383f1d6d18f..865a79b63cd8 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -988,6 +988,7 @@ if (current_toolchain == default_toolchain) {
       "errno.h",
       "exception",
       "execution",
+      "expected",
       "experimental/__config",
       "experimental/__simd/aligned_tag.h",
       "experimental/__simd/declaration.h",
-- 
GitLab


From a855eea7fe86ef09a87f6251b3b711b821ae32bf Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Mon, 15 Apr 2024 14:05:01 -0700
Subject: [PATCH 305/464] [lldb] Fix the standalone Xcode build after #88317

In #88317, the clang resource headers was converted to an interface
library. Update LLDB and fix the Xcode standalone build. Thanks Evan for
the help!
---
 lldb/cmake/modules/LLDBFramework.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lldb/cmake/modules/LLDBFramework.cmake b/lldb/cmake/modules/LLDBFramework.cmake
index 81fc596ef424..f915839f6b45 100644
--- a/lldb/cmake/modules/LLDBFramework.cmake
+++ b/lldb/cmake/modules/LLDBFramework.cmake
@@ -119,7 +119,7 @@ add_custom_command(TARGET liblldb POST_BUILD
 if(NOT APPLE_EMBEDDED)
   if (TARGET clang-resource-headers)
     add_dependencies(liblldb clang-resource-headers)
-    set(clang_resource_headers_dir $<TARGET_PROPERTY:clang-resource-headers,RUNTIME_OUTPUT_DIRECTORY>)
+    set(clang_resource_headers_dir $<TARGET_PROPERTY:clang-resource-headers,INTERFACE_INCLUDE_DIRECTORIES>)
   else()
     set(clang_resource_headers_dir ${LLDB_EXTERNAL_CLANG_RESOURCE_DIR}/include)
     if(NOT EXISTS ${clang_resource_headers_dir})
-- 
GitLab


From 21d177096f84c38cf434c21bd3ff0dbd2ca163d0 Mon Sep 17 00:00:00 2001
From: Kai Nacke <kai.peter.nacke@ibm.com>
Date: Mon, 15 Apr 2024 17:12:25 -0400
Subject: [PATCH 306/464] [NFC] Refactor looping over recomputeLiveIns into
 function (#88040)

https://github.com/llvm/llvm-project/pull/79940 put calls to
recomputeLiveIns into
a loop, to repeatedly call the function until the computation converges.
However,
this repeats a lot of code. This changes moves the loop into a function
to simplify
the handling.

Note that this changes the order in which recomputeLiveIns is called.
For example,

```
  bool anyChange = false;
  do {
    anyChange = recomputeLiveIns(*ExitMBB) || recomputeLiveIns(*LoopMBB);
  } while (anyChange);
```

only begins to recompute the live-ins for LoopMBB after the computation
for ExitMBB
has converged. With this change, all basic blocks have a recomputation
of the live-ins
for each loop iteration. This can result in less or more calls,
depending on the
situation.
---
 llvm/include/llvm/CodeGen/LivePhysRegs.h       | 18 ++++++++++++++++++
 llvm/lib/CodeGen/BranchFolding.cpp             |  8 ++------
 .../Target/AArch64/AArch64FrameLowering.cpp    |  5 +----
 llvm/lib/Target/AArch64/AArch64InstrInfo.cpp   | 11 ++---------
 llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp    |  8 +-------
 .../PowerPC/PPCExpandAtomicPseudoInsts.cpp     | 11 ++---------
 llvm/lib/Target/PowerPC/PPCFrameLowering.cpp   | 11 ++---------
 .../Target/SystemZ/SystemZFrameLowering.cpp    | 10 ++--------
 llvm/lib/Target/X86/X86FrameLowering.cpp       | 11 ++---------
 9 files changed, 32 insertions(+), 61 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/LivePhysRegs.h b/llvm/include/llvm/CodeGen/LivePhysRegs.h
index 1d40b1cbb0ea..9574a6f0c7c0 100644
--- a/llvm/include/llvm/CodeGen/LivePhysRegs.h
+++ b/llvm/include/llvm/CodeGen/LivePhysRegs.h
@@ -39,6 +39,8 @@
 
 namespace llvm {
 
+template <typename T> class ArrayRef;
+
 class MachineInstr;
 class MachineFunction;
 class MachineOperand;
@@ -207,6 +209,22 @@ static inline bool recomputeLiveIns(MachineBasicBlock &MBB) {
   return oldLiveIns != newLiveIns;
 }
 
+/// Convenience function for recomputing live-in's for a set of MBBs until the
+/// computation converges.
+inline void fullyRecomputeLiveIns(ArrayRef<MachineBasicBlock *> MBBs) {
+  MachineBasicBlock *const *Data = MBBs.data();
+  const size_t Len = MBBs.size();
+  while (true) {
+    bool AnyChange = false;
+    for (size_t I = 0; I < Len; ++I)
+      if (recomputeLiveIns(*Data[I]))
+        AnyChange = true;
+    if (!AnyChange)
+      return;
+  }
+}
+
+
 } // end namespace llvm
 
 #endif // LLVM_CODEGEN_LIVEPHYSREGS_H
diff --git a/llvm/lib/CodeGen/BranchFolding.cpp b/llvm/lib/CodeGen/BranchFolding.cpp
index ecf7bc30913f..55aa1d438b2a 100644
--- a/llvm/lib/CodeGen/BranchFolding.cpp
+++ b/llvm/lib/CodeGen/BranchFolding.cpp
@@ -2047,12 +2047,8 @@ bool BranchFolder::HoistCommonCodeInSuccs(MachineBasicBlock *MBB) {
   MBB->splice(Loc, TBB, TBB->begin(), TIB);
   FBB->erase(FBB->begin(), FIB);
 
-  if (UpdateLiveIns) {
-    bool anyChange = false;
-    do {
-      anyChange = recomputeLiveIns(*TBB) || recomputeLiveIns(*FBB);
-    } while (anyChange);
-  }
+  if (UpdateLiveIns)
+    fullyRecomputeLiveIns({TBB, FBB});
 
   ++NumHoist;
   return true;
diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index 5cc612e89162..419c141121c3 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -4325,10 +4325,7 @@ AArch64FrameLowering::inlineStackProbeLoopExactMultiple(
   ExitMBB->transferSuccessorsAndUpdatePHIs(&MBB);
   MBB.addSuccessor(LoopMBB);
   // Update liveins.
-  bool anyChange = false;
-  do {
-    anyChange = recomputeLiveIns(*ExitMBB) || recomputeLiveIns(*LoopMBB);
-  } while (anyChange);
+  fullyRecomputeLiveIns({ExitMBB, LoopMBB});
 
   return ExitMBB->begin();
 }
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 92647cb40525..9518d573bccd 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -9556,15 +9556,8 @@ AArch64InstrInfo::probedStackAlloc(MachineBasicBlock::iterator MBBI,
   MBB.addSuccessor(LoopTestMBB);
 
   // Update liveins.
-  if (MF.getRegInfo().reservedRegsFrozen()) {
-    bool anyChange = false;
-    do {
-      anyChange = recomputeLiveIns(*ExitMBB) ||
-                  recomputeLiveIns(*LoopBodyMBB) ||
-                  recomputeLiveIns(*LoopTestMBB);
-    } while (anyChange);
-    ;
-  }
+  if (MF.getRegInfo().reservedRegsFrozen())
+    fullyRecomputeLiveIns({ExitMBB, LoopBodyMBB, LoopTestMBB});
 
   return ExitMBB->begin();
 }
diff --git a/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp b/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp
index 8629551152cb..ea5dd5427ce7 100644
--- a/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp
+++ b/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp
@@ -1806,13 +1806,7 @@ void ARMLowOverheadLoops::Expand(LowOverheadLoop &LoLoop) {
   PostOrderLoopTraversal DFS(LoLoop.ML, *MLI);
   DFS.ProcessLoop();
   const SmallVectorImpl<MachineBasicBlock*> &PostOrder = DFS.getOrder();
-  bool anyChange = false;
-  do {
-    anyChange = false;
-    for (auto *MBB : PostOrder) {
-      anyChange = recomputeLiveIns(*MBB) || anyChange;
-    }
-  } while (anyChange);
+  fullyRecomputeLiveIns(PostOrder);
 
   for (auto *MBB : reverse(PostOrder))
     recomputeLivenessFlags(*MBB);
diff --git a/llvm/lib/Target/PowerPC/PPCExpandAtomicPseudoInsts.cpp b/llvm/lib/Target/PowerPC/PPCExpandAtomicPseudoInsts.cpp
index b43eee8fdd8c..b3cfcb2aa144 100644
--- a/llvm/lib/Target/PowerPC/PPCExpandAtomicPseudoInsts.cpp
+++ b/llvm/lib/Target/PowerPC/PPCExpandAtomicPseudoInsts.cpp
@@ -208,10 +208,7 @@ bool PPCExpandAtomicPseudo::expandAtomicRMW128(
       .addMBB(LoopMBB);
   CurrentMBB->addSuccessor(LoopMBB);
   CurrentMBB->addSuccessor(ExitMBB);
-  bool anyChange = false;
-  do {
-    anyChange = recomputeLiveIns(*ExitMBB) || recomputeLiveIns(*LoopMBB);
-  } while (anyChange);
+  fullyRecomputeLiveIns({ExitMBB, LoopMBB});
   NMBBI = MBB.end();
   MI.eraseFromParent();
   return true;
@@ -288,11 +285,7 @@ bool PPCExpandAtomicPseudo::expandAtomicCmpSwap128(
   CurrentMBB->addSuccessor(LoopCmpMBB);
   CurrentMBB->addSuccessor(ExitMBB);
 
-  bool anyChange = false;
-  do {
-    anyChange = recomputeLiveIns(*ExitMBB) || recomputeLiveIns(*CmpSuccMBB) ||
-                recomputeLiveIns(*LoopCmpMBB);
-  } while (anyChange);
+  fullyRecomputeLiveIns({ExitMBB, CmpSuccMBB, LoopCmpMBB});
   NMBBI = MBB.end();
   MI.eraseFromParent();
   return true;
diff --git a/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp b/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
index 6dcb59a3a57f..04e9f9e2366e 100644
--- a/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
@@ -1435,11 +1435,7 @@ void PPCFrameLowering::inlineStackProbe(MachineFunction &MF,
       ProbeLoopBodyMBB->addSuccessor(ProbeLoopBodyMBB);
     }
     // Update liveins.
-    bool anyChange = false;
-    do {
-      anyChange = recomputeLiveIns(*ProbeExitMBB) ||
-                  recomputeLiveIns(*ProbeLoopBodyMBB);
-    } while (anyChange);
+    fullyRecomputeLiveIns({ProbeExitMBB, ProbeLoopBodyMBB});
     return ProbeExitMBB;
   };
   // For case HasBP && MaxAlign > 1, we have to realign the SP by performing
@@ -1531,10 +1527,7 @@ void PPCFrameLowering::inlineStackProbe(MachineFunction &MF,
         buildDefCFAReg(*ExitMBB, ExitMBB->begin(), SPReg);
       }
       // Update liveins.
-      bool anyChange = false;
-      do {
-        anyChange = recomputeLiveIns(*ExitMBB) || recomputeLiveIns(*LoopMBB);
-      } while (anyChange);
+      fullyRecomputeLiveIns({ExitMBB, LoopMBB});
     }
   }
   ++NumPrologProbed;
diff --git a/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp b/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp
index 4897b37d8eb1..50ecd6e07441 100644
--- a/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp
@@ -824,10 +824,7 @@ void SystemZELFFrameLowering::inlineStackProbe(
   StackAllocMI->eraseFromParent();
   if (DoneMBB != nullptr) {
     // Compute the live-in lists for the new blocks.
-    bool anyChange = false;
-    do {
-      anyChange = recomputeLiveIns(*DoneMBB) || recomputeLiveIns(*LoopMBB);
-    } while (anyChange);
+    fullyRecomputeLiveIns({DoneMBB, LoopMBB});
   }
 }
 
@@ -1425,10 +1422,7 @@ void SystemZXPLINKFrameLowering::inlineStackProbe(
   StackAllocMI->eraseFromParent();
 
   // Compute the live-in lists for the new blocks.
-  bool anyChange = false;
-  do {
-    anyChange = recomputeLiveIns(*StackExtMBB) || recomputeLiveIns(*NextMBB);
-  } while (anyChange);
+  fullyRecomputeLiveIns({StackExtMBB, NextMBB});
 }
 
 bool SystemZXPLINKFrameLowering::hasFP(const MachineFunction &MF) const {
diff --git a/llvm/lib/Target/X86/X86FrameLowering.cpp b/llvm/lib/Target/X86/X86FrameLowering.cpp
index d914e1b61ab0..4521401d8741 100644
--- a/llvm/lib/Target/X86/X86FrameLowering.cpp
+++ b/llvm/lib/Target/X86/X86FrameLowering.cpp
@@ -885,10 +885,7 @@ void X86FrameLowering::emitStackProbeInlineGenericLoop(
   }
 
   // Update Live In information
-  bool anyChange = false;
-  do {
-    anyChange = recomputeLiveIns(*tailMBB) || recomputeLiveIns(*testMBB);
-  } while (anyChange);
+  fullyRecomputeLiveIns({tailMBB, testMBB});
 }
 
 void X86FrameLowering::emitStackProbeInlineWindowsCoreCLR64(
@@ -1380,11 +1377,7 @@ void X86FrameLowering::BuildStackAlignAND(MachineBasicBlock &MBB,
         footMBB->addSuccessor(&MBB);
       }
 
-      bool anyChange = false;
-      do {
-        anyChange = recomputeLiveIns(*footMBB) || recomputeLiveIns(*bodyMBB) ||
-                    recomputeLiveIns(*headMBB) || recomputeLiveIns(MBB);
-      } while (anyChange);
+      fullyRecomputeLiveIns({footMBB, bodyMBB, headMBB, &MBB});
     }
   } else {
     MachineInstr *MI = BuildMI(MBB, MBBI, DL, TII.get(AndOp), Reg)
-- 
GitLab


From 67571ffd2c51a72a23d57fb5ef746a6fadd6b09c Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Mon, 15 Apr 2024 15:00:30 -0700
Subject: [PATCH 307/464] [test][sanitizer] Compile .c file as C

---
 .../test/sanitizer_common/TestCases/Linux/mmap_56bit_test.c     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/mmap_56bit_test.c b/compiler-rt/test/sanitizer_common/TestCases/Linux/mmap_56bit_test.c
index b14ac7bcf192..a6aed77d5691 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/mmap_56bit_test.c
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/mmap_56bit_test.c
@@ -1,4 +1,4 @@
-// RUN: %clangxx %s -pie -fPIE -o %t && %run %t
+// RUN: %clang %s -pie -fPIE -o %t && %run %t
 // REQUIRES: x86_64-target-arch
 
 #include <assert.h>
-- 
GitLab


From a1ed652fb3589670180c08c2c6d5ef1ff337c658 Mon Sep 17 00:00:00 2001
From: Vitaly Buka <vitalybuka@google.com>
Date: Mon, 15 Apr 2024 15:01:09 -0700
Subject: [PATCH 308/464] [test][sanitizer] Temporarily disable test

Test, as expected, fails with Asan on system with 5lvl page tables.
Disabling the test to migrate buildbot.
---
 .../test/sanitizer_common/TestCases/Linux/mmap_56bit_test.c    | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/mmap_56bit_test.c b/compiler-rt/test/sanitizer_common/TestCases/Linux/mmap_56bit_test.c
index a6aed77d5691..02220cb78e6c 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/mmap_56bit_test.c
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/mmap_56bit_test.c
@@ -1,6 +1,9 @@
 // RUN: %clang %s -pie -fPIE -o %t && %run %t
 // REQUIRES: x86_64-target-arch
 
+// FIXME: Fails Asan, as expected, with 5lvl page tables.
+// UNSUPPORTED: x86_64-target-arch
+
 #include <assert.h>
 #include <stdio.h>
 #include <sys/mman.h>
-- 
GitLab


From 63310243537ba8830f3533a5d93e7b04b10d6c9e Mon Sep 17 00:00:00 2001
From: Argyrios Kyrtzidis <kyrtzidis@apple.com>
Date: Mon, 15 Apr 2024 15:05:55 -0700
Subject: [PATCH 309/464] [clang/DependencyScanning/ModuleDepCollector]
 Refactor part of `makeCommonInvocationForModuleBuild` into its own function
 (#88447)

The new function is about clearing out benign codegen options and can be
applied for PCH invocations as well.
---
 .../DependencyScanning/ModuleDepCollector.h   |  5 +++
 .../DependencyScanning/ModuleDepCollector.cpp | 36 ++++++++++++-------
 clang/test/ClangScanDeps/removed-args.c       | 28 +++++++++++++++
 3 files changed, 57 insertions(+), 12 deletions(-)

diff --git a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h
index 081899cc2c85..da51292296a9 100644
--- a/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h
+++ b/clang/include/clang/Tooling/DependencyScanning/ModuleDepCollector.h
@@ -308,6 +308,11 @@ private:
                                 ModuleDeps &Deps);
 };
 
+/// Resets codegen options that don't affect modules/PCH.
+void resetBenignCodeGenOptions(frontend::ActionKind ProgramAction,
+                               const LangOptions &LangOpts,
+                               CodeGenOptions &CGOpts);
+
 } // end namespace dependencies
 } // end namespace tooling
 } // end namespace clang
diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
index 94ccbd3351b0..e19f19b2528c 100644
--- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
+++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
@@ -154,6 +154,26 @@ void ModuleDepCollector::addOutputPaths(CowCompilerInvocation &CI,
   }
 }
 
+void dependencies::resetBenignCodeGenOptions(frontend::ActionKind ProgramAction,
+                                             const LangOptions &LangOpts,
+                                             CodeGenOptions &CGOpts) {
+  // TODO: Figure out better way to set options to their default value.
+  if (ProgramAction == frontend::GenerateModule) {
+    CGOpts.MainFileName.clear();
+    CGOpts.DwarfDebugFlags.clear();
+  }
+  if (ProgramAction == frontend::GeneratePCH ||
+      (ProgramAction == frontend::GenerateModule && !LangOpts.ModulesCodegen)) {
+    CGOpts.DebugCompilationDir.clear();
+    CGOpts.CoverageCompilationDir.clear();
+    CGOpts.CoverageDataFile.clear();
+    CGOpts.CoverageNotesFile.clear();
+    CGOpts.ProfileInstrumentUsePath.clear();
+    CGOpts.SampleProfileFile.clear();
+    CGOpts.ProfileRemappingFile.clear();
+  }
+}
+
 static CowCompilerInvocation
 makeCommonInvocationForModuleBuild(CompilerInvocation CI) {
   CI.resetNonModularOptions();
@@ -167,18 +187,8 @@ makeCommonInvocationForModuleBuild(CompilerInvocation CI) {
   // LLVM options are not going to affect the AST
   CI.getFrontendOpts().LLVMArgs.clear();
 
-  // TODO: Figure out better way to set options to their default value.
-  CI.getCodeGenOpts().MainFileName.clear();
-  CI.getCodeGenOpts().DwarfDebugFlags.clear();
-  if (!CI.getLangOpts().ModulesCodegen) {
-    CI.getCodeGenOpts().DebugCompilationDir.clear();
-    CI.getCodeGenOpts().CoverageCompilationDir.clear();
-    CI.getCodeGenOpts().CoverageDataFile.clear();
-    CI.getCodeGenOpts().CoverageNotesFile.clear();
-    CI.getCodeGenOpts().ProfileInstrumentUsePath.clear();
-    CI.getCodeGenOpts().SampleProfileFile.clear();
-    CI.getCodeGenOpts().ProfileRemappingFile.clear();
-  }
+  resetBenignCodeGenOptions(frontend::GenerateModule, CI.getLangOpts(),
+                            CI.getCodeGenOpts());
 
   // Map output paths that affect behaviour to "-" so their existence is in the
   // context hash. The final path will be computed in addOutputPaths.
@@ -342,6 +352,8 @@ static bool needsModules(FrontendInputFile FIF) {
 
 void ModuleDepCollector::applyDiscoveredDependencies(CompilerInvocation &CI) {
   CI.clearImplicitModuleBuildOptions();
+  resetBenignCodeGenOptions(CI.getFrontendOpts().ProgramAction,
+                            CI.getLangOpts(), CI.getCodeGenOpts());
 
   if (llvm::any_of(CI.getFrontendOpts().Inputs, needsModules)) {
     Preprocessor &PP = ScanInstance.getPreprocessor();
diff --git a/clang/test/ClangScanDeps/removed-args.c b/clang/test/ClangScanDeps/removed-args.c
index f49e4ead82f7..3e108f054945 100644
--- a/clang/test/ClangScanDeps/removed-args.c
+++ b/clang/test/ClangScanDeps/removed-args.c
@@ -93,3 +93,31 @@
 // CHECK-NOT:          "-fmodules-prune-interval=
 // CHECK-NOT:          "-fmodules-prune-after=
 // CHECK:            ],
+
+// Check for removed args for PCH invocations.
+
+// RUN: split-file %s %t
+// RUN: sed "s|DIR|%/t|g" %t/cdb-pch.json.template > %t/cdb-pch.json
+// RUN: clang-scan-deps -compilation-database %t/cdb-pch.json -format experimental-full > %t/result-pch.json
+// RUN: cat %t/result-pch.json | sed 's:\\\\\?:/:g' | FileCheck %s -DPREFIX=%/t -check-prefix=PCH
+//
+// PCH-NOT:          "-fdebug-compilation-dir="
+// PCH-NOT:          "-fcoverage-compilation-dir="
+// PCH-NOT:          "-coverage-notes-file
+// PCH-NOT:          "-coverage-data-file
+// PCH-NOT:          "-fprofile-instrument-use-path
+// PCH-NOT:          "-include"
+// PCH-NOT:          "-fmodules-cache-path=
+// PCH-NOT:          "-fmodules-validate-once-per-build-session"
+// PCH-NOT:          "-fbuild-session-timestamp=
+// PCH-NOT:          "-fmodules-prune-interval=
+// PCH-NOT:          "-fmodules-prune-after=
+
+//--- cdb-pch.json.template
+[
+  {
+    "directory": "DIR",
+    "command": "clang -x c-header DIR/header.h -fmodules -fimplicit-module-maps -fmodules-cache-path=DIR/cache -fdebug-compilation-dir=DIR/debug -fcoverage-compilation-dir=DIR/coverage -ftest-coverage -fprofile-instr-use=DIR/tu.profdata -o DIR/header.h.pch -serialize-diagnostics DIR/header.h.pch.diag ",
+    "file": "DIR/header.h.pch"
+  }
+]
-- 
GitLab


From 6d234638f90bd422078c93745bdee73d6de201bf Mon Sep 17 00:00:00 2001
From: Fr4nk1in <sh.fu@outlook.com>
Date: Tue, 16 Apr 2024 06:13:38 +0800
Subject: [PATCH 310/464] [docs][mlir] Fix broken links in 'llvm' dialects.
 (#88704)

Links to `llvm.mlir.global` and `llvm.mlir.addressof` in the ["Globals"
section of LLVM dialect
documentation](https://mlir.llvm.org/docs/Dialects/LLVM/#globals) are
broken.
---
 mlir/docs/Dialects/LLVM.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mlir/docs/Dialects/LLVM.md b/mlir/docs/Dialects/LLVM.md
index a49ba35db9a6..ba466aa6bc40 100644
--- a/mlir/docs/Dialects/LLVM.md
+++ b/mlir/docs/Dialects/LLVM.md
@@ -139,12 +139,12 @@ will be reevaluated after considering composite constants.
 ### Globals
 
 Global variables are also defined using a special operation,
-[`llvm.mlir.global`](#llvmmlirglobal-mlirllvmglobalop), located at the module
+[`llvm.mlir.global`](#llvmmlirglobal-llvmglobalop), located at the module
 level. Globals are MLIR symbols and are identified by their name.
 
 Since functions need to be isolated-from-above, i.e. values defined outside the
 function cannot be directly used inside the function, an additional operation,
-[`llvm.mlir.addressof`](#llvmmliraddressof-mlirllvmaddressofop), is provided to
+[`llvm.mlir.addressof`](#llvmmliraddressof-llvmaddressofop), is provided to
 locally define a value containing the _address_ of a global. The actual value
 can then be loaded from that pointer, or a new value can be stored into it if
 the global is not declared constant. This is similar to LLVM IR where globals
-- 
GitLab


From 206acf72c3b6c23e77716ccfc55ff94a4e7a7e3e Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Mon, 15 Apr 2024 18:48:49 -0400
Subject: [PATCH 311/464] [gn] port 8a7846fe86f95 (C++23 for libcxx, libcxxabi)

---
 llvm/utils/gn/secondary/libcxx/src/BUILD.gn    | 2 +-
 llvm/utils/gn/secondary/libcxxabi/src/BUILD.gn | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/utils/gn/secondary/libcxx/src/BUILD.gn b/llvm/utils/gn/secondary/libcxx/src/BUILD.gn
index 5da8db4574a0..1f6879358f22 100644
--- a/llvm/utils/gn/secondary/libcxx/src/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/src/BUILD.gn
@@ -49,7 +49,7 @@ config("cxx_config") {
     "-Wno-covered-switch-default",
   ]
   cflags_cc = [
-    "-std=c++20",
+    "-std=c++23",
     "-nostdinc++",
   ]
   defines = [
diff --git a/llvm/utils/gn/secondary/libcxxabi/src/BUILD.gn b/llvm/utils/gn/secondary/libcxxabi/src/BUILD.gn
index c82634e2bb06..7a923c5c854d 100644
--- a/llvm/utils/gn/secondary/libcxxabi/src/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxxabi/src/BUILD.gn
@@ -66,7 +66,7 @@ config("cxxabi_config") {
     "//libcxx/src",
   ]
   cflags_cc = [
-    "-std=c++20",
+    "-std=c++23",
     "-nostdinc++",
   ]
   defines = [
-- 
GitLab


From 466017c8dab74f66ce513c8752f0c1dcd16a8a63 Mon Sep 17 00:00:00 2001
From: Adrian Prantl <aprantl@apple.com>
Date: Mon, 15 Apr 2024 15:50:07 -0700
Subject: [PATCH 312/464] Work around test failure due to new aslr default

---
 .../TestDiagnoseDereferenceFunctionReturn.py                   | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lldb/test/API/commands/frame/diagnose/dereference-function-return/TestDiagnoseDereferenceFunctionReturn.py b/lldb/test/API/commands/frame/diagnose/dereference-function-return/TestDiagnoseDereferenceFunctionReturn.py
index d8f45161378b..4d9b036f5102 100644
--- a/lldb/test/API/commands/frame/diagnose/dereference-function-return/TestDiagnoseDereferenceFunctionReturn.py
+++ b/lldb/test/API/commands/frame/diagnose/dereference-function-return/TestDiagnoseDereferenceFunctionReturn.py
@@ -19,6 +19,9 @@ class TestDiagnoseDereferenceFunctionReturn(TestBase):
         TestBase.setUp(self)
         self.build()
         exe = self.getBuildArtifact("a.out")
+        # FIXME: This default changed in lldbtest.py and this test
+        # seems to rely on having it turned off.
+        self.runCmd("settings set target.disable-aslr true")
         self.runCmd("file " + exe, CURRENT_EXECUTABLE_SET)
         self.runCmd("run", RUN_SUCCEEDED)
         self.expect("thread list", "Thread should be stopped", substrs=["stopped"])
-- 
GitLab


From 9cb755cf5b2f48117fa34f257f386acf59ad6397 Mon Sep 17 00:00:00 2001
From: Wu Yingcong <yingcong.wu@intel.com>
Date: Mon, 15 Apr 2024 16:37:35 -0700
Subject: [PATCH 313/464] [Test][JITLink] Save rbx in
 ExecutionEngine/JITLink/x86-64/ELF_vtune.s (#86472)

The callee should preserve rbx according to the calling convention, but
it is not in the test case `ExecutionEngine/JITLink/x86-64/ELF_vtune.s`.
Not preserving the rbx register may result in some random error to the
caller function. This patch adds the missing command to preserve the
rbx.
---
 llvm/test/ExecutionEngine/JITLink/x86-64/ELF_vtune.s | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/llvm/test/ExecutionEngine/JITLink/x86-64/ELF_vtune.s b/llvm/test/ExecutionEngine/JITLink/x86-64/ELF_vtune.s
index 1c95bde51e12..936486b8a319 100644
--- a/llvm/test/ExecutionEngine/JITLink/x86-64/ELF_vtune.s
+++ b/llvm/test/ExecutionEngine/JITLink/x86-64/ELF_vtune.s
@@ -20,12 +20,14 @@ main:
         .cfi_def_cfa_offset 16
         .cfi_offset 6, -16
         movq    %rsp, %rbp
+        pushq   %rbx
         .cfi_def_cfa_register 6
-        movl    %edi, -4(%rbp)
-        movq    %rsi, -16(%rbp)
-        movl    -4(%rbp), %ebx
+        movl    %edi, -16(%rbp)
+        movq    %rsi, -24(%rbp)
+        movl    -16(%rbp), %ebx
         addl    $1, %ebx
-	movl   $0, %eax
+        movl    $0, %eax
+        popq    %rbx
         popq    %rbp
         .cfi_def_cfa 7, 8
         ret
-- 
GitLab


From c50f7e9a425bfa4ab8655c79a715c88ed3b1e830 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Tue, 16 Apr 2024 09:17:52 +0900
Subject: [PATCH 314/464] [InstCombine] Remove mul of SPF abs fold (#88675)

Remove the fold working on abs in SPF representation now that we
canonicalize SPF to intrinsics.

This is not strictly NFC because the SPF fold might fire for
non-canonical IR due to multi-use, but given the lack of test coverage,
I assume this is not important.
---
 .../InstCombine/InstCombineMulDivRem.cpp        | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
index 4dc1319f1c43..48372381a0d1 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
@@ -319,19 +319,12 @@ Instruction *InstCombinerImpl::visitMul(BinaryOperator &I) {
   }
 
   // abs(X) * abs(X) -> X * X
-  // nabs(X) * nabs(X) -> X * X
-  if (Op0 == Op1) {
-    Value *X, *Y;
-    SelectPatternFlavor SPF = matchSelectPattern(Op0, X, Y).Flavor;
-    if (SPF == SPF_ABS || SPF == SPF_NABS)
-      return BinaryOperator::CreateMul(X, X);
-
-    if (match(Op0, m_Intrinsic<Intrinsic::abs>(m_Value(X))))
-      return BinaryOperator::CreateMul(X, X);
-  }
+  Value *X;
+  if (Op0 == Op1 && match(Op0, m_Intrinsic<Intrinsic::abs>(m_Value(X))))
+    return BinaryOperator::CreateMul(X, X);
 
   {
-    Value *X, *Y;
+    Value *Y;
     // abs(X) * abs(Y) -> abs(X * Y)
     if (I.hasNoSignedWrap() &&
         match(Op0,
@@ -344,7 +337,7 @@ Instruction *InstCombinerImpl::visitMul(BinaryOperator &I) {
   }
 
   // -X * C --> X * -C
-  Value *X, *Y;
+  Value *Y;
   Constant *Op1C;
   if (match(Op0, m_Neg(m_Value(X))) && match(Op1, m_Constant(Op1C)))
     return BinaryOperator::CreateMul(X, ConstantExpr::getNeg(Op1C));
-- 
GitLab


From 2b06ff555aa32b316710b4708fbc16f36d6eab15 Mon Sep 17 00:00:00 2001
From: Philip Reames <preames@rivosinc.com>
Date: Mon, 15 Apr 2024 17:38:39 -0700
Subject: [PATCH 315/464] [RISCV] Expand mul to shNadd x, (slli x, c) in
 DAGCombine (#88524)

This expansion is directly inspired by the analogous code in the x86
backend for LEA. shXadd and (this sub-case of) LEA are largely
equivalent.

This is an alternative to
https://github.com/llvm/llvm-project/pull/87105.

This expansion is also supported via the decomposeMulByConstant
callback, but restricted because of interactions with other combines
since that code runs before legalization. As discussed in the other
review, my original plan had been to support post legalization expansion
through the same interface, but that ended up being more complicated
than seems justified.

Instead, lets go ahead and do the general expansion post-legalize. Other
targets use the combine approach, and matching that structure makes it
easier for us to adapt ideas from other targets to RISCV.
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   | 52 +++++++++++++++++--
 llvm/test/CodeGen/RISCV/addimm-mulimm.ll      |  9 ++--
 llvm/test/CodeGen/RISCV/rv32zba.ll            | 48 +++++++++++------
 .../CodeGen/RISCV/rv64-legal-i32/rv64zba.ll   | 48 +++++++++++------
 llvm/test/CodeGen/RISCV/rv64zba.ll            | 48 +++++++++++------
 5 files changed, 153 insertions(+), 52 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 27387595164a..259cc388276c 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -13363,10 +13363,56 @@ static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG,
   return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false, Subtarget);
 }
 
-static SDValue performMULCombine(SDNode *N, SelectionDAG &DAG) {
+// Try to expand a scalar multiply to a faster sequence.
+static SDValue expandMul(SDNode *N, SelectionDAG &DAG,
+                         TargetLowering::DAGCombinerInfo &DCI,
+                         const RISCVSubtarget &Subtarget) {
+
   EVT VT = N->getValueType(0);
-  if (!VT.isVector())
+
+  // LI + MUL is usually smaller than the alternative sequence.
+  if (DAG.getMachineFunction().getFunction().hasMinSize())
+    return SDValue();
+
+  if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
+    return SDValue();
+
+  if (VT != Subtarget.getXLenVT())
+    return SDValue();
+
+  if (!Subtarget.hasStdExtZba())
+    return SDValue();
+
+  ConstantSDNode *CNode = dyn_cast<ConstantSDNode>(N->getOperand(1));
+  if (!CNode)
     return SDValue();
+  uint64_t MulAmt = CNode->getZExtValue();
+
+  // If this is a power 2 + 2/4/8, we can use a shift followed by a single
+  // shXadd. First check if this a sum of two power of 2s because that's
+  // easy. Then count how many zeros are up to the first bit.
+  if (isPowerOf2_64(MulAmt & (MulAmt - 1))) {
+    unsigned ScaleShift = llvm::countr_zero(MulAmt);
+    if (ScaleShift >= 1 && ScaleShift < 4) {
+      unsigned ShiftAmt = Log2_64((MulAmt & (MulAmt - 1)));
+      SDLoc DL(N);
+      SDValue Shift1 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
+                                   DAG.getConstant(ShiftAmt, DL, VT));
+      SDValue Shift2 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
+                                   DAG.getConstant(ScaleShift, DL, VT));
+      return DAG.getNode(ISD::ADD, DL, VT, Shift1, Shift2);
+    }
+  }
+  return SDValue();
+}
+
+
+static SDValue performMULCombine(SDNode *N, SelectionDAG &DAG,
+                                 TargetLowering::DAGCombinerInfo &DCI,
+                                 const RISCVSubtarget &Subtarget) {
+  EVT VT = N->getValueType(0);
+  if (!VT.isVector())
+    return expandMul(N, DAG, DCI, Subtarget);
 
   SDLoc DL(N);
   SDValue N0 = N->getOperand(0);
@@ -15913,7 +15959,7 @@ SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
   case ISD::MUL:
     if (SDValue V = combineBinOp_VLToVWBinOp_VL(N, DCI, Subtarget))
       return V;
-    return performMULCombine(N, DAG);
+    return performMULCombine(N, DAG, DCI, Subtarget);
   case ISD::SDIV:
   case ISD::UDIV:
   case ISD::SREM:
diff --git a/llvm/test/CodeGen/RISCV/addimm-mulimm.ll b/llvm/test/CodeGen/RISCV/addimm-mulimm.ll
index 48fa69e10456..10103f071462 100644
--- a/llvm/test/CodeGen/RISCV/addimm-mulimm.ll
+++ b/llvm/test/CodeGen/RISCV/addimm-mulimm.ll
@@ -551,8 +551,9 @@ define i64 @add_mul_combine_infinite_loop(i64 %x) {
 ; RV32IMB-NEXT:    sh3add a1, a1, a2
 ; RV32IMB-NEXT:    sh1add a0, a0, a0
 ; RV32IMB-NEXT:    slli a2, a0, 3
-; RV32IMB-NEXT:    addi a0, a2, 2047
-; RV32IMB-NEXT:    addi a0, a0, 1
+; RV32IMB-NEXT:    li a3, 1
+; RV32IMB-NEXT:    slli a3, a3, 11
+; RV32IMB-NEXT:    sh3add a0, a0, a3
 ; RV32IMB-NEXT:    sltu a2, a0, a2
 ; RV32IMB-NEXT:    add a1, a1, a2
 ; RV32IMB-NEXT:    ret
@@ -561,8 +562,8 @@ define i64 @add_mul_combine_infinite_loop(i64 %x) {
 ; RV64IMB:       # %bb.0:
 ; RV64IMB-NEXT:    addi a0, a0, 86
 ; RV64IMB-NEXT:    sh1add a0, a0, a0
-; RV64IMB-NEXT:    li a1, -16
-; RV64IMB-NEXT:    sh3add a0, a0, a1
+; RV64IMB-NEXT:    slli a0, a0, 3
+; RV64IMB-NEXT:    addi a0, a0, -16
 ; RV64IMB-NEXT:    ret
   %tmp0 = mul i64 %x, 24
   %tmp1 = add i64 %tmp0, 2048
diff --git a/llvm/test/CodeGen/RISCV/rv32zba.ll b/llvm/test/CodeGen/RISCV/rv32zba.ll
index 0908a393338c..cc632a09c805 100644
--- a/llvm/test/CodeGen/RISCV/rv32zba.ll
+++ b/llvm/test/CodeGen/RISCV/rv32zba.ll
@@ -271,31 +271,49 @@ define i32 @mul288(i32 %a) {
 }
 
 define i32 @mul258(i32 %a) {
-; CHECK-LABEL: mul258:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 258
-; CHECK-NEXT:    mul a0, a0, a1
-; CHECK-NEXT:    ret
+; RV32I-LABEL: mul258:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    li a1, 258
+; RV32I-NEXT:    mul a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32ZBA-LABEL: mul258:
+; RV32ZBA:       # %bb.0:
+; RV32ZBA-NEXT:    slli a1, a0, 8
+; RV32ZBA-NEXT:    sh1add a0, a0, a1
+; RV32ZBA-NEXT:    ret
   %c = mul i32 %a, 258
   ret i32 %c
 }
 
 define i32 @mul260(i32 %a) {
-; CHECK-LABEL: mul260:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 260
-; CHECK-NEXT:    mul a0, a0, a1
-; CHECK-NEXT:    ret
+; RV32I-LABEL: mul260:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    li a1, 260
+; RV32I-NEXT:    mul a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32ZBA-LABEL: mul260:
+; RV32ZBA:       # %bb.0:
+; RV32ZBA-NEXT:    slli a1, a0, 8
+; RV32ZBA-NEXT:    sh2add a0, a0, a1
+; RV32ZBA-NEXT:    ret
   %c = mul i32 %a, 260
   ret i32 %c
 }
 
 define i32 @mul264(i32 %a) {
-; CHECK-LABEL: mul264:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 264
-; CHECK-NEXT:    mul a0, a0, a1
-; CHECK-NEXT:    ret
+; RV32I-LABEL: mul264:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    li a1, 264
+; RV32I-NEXT:    mul a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32ZBA-LABEL: mul264:
+; RV32ZBA:       # %bb.0:
+; RV32ZBA-NEXT:    slli a1, a0, 8
+; RV32ZBA-NEXT:    sh3add a0, a0, a1
+; RV32ZBA-NEXT:    ret
   %c = mul i32 %a, 264
   ret i32 %c
 }
diff --git a/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64zba.ll b/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64zba.ll
index 90cfb1fdcb77..ee9b73ca82f2 100644
--- a/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64zba.ll
+++ b/llvm/test/CodeGen/RISCV/rv64-legal-i32/rv64zba.ll
@@ -811,31 +811,49 @@ define i64 @adduw_imm(i32 signext %0) nounwind {
 }
 
 define i64 @mul258(i64 %a) {
-; CHECK-LABEL: mul258:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 258
-; CHECK-NEXT:    mul a0, a0, a1
-; CHECK-NEXT:    ret
+; RV64I-LABEL: mul258:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    li a1, 258
+; RV64I-NEXT:    mul a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64ZBA-LABEL: mul258:
+; RV64ZBA:       # %bb.0:
+; RV64ZBA-NEXT:    slli a1, a0, 8
+; RV64ZBA-NEXT:    sh1add a0, a0, a1
+; RV64ZBA-NEXT:    ret
   %c = mul i64 %a, 258
   ret i64 %c
 }
 
 define i64 @mul260(i64 %a) {
-; CHECK-LABEL: mul260:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 260
-; CHECK-NEXT:    mul a0, a0, a1
-; CHECK-NEXT:    ret
+; RV64I-LABEL: mul260:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    li a1, 260
+; RV64I-NEXT:    mul a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64ZBA-LABEL: mul260:
+; RV64ZBA:       # %bb.0:
+; RV64ZBA-NEXT:    slli a1, a0, 8
+; RV64ZBA-NEXT:    sh2add a0, a0, a1
+; RV64ZBA-NEXT:    ret
   %c = mul i64 %a, 260
   ret i64 %c
 }
 
 define i64 @mul264(i64 %a) {
-; CHECK-LABEL: mul264:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 264
-; CHECK-NEXT:    mul a0, a0, a1
-; CHECK-NEXT:    ret
+; RV64I-LABEL: mul264:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    li a1, 264
+; RV64I-NEXT:    mul a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64ZBA-LABEL: mul264:
+; RV64ZBA:       # %bb.0:
+; RV64ZBA-NEXT:    slli a1, a0, 8
+; RV64ZBA-NEXT:    sh3add a0, a0, a1
+; RV64ZBA-NEXT:    ret
   %c = mul i64 %a, 264
   ret i64 %c
 }
diff --git a/llvm/test/CodeGen/RISCV/rv64zba.ll b/llvm/test/CodeGen/RISCV/rv64zba.ll
index bb4be323ecb2..0d1d4838c611 100644
--- a/llvm/test/CodeGen/RISCV/rv64zba.ll
+++ b/llvm/test/CodeGen/RISCV/rv64zba.ll
@@ -834,31 +834,49 @@ define i64 @adduw_imm(i32 signext %0) nounwind {
 }
 
 define i64 @mul258(i64 %a) {
-; CHECK-LABEL: mul258:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 258
-; CHECK-NEXT:    mul a0, a0, a1
-; CHECK-NEXT:    ret
+; RV64I-LABEL: mul258:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    li a1, 258
+; RV64I-NEXT:    mul a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64ZBA-LABEL: mul258:
+; RV64ZBA:       # %bb.0:
+; RV64ZBA-NEXT:    slli a1, a0, 8
+; RV64ZBA-NEXT:    sh1add a0, a0, a1
+; RV64ZBA-NEXT:    ret
   %c = mul i64 %a, 258
   ret i64 %c
 }
 
 define i64 @mul260(i64 %a) {
-; CHECK-LABEL: mul260:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 260
-; CHECK-NEXT:    mul a0, a0, a1
-; CHECK-NEXT:    ret
+; RV64I-LABEL: mul260:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    li a1, 260
+; RV64I-NEXT:    mul a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64ZBA-LABEL: mul260:
+; RV64ZBA:       # %bb.0:
+; RV64ZBA-NEXT:    slli a1, a0, 8
+; RV64ZBA-NEXT:    sh2add a0, a0, a1
+; RV64ZBA-NEXT:    ret
   %c = mul i64 %a, 260
   ret i64 %c
 }
 
 define i64 @mul264(i64 %a) {
-; CHECK-LABEL: mul264:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 264
-; CHECK-NEXT:    mul a0, a0, a1
-; CHECK-NEXT:    ret
+; RV64I-LABEL: mul264:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    li a1, 264
+; RV64I-NEXT:    mul a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64ZBA-LABEL: mul264:
+; RV64ZBA:       # %bb.0:
+; RV64ZBA-NEXT:    slli a1, a0, 8
+; RV64ZBA-NEXT:    sh3add a0, a0, a1
+; RV64ZBA-NEXT:    ret
   %c = mul i64 %a, 264
   ret i64 %c
 }
-- 
GitLab


From 40bbdb609f58d6cbbae1ca525832d7a21641a347 Mon Sep 17 00:00:00 2001
From: Alina Sbirlea <asbirlea@google.com>
Date: Mon, 15 Apr 2024 17:36:15 -0700
Subject: [PATCH 316/464] Revert "[DAG] Fold
 extract_subvector(insert_subvector(x,y,c1),c2) --> extract_subvector(y,c2-c1)
 (#87925)"

This reverts commit 8c0f52e9d5a99bf96bb64ac23b5893482c292527.

Reverting to green, reproducer attached in the PR/revision comments.
---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp |   17 -
 .../any_extend_vector_inreg_of_broadcast.ll   |   42 +-
 llvm/test/CodeGen/X86/dpbusd.ll               |    2 +-
 llvm/test/CodeGen/X86/dpbusd_i4.ll            |    2 +-
 .../vector-interleaved-load-i16-stride-3.ll   | 1300 +++++++++--------
 .../vector-interleaved-store-i8-stride-7.ll   |  602 ++++----
 .../zero_extend_vector_inreg_of_broadcast.ll  |   28 +-
 7 files changed, 1006 insertions(+), 987 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index cbba3a294b3d..0fa0bf2609bb 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -24467,23 +24467,6 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode *N) {
       if (!LegalOperations || TLI.isOperationLegal(ISD::SPLAT_VECTOR, NVT))
         return DAG.getSplatVector(NVT, DL, V.getOperand(0));
 
-  // extract_subvector(insert_subvector(x,y,c1),c2)
-  //  --> extract_subvector(y,c2-c1)
-  // iff we're just extracting from the inserted subvector.
-  if (V.getOpcode() == ISD::INSERT_SUBVECTOR) {
-    SDValue InsSub = V.getOperand(1);
-    EVT InsSubVT = InsSub.getValueType();
-    unsigned NumInsElts = InsSubVT.getVectorMinNumElements();
-    unsigned InsIdx = V.getConstantOperandVal(2);
-    unsigned NumSubElts = NVT.getVectorMinNumElements();
-    if (InsIdx <= ExtIdx && (ExtIdx + NumSubElts) <= (InsIdx + NumInsElts) &&
-        TLI.isExtractSubvectorCheap(NVT, InsSubVT, ExtIdx - InsIdx)) {
-      SDLoc DL(N);
-      return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NVT, InsSub,
-                         DAG.getVectorIdxConstant(ExtIdx - InsIdx, DL));
-    }
-  }
-
   // Try to move vector bitcast after extract_subv by scaling extraction index:
   // extract_subv (bitcast X), Index --> bitcast (extract_subv X, Index')
   if (V.getOpcode() == ISD::BITCAST &&
diff --git a/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll b/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll
index 39c7ce1413d1..4242d8483e72 100644
--- a/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll
+++ b/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll
@@ -314,8 +314,8 @@ define void @vec64_i16_widen_to_i32_factor2_broadcast_to_v2i32_factor2(ptr %in.v
 ;
 ; AVX512F-LABEL: vec64_i16_widen_to_i32_factor2_broadcast_to_v2i32_factor2:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
+; AVX512F-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,10,11,0,1,14,15,u,u,u,u,u,u,u,u]
 ; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)
@@ -324,8 +324,8 @@ define void @vec64_i16_widen_to_i32_factor2_broadcast_to_v2i32_factor2(ptr %in.v
 ;
 ; AVX512DQ-LABEL: vec64_i16_widen_to_i32_factor2_broadcast_to_v2i32_factor2:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512DQ-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,10,11,0,1,14,15,u,u,u,u,u,u,u,u]
 ; AVX512DQ-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, (%rcx)
@@ -981,7 +981,7 @@ define void @vec128_i32_widen_to_i64_factor2_broadcast_to_v2i64_factor2(ptr %in.
 ; AVX512F-NEXT:    vpmovsxbd {{.*#+}} xmm0 = [0,5,0,7]
 ; AVX512F-NEXT:    vmovdqa (%rdi), %ymm1
 ; AVX512F-NEXT:    vpaddb (%rsi), %ymm1, %ymm1
-; AVX512F-NEXT:    vpermd %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpermd %zmm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)
 ; AVX512F-NEXT:    vzeroupper
@@ -992,7 +992,7 @@ define void @vec128_i32_widen_to_i64_factor2_broadcast_to_v2i64_factor2(ptr %in.
 ; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} xmm0 = [0,5,0,7]
 ; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm1
 ; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpermd %ymm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpermd %zmm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, (%rcx)
 ; AVX512DQ-NEXT:    vzeroupper
@@ -3507,12 +3507,13 @@ define void @vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12(ptr %i
 ;
 ; AVX512F-LABEL: vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512F-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512F-NEXT:    vmovdqa 48(%rdi), %xmm1
 ; AVX512F-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
+; AVX512F-NEXT:    vpbroadcastw %xmm0, %ymm2
+; AVX512F-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2],ymm1[3],ymm2[4],ymm1[5],ymm2[6],ymm1[7],ymm2[8],ymm1[9],ymm2[10],ymm1[11],ymm2[12],ymm1[13],ymm2[14],ymm1[15]
 ; AVX512F-NEXT:    vpbroadcastw %xmm0, %ymm0
-; AVX512F-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7],ymm0[8],ymm1[9],ymm0[10],ymm1[11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
 ; AVX512F-NEXT:    vpaddb (%rdx), %ymm1, %ymm1
 ; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm0, 32(%rcx)
@@ -3522,12 +3523,13 @@ define void @vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12(ptr %i
 ;
 ; AVX512DQ-LABEL: vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa 48(%rdi), %xmm1
 ; AVX512DQ-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX512DQ-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpbroadcastw %xmm0, %ymm2
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2],ymm1[3],ymm2[4],ymm1[5],ymm2[6],ymm1[7],ymm2[8],ymm1[9],ymm2[10],ymm1[11],ymm2[12],ymm1[13],ymm2[14],ymm1[15]
 ; AVX512DQ-NEXT:    vpbroadcastw %xmm0, %ymm0
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7],ymm0[8],ymm1[9],ymm0[10],ymm1[11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
 ; AVX512DQ-NEXT:    vpaddb (%rdx), %ymm1, %ymm1
 ; AVX512DQ-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, 32(%rcx)
@@ -3766,10 +3768,10 @@ define void @vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6(ptr %in.
 ;
 ; AVX512F-LABEL: vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512F-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512F-NEXT:    vmovdqa 48(%rdi), %xmm1
 ; AVX512F-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512F-NEXT:    vpbroadcastq %xmm0, %ymm2
 ; AVX512F-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1,2,3],ymm2[4],ymm1[5,6,7],ymm2[8],ymm1[9,10,11],ymm2[12],ymm1[13,14,15]
 ; AVX512F-NEXT:    vpbroadcastw %xmm0, %ymm0
@@ -3782,10 +3784,10 @@ define void @vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6(ptr %in.
 ;
 ; AVX512DQ-LABEL: vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa 48(%rdi), %xmm1
 ; AVX512DQ-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX512DQ-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpbroadcastq %xmm0, %ymm2
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1,2,3],ymm2[4],ymm1[5,6,7],ymm2[8],ymm1[9,10,11],ymm2[12],ymm1[13,14,15]
 ; AVX512DQ-NEXT:    vpbroadcastw %xmm0, %ymm0
@@ -4145,9 +4147,9 @@ define void @vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2(ptr %
 ;
 ; AVX512F-LABEL: vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512F-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512F-NEXT:    vmovdqa 48(%rdi), %xmm1
-; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512F-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
 ; AVX512F-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]
 ; AVX512F-NEXT:    vpbroadcastw %xmm0, %xmm0
@@ -4159,9 +4161,9 @@ define void @vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2(ptr %
 ;
 ; AVX512DQ-LABEL: vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa 48(%rdi), %xmm1
-; AVX512DQ-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]
 ; AVX512DQ-NEXT:    vpbroadcastw %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/dpbusd.ll b/llvm/test/CodeGen/X86/dpbusd.ll
index 04d7a9691b64..fbea08eb1e55 100644
--- a/llvm/test/CodeGen/X86/dpbusd.ll
+++ b/llvm/test/CodeGen/X86/dpbusd.ll
@@ -26,7 +26,7 @@ define i32 @no_dpbusd(ptr%a, ptr%b, i32 %c, i32 %n) {
 ; AVX512-NEXT:    vpmovzxbw {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
 ; AVX512-NEXT:    vpmaddwd %ymm0, %ymm1, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
diff --git a/llvm/test/CodeGen/X86/dpbusd_i4.ll b/llvm/test/CodeGen/X86/dpbusd_i4.ll
index a212f99680ef..906fead7f8db 100644
--- a/llvm/test/CodeGen/X86/dpbusd_i4.ll
+++ b/llvm/test/CodeGen/X86/dpbusd_i4.ll
@@ -86,7 +86,7 @@ define i32 @mul_sext_i4i4(<16 x i4> %a, <16 x i4> %b, i32 %c) {
 ; CHECK-NEXT:    vpsraw $12, %ymm0, %ymm0
 ; CHECK-NEXT:    vpmaddwd %ymm1, %ymm0, %ymm0
 ; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
 ; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-3.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-3.ll
index 6d5fc9ed0ab5..1436922f9dd1 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-3.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-3.ll
@@ -1828,22 +1828,22 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX512-LABEL: load_i16_stride3_vf32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm1 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
 ; AVX512-NEXT:    vmovdqa 128(%rdi), %ymm5
 ; AVX512-NEXT:    vmovdqa 160(%rdi), %ymm6
-; AVX512-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512-NEXT:    vpternlogq $202, %ymm5, %ymm6, %ymm1
-; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
-; AVX512-NEXT:    vmovdqa 112(%rdi), %xmm1
+; AVX512-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512-NEXT:    vpternlogq $202, %ymm5, %ymm6, %ymm0
+; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2,3],ymm2[4],ymm0[5,6],ymm2[7],ymm0[8],ymm2[9],ymm0[10,11],ymm2[12],ymm0[13,14],ymm2[15]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm3 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
+; AVX512-NEXT:    vmovdqa 112(%rdi), %xmm0
 ; AVX512-NEXT:    vmovdqa 96(%rdi), %xmm2
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm2[0],xmm0[1],xmm2[2,3],xmm0[4],xmm2[5,6],xmm0[7]
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[0,1,6,7,12,13,2,3,8,9,14,15,u,u,u,u]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm7 = ymm4[0,1,2],ymm3[3,4,5,6,7]
 ; AVX512-NEXT:    vmovdqa (%rdi), %ymm8
 ; AVX512-NEXT:    vmovdqa 32(%rdi), %ymm9
-; AVX512-NEXT:    vmovdqa %ymm0, %ymm3
+; AVX512-NEXT:    vmovdqa %ymm1, %ymm3
 ; AVX512-NEXT:    vpternlogq $202, %ymm9, %ymm8, %ymm3
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm3[2,3,0,1]
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7],ymm3[8],ymm4[9],ymm3[10,11],ymm4[12],ymm3[13,14],ymm4[15]
@@ -1857,14 +1857,14 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,6,5,4,7]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm11[4,5,6,7]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm10, %zmm7
-; AVX512-NEXT:    vmovdqa %ymm0, %ymm10
+; AVX512-NEXT:    vmovdqa %ymm1, %ymm10
 ; AVX512-NEXT:    vpternlogq $202, %ymm6, %ymm5, %ymm10
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm11 = ymm10[2,3,0,1]
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0,1],ymm11[2],ymm10[3,4],ymm11[5],ymm10[6,7,8,9],ymm11[10],ymm10[11,12],ymm11[13],ymm10[14,15]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm11 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
 ; AVX512-NEXT:    vpshufb %ymm11, %ymm10, %ymm10
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm12 = xmm12[2,3,8,9,14,15,4,5,10,11,10,11,10,11,10,11]
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]
+; AVX512-NEXT:    vpshufb %xmm11, %xmm12, %xmm12
 ; AVX512-NEXT:    vpblendw {{.*#+}} xmm12 = xmm12[0,1,2,3,4],xmm10[5,6,7]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm10 = ymm12[0,1,2,3],ymm10[4,5,6,7]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm12 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
@@ -1885,19 +1885,21 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1,2],ymm5[3],ymm12[4,5],ymm5[6],ymm12[7],ymm5[8],ymm12[9,10],ymm5[11],ymm12[12,13],ymm5[14],ymm12[15]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm6 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
 ; AVX512-NEXT:    vpshufb %ymm6, %ymm5, %ymm5
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3,4],xmm2[5],xmm1[6,7]
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm5[5,6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm5[4,5,6,7]
-; AVX512-NEXT:    vpternlogq $202, %ymm8, %ymm9, %ymm0
-; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1,2],ymm2[3],ymm0[4,5],ymm2[6],ymm0[7],ymm2[8],ymm0[9,10],ymm2[11],ymm0[12,13],ymm2[14],ymm0[15]
-; AVX512-NEXT:    vpshufb %ymm6, %ymm0, %ymm0
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]
-; AVX512-NEXT:    vpshufb %xmm6, %xmm2, %xmm2
-; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5,6,7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpternlogq $202, %ymm8, %ymm9, %ymm1
+; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm1[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm8[0],ymm1[1,2],ymm8[3],ymm1[4,5],ymm8[6],ymm1[7],ymm8[8],ymm1[9,10],ymm8[11],ymm1[12,13],ymm8[14],ymm1[15]
+; AVX512-NEXT:    vpshufb %ymm6, %ymm1, %ymm1
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]
+; AVX512-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
+; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5,6,7]
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2],xmm0[3,4],xmm2[5],xmm0[6,7]
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512-NEXT:    vextracti32x4 $2, %zmm0, %xmm0
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm5[5,6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512-NEXT:    vmovdqa64 %zmm7, (%rsi)
 ; AVX512-NEXT:    vmovdqa64 %zmm10, (%rdx)
 ; AVX512-NEXT:    vmovdqa64 %zmm0, (%rcx)
@@ -1906,22 +1908,22 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX512-FCP-LABEL: load_i16_stride3_vf32:
 ; AVX512-FCP:       # %bb.0:
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm1 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
 ; AVX512-FCP-NEXT:    vmovdqa 128(%rdi), %ymm5
 ; AVX512-FCP-NEXT:    vmovdqa 160(%rdi), %ymm6
-; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm5, %ymm6, %ymm1
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
-; AVX512-FCP-NEXT:    vmovdqa 112(%rdi), %xmm1
+; AVX512-FCP-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm5, %ymm6, %ymm0
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2,3],ymm2[4],ymm0[5,6],ymm2[7],ymm0[8],ymm2[9],ymm0[10,11],ymm2[12],ymm0[13,14],ymm2[15]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
+; AVX512-FCP-NEXT:    vmovdqa 112(%rdi), %xmm0
 ; AVX512-FCP-NEXT:    vmovdqa 96(%rdi), %xmm2
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm2[0],xmm0[1],xmm2[2,3],xmm0[4],xmm2[5,6],xmm0[7]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[0,1,6,7,12,13,2,3,8,9,14,15,u,u,u,u]
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm7 = ymm4[0,1,2],ymm3[3,4,5,6,7]
 ; AVX512-FCP-NEXT:    vmovdqa (%rdi), %ymm8
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %ymm9
-; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm3
+; AVX512-FCP-NEXT:    vmovdqa %ymm1, %ymm3
 ; AVX512-FCP-NEXT:    vpternlogq $202, %ymm9, %ymm8, %ymm3
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm3[2,3,0,1]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7],ymm3[8],ymm4[9],ymm3[10,11],ymm4[12],ymm3[13,14],ymm4[15]
@@ -1935,14 +1937,14 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,6,5,4,7]
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm11[4,5,6,7]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm10, %zmm7
-; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm10
+; AVX512-FCP-NEXT:    vmovdqa %ymm1, %ymm10
 ; AVX512-FCP-NEXT:    vpternlogq $202, %ymm6, %ymm5, %ymm10
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm11 = ymm10[2,3,0,1]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0,1],ymm11[2],ymm10[3,4],ymm11[5],ymm10[6,7,8,9],ymm11[10],ymm10[11,12],ymm11[13],ymm10[14,15]
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm11 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
 ; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm10, %ymm10
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm12 = xmm12[2,3,8,9,14,15,4,5,10,11,10,11,10,11,10,11]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]
+; AVX512-FCP-NEXT:    vpshufb %xmm11, %xmm12, %xmm12
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm12 = xmm12[0,1,2,3,4],xmm10[5,6,7]
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm10 = ymm12[0,1,2,3],ymm10[4,5,6,7]
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm12 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
@@ -1963,19 +1965,21 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1,2],ymm5[3],ymm12[4,5],ymm5[6],ymm12[7],ymm5[8],ymm12[9,10],ymm5[11],ymm12[12,13],ymm5[14],ymm12[15]
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm6 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
 ; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm5, %ymm5
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3,4],xmm2[5],xmm1[6,7]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm5[5,6,7]
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm5[4,5,6,7]
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm8, %ymm9, %ymm0
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1,2],ymm2[3],ymm0[4,5],ymm2[6],ymm0[7],ymm2[8],ymm0[9,10],ymm2[11],ymm0[12,13],ymm2[14],ymm0[15]
-; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm0, %ymm0
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]
-; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm2, %xmm2
-; AVX512-FCP-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5,6,7]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm8, %ymm9, %ymm1
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm1[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm8[0],ymm1[1,2],ymm8[3],ymm1[4,5],ymm8[6],ymm1[7],ymm8[8],ymm1[9,10],ymm8[11],ymm1[12,13],ymm8[14],ymm1[15]
+; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm1, %ymm1
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]
+; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
+; AVX512-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5,6,7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2],xmm0[3,4],xmm2[5],xmm0[6,7]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512-FCP-NEXT:    vextracti32x4 $2, %zmm0, %xmm0
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm5[5,6,7]
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm7, (%rsi)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm10, (%rdx)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, (%rcx)
@@ -1984,22 +1988,22 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX512DQ-LABEL: load_i16_stride3_vf32:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm1 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
 ; AVX512DQ-NEXT:    vmovdqa 128(%rdi), %ymm5
 ; AVX512DQ-NEXT:    vmovdqa 160(%rdi), %ymm6
-; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm5, %ymm6, %ymm1
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
-; AVX512DQ-NEXT:    vmovdqa 112(%rdi), %xmm1
+; AVX512DQ-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm5, %ymm6, %ymm0
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2,3],ymm2[4],ymm0[5,6],ymm2[7],ymm0[8],ymm2[9],ymm0[10,11],ymm2[12],ymm0[13,14],ymm2[15]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm3 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
+; AVX512DQ-NEXT:    vmovdqa 112(%rdi), %xmm0
 ; AVX512DQ-NEXT:    vmovdqa 96(%rdi), %xmm2
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm2[0],xmm0[1],xmm2[2,3],xmm0[4],xmm2[5,6],xmm0[7]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[0,1,6,7,12,13,2,3,8,9,14,15,u,u,u,u]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm7 = ymm4[0,1,2],ymm3[3,4,5,6,7]
 ; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm8
 ; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm9
-; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm3
+; AVX512DQ-NEXT:    vmovdqa %ymm1, %ymm3
 ; AVX512DQ-NEXT:    vpternlogq $202, %ymm9, %ymm8, %ymm3
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm3[2,3,0,1]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7],ymm3[8],ymm4[9],ymm3[10,11],ymm4[12],ymm3[13,14],ymm4[15]
@@ -2013,14 +2017,14 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,6,5,4,7]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm11[4,5,6,7]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm10, %zmm7
-; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm10
+; AVX512DQ-NEXT:    vmovdqa %ymm1, %ymm10
 ; AVX512DQ-NEXT:    vpternlogq $202, %ymm6, %ymm5, %ymm10
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm11 = ymm10[2,3,0,1]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0,1],ymm11[2],ymm10[3,4],ymm11[5],ymm10[6,7,8,9],ymm11[10],ymm10[11,12],ymm11[13],ymm10[14,15]
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm11 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
 ; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm10, %ymm10
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm12 = xmm12[2,3,8,9,14,15,4,5,10,11,10,11,10,11,10,11]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]
+; AVX512DQ-NEXT:    vpshufb %xmm11, %xmm12, %xmm12
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm12 = xmm12[0,1,2,3,4],xmm10[5,6,7]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm10 = ymm12[0,1,2,3],ymm10[4,5,6,7]
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm12 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
@@ -2041,19 +2045,21 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1,2],ymm5[3],ymm12[4,5],ymm5[6],ymm12[7],ymm5[8],ymm12[9,10],ymm5[11],ymm12[12,13],ymm5[14],ymm12[15]
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm6 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
 ; AVX512DQ-NEXT:    vpshufb %ymm6, %ymm5, %ymm5
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3,4],xmm2[5],xmm1[6,7]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm5[5,6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm5[4,5,6,7]
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm8, %ymm9, %ymm0
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1,2],ymm2[3],ymm0[4,5],ymm2[6],ymm0[7],ymm2[8],ymm0[9,10],ymm2[11],ymm0[12,13],ymm2[14],ymm0[15]
-; AVX512DQ-NEXT:    vpshufb %ymm6, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]
-; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm2, %xmm2
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5,6,7]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm8, %ymm9, %ymm1
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm1[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm8[0],ymm1[1,2],ymm8[3],ymm1[4,5],ymm8[6],ymm1[7],ymm8[8],ymm1[9,10],ymm8[11],ymm1[12,13],ymm8[14],ymm1[15]
+; AVX512DQ-NEXT:    vpshufb %ymm6, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]
+; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5,6,7]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2],xmm0[3,4],xmm2[5],xmm0[6,7]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512DQ-NEXT:    vextracti32x4 $2, %zmm0, %xmm0
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm5[5,6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm7, (%rsi)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm10, (%rdx)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm0, (%rcx)
@@ -2062,22 +2068,22 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX512DQ-FCP-LABEL: load_i16_stride3_vf32:
 ; AVX512DQ-FCP:       # %bb.0:
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm1 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
 ; AVX512DQ-FCP-NEXT:    vmovdqa 128(%rdi), %ymm5
 ; AVX512DQ-FCP-NEXT:    vmovdqa 160(%rdi), %ymm6
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm5, %ymm6, %ymm1
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
-; AVX512DQ-FCP-NEXT:    vmovdqa 112(%rdi), %xmm1
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm5, %ymm6, %ymm0
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2,3],ymm2[4],ymm0[5,6],ymm2[7],ymm0[8],ymm2[9],ymm0[10,11],ymm2[12],ymm0[13,14],ymm2[15]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
+; AVX512DQ-FCP-NEXT:    vmovdqa 112(%rdi), %xmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdi), %xmm2
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm2[0],xmm0[1],xmm2[2,3],xmm0[4],xmm2[5,6],xmm0[7]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[0,1,6,7,12,13,2,3,8,9,14,15,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm7 = ymm4[0,1,2],ymm3[3,4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %ymm8
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %ymm9
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm3
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm1, %ymm3
 ; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm9, %ymm8, %ymm3
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm3[2,3,0,1]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7],ymm3[8],ymm4[9],ymm3[10,11],ymm4[12],ymm3[13,14],ymm4[15]
@@ -2091,14 +2097,14 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,6,5,4,7]
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm11[4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm10, %zmm7
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm10
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm1, %ymm10
 ; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm6, %ymm5, %ymm10
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm11 = ymm10[2,3,0,1]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0,1],ymm11[2],ymm10[3,4],ymm11[5],ymm10[6,7,8,9],ymm11[10],ymm10[11,12],ymm11[13],ymm10[14,15]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm11 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm10, %ymm10
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm12 = xmm12[2,3,8,9,14,15,4,5,10,11,10,11,10,11,10,11]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm11, %xmm12, %xmm12
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm12 = xmm12[0,1,2,3,4],xmm10[5,6,7]
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm10 = ymm12[0,1,2,3],ymm10[4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm12 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
@@ -2119,19 +2125,21 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1,2],ymm5[3],ymm12[4,5],ymm5[6],ymm12[7],ymm5[8],ymm12[9,10],ymm5[11],ymm12[12,13],ymm5[14],ymm12[15]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm6 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm5, %ymm5
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3,4],xmm2[5],xmm1[6,7]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm5[5,6,7]
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm5[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm8, %ymm9, %ymm0
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1,2],ymm2[3],ymm0[4,5],ymm2[6],ymm0[7],ymm2[8],ymm0[9,10],ymm2[11],ymm0[12,13],ymm2[14],ymm0[15]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm0, %ymm0
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm2, %xmm2
-; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5,6,7]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm8, %ymm9, %ymm1
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm1[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm8[0],ymm1[1,2],ymm8[3],ymm1[4,5],ymm8[6],ymm1[7],ymm8[8],ymm1[9,10],ymm8[11],ymm1[12,13],ymm8[14],ymm1[15]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm1, %ymm1
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
+; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5,6,7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2],xmm0[3,4],xmm2[5],xmm0[6,7]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512DQ-FCP-NEXT:    vextracti32x4 $2, %zmm0, %xmm0
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm5[5,6,7]
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm7, (%rsi)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm10, (%rdx)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, (%rcx)
@@ -3492,668 +3500,688 @@ define void @load_i16_stride3_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-LABEL: load_i16_stride3_vf64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
-; AVX512-NEXT:    vmovdqa64 224(%rdi), %ymm18
-; AVX512-NEXT:    vmovdqa64 192(%rdi), %ymm20
+; AVX512-NEXT:    vmovdqa64 224(%rdi), %ymm20
+; AVX512-NEXT:    vmovdqa64 192(%rdi), %ymm21
 ; AVX512-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512-NEXT:    vpternlogq $202, %ymm18, %ymm20, %ymm1
-; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm7 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]
-; AVX512-NEXT:    vpshufb %ymm7, %ymm2, %ymm5
-; AVX512-NEXT:    vmovdqa 272(%rdi), %xmm1
+; AVX512-NEXT:    vpternlogq $202, %ymm20, %ymm21, %ymm1
+; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm1[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm3[1],ymm1[2,3],ymm3[4],ymm1[5,6],ymm3[7],ymm1[8],ymm3[9],ymm1[10,11],ymm3[12],ymm1[13,14],ymm3[15]
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]
+; AVX512-NEXT:    vpshufb %ymm3, %ymm1, %ymm5
+; AVX512-NEXT:    vmovdqa 272(%rdi), %xmm8
 ; AVX512-NEXT:    vmovdqa 256(%rdi), %xmm2
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm6 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]
-; AVX512-NEXT:    vmovdqa %xmm2, %xmm3
-; AVX512-NEXT:    vmovdqa64 %xmm1, %xmm19
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm13 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]
-; AVX512-NEXT:    vpshufb %xmm13, %xmm6, %xmm6
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm6 = xmm2[0,1],xmm8[2],xmm2[3,4],xmm8[5],xmm2[6,7]
+; AVX512-NEXT:    vmovdqa %xmm2, %xmm14
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm9 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]
+; AVX512-NEXT:    vpshufb %xmm9, %xmm6, %xmm6
 ; AVX512-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm6 = ymm5[0,1,2],ymm6[3,4,5,6,7],ymm5[8,9,10],ymm6[11,12,13,14,15]
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,6,5,4,7]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4,5,6,7]
-; AVX512-NEXT:    vmovdqa64 320(%rdi), %ymm21
-; AVX512-NEXT:    vmovdqa64 352(%rdi), %ymm22
-; AVX512-NEXT:    vmovdqa %ymm0, %ymm8
-; AVX512-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm8
-; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm8[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm8 = ymm8[0],ymm9[1],ymm8[2,3],ymm9[4],ymm8[5,6],ymm9[7],ymm8[8],ymm9[9],ymm8[10,11],ymm9[12],ymm8[13,14],ymm9[15]
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm10 = [0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
-; AVX512-NEXT:    vpshufb %ymm10, %ymm8, %ymm11
+; AVX512-NEXT:    vmovdqa64 320(%rdi), %ymm22
+; AVX512-NEXT:    vmovdqa64 352(%rdi), %ymm23
+; AVX512-NEXT:    vmovdqa %ymm0, %ymm6
+; AVX512-NEXT:    vpternlogq $202, %ymm22, %ymm23, %ymm6
+; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0],ymm7[1],ymm6[2,3],ymm7[4],ymm6[5,6],ymm7[7],ymm6[8],ymm7[9],ymm6[10,11],ymm7[12],ymm6[13,14],ymm7[15]
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm11 = [0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
+; AVX512-NEXT:    vpshufb %ymm11, %ymm6, %ymm12
 ; AVX512-NEXT:    vmovdqa 304(%rdi), %xmm1
 ; AVX512-NEXT:    vmovdqa 288(%rdi), %xmm2
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm13 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
 ; AVX512-NEXT:    vmovdqa %xmm2, %xmm4
-; AVX512-NEXT:    vmovdqa %xmm1, %xmm8
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm14 = [0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]
-; AVX512-NEXT:    vpshufb %xmm14, %xmm12, %xmm12
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm11 = ymm12[0,1,2],ymm11[3,4,5,6,7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm11, %zmm5, %zmm16
-; AVX512-NEXT:    vmovdqa64 128(%rdi), %ymm23
-; AVX512-NEXT:    vmovdqa 160(%rdi), %ymm11
+; AVX512-NEXT:    vmovdqa %xmm1, %xmm6
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm15 = [0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]
+; AVX512-NEXT:    vpshufb %xmm15, %xmm13, %xmm13
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm12 = ymm13[0,1,2],ymm12[3,4,5,6,7]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm12, %zmm5, %zmm16
+; AVX512-NEXT:    vmovdqa64 128(%rdi), %ymm24
+; AVX512-NEXT:    vmovdqa 160(%rdi), %ymm13
 ; AVX512-NEXT:    vmovdqa %ymm0, %ymm5
-; AVX512-NEXT:    vpternlogq $202, %ymm23, %ymm11, %ymm5
+; AVX512-NEXT:    vpternlogq $202, %ymm24, %ymm13, %ymm5
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm12 = ymm5[2,3,0,1]
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1],ymm5[2,3],ymm12[4],ymm5[5,6],ymm12[7],ymm5[8],ymm12[9],ymm5[10,11],ymm12[12],ymm5[13,14],ymm12[15]
-; AVX512-NEXT:    vpshufb %ymm10, %ymm5, %ymm10
-; AVX512-NEXT:    vmovdqa 112(%rdi), %xmm15
-; AVX512-NEXT:    vmovdqa 96(%rdi), %xmm5
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm12 = xmm5[0],xmm15[1],xmm5[2,3],xmm15[4],xmm5[5,6],xmm15[7]
-; AVX512-NEXT:    vpshufb %xmm14, %xmm12, %xmm12
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm6 = ymm12[0,1,2],ymm10[3,4,5,6,7]
-; AVX512-NEXT:    vmovdqa64 (%rdi), %ymm24
-; AVX512-NEXT:    vmovdqa 32(%rdi), %ymm12
+; AVX512-NEXT:    vpshufb %ymm11, %ymm5, %ymm5
+; AVX512-NEXT:    vmovdqa 112(%rdi), %xmm11
+; AVX512-NEXT:    vmovdqa 96(%rdi), %xmm12
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm10 = xmm12[0],xmm11[1],xmm12[2,3],xmm11[4],xmm12[5,6],xmm11[7]
+; AVX512-NEXT:    vpshufb %xmm15, %xmm10, %xmm10
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm10[0,1,2],ymm5[3,4,5,6,7]
+; AVX512-NEXT:    vmovdqa64 (%rdi), %ymm17
+; AVX512-NEXT:    vmovdqa 32(%rdi), %ymm5
 ; AVX512-NEXT:    vmovdqa %ymm0, %ymm10
-; AVX512-NEXT:    vpternlogq $202, %ymm12, %ymm24, %ymm10
-; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm10[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm10[0],ymm1[1],ymm10[2,3],ymm1[4],ymm10[5,6],ymm1[7],ymm10[8],ymm1[9],ymm10[10,11],ymm1[12],ymm10[13,14],ymm1[15]
-; AVX512-NEXT:    vpshufb %ymm7, %ymm1, %ymm7
+; AVX512-NEXT:    vpternlogq $202, %ymm5, %ymm17, %ymm10
+; AVX512-NEXT:    vpermq {{.*#+}} ymm15 = ymm10[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0],ymm15[1],ymm10[2,3],ymm15[4],ymm10[5,6],ymm15[7],ymm10[8],ymm15[9],ymm10[10,11],ymm15[12],ymm10[13,14],ymm15[15]
+; AVX512-NEXT:    vpshufb %ymm3, %ymm10, %ymm2
 ; AVX512-NEXT:    vmovdqa 80(%rdi), %xmm10
-; AVX512-NEXT:    vmovdqa 64(%rdi), %xmm1
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm2 = xmm1[0,1],xmm10[2],xmm1[3,4],xmm10[5],xmm1[6,7]
-; AVX512-NEXT:    vpshufb %xmm13, %xmm2, %xmm2
-; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm7[0,1,2],ymm2[3,4,5,6,7],ymm7[8,9,10],ymm2[11,12,13,14,15]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,6,5,4,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm2[4,5,6,7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm2, %zmm17
-; AVX512-NEXT:    vmovdqa %ymm0, %ymm2
-; AVX512-NEXT:    vpternlogq $202, %ymm22, %ymm21, %ymm2
-; AVX512-NEXT:    vpermq {{.*#+}} ymm6 = ymm2[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm6[2],ymm2[3,4],ymm6[5],ymm2[6,7,8,9],ymm6[10],ymm2[11,12],ymm6[13],ymm2[14,15]
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm9 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
-; AVX512-NEXT:    vpshufb %ymm9, %ymm2, %ymm2
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm7 = xmm4[0,1],xmm8[2],xmm4[3,4],xmm8[5],xmm4[6,7]
-; AVX512-NEXT:    vmovdqa64 %xmm8, %xmm25
+; AVX512-NEXT:    vmovdqa 64(%rdi), %xmm15
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm3 = xmm15[0,1],xmm10[2],xmm15[3,4],xmm10[5],xmm15[6,7]
+; AVX512-NEXT:    vpshufb %xmm9, %xmm3, %xmm3
+; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm2[0,1,2],ymm3[3,4,5,6,7],ymm2[8,9,10],ymm3[11,12,13,14,15]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,4,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm18
+; AVX512-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512-NEXT:    vpternlogq $202, %ymm23, %ymm22, %ymm1
+; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7,8,9],ymm2[10],ymm1[11,12],ymm2[13],ymm1[14,15]
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm2 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
+; AVX512-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
+; AVX512-NEXT:    vmovdqa64 %ymm2, %ymm28
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1],xmm6[2],xmm4[3,4],xmm6[5],xmm4[6,7]
+; AVX512-NEXT:    vmovdqa64 %xmm6, %xmm25
 ; AVX512-NEXT:    vmovdqa64 %xmm4, %xmm26
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm6 = [2,3,8,9,14,15,4,5,10,11,10,11,10,11,10,11]
-; AVX512-NEXT:    vpshufb %xmm6, %xmm7, %xmm7
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm7 = xmm7[0,1,2,3,4],xmm2[5,6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm2[4,5,6,7]
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm13 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
-; AVX512-NEXT:    vmovdqa %ymm13, %ymm2
-; AVX512-NEXT:    vpternlogq $202, %ymm20, %ymm18, %ymm2
-; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm2[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm4[2],ymm2[3,4],ymm4[5],ymm2[6,7,8,9],ymm4[10],ymm2[11,12],ymm4[13],ymm2[14,15]
+; AVX512-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm1[5,6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm9 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
+; AVX512-NEXT:    vmovdqa %ymm9, %ymm1
+; AVX512-NEXT:    vpternlogq $202, %ymm21, %ymm20, %ymm1
+; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm1[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm4[2],ymm1[3,4],ymm4[5],ymm1[6,7,8,9],ymm4[10],ymm1[11,12],ymm4[13],ymm1[14,15]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm4 = [2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]
-; AVX512-NEXT:    vpshufb %ymm4, %ymm2, %ymm2
-; AVX512-NEXT:    vmovdqa64 %xmm19, %xmm8
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm14 = xmm8[0,1],xmm3[2],xmm8[3,4],xmm3[5],xmm8[6,7]
-; AVX512-NEXT:    vmovdqa64 %xmm3, %xmm27
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]
-; AVX512-NEXT:    vpshufb %xmm3, %xmm14, %xmm14
+; AVX512-NEXT:    vpshufb %ymm4, %ymm1, %ymm1
+; AVX512-NEXT:    vmovdqa %xmm14, %xmm7
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm14 = xmm8[0,1],xmm14[2],xmm8[3,4],xmm14[5],xmm8[6,7]
+; AVX512-NEXT:    vmovdqa64 %xmm8, %xmm27
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]
+; AVX512-NEXT:    vpshufb %xmm2, %xmm14, %xmm14
 ; AVX512-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm14 = ymm2[0,1,2],ymm14[3,4,5,6,7],ymm2[8,9,10],ymm14[11,12,13,14,15]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,6,7,4]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm14[4,5,6,7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm2, %zmm19
-; AVX512-NEXT:    vmovdqa %ymm0, %ymm2
-; AVX512-NEXT:    vpternlogq $202, %ymm11, %ymm23, %ymm2
-; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm2[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm7[2],ymm2[3,4],ymm7[5],ymm2[6,7,8,9],ymm7[10],ymm2[11,12],ymm7[13],ymm2[14,15]
-; AVX512-NEXT:    vpshufb %ymm9, %ymm2, %ymm2
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm7 = xmm5[0,1],xmm15[2],xmm5[3,4],xmm15[5],xmm5[6,7]
-; AVX512-NEXT:    vpshufb %xmm6, %xmm7, %xmm6
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4],xmm2[5,6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm2[4,5,6,7]
-; AVX512-NEXT:    vmovdqa %ymm13, %ymm6
-; AVX512-NEXT:    vpternlogq $202, %ymm24, %ymm12, %ymm6
-; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0,1],ymm7[2],ymm6[3,4],ymm7[5],ymm6[6,7,8,9],ymm7[10],ymm6[11,12],ymm7[13],ymm6[14,15]
-; AVX512-NEXT:    vpshufb %ymm4, %ymm6, %ymm4
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm6 = xmm10[0,1],xmm1[2],xmm10[3,4],xmm1[5],xmm10[6,7]
-; AVX512-NEXT:    vpshufb %xmm3, %xmm6, %xmm3
-; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3,4,5,6,7],ymm4[8,9,10],ymm3[11,12,13,14,15]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,6,7,4]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
-; AVX512-NEXT:    vpternlogq $226, %ymm23, %ymm13, %ymm11
-; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm11[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm11[1,2],ymm3[3],ymm11[4,5],ymm3[6],ymm11[7],ymm3[8],ymm11[9,10],ymm3[11],ymm11[12,13],ymm3[14],ymm11[15]
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm11 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
-; AVX512-NEXT:    vpshufb %ymm11, %ymm3, %ymm3
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm15[0,1],xmm5[2],xmm15[3,4],xmm5[5],xmm15[6,7]
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm5 = [4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
-; AVX512-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
-; AVX512-NEXT:    vpternlogq $226, %ymm24, %ymm0, %ymm12
-; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm12[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0],ymm12[1,2],ymm4[3],ymm12[4,5],ymm4[6],ymm12[7],ymm4[8],ymm12[9,10],ymm4[11],ymm12[12,13],ymm4[14],ymm12[15]
-; AVX512-NEXT:    vpshufb %ymm11, %ymm4, %ymm4
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm10[1],xmm1[2,3],xmm10[4],xmm1[5,6],xmm10[7]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm14 = ymm1[0,1,2],ymm14[3,4,5,6,7],ymm1[8,9,10],ymm14[11,12,13,14,15]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,6,7,4]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm14[4,5,6,7]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm19
+; AVX512-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512-NEXT:    vpternlogq $202, %ymm13, %ymm24, %ymm1
+; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm1[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm3[2],ymm1[3,4],ymm3[5],ymm1[6,7,8,9],ymm3[10],ymm1[11,12],ymm3[13],ymm1[14,15]
+; AVX512-NEXT:    vmovdqa64 %ymm28, %ymm3
+; AVX512-NEXT:    vpshufb %ymm3, %ymm1, %ymm1
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm3 = xmm12[0,1],xmm11[2],xmm12[3,4],xmm11[5],xmm12[6,7]
+; AVX512-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm1[5,6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX512-NEXT:    vmovdqa %ymm9, %ymm3
+; AVX512-NEXT:    vpternlogq $202, %ymm17, %ymm5, %ymm3
+; AVX512-NEXT:    vpermq {{.*#+}} ymm6 = ymm3[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1],ymm6[2],ymm3[3,4],ymm6[5],ymm3[6,7,8,9],ymm6[10],ymm3[11,12],ymm6[13],ymm3[14,15]
+; AVX512-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm10[0,1],xmm15[2],xmm10[3,4],xmm15[5],xmm10[6,7]
+; AVX512-NEXT:    vpshufb %xmm2, %xmm4, %xmm2
+; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3,4,5,6,7],ymm3[8,9,10],ymm2[11,12,13,14,15]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,6,7,4]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
+; AVX512-NEXT:    vpternlogq $226, %ymm24, %ymm9, %ymm13
+; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm13[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm13[1,2],ymm2[3],ymm13[4,5],ymm2[6],ymm13[7],ymm2[8],ymm13[9,10],ymm2[11],ymm13[12,13],ymm2[14],ymm13[15]
+; AVX512-NEXT:    vpternlogq $226, %ymm17, %ymm0, %ymm5
+; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm5[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm5[1,2],ymm3[3],ymm5[4,5],ymm3[6],ymm5[7],ymm3[8],ymm5[9,10],ymm3[11],ymm5[12,13],ymm3[14],ymm5[15]
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm4 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
+; AVX512-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm5 = xmm15[0],xmm10[1],xmm15[2,3],xmm10[4],xmm15[5,6],xmm10[7]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm6 = [0,1,2,3,0,1,6,7,12,13,2,3,8,9,14,15]
-; AVX512-NEXT:    vpshufb %xmm6, %xmm1, %xmm1
-; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4],ymm1[5,6,7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm13
-; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm13[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm13[1,2],ymm3[3],ymm13[4,5],ymm3[6],ymm13[7],ymm3[8],ymm13[9,10],ymm3[11],ymm13[12,13],ymm3[14],ymm13[15]
-; AVX512-NEXT:    vmovdqa64 %xmm25, %xmm4
-; AVX512-NEXT:    vmovdqa64 %xmm26, %xmm7
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm7[2],xmm4[3,4],xmm7[5],xmm4[6,7]
-; AVX512-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
-; AVX512-NEXT:    vpshufb %ymm11, %ymm3, %ymm3
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
-; AVX512-NEXT:    vpternlogq $202, %ymm20, %ymm18, %ymm0
-; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm0[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm4[0],ymm0[1,2],ymm4[3],ymm0[4,5],ymm4[6],ymm0[7],ymm4[8],ymm0[9,10],ymm4[11],ymm0[12,13],ymm4[14],ymm0[15]
-; AVX512-NEXT:    vpshufb %ymm11, %ymm0, %ymm0
+; AVX512-NEXT:    vpshufb %xmm6, %xmm5, %xmm5
+; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm5[5,6,7]
+; AVX512-NEXT:    vpshufb %ymm4, %ymm2, %ymm2
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm5 = xmm11[0,1],xmm12[2],xmm11[3,4],xmm12[5],xmm11[6,7]
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm8 = [4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
+; AVX512-NEXT:    vpshufb %xmm8, %xmm5, %xmm5
+; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm3, %zmm5
+; AVX512-NEXT:    vextracti32x4 $2, %zmm5, %xmm5
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4],xmm2[5,6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
+; AVX512-NEXT:    vpternlogq $202, %ymm22, %ymm23, %ymm9
+; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm9[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm9[1,2],ymm3[3],ymm9[4,5],ymm3[6],ymm9[7],ymm3[8],ymm9[9,10],ymm3[11],ymm9[12,13],ymm3[14],ymm9[15]
+; AVX512-NEXT:    vpternlogq $202, %ymm21, %ymm20, %ymm0
+; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm0[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm5[0],ymm0[1,2],ymm5[3],ymm0[4,5],ymm5[6],ymm0[7],ymm5[8],ymm0[9,10],ymm5[11],ymm0[12,13],ymm5[14],ymm0[15]
+; AVX512-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
+; AVX512-NEXT:    vpshufb %ymm4, %ymm0, %ymm0
 ; AVX512-NEXT:    vmovdqa64 %xmm27, %xmm4
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0],xmm8[1],xmm4[2,3],xmm8[4],xmm4[5,6],xmm8[7]
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm7[0],xmm4[1],xmm7[2,3],xmm4[4],xmm7[5,6],xmm4[7]
 ; AVX512-NEXT:    vpshufb %xmm6, %xmm4, %xmm4
 ; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm4[5,6,7]
+; AVX512-NEXT:    vmovdqa64 %xmm25, %xmm4
+; AVX512-NEXT:    vmovdqa64 %xmm26, %xmm5
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2],xmm4[3,4],xmm5[5],xmm4[6,7]
+; AVX512-NEXT:    vpshufb %xmm8, %xmm4, %xmm4
+; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm4
+; AVX512-NEXT:    vextracti32x4 $2, %zmm4, %xmm4
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
-; AVX512-NEXT:    vmovdqa64 %zmm17, (%rsi)
+; AVX512-NEXT:    vmovdqa64 %zmm18, (%rsi)
 ; AVX512-NEXT:    vmovdqa64 %zmm16, 64(%rsi)
 ; AVX512-NEXT:    vmovdqa64 %zmm19, 64(%rdx)
-; AVX512-NEXT:    vmovdqa64 %zmm2, (%rdx)
+; AVX512-NEXT:    vmovdqa64 %zmm1, (%rdx)
 ; AVX512-NEXT:    vmovdqa64 %zmm0, 64(%rcx)
-; AVX512-NEXT:    vmovdqa64 %zmm1, (%rcx)
+; AVX512-NEXT:    vmovdqa64 %zmm2, (%rcx)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
 ; AVX512-FCP-LABEL: load_i16_stride3_vf64:
 ; AVX512-FCP:       # %bb.0:
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
-; AVX512-FCP-NEXT:    vmovdqa64 224(%rdi), %ymm18
-; AVX512-FCP-NEXT:    vmovdqa64 192(%rdi), %ymm20
+; AVX512-FCP-NEXT:    vmovdqa64 224(%rdi), %ymm20
+; AVX512-FCP-NEXT:    vmovdqa64 192(%rdi), %ymm21
 ; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm18, %ymm20, %ymm1
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm7 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]
-; AVX512-FCP-NEXT:    vpshufb %ymm7, %ymm2, %ymm5
-; AVX512-FCP-NEXT:    vmovdqa 272(%rdi), %xmm1
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm20, %ymm21, %ymm1
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm1[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm3[1],ymm1[2,3],ymm3[4],ymm1[5,6],ymm3[7],ymm1[8],ymm3[9],ymm1[10,11],ymm3[12],ymm1[13,14],ymm3[15]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]
+; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm1, %ymm5
+; AVX512-FCP-NEXT:    vmovdqa 272(%rdi), %xmm8
 ; AVX512-FCP-NEXT:    vmovdqa 256(%rdi), %xmm2
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm6 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]
-; AVX512-FCP-NEXT:    vmovdqa %xmm2, %xmm3
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm1, %xmm19
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm13 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]
-; AVX512-FCP-NEXT:    vpshufb %xmm13, %xmm6, %xmm6
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm6 = xmm2[0,1],xmm8[2],xmm2[3,4],xmm8[5],xmm2[6,7]
+; AVX512-FCP-NEXT:    vmovdqa %xmm2, %xmm14
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm9 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]
+; AVX512-FCP-NEXT:    vpshufb %xmm9, %xmm6, %xmm6
 ; AVX512-FCP-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm5[0,1,2],ymm6[3,4,5,6,7],ymm5[8,9,10],ymm6[11,12,13,14,15]
 ; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,6,5,4,7]
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4,5,6,7]
-; AVX512-FCP-NEXT:    vmovdqa64 320(%rdi), %ymm21
-; AVX512-FCP-NEXT:    vmovdqa64 352(%rdi), %ymm22
-; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm8
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm8
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm8[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm8 = ymm8[0],ymm9[1],ymm8[2,3],ymm9[4],ymm8[5,6],ymm9[7],ymm8[8],ymm9[9],ymm8[10,11],ymm9[12],ymm8[13,14],ymm9[15]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm10 = [0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
-; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm8, %ymm11
+; AVX512-FCP-NEXT:    vmovdqa64 320(%rdi), %ymm22
+; AVX512-FCP-NEXT:    vmovdqa64 352(%rdi), %ymm23
+; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm6
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm22, %ymm23, %ymm6
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0],ymm7[1],ymm6[2,3],ymm7[4],ymm6[5,6],ymm7[7],ymm6[8],ymm7[9],ymm6[10,11],ymm7[12],ymm6[13,14],ymm7[15]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm11 = [0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
+; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm6, %ymm12
 ; AVX512-FCP-NEXT:    vmovdqa 304(%rdi), %xmm1
 ; AVX512-FCP-NEXT:    vmovdqa 288(%rdi), %xmm2
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm13 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
 ; AVX512-FCP-NEXT:    vmovdqa %xmm2, %xmm4
-; AVX512-FCP-NEXT:    vmovdqa %xmm1, %xmm8
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm14 = [0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]
-; AVX512-FCP-NEXT:    vpshufb %xmm14, %xmm12, %xmm12
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm11 = ymm12[0,1,2],ymm11[3,4,5,6,7]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm5, %zmm16
-; AVX512-FCP-NEXT:    vmovdqa64 128(%rdi), %ymm23
-; AVX512-FCP-NEXT:    vmovdqa 160(%rdi), %ymm11
+; AVX512-FCP-NEXT:    vmovdqa %xmm1, %xmm6
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm15 = [0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]
+; AVX512-FCP-NEXT:    vpshufb %xmm15, %xmm13, %xmm13
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm12 = ymm13[0,1,2],ymm12[3,4,5,6,7]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm5, %zmm16
+; AVX512-FCP-NEXT:    vmovdqa64 128(%rdi), %ymm24
+; AVX512-FCP-NEXT:    vmovdqa 160(%rdi), %ymm13
 ; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm5
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm23, %ymm11, %ymm5
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm24, %ymm13, %ymm5
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm12 = ymm5[2,3,0,1]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1],ymm5[2,3],ymm12[4],ymm5[5,6],ymm12[7],ymm5[8],ymm12[9],ymm5[10,11],ymm12[12],ymm5[13,14],ymm12[15]
-; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm5, %ymm10
-; AVX512-FCP-NEXT:    vmovdqa 112(%rdi), %xmm15
-; AVX512-FCP-NEXT:    vmovdqa 96(%rdi), %xmm5
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm12 = xmm5[0],xmm15[1],xmm5[2,3],xmm15[4],xmm5[5,6],xmm15[7]
-; AVX512-FCP-NEXT:    vpshufb %xmm14, %xmm12, %xmm12
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm6 = ymm12[0,1,2],ymm10[3,4,5,6,7]
-; AVX512-FCP-NEXT:    vmovdqa64 (%rdi), %ymm24
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %ymm12
+; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm5, %ymm5
+; AVX512-FCP-NEXT:    vmovdqa 112(%rdi), %xmm11
+; AVX512-FCP-NEXT:    vmovdqa 96(%rdi), %xmm12
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm10 = xmm12[0],xmm11[1],xmm12[2,3],xmm11[4],xmm12[5,6],xmm11[7]
+; AVX512-FCP-NEXT:    vpshufb %xmm15, %xmm10, %xmm10
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm10[0,1,2],ymm5[3,4,5,6,7]
+; AVX512-FCP-NEXT:    vmovdqa64 (%rdi), %ymm17
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %ymm5
 ; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm10
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm12, %ymm24, %ymm10
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm10[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm10[0],ymm1[1],ymm10[2,3],ymm1[4],ymm10[5,6],ymm1[7],ymm10[8],ymm1[9],ymm10[10,11],ymm1[12],ymm10[13,14],ymm1[15]
-; AVX512-FCP-NEXT:    vpshufb %ymm7, %ymm1, %ymm7
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm5, %ymm17, %ymm10
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm15 = ymm10[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0],ymm15[1],ymm10[2,3],ymm15[4],ymm10[5,6],ymm15[7],ymm10[8],ymm15[9],ymm10[10,11],ymm15[12],ymm10[13,14],ymm15[15]
+; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm10, %ymm2
 ; AVX512-FCP-NEXT:    vmovdqa 80(%rdi), %xmm10
-; AVX512-FCP-NEXT:    vmovdqa 64(%rdi), %xmm1
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm2 = xmm1[0,1],xmm10[2],xmm1[3,4],xmm10[5],xmm1[6,7]
-; AVX512-FCP-NEXT:    vpshufb %xmm13, %xmm2, %xmm2
-; AVX512-FCP-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm7[0,1,2],ymm2[3,4,5,6,7],ymm7[8,9,10],ymm2[11,12,13,14,15]
-; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,6,5,4,7]
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm2[4,5,6,7]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm2, %zmm17
-; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm2
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm22, %ymm21, %ymm2
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm2[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm6[2],ymm2[3,4],ymm6[5],ymm2[6,7,8,9],ymm6[10],ymm2[11,12],ymm6[13],ymm2[14,15]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm9 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
-; AVX512-FCP-NEXT:    vpshufb %ymm9, %ymm2, %ymm2
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm7 = xmm4[0,1],xmm8[2],xmm4[3,4],xmm8[5],xmm4[6,7]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm8, %xmm25
+; AVX512-FCP-NEXT:    vmovdqa 64(%rdi), %xmm15
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm15[0,1],xmm10[2],xmm15[3,4],xmm10[5],xmm15[6,7]
+; AVX512-FCP-NEXT:    vpshufb %xmm9, %xmm3, %xmm3
+; AVX512-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm2[0,1,2],ymm3[3,4,5,6,7],ymm2[8,9,10],ymm3[11,12,13,14,15]
+; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,4,7]
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm18
+; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm23, %ymm22, %ymm1
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7,8,9],ymm2[10],ymm1[11,12],ymm2[13],ymm1[14,15]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm2 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
+; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm2, %ymm28
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1],xmm6[2],xmm4[3,4],xmm6[5],xmm4[6,7]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm6, %xmm25
 ; AVX512-FCP-NEXT:    vmovdqa64 %xmm4, %xmm26
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm6 = [2,3,8,9,14,15,4,5,10,11,10,11,10,11,10,11]
-; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm7, %xmm7
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm7 = xmm7[0,1,2,3,4],xmm2[5,6,7]
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm2[4,5,6,7]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm13 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
-; AVX512-FCP-NEXT:    vmovdqa %ymm13, %ymm2
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm20, %ymm18, %ymm2
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm2[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm4[2],ymm2[3,4],ymm4[5],ymm2[6,7,8,9],ymm4[10],ymm2[11,12],ymm4[13],ymm2[14,15]
+; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm1[5,6,7]
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm9 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
+; AVX512-FCP-NEXT:    vmovdqa %ymm9, %ymm1
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm21, %ymm20, %ymm1
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm1[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm4[2],ymm1[3,4],ymm4[5],ymm1[6,7,8,9],ymm4[10],ymm1[11,12],ymm4[13],ymm1[14,15]
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm4 = [2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]
-; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm2, %ymm2
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm19, %xmm8
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm14 = xmm8[0,1],xmm3[2],xmm8[3,4],xmm3[5],xmm8[6,7]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm3, %xmm27
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]
-; AVX512-FCP-NEXT:    vpshufb %xmm3, %xmm14, %xmm14
+; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm1, %ymm1
+; AVX512-FCP-NEXT:    vmovdqa %xmm14, %xmm7
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm14 = xmm8[0,1],xmm14[2],xmm8[3,4],xmm14[5],xmm8[6,7]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm8, %xmm27
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]
+; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm14, %xmm14
 ; AVX512-FCP-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm14 = ymm2[0,1,2],ymm14[3,4,5,6,7],ymm2[8,9,10],ymm14[11,12,13,14,15]
-; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,6,7,4]
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm14[4,5,6,7]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm2, %zmm19
-; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm2
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm11, %ymm23, %ymm2
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm2[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm7[2],ymm2[3,4],ymm7[5],ymm2[6,7,8,9],ymm7[10],ymm2[11,12],ymm7[13],ymm2[14,15]
-; AVX512-FCP-NEXT:    vpshufb %ymm9, %ymm2, %ymm2
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm7 = xmm5[0,1],xmm15[2],xmm5[3,4],xmm15[5],xmm5[6,7]
-; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm7, %xmm6
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4],xmm2[5,6,7]
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm2[4,5,6,7]
-; AVX512-FCP-NEXT:    vmovdqa %ymm13, %ymm6
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm24, %ymm12, %ymm6
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0,1],ymm7[2],ymm6[3,4],ymm7[5],ymm6[6,7,8,9],ymm7[10],ymm6[11,12],ymm7[13],ymm6[14,15]
-; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm6, %ymm4
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm6 = xmm10[0,1],xmm1[2],xmm10[3,4],xmm1[5],xmm10[6,7]
-; AVX512-FCP-NEXT:    vpshufb %xmm3, %xmm6, %xmm3
-; AVX512-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3,4,5,6,7],ymm4[8,9,10],ymm3[11,12,13,14,15]
-; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,6,7,4]
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
-; AVX512-FCP-NEXT:    vpternlogq $226, %ymm23, %ymm13, %ymm11
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm11[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm11[1,2],ymm3[3],ymm11[4,5],ymm3[6],ymm11[7],ymm3[8],ymm11[9,10],ymm3[11],ymm11[12,13],ymm3[14],ymm11[15]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm11 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
-; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm3, %ymm3
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm15[0,1],xmm5[2],xmm15[3,4],xmm5[5],xmm15[6,7]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm5 = [4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
-; AVX512-FCP-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
-; AVX512-FCP-NEXT:    vpternlogq $226, %ymm24, %ymm0, %ymm12
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm12[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0],ymm12[1,2],ymm4[3],ymm12[4,5],ymm4[6],ymm12[7],ymm4[8],ymm12[9,10],ymm4[11],ymm12[12,13],ymm4[14],ymm12[15]
-; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm4, %ymm4
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm10[1],xmm1[2,3],xmm10[4],xmm1[5,6],xmm10[7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm14 = ymm1[0,1,2],ymm14[3,4,5,6,7],ymm1[8,9,10],ymm14[11,12,13,14,15]
+; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,6,7,4]
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm14[4,5,6,7]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm19
+; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm13, %ymm24, %ymm1
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm1[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm3[2],ymm1[3,4],ymm3[5],ymm1[6,7,8,9],ymm3[10],ymm1[11,12],ymm3[13],ymm1[14,15]
+; AVX512-FCP-NEXT:    vmovdqa64 %ymm28, %ymm3
+; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm1, %ymm1
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm12[0,1],xmm11[2],xmm12[3,4],xmm11[5],xmm12[6,7]
+; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm1[5,6,7]
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX512-FCP-NEXT:    vmovdqa %ymm9, %ymm3
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm17, %ymm5, %ymm3
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm3[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1],ymm6[2],ymm3[3,4],ymm6[5],ymm3[6,7,8,9],ymm6[10],ymm3[11,12],ymm6[13],ymm3[14,15]
+; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm10[0,1],xmm15[2],xmm10[3,4],xmm15[5],xmm10[6,7]
+; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm4, %xmm2
+; AVX512-FCP-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3,4,5,6,7],ymm3[8,9,10],ymm2[11,12,13,14,15]
+; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,6,7,4]
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
+; AVX512-FCP-NEXT:    vpternlogq $226, %ymm24, %ymm9, %ymm13
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm13[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm13[1,2],ymm2[3],ymm13[4,5],ymm2[6],ymm13[7],ymm2[8],ymm13[9,10],ymm2[11],ymm13[12,13],ymm2[14],ymm13[15]
+; AVX512-FCP-NEXT:    vpternlogq $226, %ymm17, %ymm0, %ymm5
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm5[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm5[1,2],ymm3[3],ymm5[4,5],ymm3[6],ymm5[7],ymm3[8],ymm5[9,10],ymm3[11],ymm5[12,13],ymm3[14],ymm5[15]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm4 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
+; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm5 = xmm15[0],xmm10[1],xmm15[2,3],xmm10[4],xmm15[5,6],xmm10[7]
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm6 = [0,1,2,3,0,1,6,7,12,13,2,3,8,9,14,15]
-; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm1, %xmm1
-; AVX512-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4],ymm1[5,6,7]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm13
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm13[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm13[1,2],ymm3[3],ymm13[4,5],ymm3[6],ymm13[7],ymm3[8],ymm13[9,10],ymm3[11],ymm13[12,13],ymm3[14],ymm13[15]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm25, %xmm4
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm26, %xmm7
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm7[2],xmm4[3,4],xmm7[5],xmm4[6,7]
-; AVX512-FCP-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
-; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm3, %ymm3
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm20, %ymm18, %ymm0
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm0[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm4[0],ymm0[1,2],ymm4[3],ymm0[4,5],ymm4[6],ymm0[7],ymm4[8],ymm0[9,10],ymm4[11],ymm0[12,13],ymm4[14],ymm0[15]
-; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm0, %ymm0
+; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm5, %xmm5
+; AVX512-FCP-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm5[5,6,7]
+; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm2, %ymm2
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm5 = xmm11[0,1],xmm12[2],xmm11[3,4],xmm12[5],xmm11[6,7]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm8 = [4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
+; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm5, %xmm5
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm3, %zmm5
+; AVX512-FCP-NEXT:    vextracti32x4 $2, %zmm5, %xmm5
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4],xmm2[5,6,7]
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm22, %ymm23, %ymm9
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm9[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm9[1,2],ymm3[3],ymm9[4,5],ymm3[6],ymm9[7],ymm3[8],ymm9[9,10],ymm3[11],ymm9[12,13],ymm3[14],ymm9[15]
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm21, %ymm20, %ymm0
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm5 = ymm0[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm5[0],ymm0[1,2],ymm5[3],ymm0[4,5],ymm5[6],ymm0[7],ymm5[8],ymm0[9,10],ymm5[11],ymm0[12,13],ymm5[14],ymm0[15]
+; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
+; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm0, %ymm0
 ; AVX512-FCP-NEXT:    vmovdqa64 %xmm27, %xmm4
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0],xmm8[1],xmm4[2,3],xmm8[4],xmm4[5,6],xmm8[7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm7[0],xmm4[1],xmm7[2,3],xmm4[4],xmm7[5,6],xmm4[7]
 ; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm4, %xmm4
 ; AVX512-FCP-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm4[5,6,7]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm25, %xmm4
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm26, %xmm5
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2],xmm4[3,4],xmm5[5],xmm4[6,7]
+; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm4, %xmm4
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm4
+; AVX512-FCP-NEXT:    vextracti32x4 $2, %zmm4, %xmm4
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm17, (%rsi)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm18, (%rsi)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm16, 64(%rsi)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm19, 64(%rdx)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, (%rdx)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, (%rdx)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, 64(%rcx)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, (%rcx)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, (%rcx)
 ; AVX512-FCP-NEXT:    vzeroupper
 ; AVX512-FCP-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: load_i16_stride3_vf64:
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
-; AVX512DQ-NEXT:    vmovdqa64 224(%rdi), %ymm18
-; AVX512DQ-NEXT:    vmovdqa64 192(%rdi), %ymm20
+; AVX512DQ-NEXT:    vmovdqa64 224(%rdi), %ymm20
+; AVX512DQ-NEXT:    vmovdqa64 192(%rdi), %ymm21
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm18, %ymm20, %ymm1
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm7 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]
-; AVX512DQ-NEXT:    vpshufb %ymm7, %ymm2, %ymm5
-; AVX512DQ-NEXT:    vmovdqa 272(%rdi), %xmm1
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm20, %ymm21, %ymm1
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm1[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm3[1],ymm1[2,3],ymm3[4],ymm1[5,6],ymm3[7],ymm1[8],ymm3[9],ymm1[10,11],ymm3[12],ymm1[13,14],ymm3[15]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]
+; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm1, %ymm5
+; AVX512DQ-NEXT:    vmovdqa 272(%rdi), %xmm8
 ; AVX512DQ-NEXT:    vmovdqa 256(%rdi), %xmm2
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm6 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]
-; AVX512DQ-NEXT:    vmovdqa %xmm2, %xmm3
-; AVX512DQ-NEXT:    vmovdqa64 %xmm1, %xmm19
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm13 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]
-; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm6, %xmm6
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm6 = xmm2[0,1],xmm8[2],xmm2[3,4],xmm8[5],xmm2[6,7]
+; AVX512DQ-NEXT:    vmovdqa %xmm2, %xmm14
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm9 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]
+; AVX512DQ-NEXT:    vpshufb %xmm9, %xmm6, %xmm6
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm6 = ymm5[0,1,2],ymm6[3,4,5,6,7],ymm5[8,9,10],ymm6[11,12,13,14,15]
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,6,5,4,7]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa64 320(%rdi), %ymm21
-; AVX512DQ-NEXT:    vmovdqa64 352(%rdi), %ymm22
-; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm8
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm8
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm8[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm8 = ymm8[0],ymm9[1],ymm8[2,3],ymm9[4],ymm8[5,6],ymm9[7],ymm8[8],ymm9[9],ymm8[10,11],ymm9[12],ymm8[13,14],ymm9[15]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm10 = [0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
-; AVX512DQ-NEXT:    vpshufb %ymm10, %ymm8, %ymm11
+; AVX512DQ-NEXT:    vmovdqa64 320(%rdi), %ymm22
+; AVX512DQ-NEXT:    vmovdqa64 352(%rdi), %ymm23
+; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm6
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm22, %ymm23, %ymm6
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0],ymm7[1],ymm6[2,3],ymm7[4],ymm6[5,6],ymm7[7],ymm6[8],ymm7[9],ymm6[10,11],ymm7[12],ymm6[13,14],ymm7[15]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm11 = [0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
+; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm6, %ymm12
 ; AVX512DQ-NEXT:    vmovdqa 304(%rdi), %xmm1
 ; AVX512DQ-NEXT:    vmovdqa 288(%rdi), %xmm2
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm13 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
 ; AVX512DQ-NEXT:    vmovdqa %xmm2, %xmm4
-; AVX512DQ-NEXT:    vmovdqa %xmm1, %xmm8
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm14 = [0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]
-; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm12, %xmm12
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm11 = ymm12[0,1,2],ymm11[3,4,5,6,7]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm11, %zmm5, %zmm16
-; AVX512DQ-NEXT:    vmovdqa64 128(%rdi), %ymm23
-; AVX512DQ-NEXT:    vmovdqa 160(%rdi), %ymm11
+; AVX512DQ-NEXT:    vmovdqa %xmm1, %xmm6
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm15 = [0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufb %xmm15, %xmm13, %xmm13
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm12 = ymm13[0,1,2],ymm12[3,4,5,6,7]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm12, %zmm5, %zmm16
+; AVX512DQ-NEXT:    vmovdqa64 128(%rdi), %ymm24
+; AVX512DQ-NEXT:    vmovdqa 160(%rdi), %ymm13
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm5
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm23, %ymm11, %ymm5
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm24, %ymm13, %ymm5
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm12 = ymm5[2,3,0,1]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1],ymm5[2,3],ymm12[4],ymm5[5,6],ymm12[7],ymm5[8],ymm12[9],ymm5[10,11],ymm12[12],ymm5[13,14],ymm12[15]
-; AVX512DQ-NEXT:    vpshufb %ymm10, %ymm5, %ymm10
-; AVX512DQ-NEXT:    vmovdqa 112(%rdi), %xmm15
-; AVX512DQ-NEXT:    vmovdqa 96(%rdi), %xmm5
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm12 = xmm5[0],xmm15[1],xmm5[2,3],xmm15[4],xmm5[5,6],xmm15[7]
-; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm12, %xmm12
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm6 = ymm12[0,1,2],ymm10[3,4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa64 (%rdi), %ymm24
-; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm12
+; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm5, %ymm5
+; AVX512DQ-NEXT:    vmovdqa 112(%rdi), %xmm11
+; AVX512DQ-NEXT:    vmovdqa 96(%rdi), %xmm12
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm10 = xmm12[0],xmm11[1],xmm12[2,3],xmm11[4],xmm12[5,6],xmm11[7]
+; AVX512DQ-NEXT:    vpshufb %xmm15, %xmm10, %xmm10
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm10[0,1,2],ymm5[3,4,5,6,7]
+; AVX512DQ-NEXT:    vmovdqa64 (%rdi), %ymm17
+; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm5
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm10
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm12, %ymm24, %ymm10
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm10[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm10[0],ymm1[1],ymm10[2,3],ymm1[4],ymm10[5,6],ymm1[7],ymm10[8],ymm1[9],ymm10[10,11],ymm1[12],ymm10[13,14],ymm1[15]
-; AVX512DQ-NEXT:    vpshufb %ymm7, %ymm1, %ymm7
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm5, %ymm17, %ymm10
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm15 = ymm10[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0],ymm15[1],ymm10[2,3],ymm15[4],ymm10[5,6],ymm15[7],ymm10[8],ymm15[9],ymm10[10,11],ymm15[12],ymm10[13,14],ymm15[15]
+; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm10, %ymm2
 ; AVX512DQ-NEXT:    vmovdqa 80(%rdi), %xmm10
-; AVX512DQ-NEXT:    vmovdqa 64(%rdi), %xmm1
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm2 = xmm1[0,1],xmm10[2],xmm1[3,4],xmm10[5],xmm1[6,7]
-; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm2, %xmm2
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm7[0,1,2],ymm2[3,4,5,6,7],ymm7[8,9,10],ymm2[11,12,13,14,15]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,6,5,4,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm6, %zmm2, %zmm17
-; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm2
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm22, %ymm21, %ymm2
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm6 = ymm2[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm6[2],ymm2[3,4],ymm6[5],ymm2[6,7,8,9],ymm6[10],ymm2[11,12],ymm6[13],ymm2[14,15]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm9 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
-; AVX512DQ-NEXT:    vpshufb %ymm9, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm7 = xmm4[0,1],xmm8[2],xmm4[3,4],xmm8[5],xmm4[6,7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm8, %xmm25
+; AVX512DQ-NEXT:    vmovdqa 64(%rdi), %xmm15
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm3 = xmm15[0,1],xmm10[2],xmm15[3,4],xmm10[5],xmm15[6,7]
+; AVX512DQ-NEXT:    vpshufb %xmm9, %xmm3, %xmm3
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm2[0,1,2],ymm3[3,4,5,6,7],ymm2[8,9,10],ymm3[11,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,4,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm18
+; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm23, %ymm22, %ymm1
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7,8,9],ymm2[10],ymm1[11,12],ymm2[13],ymm1[14,15]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
+; AVX512DQ-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vmovdqa64 %ymm2, %ymm28
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1],xmm6[2],xmm4[3,4],xmm6[5],xmm4[6,7]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm6, %xmm25
 ; AVX512DQ-NEXT:    vmovdqa64 %xmm4, %xmm26
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm6 = [2,3,8,9,14,15,4,5,10,11,10,11,10,11,10,11]
-; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm7, %xmm7
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm7 = xmm7[0,1,2,3,4],xmm2[5,6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm13 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
-; AVX512DQ-NEXT:    vmovdqa %ymm13, %ymm2
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm20, %ymm18, %ymm2
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm2[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm4[2],ymm2[3,4],ymm4[5],ymm2[6,7,8,9],ymm4[10],ymm2[11,12],ymm4[13],ymm2[14,15]
+; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm1[5,6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm9 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
+; AVX512DQ-NEXT:    vmovdqa %ymm9, %ymm1
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm21, %ymm20, %ymm1
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm1[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm4[2],ymm1[3,4],ymm4[5],ymm1[6,7,8,9],ymm4[10],ymm1[11,12],ymm4[13],ymm1[14,15]
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm4 = [2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]
-; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vmovdqa64 %xmm19, %xmm8
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm14 = xmm8[0,1],xmm3[2],xmm8[3,4],xmm3[5],xmm8[6,7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm3, %xmm27
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm3 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm14, %xmm14
+; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vmovdqa %xmm14, %xmm7
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm14 = xmm8[0,1],xmm14[2],xmm8[3,4],xmm14[5],xmm8[6,7]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm8, %xmm27
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm2 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm14, %xmm14
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm14 = ymm2[0,1,2],ymm14[3,4,5,6,7],ymm2[8,9,10],ymm14[11,12,13,14,15]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,6,7,4]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm14[4,5,6,7]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm2, %zmm19
-; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm2
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm11, %ymm23, %ymm2
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm2[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm7[2],ymm2[3,4],ymm7[5],ymm2[6,7,8,9],ymm7[10],ymm2[11,12],ymm7[13],ymm2[14,15]
-; AVX512DQ-NEXT:    vpshufb %ymm9, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm7 = xmm5[0,1],xmm15[2],xmm5[3,4],xmm15[5],xmm5[6,7]
-; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm7, %xmm6
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4],xmm2[5,6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa %ymm13, %ymm6
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm24, %ymm12, %ymm6
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0,1],ymm7[2],ymm6[3,4],ymm7[5],ymm6[6,7,8,9],ymm7[10],ymm6[11,12],ymm7[13],ymm6[14,15]
-; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm6, %ymm4
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm6 = xmm10[0,1],xmm1[2],xmm10[3,4],xmm1[5],xmm10[6,7]
-; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm6, %xmm3
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3,4,5,6,7],ymm4[8,9,10],ymm3[11,12,13,14,15]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,6,7,4]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
-; AVX512DQ-NEXT:    vpternlogq $226, %ymm23, %ymm13, %ymm11
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm11[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm11[1,2],ymm3[3],ymm11[4,5],ymm3[6],ymm11[7],ymm3[8],ymm11[9,10],ymm3[11],ymm11[12,13],ymm3[14],ymm11[15]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm11 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
-; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm3, %ymm3
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm15[0,1],xmm5[2],xmm15[3,4],xmm5[5],xmm15[6,7]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm5 = [4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
-; AVX512DQ-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
-; AVX512DQ-NEXT:    vpternlogq $226, %ymm24, %ymm0, %ymm12
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm12[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0],ymm12[1,2],ymm4[3],ymm12[4,5],ymm4[6],ymm12[7],ymm4[8],ymm12[9,10],ymm4[11],ymm12[12,13],ymm4[14],ymm12[15]
-; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm4, %ymm4
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm10[1],xmm1[2,3],xmm10[4],xmm1[5,6],xmm10[7]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm14 = ymm1[0,1,2],ymm14[3,4,5,6,7],ymm1[8,9,10],ymm14[11,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,6,7,4]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm14[4,5,6,7]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm19
+; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm13, %ymm24, %ymm1
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm1[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm3[2],ymm1[3,4],ymm3[5],ymm1[6,7,8,9],ymm3[10],ymm1[11,12],ymm3[13],ymm1[14,15]
+; AVX512DQ-NEXT:    vmovdqa64 %ymm28, %ymm3
+; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm3 = xmm12[0,1],xmm11[2],xmm12[3,4],xmm11[5],xmm12[6,7]
+; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm1[5,6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQ-NEXT:    vmovdqa %ymm9, %ymm3
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm17, %ymm5, %ymm3
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm6 = ymm3[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1],ymm6[2],ymm3[3,4],ymm6[5],ymm3[6,7,8,9],ymm6[10],ymm3[11,12],ymm6[13],ymm3[14,15]
+; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm10[0,1],xmm15[2],xmm10[3,4],xmm15[5],xmm10[6,7]
+; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm4, %xmm2
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3,4,5,6,7],ymm3[8,9,10],ymm2[11,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,6,7,4]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
+; AVX512DQ-NEXT:    vpternlogq $226, %ymm24, %ymm9, %ymm13
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm13[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm13[1,2],ymm2[3],ymm13[4,5],ymm2[6],ymm13[7],ymm2[8],ymm13[9,10],ymm2[11],ymm13[12,13],ymm2[14],ymm13[15]
+; AVX512DQ-NEXT:    vpternlogq $226, %ymm17, %ymm0, %ymm5
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm5[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm5[1,2],ymm3[3],ymm5[4,5],ymm3[6],ymm5[7],ymm3[8],ymm5[9,10],ymm3[11],ymm5[12,13],ymm3[14],ymm5[15]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm4 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
+; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm5 = xmm15[0],xmm10[1],xmm15[2,3],xmm10[4],xmm15[5,6],xmm10[7]
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm6 = [0,1,2,3,0,1,6,7,12,13,2,3,8,9,14,15]
-; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm1, %xmm1
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4],ymm1[5,6,7]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm13
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm13[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm13[1,2],ymm3[3],ymm13[4,5],ymm3[6],ymm13[7],ymm3[8],ymm13[9,10],ymm3[11],ymm13[12,13],ymm3[14],ymm13[15]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm25, %xmm4
-; AVX512DQ-NEXT:    vmovdqa64 %xmm26, %xmm7
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm7[2],xmm4[3,4],xmm7[5],xmm4[6,7]
-; AVX512DQ-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
-; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm3, %ymm3
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm20, %ymm18, %ymm0
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm0[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm4[0],ymm0[1,2],ymm4[3],ymm0[4,5],ymm4[6],ymm0[7],ymm4[8],ymm0[9,10],ymm4[11],ymm0[12,13],ymm4[14],ymm0[15]
-; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm5, %xmm5
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm5[5,6,7]
+; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm5 = xmm11[0,1],xmm12[2],xmm11[3,4],xmm12[5],xmm11[6,7]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm8 = [4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
+; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm5, %xmm5
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm3, %zmm5
+; AVX512DQ-NEXT:    vextracti32x4 $2, %zmm5, %xmm5
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4],xmm2[5,6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm22, %ymm23, %ymm9
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm9[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm9[1,2],ymm3[3],ymm9[4,5],ymm3[6],ymm9[7],ymm3[8],ymm9[9,10],ymm3[11],ymm9[12,13],ymm3[14],ymm9[15]
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm21, %ymm20, %ymm0
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm5 = ymm0[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm5[0],ymm0[1,2],ymm5[3],ymm0[4,5],ymm5[6],ymm0[7],ymm5[8],ymm0[9,10],ymm5[11],ymm0[12,13],ymm5[14],ymm0[15]
+; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa64 %xmm27, %xmm4
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0],xmm8[1],xmm4[2,3],xmm8[4],xmm4[5,6],xmm8[7]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm7[0],xmm4[1],xmm7[2,3],xmm4[4],xmm7[5,6],xmm4[7]
 ; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm4, %xmm4
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm4[5,6,7]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm25, %xmm4
+; AVX512DQ-NEXT:    vmovdqa64 %xmm26, %xmm5
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2],xmm4[3,4],xmm5[5],xmm4[6,7]
+; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm4, %xmm4
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm4
+; AVX512DQ-NEXT:    vextracti32x4 $2, %zmm4, %xmm4
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vmovdqa64 %zmm17, (%rsi)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm18, (%rsi)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm16, 64(%rsi)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm19, 64(%rdx)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm2, (%rdx)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm1, (%rdx)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm0, 64(%rcx)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm1, (%rcx)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm2, (%rcx)
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512DQ-FCP-LABEL: load_i16_stride3_vf64:
 ; AVX512DQ-FCP:       # %bb.0:
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 224(%rdi), %ymm18
-; AVX512DQ-FCP-NEXT:    vmovdqa64 192(%rdi), %ymm20
+; AVX512DQ-FCP-NEXT:    vmovdqa64 224(%rdi), %ymm20
+; AVX512DQ-FCP-NEXT:    vmovdqa64 192(%rdi), %ymm21
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm18, %ymm20, %ymm1
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm7 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm7, %ymm2, %ymm5
-; AVX512DQ-FCP-NEXT:    vmovdqa 272(%rdi), %xmm1
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm20, %ymm21, %ymm1
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm1[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm3[1],ymm1[2,3],ymm3[4],ymm1[5,6],ymm3[7],ymm1[8],ymm3[9],ymm1[10,11],ymm3[12],ymm1[13,14],ymm3[15]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm1, %ymm5
+; AVX512DQ-FCP-NEXT:    vmovdqa 272(%rdi), %xmm8
 ; AVX512DQ-FCP-NEXT:    vmovdqa 256(%rdi), %xmm2
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm6 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm2, %xmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm1, %xmm19
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm13 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm13, %xmm6, %xmm6
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm6 = xmm2[0,1],xmm8[2],xmm2[3,4],xmm8[5],xmm2[6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm2, %xmm14
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm9 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm9, %xmm6, %xmm6
 ; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm5[0,1,2],ymm6[3,4,5,6,7],ymm5[8,9,10],ymm6[11,12,13,14,15]
 ; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,6,5,4,7]
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 320(%rdi), %ymm21
-; AVX512DQ-FCP-NEXT:    vmovdqa64 352(%rdi), %ymm22
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm8
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm8
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm8[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm8 = ymm8[0],ymm9[1],ymm8[2,3],ymm9[4],ymm8[5,6],ymm9[7],ymm8[8],ymm9[9],ymm8[10,11],ymm9[12],ymm8[13,14],ymm9[15]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm10 = [0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm8, %ymm11
+; AVX512DQ-FCP-NEXT:    vmovdqa64 320(%rdi), %ymm22
+; AVX512DQ-FCP-NEXT:    vmovdqa64 352(%rdi), %ymm23
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm6
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm22, %ymm23, %ymm6
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0],ymm7[1],ymm6[2,3],ymm7[4],ymm6[5,6],ymm7[7],ymm6[8],ymm7[9],ymm6[10,11],ymm7[12],ymm6[13,14],ymm7[15]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm11 = [0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm6, %ymm12
 ; AVX512DQ-FCP-NEXT:    vmovdqa 304(%rdi), %xmm1
 ; AVX512DQ-FCP-NEXT:    vmovdqa 288(%rdi), %xmm2
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm13 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm2, %xmm4
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, %xmm8
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm14 = [0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm14, %xmm12, %xmm12
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm11 = ymm12[0,1,2],ymm11[3,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm5, %zmm16
-; AVX512DQ-FCP-NEXT:    vmovdqa64 128(%rdi), %ymm23
-; AVX512DQ-FCP-NEXT:    vmovdqa 160(%rdi), %ymm11
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, %xmm6
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm15 = [0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm15, %xmm13, %xmm13
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm12 = ymm13[0,1,2],ymm12[3,4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm5, %zmm16
+; AVX512DQ-FCP-NEXT:    vmovdqa64 128(%rdi), %ymm24
+; AVX512DQ-FCP-NEXT:    vmovdqa 160(%rdi), %ymm13
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm5
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm23, %ymm11, %ymm5
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm24, %ymm13, %ymm5
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm12 = ymm5[2,3,0,1]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1],ymm5[2,3],ymm12[4],ymm5[5,6],ymm12[7],ymm5[8],ymm12[9],ymm5[10,11],ymm12[12],ymm5[13,14],ymm12[15]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm5, %ymm10
-; AVX512DQ-FCP-NEXT:    vmovdqa 112(%rdi), %xmm15
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdi), %xmm5
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm12 = xmm5[0],xmm15[1],xmm5[2,3],xmm15[4],xmm5[5,6],xmm15[7]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm14, %xmm12, %xmm12
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm6 = ymm12[0,1,2],ymm10[3,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 (%rdi), %ymm24
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %ymm12
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm5, %ymm5
+; AVX512DQ-FCP-NEXT:    vmovdqa 112(%rdi), %xmm11
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdi), %xmm12
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm10 = xmm12[0],xmm11[1],xmm12[2,3],xmm11[4],xmm12[5,6],xmm11[7]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm15, %xmm10, %xmm10
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm10[0,1,2],ymm5[3,4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 (%rdi), %ymm17
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %ymm5
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm10
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm12, %ymm24, %ymm10
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm10[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm10[0],ymm1[1],ymm10[2,3],ymm1[4],ymm10[5,6],ymm1[7],ymm10[8],ymm1[9],ymm10[10,11],ymm1[12],ymm10[13,14],ymm1[15]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm7, %ymm1, %ymm7
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm5, %ymm17, %ymm10
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm15 = ymm10[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0],ymm15[1],ymm10[2,3],ymm15[4],ymm10[5,6],ymm15[7],ymm10[8],ymm15[9],ymm10[10,11],ymm15[12],ymm10[13,14],ymm15[15]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm10, %ymm2
 ; AVX512DQ-FCP-NEXT:    vmovdqa 80(%rdi), %xmm10
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdi), %xmm1
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm2 = xmm1[0,1],xmm10[2],xmm1[3,4],xmm10[5],xmm1[6,7]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm13, %xmm2, %xmm2
-; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm7[0,1,2],ymm2[3,4,5,6,7],ymm7[8,9,10],ymm2[11,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,6,5,4,7]
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm2, %zmm17
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm2
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm22, %ymm21, %ymm2
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm2[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm6[2],ymm2[3,4],ymm6[5],ymm2[6,7,8,9],ymm6[10],ymm2[11,12],ymm6[13],ymm2[14,15]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm9 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm9, %ymm2, %ymm2
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm7 = xmm4[0,1],xmm8[2],xmm4[3,4],xmm8[5],xmm4[6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm8, %xmm25
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdi), %xmm15
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm15[0,1],xmm10[2],xmm15[3,4],xmm10[5],xmm15[6,7]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm9, %xmm3, %xmm3
+; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm2[0,1,2],ymm3[3,4,5,6,7],ymm2[8,9,10],ymm3[11,12,13,14,15]
+; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,4,7]
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm18
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm23, %ymm22, %ymm1
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7,8,9],ymm2[10],ymm1[11,12],ymm2[13],ymm1[14,15]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm2 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm2, %ymm28
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1],xmm6[2],xmm4[3,4],xmm6[5],xmm4[6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm6, %xmm25
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm4, %xmm26
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm6 = [2,3,8,9,14,15,4,5,10,11,10,11,10,11,10,11]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm7, %xmm7
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm7 = xmm7[0,1,2,3,4],xmm2[5,6,7]
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm13 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm13, %ymm2
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm20, %ymm18, %ymm2
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm2[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm4[2],ymm2[3,4],ymm4[5],ymm2[6,7,8,9],ymm4[10],ymm2[11,12],ymm4[13],ymm2[14,15]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm1[5,6,7]
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm9 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm9, %ymm1
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm21, %ymm20, %ymm1
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm1[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm4[2],ymm1[3,4],ymm4[5],ymm1[6,7,8,9],ymm4[10],ymm1[11,12],ymm4[13],ymm1[14,15]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm4 = [2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm2, %ymm2
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm19, %xmm8
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm14 = xmm8[0,1],xmm3[2],xmm8[3,4],xmm3[5],xmm8[6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm3, %xmm27
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm3, %xmm14, %xmm14
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm1, %ymm1
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm14, %xmm7
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm14 = xmm8[0,1],xmm14[2],xmm8[3,4],xmm14[5],xmm8[6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm8, %xmm27
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm2, %xmm14, %xmm14
 ; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm14 = ymm2[0,1,2],ymm14[3,4,5,6,7],ymm2[8,9,10],ymm14[11,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,6,7,4]
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm14[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm2, %zmm19
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm2
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm11, %ymm23, %ymm2
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm2[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm7[2],ymm2[3,4],ymm7[5],ymm2[6,7,8,9],ymm7[10],ymm2[11,12],ymm7[13],ymm2[14,15]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm9, %ymm2, %ymm2
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm7 = xmm5[0,1],xmm15[2],xmm5[3,4],xmm15[5],xmm5[6,7]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm7, %xmm6
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4],xmm2[5,6,7]
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm13, %ymm6
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm24, %ymm12, %ymm6
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0,1],ymm7[2],ymm6[3,4],ymm7[5],ymm6[6,7,8,9],ymm7[10],ymm6[11,12],ymm7[13],ymm6[14,15]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm6, %ymm4
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm6 = xmm10[0,1],xmm1[2],xmm10[3,4],xmm1[5],xmm10[6,7]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm3, %xmm6, %xmm3
-; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3,4,5,6,7],ymm4[8,9,10],ymm3[11,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,6,7,4]
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %ymm23, %ymm13, %ymm11
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm11[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm11[1,2],ymm3[3],ymm11[4,5],ymm3[6],ymm11[7],ymm3[8],ymm11[9,10],ymm3[11],ymm11[12,13],ymm3[14],ymm11[15]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm11 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm3, %ymm3
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm15[0,1],xmm5[2],xmm15[3,4],xmm5[5],xmm15[6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm5 = [4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %ymm24, %ymm0, %ymm12
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm12[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0],ymm12[1,2],ymm4[3],ymm12[4,5],ymm4[6],ymm12[7],ymm4[8],ymm12[9,10],ymm4[11],ymm12[12,13],ymm4[14],ymm12[15]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm4, %ymm4
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm10[1],xmm1[2,3],xmm10[4],xmm1[5,6],xmm10[7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm14 = ymm1[0,1,2],ymm14[3,4,5,6,7],ymm1[8,9,10],ymm14[11,12,13,14,15]
+; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,6,7,4]
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm14[4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm19
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm13, %ymm24, %ymm1
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm1[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm3[2],ymm1[3,4],ymm3[5],ymm1[6,7,8,9],ymm3[10],ymm1[11,12],ymm3[13],ymm1[14,15]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm28, %ymm3
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm1, %ymm1
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm12[0,1],xmm11[2],xmm12[3,4],xmm11[5],xmm12[6,7]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm1[5,6,7]
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm9, %ymm3
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm17, %ymm5, %ymm3
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm3[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1],ymm6[2],ymm3[3,4],ymm6[5],ymm3[6,7,8,9],ymm6[10],ymm3[11,12],ymm6[13],ymm3[14,15]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm10[0,1],xmm15[2],xmm10[3,4],xmm15[5],xmm10[6,7]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm2, %xmm4, %xmm2
+; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3,4,5,6,7],ymm3[8,9,10],ymm2[11,12,13,14,15]
+; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,6,7,4]
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %ymm24, %ymm9, %ymm13
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm13[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm13[1,2],ymm2[3],ymm13[4,5],ymm2[6],ymm13[7],ymm2[8],ymm13[9,10],ymm2[11],ymm13[12,13],ymm2[14],ymm13[15]
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %ymm17, %ymm0, %ymm5
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm5[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm5[1,2],ymm3[3],ymm5[4,5],ymm3[6],ymm5[7],ymm3[8],ymm5[9,10],ymm3[11],ymm5[12,13],ymm3[14],ymm5[15]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm4 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm5 = xmm15[0],xmm10[1],xmm15[2,3],xmm10[4],xmm15[5,6],xmm10[7]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm6 = [0,1,2,3,0,1,6,7,12,13,2,3,8,9,14,15]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm1, %xmm1
-; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4],ymm1[5,6,7]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm13
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm13[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm13[1,2],ymm3[3],ymm13[4,5],ymm3[6],ymm13[7],ymm3[8],ymm13[9,10],ymm3[11],ymm13[12,13],ymm3[14],ymm13[15]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm25, %xmm4
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm26, %xmm7
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm7[2],xmm4[3,4],xmm7[5],xmm4[6,7]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm3, %ymm3
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm20, %ymm18, %ymm0
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm0[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm4[0],ymm0[1,2],ymm4[3],ymm0[4,5],ymm4[6],ymm0[7],ymm4[8],ymm0[9,10],ymm4[11],ymm0[12,13],ymm4[14],ymm0[15]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm0, %ymm0
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm5, %xmm5
+; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm5[5,6,7]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm2, %ymm2
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm5 = xmm11[0,1],xmm12[2],xmm11[3,4],xmm12[5],xmm11[6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm8 = [4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm5, %xmm5
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm3, %zmm5
+; AVX512DQ-FCP-NEXT:    vextracti32x4 $2, %zmm5, %xmm5
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4],xmm2[5,6,7]
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm22, %ymm23, %ymm9
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm9[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm9[1,2],ymm3[3],ymm9[4,5],ymm3[6],ymm9[7],ymm3[8],ymm9[9,10],ymm3[11],ymm9[12,13],ymm3[14],ymm9[15]
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm21, %ymm20, %ymm0
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm5 = ymm0[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm5[0],ymm0[1,2],ymm5[3],ymm0[4,5],ymm5[6],ymm0[7],ymm5[8],ymm0[9,10],ymm5[11],ymm0[12,13],ymm5[14],ymm0[15]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm0, %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm27, %xmm4
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0],xmm8[1],xmm4[2,3],xmm8[4],xmm4[5,6],xmm8[7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm7[0],xmm4[1],xmm7[2,3],xmm4[4],xmm7[5,6],xmm4[7]
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm4, %xmm4
 ; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm4[5,6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm25, %xmm4
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm26, %xmm5
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2],xmm4[3,4],xmm5[5],xmm4[6,7]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm4, %xmm4
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm4
+; AVX512DQ-FCP-NEXT:    vextracti32x4 $2, %zmm4, %xmm4
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm17, (%rsi)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm18, (%rsi)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm16, 64(%rsi)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm19, 64(%rdx)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm2, (%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, (%rdx)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, 64(%rcx)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, (%rcx)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm2, (%rcx)
 ; AVX512DQ-FCP-NEXT:    vzeroupper
 ; AVX512DQ-FCP-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
index 8091afbbfd70..8b6ba51506ab 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
@@ -1246,28 +1246,29 @@ define void @store_i8_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecp
 ; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
 ; AVX512BW-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm1 = [0,2,4,0]
 ; AVX512BW-FCP-NEXT:    vpermi2q %ymm3, %ymm0, %ymm1
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm2[0,8],zero,zero,zero,zero,zero,ymm2[1,9],zero,zero,zero,zero,zero,ymm2[2,10,18,26],zero,zero,zero,zero,zero,ymm2[19,27],zero,zero,zero,zero,zero,ymm2[20,28]
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm2[2,3,0,1]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,ymm3[0,8],zero,zero,zero,zero,zero,ymm3[1,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[19,27],zero,zero,zero,zero,zero,ymm3[20,28],zero,zero
+; AVX512BW-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [1,3,5,7,1,3,5,7]
+; AVX512BW-FCP-NEXT:    # ymm0 = mem[0,1,0,1]
+; AVX512BW-FCP-NEXT:    vpermd %ymm2, %ymm0, %ymm0
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm0[1,5,9,13],zero,zero,zero,ymm0[2,6,10,14],zero,zero,zero,ymm0[19,23,27,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm3 = [1,3,5,0,5,1,3,0]
+; AVX512BW-FCP-NEXT:    vpermd %ymm1, %ymm3, %ymm3
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[0,4,8],zero,zero,zero,zero,ymm3[1,5,9],zero,zero,zero,zero,ymm3[2,6,18],zero,zero,zero,zero,ymm3[23,27,19],zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512BW-FCP-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm2[0,8],zero,zero,zero,zero,zero,ymm2[1,9],zero,zero,zero,zero,zero,ymm2[2,10,18,26],zero,zero,zero,zero,zero,ymm2[19,27],zero,zero,zero,zero,zero,ymm2[20,28]
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,3,0,1]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,ymm2[0,8],zero,zero,zero,zero,zero,ymm2[1,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[19,27],zero,zero,zero,zero,zero,ymm2[20,28],zero,zero
+; AVX512BW-FCP-NEXT:    vpor %ymm3, %ymm2, %ymm2
 ; AVX512BW-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [0,2,4,6,0,2,4,6]
 ; AVX512BW-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
-; AVX512BW-FCP-NEXT:    vpermd %ymm1, %ymm3, %ymm3
+; AVX512BW-FCP-NEXT:    vpermd %ymm1, %ymm3, %ymm1
 ; AVX512BW-FCP-NEXT:    movl $236730480, %ecx # imm = 0xE1C3870
 ; AVX512BW-FCP-NEXT:    kmovd %ecx, %k1
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm0 {%k1} = ymm3[u,u,u,u,0,4,8,u,u,u,u,1,5,9,u,u,u,u,18,22,26,u,u,u,u,19,23,27,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [1,3,5,7,1,3,5,7]
-; AVX512BW-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
-; AVX512BW-FCP-NEXT:    vpermd %ymm2, %ymm3, %ymm2
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,zero,ymm2[1,5,9,13],zero,zero,zero,ymm2[2,6,10,14],zero,zero,zero,ymm2[19,23,27,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm3 = [1,3,5,0,5,1,3,0]
-; AVX512BW-FCP-NEXT:    vpermd %ymm1, %ymm3, %ymm1
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[0,4,8],zero,zero,zero,zero,ymm1[1,5,9],zero,zero,zero,zero,ymm1[2,6,18],zero,zero,zero,zero,ymm1[23,27,19],zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512BW-FCP-NEXT:    vpor %ymm2, %ymm1, %ymm1
-; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512BW-FCP-NEXT:    vmovq %xmm2, 48(%rax)
-; AVX512BW-FCP-NEXT:    vmovdqa %xmm1, 32(%rax)
-; AVX512BW-FCP-NEXT:    vmovdqa %ymm0, (%rax)
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm2 {%k1} = ymm1[u,u,u,u,0,4,8,u,u,u,u,1,5,9,u,u,u,u,18,22,26,u,u,u,u,19,23,27,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm0
+; AVX512BW-FCP-NEXT:    vmovdqa %ymm2, (%rax)
+; AVX512BW-FCP-NEXT:    vextracti32x4 $2, %zmm0, 32(%rax)
+; AVX512BW-FCP-NEXT:    vextracti32x4 $3, %zmm0, %xmm0
+; AVX512BW-FCP-NEXT:    vmovq %xmm0, 48(%rax)
 ; AVX512BW-FCP-NEXT:    vzeroupper
 ; AVX512BW-FCP-NEXT:    retq
 ;
@@ -1325,28 +1326,29 @@ define void @store_i8_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecp
 ; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
 ; AVX512DQ-BW-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm1 = [0,2,4,0]
 ; AVX512DQ-BW-FCP-NEXT:    vpermi2q %ymm3, %ymm0, %ymm1
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm2[0,8],zero,zero,zero,zero,zero,ymm2[1,9],zero,zero,zero,zero,zero,ymm2[2,10,18,26],zero,zero,zero,zero,zero,ymm2[19,27],zero,zero,zero,zero,zero,ymm2[20,28]
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm2[2,3,0,1]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,ymm3[0,8],zero,zero,zero,zero,zero,ymm3[1,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[19,27],zero,zero,zero,zero,zero,ymm3[20,28],zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [1,3,5,7,1,3,5,7]
+; AVX512DQ-BW-FCP-NEXT:    # ymm0 = mem[0,1,0,1]
+; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm2, %ymm0, %ymm0
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm0[1,5,9,13],zero,zero,zero,ymm0[2,6,10,14],zero,zero,zero,ymm0[19,23,27,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm3 = [1,3,5,0,5,1,3,0]
+; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm1, %ymm3, %ymm3
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[0,4,8],zero,zero,zero,zero,ymm3[1,5,9],zero,zero,zero,zero,ymm3[2,6,18],zero,zero,zero,zero,ymm3[23,27,19],zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQ-BW-FCP-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm2[0,8],zero,zero,zero,zero,zero,ymm2[1,9],zero,zero,zero,zero,zero,ymm2[2,10,18,26],zero,zero,zero,zero,zero,ymm2[19,27],zero,zero,zero,zero,zero,ymm2[20,28]
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,3,0,1]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,ymm2[0,8],zero,zero,zero,zero,zero,ymm2[1,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[19,27],zero,zero,zero,zero,zero,ymm2[20,28],zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vpor %ymm3, %ymm2, %ymm2
 ; AVX512DQ-BW-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [0,2,4,6,0,2,4,6]
 ; AVX512DQ-BW-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
-; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm1, %ymm3, %ymm3
+; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm1, %ymm3, %ymm1
 ; AVX512DQ-BW-FCP-NEXT:    movl $236730480, %ecx # imm = 0xE1C3870
 ; AVX512DQ-BW-FCP-NEXT:    kmovd %ecx, %k1
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm0 {%k1} = ymm3[u,u,u,u,0,4,8,u,u,u,u,1,5,9,u,u,u,u,18,22,26,u,u,u,u,19,23,27,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [1,3,5,7,1,3,5,7]
-; AVX512DQ-BW-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
-; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm2, %ymm3, %ymm2
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,zero,ymm2[1,5,9,13],zero,zero,zero,ymm2[2,6,10,14],zero,zero,zero,ymm2[19,23,27,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm3 = [1,3,5,0,5,1,3,0]
-; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm1, %ymm3, %ymm1
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[0,4,8],zero,zero,zero,zero,ymm1[1,5,9],zero,zero,zero,zero,ymm1[2,6,18],zero,zero,zero,zero,ymm1[23,27,19],zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vpor %ymm2, %ymm1, %ymm1
-; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512DQ-BW-FCP-NEXT:    vmovq %xmm2, 48(%rax)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa %xmm1, 32(%rax)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa %ymm0, (%rax)
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm2 {%k1} = ymm1[u,u,u,u,0,4,8,u,u,u,u,1,5,9,u,u,u,u,18,22,26,u,u,u,u,19,23,27,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm0
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa %ymm2, (%rax)
+; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $2, %zmm0, 32(%rax)
+; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $3, %zmm0, %xmm0
+; AVX512DQ-BW-FCP-NEXT:    vmovq %xmm0, 48(%rax)
 ; AVX512DQ-BW-FCP-NEXT:    vzeroupper
 ; AVX512DQ-BW-FCP-NEXT:    retq
   %in.vec0 = load <8 x i8>, ptr %in.vecptr0, align 64
@@ -2051,76 +2053,77 @@ define void @store_i8_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512-NEXT:    vmovdqa (%rsi), %xmm1
-; AVX512-NEXT:    vmovdqa (%rdx), %xmm5
-; AVX512-NEXT:    vmovdqa (%rcx), %xmm6
-; AVX512-NEXT:    vmovdqa (%r8), %xmm3
-; AVX512-NEXT:    vmovdqa (%r9), %xmm4
-; AVX512-NEXT:    vmovdqa (%r10), %xmm2
-; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm8
-; AVX512-NEXT:    vinserti128 $1, %xmm6, %ymm5, %ymm9
-; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm7
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm10 = ymm9[u,u,u,u,u,5],zero,ymm9[u,u,u,u,u,6],zero,ymm9[u,u,u,u,u],zero,ymm9[23,u,u,u,u,u],zero,ymm9[24,u,u,u,u]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm11 = ymm9[2,3,0,1]
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[u,u,u,u,u],zero,ymm11[5,u,u,u,u,u],zero,ymm11[6,u,u,u,u,u,23],zero,ymm11[u,u,u,u,u,24],zero,ymm11[u,u,u,u]
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm12 = [255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255]
-; AVX512-NEXT:    vpternlogq $50, %ymm10, %ymm12, %ymm11
-; AVX512-NEXT:    vpermq {{.*#+}} ymm10 = ymm9[0,2,0,2]
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm10 = zero,zero,ymm10[0,8,u,u,u],zero,zero,ymm10[1,9,u,u,u],zero,zero,ymm10[18,26,u,u,u],zero,zero,ymm10[19,27,u,u,u],zero,zero,ymm10[20,28]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm11, %zmm10, %zmm10
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm11 = ymm8[u,u,u,5],zero,ymm8[u,u,u,u,u,6],zero,ymm8[u,u,u,u,u],zero,ymm8[23,u,u,u,u,u],zero,ymm8[24,u,u,u,u,u],zero
-; AVX512-NEXT:    vpermq {{.*#+}} ymm13 = ymm8[2,3,0,1]
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm13 = ymm13[u,u,u],zero,ymm13[5,u,u,u,u,u],zero,ymm13[6,u,u,u,u,u,23],zero,ymm13[u,u,u,u,u,24],zero,ymm13[u,u,u,u,u,25]
-; AVX512-NEXT:    vpternlogq $200, %ymm11, %ymm12, %ymm13
-; AVX512-NEXT:    vpermq {{.*#+}} ymm11 = ymm8[0,2,0,2]
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[0,8],zero,zero,ymm11[u,u,u,1,9],zero,zero,ymm11[u,u,u,2,10],zero,zero,ymm11[u,u,u,19,27],zero,zero,ymm11[u,u,u,20,28],zero,zero
-; AVX512-NEXT:    vinserti64x4 $1, %ymm13, %zmm11, %zmm11
-; AVX512-NEXT:    vporq %zmm10, %zmm11, %zmm10
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm11 = ymm7[4],zero,ymm7[u,u,u,u,u,5],zero,ymm7[u,u,u,u,u,6],zero,ymm7[u,u,u,u,u],zero,ymm7[23,u,u,u,u,u],zero,ymm7[24,u,u]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm12 = ymm7[2,3,0,1]
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm12 = zero,ymm12[4,u,u,u,u,u],zero,ymm12[5,u,u,u,u,u],zero,ymm12[6,u,u,u,u,u,23],zero,ymm12[u,u,u,u,u,24],zero,ymm12[u,u]
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm13 = [255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255]
-; AVX512-NEXT:    vpternlogq $200, %ymm11, %ymm13, %ymm12
-; AVX512-NEXT:    vpermq {{.*#+}} ymm11 = ymm7[0,2,0,2]
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[u,u,u,u,0,8],zero,ymm11[u,u,u,u,1,9],zero,ymm11[u,u,u,u,18,26],zero,ymm11[u,u,u,u,19,27],zero,ymm11[u,u,u,u]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm12, %zmm11, %zmm11
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm12 = xmm2[4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,0,1,0]
-; AVX512-NEXT:    vpandn %ymm12, %ymm13, %ymm12
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm13 = xmm2[1,1,0,0,4,5,6,7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm13 = xmm13[0,1,2,0]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,0,1,0]
-; AVX512-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm13, %ymm13
-; AVX512-NEXT:    vinserti64x4 $1, %ymm12, %zmm13, %zmm12
-; AVX512-NEXT:    vporq %zmm12, %zmm11, %zmm11
-; AVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm11
-; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[3,1,1,3]
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm8 = ymm8[1],zero,zero,ymm8[u,u,u,10,2],zero,zero,ymm8[u,u,u,11,3],zero,zero,ymm8[u,u,u,20,28],zero,zero,ymm8[u,u,u,21,29],zero,zero,ymm8[u]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[1,3,3,1]
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm9 = zero,ymm9[1,9,u,u,u],zero,zero,ymm9[2,10,u,u,u],zero,zero,ymm9[3,19,u,u,u],zero,zero,ymm9[28,20,u,u,u],zero,zero,ymm9[29,21,u]
-; AVX512-NEXT:    vpor %ymm8, %ymm9, %ymm8
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm9 = xmm2[0,1,2,3,4,5,5,6]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm9 = xmm9[2,2,3,3]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,1,0,1]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[1,3,1,3]
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm4
+; AVX512-NEXT:    vmovdqa (%rsi), %xmm5
+; AVX512-NEXT:    vmovdqa (%rdx), %xmm6
+; AVX512-NEXT:    vmovdqa (%rcx), %xmm7
+; AVX512-NEXT:    vmovdqa (%r8), %xmm0
+; AVX512-NEXT:    vmovdqa (%r10), %xmm1
+; AVX512-NEXT:    vinserti128 $1, %xmm7, %ymm6, %ymm3
+; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm4, %ymm2
+; AVX512-NEXT:    vinserti128 $1, (%r9), %ymm0, %ymm0
+; AVX512-NEXT:    vinserti32x4 $2, %xmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm6[8],xmm7[8],xmm6[9],xmm7[9],xmm6[10],xmm7[10],xmm6[11],xmm7[11],xmm6[12],xmm7[12],xmm6[13],xmm7[13],xmm6[14],xmm7[14],xmm6[15],xmm7[15]
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[u,u],zero,zero,xmm6[12,13,u,u,u],zero,zero,xmm6[14,15,u,u,u]
+; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm4[8],xmm5[8],xmm4[9],xmm5[9],xmm4[10],xmm5[10],xmm4[11],xmm5[11],xmm4[12],xmm5[12],xmm4[13],xmm5[13],xmm4[14],xmm5[14],xmm4[15],xmm5[15]
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,12,13],zero,zero,xmm4[u,u,u,14,15],zero,zero,xmm4[u,u,u]
+; AVX512-NEXT:    vpor %xmm6, %xmm4, %xmm4
+; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm5
+; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[10],zero,xmm5[u,u,u,u,13,12],zero,xmm5[u,u,u,u,15,14],zero
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm6 = zero,xmm1[13,u,u,u,u],zero,zero,xmm1[14,u,u,u,u],zero,zero,xmm1[15]
+; AVX512-NEXT:    vpor %xmm6, %xmm5, %xmm5
+; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4, %xmm5
+; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm2[3,1,1,3]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm4 = ymm4[1],zero,zero,ymm4[u,u,u,10,2],zero,zero,ymm4[u,u,u,11,3],zero,zero,ymm4[u,u,u,20,28],zero,zero,ymm4[u,u,u,21,29],zero,zero,ymm4[u]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm6 = ymm3[1,3,3,1]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm6 = zero,ymm6[1,9,u,u,u],zero,zero,ymm6[2,10,u,u,u],zero,zero,ymm6[3,19,u,u,u],zero,zero,ymm6[28,20,u,u,u],zero,zero,ymm6[29,21,u]
+; AVX512-NEXT:    vpor %ymm4, %ymm6, %ymm4
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm6 = xmm1[0,1,2,3,4,5,5,6]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,2,3,3]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,1,0,1]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm0[1,3,1,3]
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,1,9],zero,ymm7[u,u,u,u,2,10],zero,ymm7[u,u,u,u,19,27],zero,ymm7[u,u,u,u,20,28],zero,ymm7[u,u,u,u,21]
-; AVX512-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm9, %ymm7
-; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm8, %ymm7
-; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm5[8],xmm6[8],xmm5[9],xmm6[9],xmm5[10],xmm6[10],xmm5[11],xmm6[11],xmm5[12],xmm6[12],xmm5[13],xmm6[13],xmm5[14],xmm6[14],xmm5[15],xmm6[15]
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u],zero,zero,xmm5[12,13,u,u,u],zero,zero,xmm5[14,15,u,u,u]
-; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,12,13],zero,zero,xmm0[u,u,u,14,15],zero,zero,xmm0[u,u,u]
-; AVX512-NEXT:    vpor %xmm5, %xmm0, %xmm0
-; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm4[8],xmm3[8],xmm4[9],xmm3[9],xmm4[10],xmm3[10],xmm4[11],xmm3[11],xmm4[12],xmm3[12],xmm4[13],xmm3[13],xmm4[14],xmm3[14],xmm4[15],xmm3[15]
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[10],zero,xmm1[u,u,u,u,13,12],zero,xmm1[u,u,u,u,15,14],zero
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm2 = zero,xmm2[13,u,u,u,u],zero,zero,xmm2[14,u,u,u,u],zero,zero,xmm2[15]
-; AVX512-NEXT:    vpor %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
-; AVX512-NEXT:    vinserti32x4 $2, %xmm1, %zmm7, %zmm0
-; AVX512-NEXT:    vmovdqa %xmm1, 96(%rax)
-; AVX512-NEXT:    vmovdqa %ymm0, 64(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm11, (%rax)
+; AVX512-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm6, %ymm7
+; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm7
+; AVX512-NEXT:    vinserti32x4 $2, %xmm5, %zmm7, %zmm4
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm6 = ymm3[u,u,u,u,u,5],zero,ymm3[u,u,u,u,u,6],zero,ymm3[u,u,u,u,u],zero,ymm3[23,u,u,u,u,u],zero,ymm3[24,u,u,u,u]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm3[2,3,0,1]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,u,u],zero,ymm7[5,u,u,u,u,u],zero,ymm7[6,u,u,u,u,u,23],zero,ymm7[u,u,u,u,u,24],zero,ymm7[u,u,u,u]
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm8 = [255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255]
+; AVX512-NEXT:    vpternlogq $50, %ymm6, %ymm8, %ymm7
+; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,2,0,2]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,ymm3[0,8,u,u,u],zero,zero,ymm3[1,9,u,u,u],zero,zero,ymm3[18,26,u,u,u],zero,zero,ymm3[19,27,u,u,u],zero,zero,ymm3[20,28]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm3, %zmm3
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm6 = ymm2[u,u,u,5],zero,ymm2[u,u,u,u,u,6],zero,ymm2[u,u,u,u,u],zero,ymm2[23,u,u,u,u,u],zero,ymm2[24,u,u,u,u,u],zero
+; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm2[2,3,0,1]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u],zero,ymm7[5,u,u,u,u,u],zero,ymm7[6,u,u,u,u,u,23],zero,ymm7[u,u,u,u,u,24],zero,ymm7[u,u,u,u,u,25]
+; AVX512-NEXT:    vpternlogq $200, %ymm6, %ymm8, %ymm7
+; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,0,2]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[0,8],zero,zero,ymm2[u,u,u,1,9],zero,zero,ymm2[u,u,u,2,10],zero,zero,ymm2[u,u,u,19,27],zero,zero,ymm2[u,u,u,20,28],zero,zero
+; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm2, %zmm2
+; AVX512-NEXT:    vporq %zmm3, %zmm2, %zmm2
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm3 = xmm1[4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,1,0]
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm6 = [255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255]
+; AVX512-NEXT:    vpandn %ymm3, %ymm6, %ymm3
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[1,1,0,0,4,5,6,7]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,0]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,1,0]
+; AVX512-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm3 = ymm0[4],zero,ymm0[u,u,u,u,u,5],zero,ymm0[u,u,u,u,u,6],zero,ymm0[u,u,u,u,u],zero,ymm0[23,u,u,u,u,u],zero,ymm0[24,u,u]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm0[2,3,0,1]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm7 = zero,ymm7[4,u,u,u,u,u],zero,ymm7[5,u,u,u,u,u],zero,ymm7[6,u,u,u,u,u,23],zero,ymm7[u,u,u,u,u,24],zero,ymm7[u,u]
+; AVX512-NEXT:    vpternlogq $200, %ymm3, %ymm6, %ymm7
+; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,0,2]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,0,8],zero,ymm0[u,u,u,u,1,9],zero,ymm0[u,u,u,u,18,26],zero,ymm0[u,u,u,u,19,27],zero,ymm0[u,u,u,u]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm0, %zmm0
+; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0
+; AVX512-NEXT:    vmovdqa %xmm5, 96(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm0, (%rax)
+; AVX512-NEXT:    vmovdqa %ymm4, 64(%rax)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
@@ -2128,69 +2131,70 @@ define void @store_i8_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP:       # %bb.0:
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm1
-; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm5
-; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm6
-; AVX512-FCP-NEXT:    vmovdqa (%r8), %xmm3
-; AVX512-FCP-NEXT:    vmovdqa (%r9), %xmm4
-; AVX512-FCP-NEXT:    vmovdqa (%r10), %xmm2
-; AVX512-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm7
-; AVX512-FCP-NEXT:    vinserti128 $1, %xmm6, %ymm5, %ymm8
-; AVX512-FCP-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm9
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm8[0,2,0,2]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm10 = zero,zero,ymm10[0,8,u,u,u],zero,zero,ymm10[1,9,u,u,u],zero,zero,ymm10[18,26,u,u,u],zero,zero,ymm10[19,27,u,u,u],zero,zero,ymm10[20,28]
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm11 = [1,5,2,6,1,5,2,6]
-; AVX512-FCP-NEXT:    # ymm11 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpermd %ymm8, %ymm11, %ymm12
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm12 = ymm12[u,u,u],zero,zero,ymm12[1,5,u,u,u],zero,zero,ymm12[2,6,u,u,u],zero,zero,ymm12[19,23,u,u,u],zero,zero,ymm12[24,28,u,u,u],zero
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm10, %zmm10
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm12 = ymm7[0,2,0,2]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm12 = ymm12[0,8],zero,zero,ymm12[u,u,u,1,9],zero,zero,ymm12[u,u,u,2,10],zero,zero,ymm12[u,u,u,19,27],zero,zero,ymm12[u,u,u,20,28],zero,zero
-; AVX512-FCP-NEXT:    vpermd %ymm7, %ymm11, %ymm13
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm13 = ymm13[u,u,u,1,5],zero,zero,ymm13[u,u,u,2,6],zero,zero,ymm13[u,u,u,19,23],zero,zero,ymm13[u,u,u,24,28],zero,zero,ymm13[u,u,u,25]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm12, %zmm12
-; AVX512-FCP-NEXT:    vporq %zmm10, %zmm12, %zmm10
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm12 = xmm2[1,1,0,0,4,5,6,7]
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm13 = [0,1,0,1,0,0,0,0]
-; AVX512-FCP-NEXT:    vpermd %ymm12, %ymm13, %ymm12
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm13 = xmm2[4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,0,1,0]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm12, %zmm12
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm13 = ymm9[0,2,0,2]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm13 = ymm13[u,u,u,u,0,8],zero,ymm13[u,u,u,u,1,9],zero,ymm13[u,u,u,u,18,26],zero,ymm13[u,u,u,u,19,27],zero,ymm13[u,u,u,u]
-; AVX512-FCP-NEXT:    vpermd %ymm9, %ymm11, %ymm11
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[0,4],zero,ymm11[u,u,u,u,1,5],zero,ymm11[u,u,u,u,2,6],zero,ymm11[u,u,u,u,19,23],zero,ymm11[u,u,u,u,24,28],zero,ymm11[u]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm13, %zmm11
-; AVX512-FCP-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm12, %zmm11
-; AVX512-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm11
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[3,1,1,3]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[1],zero,zero,ymm7[u,u,u,10,2],zero,zero,ymm7[u,u,u,11,3],zero,zero,ymm7[u,u,u,20,28],zero,zero,ymm7[u,u,u,21,29],zero,zero,ymm7[u]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[1,3,3,1]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm8 = zero,ymm8[1,9,u,u,u],zero,zero,ymm8[2,10,u,u,u],zero,zero,ymm8[3,19,u,u,u],zero,zero,ymm8[28,20,u,u,u],zero,zero,ymm8[29,21,u]
-; AVX512-FCP-NEXT:    vpor %ymm7, %ymm8, %ymm7
-; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm8 = xmm2[0,1,2,3,4,5,5,6]
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [2,2,3,3,2,2,3,3]
-; AVX512-FCP-NEXT:    # ymm10 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpermd %ymm8, %ymm10, %ymm8
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[1,3,1,3]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm9 = ymm9[u,u,u,1,9],zero,ymm9[u,u,u,u,2,10],zero,ymm9[u,u,u,u,19,27],zero,ymm9[u,u,u,u,20,28],zero,ymm9[u,u,u,u,21]
-; AVX512-FCP-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm8, %ymm9
-; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm7, %ymm9
-; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm5[8],xmm6[8],xmm5[9],xmm6[9],xmm5[10],xmm6[10],xmm5[11],xmm6[11],xmm5[12],xmm6[12],xmm5[13],xmm6[13],xmm5[14],xmm6[14],xmm5[15],xmm6[15]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u],zero,zero,xmm5[12,13,u,u,u],zero,zero,xmm5[14,15,u,u,u]
-; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,12,13],zero,zero,xmm0[u,u,u,14,15],zero,zero,xmm0[u,u,u]
-; AVX512-FCP-NEXT:    vpor %xmm5, %xmm0, %xmm0
-; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm4[8],xmm3[8],xmm4[9],xmm3[9],xmm4[10],xmm3[10],xmm4[11],xmm3[11],xmm4[12],xmm3[12],xmm4[13],xmm3[13],xmm4[14],xmm3[14],xmm4[15],xmm3[15]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[10],zero,xmm1[u,u,u,u,13,12],zero,xmm1[u,u,u,u,15,14],zero
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = zero,xmm2[13,u,u,u,u],zero,zero,xmm2[14,u,u,u,u],zero,zero,xmm2[15]
-; AVX512-FCP-NEXT:    vpor %xmm2, %xmm1, %xmm1
-; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
-; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm1, %zmm9, %zmm0
-; AVX512-FCP-NEXT:    vmovdqa %xmm1, 96(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm11, (%rax)
-; AVX512-FCP-NEXT:    vmovdqa %ymm0, 64(%rax)
+; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm2
+; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm3
+; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm4
+; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm5
+; AVX512-FCP-NEXT:    vmovdqa (%r8), %xmm1
+; AVX512-FCP-NEXT:    vmovdqa (%r10), %xmm0
+; AVX512-FCP-NEXT:    vinserti128 $1, %xmm5, %ymm4, %ymm6
+; AVX512-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm7
+; AVX512-FCP-NEXT:    vinserti128 $1, (%r9), %ymm1, %ymm1
+; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm1
+; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm4[8],xmm5[8],xmm4[9],xmm5[9],xmm4[10],xmm5[10],xmm4[11],xmm5[11],xmm4[12],xmm5[12],xmm4[13],xmm5[13],xmm4[14],xmm5[14],xmm4[15],xmm5[15]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u],zero,zero,xmm4[12,13,u,u,u],zero,zero,xmm4[14,15,u,u,u]
+; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,12,13],zero,zero,xmm2[u,u,u,14,15],zero,zero,xmm2[u,u,u]
+; AVX512-FCP-NEXT:    vpor %xmm4, %xmm2, %xmm2
+; AVX512-FCP-NEXT:    vextracti128 $1, %ymm1, %xmm3
+; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[10],zero,xmm3[u,u,u,u,13,12],zero,xmm3[u,u,u,u,15,14],zero
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = zero,xmm0[13,u,u,u,u],zero,zero,xmm0[14,u,u,u,u],zero,zero,xmm0[15]
+; AVX512-FCP-NEXT:    vpor %xmm4, %xmm3, %xmm3
+; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm3
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm7[3,1,1,3]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[1],zero,zero,ymm2[u,u,u,10,2],zero,zero,ymm2[u,u,u,11,3],zero,zero,ymm2[u,u,u,20,28],zero,zero,ymm2[u,u,u,21,29],zero,zero,ymm2[u]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm6[1,3,3,1]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = zero,ymm4[1,9,u,u,u],zero,zero,ymm4[2,10,u,u,u],zero,zero,ymm4[3,19,u,u,u],zero,zero,ymm4[28,20,u,u,u],zero,zero,ymm4[29,21,u]
+; AVX512-FCP-NEXT:    vpor %ymm2, %ymm4, %ymm2
+; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm0[0,1,2,3,4,5,5,6]
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [2,2,3,3,2,2,3,3]
+; AVX512-FCP-NEXT:    # ymm5 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vpermd %ymm4, %ymm5, %ymm4
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm5 = ymm1[1,3,1,3]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm5 = ymm5[u,u,u,1,9],zero,ymm5[u,u,u,u,2,10],zero,ymm5[u,u,u,u,19,27],zero,ymm5[u,u,u,u,20,28],zero,ymm5[u,u,u,u,21]
+; AVX512-FCP-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm5
+; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm5
+; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm3, %zmm5, %zmm2
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm6[0,2,0,2]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = zero,zero,ymm4[0,8,u,u,u],zero,zero,ymm4[1,9,u,u,u],zero,zero,ymm4[18,26,u,u,u],zero,zero,ymm4[19,27,u,u,u],zero,zero,ymm4[20,28]
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [1,5,2,6,1,5,2,6]
+; AVX512-FCP-NEXT:    # ymm5 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vpermd %ymm6, %ymm5, %ymm6
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[u,u,u],zero,zero,ymm6[1,5,u,u,u],zero,zero,ymm6[2,6,u,u,u],zero,zero,ymm6[19,23,u,u,u],zero,zero,ymm6[24,28,u,u,u],zero
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm4, %zmm4
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm7[0,2,0,2]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[0,8],zero,zero,ymm6[u,u,u,1,9],zero,zero,ymm6[u,u,u,2,10],zero,zero,ymm6[u,u,u,19,27],zero,zero,ymm6[u,u,u,20,28],zero,zero
+; AVX512-FCP-NEXT:    vpermd %ymm7, %ymm5, %ymm7
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,1,5],zero,zero,ymm7[u,u,u,2,6],zero,zero,ymm7[u,u,u,19,23],zero,zero,ymm7[u,u,u,24,28],zero,zero,ymm7[u,u,u,25]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm6, %zmm6
+; AVX512-FCP-NEXT:    vporq %zmm4, %zmm6, %zmm4
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm0[1,1,0,0,4,5,6,7]
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm7 = [0,1,0,1,0,0,0,0]
+; AVX512-FCP-NEXT:    vpermd %ymm6, %ymm7, %ymm6
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm6, %zmm0
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm1[0,2,0,2]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[u,u,u,u,0,8],zero,ymm6[u,u,u,u,1,9],zero,ymm6[u,u,u,u,18,26],zero,ymm6[u,u,u,u,19,27],zero,ymm6[u,u,u,u]
+; AVX512-FCP-NEXT:    vpermd %ymm1, %ymm5, %ymm1
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[0,4],zero,ymm1[u,u,u,u,1,5],zero,ymm1[u,u,u,u,2,6],zero,ymm1[u,u,u,u,19,23],zero,ymm1[u,u,u,u,24,28],zero,ymm1[u]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm6, %zmm1
+; AVX512-FCP-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1
+; AVX512-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm1
+; AVX512-FCP-NEXT:    vmovdqa %xmm3, 96(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, (%rax)
+; AVX512-FCP-NEXT:    vmovdqa %ymm2, 64(%rax)
 ; AVX512-FCP-NEXT:    vzeroupper
 ; AVX512-FCP-NEXT:    retq
 ;
@@ -2198,76 +2202,77 @@ define void @store_i8_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512DQ-NEXT:    vmovdqa (%rsi), %xmm1
-; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm5
-; AVX512DQ-NEXT:    vmovdqa (%rcx), %xmm6
-; AVX512DQ-NEXT:    vmovdqa (%r8), %xmm3
-; AVX512DQ-NEXT:    vmovdqa (%r9), %xmm4
-; AVX512DQ-NEXT:    vmovdqa (%r10), %xmm2
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm8
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm6, %ymm5, %ymm9
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm7
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm10 = ymm9[u,u,u,u,u,5],zero,ymm9[u,u,u,u,u,6],zero,ymm9[u,u,u,u,u],zero,ymm9[23,u,u,u,u,u],zero,ymm9[24,u,u,u,u]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm11 = ymm9[2,3,0,1]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[u,u,u,u,u],zero,ymm11[5,u,u,u,u,u],zero,ymm11[6,u,u,u,u,u,23],zero,ymm11[u,u,u,u,u,24],zero,ymm11[u,u,u,u]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm12 = [255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255]
-; AVX512DQ-NEXT:    vpternlogq $50, %ymm10, %ymm12, %ymm11
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm10 = ymm9[0,2,0,2]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm10 = zero,zero,ymm10[0,8,u,u,u],zero,zero,ymm10[1,9,u,u,u],zero,zero,ymm10[18,26,u,u,u],zero,zero,ymm10[19,27,u,u,u],zero,zero,ymm10[20,28]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm11, %zmm10, %zmm10
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm11 = ymm8[u,u,u,5],zero,ymm8[u,u,u,u,u,6],zero,ymm8[u,u,u,u,u],zero,ymm8[23,u,u,u,u,u],zero,ymm8[24,u,u,u,u,u],zero
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm13 = ymm8[2,3,0,1]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm13 = ymm13[u,u,u],zero,ymm13[5,u,u,u,u,u],zero,ymm13[6,u,u,u,u,u,23],zero,ymm13[u,u,u,u,u,24],zero,ymm13[u,u,u,u,u,25]
-; AVX512DQ-NEXT:    vpternlogq $200, %ymm11, %ymm12, %ymm13
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm11 = ymm8[0,2,0,2]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[0,8],zero,zero,ymm11[u,u,u,1,9],zero,zero,ymm11[u,u,u,2,10],zero,zero,ymm11[u,u,u,19,27],zero,zero,ymm11[u,u,u,20,28],zero,zero
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm13, %zmm11, %zmm11
-; AVX512DQ-NEXT:    vporq %zmm10, %zmm11, %zmm10
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm11 = ymm7[4],zero,ymm7[u,u,u,u,u,5],zero,ymm7[u,u,u,u,u,6],zero,ymm7[u,u,u,u,u],zero,ymm7[23,u,u,u,u,u],zero,ymm7[24,u,u]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm12 = ymm7[2,3,0,1]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm12 = zero,ymm12[4,u,u,u,u,u],zero,ymm12[5,u,u,u,u,u],zero,ymm12[6,u,u,u,u,u,23],zero,ymm12[u,u,u,u,u,24],zero,ymm12[u,u]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm13 = [255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255]
-; AVX512DQ-NEXT:    vpternlogq $200, %ymm11, %ymm13, %ymm12
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm11 = ymm7[0,2,0,2]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[u,u,u,u,0,8],zero,ymm11[u,u,u,u,1,9],zero,ymm11[u,u,u,u,18,26],zero,ymm11[u,u,u,u,19,27],zero,ymm11[u,u,u,u]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm12, %zmm11, %zmm11
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm12 = xmm2[4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,0,1,0]
-; AVX512DQ-NEXT:    vpandn %ymm12, %ymm13, %ymm12
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm13 = xmm2[1,1,0,0,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm13 = xmm13[0,1,2,0]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,0,1,0]
-; AVX512DQ-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm13, %ymm13
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm12, %zmm13, %zmm12
-; AVX512DQ-NEXT:    vporq %zmm12, %zmm11, %zmm11
-; AVX512DQ-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm11
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[3,1,1,3]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm8 = ymm8[1],zero,zero,ymm8[u,u,u,10,2],zero,zero,ymm8[u,u,u,11,3],zero,zero,ymm8[u,u,u,20,28],zero,zero,ymm8[u,u,u,21,29],zero,zero,ymm8[u]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[1,3,3,1]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm9 = zero,ymm9[1,9,u,u,u],zero,zero,ymm9[2,10,u,u,u],zero,zero,ymm9[3,19,u,u,u],zero,zero,ymm9[28,20,u,u,u],zero,zero,ymm9[29,21,u]
-; AVX512DQ-NEXT:    vpor %ymm8, %ymm9, %ymm8
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm9 = xmm2[0,1,2,3,4,5,5,6]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm9 = xmm9[2,2,3,3]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,1,0,1]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[1,3,1,3]
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm4
+; AVX512DQ-NEXT:    vmovdqa (%rsi), %xmm5
+; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm6
+; AVX512DQ-NEXT:    vmovdqa (%rcx), %xmm7
+; AVX512DQ-NEXT:    vmovdqa (%r8), %xmm0
+; AVX512DQ-NEXT:    vmovdqa (%r10), %xmm1
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm7, %ymm6, %ymm3
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm5, %ymm4, %ymm2
+; AVX512DQ-NEXT:    vinserti128 $1, (%r9), %ymm0, %ymm0
+; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm6[8],xmm7[8],xmm6[9],xmm7[9],xmm6[10],xmm7[10],xmm6[11],xmm7[11],xmm6[12],xmm7[12],xmm6[13],xmm7[13],xmm6[14],xmm7[14],xmm6[15],xmm7[15]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[u,u],zero,zero,xmm6[12,13,u,u,u],zero,zero,xmm6[14,15,u,u,u]
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm4[8],xmm5[8],xmm4[9],xmm5[9],xmm4[10],xmm5[10],xmm4[11],xmm5[11],xmm4[12],xmm5[12],xmm4[13],xmm5[13],xmm4[14],xmm5[14],xmm4[15],xmm5[15]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,12,13],zero,zero,xmm4[u,u,u,14,15],zero,zero,xmm4[u,u,u]
+; AVX512DQ-NEXT:    vpor %xmm6, %xmm4, %xmm4
+; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm5
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[10],zero,xmm5[u,u,u,u,13,12],zero,xmm5[u,u,u,u,15,14],zero
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm6 = zero,xmm1[13,u,u,u,u],zero,zero,xmm1[14,u,u,u,u],zero,zero,xmm1[15]
+; AVX512DQ-NEXT:    vpor %xmm6, %xmm5, %xmm5
+; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4, %xmm5
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm2[3,1,1,3]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm4 = ymm4[1],zero,zero,ymm4[u,u,u,10,2],zero,zero,ymm4[u,u,u,11,3],zero,zero,ymm4[u,u,u,20,28],zero,zero,ymm4[u,u,u,21,29],zero,zero,ymm4[u]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm6 = ymm3[1,3,3,1]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm6 = zero,ymm6[1,9,u,u,u],zero,zero,ymm6[2,10,u,u,u],zero,zero,ymm6[3,19,u,u,u],zero,zero,ymm6[28,20,u,u,u],zero,zero,ymm6[29,21,u]
+; AVX512DQ-NEXT:    vpor %ymm4, %ymm6, %ymm4
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm6 = xmm1[0,1,2,3,4,5,5,6]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,2,3,3]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,1,0,1]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm0[1,3,1,3]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,1,9],zero,ymm7[u,u,u,u,2,10],zero,ymm7[u,u,u,u,19,27],zero,ymm7[u,u,u,u,20,28],zero,ymm7[u,u,u,u,21]
-; AVX512DQ-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm9, %ymm7
-; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm8, %ymm7
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm5[8],xmm6[8],xmm5[9],xmm6[9],xmm5[10],xmm6[10],xmm5[11],xmm6[11],xmm5[12],xmm6[12],xmm5[13],xmm6[13],xmm5[14],xmm6[14],xmm5[15],xmm6[15]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u],zero,zero,xmm5[12,13,u,u,u],zero,zero,xmm5[14,15,u,u,u]
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,12,13],zero,zero,xmm0[u,u,u,14,15],zero,zero,xmm0[u,u,u]
-; AVX512DQ-NEXT:    vpor %xmm5, %xmm0, %xmm0
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm4[8],xmm3[8],xmm4[9],xmm3[9],xmm4[10],xmm3[10],xmm4[11],xmm3[11],xmm4[12],xmm3[12],xmm4[13],xmm3[13],xmm4[14],xmm3[14],xmm4[15],xmm3[15]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[10],zero,xmm1[u,u,u,u,13,12],zero,xmm1[u,u,u,u,15,14],zero
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm2 = zero,xmm2[13,u,u,u,u],zero,zero,xmm2[14,u,u,u,u],zero,zero,xmm2[15]
-; AVX512DQ-NEXT:    vpor %xmm2, %xmm1, %xmm1
-; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
-; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm1, %zmm7, %zmm0
-; AVX512DQ-NEXT:    vmovdqa %xmm1, 96(%rax)
-; AVX512DQ-NEXT:    vmovdqa %ymm0, 64(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm11, (%rax)
+; AVX512DQ-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm6, %ymm7
+; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm7
+; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm5, %zmm7, %zmm4
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm6 = ymm3[u,u,u,u,u,5],zero,ymm3[u,u,u,u,u,6],zero,ymm3[u,u,u,u,u],zero,ymm3[23,u,u,u,u,u],zero,ymm3[24,u,u,u,u]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm3[2,3,0,1]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,u,u],zero,ymm7[5,u,u,u,u,u],zero,ymm7[6,u,u,u,u,u,23],zero,ymm7[u,u,u,u,u,24],zero,ymm7[u,u,u,u]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm8 = [255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255]
+; AVX512DQ-NEXT:    vpternlogq $50, %ymm6, %ymm8, %ymm7
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,2,0,2]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,ymm3[0,8,u,u,u],zero,zero,ymm3[1,9,u,u,u],zero,zero,ymm3[18,26,u,u,u],zero,zero,ymm3[19,27,u,u,u],zero,zero,ymm3[20,28]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm3, %zmm3
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm6 = ymm2[u,u,u,5],zero,ymm2[u,u,u,u,u,6],zero,ymm2[u,u,u,u,u],zero,ymm2[23,u,u,u,u,u],zero,ymm2[24,u,u,u,u,u],zero
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm2[2,3,0,1]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u],zero,ymm7[5,u,u,u,u,u],zero,ymm7[6,u,u,u,u,u,23],zero,ymm7[u,u,u,u,u,24],zero,ymm7[u,u,u,u,u,25]
+; AVX512DQ-NEXT:    vpternlogq $200, %ymm6, %ymm8, %ymm7
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,0,2]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[0,8],zero,zero,ymm2[u,u,u,1,9],zero,zero,ymm2[u,u,u,2,10],zero,zero,ymm2[u,u,u,19,27],zero,zero,ymm2[u,u,u,20,28],zero,zero
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm2, %zmm2
+; AVX512DQ-NEXT:    vporq %zmm3, %zmm2, %zmm2
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm3 = xmm1[4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,1,0]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm6 = [255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255]
+; AVX512DQ-NEXT:    vpandn %ymm3, %ymm6, %ymm3
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[1,1,0,0,4,5,6,7]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,0]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,1,0]
+; AVX512DQ-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm3 = ymm0[4],zero,ymm0[u,u,u,u,u,5],zero,ymm0[u,u,u,u,u,6],zero,ymm0[u,u,u,u,u],zero,ymm0[23,u,u,u,u,u],zero,ymm0[24,u,u]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm0[2,3,0,1]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm7 = zero,ymm7[4,u,u,u,u,u],zero,ymm7[5,u,u,u,u,u],zero,ymm7[6,u,u,u,u,u,23],zero,ymm7[u,u,u,u,u,24],zero,ymm7[u,u]
+; AVX512DQ-NEXT:    vpternlogq $200, %ymm3, %ymm6, %ymm7
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,0,2]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,0,8],zero,ymm0[u,u,u,u,1,9],zero,ymm0[u,u,u,u,18,26],zero,ymm0[u,u,u,u,19,27],zero,ymm0[u,u,u,u]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0
+; AVX512DQ-NEXT:    vmovdqa %xmm5, 96(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm0, (%rax)
+; AVX512DQ-NEXT:    vmovdqa %ymm4, 64(%rax)
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
@@ -2275,69 +2280,70 @@ define void @store_i8_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP:       # %bb.0:
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm5
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm6
-; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %xmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %xmm4
-; AVX512DQ-FCP-NEXT:    vmovdqa (%r10), %xmm2
-; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm7
-; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm6, %ymm5, %ymm8
-; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm9
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm8[0,2,0,2]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm10 = zero,zero,ymm10[0,8,u,u,u],zero,zero,ymm10[1,9,u,u,u],zero,zero,ymm10[18,26,u,u,u],zero,zero,ymm10[19,27,u,u,u],zero,zero,ymm10[20,28]
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm11 = [1,5,2,6,1,5,2,6]
-; AVX512DQ-FCP-NEXT:    # ymm11 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm8, %ymm11, %ymm12
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm12 = ymm12[u,u,u],zero,zero,ymm12[1,5,u,u,u],zero,zero,ymm12[2,6,u,u,u],zero,zero,ymm12[19,23,u,u,u],zero,zero,ymm12[24,28,u,u,u],zero
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm10, %zmm10
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm12 = ymm7[0,2,0,2]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm12 = ymm12[0,8],zero,zero,ymm12[u,u,u,1,9],zero,zero,ymm12[u,u,u,2,10],zero,zero,ymm12[u,u,u,19,27],zero,zero,ymm12[u,u,u,20,28],zero,zero
-; AVX512DQ-FCP-NEXT:    vpermd %ymm7, %ymm11, %ymm13
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm13 = ymm13[u,u,u,1,5],zero,zero,ymm13[u,u,u,2,6],zero,zero,ymm13[u,u,u,19,23],zero,zero,ymm13[u,u,u,24,28],zero,zero,ymm13[u,u,u,25]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm12, %zmm12
-; AVX512DQ-FCP-NEXT:    vporq %zmm10, %zmm12, %zmm10
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm12 = xmm2[1,1,0,0,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm13 = [0,1,0,1,0,0,0,0]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm12, %ymm13, %ymm12
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm13 = xmm2[4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,0,1,0]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm12, %zmm12
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm13 = ymm9[0,2,0,2]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm13 = ymm13[u,u,u,u,0,8],zero,ymm13[u,u,u,u,1,9],zero,ymm13[u,u,u,u,18,26],zero,ymm13[u,u,u,u,19,27],zero,ymm13[u,u,u,u]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm9, %ymm11, %ymm11
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[0,4],zero,ymm11[u,u,u,u,1,5],zero,ymm11[u,u,u,u,2,6],zero,ymm11[u,u,u,u,19,23],zero,ymm11[u,u,u,u,24,28],zero,ymm11[u]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm13, %zmm11
-; AVX512DQ-FCP-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm12, %zmm11
-; AVX512DQ-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm11
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[3,1,1,3]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[1],zero,zero,ymm7[u,u,u,10,2],zero,zero,ymm7[u,u,u,11,3],zero,zero,ymm7[u,u,u,20,28],zero,zero,ymm7[u,u,u,21,29],zero,zero,ymm7[u]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[1,3,3,1]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm8 = zero,ymm8[1,9,u,u,u],zero,zero,ymm8[2,10,u,u,u],zero,zero,ymm8[3,19,u,u,u],zero,zero,ymm8[28,20,u,u,u],zero,zero,ymm8[29,21,u]
-; AVX512DQ-FCP-NEXT:    vpor %ymm7, %ymm8, %ymm7
-; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm8 = xmm2[0,1,2,3,4,5,5,6]
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [2,2,3,3,2,2,3,3]
-; AVX512DQ-FCP-NEXT:    # ymm10 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm8, %ymm10, %ymm8
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[1,3,1,3]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm9 = ymm9[u,u,u,1,9],zero,ymm9[u,u,u,u,2,10],zero,ymm9[u,u,u,u,19,27],zero,ymm9[u,u,u,u,20,28],zero,ymm9[u,u,u,u,21]
-; AVX512DQ-FCP-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm8, %ymm9
-; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm7, %ymm9
-; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm5[8],xmm6[8],xmm5[9],xmm6[9],xmm5[10],xmm6[10],xmm5[11],xmm6[11],xmm5[12],xmm6[12],xmm5[13],xmm6[13],xmm5[14],xmm6[14],xmm5[15],xmm6[15]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u],zero,zero,xmm5[12,13,u,u,u],zero,zero,xmm5[14,15,u,u,u]
-; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,12,13],zero,zero,xmm0[u,u,u,14,15],zero,zero,xmm0[u,u,u]
-; AVX512DQ-FCP-NEXT:    vpor %xmm5, %xmm0, %xmm0
-; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm4[8],xmm3[8],xmm4[9],xmm3[9],xmm4[10],xmm3[10],xmm4[11],xmm3[11],xmm4[12],xmm3[12],xmm4[13],xmm3[13],xmm4[14],xmm3[14],xmm4[15],xmm3[15]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[10],zero,xmm1[u,u,u,u,13,12],zero,xmm1[u,u,u,u,15,14],zero
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = zero,xmm2[13,u,u,u,u],zero,zero,xmm2[14,u,u,u,u],zero,zero,xmm2[15]
-; AVX512DQ-FCP-NEXT:    vpor %xmm2, %xmm1, %xmm1
-; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
-; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm1, %zmm9, %zmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, 96(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm11, (%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, 64(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm2
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm4
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm5
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %xmm1
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r10), %xmm0
+; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm5, %ymm4, %ymm6
+; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm7
+; AVX512DQ-FCP-NEXT:    vinserti128 $1, (%r9), %ymm1, %ymm1
+; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm1
+; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm4[8],xmm5[8],xmm4[9],xmm5[9],xmm4[10],xmm5[10],xmm4[11],xmm5[11],xmm4[12],xmm5[12],xmm4[13],xmm5[13],xmm4[14],xmm5[14],xmm4[15],xmm5[15]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u],zero,zero,xmm4[12,13,u,u,u],zero,zero,xmm4[14,15,u,u,u]
+; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,12,13],zero,zero,xmm2[u,u,u,14,15],zero,zero,xmm2[u,u,u]
+; AVX512DQ-FCP-NEXT:    vpor %xmm4, %xmm2, %xmm2
+; AVX512DQ-FCP-NEXT:    vextracti128 $1, %ymm1, %xmm3
+; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[10],zero,xmm3[u,u,u,u,13,12],zero,xmm3[u,u,u,u,15,14],zero
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = zero,xmm0[13,u,u,u,u],zero,zero,xmm0[14,u,u,u,u],zero,zero,xmm0[15]
+; AVX512DQ-FCP-NEXT:    vpor %xmm4, %xmm3, %xmm3
+; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm3
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm7[3,1,1,3]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[1],zero,zero,ymm2[u,u,u,10,2],zero,zero,ymm2[u,u,u,11,3],zero,zero,ymm2[u,u,u,20,28],zero,zero,ymm2[u,u,u,21,29],zero,zero,ymm2[u]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm6[1,3,3,1]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = zero,ymm4[1,9,u,u,u],zero,zero,ymm4[2,10,u,u,u],zero,zero,ymm4[3,19,u,u,u],zero,zero,ymm4[28,20,u,u,u],zero,zero,ymm4[29,21,u]
+; AVX512DQ-FCP-NEXT:    vpor %ymm2, %ymm4, %ymm2
+; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm0[0,1,2,3,4,5,5,6]
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [2,2,3,3,2,2,3,3]
+; AVX512DQ-FCP-NEXT:    # ymm5 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm4, %ymm5, %ymm4
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm5 = ymm1[1,3,1,3]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm5 = ymm5[u,u,u,1,9],zero,ymm5[u,u,u,u,2,10],zero,ymm5[u,u,u,u,19,27],zero,ymm5[u,u,u,u,20,28],zero,ymm5[u,u,u,u,21]
+; AVX512DQ-FCP-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm5
+; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm5
+; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm3, %zmm5, %zmm2
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm6[0,2,0,2]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = zero,zero,ymm4[0,8,u,u,u],zero,zero,ymm4[1,9,u,u,u],zero,zero,ymm4[18,26,u,u,u],zero,zero,ymm4[19,27,u,u,u],zero,zero,ymm4[20,28]
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [1,5,2,6,1,5,2,6]
+; AVX512DQ-FCP-NEXT:    # ymm5 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm6, %ymm5, %ymm6
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[u,u,u],zero,zero,ymm6[1,5,u,u,u],zero,zero,ymm6[2,6,u,u,u],zero,zero,ymm6[19,23,u,u,u],zero,zero,ymm6[24,28,u,u,u],zero
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm4, %zmm4
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm7[0,2,0,2]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[0,8],zero,zero,ymm6[u,u,u,1,9],zero,zero,ymm6[u,u,u,2,10],zero,zero,ymm6[u,u,u,19,27],zero,zero,ymm6[u,u,u,20,28],zero,zero
+; AVX512DQ-FCP-NEXT:    vpermd %ymm7, %ymm5, %ymm7
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,1,5],zero,zero,ymm7[u,u,u,2,6],zero,zero,ymm7[u,u,u,19,23],zero,zero,ymm7[u,u,u,24,28],zero,zero,ymm7[u,u,u,25]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm6, %zmm6
+; AVX512DQ-FCP-NEXT:    vporq %zmm4, %zmm6, %zmm4
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm0[1,1,0,0,4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm7 = [0,1,0,1,0,0,0,0]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm6, %ymm7, %ymm6
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm6, %zmm0
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm1[0,2,0,2]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[u,u,u,u,0,8],zero,ymm6[u,u,u,u,1,9],zero,ymm6[u,u,u,u,18,26],zero,ymm6[u,u,u,u,19,27],zero,ymm6[u,u,u,u]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm1, %ymm5, %ymm1
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[0,4],zero,ymm1[u,u,u,u,1,5],zero,ymm1[u,u,u,u,2,6],zero,ymm1[u,u,u,u,19,23],zero,ymm1[u,u,u,u,24,28],zero,ymm1[u]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm6, %zmm1
+; AVX512DQ-FCP-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1
+; AVX512DQ-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm1
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm3, 96(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, (%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm2, 64(%rax)
 ; AVX512DQ-FCP-NEXT:    vzeroupper
 ; AVX512DQ-FCP-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll b/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
index 99e8cdb179c8..11f422d67154 100644
--- a/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
+++ b/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
@@ -314,8 +314,8 @@ define void @vec64_i16_widen_to_i32_factor2_broadcast_to_v2i32_factor2(ptr %in.v
 ;
 ; AVX512F-LABEL: vec64_i16_widen_to_i32_factor2_broadcast_to_v2i32_factor2:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
+; AVX512F-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,10,11,0,1,14,15,u,u,u,u,u,u,u,u]
 ; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)
@@ -324,8 +324,8 @@ define void @vec64_i16_widen_to_i32_factor2_broadcast_to_v2i32_factor2(ptr %in.v
 ;
 ; AVX512DQ-LABEL: vec64_i16_widen_to_i32_factor2_broadcast_to_v2i32_factor2:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512DQ-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,10,11,0,1,14,15,u,u,u,u,u,u,u,u]
 ; AVX512DQ-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, (%rcx)
@@ -981,7 +981,7 @@ define void @vec128_i32_widen_to_i64_factor2_broadcast_to_v2i64_factor2(ptr %in.
 ; AVX512F-NEXT:    vpmovsxbd {{.*#+}} xmm0 = [0,5,0,7]
 ; AVX512F-NEXT:    vmovdqa (%rdi), %ymm1
 ; AVX512F-NEXT:    vpaddb (%rsi), %ymm1, %ymm1
-; AVX512F-NEXT:    vpermd %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpermd %zmm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)
 ; AVX512F-NEXT:    vzeroupper
@@ -992,7 +992,7 @@ define void @vec128_i32_widen_to_i64_factor2_broadcast_to_v2i64_factor2(ptr %in.
 ; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} xmm0 = [0,5,0,7]
 ; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm1
 ; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpermd %ymm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpermd %zmm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, (%rcx)
 ; AVX512DQ-NEXT:    vzeroupper
@@ -4026,10 +4026,10 @@ define void @vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6(ptr %in.
 ;
 ; AVX512F-FAST-LABEL: vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6:
 ; AVX512F-FAST:       # %bb.0:
-; AVX512F-FAST-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512F-FAST-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512F-FAST-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512F-FAST-NEXT:    vmovdqa 48(%rdi), %xmm1
 ; AVX512F-FAST-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX512F-FAST-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512F-FAST-NEXT:    vpbroadcastq %xmm0, %ymm2
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1,2,3],ymm2[4],ymm1[5,6,7],ymm2[8],ymm1[9,10,11],ymm2[12],ymm1[13,14,15]
 ; AVX512F-FAST-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
@@ -4062,10 +4062,10 @@ define void @vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6(ptr %in.
 ;
 ; AVX512DQ-FAST-LABEL: vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6:
 ; AVX512DQ-FAST:       # %bb.0:
-; AVX512DQ-FAST-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512DQ-FAST-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512DQ-FAST-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512DQ-FAST-NEXT:    vmovdqa 48(%rdi), %xmm1
 ; AVX512DQ-FAST-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX512DQ-FAST-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512DQ-FAST-NEXT:    vpbroadcastq %xmm0, %ymm2
 ; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1,2,3],ymm2[4],ymm1[5,6,7],ymm2[8],ymm1[9,10,11],ymm2[12],ymm1[13,14,15]
 ; AVX512DQ-FAST-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
@@ -4541,9 +4541,9 @@ define void @vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2(ptr %
 ;
 ; AVX512F-LABEL: vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512F-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512F-NEXT:    vmovdqa 48(%rdi), %xmm1
-; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512F-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
 ; AVX512F-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]
 ; AVX512F-NEXT:    vpbroadcastw %xmm0, %ymm0
@@ -4559,9 +4559,9 @@ define void @vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2(ptr %
 ;
 ; AVX512DQ-LABEL: vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm0
+; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa 48(%rdi), %xmm1
-; AVX512DQ-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]
 ; AVX512DQ-NEXT:    vpbroadcastw %xmm0, %ymm0
-- 
GitLab


From 694c444b5bbb56dcba8978d283fe5385237c309a Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Mon, 15 Apr 2024 20:58:08 -0400
Subject: [PATCH 317/464] [gn] port 311ff3917827 more

---
 llvm/utils/gn/secondary/libcxx/include/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 865a79b63cd8..ee44558a4e99 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -982,6 +982,7 @@ if (current_toolchain == default_toolchain) {
       "ctgmath",
       "ctime",
       "ctype.h",
+      "cuchar",
       "cwchar",
       "cwctype",
       "deque",
-- 
GitLab


From 2ac562ab784c6bf04e6d3026c567d3552a735668 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Mon, 15 Apr 2024 17:58:39 -0700
Subject: [PATCH 318/464] [Sema] Mark alias/ifunc targets used and consider
 mangled names

https://reviews.llvm.org/D54188 marked "alias" targets as used in C to
fix -Wunused false positives. This patch extends the approach to handle
mangled names to support global scope names in C++ and the
`overloadable` attribute in C.

In addition, we mark ifunc targets as used to fix #63957.

While our approach has false negatives for namespace scope names, the
majority of alias/ifunc C++ uses (global scope with no overloads) are
handled.

Note: The following function with internal linkage but C language
linkage type is mangled in Clang but not in GCC. This inconsistency
makes alias/ifunc difficult to use in C++ with portability (#88593).
```
extern "C" {
static void f0() {}
// GCC: void g0() __attribute__((alias("_ZL2f0v")));
// Clang: void g0() __attribute__((alias("f0")));
}
```

Pull Request: https://github.com/llvm/llvm-project/pull/87130
---
 clang/lib/Sema/CMakeLists.txt                 |  1 +
 clang/lib/Sema/SemaDeclAttr.cpp               | 44 ++++++++++++++-----
 clang/test/AST/ast-dump-attr-json.cpp         |  1 +
 clang/test/Sema/alias-unused-win.cpp          |  2 +-
 clang/test/Sema/alias-unused.cpp              | 16 ++++---
 .../llvm-project-overlay/clang/BUILD.bazel    |  1 +
 6 files changed, 46 insertions(+), 19 deletions(-)

diff --git a/clang/lib/Sema/CMakeLists.txt b/clang/lib/Sema/CMakeLists.txt
index ab3b813a9ccd..a96439df6642 100644
--- a/clang/lib/Sema/CMakeLists.txt
+++ b/clang/lib/Sema/CMakeLists.txt
@@ -1,5 +1,6 @@
 set(LLVM_LINK_COMPONENTS
   Core
+  Demangle
   FrontendHLSL
   FrontendOpenMP
   MC
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index b7b1fbc625a1..d26f130b5774 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -45,6 +45,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/STLForwardCompat.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/Demangle/Demangle.h"
 #include "llvm/IR/Assumptions.h"
 #include "llvm/MC/MCSectionMachO.h"
 #include "llvm/Support/Error.h"
@@ -1983,6 +1984,36 @@ static void handleWeakRefAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   D->addAttr(::new (S.Context) WeakRefAttr(S.Context, AL));
 }
 
+// Mark alias/ifunc target as used. Due to name mangling, we look up the
+// demangled name ignoring parameters (not supported by microsoftDemangle
+// https://github.com/llvm/llvm-project/issues/88825). This should handle the
+// majority of use cases while leaving namespace scope names unmarked.
+static void markUsedForAliasOrIfunc(Sema &S, Decl *D, const ParsedAttr &AL,
+                                    StringRef Str) {
+  std::unique_ptr<char, llvm::FreeDeleter> Demangled;
+  if (S.getASTContext().getCXXABIKind() != TargetCXXABI::Microsoft)
+    Demangled.reset(llvm::itaniumDemangle(Str, /*ParseParams=*/false));
+  std::unique_ptr<MangleContext> MC(S.Context.createMangleContext());
+  SmallString<256> Name;
+
+  const DeclarationNameInfo Target(
+      &S.Context.Idents.get(Demangled ? Demangled.get() : Str), AL.getLoc());
+  LookupResult LR(S, Target, Sema::LookupOrdinaryName);
+  if (S.LookupName(LR, S.TUScope)) {
+    for (NamedDecl *ND : LR) {
+      if (MC->shouldMangleDeclName(ND)) {
+        llvm::raw_svector_ostream Out(Name);
+        Name.clear();
+        MC->mangleName(GlobalDecl(ND), Out);
+      } else {
+        Name = ND->getIdentifier()->getName();
+      }
+      if (Name == Str)
+        ND->markUsed(S.Context);
+    }
+  }
+}
+
 static void handleIFuncAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   StringRef Str;
   if (!S.checkStringLiteralArgumentAttr(AL, 0, Str))
@@ -1995,6 +2026,7 @@ static void handleIFuncAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
     return;
   }
 
+  markUsedForAliasOrIfunc(S, D, AL, Str);
   D->addAttr(::new (S.Context) IFuncAttr(S.Context, AL, Str));
 }
 
@@ -2029,17 +2061,7 @@ static void handleAliasAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
     }
   }
 
-  // Mark target used to prevent unneeded-internal-declaration warnings.
-  if (!S.LangOpts.CPlusPlus) {
-    // FIXME: demangle Str for C++, as the attribute refers to the mangled
-    // linkage name, not the pre-mangled identifier.
-    const DeclarationNameInfo target(&S.Context.Idents.get(Str), AL.getLoc());
-    LookupResult LR(S, target, Sema::LookupOrdinaryName);
-    if (S.LookupQualifiedName(LR, S.getCurLexicalContext()))
-      for (NamedDecl *ND : LR)
-        ND->markUsed(S.Context);
-  }
-
+  markUsedForAliasOrIfunc(S, D, AL, Str);
   D->addAttr(::new (S.Context) AliasAttr(S.Context, AL, Str));
 }
 
diff --git a/clang/test/AST/ast-dump-attr-json.cpp b/clang/test/AST/ast-dump-attr-json.cpp
index 051c2956abfd..883e584bfedf 100644
--- a/clang/test/AST/ast-dump-attr-json.cpp
+++ b/clang/test/AST/ast-dump-attr-json.cpp
@@ -46,6 +46,7 @@ __thread __attribute__ ((tls_model ("local-exec"))) int tls_model_var;
 // CHECK-NEXT:    "tokLen": 11
 // CHECK-NEXT:   }
 // CHECK-NEXT:  },
+// CHECK-NEXT:  "isUsed": true,
 // CHECK-NEXT:  "name": "global_decl",
 // CHECK-NEXT:  "mangledName": "global_decl",
 // CHECK-NEXT:  "type": {
diff --git a/clang/test/Sema/alias-unused-win.cpp b/clang/test/Sema/alias-unused-win.cpp
index 47c96d411751..97d57a3bbd1e 100644
--- a/clang/test/Sema/alias-unused-win.cpp
+++ b/clang/test/Sema/alias-unused-win.cpp
@@ -7,7 +7,7 @@ extern "C" {
 static int f(void) { return 42; } // cxx-warning{{unused function 'f'}}
 int g(void) __attribute__((alias("f")));
 
-static int foo [] = { 42, 0xDEAD }; // cxx-warning{{variable 'foo' is not needed and will not be emitted}}
+static int foo [] = { 42, 0xDEAD };
 extern typeof(foo) bar __attribute__((unused, alias("foo")));
 
 static int __attribute__((overloadable)) f0(int x) { return x; } // expected-warning{{unused function 'f0'}}
diff --git a/clang/test/Sema/alias-unused.cpp b/clang/test/Sema/alias-unused.cpp
index dc8e46f072d7..c0b541c880e5 100644
--- a/clang/test/Sema/alias-unused.cpp
+++ b/clang/test/Sema/alias-unused.cpp
@@ -14,24 +14,26 @@ extern typeof(foo) bar __attribute__((unused, alias("foo")));
 /// We report a warning in C++ mode because the internal linkage `resolver` gets
 /// mangled as it does not have a language linkage. GCC does not mangle
 /// `resolver` or report a warning.
-static int (*resolver(void))(void) { return f; } // expected-warning{{unused function 'resolver'}}
+static int (*resolver(void))(void) { return f; } // cxx-warning{{unused function 'resolver'}}
 int ifunc(void) __attribute__((ifunc("resolver")));
 
-static int __attribute__((overloadable)) f0(int x) { return x; } // expected-warning{{unused function 'f0'}}
+static int __attribute__((overloadable)) f0(int x) { return x; }
 static float __attribute__((overloadable)) f0(float x) { return x; } // expected-warning{{unused function 'f0'}}
 int g0(void) __attribute__((alias("_ZL2f0i")));
 
 #ifdef __cplusplus
-static int f1() { return 42; } // expected-warning{{unused function 'f1'}}
+static int f1() { return 42; }
 int g1(void) __attribute__((alias("_ZL2f1v")));
 }
 
-static int f2(int) { return 42; } // expected-warning{{unused function 'f2'}}
-static int f2() { return 42; } // expected-warning{{unused function 'f2'}}
+/// We demangle alias/ifunc target and mark all found functions as used.
+
+static int f2(int) { return 42; } // cxx-warning{{unused function 'f2'}}
+static int f2() { return 42; }
 int g2() __attribute__((alias("_ZL2f2v")));
 
-static int (*resolver1())() { return f; } // expected-warning{{unused function 'resolver1'}}
-static int (*resolver1(int))() { return f; } // expected-warning{{unused function 'resolver1'}}
+static int (*resolver1())() { return f; } // cxx-warning{{unused function 'resolver1'}}
+static int (*resolver1(int))() { return f; }
 int ifunc1() __attribute__((ifunc("_ZL9resolver1i")));
 
 /// TODO: We should report "unused function" for f3(int).
diff --git a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
index c2f77e3abca0..725ac6bb3812 100644
--- a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
@@ -1136,6 +1136,7 @@ cc_library(
         "//llvm:AllTargetsAsmParsers",
         "//llvm:AllTargetsCodeGens",
         "//llvm:Core",
+        "//llvm:Demangle",
         "//llvm:FrontendHLSL",
         "//llvm:FrontendOpenMP",
         "//llvm:MC",
-- 
GitLab


From 8aa7e378dee27ec81959ef6750a7dd07cefdc77d Mon Sep 17 00:00:00 2001
From: Allen <zhongyunde@huawei.com>
Date: Tue, 16 Apr 2024 09:00:03 +0800
Subject: [PATCH 319/464] [InterleavedAccessPass] Get round the unsupported
 large scalarize vectors (#88643)

When build with option -msve-vector-bits=512, the return vaule of
Subtarget->getMinSVEVectorSizeInBits() is 512;
While the MinElts is still 4 for <vscale x 4 x double> in
getNumInterleavedAccesses, so it creates invalid
llvm.aarch64.sve.ld2.sret.nxv4f64, which need be splited.
Unlikely, the related custom spilting is not supported now.

Fix https://github.com/llvm/llvm-project/issues/88247
---
 .../Target/AArch64/AArch64ISelLowering.cpp    |  2 +-
 .../AArch64/sve-interleaved-accesses.ll       | 24 +++++++++++++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index eee67a0f823c..7947d73f9a4d 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -15888,7 +15888,7 @@ unsigned AArch64TargetLowering::getNumInterleavedAccesses(
   unsigned VecSize = 128;
   unsigned ElSize = DL.getTypeSizeInBits(VecTy->getElementType());
   unsigned MinElts = VecTy->getElementCount().getKnownMinValue();
-  if (UseScalable)
+  if (UseScalable && isa<FixedVectorType>(VecTy))
     VecSize = std::max(Subtarget->getMinSVEVectorSizeInBits(), 128u);
   return std::max<unsigned>(1, (MinElts * ElSize + 127) / VecSize);
 }
diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleaved-accesses.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleaved-accesses.ll
index feb22aa1a376..45e2c36836ff 100644
--- a/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleaved-accesses.ll
+++ b/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleaved-accesses.ll
@@ -491,6 +491,30 @@ define void @store_bfloat_factor2(ptr %ptr, <16 x bfloat> %v0, <16 x bfloat> %v1
   ret void
 }
 
+; Ensure vscale_range property does not affect scalable vector types.
+define { <vscale x 4 x double>, <vscale x 4 x double> } @deinterleave_nxptr_factor2(ptr %ptr) #2 {
+; CHECK-LABEL: define { <vscale x 4 x double>, <vscale x 4 x double> } @deinterleave_nxptr_factor2(
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr <vscale x 2 x double>, ptr [[PTR]], i64 0
+; CHECK-NEXT:    [[LDN1:%.*]] = call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld2.sret.nxv2f64(<vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), ptr [[TMP1]])
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[LDN1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> poison, <vscale x 2 x double> [[TMP2]], i64 0)
+; CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[LDN1]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> poison, <vscale x 2 x double> [[TMP4]], i64 0)
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr <vscale x 2 x double>, ptr [[PTR]], i64 2
+; CHECK-NEXT:    [[LDN2:%.*]] = call { <vscale x 2 x double>, <vscale x 2 x double> } @llvm.aarch64.sve.ld2.sret.nxv2f64(<vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i64 0), <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer), ptr [[TMP6]])
+; CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[LDN2]], 0
+; CHECK-NEXT:    [[TMP8:%.*]] = call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> [[TMP3]], <vscale x 2 x double> [[TMP7]], i64 2)
+; CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x double>, <vscale x 2 x double> } [[LDN2]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = call <vscale x 4 x double> @llvm.vector.insert.nxv4f64.nxv2f64(<vscale x 4 x double> [[TMP5]], <vscale x 2 x double> [[TMP9]], i64 2)
+; CHECK-NEXT:    [[TMP11:%.*]] = insertvalue { <vscale x 4 x double>, <vscale x 4 x double> } poison, <vscale x 4 x double> [[TMP8]], 0
+; CHECK-NEXT:    [[TMP12:%.*]] = insertvalue { <vscale x 4 x double>, <vscale x 4 x double> } [[TMP11]], <vscale x 4 x double> [[TMP10]], 1
+; CHECK-NEXT:    ret { <vscale x 4 x double>, <vscale x 4 x double> } [[TMP12]]
+;
+  %wide.vec = load <vscale x 8 x double>, ptr %ptr, align 8
+  %ldN = tail call { <vscale x 4 x double>, <vscale x 4 x double> } @llvm.experimental.vector.deinterleave2.nxv8f64(<vscale x 8 x double> %wide.vec)
+  ret { <vscale x 4 x double>, <vscale x 4 x double> } %ldN
+}
+
 attributes #0 = { vscale_range(2,2) "target-features"="+sve" }
 attributes #1 = { vscale_range(2,4) "target-features"="+sve" }
 attributes #2 = { vscale_range(4,4) "target-features"="+sve" }
-- 
GitLab


From d23a85066b5d89a47fde7b6777e0fc5a711b99d4 Mon Sep 17 00:00:00 2001
From: Matthias Braun <matze@braunis.de>
Date: Mon, 15 Apr 2024 18:19:06 -0700
Subject: [PATCH 320/464] InstCombine: Increase threadlocal.address alignment
 if pointee is more aligned (#88435)

Increase alignment of `llvm.threadlocal.address` if the pointed to
global has higher alignment.
---
 .../InstCombine/InstCombineCalls.cpp          |  9 ++++
 .../InstCombine/threadlocal_address.ll        | 41 +++++++++++++++++++
 2 files changed, 50 insertions(+)
 create mode 100644 llvm/test/Transforms/InstCombine/threadlocal_address.ll

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 20f51c8af617..bae8579fc365 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -3408,6 +3408,15 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
       return I;
     break;
   }
+  case Intrinsic::threadlocal_address: {
+    Align MinAlign = getKnownAlignment(II->getArgOperand(0), DL, II, &AC, &DT);
+    MaybeAlign Align = II->getRetAlign();
+    if (MinAlign > Align.valueOrOne()) {
+      II->addRetAttr(Attribute::getWithAlignment(II->getContext(), MinAlign));
+      return II;
+    }
+    break;
+  }
   default: {
     // Handle target specific intrinsics
     std::optional<Instruction *> V = targetInstCombineIntrinsic(*II);
diff --git a/llvm/test/Transforms/InstCombine/threadlocal_address.ll b/llvm/test/Transforms/InstCombine/threadlocal_address.ll
new file mode 100644
index 000000000000..0c220d996839
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/threadlocal_address.ll
@@ -0,0 +1,41 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -o - -S %s -passes=instcombine | FileCheck %s
+
+@tlsvar_a4 = thread_local global i32 4, align 4
+
+define void @func_increase_alignment() {
+; CHECK-LABEL: define void @func_increase_alignment() {
+; CHECK-NEXT:    [[P:%.*]] = call align 4 ptr @llvm.threadlocal.address.p0(ptr @tlsvar_a4)
+; CHECK-NEXT:    store i32 42, ptr [[P]], align 2
+; CHECK-NEXT:    ret void
+;
+  %p = call align 2 ptr @llvm.threadlocal.address(ptr @tlsvar_a4)
+  store i32 42, ptr %p, align 2
+  ret void
+}
+
+@tlsvar_a32 = thread_local global i32 5, align 32
+
+define i1 @func_add_alignment() {
+; CHECK-LABEL: define i1 @func_add_alignment() {
+; CHECK-NEXT:    ret i1 true
+;
+  %p = call ptr @llvm.threadlocal.address(ptr @tlsvar_a32)
+  %p_int = ptrtoint ptr %p to i32
+  %lowbits = and i32 %p_int, 31
+  %zero = icmp eq i32 %lowbits, 0
+  ret i1 %zero
+}
+
+@tlsvar_a1 = thread_local global i8 6, align 1
+
+define i1 @func_dont_reduce_alignment() {
+; CHECK-LABEL: define i1 @func_dont_reduce_alignment() {
+; CHECK-NEXT:    ret i1 true
+;
+  %p = call align 4 ptr @llvm.threadlocal.address(ptr @tlsvar_a1)
+  %p_int = ptrtoint ptr %p to i32
+  %lowbits = and i32 %p_int, 3
+  %zero = icmp eq i32 %lowbits, 0
+  ret i1 %zero
+}
-- 
GitLab


From 2e26ee9dce32d12ffa9bfb7f7d3e97778f0b7a75 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Mon, 15 Apr 2024 18:22:15 -0700
Subject: [PATCH 321/464] [DWARF] Clarify a variable name. NFC (#88814)

The parameter of `findDebugNamesOffsets` has been renamed to
`EndOfHeaderOffset` in #88064 to make it clear it is a section offset
instead of an offset relative to the current name index. Rename the call
site variable as well.
---
 llvm/lib/DebugInfo/DWARF/DWARFAcceleratorTable.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/DebugInfo/DWARF/DWARFAcceleratorTable.cpp b/llvm/lib/DebugInfo/DWARF/DWARFAcceleratorTable.cpp
index 22c9e8cd143c..ac19ac793297 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFAcceleratorTable.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFAcceleratorTable.cpp
@@ -572,12 +572,12 @@ dwarf::findDebugNamesOffsets(uint64_t EndOfHeaderOffset,
 
 Error DWARFDebugNames::NameIndex::extract() {
   const DWARFDataExtractor &AS = Section.AccelSection;
-  uint64_t hdrSize = Base;
-  if (Error E = Hdr.extract(AS, &hdrSize))
+  uint64_t EndOfHeaderOffset = Base;
+  if (Error E = Hdr.extract(AS, &EndOfHeaderOffset))
     return E;
 
   const unsigned SectionOffsetSize = dwarf::getDwarfOffsetByteSize(Hdr.Format);
-  Offsets = dwarf::findDebugNamesOffsets(hdrSize, Hdr);
+  Offsets = dwarf::findDebugNamesOffsets(EndOfHeaderOffset, Hdr);
 
   uint64_t Offset =
       Offsets.EntryOffsetsBase + (Hdr.NameCount * SectionOffsetSize);
-- 
GitLab


From f3a8112d9839a166f7eb240c6c72d7ecd47d3560 Mon Sep 17 00:00:00 2001
From: Shih-Po Hung <shihpo.hung@sifive.com>
Date: Tue, 16 Apr 2024 09:37:32 +0800
Subject: [PATCH 322/464] [RISCV][TTI] Scale the cost of ICmp with LMUL
 (#88235)

Use the Val type to estimate the instruction cost for ICmp.
---
 .../Target/RISCV/RISCVTargetTransformInfo.cpp |  17 +-
 llvm/test/Analysis/CostModel/RISCV/rvv-cmp.ll | 320 +++++++++---------
 .../LoopVectorize/RISCV/illegal-type.ll       |  13 +-
 3 files changed, 174 insertions(+), 176 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index bc9756c5e6dd..56f5bd8794ae 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -1335,8 +1335,8 @@ InstructionCost RISCVTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
     return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
                                      I);
 
+  std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(ValTy);
   if (Opcode == Instruction::Select && ValTy->isVectorTy()) {
-    std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(ValTy);
     if (CondTy->isVectorTy()) {
       if (ValTy->getScalarSizeInBits() == 1) {
         // vmandn.mm v8, v8, v9
@@ -1375,14 +1375,15 @@ InstructionCost RISCVTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
                           LT.second, CostKind);
   }
 
-  if ((Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) &&
-      ValTy->isVectorTy()) {
-    std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(ValTy);
-
-    // Support natively.
-    if (CmpInst::isIntPredicate(VecPred))
-      return LT.first * 1;
+  if ((Opcode == Instruction::ICmp) && ValTy->isVectorTy() &&
+      CmpInst::isIntPredicate(VecPred)) {
+    // Use VMSLT_VV to represent VMSEQ, VMSNE, VMSLTU, VMSLEU, VMSLT, VMSLE
+    // provided they incur the same cost across all implementations
+    return LT.first *
+           getRISCVInstructionCost(RISCV::VMSLT_VV, LT.second, CostKind);
+  }
 
+  if ((Opcode == Instruction::FCmp) && ValTy->isVectorTy()) {
     // If we do not support the input floating point vector type, use the base
     // one which will calculate as:
     // ScalarizeCost + Num * Cost for fixed vector,
diff --git a/llvm/test/Analysis/CostModel/RISCV/rvv-cmp.ll b/llvm/test/Analysis/CostModel/RISCV/rvv-cmp.ll
index 27d24faf0a8d..caa6d6f483a2 100644
--- a/llvm/test/Analysis/CostModel/RISCV/rvv-cmp.ll
+++ b/llvm/test/Analysis/CostModel/RISCV/rvv-cmp.ll
@@ -9,38 +9,38 @@ define void @icmp_eq() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8 = icmp eq <4 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8 = icmp eq <8 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8 = icmp eq <16 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8 = icmp eq <32 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i8 = icmp eq <32 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8 = icmp eq <vscale x 1 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8 = icmp eq <vscale x 2 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8 = icmp eq <vscale x 4 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8 = icmp eq <vscale x 8 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8 = icmp eq <vscale x 16 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv32i8 = icmp eq <vscale x 32 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i8 = icmp eq <vscale x 16 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i8 = icmp eq <vscale x 32 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i16 = icmp eq <2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16 = icmp eq <4 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16 = icmp eq <8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16 = icmp eq <16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16 = icmp eq <16 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i16 = icmp eq <vscale x 1 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16 = icmp eq <vscale x 2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16 = icmp eq <vscale x 4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16 = icmp eq <vscale x 8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i16 = icmp eq <vscale x 16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i16 = icmp eq <vscale x 8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i16 = icmp eq <vscale x 16 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32 = icmp eq <2 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32 = icmp eq <4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32 = icmp eq <8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32 = icmp eq <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32 = icmp eq <8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32 = icmp eq <16 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i32 = icmp eq <vscale x 1 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i32 = icmp eq <vscale x 2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i32 = icmp eq <vscale x 4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i32 = icmp eq <vscale x 8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i32 = icmp eq <vscale x 16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i32 = icmp eq <vscale x 4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i32 = icmp eq <vscale x 8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i32 = icmp eq <vscale x 16 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64 = icmp eq <2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64 = icmp eq <4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64 = icmp eq <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64 = icmp eq <4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64 = icmp eq <8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i64 = icmp eq <vscale x 1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i64 = icmp eq <vscale x 2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i64 = icmp eq <vscale x 4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i64 = icmp eq <vscale x 8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i64 = icmp eq <vscale x 2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i64 = icmp eq <vscale x 4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i64 = icmp eq <vscale x 8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %v2i8 = icmp eq <2 x i8> undef, undef
@@ -96,38 +96,38 @@ define void @icmp_ne() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8 = icmp ne <4 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8 = icmp ne <8 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8 = icmp ne <16 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8 = icmp ne <32 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i8 = icmp ne <32 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8 = icmp ne <vscale x 1 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8 = icmp ne <vscale x 2 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8 = icmp ne <vscale x 4 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8 = icmp ne <vscale x 8 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8 = icmp ne <vscale x 16 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv32i8 = icmp ne <vscale x 32 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i8 = icmp ne <vscale x 16 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i8 = icmp ne <vscale x 32 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i16 = icmp ne <2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16 = icmp ne <4 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16 = icmp ne <8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16 = icmp ne <16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16 = icmp ne <16 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i16 = icmp ne <vscale x 1 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16 = icmp ne <vscale x 2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16 = icmp ne <vscale x 4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16 = icmp ne <vscale x 8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i16 = icmp ne <vscale x 16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i16 = icmp ne <vscale x 8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i16 = icmp ne <vscale x 16 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32 = icmp ne <2 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32 = icmp ne <4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32 = icmp ne <8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32 = icmp ne <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32 = icmp ne <8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32 = icmp ne <16 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i32 = icmp ne <vscale x 1 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i32 = icmp ne <vscale x 2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i32 = icmp ne <vscale x 4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i32 = icmp ne <vscale x 8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i32 = icmp ne <vscale x 16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i32 = icmp ne <vscale x 4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i32 = icmp ne <vscale x 8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i32 = icmp ne <vscale x 16 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64 = icmp ne <2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64 = icmp ne <4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64 = icmp ne <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64 = icmp ne <4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64 = icmp ne <8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i64 = icmp ne <vscale x 1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i64 = icmp ne <vscale x 2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i64 = icmp ne <vscale x 4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i64 = icmp ne <vscale x 8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i64 = icmp ne <vscale x 2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i64 = icmp ne <vscale x 4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i64 = icmp ne <vscale x 8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %v2i8 = icmp ne <2 x i8> undef, undef
@@ -183,38 +183,38 @@ define void @icmp_ugt() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8 = icmp ugt <4 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8 = icmp ugt <8 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8 = icmp ugt <16 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8 = icmp ugt <32 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i8 = icmp ugt <32 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8 = icmp ugt <vscale x 1 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8 = icmp ugt <vscale x 2 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8 = icmp ugt <vscale x 4 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8 = icmp ugt <vscale x 8 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8 = icmp ugt <vscale x 16 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv32i8 = icmp ugt <vscale x 32 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i8 = icmp ugt <vscale x 16 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i8 = icmp ugt <vscale x 32 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i16 = icmp ugt <2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16 = icmp ugt <4 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16 = icmp ugt <8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16 = icmp ugt <16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16 = icmp ugt <16 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i16 = icmp ugt <vscale x 1 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16 = icmp ugt <vscale x 2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16 = icmp ugt <vscale x 4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16 = icmp ugt <vscale x 8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i16 = icmp ugt <vscale x 16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i16 = icmp ugt <vscale x 8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i16 = icmp ugt <vscale x 16 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32 = icmp ugt <2 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32 = icmp ugt <4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32 = icmp ugt <8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32 = icmp ugt <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32 = icmp ugt <8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32 = icmp ugt <16 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i32 = icmp ugt <vscale x 1 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i32 = icmp ugt <vscale x 2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i32 = icmp ugt <vscale x 4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i32 = icmp ugt <vscale x 8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i32 = icmp ugt <vscale x 16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i32 = icmp ugt <vscale x 4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i32 = icmp ugt <vscale x 8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i32 = icmp ugt <vscale x 16 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64 = icmp ugt <2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64 = icmp ugt <4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64 = icmp ugt <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64 = icmp ugt <4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64 = icmp ugt <8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i64 = icmp ugt <vscale x 1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i64 = icmp ugt <vscale x 2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i64 = icmp ugt <vscale x 4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i64 = icmp ugt <vscale x 8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i64 = icmp ugt <vscale x 2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i64 = icmp ugt <vscale x 4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i64 = icmp ugt <vscale x 8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %v2i8 = icmp ugt <2 x i8> undef, undef
@@ -270,38 +270,38 @@ define void @icmp_uge() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8 = icmp uge <4 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8 = icmp uge <8 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8 = icmp uge <16 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8 = icmp uge <32 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i8 = icmp uge <32 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8 = icmp uge <vscale x 1 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8 = icmp uge <vscale x 2 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8 = icmp uge <vscale x 4 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8 = icmp uge <vscale x 8 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8 = icmp uge <vscale x 16 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv32i8 = icmp uge <vscale x 32 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i8 = icmp uge <vscale x 16 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i8 = icmp uge <vscale x 32 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i16 = icmp uge <2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16 = icmp uge <4 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16 = icmp uge <8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16 = icmp uge <16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16 = icmp uge <16 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i16 = icmp uge <vscale x 1 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16 = icmp uge <vscale x 2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16 = icmp uge <vscale x 4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16 = icmp uge <vscale x 8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i16 = icmp uge <vscale x 16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i16 = icmp uge <vscale x 8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i16 = icmp uge <vscale x 16 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32 = icmp uge <2 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32 = icmp uge <4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32 = icmp uge <8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32 = icmp uge <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32 = icmp uge <8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32 = icmp uge <16 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i32 = icmp uge <vscale x 1 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i32 = icmp uge <vscale x 2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i32 = icmp uge <vscale x 4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i32 = icmp uge <vscale x 8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i32 = icmp uge <vscale x 16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i32 = icmp uge <vscale x 4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i32 = icmp uge <vscale x 8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i32 = icmp uge <vscale x 16 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64 = icmp uge <2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64 = icmp uge <4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64 = icmp uge <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64 = icmp uge <4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64 = icmp uge <8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i64 = icmp uge <vscale x 1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i64 = icmp uge <vscale x 2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i64 = icmp uge <vscale x 4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i64 = icmp uge <vscale x 8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i64 = icmp uge <vscale x 2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i64 = icmp uge <vscale x 4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i64 = icmp uge <vscale x 8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %v2i8 = icmp uge <2 x i8> undef, undef
@@ -357,38 +357,38 @@ define void @icmp_ult() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8 = icmp ult <4 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8 = icmp ult <8 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8 = icmp ult <16 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8 = icmp ult <32 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i8 = icmp ult <32 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8 = icmp ult <vscale x 1 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8 = icmp ult <vscale x 2 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8 = icmp ult <vscale x 4 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8 = icmp ult <vscale x 8 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8 = icmp ult <vscale x 16 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv32i8 = icmp ult <vscale x 32 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i8 = icmp ult <vscale x 16 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i8 = icmp ult <vscale x 32 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i16 = icmp ult <2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16 = icmp ult <4 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16 = icmp ult <8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16 = icmp ult <16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16 = icmp ult <16 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i16 = icmp ult <vscale x 1 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16 = icmp ult <vscale x 2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16 = icmp ult <vscale x 4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16 = icmp ult <vscale x 8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i16 = icmp ult <vscale x 16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i16 = icmp ult <vscale x 8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i16 = icmp ult <vscale x 16 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32 = icmp ult <2 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32 = icmp ult <4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32 = icmp ult <8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32 = icmp ult <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32 = icmp ult <8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32 = icmp ult <16 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i32 = icmp ult <vscale x 1 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i32 = icmp ult <vscale x 2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i32 = icmp ult <vscale x 4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i32 = icmp ult <vscale x 8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i32 = icmp ult <vscale x 16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i32 = icmp ult <vscale x 4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i32 = icmp ult <vscale x 8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i32 = icmp ult <vscale x 16 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64 = icmp ult <2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64 = icmp ult <4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64 = icmp ult <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64 = icmp ult <4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64 = icmp ult <8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i64 = icmp ult <vscale x 1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i64 = icmp ult <vscale x 2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i64 = icmp ult <vscale x 4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i64 = icmp ult <vscale x 8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i64 = icmp ult <vscale x 2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i64 = icmp ult <vscale x 4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i64 = icmp ult <vscale x 8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %v2i8 = icmp ult <2 x i8> undef, undef
@@ -444,38 +444,38 @@ define void @icmp_ule() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8 = icmp ule <4 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8 = icmp ule <8 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8 = icmp ule <16 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8 = icmp ule <32 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i8 = icmp ule <32 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8 = icmp ule <vscale x 1 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8 = icmp ule <vscale x 2 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8 = icmp ule <vscale x 4 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8 = icmp ule <vscale x 8 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8 = icmp ule <vscale x 16 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv32i8 = icmp ule <vscale x 32 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i8 = icmp ule <vscale x 16 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i8 = icmp ule <vscale x 32 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i16 = icmp ule <2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16 = icmp ule <4 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16 = icmp ule <8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16 = icmp ule <16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16 = icmp ule <16 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i16 = icmp ule <vscale x 1 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16 = icmp ule <vscale x 2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16 = icmp ule <vscale x 4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16 = icmp ule <vscale x 8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i16 = icmp ule <vscale x 16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i16 = icmp ule <vscale x 8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i16 = icmp ule <vscale x 16 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32 = icmp ule <2 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32 = icmp ule <4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32 = icmp ule <8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32 = icmp ule <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32 = icmp ule <8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32 = icmp ule <16 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i32 = icmp ule <vscale x 1 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i32 = icmp ule <vscale x 2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i32 = icmp ule <vscale x 4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i32 = icmp ule <vscale x 8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i32 = icmp ule <vscale x 16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i32 = icmp ule <vscale x 4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i32 = icmp ule <vscale x 8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i32 = icmp ule <vscale x 16 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64 = icmp ule <2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64 = icmp ule <4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64 = icmp ule <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64 = icmp ule <4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64 = icmp ule <8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i64 = icmp ule <vscale x 1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i64 = icmp ule <vscale x 2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i64 = icmp ule <vscale x 4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i64 = icmp ule <vscale x 8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i64 = icmp ule <vscale x 2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i64 = icmp ule <vscale x 4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i64 = icmp ule <vscale x 8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %v2i8 = icmp ule <2 x i8> undef, undef
@@ -531,38 +531,38 @@ define void @icmp_sgt() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8 = icmp sgt <4 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8 = icmp sgt <8 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8 = icmp sgt <16 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8 = icmp sgt <32 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i8 = icmp sgt <32 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8 = icmp sgt <vscale x 1 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8 = icmp sgt <vscale x 2 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8 = icmp sgt <vscale x 4 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8 = icmp sgt <vscale x 8 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8 = icmp sgt <vscale x 16 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv32i8 = icmp sgt <vscale x 32 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i8 = icmp sgt <vscale x 16 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i8 = icmp sgt <vscale x 32 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i16 = icmp sgt <2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16 = icmp sgt <4 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16 = icmp sgt <8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16 = icmp sgt <16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16 = icmp sgt <16 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i16 = icmp sgt <vscale x 1 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16 = icmp sgt <vscale x 2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16 = icmp sgt <vscale x 4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16 = icmp sgt <vscale x 8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i16 = icmp sgt <vscale x 16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i16 = icmp sgt <vscale x 8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i16 = icmp sgt <vscale x 16 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32 = icmp sgt <2 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32 = icmp sgt <4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32 = icmp sgt <8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32 = icmp sgt <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32 = icmp sgt <8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32 = icmp sgt <16 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i32 = icmp sgt <vscale x 1 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i32 = icmp sgt <vscale x 2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i32 = icmp sgt <vscale x 4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i32 = icmp sgt <vscale x 8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i32 = icmp sgt <vscale x 16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i32 = icmp sgt <vscale x 4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i32 = icmp sgt <vscale x 8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i32 = icmp sgt <vscale x 16 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64 = icmp sgt <2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64 = icmp sgt <4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64 = icmp sgt <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64 = icmp sgt <4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64 = icmp sgt <8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i64 = icmp sgt <vscale x 1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i64 = icmp sgt <vscale x 2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i64 = icmp sgt <vscale x 4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i64 = icmp sgt <vscale x 8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i64 = icmp sgt <vscale x 2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i64 = icmp sgt <vscale x 4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i64 = icmp sgt <vscale x 8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %v2i8 = icmp sgt <2 x i8> undef, undef
@@ -618,38 +618,38 @@ define void @icmp_sge() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8 = icmp sge <4 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8 = icmp sge <8 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8 = icmp sge <16 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8 = icmp sge <32 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i8 = icmp sge <32 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8 = icmp sge <vscale x 1 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8 = icmp sge <vscale x 2 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8 = icmp sge <vscale x 4 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8 = icmp sge <vscale x 8 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8 = icmp sge <vscale x 16 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv32i8 = icmp sge <vscale x 32 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i8 = icmp sge <vscale x 16 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i8 = icmp sge <vscale x 32 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i16 = icmp sge <2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16 = icmp sge <4 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16 = icmp sge <8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16 = icmp sge <16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16 = icmp sge <16 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i16 = icmp sge <vscale x 1 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16 = icmp sge <vscale x 2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16 = icmp sge <vscale x 4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16 = icmp sge <vscale x 8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i16 = icmp sge <vscale x 16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i16 = icmp sge <vscale x 8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i16 = icmp sge <vscale x 16 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32 = icmp sge <2 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32 = icmp sge <4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32 = icmp sge <8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32 = icmp sge <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32 = icmp sge <8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32 = icmp sge <16 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i32 = icmp sge <vscale x 1 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i32 = icmp sge <vscale x 2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i32 = icmp sge <vscale x 4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i32 = icmp sge <vscale x 8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i32 = icmp sge <vscale x 16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i32 = icmp sge <vscale x 4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i32 = icmp sge <vscale x 8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i32 = icmp sge <vscale x 16 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64 = icmp sge <2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64 = icmp sge <4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64 = icmp sge <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64 = icmp sge <4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64 = icmp sge <8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i64 = icmp sge <vscale x 1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i64 = icmp sge <vscale x 2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i64 = icmp sge <vscale x 4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i64 = icmp sge <vscale x 8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i64 = icmp sge <vscale x 2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i64 = icmp sge <vscale x 4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i64 = icmp sge <vscale x 8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %v2i8 = icmp sge <2 x i8> undef, undef
@@ -705,38 +705,38 @@ define void @icmp_slt() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8 = icmp slt <4 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8 = icmp slt <8 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8 = icmp slt <16 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8 = icmp slt <32 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i8 = icmp slt <32 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8 = icmp slt <vscale x 1 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8 = icmp slt <vscale x 2 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8 = icmp slt <vscale x 4 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8 = icmp slt <vscale x 8 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8 = icmp slt <vscale x 16 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv32i8 = icmp slt <vscale x 32 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i8 = icmp slt <vscale x 16 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i8 = icmp slt <vscale x 32 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i16 = icmp slt <2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16 = icmp slt <4 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16 = icmp slt <8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16 = icmp slt <16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16 = icmp slt <16 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i16 = icmp slt <vscale x 1 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16 = icmp slt <vscale x 2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16 = icmp slt <vscale x 4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16 = icmp slt <vscale x 8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i16 = icmp slt <vscale x 16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i16 = icmp slt <vscale x 8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i16 = icmp slt <vscale x 16 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32 = icmp slt <2 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32 = icmp slt <4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32 = icmp slt <8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32 = icmp slt <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32 = icmp slt <8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32 = icmp slt <16 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i32 = icmp slt <vscale x 1 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i32 = icmp slt <vscale x 2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i32 = icmp slt <vscale x 4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i32 = icmp slt <vscale x 8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i32 = icmp slt <vscale x 16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i32 = icmp slt <vscale x 4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i32 = icmp slt <vscale x 8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i32 = icmp slt <vscale x 16 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64 = icmp slt <2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64 = icmp slt <4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64 = icmp slt <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64 = icmp slt <4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64 = icmp slt <8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i64 = icmp slt <vscale x 1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i64 = icmp slt <vscale x 2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i64 = icmp slt <vscale x 4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i64 = icmp slt <vscale x 8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i64 = icmp slt <vscale x 2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i64 = icmp slt <vscale x 4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i64 = icmp slt <vscale x 8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %v2i8 = icmp slt <2 x i8> undef, undef
@@ -792,38 +792,38 @@ define void @icmp_sle() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i8 = icmp sle <4 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i8 = icmp sle <8 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i8 = icmp sle <16 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v32i8 = icmp sle <32 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v32i8 = icmp sle <32 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i8 = icmp sle <vscale x 1 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i8 = icmp sle <vscale x 2 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i8 = icmp sle <vscale x 4 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i8 = icmp sle <vscale x 8 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i8 = icmp sle <vscale x 16 x i8> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv32i8 = icmp sle <vscale x 32 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv16i8 = icmp sle <vscale x 16 x i8> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv32i8 = icmp sle <vscale x 32 x i8> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i16 = icmp sle <2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i16 = icmp sle <4 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i16 = icmp sle <8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i16 = icmp sle <16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v16i16 = icmp sle <16 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i16 = icmp sle <vscale x 1 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i16 = icmp sle <vscale x 2 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i16 = icmp sle <vscale x 4 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i16 = icmp sle <vscale x 8 x i16> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i16 = icmp sle <vscale x 16 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv8i16 = icmp sle <vscale x 8 x i16> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv16i16 = icmp sle <vscale x 16 x i16> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i32 = icmp sle <2 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i32 = icmp sle <4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i32 = icmp sle <8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16i32 = icmp sle <16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8i32 = icmp sle <8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v16i32 = icmp sle <16 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i32 = icmp sle <vscale x 1 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i32 = icmp sle <vscale x 2 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i32 = icmp sle <vscale x 4 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i32 = icmp sle <vscale x 8 x i32> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv16i32 = icmp sle <vscale x 16 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv4i32 = icmp sle <vscale x 4 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv8i32 = icmp sle <vscale x 8 x i32> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv16i32 = icmp sle <vscale x 16 x i32> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2i64 = icmp sle <2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4i64 = icmp sle <4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8i64 = icmp sle <8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4i64 = icmp sle <4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %v8i64 = icmp sle <8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv1i64 = icmp sle <vscale x 1 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv2i64 = icmp sle <vscale x 2 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv4i64 = icmp sle <vscale x 4 x i64> undef, undef
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %nxv8i64 = icmp sle <vscale x 8 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %nxv2i64 = icmp sle <vscale x 2 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %nxv4i64 = icmp sle <vscale x 4 x i64> undef, undef
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %nxv8i64 = icmp sle <vscale x 8 x i64> undef, undef
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %v2i8 = icmp sle <2 x i8> undef, undef
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/illegal-type.ll b/llvm/test/Transforms/LoopVectorize/RISCV/illegal-type.ll
index c49f7d4a5d5f..eeef8f199353 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/illegal-type.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/illegal-type.ll
@@ -102,10 +102,10 @@ define void @uniform_store_i1(ptr noalias %dst, ptr noalias %start, i64 %N) {
 ; CHECK-LABEL: @uniform_store_i1(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[N:%.*]], 1
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 64
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 32
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
-; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 64
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 32
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[N_VEC]], 8
 ; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[START:%.*]], i64 [[TMP1]]
@@ -116,15 +116,12 @@ define void @uniform_store_i1(ptr noalias %dst, ptr noalias %start, i64 %N) {
 ; CHECK-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[START]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <32 x i64> <i64 0, i64 8, i64 16, i64 24, i64 32, i64 40, i64 48, i64 56, i64 64, i64 72, i64 80, i64 88, i64 96, i64 104, i64 112, i64 120, i64 128, i64 136, i64 144, i64 152, i64 160, i64 168, i64 176, i64 184, i64 192, i64 200, i64 208, i64 216, i64 224, i64 232, i64 240, i64 248>
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <32 x i64> <i64 256, i64 264, i64 272, i64 280, i64 288, i64 296, i64 304, i64 312, i64 320, i64 328, i64 336, i64 344, i64 352, i64 360, i64 368, i64 376, i64 384, i64 392, i64 400, i64 408, i64 416, i64 424, i64 432, i64 440, i64 448, i64 456, i64 464, i64 472, i64 480, i64 488, i64 496, i64 504>
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, <32 x ptr> [[TMP2]], i64 1
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, <32 x ptr> [[TMP3]], i64 1
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq <32 x ptr> [[TMP4]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, <32 x ptr> [[TMP2]], i64 1
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq <32 x ptr> [[TMP5]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <32 x i1> [[TMP7]], i32 31
 ; CHECK-NEXT:    store i1 [[TMP8]], ptr [[DST:%.*]], align 1
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64
-; CHECK-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 512
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; CHECK-NEXT:    [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 256
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
 ; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
 ; CHECK:       middle.block:
-- 
GitLab


From dfe12b3fd193318403622a8ae51e0362c27502d1 Mon Sep 17 00:00:00 2001
From: Benji Smith <6193112+Benjins@users.noreply.github.com>
Date: Mon, 15 Apr 2024 21:37:55 -0400
Subject: [PATCH 323/464] [C API] Support uinc_wrap/udec_wrap in atomicrmw when
 accessing the bin op (#87163)

These previously were added in the C++ API in
778cf5431cafc243f81dd5c8cbd27701ff7f9120, but without updating the enum
in the C API or mapping functions.

Corresponding tests for all current atomicrmw bin ops have been added as
well.
---
 llvm/docs/ReleaseNotes.rst           |  3 ++
 llvm/include/llvm-c/Core.h           | 60 +++++++++++++++-------------
 llvm/lib/IR/Core.cpp                 |  8 ++++
 llvm/test/Bindings/llvm-c/atomics.ll | 25 ++++++++++++
 4 files changed, 68 insertions(+), 28 deletions(-)

diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst
index 11add9274a7a..76ef6ceb9407 100644
--- a/llvm/docs/ReleaseNotes.rst
+++ b/llvm/docs/ReleaseNotes.rst
@@ -149,6 +149,9 @@ Changes to the C API
 
 * Deprecated ``LLVMConstNUWNeg`` and ``LLVMBuildNUWNeg``.
 
+* Added ``LLVMAtomicRMWBinOpUIncWrap`` and ``LLVMAtomicRMWBinOpUDecWrap`` to
+  ``LLVMAtomicRMWBinOp`` enum for AtomicRMW instructions.
+
 Changes to the CodeGen infrastructure
 -------------------------------------
 
diff --git a/llvm/include/llvm-c/Core.h b/llvm/include/llvm-c/Core.h
index 6be5957ce610..0b03f3b36fcd 100644
--- a/llvm/include/llvm-c/Core.h
+++ b/llvm/include/llvm-c/Core.h
@@ -361,35 +361,39 @@ typedef enum {
 } LLVMAtomicOrdering;
 
 typedef enum {
-    LLVMAtomicRMWBinOpXchg, /**< Set the new value and return the one old */
-    LLVMAtomicRMWBinOpAdd, /**< Add a value and return the old one */
-    LLVMAtomicRMWBinOpSub, /**< Subtract a value and return the old one */
-    LLVMAtomicRMWBinOpAnd, /**< And a value and return the old one */
-    LLVMAtomicRMWBinOpNand, /**< Not-And a value and return the old one */
-    LLVMAtomicRMWBinOpOr, /**< OR a value and return the old one */
-    LLVMAtomicRMWBinOpXor, /**< Xor a value and return the old one */
-    LLVMAtomicRMWBinOpMax, /**< Sets the value if it's greater than the
-                             original using a signed comparison and return
-                             the old one */
-    LLVMAtomicRMWBinOpMin, /**< Sets the value if it's Smaller than the
-                             original using a signed comparison and return
-                             the old one */
-    LLVMAtomicRMWBinOpUMax, /**< Sets the value if it's greater than the
-                             original using an unsigned comparison and return
-                             the old one */
-    LLVMAtomicRMWBinOpUMin, /**< Sets the value if it's greater than the
-                              original using an unsigned comparison and return
-                              the old one */
-    LLVMAtomicRMWBinOpFAdd, /**< Add a floating point value and return the
-                              old one */
-    LLVMAtomicRMWBinOpFSub, /**< Subtract a floating point value and return the
+  LLVMAtomicRMWBinOpXchg, /**< Set the new value and return the one old */
+  LLVMAtomicRMWBinOpAdd,  /**< Add a value and return the old one */
+  LLVMAtomicRMWBinOpSub,  /**< Subtract a value and return the old one */
+  LLVMAtomicRMWBinOpAnd,  /**< And a value and return the old one */
+  LLVMAtomicRMWBinOpNand, /**< Not-And a value and return the old one */
+  LLVMAtomicRMWBinOpOr,   /**< OR a value and return the old one */
+  LLVMAtomicRMWBinOpXor,  /**< Xor a value and return the old one */
+  LLVMAtomicRMWBinOpMax,  /**< Sets the value if it's greater than the
+                            original using a signed comparison and return
+                            the old one */
+  LLVMAtomicRMWBinOpMin,  /**< Sets the value if it's Smaller than the
+                            original using a signed comparison and return
+                            the old one */
+  LLVMAtomicRMWBinOpUMax, /**< Sets the value if it's greater than the
+                           original using an unsigned comparison and return
+                           the old one */
+  LLVMAtomicRMWBinOpUMin, /**< Sets the value if it's greater than the
+                            original using an unsigned comparison and return
+                            the old one */
+  LLVMAtomicRMWBinOpFAdd, /**< Add a floating point value and return the
                             old one */
-    LLVMAtomicRMWBinOpFMax, /**< Sets the value if it's greater than the
-                             original using an floating point comparison and
-                             return the old one */
-    LLVMAtomicRMWBinOpFMin, /**< Sets the value if it's smaller than the
-                             original using an floating point comparison and
-                             return the old one */
+  LLVMAtomicRMWBinOpFSub, /**< Subtract a floating point value and return the
+                          old one */
+  LLVMAtomicRMWBinOpFMax, /**< Sets the value if it's greater than the
+                           original using an floating point comparison and
+                           return the old one */
+  LLVMAtomicRMWBinOpFMin, /**< Sets the value if it's smaller than the
+                           original using an floating point comparison and
+                           return the old one */
+  LLVMAtomicRMWBinOpUIncWrap, /**< Increments the value, wrapping back to zero
+                               when incremented above input value */
+  LLVMAtomicRMWBinOpUDecWrap, /**< Decrements the value, wrapping back to
+                               the input value when decremented below zero */
 } LLVMAtomicRMWBinOp;
 
 typedef enum {
diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp
index 8ce9c5ca63be..6aff94f39d9c 100644
--- a/llvm/lib/IR/Core.cpp
+++ b/llvm/lib/IR/Core.cpp
@@ -3769,6 +3769,10 @@ static AtomicRMWInst::BinOp mapFromLLVMRMWBinOp(LLVMAtomicRMWBinOp BinOp) {
     case LLVMAtomicRMWBinOpFSub: return AtomicRMWInst::FSub;
     case LLVMAtomicRMWBinOpFMax: return AtomicRMWInst::FMax;
     case LLVMAtomicRMWBinOpFMin: return AtomicRMWInst::FMin;
+    case LLVMAtomicRMWBinOpUIncWrap:
+      return AtomicRMWInst::UIncWrap;
+    case LLVMAtomicRMWBinOpUDecWrap:
+      return AtomicRMWInst::UDecWrap;
   }
 
   llvm_unreachable("Invalid LLVMAtomicRMWBinOp value!");
@@ -3791,6 +3795,10 @@ static LLVMAtomicRMWBinOp mapToLLVMRMWBinOp(AtomicRMWInst::BinOp BinOp) {
     case AtomicRMWInst::FSub: return LLVMAtomicRMWBinOpFSub;
     case AtomicRMWInst::FMax: return LLVMAtomicRMWBinOpFMax;
     case AtomicRMWInst::FMin: return LLVMAtomicRMWBinOpFMin;
+    case AtomicRMWInst::UIncWrap:
+      return LLVMAtomicRMWBinOpUIncWrap;
+    case AtomicRMWInst::UDecWrap:
+      return LLVMAtomicRMWBinOpUDecWrap;
     default: break;
   }
 
diff --git a/llvm/test/Bindings/llvm-c/atomics.ll b/llvm/test/Bindings/llvm-c/atomics.ll
index e64a29944ef9..162368c9d98d 100644
--- a/llvm/test/Bindings/llvm-c/atomics.ll
+++ b/llvm/test/Bindings/llvm-c/atomics.ll
@@ -36,6 +36,31 @@ define void @atomic_load_store(ptr %word) {
   ret void
 }
 
+define void @atomic_rmw_ops(ptr %p, i32 %i, float %f) {
+  ; Test all atomicrmw operations
+  %a.xchg      = atomicrmw xchg      ptr %p, i32 %i acq_rel, align 8
+  %a.add       = atomicrmw add       ptr %p, i32 %i acq_rel, align 8
+  %a.sub       = atomicrmw sub       ptr %p, i32 %i acq_rel, align 8
+  %a.and       = atomicrmw and       ptr %p, i32 %i acq_rel, align 8
+  %a.nand      = atomicrmw nand      ptr %p, i32 %i acq_rel, align 8
+  %a.or        = atomicrmw or        ptr %p, i32 %i acq_rel, align 8
+  %a.xor       = atomicrmw xor       ptr %p, i32 %i acq_rel, align 8
+  %a.max       = atomicrmw max       ptr %p, i32 %i acq_rel, align 8
+  %a.min       = atomicrmw min       ptr %p, i32 %i acq_rel, align 8
+  %a.umax      = atomicrmw umax      ptr %p, i32 %i acq_rel, align 8
+  %a.umin      = atomicrmw umin      ptr %p, i32 %i acq_rel, align 8
+
+  %a.fadd      = atomicrmw fadd      ptr %p, float %f acq_rel, align 8
+  %a.fsub      = atomicrmw fsub      ptr %p, float %f acq_rel, align 8
+  %a.fmax      = atomicrmw fmax      ptr %p, float %f acq_rel, align 8
+  %a.fmin      = atomicrmw fmin      ptr %p, float %f acq_rel, align 8
+
+  %a.uinc_wrap = atomicrmw uinc_wrap ptr %p, i32 %i acq_rel, align 8
+  %a.udec_wrap = atomicrmw udec_wrap ptr %p, i32 %i acq_rel, align 8
+
+  ret void
+}
+
 define i32 @main() {
   %1 = alloca i32, align 4
   %2 = cmpxchg ptr %1, i32 2, i32 3 seq_cst acquire
-- 
GitLab


From 5b9af38a03c8119cc2a42ae80d4a25e6f454c721 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 15 Apr 2024 18:38:54 -0700
Subject: [PATCH 324/464] [RISCV] Provide a more efficient lowering for
 experimental.cttz.elts. (#88552)

For experimental.cttz.elts, we can use a vfirst instruction, but we need
to correct the result if input vector can be 0. cttz.elts returns the
vector length while vfirst returns -1.
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   |  36 ++++++
 llvm/lib/Target/RISCV/RISCVISelLowering.h     |   2 +
 .../RISCV/intrinsic-cttz-elts-vscale.ll       | 114 ++++++++++++++----
 .../test/CodeGen/RISCV/intrinsic-cttz-elts.ll |  22 +---
 4 files changed, 132 insertions(+), 42 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 259cc388276c..f6ed6420c9e1 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -1484,6 +1484,11 @@ bool RISCVTargetLowering::shouldExpandGetVectorLength(EVT TripCountVT,
   return VF > MaxVF || !isPowerOf2_32(VF);
 }
 
+bool RISCVTargetLowering::shouldExpandCttzElements(EVT VT) const {
+  return !Subtarget.hasVInstructions() ||
+         VT.getVectorElementType() != MVT::i1 || !isTypeLegal(VT);
+}
+
 bool RISCVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
                                              const CallInst &I,
                                              MachineFunction &MF,
@@ -8718,6 +8723,29 @@ static SDValue lowerGetVectorLength(SDNode *N, SelectionDAG &DAG,
   return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), Res);
 }
 
+static SDValue lowerCttzElts(SDNode *N, SelectionDAG &DAG,
+                             const RISCVSubtarget &Subtarget) {
+  SDValue Op0 = N->getOperand(1);
+  MVT OpVT = Op0.getSimpleValueType();
+  MVT ContainerVT = OpVT;
+  if (OpVT.isFixedLengthVector()) {
+    ContainerVT = getContainerForFixedLengthVector(DAG, OpVT, Subtarget);
+    Op0 = convertToScalableVector(ContainerVT, Op0, DAG, Subtarget);
+  }
+  MVT XLenVT = Subtarget.getXLenVT();
+  SDLoc DL(N);
+  auto [Mask, VL] = getDefaultVLOps(OpVT, ContainerVT, DL, DAG, Subtarget);
+  SDValue Res = DAG.getNode(RISCVISD::VFIRST_VL, DL, XLenVT, Op0, Mask, VL);
+  if (isOneConstant(N->getOperand(2)))
+    return Res;
+
+  // Convert -1 to VL.
+  SDValue Setcc =
+      DAG.getSetCC(DL, XLenVT, Res, DAG.getConstant(0, DL, XLenVT), ISD::SETLT);
+  VL = DAG.getElementCount(DL, XLenVT, OpVT.getVectorElementCount());
+  return DAG.getSelect(DL, XLenVT, Setcc, VL, Res);
+}
+
 static inline void promoteVCIXScalar(const SDValue &Op,
                                      SmallVectorImpl<SDValue> &Operands,
                                      SelectionDAG &DAG) {
@@ -8913,6 +8941,8 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
   }
   case Intrinsic::experimental_get_vector_length:
     return lowerGetVectorLength(Op.getNode(), DAG, Subtarget);
+  case Intrinsic::experimental_cttz_elts:
+    return lowerCttzElts(Op.getNode(), DAG, Subtarget);
   case Intrinsic::riscv_vmv_x_s: {
     SDValue Res = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Op.getOperand(1));
     return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Res);
@@ -12336,6 +12366,12 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
       Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
       return;
     }
+    case Intrinsic::experimental_cttz_elts: {
+      SDValue Res = lowerCttzElts(N, DAG, Subtarget);
+      Results.push_back(
+          DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), Res));
+      return;
+    }
     case Intrinsic::riscv_orc_b:
     case Intrinsic::riscv_brev8:
     case Intrinsic::riscv_sha256sig0:
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index ace5b3fd2b95..e2633733c31b 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -986,6 +986,8 @@ private:
   bool shouldExpandGetVectorLength(EVT TripCountVT, unsigned VF,
                                    bool IsScalable) const override;
 
+  bool shouldExpandCttzElements(EVT VT) const override;
+
   /// RVV code generation for fixed length vectors does not lower all
   /// BUILD_VECTORs. This makes BUILD_VECTOR legalisation a source of stores to
   /// merge. However, merging them creates a BUILD_VECTOR that is just as
diff --git a/llvm/test/CodeGen/RISCV/intrinsic-cttz-elts-vscale.ll b/llvm/test/CodeGen/RISCV/intrinsic-cttz-elts-vscale.ll
index 65d0768c6088..ea8feef33298 100644
--- a/llvm/test/CodeGen/RISCV/intrinsic-cttz-elts-vscale.ll
+++ b/llvm/test/CodeGen/RISCV/intrinsic-cttz-elts-vscale.ll
@@ -128,43 +128,113 @@ define i64 @ctz_nxv8i1_no_range(<vscale x 8 x i16> %a) {
 define i32 @ctz_nxv16i1(<vscale x 16 x i1> %pg, <vscale x 16 x i1> %a) {
 ; RV32-LABEL: ctz_nxv16i1:
 ; RV32:       # %bb.0:
-; RV32-NEXT:    vmv1r.v v0, v8
+; RV32-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; RV32-NEXT:    vfirst.m a0, v8
+; RV32-NEXT:    bgez a0, .LBB2_2
+; RV32-NEXT:  # %bb.1:
 ; RV32-NEXT:    csrr a0, vlenb
 ; RV32-NEXT:    slli a0, a0, 1
-; RV32-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
-; RV32-NEXT:    vmv.v.x v8, a0
-; RV32-NEXT:    vid.v v16
-; RV32-NEXT:    li a1, -1
-; RV32-NEXT:    vmadd.vx v16, a1, v8
-; RV32-NEXT:    vmv.v.i v8, 0
-; RV32-NEXT:    vmerge.vvm v8, v8, v16, v0
-; RV32-NEXT:    vredmaxu.vs v8, v8, v8
-; RV32-NEXT:    vmv.x.s a1, v8
-; RV32-NEXT:    sub a0, a0, a1
+; RV32-NEXT:  .LBB2_2:
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: ctz_nxv16i1:
 ; RV64:       # %bb.0:
-; RV64-NEXT:    vmv1r.v v0, v8
+; RV64-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; RV64-NEXT:    vfirst.m a0, v8
+; RV64-NEXT:    bgez a0, .LBB2_2
+; RV64-NEXT:  # %bb.1:
 ; RV64-NEXT:    csrr a0, vlenb
 ; RV64-NEXT:    slli a0, a0, 1
-; RV64-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
-; RV64-NEXT:    vmv.v.x v8, a0
-; RV64-NEXT:    vid.v v16
-; RV64-NEXT:    li a1, -1
-; RV64-NEXT:    vmadd.vx v16, a1, v8
-; RV64-NEXT:    vmv.v.i v8, 0
-; RV64-NEXT:    vmerge.vvm v8, v8, v16, v0
-; RV64-NEXT:    vredmaxu.vs v8, v8, v8
-; RV64-NEXT:    vmv.x.s a1, v8
-; RV64-NEXT:    subw a0, a0, a1
+; RV64-NEXT:  .LBB2_2:
 ; RV64-NEXT:    ret
   %res = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> %a, i1 0)
   ret i32 %res
 }
 
+define i32 @ctz_nxv16i1_poison(<vscale x 16 x i1> %pg, <vscale x 16 x i1> %a) {
+; RV32-LABEL: ctz_nxv16i1_poison:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; RV32-NEXT:    vfirst.m a0, v8
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: ctz_nxv16i1_poison:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
+; RV64-NEXT:    vfirst.m a0, v8
+; RV64-NEXT:    ret
+  %res = call i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1> %a, i1 1)
+  ret i32 %res
+}
+
+define i32 @ctz_v16i1(<16 x i1> %pg, <16 x i1> %a) {
+; RV32-LABEL: ctz_v16i1:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; RV32-NEXT:    vfirst.m a0, v8
+; RV32-NEXT:    bgez a0, .LBB4_2
+; RV32-NEXT:  # %bb.1:
+; RV32-NEXT:    li a0, 16
+; RV32-NEXT:  .LBB4_2:
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: ctz_v16i1:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; RV64-NEXT:    vfirst.m a0, v8
+; RV64-NEXT:    bgez a0, .LBB4_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    li a0, 16
+; RV64-NEXT:  .LBB4_2:
+; RV64-NEXT:    ret
+  %res = call i32 @llvm.experimental.cttz.elts.i32.v16i1(<16 x i1> %a, i1 0)
+  ret i32 %res
+}
+
+define i32 @ctz_v16i1_poison(<16 x i1> %pg, <16 x i1> %a) {
+; RV32-LABEL: ctz_v16i1_poison:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; RV32-NEXT:    vfirst.m a0, v8
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: ctz_v16i1_poison:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; RV64-NEXT:    vfirst.m a0, v8
+; RV64-NEXT:    ret
+  %res = call i32 @llvm.experimental.cttz.elts.i32.v16i1(<16 x i1> %a, i1 1)
+  ret i32 %res
+}
+
+define i16 @ctz_v8i1_i16_ret(<8 x i1> %a) {
+; RV32-LABEL: ctz_v8i1_i16_ret:
+; RV32:       # %bb.0:
+; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV32-NEXT:    vfirst.m a0, v0
+; RV32-NEXT:    bgez a0, .LBB6_2
+; RV32-NEXT:  # %bb.1:
+; RV32-NEXT:    li a0, 8
+; RV32-NEXT:  .LBB6_2:
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: ctz_v8i1_i16_ret:
+; RV64:       # %bb.0:
+; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; RV64-NEXT:    vfirst.m a0, v0
+; RV64-NEXT:    bgez a0, .LBB6_2
+; RV64-NEXT:  # %bb.1:
+; RV64-NEXT:    li a0, 8
+; RV64-NEXT:  .LBB6_2:
+; RV64-NEXT:    ret
+  %res = call i16 @llvm.experimental.cttz.elts.i16.v8i1(<8 x i1> %a, i1 0)
+  ret i16 %res
+}
+
 declare i64 @llvm.experimental.cttz.elts.i64.nxv8i16(<vscale x 8 x i16>, i1)
 declare i32 @llvm.experimental.cttz.elts.i32.nxv16i1(<vscale x 16 x i1>, i1)
 declare i32 @llvm.experimental.cttz.elts.i32.nxv4i32(<vscale x 4 x i32>, i1)
+declare i32 @llvm.experimental.cttz.elts.i32.v16i1(<16 x i1>, i1)
+declare i16 @llvm.experimental.cttz.elts.i16.v16i1(<8 x i1>, i1)
 
 attributes #0 = { vscale_range(2,1024) }
diff --git a/llvm/test/CodeGen/RISCV/intrinsic-cttz-elts.ll b/llvm/test/CodeGen/RISCV/intrinsic-cttz-elts.ll
index 49d4760a2e9a..94b717b42e92 100644
--- a/llvm/test/CodeGen/RISCV/intrinsic-cttz-elts.ll
+++ b/llvm/test/CodeGen/RISCV/intrinsic-cttz-elts.ll
@@ -48,31 +48,13 @@ define i32 @ctz_v2i1_poison(<2 x i1> %a) {
 ; RV32-LABEL: ctz_v2i1_poison:
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
-; RV32-NEXT:    vmv.v.i v8, 0
-; RV32-NEXT:    vmerge.vim v8, v8, -1, v0
-; RV32-NEXT:    vid.v v9
-; RV32-NEXT:    vrsub.vi v9, v9, 2
-; RV32-NEXT:    vand.vv v8, v8, v9
-; RV32-NEXT:    vredmaxu.vs v8, v8, v8
-; RV32-NEXT:    vmv.x.s a0, v8
-; RV32-NEXT:    li a1, 2
-; RV32-NEXT:    sub a1, a1, a0
-; RV32-NEXT:    andi a0, a1, 255
+; RV32-NEXT:    vfirst.m a0, v0
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: ctz_v2i1_poison:
 ; RV64:       # %bb.0:
 ; RV64-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
-; RV64-NEXT:    vmv.v.i v8, 0
-; RV64-NEXT:    vmerge.vim v8, v8, -1, v0
-; RV64-NEXT:    vid.v v9
-; RV64-NEXT:    vrsub.vi v9, v9, 2
-; RV64-NEXT:    vand.vv v8, v8, v9
-; RV64-NEXT:    vredmaxu.vs v8, v8, v8
-; RV64-NEXT:    vmv.x.s a0, v8
-; RV64-NEXT:    li a1, 2
-; RV64-NEXT:    subw a1, a1, a0
-; RV64-NEXT:    andi a0, a1, 255
+; RV64-NEXT:    vfirst.m a0, v0
 ; RV64-NEXT:    ret
   %res = call i32 @llvm.experimental.cttz.elts.i32.v2i1(<2 x i1> %a, i1 1)
   ret i32 %res
-- 
GitLab


From 4b22a923c4bfd0aa1d483149f84b6787263c2d76 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Tue, 16 Apr 2024 10:42:32 +0900
Subject: [PATCH 325/464] [IndVars] Mark truncs as nuw/nsw (#88686)

When inserting truncs during IV widening, mark the trunc as either nuw
or nsw depending on whether zext or sext widening was used. For
non-negative IVs both nuw and nsw apply.
---
 llvm/include/llvm/IR/IRBuilder.h              | 14 ++++++-
 llvm/lib/Transforms/Utils/SimplifyIndVar.cpp  | 40 ++++++++++---------
 .../IndVarSimplify/AArch64/widen-loop-comp.ll |  8 ++--
 .../Transforms/IndVarSimplify/X86/iv-widen.ll | 16 ++++----
 .../Transforms/IndVarSimplify/elim-extend.ll  |  2 +-
 ...wide-inc-for-narrow-use-recompute-flags.ll |  2 +-
 .../test/Transforms/IndVarSimplify/iv-sext.ll |  2 +-
 .../IndVarSimplify/iv-widen-elim-ext.ll       |  8 ++--
 llvm/test/Transforms/IndVarSimplify/lftr.ll   |  4 +-
 .../IndVarSimplify/no-iv-rewrite.ll           |  2 +-
 .../IndVarSimplify/post-inc-range.ll          |  2 +-
 .../test/Transforms/IndVarSimplify/pr25578.ll |  2 +-
 .../test/Transforms/IndVarSimplify/pr55925.ll |  8 ++--
 .../widen-nonnegative-countdown.ll            | 22 +++++-----
 .../IndVarSimplify/widen-nonnegative.ll       | 20 +++++-----
 llvm/test/Transforms/LoopFlatten/widen-iv3.ll |  2 +-
 16 files changed, 84 insertions(+), 70 deletions(-)

diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h
index f381273c46cf..b6534a1962a2 100644
--- a/llvm/include/llvm/IR/IRBuilder.h
+++ b/llvm/include/llvm/IR/IRBuilder.h
@@ -2004,8 +2004,18 @@ public:
   // Instruction creation methods: Cast/Conversion Operators
   //===--------------------------------------------------------------------===//
 
-  Value *CreateTrunc(Value *V, Type *DestTy, const Twine &Name = "") {
-    return CreateCast(Instruction::Trunc, V, DestTy, Name);
+  Value *CreateTrunc(Value *V, Type *DestTy, const Twine &Name = "",
+                     bool IsNUW = false, bool IsNSW = false) {
+    if (V->getType() == DestTy)
+      return V;
+    if (Value *Folded = Folder.FoldCast(Instruction::Trunc, V, DestTy))
+      return Folded;
+    Instruction *I = CastInst::Create(Instruction::Trunc, V, DestTy);
+    if (IsNUW)
+      I->setHasNoUnsignedWrap();
+    if (IsNSW)
+      I->setHasNoSignedWrap();
+    return Insert(I, Name);
   }
 
   Value *CreateZExt(Value *V, Type *DestTy, const Twine &Name = "",
diff --git a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
index 440fe0790d79..31be7d62c8d1 100644
--- a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp
@@ -1153,6 +1153,7 @@ protected:
 
   Instruction *widenIVUse(NarrowIVDefUse DU, SCEVExpander &Rewriter,
                           PHINode *OrigPhi, PHINode *WidePhi);
+  void truncateIVUse(NarrowIVDefUse DU);
 
   bool widenLoopCompare(NarrowIVDefUse DU);
   bool widenWithVariantUse(NarrowIVDefUse DU);
@@ -1569,15 +1570,18 @@ WidenIV::WidenedRecTy WidenIV::getWideRecurrence(WidenIV::NarrowIVDefUse DU) {
 
 /// This IV user cannot be widened. Replace this use of the original narrow IV
 /// with a truncation of the new wide IV to isolate and eliminate the narrow IV.
-static void truncateIVUse(WidenIV::NarrowIVDefUse DU, DominatorTree *DT,
-                          LoopInfo *LI) {
+void WidenIV::truncateIVUse(NarrowIVDefUse DU) {
   auto *InsertPt = getInsertPointForUses(DU.NarrowUse, DU.NarrowDef, DT, LI);
   if (!InsertPt)
     return;
   LLVM_DEBUG(dbgs() << "INDVARS: Truncate IV " << *DU.WideDef << " for user "
                     << *DU.NarrowUse << "\n");
+  ExtendKind ExtKind = getExtendKind(DU.NarrowDef);
   IRBuilder<> Builder(InsertPt);
-  Value *Trunc = Builder.CreateTrunc(DU.WideDef, DU.NarrowDef->getType());
+  Value *Trunc =
+      Builder.CreateTrunc(DU.WideDef, DU.NarrowDef->getType(), "",
+                          DU.NeverNegative || ExtKind == ExtendKind::Zero,
+                          DU.NeverNegative || ExtKind == ExtendKind::Sign);
   DU.NarrowUse->replaceUsesOfWith(DU.NarrowDef, Trunc);
 }
 
@@ -1826,6 +1830,13 @@ Instruction *WidenIV::widenIVUse(WidenIV::NarrowIVDefUse DU,
   assert(ExtendKindMap.count(DU.NarrowDef) &&
          "Should already know the kind of extension used to widen NarrowDef");
 
+  // This narrow use can be widened by a sext if it's non-negative or its narrow
+  // def was widened by a sext. Same for zext.
+  bool CanWidenBySExt =
+      DU.NeverNegative || getExtendKind(DU.NarrowDef) == ExtendKind::Sign;
+  bool CanWidenByZExt =
+      DU.NeverNegative || getExtendKind(DU.NarrowDef) == ExtendKind::Zero;
+
   // Stop traversing the def-use chain at inner-loop phis or post-loop phis.
   if (PHINode *UsePhi = dyn_cast<PHINode>(DU.NarrowUse)) {
     if (LI->getLoopFor(UsePhi->getParent()) != L) {
@@ -1833,7 +1844,7 @@ Instruction *WidenIV::widenIVUse(WidenIV::NarrowIVDefUse DU,
       // After SimplifyCFG most loop exit targets have a single predecessor.
       // Otherwise fall back to a truncate within the loop.
       if (UsePhi->getNumOperands() != 1)
-        truncateIVUse(DU, DT, LI);
+        truncateIVUse(DU);
       else {
         // Widening the PHI requires us to insert a trunc.  The logical place
         // for this trunc is in the same BB as the PHI.  This is not possible if
@@ -1847,7 +1858,8 @@ Instruction *WidenIV::widenIVUse(WidenIV::NarrowIVDefUse DU,
         WidePhi->addIncoming(DU.WideDef, UsePhi->getIncomingBlock(0));
         BasicBlock *WidePhiBB = WidePhi->getParent();
         IRBuilder<> Builder(WidePhiBB, WidePhiBB->getFirstInsertionPt());
-        Value *Trunc = Builder.CreateTrunc(WidePhi, DU.NarrowDef->getType());
+        Value *Trunc = Builder.CreateTrunc(WidePhi, DU.NarrowDef->getType(), "",
+                                           CanWidenByZExt, CanWidenBySExt);
         UsePhi->replaceAllUsesWith(Trunc);
         DeadInsts.emplace_back(UsePhi);
         LLVM_DEBUG(dbgs() << "INDVARS: Widen lcssa phi " << *UsePhi << " to "
@@ -1857,18 +1869,9 @@ Instruction *WidenIV::widenIVUse(WidenIV::NarrowIVDefUse DU,
     }
   }
 
-  // This narrow use can be widened by a sext if it's non-negative or its narrow
-  // def was widened by a sext. Same for zext.
-  auto canWidenBySExt = [&]() {
-    return DU.NeverNegative || getExtendKind(DU.NarrowDef) == ExtendKind::Sign;
-  };
-  auto canWidenByZExt = [&]() {
-    return DU.NeverNegative || getExtendKind(DU.NarrowDef) == ExtendKind::Zero;
-  };
-
   // Our raison d'etre! Eliminate sign and zero extension.
-  if ((match(DU.NarrowUse, m_SExtLike(m_Value())) && canWidenBySExt()) ||
-      (isa<ZExtInst>(DU.NarrowUse) && canWidenByZExt())) {
+  if ((match(DU.NarrowUse, m_SExtLike(m_Value())) && CanWidenBySExt) ||
+      (isa<ZExtInst>(DU.NarrowUse) && CanWidenByZExt)) {
     Value *NewDef = DU.WideDef;
     if (DU.NarrowUse->getType() != WideType) {
       unsigned CastWidth = SE->getTypeSizeInBits(DU.NarrowUse->getType());
@@ -1876,7 +1879,8 @@ Instruction *WidenIV::widenIVUse(WidenIV::NarrowIVDefUse DU,
       if (CastWidth < IVWidth) {
         // The cast isn't as wide as the IV, so insert a Trunc.
         IRBuilder<> Builder(DU.NarrowUse);
-        NewDef = Builder.CreateTrunc(DU.WideDef, DU.NarrowUse->getType());
+        NewDef = Builder.CreateTrunc(DU.WideDef, DU.NarrowUse->getType(), "",
+                                     CanWidenByZExt, CanWidenBySExt);
       }
       else {
         // A wider extend was hidden behind a narrower one. This may induce
@@ -1975,7 +1979,7 @@ Instruction *WidenIV::widenIVUse(WidenIV::NarrowIVDefUse DU,
   // This user does not evaluate to a recurrence after widening, so don't
   // follow it. Instead insert a Trunc to kill off the original use,
   // eventually isolating the original narrow IV so it can be removed.
-  truncateIVUse(DU, DT, LI);
+  truncateIVUse(DU);
   return nullptr;
 }
 
diff --git a/llvm/test/Transforms/IndVarSimplify/AArch64/widen-loop-comp.ll b/llvm/test/Transforms/IndVarSimplify/AArch64/widen-loop-comp.ll
index 6f659a88da2e..c5f656c870a2 100644
--- a/llvm/test/Transforms/IndVarSimplify/AArch64/widen-loop-comp.ll
+++ b/llvm/test/Transforms/IndVarSimplify/AArch64/widen-loop-comp.ll
@@ -41,7 +41,7 @@ define i32 @test1() {
 ; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[FOR_COND]]
 ; CHECK:       if.then:
 ; CHECK-NEXT:    [[I_05_LCSSA_WIDE:%.*]] = phi i64 [ [[INDVARS_IV]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[TMP5:%.*]] = trunc i64 [[I_05_LCSSA_WIDE]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc nuw nsw i64 [[I_05_LCSSA_WIDE]] to i32
 ; CHECK-NEXT:    store i32 [[TMP5]], ptr @idx, align 4
 ; CHECK-NEXT:    br label [[FOR_END:%.*]]
 ; CHECK:       for.cond.for.end.loopexit_crit_edge:
@@ -237,7 +237,7 @@ define i32 @test4(i32 %a) {
 ; CHECK-NEXT:    [[CONV3:%.*]] = trunc i32 [[OR]] to i8
 ; CHECK-NEXT:    [[CALL:%.*]] = call i32 @fn1(i8 signext [[CONV3]])
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nsw i32 [[INDVARS_IV]], -1
-; CHECK-NEXT:    [[TMP0:%.*]] = trunc i32 [[INDVARS_IV_NEXT]] to i8
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc nuw i32 [[INDVARS_IV_NEXT]] to i8
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[TMP0]], -14
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
 ; CHECK:       for.end:
@@ -466,7 +466,7 @@ define i32 @test9(ptr %a, i32 %b, i32 %init) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ADD]] = add nsw i32 [[SUM_0]], [[TMP1]]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc nuw i64 [[INDVARS_IV_NEXT]] to i32
 ; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 0, [[TMP2]]
 ; CHECK-NEXT:    br i1 [[CMP2]], label [[FOR_COND]], label [[FOR_END]]
 ; CHECK:       for.end:
@@ -997,7 +997,7 @@ define i32 @test16_unsigned_neg(i32 %start, ptr %p, ptr %q, i32 %x) {
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[BACKEDGE:%.*]] ], [ [[TMP0]], [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[INDVARS_IV]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc nuw i64 [[INDVARS_IV]] to i32
 ; CHECK-NEXT:    [[FOO:%.*]] = add i32 [[TMP1]], -1
 ; CHECK-NEXT:    br i1 [[COND]], label [[EXIT:%.*]], label [[GUARDED:%.*]]
 ; CHECK:       guarded:
diff --git a/llvm/test/Transforms/IndVarSimplify/X86/iv-widen.ll b/llvm/test/Transforms/IndVarSimplify/X86/iv-widen.ll
index d05755bea0dd..4e0c503794bf 100644
--- a/llvm/test/Transforms/IndVarSimplify/X86/iv-widen.ll
+++ b/llvm/test/Transforms/IndVarSimplify/X86/iv-widen.ll
@@ -23,7 +23,7 @@ define void @loop_0(ptr %a) {
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[B18_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[B24:%.*]] ]
 ; CHECK-NEXT:    call void @use(i64 [[INDVARS_IV]])
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK-NEXT:    [[TMP0:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc nuw nsw i64 [[INDVARS_IV]] to i32
 ; CHECK-NEXT:    [[O:%.*]] = getelementptr i32, ptr [[A:%.*]], i32 [[TMP0]]
 ; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[O]], align 4
 ; CHECK-NEXT:    [[T:%.*]] = icmp eq i32 [[V]], 0
@@ -37,7 +37,7 @@ define void @loop_0(ptr %a) {
 ; CHECK-NEXT:    ret void
 ; CHECK:       exit24:
 ; CHECK-NEXT:    [[DOT02_LCSSA_WIDE:%.*]] = phi i64 [ [[INDVARS_IV]], [[B18]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[DOT02_LCSSA_WIDE]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc nuw nsw i64 [[DOT02_LCSSA_WIDE]] to i32
 ; CHECK-NEXT:    call void @dummy(i32 [[TMP1]])
 ; CHECK-NEXT:    unreachable
 ;
@@ -159,7 +159,7 @@ declare void @dummy(i32)
 declare void @dummy.i64(i64)
 
 
-define void @loop_2(i32 %size, i32 %nsteps, i32 %hsize, ptr %lined, i8 %tmp1) {
+define void @loop_2(i32 %size, i32 %nsteps, i32 %hsize, ptr %lined, i8 %arg) {
 ; CHECK-LABEL: @loop_2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CMP215:%.*]] = icmp sgt i32 [[SIZE:%.*]], 1
@@ -180,12 +180,12 @@ define void @loop_2(i32 %size, i32 %nsteps, i32 %hsize, ptr %lined, i8 %tmp1) {
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 1, [[FOR_BODY2_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY2]] ]
 ; CHECK-NEXT:    [[TMP4:%.*]] = add nsw i64 [[TMP3]], [[INDVARS_IV]]
 ; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i8, ptr [[LINED:%.*]], i64 [[TMP4]]
-; CHECK-NEXT:    store i8 [[TMP1:%.*]], ptr [[ADD_PTR]], align 1
+; CHECK-NEXT:    store i8 [[ARG:%.*]], ptr [[ADD_PTR]], align 1
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT]], [[WIDE_TRIP_COUNT]]
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_BODY2]], label [[FOR_BODY3_PREHEADER:%.*]]
 ; CHECK:       for.body3.preheader:
-; CHECK-NEXT:    [[TMP5:%.*]] = trunc i64 [[TMP3]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc nsw i64 [[TMP3]] to i32
 ; CHECK-NEXT:    [[TMP6:%.*]] = zext i32 [[TMP5]] to i64
 ; CHECK-NEXT:    [[WIDE_TRIP_COUNT7:%.*]] = zext i32 [[SIZE]] to i64
 ; CHECK-NEXT:    br label [[FOR_BODY3:%.*]]
@@ -193,7 +193,7 @@ define void @loop_2(i32 %size, i32 %nsteps, i32 %hsize, ptr %lined, i8 %tmp1) {
 ; CHECK-NEXT:    [[INDVARS_IV3:%.*]] = phi i64 [ 1, [[FOR_BODY3_PREHEADER]] ], [ [[INDVARS_IV_NEXT4:%.*]], [[FOR_BODY3]] ]
 ; CHECK-NEXT:    [[TMP7:%.*]] = add nuw nsw i64 [[TMP6]], [[INDVARS_IV3]]
 ; CHECK-NEXT:    [[ADD_PTR2:%.*]] = getelementptr inbounds i8, ptr [[LINED]], i64 [[TMP7]]
-; CHECK-NEXT:    store i8 [[TMP1]], ptr [[ADD_PTR2]], align 1
+; CHECK-NEXT:    store i8 [[ARG]], ptr [[ADD_PTR2]], align 1
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT4]] = add nuw nsw i64 [[INDVARS_IV3]], 1
 ; CHECK-NEXT:    [[EXITCOND8:%.*]] = icmp ne i64 [[INDVARS_IV_NEXT4]], [[WIDE_TRIP_COUNT7]]
 ; CHECK-NEXT:    br i1 [[EXITCOND8]], label [[FOR_BODY3]], label [[FOR_INC_LOOPEXIT:%.*]]
@@ -222,7 +222,7 @@ for.body2:
   %add4 = add nsw i32 %add, %k
   %idx.ext = sext i32 %add4 to i64
   %add.ptr = getelementptr inbounds i8, ptr %lined, i64 %idx.ext
-  store i8 %tmp1, ptr %add.ptr, align 1
+  store i8 %arg, ptr %add.ptr, align 1
   %inc = add nsw i32 %k, 1
   %cmp2 = icmp slt i32 %inc, %size
   br i1 %cmp2, label %for.body2, label %for.body3
@@ -233,7 +233,7 @@ for.body3:
   %add5 = add nuw i32 %add, %l
   %idx.ext2 = zext i32 %add5 to i64
   %add.ptr2 = getelementptr inbounds i8, ptr %lined, i64 %idx.ext2
-  store i8 %tmp1, ptr %add.ptr2, align 1
+  store i8 %arg, ptr %add.ptr2, align 1
   %inc2 = add nsw i32 %l, 1
   %cmp3 = icmp slt i32 %inc2, %size
   br i1 %cmp3, label %for.body3, label %for.inc
diff --git a/llvm/test/Transforms/IndVarSimplify/elim-extend.ll b/llvm/test/Transforms/IndVarSimplify/elim-extend.ll
index 54bb9951ff66..01c95dadd162 100644
--- a/llvm/test/Transforms/IndVarSimplify/elim-extend.ll
+++ b/llvm/test/Transforms/IndVarSimplify/elim-extend.ll
@@ -142,7 +142,7 @@ define void @nestedIV(ptr %address, i32 %limit) nounwind {
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[INNERLOOP]], label [[INNEREXIT:%.*]]
 ; CHECK:       innerexit:
 ; CHECK-NEXT:    [[INNERCOUNT_LCSSA_WIDE:%.*]] = phi i64 [ [[INDVARS_IV_NEXT]], [[INNERLOOP]] ]
-; CHECK-NEXT:    [[TMP3:%.*]] = trunc i64 [[INNERCOUNT_LCSSA_WIDE]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc nsw i64 [[INNERCOUNT_LCSSA_WIDE]] to i32
 ; CHECK-NEXT:    br label [[OUTERMERGE]]
 ; CHECK:       outermerge:
 ; CHECK-NEXT:    [[INNERCOUNT_MERGE]] = phi i32 [ [[TMP3]], [[INNEREXIT]] ], [ [[INNERCOUNT]], [[INNERPREHEADER]] ]
diff --git a/llvm/test/Transforms/IndVarSimplify/hoist-wide-inc-for-narrow-use-recompute-flags.ll b/llvm/test/Transforms/IndVarSimplify/hoist-wide-inc-for-narrow-use-recompute-flags.ll
index cc99ee312ccb..1135ca9dbf00 100644
--- a/llvm/test/Transforms/IndVarSimplify/hoist-wide-inc-for-narrow-use-recompute-flags.ll
+++ b/llvm/test/Transforms/IndVarSimplify/hoist-wide-inc-for-narrow-use-recompute-flags.ll
@@ -15,7 +15,7 @@ define void @test_pr82243(ptr %f) {
 ; CHECK-NEXT:    [[GEP_IV_EXT:%.*]] = getelementptr i32, ptr [[F]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    store i32 1, ptr [[GEP_IV_EXT]], align 4
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
-; CHECK-NEXT:    [[TMP0:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc nuw nsw i64 [[INDVARS_IV_NEXT]] to i32
 ; CHECK-NEXT:    [[SHL:%.*]] = shl i32 123, [[TMP0]]
 ; CHECK-NEXT:    [[GEP_SHL:%.*]] = getelementptr i32, ptr [[F]], i32 [[SHL]]
 ; CHECK-NEXT:    br label [[INNER_HEADER:%.*]]
diff --git a/llvm/test/Transforms/IndVarSimplify/iv-sext.ll b/llvm/test/Transforms/IndVarSimplify/iv-sext.ll
index 450913f16baa..95a036f0e54c 100644
--- a/llvm/test/Transforms/IndVarSimplify/iv-sext.ll
+++ b/llvm/test/Transforms/IndVarSimplify/iv-sext.ll
@@ -99,7 +99,7 @@ define void @t(ptr %pval1, ptr %peakWeight, ptr %nrgReducePeakrate, i32 %bandEdg
 ; CHECK-NEXT:    [[VAL35_LCSSA:%.*]] = phi float [ [[VAL35]], [[BB5]] ]
 ; CHECK-NEXT:    [[VAL31_LCSSA_WIDE:%.*]] = phi i64 [ [[INDVARS_IV_NEXT]], [[BB5]] ]
 ; CHECK-NEXT:    [[VAL30_LCSSA:%.*]] = phi float [ [[VAL30]], [[BB5]] ]
-; CHECK-NEXT:    [[TMP4:%.*]] = trunc i64 [[VAL31_LCSSA_WIDE]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc nsw i64 [[VAL31_LCSSA_WIDE]] to i32
 ; CHECK-NEXT:    br label [[BB7]]
 ; CHECK:       bb7:
 ; CHECK-NEXT:    [[DISTERBHI_2_LCSSA]] = phi float [ [[VAL30_LCSSA]], [[BB5_BB7_CRIT_EDGE]] ], [ [[DISTERBHI_0_PH]], [[BB5_PREHEADER]] ]
diff --git a/llvm/test/Transforms/IndVarSimplify/iv-widen-elim-ext.ll b/llvm/test/Transforms/IndVarSimplify/iv-widen-elim-ext.ll
index 59a0241bfe9f..a83e9ce74b12 100644
--- a/llvm/test/Transforms/IndVarSimplify/iv-widen-elim-ext.ll
+++ b/llvm/test/Transforms/IndVarSimplify/iv-widen-elim-ext.ll
@@ -22,7 +22,7 @@ define void @foo(ptr %A, ptr %B, ptr %C, i32 %N) {
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[C:%.*]], i64 [[TMP1]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP0]], [[TMP2]]
-; CHECK-NEXT:    [[TMP3:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc nuw nsw i64 [[TMP1]] to i32
 ; CHECK-NEXT:    [[DIV0:%.*]] = udiv i32 5, [[TMP3]]
 ; CHECK-NEXT:    [[ADD4:%.*]] = add nsw i32 [[ADD3]], [[DIV0]]
 ; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
@@ -224,7 +224,7 @@ define i32 @foo3(i32 %M) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = add nsw i64 [[INDVARS_IV]], [[TMP0]]
-; CHECK-NEXT:    [[TMP4:%.*]] = trunc i64 [[TMP3]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc nsw i64 [[TMP3]] to i32
 ; CHECK-NEXT:    [[IDXPROM4:%.*]] = zext i32 [[TMP4]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [100 x i32], ptr @a, i64 0, i64 [[IDXPROM4]]
 ; CHECK-NEXT:    store i32 [[ADD]], ptr [[ARRAYIDX5]], align 4
@@ -365,7 +365,7 @@ define i32 @foo5(ptr %input, i32 %length, ptr %in) {
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ 1, [[FOR_BODY_LR_PH]] ]
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[INPUT]], align 8
-; CHECK-NEXT:    [[TMP5:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc nuw nsw i64 [[INDVARS_IV_NEXT]] to i32
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP4]], [[TMP5]]
 ; CHECK-NEXT:    [[IDX_EXT:%.*]] = sext i32 [[MUL]] to i64
 ; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i32, ptr [[IN:%.*]], i64 [[IDX_EXT]]
@@ -514,7 +514,7 @@ define void @foo7(i32 %n, ptr %a, i32 %x) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = shl nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = or disjoint i64 [[TMP2]], 1
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP3]]
-; CHECK-NEXT:    [[TMP4:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc nsw i64 [[INDVARS_IV]] to i32
 ; CHECK-NEXT:    store i32 [[TMP4]], ptr [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], [[TMP0]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], [[TMP1]]
diff --git a/llvm/test/Transforms/IndVarSimplify/lftr.ll b/llvm/test/Transforms/IndVarSimplify/lftr.ll
index 41db925de577..7f4820f093e5 100644
--- a/llvm/test/Transforms/IndVarSimplify/lftr.ll
+++ b/llvm/test/Transforms/IndVarSimplify/lftr.ll
@@ -525,7 +525,7 @@ define float @wide_trip_count_test3(ptr %b,
 ; CHECK-NEXT:    [[TMP0:%.*]] = add nsw i64 [[INDVARS_IV]], 20
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[TEMP:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc nsw i64 [[INDVARS_IV]] to i32
 ; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP1]] to float
 ; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[CONV]], [[TEMP]]
 ; CHECK-NEXT:    [[ADD1]] = fadd float [[SUM_07]], [[MUL]]
@@ -584,7 +584,7 @@ define float @wide_trip_count_test4(ptr %b,
 ; CHECK-NEXT:    [[TMP0:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 20
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[TEMP:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc nuw nsw i64 [[INDVARS_IV]] to i32
 ; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP1]] to float
 ; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[CONV]], [[TEMP]]
 ; CHECK-NEXT:    [[ADD1]] = fadd float [[SUM_07]], [[MUL]]
diff --git a/llvm/test/Transforms/IndVarSimplify/no-iv-rewrite.ll b/llvm/test/Transforms/IndVarSimplify/no-iv-rewrite.ll
index c35c5bacf68c..579b8536cedf 100644
--- a/llvm/test/Transforms/IndVarSimplify/no-iv-rewrite.ll
+++ b/llvm/test/Transforms/IndVarSimplify/no-iv-rewrite.ll
@@ -213,7 +213,7 @@ define void @maxvisitor(i32 %limit, ptr %base) nounwind {
 ; CHECK-NEXT:    [[CMP19:%.*]] = icmp sgt i32 [[VAL]], [[MAX]]
 ; CHECK-NEXT:    br i1 [[CMP19]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
 ; CHECK:       if.then:
-; CHECK-NEXT:    [[TMP0:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc nuw nsw i64 [[INDVARS_IV]] to i32
 ; CHECK-NEXT:    br label [[LOOP_INC]]
 ; CHECK:       if.else:
 ; CHECK-NEXT:    br label [[LOOP_INC]]
diff --git a/llvm/test/Transforms/IndVarSimplify/post-inc-range.ll b/llvm/test/Transforms/IndVarSimplify/post-inc-range.ll
index 5c22ba1044b6..bbdee0267eff 100644
--- a/llvm/test/Transforms/IndVarSimplify/post-inc-range.ll
+++ b/llvm/test/Transforms/IndVarSimplify/post-inc-range.ll
@@ -180,7 +180,7 @@ define void @test_neg(ptr %array_length_ptr, ptr %base,
 ; CHECK-NEXT:    br label [[FOR_INC]]
 ; CHECK:       for.inc:
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc nuw i64 [[INDVARS_IV_NEXT]] to i32
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP2]], [[LIMIT:%.*]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END]]
 ; CHECK:       for.end:
diff --git a/llvm/test/Transforms/IndVarSimplify/pr25578.ll b/llvm/test/Transforms/IndVarSimplify/pr25578.ll
index d8adc178474c..380e8171798b 100644
--- a/llvm/test/Transforms/IndVarSimplify/pr25578.ll
+++ b/llvm/test/Transforms/IndVarSimplify/pr25578.ll
@@ -13,7 +13,7 @@ L1_header:
 
 ; CHECK: L2_header:
 ; CHECK: %[[INDVAR:.*]] = phi i64
-; CHECK: %[[TRUNC:.*]] = trunc i64 %[[INDVAR]] to i32
+; CHECK: %[[TRUNC:.*]] = trunc nuw nsw i64 %[[INDVAR]] to i32
 L2_header:
   %i = phi i32 [ 0, %L1_header ], [ %i_next, %L2_latch ]
   %i_prom = sext i32 %i to i64
diff --git a/llvm/test/Transforms/IndVarSimplify/pr55925.ll b/llvm/test/Transforms/IndVarSimplify/pr55925.ll
index 312a8295ccdc..2ad187add4e1 100644
--- a/llvm/test/Transforms/IndVarSimplify/pr55925.ll
+++ b/llvm/test/Transforms/IndVarSimplify/pr55925.ll
@@ -14,11 +14,11 @@ define void @test(ptr %p) personality ptr undef {
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc nuw i64 [[INDVARS_IV]] to i32
 ; CHECK-NEXT:    [[RES:%.*]] = invoke i32 @foo(i32 returned [[TMP0]])
 ; CHECK-NEXT:            to label [[LOOP_LATCH]] unwind label [[EXIT:%.*]]
 ; CHECK:       loop.latch:
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc nuw i64 [[INDVARS_IV]] to i32
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @foo(i32 [[TMP1]])
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    br label [[LOOP]]
@@ -56,8 +56,8 @@ define void @test_critedge(i1 %c, ptr %p) personality ptr undef {
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    br i1 [[C:%.*]], label [[LOOP_INVOKE:%.*]], label [[LOOP_OTHER:%.*]]
 ; CHECK:       loop.invoke:
-; CHECK-NEXT:    [[TMP0:%.*]] = trunc i64 [[INDVARS_IV]] to i32
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc nuw i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc nuw i64 [[INDVARS_IV]] to i32
 ; CHECK-NEXT:    [[RES:%.*]] = invoke i32 @foo(i32 returned [[TMP0]])
 ; CHECK-NEXT:            to label [[LOOP_LATCH]] unwind label [[EXIT:%.*]]
 ; CHECK:       loop.other:
diff --git a/llvm/test/Transforms/IndVarSimplify/widen-nonnegative-countdown.ll b/llvm/test/Transforms/IndVarSimplify/widen-nonnegative-countdown.ll
index d473103f5824..9c8983421029 100644
--- a/llvm/test/Transforms/IndVarSimplify/widen-nonnegative-countdown.ll
+++ b/llvm/test/Transforms/IndVarSimplify/widen-nonnegative-countdown.ll
@@ -223,7 +223,7 @@ define void @sext_postinc(ptr %A, i32 %start) {
 ; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    tail call void @use_ptr(ptr [[ARRAYIDX_US]])
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc nsw i64 [[INDVARS_IV_NEXT]] to i32
 ; CHECK-NEXT:    [[CMP2_US:%.*]] = icmp ugt i32 [[TMP1]], 6
 ; CHECK-NEXT:    br i1 [[CMP2_US]], label [[FOR_BODY]], label [[EXIT_LOOPEXIT:%.*]]
 ; CHECK:       exit.loopexit:
@@ -262,7 +262,7 @@ define void @sext_preinc(ptr %A, i32 %start) {
 ; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    tail call void @use_ptr(ptr [[ARRAYIDX_US]])
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc nsw i64 [[INDVARS_IV]] to i32
 ; CHECK-NEXT:    [[CMP2_US:%.*]] = icmp ugt i32 [[TMP1]], 6
 ; CHECK-NEXT:    br i1 [[CMP2_US]], label [[FOR_BODY]], label [[EXIT_LOOPEXIT:%.*]]
 ; CHECK:       exit.loopexit:
@@ -366,7 +366,7 @@ define void @zext_postinc_offset_constant_one(ptr %A, i32 %start) {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[TMP0]], [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[J_016_US:%.*]] = phi i32 [ [[INC_US:%.*]], [[FOR_BODY]] ], [ [[START]], [[FOR_BODY_PREHEADER]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc nuw i64 [[INDVARS_IV]] to i32
 ; CHECK-NEXT:    [[ADD_US:%.*]] = add i32 [[TMP1]], 1
 ; CHECK-NEXT:    [[IDXPROM_US:%.*]] = zext i32 [[ADD_US]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[IDXPROM_US]]
@@ -513,13 +513,13 @@ define void @sext_postinc_offset_constant_one(ptr %A, i32 %start) {
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[TMP0]], [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc nsw i64 [[INDVARS_IV]] to i32
 ; CHECK-NEXT:    [[ADD_US:%.*]] = add i32 [[TMP1]], 1
 ; CHECK-NEXT:    [[IDXPROM_US:%.*]] = sext i32 [[ADD_US]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[IDXPROM_US]]
 ; CHECK-NEXT:    tail call void @use_ptr(ptr [[ARRAYIDX_US]])
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
-; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc nsw i64 [[INDVARS_IV_NEXT]] to i32
 ; CHECK-NEXT:    [[CMP2_US:%.*]] = icmp ugt i32 [[TMP2]], 6
 ; CHECK-NEXT:    br i1 [[CMP2_US]], label [[FOR_BODY]], label [[EXIT_LOOPEXIT:%.*]]
 ; CHECK:       exit.loopexit:
@@ -556,13 +556,13 @@ define void @sext_preinc_offset_constant_one(ptr %A, i32 %start) {
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[TMP0]], [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc nsw i64 [[INDVARS_IV]] to i32
 ; CHECK-NEXT:    [[ADD_US:%.*]] = add nuw i32 [[TMP1]], 1
 ; CHECK-NEXT:    [[IDXPROM_US:%.*]] = sext i32 [[ADD_US]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[IDXPROM_US]]
 ; CHECK-NEXT:    tail call void @use_ptr(ptr [[ARRAYIDX_US]])
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
-; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc nsw i64 [[INDVARS_IV]] to i32
 ; CHECK-NEXT:    [[CMP2_US:%.*]] = icmp ugt i32 [[TMP2]], 6
 ; CHECK-NEXT:    br i1 [[CMP2_US]], label [[FOR_BODY]], label [[EXIT_LOOPEXIT:%.*]]
 ; CHECK:       exit.loopexit:
@@ -808,13 +808,13 @@ define void @sext_postinc_offset_constant_minus_one(ptr %A, i32 %start) {
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[TMP0]], [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc nsw i64 [[INDVARS_IV]] to i32
 ; CHECK-NEXT:    [[ADD_US:%.*]] = add i32 [[TMP1]], -1
 ; CHECK-NEXT:    [[IDXPROM_US:%.*]] = sext i32 [[ADD_US]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[IDXPROM_US]]
 ; CHECK-NEXT:    tail call void @use_ptr(ptr [[ARRAYIDX_US]])
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
-; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc nsw i64 [[INDVARS_IV_NEXT]] to i32
 ; CHECK-NEXT:    [[CMP2_US:%.*]] = icmp ugt i32 [[TMP2]], 6
 ; CHECK-NEXT:    br i1 [[CMP2_US]], label [[FOR_BODY]], label [[EXIT_LOOPEXIT:%.*]]
 ; CHECK:       exit.loopexit:
@@ -851,13 +851,13 @@ define void @sext_preinc_offset_constant_minus_one(ptr %A, i32 %start) {
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[TMP0]], [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc nsw i64 [[INDVARS_IV]] to i32
 ; CHECK-NEXT:    [[ADD_US:%.*]] = add i32 [[TMP1]], -1
 ; CHECK-NEXT:    [[IDXPROM_US:%.*]] = sext i32 [[ADD_US]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[IDXPROM_US]]
 ; CHECK-NEXT:    tail call void @use_ptr(ptr [[ARRAYIDX_US]])
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], -1
-; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc nsw i64 [[INDVARS_IV]] to i32
 ; CHECK-NEXT:    [[CMP2_US:%.*]] = icmp ugt i32 [[TMP2]], 6
 ; CHECK-NEXT:    br i1 [[CMP2_US]], label [[FOR_BODY]], label [[EXIT_LOOPEXIT:%.*]]
 ; CHECK:       exit.loopexit:
diff --git a/llvm/test/Transforms/IndVarSimplify/widen-nonnegative.ll b/llvm/test/Transforms/IndVarSimplify/widen-nonnegative.ll
index 739db26311f4..e00eaafa3f19 100644
--- a/llvm/test/Transforms/IndVarSimplify/widen-nonnegative.ll
+++ b/llvm/test/Transforms/IndVarSimplify/widen-nonnegative.ll
@@ -150,7 +150,7 @@ define void @sext_add_nuw(ptr %A, i32 %offset, i32 %M) {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = add nuw nsw i64 [[INDVARS_IV]], [[TMP0]]
-; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc nuw i64 [[TMP1]] to i32
 ; CHECK-NEXT:    [[IDXPROM_US:%.*]] = sext i32 [[TMP2]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[IDXPROM_US]]
 ; CHECK-NEXT:    tail call void @use_ptr(ptr [[ARRAYIDX_US]])
@@ -185,7 +185,7 @@ define void @sext_add_noflags(ptr %A, i32 %offset, i32 %M) {
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc nuw nsw i64 [[INDVARS_IV]] to i32
 ; CHECK-NEXT:    [[ADD_US:%.*]] = add i32 [[TMP0]], [[OFFSET:%.*]]
 ; CHECK-NEXT:    [[IDXPROM_US:%.*]] = sext i32 [[ADD_US]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[IDXPROM_US]]
@@ -223,7 +223,7 @@ define void @zext_add_nsw(ptr %A, i32 %offset, i32 %M) {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = add nsw i64 [[INDVARS_IV]], [[TMP0]]
-; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc nsw i64 [[TMP1]] to i32
 ; CHECK-NEXT:    [[IDXPROM_US:%.*]] = zext i32 [[TMP2]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[IDXPROM_US]]
 ; CHECK-NEXT:    tail call void @use_ptr(ptr [[ARRAYIDX_US]])
@@ -293,7 +293,7 @@ define void @zext_add_noflags(ptr %A, i32 %offset, i32 %M) {
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc nuw nsw i64 [[INDVARS_IV]] to i32
 ; CHECK-NEXT:    [[ADD_US:%.*]] = add i32 [[TMP0]], [[OFFSET:%.*]]
 ; CHECK-NEXT:    [[IDXPROM_US:%.*]] = zext i32 [[ADD_US]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[IDXPROM_US]]
@@ -399,7 +399,7 @@ define void @zext_nneg_add_noflags(ptr %A, i32 %offset, i32 %M) {
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc nuw nsw i64 [[INDVARS_IV]] to i32
 ; CHECK-NEXT:    [[ADD_US:%.*]] = add i32 [[TMP0]], [[OFFSET:%.*]]
 ; CHECK-NEXT:    [[IDXPROM_US:%.*]] = zext nneg i32 [[ADD_US]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[IDXPROM_US]]
@@ -475,7 +475,7 @@ define void @sext_mul_nuw(ptr %A, i32 %multiple, i32 %M) {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw nsw i64 [[INDVARS_IV]], [[TMP0]]
-; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc nuw i64 [[TMP1]] to i32
 ; CHECK-NEXT:    [[IDXPROM_US:%.*]] = sext i32 [[TMP2]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[IDXPROM_US]]
 ; CHECK-NEXT:    tail call void @use_ptr(ptr [[ARRAYIDX_US]])
@@ -510,7 +510,7 @@ define void @sext_mul_noflags(ptr %A, i32 %multiple, i32 %M) {
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc nuw nsw i64 [[INDVARS_IV]] to i32
 ; CHECK-NEXT:    [[MUL_US:%.*]] = mul i32 [[TMP0]], [[MULTIPLE:%.*]]
 ; CHECK-NEXT:    [[IDXPROM_US:%.*]] = sext i32 [[MUL_US]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[IDXPROM_US]]
@@ -548,7 +548,7 @@ define void @zext_mul_nsw(ptr %A, i32 %multiple, i32 %M) {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = mul nsw i64 [[INDVARS_IV]], [[TMP0]]
-; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc nsw i64 [[TMP1]] to i32
 ; CHECK-NEXT:    [[IDXPROM_US:%.*]] = zext i32 [[TMP2]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[IDXPROM_US]]
 ; CHECK-NEXT:    tail call void @use_ptr(ptr [[ARRAYIDX_US]])
@@ -618,7 +618,7 @@ define void @zext_mul_noflags(ptr %A, i32 %multiple, i32 %M) {
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc nuw nsw i64 [[INDVARS_IV]] to i32
 ; CHECK-NEXT:    [[MUL_US:%.*]] = mul i32 [[TMP0]], [[MULTIPLE:%.*]]
 ; CHECK-NEXT:    [[IDXPROM_US:%.*]] = zext i32 [[MUL_US]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[IDXPROM_US]]
@@ -724,7 +724,7 @@ define void @zext_nneg_mul_noflags(ptr %A, i32 %multiple, i32 %M) {
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP0:%.*]] = trunc nuw nsw i64 [[INDVARS_IV]] to i32
 ; CHECK-NEXT:    [[MUL_US:%.*]] = mul i32 [[TMP0]], [[MULTIPLE:%.*]]
 ; CHECK-NEXT:    [[IDXPROM_US:%.*]] = zext nneg i32 [[MUL_US]] to i64
 ; CHECK-NEXT:    [[ARRAYIDX_US:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[IDXPROM_US]]
diff --git a/llvm/test/Transforms/LoopFlatten/widen-iv3.ll b/llvm/test/Transforms/LoopFlatten/widen-iv3.ll
index 6e6c045661c2..3ac5a69a496f 100644
--- a/llvm/test/Transforms/LoopFlatten/widen-iv3.ll
+++ b/llvm/test/Transforms/LoopFlatten/widen-iv3.ll
@@ -35,7 +35,7 @@ define i16 @foo() {
 ; CHECK-NEXT:    [[SUM_110:%.*]] = phi i16 [ [[SUM_012]], [[FOR_COND1_PREHEADER]] ], [ [[ADD5]], [[FOR_BODY4]] ]
 ; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i32 [[INDVAR]], [[TMP0]]
 ; CHECK-NEXT:    [[ADD:%.*]] = add nuw nsw i16 [[J_011]], [[MUL]]
-; CHECK-NEXT:    [[TMP3:%.*]] = trunc i32 [[TMP2]] to i16
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc nuw nsw i32 [[TMP2]] to i16
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [64 x i16], ptr @v, i16 0, i16 [[TMP3]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i16, ptr [[ARRAYIDX]], align 1
 ; CHECK-NEXT:    [[ADD5]] = add nsw i16 [[TMP4]], [[SUM_110]]
-- 
GitLab


From 5f680724838188f516d349bd9459710308d721e0 Mon Sep 17 00:00:00 2001
From: Qizhi Hu <836744285@qq.com>
Date: Tue, 16 Apr 2024 09:57:23 +0800
Subject: [PATCH 326/464] [Clang][Sema] Fix issue on requires expression with
 templated base class member function (#85198)

Fix https://github.com/llvm/llvm-project/issues/84020
Skip checking implicit object parameter in the context of
`RequiresExprBodyDecl`.

Co-authored-by: huqizhi <836744285@qq.com>
---
 clang/docs/ReleaseNotes.rst    |  1 +
 clang/lib/Sema/SemaExpr.cpp    |  3 ++-
 clang/test/SemaCXX/PR84020.cpp | 23 +++++++++++++++++++++++
 3 files changed, 26 insertions(+), 1 deletion(-)
 create mode 100644 clang/test/SemaCXX/PR84020.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index dc108785f6cc..76701dc723b6 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -538,6 +538,7 @@ Bug Fixes to C++ Support
   object parameter.
   Fixes (#GH70604), (#GH79754), (#GH84163), (#GH84425), (#GH86054), (#GH86398), and (#GH86399).
 - Fix a crash when deducing ``auto`` from an invalid dereference (#GH88329).
+- Fix a crash in requires expression with templated base class member function. Fixes (#GH84020).
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 24f354f1c724..189764cb4b6b 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -7739,7 +7739,8 @@ ExprResult Sema::BuildResolvedCallExpr(Expr *Fn, NamedDecl *NDecl,
   }
 
   if (CXXMethodDecl *Method = dyn_cast_or_null<CXXMethodDecl>(FDecl))
-    if (Method->isImplicitObjectMemberFunction())
+    if (!isa<RequiresExprBodyDecl>(CurContext) &&
+        Method->isImplicitObjectMemberFunction())
       return ExprError(Diag(LParenLoc, diag::err_member_call_without_object)
                        << Fn->getSourceRange() << 0);
 
diff --git a/clang/test/SemaCXX/PR84020.cpp b/clang/test/SemaCXX/PR84020.cpp
new file mode 100644
index 000000000000..8ea5dcc4527a
--- /dev/null
+++ b/clang/test/SemaCXX/PR84020.cpp
@@ -0,0 +1,23 @@
+// RUN: %clang_cc1 -std=c++20 -verify %s
+// RUN: %clang_cc1 -std=c++23 -verify %s
+// expected-no-diagnostics
+
+struct B {
+    template <typename S>
+    void foo();
+
+    void bar();
+};
+
+template <typename T, typename S>
+struct A : T {
+    auto foo() {
+        static_assert(requires { T::template foo<S>(); });
+        static_assert(requires { T::bar(); });
+    }
+};
+
+int main() {
+    A<B, double> a;
+    a.foo();
+}
-- 
GitLab


From 568368a43e5b4adb3c5d105a0eff3e0c13c0af8c Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Mon, 15 Apr 2024 19:05:30 -0700
Subject: [PATCH 327/464] [Support] Make readNext default to unaligned (#88808)

Without this patch, you would typically use readNext as:

  readNext<uint32_t, llvm::endianness::little, unaligned>(Ptr)

which is quite mouthful.  Since most serialization/deserialization
operations are unaligned accesses, this patch makes the alignment
template parameter default to unaligned, allowing us to say:

  readNext<uint32_t, llvm::endianness::little>(Ptr)

I'm including a few examples of migration in this patch.  I'll do the
rest in a separate patch.

Note that writeNext already has the same trick for the alignment
template parameter.
---
 llvm/include/llvm/Support/Endian.h          | 4 ++--
 llvm/include/llvm/Support/OnDiskHashTable.h | 9 +++------
 2 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/llvm/include/llvm/Support/Endian.h b/llvm/include/llvm/Support/Endian.h
index 1cdb5ca0d5ea..30e0852b972c 100644
--- a/llvm/include/llvm/Support/Endian.h
+++ b/llvm/include/llvm/Support/Endian.h
@@ -80,8 +80,8 @@ template <typename value_type, std::size_t alignment, typename CharT>
   return ret;
 }
 
-template <typename value_type, endianness endian, std::size_t alignment,
-          typename CharT>
+template <typename value_type, endianness endian,
+          std::size_t alignment = unaligned, typename CharT>
 [[nodiscard]] inline value_type readNext(const CharT *&memory) {
   return readNext<value_type, alignment, CharT>(memory, endian);
 }
diff --git a/llvm/include/llvm/Support/OnDiskHashTable.h b/llvm/include/llvm/Support/OnDiskHashTable.h
index 0a8cbbd8b188..f6b4055e74de 100644
--- a/llvm/include/llvm/Support/OnDiskHashTable.h
+++ b/llvm/include/llvm/Support/OnDiskHashTable.h
@@ -368,14 +368,12 @@ public:
 
     // 'Items' starts with a 16-bit unsigned integer representing the
     // number of items in this bucket.
-    unsigned Len =
-        endian::readNext<uint16_t, llvm::endianness::little, unaligned>(Items);
+    unsigned Len = endian::readNext<uint16_t, llvm::endianness::little>(Items);
 
     for (unsigned i = 0; i < Len; ++i) {
       // Read the hash.
       hash_value_type ItemHash =
-          endian::readNext<hash_value_type, llvm::endianness::little,
-                           unaligned>(Items);
+          endian::readNext<hash_value_type, llvm::endianness::little>(Items);
 
       // Determine the length of the key and the data.
       const std::pair<offset_type, offset_type> &L =
@@ -473,8 +471,7 @@ private:
         // 'Items' starts with a 16-bit unsigned integer representing the
         // number of items in this bucket.
         NumItemsInBucketLeft =
-            endian::readNext<uint16_t, llvm::endianness::little, unaligned>(
-                Ptr);
+            endian::readNext<uint16_t, llvm::endianness::little>(Ptr);
       }
       Ptr += sizeof(hash_value_type); // Skip the hash.
       // Determine the length of the key and the data.
-- 
GitLab


From b4cf63d26f4c41dd9403c4e62500d82a6d31d692 Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Mon, 15 Apr 2024 20:27:04 -0600
Subject: [PATCH 328/464] [X86] Remove obsolete tablegen rules for near data in
 small static code model (#84523)

These should be already handled by other code.

Removing the kernel code model rules right above it cause
bss_pagealigned.ll to fail by using a movabsq to get the address of a
global, haven't figured out where that code is yet.
---
 llvm/lib/Target/X86/X86InstrCompiler.td | 22 ----------------------
 1 file changed, 22 deletions(-)

diff --git a/llvm/lib/Target/X86/X86InstrCompiler.td b/llvm/lib/Target/X86/X86InstrCompiler.td
index 270dd32c7235..2350636be10e 100644
--- a/llvm/lib/Target/X86/X86InstrCompiler.td
+++ b/llvm/lib/Target/X86/X86InstrCompiler.td
@@ -1247,28 +1247,6 @@ def : Pat<(i64 (X86Wrapper mcsym:$dst)),
 def : Pat<(i64 (X86Wrapper tblockaddress:$dst)),
           (MOV64ri32 tblockaddress:$dst)>, Requires<[KernelCode]>;
 
-// If we have small model and -static mode, it is safe to store global addresses
-// directly as immediates.  FIXME: This is really a hack, the 'imm' predicate
-// for MOV64mi32 should handle this sort of thing.
-def : Pat<(store (i64 (X86Wrapper tconstpool:$src)), addr:$dst),
-          (MOV64mi32 addr:$dst, tconstpool:$src)>,
-          Requires<[NearData, IsNotPIC]>;
-def : Pat<(store (i64 (X86Wrapper tjumptable:$src)), addr:$dst),
-          (MOV64mi32 addr:$dst, tjumptable:$src)>,
-          Requires<[NearData, IsNotPIC]>;
-def : Pat<(store (i64 (X86Wrapper tglobaladdr:$src)), addr:$dst),
-          (MOV64mi32 addr:$dst, tglobaladdr:$src)>,
-          Requires<[NearData, IsNotPIC]>;
-def : Pat<(store (i64 (X86Wrapper texternalsym:$src)), addr:$dst),
-          (MOV64mi32 addr:$dst, texternalsym:$src)>,
-          Requires<[NearData, IsNotPIC]>;
-def : Pat<(store (i64 (X86Wrapper mcsym:$src)), addr:$dst),
-          (MOV64mi32 addr:$dst, mcsym:$src)>,
-          Requires<[NearData, IsNotPIC]>;
-def : Pat<(store (i64 (X86Wrapper tblockaddress:$src)), addr:$dst),
-          (MOV64mi32 addr:$dst, tblockaddress:$src)>,
-          Requires<[NearData, IsNotPIC]>;
-
 def : Pat<(i32 (X86RecoverFrameAlloc mcsym:$dst)), (MOV32ri mcsym:$dst)>;
 def : Pat<(i64 (X86RecoverFrameAlloc mcsym:$dst)), (MOV64ri mcsym:$dst)>;
 
-- 
GitLab


From 82f479ba315a417b6cd01a8c2efdc15c26689f2e Mon Sep 17 00:00:00 2001
From: Usama Hameed <u_hameed@apple.com>
Date: Mon, 15 Apr 2024 19:42:45 -0700
Subject: [PATCH 329/464] Add asan tests for libsanitizers. (#88349)

This patch tests LLDB integration with libsanitizers for ASan.

rdar://111856681
---
 lldb/test/API/functionalities/asan/Makefile   |  6 +-
 .../functionalities/asan/TestMemoryHistory.py | 73 ++++++++++++++++++-
 .../functionalities/asan/TestReportData.py    | 20 ++++-
 .../API/functionalities/libsanitizers/util.py |  3 +
 4 files changed, 97 insertions(+), 5 deletions(-)
 create mode 100644 lldb/test/API/functionalities/libsanitizers/util.py

diff --git a/lldb/test/API/functionalities/asan/Makefile b/lldb/test/API/functionalities/asan/Makefile
index 4913a18d8cc6..d66696fed707 100644
--- a/lldb/test/API/functionalities/asan/Makefile
+++ b/lldb/test/API/functionalities/asan/Makefile
@@ -1,4 +1,8 @@
 C_SOURCES := main.c
-CFLAGS_EXTRAS := -fsanitize=address -g -gcolumn-info
+asan: CFLAGS_EXTRAS := -fsanitize=address -g -gcolumn-info
+asan: all
+
+libsanitizers: CFLAGS_EXTRAS := -fsanitize=address -fsanitize-stable-abi -g -gcolumn-info
+libsanitizers: all
 
 include Makefile.rules
diff --git a/lldb/test/API/functionalities/asan/TestMemoryHistory.py b/lldb/test/API/functionalities/asan/TestMemoryHistory.py
index 00162ae8822c..ee7939203ead 100644
--- a/lldb/test/API/functionalities/asan/TestMemoryHistory.py
+++ b/lldb/test/API/functionalities/asan/TestMemoryHistory.py
@@ -9,15 +9,21 @@ from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbplatform
 from lldbsuite.test import lldbutil
 
+from functionalities.libsanitizers.util import no_libsanitizers
 
 class AsanTestCase(TestBase):
     @skipIfFreeBSD  # llvm.org/pr21136 runtimes not yet available by default
     @expectedFailureNetBSD
     @skipUnlessAddressSanitizer
     def test(self):
-        self.build()
+        self.build(make_targets=["asan"])
         self.asan_tests()
 
+    @skipIf(oslist=no_match(["macosx"]))
+    def test_libsanitizers_asan(self):
+        self.build(make_targets=["libsanitizers"])
+        self.libsanitizer_tests()
+
     def setUp(self):
         # Call super's setUp().
         TestBase.setUp(self)
@@ -26,6 +32,71 @@ class AsanTestCase(TestBase):
         self.line_free = line_number("main.c", "// free line")
         self.line_breakpoint = line_number("main.c", "// break line")
 
+    # Test line numbers: rdar://126237493
+    def libsanitizer_tests(self):
+        target = self.createTestTarget()
+
+        if no_libsanitizers(self):
+            self.skipTest("libsanitizers not found")
+
+        self.runCmd(
+            "env SanitizersAddress=1 MallocSanitizerZone=1 MallocSecureAllocator=0"
+        )
+
+        self.runCmd("run")
+
+        # In libsanitizers, memory history is not supported until a report has been generated
+        self.expect(
+            "thread list",
+            "Process should be stopped due to ASan report",
+            substrs=["stopped", "stop reason = Use of deallocated memory"],
+        )
+
+        # test the 'memory history' command
+        self.expect(
+            "memory history 'pointer'",
+            substrs=[
+                "Memory deallocated by Thread",
+                "a.out`f2",
+                "main.c",
+                "Memory allocated by Thread",
+                "a.out`f1",
+                "main.c",
+            ],
+        )
+
+        # do the same using SB API
+        process = self.dbg.GetSelectedTarget().process
+        val = (
+            process.GetSelectedThread().GetSelectedFrame().EvaluateExpression("pointer")
+        )
+        addr = val.GetValueAsUnsigned()
+        threads = process.GetHistoryThreads(addr)
+        self.assertEqual(threads.GetSize(), 2)
+
+        history_thread = threads.GetThreadAtIndex(0)
+        self.assertTrue(history_thread.num_frames >= 2)
+        self.assertEqual(
+            history_thread.frames[1].GetLineEntry().GetFileSpec().GetFilename(),
+            "main.c",
+        )
+
+        history_thread = threads.GetThreadAtIndex(1)
+        self.assertTrue(history_thread.num_frames >= 2)
+        self.assertEqual(
+            history_thread.frames[1].GetLineEntry().GetFileSpec().GetFilename(),
+            "main.c",
+        )
+
+        # let's free the container (SBThreadCollection) and see if the
+        # SBThreads still live
+        threads = None
+        self.assertTrue(history_thread.num_frames >= 2)
+        self.assertEqual(
+            history_thread.frames[1].GetLineEntry().GetFileSpec().GetFilename(),
+            "main.c",
+        )
+
     def asan_tests(self):
         target = self.createTestTarget()
 
diff --git a/lldb/test/API/functionalities/asan/TestReportData.py b/lldb/test/API/functionalities/asan/TestReportData.py
index 543c5fe66a20..de0c1206a57a 100644
--- a/lldb/test/API/functionalities/asan/TestReportData.py
+++ b/lldb/test/API/functionalities/asan/TestReportData.py
@@ -9,6 +9,7 @@ from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
 
+from functionalities.libsanitizers.util import no_libsanitizers
 
 class AsanTestReportDataCase(TestBase):
     @skipIfFreeBSD  # llvm.org/pr21136 runtimes not yet available by default
@@ -16,9 +17,14 @@ class AsanTestReportDataCase(TestBase):
     @skipUnlessAddressSanitizer
     @skipIf(archs=["i386"], bugnumber="llvm.org/PR36710")
     def test(self):
-        self.build()
+        self.build(make_targets=["asan"])
         self.asan_tests()
 
+    @skipIf(oslist=no_match(["macosx"]))
+    def test_libsanitizers_asan(self):
+        self.build(make_targets=["libsanitizers"])
+        self.asan_tests(libsanitizers=True)
+
     def setUp(self):
         # Call super's setUp().
         TestBase.setUp(self)
@@ -29,10 +35,18 @@ class AsanTestReportDataCase(TestBase):
         self.line_crash = line_number("main.c", "// BOOM line")
         self.col_crash = 16
 
-    def asan_tests(self):
+    def asan_tests(self, libsanitizers=False):
         target = self.createTestTarget()
 
-        self.registerSanitizerLibrariesWithTarget(target)
+        if libsanitizers and no_libsanitizers(self):
+            self.skipTest("libsanitizers not found")
+
+        if libsanitizers:
+            self.runCmd(
+                "env SanitizersAddress=1 MallocSanitizerZone=1 MallocSecureAllocator=0"
+            )
+        else:
+            self.registerSanitizerLibrariesWithTarget(target)
 
         self.runCmd("run")
 
diff --git a/lldb/test/API/functionalities/libsanitizers/util.py b/lldb/test/API/functionalities/libsanitizers/util.py
new file mode 100644
index 000000000000..ad68541aba8d
--- /dev/null
+++ b/lldb/test/API/functionalities/libsanitizers/util.py
@@ -0,0 +1,3 @@
+def no_libsanitizers(testbase):
+    testbase.runCmd("image list libsystem_sanitizers.dylib", check=False)
+    return not "libsystem_sanitizers.dylib" in testbase.res.GetOutput()
-- 
GitLab


From 00ae4b738be6e840cfdb3e7461409c3e49403e50 Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Tue, 16 Apr 2024 03:49:28 +0000
Subject: [PATCH 330/464] Revert "[X86] Remove obsolete tablegen rules for near
 data in small static code model (#84523)"

This reverts commit b4cf63d26f4c41dd9403c4e62500d82a6d31d692.

Breaks indirect-branch-tracking-eh2.ll.
---
 llvm/lib/Target/X86/X86InstrCompiler.td | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/llvm/lib/Target/X86/X86InstrCompiler.td b/llvm/lib/Target/X86/X86InstrCompiler.td
index 2350636be10e..270dd32c7235 100644
--- a/llvm/lib/Target/X86/X86InstrCompiler.td
+++ b/llvm/lib/Target/X86/X86InstrCompiler.td
@@ -1247,6 +1247,28 @@ def : Pat<(i64 (X86Wrapper mcsym:$dst)),
 def : Pat<(i64 (X86Wrapper tblockaddress:$dst)),
           (MOV64ri32 tblockaddress:$dst)>, Requires<[KernelCode]>;
 
+// If we have small model and -static mode, it is safe to store global addresses
+// directly as immediates.  FIXME: This is really a hack, the 'imm' predicate
+// for MOV64mi32 should handle this sort of thing.
+def : Pat<(store (i64 (X86Wrapper tconstpool:$src)), addr:$dst),
+          (MOV64mi32 addr:$dst, tconstpool:$src)>,
+          Requires<[NearData, IsNotPIC]>;
+def : Pat<(store (i64 (X86Wrapper tjumptable:$src)), addr:$dst),
+          (MOV64mi32 addr:$dst, tjumptable:$src)>,
+          Requires<[NearData, IsNotPIC]>;
+def : Pat<(store (i64 (X86Wrapper tglobaladdr:$src)), addr:$dst),
+          (MOV64mi32 addr:$dst, tglobaladdr:$src)>,
+          Requires<[NearData, IsNotPIC]>;
+def : Pat<(store (i64 (X86Wrapper texternalsym:$src)), addr:$dst),
+          (MOV64mi32 addr:$dst, texternalsym:$src)>,
+          Requires<[NearData, IsNotPIC]>;
+def : Pat<(store (i64 (X86Wrapper mcsym:$src)), addr:$dst),
+          (MOV64mi32 addr:$dst, mcsym:$src)>,
+          Requires<[NearData, IsNotPIC]>;
+def : Pat<(store (i64 (X86Wrapper tblockaddress:$src)), addr:$dst),
+          (MOV64mi32 addr:$dst, tblockaddress:$src)>,
+          Requires<[NearData, IsNotPIC]>;
+
 def : Pat<(i32 (X86RecoverFrameAlloc mcsym:$dst)), (MOV32ri mcsym:$dst)>;
 def : Pat<(i64 (X86RecoverFrameAlloc mcsym:$dst)), (MOV64ri mcsym:$dst)>;
 
-- 
GitLab


From 8ee7d9732204a7415967bce2c1d2c0fa63af264f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Valentin=20Clement=20=28=E3=83=90=E3=83=AC=E3=83=B3?=
 =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=B3=20=E3=82=AF=E3=83=AC=E3=83=A1?=
 =?UTF-8?q?=E3=83=B3=29?= <clementval@gmail.com>
Date: Mon, 15 Apr 2024 20:51:31 -0700
Subject: [PATCH 331/464] [flang][cuda] Add fir.cuda_allocate operation
 (#88586)

Allocatable with cuda device attribute have special semantic for the
allocate statement. In flang the allocate statement is lowered to a
sequence of runtime call initializing the descriptor and then allocating
the descriptor data. This new operation will replace the last runtime
call and abstract all the device memory allocation needed.
The lowering patch will follow.
---
 .../include/flang/Optimizer/Dialect/FIROps.td | 32 +++++++++
 .../flang/Optimizer/Dialect/FIRTypes.td       |  1 +
 flang/lib/Optimizer/Dialect/FIROps.cpp        | 19 +++++
 flang/test/Fir/cuf-invalid.fir                | 50 +++++++++++++
 flang/test/Fir/cuf.mlir                       | 70 +++++++++++++++++++
 5 files changed, 172 insertions(+)
 create mode 100644 flang/test/Fir/cuf-invalid.fir
 create mode 100644 flang/test/Fir/cuf.mlir

diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index dff1cdb20cbf..c181c7ed62df 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -3190,4 +3190,36 @@ def fir_CUDADataTransferOp : fir_Op<"cuda_data_transfer", []> {
   }];
 }
 
+def fir_CUDAAllocateOp : fir_Op<"cuda_allocate", [AttrSizedOperandSegments,
+    MemoryEffects<[MemAlloc<DefaultResource>]>]> {
+  let summary = "Perform the device allocation of data of an allocatable";
+
+  let description = [{
+    The fir.cuda_allocate operation performs the allocation on the device
+    of the data of an allocatable. The descriptor passed to the operation
+    is initialized before with the standard flang runtime calls.
+  }];
+
+  let arguments = (ins Arg<AnyRefOrBoxType, "", [MemWrite]>:$box,
+                       Arg<Optional<AnyRefOrBoxType>, "", [MemWrite]>:$errmsg,
+                       Optional<AnyIntegerType>:$stream,
+                       Arg<Optional<AnyRefOrBoxType>, "", [MemWrite]>:$pinned,
+                       Arg<Optional<AnyRefOrBoxType>, "", [MemRead]>:$source,
+                       fir_CUDADataAttributeAttr:$cuda_attr,
+                       UnitAttr:$hasStat);
+
+  let results = (outs AnyIntegerType:$stat);
+
+  let assemblyFormat = [{
+    $box `:` qualified(type($box))
+    ( `source` `(` $source^ `:` qualified(type($source) )`)` )?
+    ( `errmsg` `(` $errmsg^ `:` type($errmsg) `)` )?
+    ( `stream` `(` $stream^ `:` type($stream) `)` )?
+    ( `pinned` `(` $pinned^ `:` type($pinned) `)` )?
+    attr-dict `->` type($stat)
+  }];
+
+  let hasVerifier = 1;
+}
+
 #endif
diff --git a/flang/include/flang/Optimizer/Dialect/FIRTypes.td b/flang/include/flang/Optimizer/Dialect/FIRTypes.td
index 4c6a8064991a..3b876e4642da 100644
--- a/flang/include/flang/Optimizer/Dialect/FIRTypes.td
+++ b/flang/include/flang/Optimizer/Dialect/FIRTypes.td
@@ -625,6 +625,7 @@ def AnyRefOrBoxLike : TypeConstraint<Or<[AnyReferenceLike.predicate,
 def AnyRefOrBox : TypeConstraint<Or<[fir_ReferenceType.predicate,
     fir_HeapType.predicate, fir_PointerType.predicate,
     IsBaseBoxTypePred]>, "any reference or box">;
+def AnyRefOrBoxType : Type<AnyRefOrBox.predicate, "any legal ref or box type">;
 
 def AnyShapeLike : TypeConstraint<Or<[fir_ShapeType.predicate,
     fir_ShapeShiftType.predicate]>, "any legal shape type">;
diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index 8ab74103cb6a..88710880174d 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -3993,6 +3993,25 @@ mlir::LogicalResult fir::CUDAKernelOp::verify() {
   return mlir::success();
 }
 
+mlir::LogicalResult fir::CUDAAllocateOp::verify() {
+  if (getPinned() && getStream())
+    return emitOpError("pinned and stream cannot appears at the same time");
+  if (!fir::unwrapRefType(getBox().getType()).isa<fir::BaseBoxType>())
+    return emitOpError(
+        "expect box to be a reference to/or a class or box type value");
+  if (getSource() &&
+      !fir::unwrapRefType(getSource().getType()).isa<fir::BaseBoxType>())
+    return emitOpError(
+        "expect source to be a reference to/or a class or box type value");
+  if (getErrmsg() &&
+      !fir::unwrapRefType(getErrmsg().getType()).isa<fir::BoxType>())
+    return emitOpError(
+        "expect errmsg to be a reference to/or a box type value");
+  if (getErrmsg() && !getHasStat())
+    return emitOpError("expect stat attribute when errmsg is provided");
+  return mlir::success();
+}
+
 //===----------------------------------------------------------------------===//
 // FIROpsDialect
 //===----------------------------------------------------------------------===//
diff --git a/flang/test/Fir/cuf-invalid.fir b/flang/test/Fir/cuf-invalid.fir
new file mode 100644
index 000000000000..9c5ffe7176a3
--- /dev/null
+++ b/flang/test/Fir/cuf-invalid.fir
@@ -0,0 +1,50 @@
+// RUN: fir-opt -split-input-file -verify-diagnostics %s
+
+func.func @_QPsub1() {
+  %0 = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", uniq_name = "_QFsub1Ea"}
+  %1 = fir.alloca i32
+  %pinned = fir.alloca i1
+  %4:2 = hlfir.declare %0 {cuda_attr = #fir.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
+  %11 = fir.convert %4#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<none>>
+  %s = fir.load %1 : !fir.ref<i32>
+  // expected-error@+1{{'fir.cuda_allocate' op pinned and stream cannot appears at the same time}}
+  %13 = fir.cuda_allocate %11 : !fir.ref<!fir.box<none>> stream(%s : i32) pinned(%pinned : !fir.ref<i1>) {cuda_attr = #fir.cuda<device>} -> i32
+  return
+}
+
+// -----
+
+func.func @_QPsub1() {
+  %1 = fir.alloca i32
+  // expected-error@+1{{'fir.cuda_allocate' op expect box to be a reference to/or a class or box type value}}
+  %2 = fir.cuda_allocate %1 : !fir.ref<i32> {cuda_attr = #fir.cuda<device>} -> i32
+  return
+}
+
+// -----
+
+func.func @_QPsub1() {
+  %0 = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", uniq_name = "_QFsub1Ea"}
+  %4:2 = hlfir.declare %0 {cuda_attr = #fir.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
+  %c100 = arith.constant 100 : index
+  %7 = fir.alloca !fir.char<1,100> {bindc_name = "msg", uniq_name = "_QFsub1Emsg"}
+  %8:2 = hlfir.declare %7 typeparams %c100 {uniq_name = "_QFsub1Emsg"} : (!fir.ref<!fir.char<1,100>>, index) -> (!fir.ref<!fir.char<1,100>>, !fir.ref<!fir.char<1,100>>)
+  %9 = fir.embox %8#1 : (!fir.ref<!fir.char<1,100>>) -> !fir.box<!fir.char<1,100>>
+  %11 = fir.convert %4#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<none>>
+  %16 = fir.convert %9 : (!fir.box<!fir.char<1,100>>) -> !fir.box<none>
+  // expected-error@+1{{'fir.cuda_allocate' op expect stat attribute when errmsg is provided}}
+  %13 = fir.cuda_allocate %11 : !fir.ref<!fir.box<none>> errmsg(%16 : !fir.box<none>) {cuda_attr = #fir.cuda<device>} -> i32
+  return
+}
+
+// -----
+
+func.func @_QPsub1() {
+  %0 = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", uniq_name = "_QFsub1Ea"}
+  %4:2 = hlfir.declare %0 {cuda_attr = #fir.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
+  %1 = fir.alloca i32
+  %11 = fir.convert %4#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<none>>
+  // expected-error@+1{{'fir.cuda_allocate' op expect errmsg to be a reference to/or a box type value}}
+  %13 = fir.cuda_allocate %11 : !fir.ref<!fir.box<none>> errmsg(%1 : !fir.ref<i32>) {cuda_attr = #fir.cuda<device>, hasStat} -> i32
+  return
+}
diff --git a/flang/test/Fir/cuf.mlir b/flang/test/Fir/cuf.mlir
new file mode 100644
index 000000000000..67eff31b35b2
--- /dev/null
+++ b/flang/test/Fir/cuf.mlir
@@ -0,0 +1,70 @@
+// RUN: fir-opt --split-input-file %s | fir-opt --split-input-file | FileCheck %s
+
+// Simple round trip test of operations.
+
+func.func @_QPsub1() {
+  %0 = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", uniq_name = "_QFsub1Ea"}
+  %4:2 = hlfir.declare %0 {cuda_attr = #fir.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
+  %11 = fir.convert %4#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<none>>
+  %13 = fir.cuda_allocate %11 : !fir.ref<!fir.box<none>> {cuda_attr = #fir.cuda<device>} -> i32
+  return
+}
+
+// CHECK: fir.cuda_allocate %{{.*}} : !fir.ref<!fir.box<none>> {cuda_attr = #fir.cuda<device>} -> i32
+
+// -----
+
+func.func @_QPsub1() {
+  %0 = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", uniq_name = "_QFsub1Ea"}
+  %1 = fir.alloca i32
+  %4:2 = hlfir.declare %0 {cuda_attr = #fir.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
+  %11 = fir.convert %4#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<none>>
+  %s = fir.load %1 : !fir.ref<i32>
+  %13 = fir.cuda_allocate %11 : !fir.ref<!fir.box<none>> stream(%s : i32) {cuda_attr = #fir.cuda<device>} -> i32
+  return
+}
+
+// CHECK: fir.cuda_allocate %{{.*}} : !fir.ref<!fir.box<none>> stream(%{{.*}} : i32) {cuda_attr = #fir.cuda<device>} -> i32
+
+// -----
+
+func.func @_QPsub1() {
+  %0 = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", uniq_name = "_QFsub1Ea"}
+  %1 = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "b", uniq_name = "_QFsub1Eb"}
+  %4:2 = hlfir.declare %0 {cuda_attr = #fir.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
+  %5:2 = hlfir.declare %1 {fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
+  %11 = fir.convert %4#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<none>>
+  %12 = fir.convert %5#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<none>>
+  %13 = fir.cuda_allocate %11 : !fir.ref<!fir.box<none>> source(%12 : !fir.ref<!fir.box<none>>) {cuda_attr = #fir.cuda<device>} -> i32
+  return
+}
+
+// CHECK: fir.cuda_allocate %{{.*}} : !fir.ref<!fir.box<none>> source(%{{.*}} : !fir.ref<!fir.box<none>>) {cuda_attr = #fir.cuda<device>} -> i32
+
+// -----
+
+func.func @_QPsub1() {
+  %0 = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", uniq_name = "_QFsub1Ea"}
+  %pinned = fir.alloca i1
+  %4:2 = hlfir.declare %0 {cuda_attr = #fir.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
+  %11 = fir.convert %4#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<none>>
+  %13 = fir.cuda_allocate %11 : !fir.ref<!fir.box<none>> pinned(%pinned : !fir.ref<i1>) {cuda_attr = #fir.cuda<device>} -> i32
+  return
+}
+
+// CHECK: fir.cuda_allocate %{{.*}} : !fir.ref<!fir.box<none>> pinned(%{{.*}} : !fir.ref<i1>) {cuda_attr = #fir.cuda<device>} -> i32
+
+// -----
+
+func.func @_QPsub1() {
+  %0 = fir.alloca !fir.box<!fir.heap<!fir.array<?xf32>>> {bindc_name = "a", uniq_name = "_QFsub1Ea"}
+  %4:2 = hlfir.declare %0 {cuda_attr = #fir.cuda<device>, fortran_attrs = #fir.var_attrs<allocatable>, uniq_name = "_QFsub1Ea"} : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>, !fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>)
+  %c100 = arith.constant 100 : index
+  %7 = fir.alloca !fir.char<1,100> {bindc_name = "msg", uniq_name = "_QFsub1Emsg"}
+  %8:2 = hlfir.declare %7 typeparams %c100 {uniq_name = "_QFsub1Emsg"} : (!fir.ref<!fir.char<1,100>>, index) -> (!fir.ref<!fir.char<1,100>>, !fir.ref<!fir.char<1,100>>)
+  %9 = fir.embox %8#1 : (!fir.ref<!fir.char<1,100>>) -> !fir.box<!fir.char<1,100>>
+  %11 = fir.convert %4#1 : (!fir.ref<!fir.box<!fir.heap<!fir.array<?xf32>>>>) -> !fir.ref<!fir.box<none>>
+  %16 = fir.convert %9 : (!fir.box<!fir.char<1,100>>) -> !fir.box<none>
+  %13 = fir.cuda_allocate %11 : !fir.ref<!fir.box<none>> errmsg(%16 : !fir.box<none>) {cuda_attr = #fir.cuda<device>, hasStat} -> i32
+  return
+}
-- 
GitLab


From fe48bf672e1ab293368a3212203db94a4e21c533 Mon Sep 17 00:00:00 2001
From: Chelsea Cassanova <chelsea_cassanova@apple.com>
Date: Mon, 15 Apr 2024 21:26:18 -0700
Subject: [PATCH 332/464] [lldb][lit] Guard MallocNanoZone envvar in shell
 tests (#88824)

Previously the MallocNanoZone envvar would be set to 0 on Darwin for the
LLDB shell tests, but this should guarded behind ASan being enabled as
opposed to simply running the test suite behind Darwin. This required
that the LLVM_USE_SANITIZER option be added as an attribute to the lit
config for shell tests.
---
 lldb/test/Shell/lit.cfg.py         | 12 ++++++++----
 lldb/test/Shell/lit.site.cfg.py.in |  1 +
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/lldb/test/Shell/lit.cfg.py b/lldb/test/Shell/lit.cfg.py
index 290569576ac8..e24f3fbb4d93 100644
--- a/lldb/test/Shell/lit.cfg.py
+++ b/lldb/test/Shell/lit.cfg.py
@@ -50,10 +50,14 @@ llvm_config.with_system_environment(
 )
 
 # Enable sanitizer runtime flags.
-config.environment["ASAN_OPTIONS"] = "detect_stack_use_after_return=1"
-config.environment["TSAN_OPTIONS"] = "halt_on_error=1"
-if platform.system() == "Darwin":
-    config.environment["MallocNanoZone"] = "0"
+if "Address" in config.llvm_use_sanitizer:
+    config.environment["ASAN_OPTIONS"] = "detect_stack_use_after_return=1"
+    if platform.system() == "Darwin":
+        config.environment["MallocNanoZone"] = "0"
+
+if "Thread" in config.llvm_use_sanitizer:
+    config.environment["TSAN_OPTIONS"] = "halt_on_error=1"
+
 
 # Support running the test suite under the lldb-repro wrapper. This makes it
 # possible to capture a test suite run and then rerun all the test from the
diff --git a/lldb/test/Shell/lit.site.cfg.py.in b/lldb/test/Shell/lit.site.cfg.py.in
index 736dfc335732..b69e7bce1bc0 100644
--- a/lldb/test/Shell/lit.site.cfg.py.in
+++ b/lldb/test/Shell/lit.site.cfg.py.in
@@ -26,6 +26,7 @@ config.lldb_enable_lua = @LLDB_ENABLE_LUA@
 config.lldb_build_directory = "@LLDB_TEST_BUILD_DIRECTORY@"
 config.have_lldb_server = @LLDB_TOOL_LLDB_SERVER_BUILD@
 config.lldb_system_debugserver = @LLDB_USE_SYSTEM_DEBUGSERVER@
+config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
 # The shell tests use their own module caches.
 config.lldb_module_cache = os.path.join("@LLDB_TEST_MODULE_CACHE_LLDB@", "lldb-shell")
 config.clang_module_cache = os.path.join("@LLDB_TEST_MODULE_CACHE_CLANG@", "lldb-shell")
-- 
GitLab


From 65b0cc610f80d9b9724a98cf7c5bcfd38e1cf799 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 15 Apr 2024 21:54:26 -0700
Subject: [PATCH 333/464] [RISCV] Add FeatureStdExtI to all CPUs in
 RISCVProcessors.td. NFC (#88805)

This is currently being implied in RISCVISAInfo.cpp. Make it explicit.

I'm planning to move all extension information to RISCVFeatures.td and
have tablegen create the tables for RISCVISAInfo.cpp. This requires
making the creation of RISCVTargetParserDef.inc in tablegen independent
of RISCVISAInfo.cpp. So we need an accurate extension list for CPUs in
tablegen.
---
 llvm/lib/Target/RISCV/RISCVProcessors.td | 27 ++++++++++++++++++++++--
 1 file changed, 25 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVProcessors.td b/llvm/lib/Target/RISCV/RISCVProcessors.td
index fd6d6078ec23..739b50749e13 100644
--- a/llvm/lib/Target/RISCV/RISCVProcessors.td
+++ b/llvm/lib/Target/RISCV/RISCVProcessors.td
@@ -56,11 +56,13 @@ class RISCVTuneProcessorModel<string n,
 
 def GENERIC_RV32 : RISCVProcessorModel<"generic-rv32",
                                        NoSchedModel,
-                                       [Feature32Bit]>,
+                                       [Feature32Bit,
+                                        FeatureStdExtI]>,
                    GenericTuneInfo;
 def GENERIC_RV64 : RISCVProcessorModel<"generic-rv64",
                                        NoSchedModel,
-                                       [Feature64Bit]>,
+                                       [Feature64Bit,
+                                        FeatureStdExtI]>,
                    GenericTuneInfo;
 // Support generic for compatibility with other targets. The triple will be used
 // to change to the appropriate rv32/rv64 version.
@@ -69,11 +71,13 @@ def : ProcessorModel<"generic", NoSchedModel, []>, GenericTuneInfo;
 def ROCKET_RV32 : RISCVProcessorModel<"rocket-rv32",
                                       RocketModel,
                                       [Feature32Bit,
+                                       FeatureStdExtI,
                                        FeatureStdExtZifencei,
                                        FeatureStdExtZicsr]>;
 def ROCKET_RV64 : RISCVProcessorModel<"rocket-rv64",
                                       RocketModel,
                                       [Feature64Bit,
+                                       FeatureStdExtI,
                                        FeatureStdExtZifencei,
                                        FeatureStdExtZicsr]>;
 def ROCKET : RISCVTuneProcessorModel<"rocket",
@@ -86,6 +90,7 @@ def SIFIVE_7 : RISCVTuneProcessorModel<"sifive-7-series",
 def SIFIVE_E20 : RISCVProcessorModel<"sifive-e20",
                                      RocketModel,
                                      [Feature32Bit,
+                                      FeatureStdExtI,
                                       FeatureStdExtZicsr,
                                       FeatureStdExtZifencei,
                                       FeatureStdExtM,
@@ -94,6 +99,7 @@ def SIFIVE_E20 : RISCVProcessorModel<"sifive-e20",
 def SIFIVE_E21 : RISCVProcessorModel<"sifive-e21",
                                      RocketModel,
                                      [Feature32Bit,
+                                      FeatureStdExtI,
                                       FeatureStdExtZicsr,
                                       FeatureStdExtZifencei,
                                       FeatureStdExtM,
@@ -103,6 +109,7 @@ def SIFIVE_E21 : RISCVProcessorModel<"sifive-e21",
 def SIFIVE_E24 : RISCVProcessorModel<"sifive-e24",
                                      RocketModel,
                                      [Feature32Bit,
+                                      FeatureStdExtI,
                                       FeatureStdExtZifencei,
                                       FeatureStdExtM,
                                       FeatureStdExtA,
@@ -112,6 +119,7 @@ def SIFIVE_E24 : RISCVProcessorModel<"sifive-e24",
 def SIFIVE_E31 : RISCVProcessorModel<"sifive-e31",
                                      RocketModel,
                                      [Feature32Bit,
+                                      FeatureStdExtI,
                                       FeatureStdExtZifencei,
                                       FeatureStdExtZicsr,
                                       FeatureStdExtM,
@@ -121,6 +129,7 @@ def SIFIVE_E31 : RISCVProcessorModel<"sifive-e31",
 def SIFIVE_E34 : RISCVProcessorModel<"sifive-e34",
                                      RocketModel,
                                      [Feature32Bit,
+                                      FeatureStdExtI,
                                       FeatureStdExtZifencei,
                                       FeatureStdExtM,
                                       FeatureStdExtA,
@@ -130,6 +139,7 @@ def SIFIVE_E34 : RISCVProcessorModel<"sifive-e34",
 def SIFIVE_E76 : RISCVProcessorModel<"sifive-e76",
                                      SiFive7Model,
                                      [Feature32Bit,
+                                      FeatureStdExtI,
                                       FeatureStdExtZifencei,
                                       FeatureStdExtM,
                                       FeatureStdExtA,
@@ -140,6 +150,7 @@ def SIFIVE_E76 : RISCVProcessorModel<"sifive-e76",
 def SIFIVE_S21 : RISCVProcessorModel<"sifive-s21",
                                      RocketModel,
                                      [Feature64Bit,
+                                      FeatureStdExtI,
                                       FeatureStdExtZicsr,
                                       FeatureStdExtZifencei,
                                       FeatureStdExtM,
@@ -149,6 +160,7 @@ def SIFIVE_S21 : RISCVProcessorModel<"sifive-s21",
 def SIFIVE_S51 : RISCVProcessorModel<"sifive-s51",
                                      RocketModel,
                                      [Feature64Bit,
+                                      FeatureStdExtI,
                                       FeatureStdExtZicsr,
                                       FeatureStdExtZifencei,
                                       FeatureStdExtM,
@@ -158,6 +170,7 @@ def SIFIVE_S51 : RISCVProcessorModel<"sifive-s51",
 def SIFIVE_S54 : RISCVProcessorModel<"sifive-s54",
                                       RocketModel,
                                       [Feature64Bit,
+                                       FeatureStdExtI,
                                        FeatureStdExtZifencei,
                                        FeatureStdExtM,
                                        FeatureStdExtA,
@@ -168,6 +181,7 @@ def SIFIVE_S54 : RISCVProcessorModel<"sifive-s54",
 def SIFIVE_S76 : RISCVProcessorModel<"sifive-s76",
                                      SiFive7Model,
                                      [Feature64Bit,
+                                      FeatureStdExtI,
                                       FeatureStdExtZifencei,
                                       FeatureStdExtM,
                                       FeatureStdExtA,
@@ -180,6 +194,7 @@ def SIFIVE_S76 : RISCVProcessorModel<"sifive-s76",
 def SIFIVE_U54 : RISCVProcessorModel<"sifive-u54",
                                      RocketModel,
                                      [Feature64Bit,
+                                      FeatureStdExtI,
                                       FeatureStdExtZifencei,
                                       FeatureStdExtM,
                                       FeatureStdExtA,
@@ -190,6 +205,7 @@ def SIFIVE_U54 : RISCVProcessorModel<"sifive-u54",
 def SIFIVE_U74 : RISCVProcessorModel<"sifive-u74",
                                      SiFive7Model,
                                      [Feature64Bit,
+                                      FeatureStdExtI,
                                       FeatureStdExtZifencei,
                                       FeatureStdExtM,
                                       FeatureStdExtA,
@@ -200,6 +216,7 @@ def SIFIVE_U74 : RISCVProcessorModel<"sifive-u74",
 
 def SIFIVE_X280 : RISCVProcessorModel<"sifive-x280", SiFive7Model,
                                       [Feature64Bit,
+                                       FeatureStdExtI,
                                        FeatureStdExtZifencei,
                                        FeatureStdExtM,
                                        FeatureStdExtA,
@@ -217,6 +234,7 @@ def SIFIVE_X280 : RISCVProcessorModel<"sifive-x280", SiFive7Model,
 
 def SIFIVE_P450 : RISCVProcessorModel<"sifive-p450", SiFiveP400Model,
                                       [Feature64Bit,
+                                       FeatureStdExtI,
                                        FeatureStdExtZifencei,
                                        FeatureStdExtM,
                                        FeatureStdExtA,
@@ -247,6 +265,7 @@ def SIFIVE_P450 : RISCVProcessorModel<"sifive-p450", SiFiveP400Model,
 
 def SIFIVE_P670 : RISCVProcessorModel<"sifive-p670", SiFiveP600Model,
                                       [Feature64Bit,
+                                       FeatureStdExtI,
                                        FeatureStdExtZifencei,
                                        FeatureStdExtM,
                                        FeatureStdExtA,
@@ -286,6 +305,7 @@ def SIFIVE_P670 : RISCVProcessorModel<"sifive-p670", SiFiveP600Model,
 def SYNTACORE_SCR1_BASE : RISCVProcessorModel<"syntacore-scr1-base",
                                               SyntacoreSCR1Model,
                                               [Feature32Bit,
+                                               FeatureStdExtI,
                                                FeatureStdExtZicsr,
                                                FeatureStdExtZifencei,
                                                FeatureStdExtC],
@@ -294,6 +314,7 @@ def SYNTACORE_SCR1_BASE : RISCVProcessorModel<"syntacore-scr1-base",
 def SYNTACORE_SCR1_MAX : RISCVProcessorModel<"syntacore-scr1-max",
                                              SyntacoreSCR1Model,
                                              [Feature32Bit,
+                                              FeatureStdExtI,
                                               FeatureStdExtZicsr,
                                               FeatureStdExtZifencei,
                                               FeatureStdExtM,
@@ -303,6 +324,7 @@ def SYNTACORE_SCR1_MAX : RISCVProcessorModel<"syntacore-scr1-max",
 def VENTANA_VEYRON_V1 : RISCVProcessorModel<"veyron-v1",
                                             NoSchedModel,
                                             [Feature64Bit,
+                                             FeatureStdExtI,
                                              FeatureStdExtZifencei,
                                              FeatureStdExtZicsr,
                                              FeatureStdExtZicntr,
@@ -332,6 +354,7 @@ def VENTANA_VEYRON_V1 : RISCVProcessorModel<"veyron-v1",
 def XIANGSHAN_NANHU : RISCVProcessorModel<"xiangshan-nanhu",
                                           XiangShanNanHuModel,
                                           [Feature64Bit,
+                                           FeatureStdExtI,
                                            FeatureStdExtZicsr,
                                            FeatureStdExtZifencei,
                                            FeatureStdExtM,
-- 
GitLab


From 4e2d11f87a595d658f25ec5838b002e4bf5a26fc Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 15 Apr 2024 21:55:50 -0700
Subject: [PATCH 334/464] [RISCV] Remove unnecessary NoHasStdExtZicfiss
 Predicate from cmop.1 and cmop.5. NFC (#88822)

I'm not sure what this was supposed do. Maybe it was for the
disassembler, but that should be managed through DecoderNamespace.

So let's remove it and simplify the code.
---
 llvm/lib/Target/RISCV/RISCVInstrInfoZcmop.td | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZcmop.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZcmop.td
index dd13a07d606d..65091aad9100 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZcmop.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZcmop.td
@@ -20,13 +20,7 @@ class CMOPInst<bits<3> imm3, string opcodestr>
   let Inst{12-11} = 0;
 }
 
-// CMOP1, CMOP5 is used by Zicfiss.
-let Predicates = [HasStdExtZcmop, NoHasStdExtZicfiss] in {
-  def CMOP1 : CMOPInst<0, "cmop.1">, Sched<[]>;
-  def CMOP5 : CMOPInst<2, "cmop.5">, Sched<[]>;
-}
-
-foreach n = [3, 7, 9, 11, 13, 15] in {
+foreach n = [1, 3, 5, 7, 9, 11, 13, 15] in {
   let Predicates = [HasStdExtZcmop] in
   def CMOP # n : CMOPInst<!srl(n, 1), "cmop." # n>, Sched<[]>;
 }
-- 
GitLab


From edb0708dc1ceeaeb3356311a4ddf72a0dc9b224f Mon Sep 17 00:00:00 2001
From: Vlad Mishel <43666597+vmishelcs@users.noreply.github.com>
Date: Mon, 15 Apr 2024 22:05:29 -0700
Subject: [PATCH 335/464] [InstCombine] Implement `fcmp (fadd x, 0.0), y` =>
 `fcmp x, y` optimization (#88476)

This PR addresses issue #88168. It implements an optimization for
the case of

```
define i1 @fcmp_fadd_zero_ugt(float %x, float %y) {
  %add = fadd float %x, 0.000000e+00
  %cmp = fcmp ugt float %add, %y
  ret i1 %cmp
}
```
`=>`
```
define i1 @fcmp_fadd_zero_ugt(float %x, float %y) {
  %cmp = fcmp ugt float %x, %y
  ret i1 %cmp
}
```
and all other types of `fcmp` instructions (`uge`, `ogt`, etc).

Proofs:
`fadd x, 0.0` https://alive2.llvm.org/ce/z/7FzNnM
`fsub x, 0.0` https://alive2.llvm.org/ce/z/puUxLK
---
 .../InstCombine/InstCombineCompares.cpp       |   8 +
 llvm/test/Transforms/InstCombine/fcmp.ll      | 202 ++++++++++++++++++
 2 files changed, 210 insertions(+)

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index 90550cdbdf89..ee783eed190a 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -8097,6 +8097,14 @@ Instruction *InstCombinerImpl::visitFCmpInst(FCmpInst &I) {
         return new FCmpInst(I.getSwappedPredicate(), X, NegC, "", &I);
   }
 
+  // fcmp (fadd X, 0.0), Y --> fcmp X, Y
+  if (match(Op0, m_FAdd(m_Value(X), m_AnyZeroFP())))
+    return new FCmpInst(Pred, X, Op1, "", &I);
+
+  // fcmp X, (fadd Y, 0.0) --> fcmp X, Y
+  if (match(Op1, m_FAdd(m_Value(Y), m_AnyZeroFP())))
+    return new FCmpInst(Pred, Op0, Y, "", &I);
+
   if (match(Op0, m_FPExt(m_Value(X)))) {
     // fcmp (fpext X), (fpext Y) -> fcmp X, Y
     if (match(Op1, m_FPExt(m_Value(Y))) && X->getType() == Y->getType())
diff --git a/llvm/test/Transforms/InstCombine/fcmp.ll b/llvm/test/Transforms/InstCombine/fcmp.ll
index f2701d16d0f3..069512b0f2d8 100644
--- a/llvm/test/Transforms/InstCombine/fcmp.ll
+++ b/llvm/test/Transforms/InstCombine/fcmp.ll
@@ -1284,3 +1284,205 @@ define <1 x i1> @bitcast_1vec_eq0(i32 %x) {
   %cmp = fcmp oeq <1 x float> %f, zeroinitializer
   ret <1 x i1> %cmp
 }
+
+; Simplify fcmp (x + 0.0), y => fcmp x, y
+
+define i1 @fcmp_fadd_zero_ugt(float %x, float %y) {
+; CHECK-LABEL: @fcmp_fadd_zero_ugt(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ugt float [[ADD:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = fadd float %x, 0.000000e+00
+  %cmp = fcmp ugt float %add, %y
+  ret i1 %cmp
+}
+
+define i1 @fcmp_fadd_zero_uge(float %x, float %y) {
+; CHECK-LABEL: @fcmp_fadd_zero_uge(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp uge float [[ADD:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = fadd float %x, 0.000000e+00
+  %cmp = fcmp uge float %add, %y
+  ret i1 %cmp
+}
+
+define i1 @fcmp_fadd_zero_ogt(float %x, float %y) {
+; CHECK-LABEL: @fcmp_fadd_zero_ogt(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt float [[ADD:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = fadd float %x, 0.000000e+00
+  %cmp = fcmp ogt float %add, %y
+  ret i1 %cmp
+}
+
+define i1 @fcmp_fadd_zero_oge(float %x, float %y) {
+; CHECK-LABEL: @fcmp_fadd_zero_oge(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oge float [[ADD:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = fadd float %x, 0.000000e+00
+  %cmp = fcmp oge float %add, %y
+  ret i1 %cmp
+}
+
+define i1 @fcmp_fadd_zero_ult(float %x, float %y) {
+; CHECK-LABEL: @fcmp_fadd_zero_ult(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ult float [[ADD:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = fadd float %x, 0.000000e+00
+  %cmp = fcmp ult float %add, %y
+  ret i1 %cmp
+}
+
+define i1 @fcmp_fadd_zero_ule(float %x, float %y) {
+; CHECK-LABEL: @fcmp_fadd_zero_ule(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ule float [[ADD:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = fadd float %x, 0.000000e+00
+  %cmp = fcmp ule float %add, %y
+  ret i1 %cmp
+}
+
+define i1 @fcmp_fadd_zero_olt(float %x, float %y) {
+; CHECK-LABEL: @fcmp_fadd_zero_olt(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp olt float [[ADD:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = fadd float %x, 0.000000e+00
+  %cmp = fcmp olt float %add, %y
+  ret i1 %cmp
+}
+
+define i1 @fcmp_fadd_zero_ole(float %x, float %y) {
+; CHECK-LABEL: @fcmp_fadd_zero_ole(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ole float [[ADD:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = fadd float %x, 0.000000e+00
+  %cmp = fcmp ole float %add, %y
+  ret i1 %cmp
+}
+
+define i1 @fcmp_fadd_zero_oeq(float %x, float %y) {
+; CHECK-LABEL: @fcmp_fadd_zero_oeq(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[ADD:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = fadd float %x, 0.000000e+00
+  %cmp = fcmp oeq float %add, %y
+  ret i1 %cmp
+}
+
+define i1 @fcmp_fadd_zero_one(float %x, float %y) {
+; CHECK-LABEL: @fcmp_fadd_zero_one(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp one float [[ADD:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = fadd float %x, 0.000000e+00
+  %cmp = fcmp one float %add, %y
+  ret i1 %cmp
+}
+
+define i1 @fcmp_fadd_zero_ueq(float %x, float %y) {
+; CHECK-LABEL: @fcmp_fadd_zero_ueq(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ueq float [[ADD:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = fadd float %x, 0.000000e+00
+  %cmp = fcmp ueq float %add, %y
+  ret i1 %cmp
+}
+
+define i1 @fcmp_fadd_zero_une(float %x, float %y) {
+; CHECK-LABEL: @fcmp_fadd_zero_une(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp une float [[ADD:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = fadd float %x, 0.000000e+00
+  %cmp = fcmp une float %add, %y
+  ret i1 %cmp
+}
+
+define i1 @fcmp_fadd_zero_ord(float %x, float %y) {
+; CHECK-LABEL: @fcmp_fadd_zero_ord(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ord float [[ADD:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = fadd float %x, 0.000000e+00
+  %cmp = fcmp ord float %add, %y
+  ret i1 %cmp
+}
+
+define i1 @fcmp_fadd_zero_uno(float %x, float %y) {
+; CHECK-LABEL: @fcmp_fadd_zero_uno(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp uno float [[ADD:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = fadd float %x, 0.000000e+00
+  %cmp = fcmp uno float %add, %y
+  ret i1 %cmp
+}
+
+define i1 @fcmp_fadd_neg_zero(float %x, float %y) {
+; CHECK-LABEL: @fcmp_fadd_neg_zero(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ugt float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = fadd float %x, -0.000000e+00
+  %cmp = fcmp ugt float %add, %y
+  ret i1 %cmp
+}
+
+define i1 @fcmp_fadd_zero_switched(float %x, float %y) {
+; CHECK-LABEL: @fcmp_fadd_zero_switched(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ult float [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = fadd float %y, 0.000000e+00
+  %cmp = fcmp ugt float %x, %add
+  ret i1 %cmp
+}
+
+define <2 x i1> @fcmp_fadd_zero_vec(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @fcmp_fadd_zero_vec(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ugt <2 x float> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %add = fadd <2 x float> %x, <float 0.0, float -0.0>
+  %cmp = fcmp ugt <2 x float> %add, %y
+  ret <2 x i1> %cmp
+}
+
+define i1 @fcmp_fast_fadd_fast_zero(float %x, float %y) {
+; CHECK-LABEL: @fcmp_fast_fadd_fast_zero(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp fast ugt float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = fadd fast float %x, 0.000000e+00
+  %cmp = fcmp fast ugt float %add, %y
+  ret i1 %cmp
+}
+
+define i1 @fcmp_fast_fadd_zero(float %x, float %y) {
+; CHECK-LABEL: @fcmp_fast_fadd_zero(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp fast ugt float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = fadd float %x, 0.000000e+00
+  %cmp = fcmp fast ugt float %add, %y
+  ret i1 %cmp
+}
+
+define i1 @fcmp_fadd_fast_zero(float %x, float %y) {
+; CHECK-LABEL: @fcmp_fadd_fast_zero(
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ugt float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %add = fadd fast float %x, 0.000000e+00
+  %cmp = fcmp ugt float %add, %y
+  ret i1 %cmp
+}
-- 
GitLab


From 39016e33b0fe78ddb1f11822f71a8a233af4dca9 Mon Sep 17 00:00:00 2001
From: Chuanqi Xu <yedeng.yd@linux.alibaba.com>
Date: Tue, 16 Apr 2024 13:00:06 +0800
Subject: [PATCH 336/464] [C++20] [Modules] Don't import non-inline function
 bodies even if it is always-inline

Recommit
https://github.com/llvm/llvm-project/commit/1ecbab56dcbb78268c8d19af34a50591f90b12a0

Close https://github.com/llvm/llvm-project/issues/80949

The new thing in this commit is to allow to import the function body
from instantiations if it is marked with always-inline. See the
discussion in https://github.com/llvm/llvm-project/issues/86893 for
details.
---
 clang/lib/CodeGen/CodeGenModule.cpp           | 17 ++++++++--
 .../CodeGenCXX/module-funcs-from-imports.cppm | 31 ++++++++++++++++---
 2 files changed, 40 insertions(+), 8 deletions(-)

diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index e44749672d58..0c447b20cef4 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -3952,9 +3952,20 @@ bool CodeGenModule::shouldEmitFunction(GlobalDecl GD) {
   // behavior may break ABI compatibility of the current unit.
   if (const Module *M = F->getOwningModule();
       M && M->getTopLevelModule()->isNamedModule() &&
-      getContext().getCurrentNamedModule() != M->getTopLevelModule() &&
-      !F->hasAttr<AlwaysInlineAttr>())
-    return false;
+      getContext().getCurrentNamedModule() != M->getTopLevelModule()) {
+    // There are practices to mark template member function as always-inline
+    // and mark the template as extern explicit instantiation but not give
+    // the definition for member function. So we have to emit the function
+    // from explicitly instantiation with always-inline.
+    //
+    // See https://github.com/llvm/llvm-project/issues/86893 for details.
+    //
+    // TODO: Maybe it is better to give it a warning if we call a non-inline
+    // function from other module units which is marked as always-inline.
+    if (!F->isTemplateInstantiation() || !F->hasAttr<AlwaysInlineAttr>()) {
+      return false;
+    }
+  }
 
   if (F->hasAttr<NoInlineAttr>())
     return false;
diff --git a/clang/test/CodeGenCXX/module-funcs-from-imports.cppm b/clang/test/CodeGenCXX/module-funcs-from-imports.cppm
index 33cdf437110a..a2a9122fc391 100644
--- a/clang/test/CodeGenCXX/module-funcs-from-imports.cppm
+++ b/clang/test/CodeGenCXX/module-funcs-from-imports.cppm
@@ -23,6 +23,21 @@ int func_in_gmf_not_called() {
     return 44;
 }
 
+template <class T>
+class A {
+public:
+    __attribute__((always_inline))
+    inline constexpr int getValue() {
+        return 43;
+    }
+
+    inline constexpr int getValue2() {
+        return 43;
+    }
+};
+
+extern template class A<char>;
+
 //--- M.cppm
 module;
 #include "foo.h"
@@ -47,17 +62,21 @@ int always_inline_func() {
     return 45;
 }
 
+export using ::A;
+
 //--- Use.cpp
 import M;
 int use() {
-    return exported_func() + always_inline_func();
+    A<char> a;
+    return exported_func() + always_inline_func() +
+           a.getValue() + a.getValue2();
 }
 
-// Checks that none of the function (except the always_inline_func) in the importees
-// are generated in the importer's code.
 // CHECK-O0: define{{.*}}_Z3usev(
 // CHECK-O0: declare{{.*}}_ZW1M13exported_funcv(
-// CHECK-O0: define{{.*}}available_externally{{.*}}_ZW1M18always_inline_funcv(
+// CHECK-O0: declare{{.*}}_ZW1M18always_inline_funcv(
+// CHECK-O0: define{{.*}}@_ZN1AIcE8getValueEv(
+// CHECK-O0: declare{{.*}}@_ZN1AIcE9getValue2Ev(
 // CHECK-O0-NOT: func_in_gmf
 // CHECK-O0-NOT: func_in_gmf_not_called
 // CHECK-O0-NOT: non_exported_func
@@ -68,7 +87,9 @@ int use() {
 // O0 to keep consistent ABI.
 // CHECK-O1: define{{.*}}_Z3usev(
 // CHECK-O1: declare{{.*}}_ZW1M13exported_funcv(
-// CHECK-O1: define{{.*}}available_externally{{.*}}_ZW1M18always_inline_funcv(
+// CHECK-O1: declare{{.*}}_ZW1M18always_inline_funcv(
+// CHECK-O1: define{{.*}}@_ZN1AIcE8getValueEv(
+// CHECK-O1: declare{{.*}}@_ZN1AIcE9getValue2Ev(
 // CHECK-O1-NOT: func_in_gmf
 // CHECK-O1-NOT: func_in_gmf_not_called
 // CHECK-O1-NOT: non_exported_func
-- 
GitLab


From f14146fd46dd8bf7ef1e64d3c13af73dc5668045 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 15 Apr 2024 22:33:14 -0700
Subject: [PATCH 337/464] [RISCV] Correct the spelling of the Zcmop mnemonics.
 (#88826)

The instruction names should be c.mop.1 instead of cmop.1.
---
 llvm/lib/Target/RISCV/RISCVInstrInfoZcmop.td |  2 +-
 llvm/test/MC/RISCV/rv32zcmop-invalid.s       |  6 ++--
 llvm/test/MC/RISCV/rvzcmop-valid.s           | 32 ++++++++++----------
 3 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZcmop.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZcmop.td
index 65091aad9100..32e7f962aa2a 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZcmop.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZcmop.td
@@ -22,5 +22,5 @@ class CMOPInst<bits<3> imm3, string opcodestr>
 
 foreach n = [1, 3, 5, 7, 9, 11, 13, 15] in {
   let Predicates = [HasStdExtZcmop] in
-  def CMOP # n : CMOPInst<!srl(n, 1), "cmop." # n>, Sched<[]>;
+  def C_MOP # n : CMOPInst<!srl(n, 1), "c.mop." # n>, Sched<[]>;
 }
diff --git a/llvm/test/MC/RISCV/rv32zcmop-invalid.s b/llvm/test/MC/RISCV/rv32zcmop-invalid.s
index 71d72d59b020..fb6252f7f076 100644
--- a/llvm/test/MC/RISCV/rv32zcmop-invalid.s
+++ b/llvm/test/MC/RISCV/rv32zcmop-invalid.s
@@ -1,7 +1,7 @@
 # RUN: not llvm-mc -triple riscv32 -mattr=+zcmop < %s 2>&1 | FileCheck %s
 
-cmop.0 # CHECK: :[[@LINE]]:1: error: unrecognized instruction mnemonic
+c.mop.0 # CHECK: :[[@LINE]]:1: error: unrecognized instruction mnemonic
 
-cmop.1 t0 # CHECK: :[[@LINE]]:8: error: invalid operand for instruction
+c.mop.1 t0 # CHECK: :[[@LINE]]:9: error: invalid operand for instruction
 
-cmop.1 0x0 # CHECK: :[[@LINE]]:8: error: invalid operand for instruction
+c.mop.1 0x0 # CHECK: :[[@LINE]]:9: error: invalid operand for instruction
diff --git a/llvm/test/MC/RISCV/rvzcmop-valid.s b/llvm/test/MC/RISCV/rvzcmop-valid.s
index c6bb4a158082..dd5d26ac5dd0 100644
--- a/llvm/test/MC/RISCV/rvzcmop-valid.s
+++ b/llvm/test/MC/RISCV/rvzcmop-valid.s
@@ -9,34 +9,34 @@
 # RUN:     | llvm-objdump --mattr=+zcmop -d -r - \
 # RUN:     | FileCheck --check-prefix=CHECK-ASM-AND-OBJ %s
 
-# CHECK-ASM-AND-OBJ: cmop.1
+# CHECK-ASM-AND-OBJ: c.mop.1
 # CHECK-ASM: encoding: [0x81,0x60]
-cmop.1
+c.mop.1
 
-# CHECK-ASM-AND-OBJ: cmop.3
+# CHECK-ASM-AND-OBJ: c.mop.3
 # CHECK-ASM: encoding: [0x81,0x61]
-cmop.3
+c.mop.3
 
-# CHECK-ASM-AND-OBJ: cmop.5
+# CHECK-ASM-AND-OBJ: c.mop.5
 # CHECK-ASM: encoding: [0x81,0x62]
-cmop.5
+c.mop.5
 
-# CHECK-ASM-AND-OBJ: cmop.7
+# CHECK-ASM-AND-OBJ: c.mop.7
 # CHECK-ASM: encoding: [0x81,0x63]
-cmop.7
+c.mop.7
 
-# CHECK-ASM-AND-OBJ: cmop.9
+# CHECK-ASM-AND-OBJ: c.mop.9
 # CHECK-ASM: encoding: [0x81,0x64]
-cmop.9
+c.mop.9
 
-# CHECK-ASM-AND-OBJ: cmop.11
+# CHECK-ASM-AND-OBJ: c.mop.11
 # CHECK-ASM: encoding: [0x81,0x65]
-cmop.11
+c.mop.11
 
-# CHECK-ASM-AND-OBJ: cmop.13
+# CHECK-ASM-AND-OBJ: c.mop.13
 # CHECK-ASM: encoding: [0x81,0x66]
-cmop.13
+c.mop.13
 
-# CHECK-ASM-AND-OBJ: cmop.15
+# CHECK-ASM-AND-OBJ: c.mop.15
 # CHECK-ASM: encoding: [0x81,0x67]
-cmop.15
+c.mop.15
-- 
GitLab


From 17d6bf046cea381413895f91e24d26d65763b59a Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Mon, 15 Apr 2024 22:39:09 -0700
Subject: [PATCH 338/464] [RISCV] Change how MMO is rebuilt in
 lowerFixedLengthVectorLoadToRVV/lowerFixedLengthVectorStoreToRVV (#88811)

Copy the pointer info, flags, alignment, AAInfo, and ranges, but let
getLoad rebuild the MMO using the scalable type used for the the new
load/store. This makes sure the LLT minimum size matches the ContainerVT
minimum size. This is important since vscale_range may have been used to
determine that the fixed vector was the exact size of a scalable vector.

Fixes #88799
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 19 +++++--------------
 llvm/test/CodeGen/RISCV/rvv/pr88799.ll      | 19 +++++++++++++++++++
 2 files changed, 24 insertions(+), 14 deletions(-)
 create mode 100644 llvm/test/CodeGen/RISCV/rvv/pr88799.ll

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index f6ed6420c9e1..1d1ea6bae6c1 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -10433,14 +10433,10 @@ RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(SDValue Op,
   if (MinVLMAX == MaxVLMAX && MinVLMAX == VT.getVectorNumElements() &&
       getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
     MachineMemOperand *MMO = Load->getMemOperand();
-    MachineFunction &MF = DAG.getMachineFunction();
-    MMO = MF.getMachineMemOperand(
-        MMO, MMO->getPointerInfo(),
-        MMO->getMemoryType().isValid()
-            ? LLT::scalable_vector(1, MMO->getMemoryType().getSizeInBits())
-            : MMO->getMemoryType());
     SDValue NewLoad =
-        DAG.getLoad(ContainerVT, DL, Load->getChain(), Load->getBasePtr(), MMO);
+        DAG.getLoad(ContainerVT, DL, Load->getChain(), Load->getBasePtr(),
+                    MMO->getPointerInfo(), MMO->getBaseAlign(), MMO->getFlags(),
+                    MMO->getAAInfo(), MMO->getRanges());
     SDValue Result = convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
     return DAG.getMergeValues({Result, NewLoad.getValue(1)}, DL);
   }
@@ -10500,14 +10496,9 @@ RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(SDValue Op,
   if (MinVLMAX == MaxVLMAX && MinVLMAX == VT.getVectorNumElements() &&
       getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
     MachineMemOperand *MMO = Store->getMemOperand();
-    MachineFunction &MF = DAG.getMachineFunction();
-    MMO = MF.getMachineMemOperand(
-        MMO, MMO->getPointerInfo(),
-        MMO->getMemoryType().isValid()
-            ? LLT::scalable_vector(1, MMO->getMemoryType().getSizeInBits())
-            : MMO->getMemoryType());
     return DAG.getStore(Store->getChain(), DL, NewValue, Store->getBasePtr(),
-                        MMO);
+                        MMO->getPointerInfo(), MMO->getBaseAlign(),
+                        MMO->getFlags(), MMO->getAAInfo());
   }
 
   SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG,
diff --git a/llvm/test/CodeGen/RISCV/rvv/pr88799.ll b/llvm/test/CodeGen/RISCV/rvv/pr88799.ll
new file mode 100644
index 000000000000..7212a789f9e7
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/pr88799.ll
@@ -0,0 +1,19 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc < %s -mtriple=riscv64-unknown-linux-gnu -mattr=+v | FileCheck %s
+
+define i32 @main() vscale_range(2,2) {
+; CHECK-LABEL: main:
+; CHECK:       # %bb.0: # %vector.body
+; CHECK-NEXT:    lui a0, 1040368
+; CHECK-NEXT:    addiw a0, a0, -144
+; CHECK-NEXT:    vl2re16.v v8, (a0)
+; CHECK-NEXT:    vs2r.v v8, (zero)
+; CHECK-NEXT:    li a0, 0
+; CHECK-NEXT:    ret
+vector.body:
+  %0 = load <16 x i16>, ptr getelementptr ([3 x [23 x [23 x i16]]], ptr null, i64 -10593, i64 1, i64 22, i64 0), align 16
+  store <16 x i16> %0, ptr null, align 2
+  %wide.load = load <vscale x 8 x i16>, ptr getelementptr ([3 x [23 x [23 x i16]]], ptr null, i64 -10593, i64 1, i64 22, i64 0), align 16
+  store <vscale x 8 x i16> %wide.load, ptr null, align 2
+  ret i32 0
+}
-- 
GitLab


From 6da1966bc503e1ce44ef36e7107c9db482fac6ab Mon Sep 17 00:00:00 2001
From: Michal Terepeta <michalt@google.com>
Date: Tue, 16 Apr 2024 07:55:45 +0200
Subject: [PATCH 339/464] [RISCV] Add scheduling information for SiFive VCIX
 (#86093)

This adds `RISCVScheduleXSf.td` with `SchedWrite` definitions for all
VCIX instructions and uses it in `RISCVSchedSiFive7.td` to set default
latencies for these instructions, helping with issue
https://github.com/llvm/llvm-project/issues/83391. Of course these
default latencies cannot be accurate (since each coprocessor will have
different latencies), but this seems to be enough to avoid some of the
problematic behavior described in the bug.

In any case, this seems to be enough to help with #83391 in our internal
testing.

A subsequent discussion is how to structure the code such that it's
easier for downstream consumers of this to use `SiFive7` scheduling
model with accurate VCIX latencies. But we can probably have a separate
issue to discuss that.
---
 llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td    | 48 +++++++++++----
 llvm/lib/Target/RISCV/RISCVSchedRocket.td     |  1 +
 llvm/lib/Target/RISCV/RISCVSchedSiFive7.td    | 48 +++++++++++++++
 llvm/lib/Target/RISCV/RISCVSchedSiFiveP400.td |  1 +
 llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td |  1 +
 .../Target/RISCV/RISCVSchedSyntacoreSCR1.td   |  1 +
 .../Target/RISCV/RISCVSchedXiangShanNanHu.td  |  1 +
 llvm/lib/Target/RISCV/RISCVSchedule.td        |  1 +
 llvm/lib/Target/RISCV/RISCVScheduleXSf.td     | 59 +++++++++++++++++++
 9 files changed, 149 insertions(+), 12 deletions(-)
 create mode 100644 llvm/lib/Target/RISCV/RISCVScheduleXSf.td

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td
index 9a6818c99af2..71aa1f19e089 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXSf.td
@@ -307,10 +307,16 @@ multiclass VPseudoVC_X<LMULInfo m, DAGOperand RS1Class,
                        Operand OpClass = payload2> {
   let VLMul = m.value in {
     let Defs = [VCIX_STATE], Uses = [VCIX_STATE] in {
-      def "PseudoVC_" # NAME # "_SE_" # m.MX : VPseudoVC_X<OpClass, RS1Class>;
-      def "PseudoVC_V_" # NAME # "_SE_" # m.MX : VPseudoVC_V_X<OpClass, m.vrclass, RS1Class>;
+      def "PseudoVC_" # NAME # "_SE_" # m.MX
+        : VPseudoVC_X<OpClass, RS1Class>,
+          Sched<[!cast<SchedWrite>("WriteVC_" # NAME # "_" # m.MX)]>;
+      def "PseudoVC_V_" # NAME # "_SE_" # m.MX
+        : VPseudoVC_V_X<OpClass, m.vrclass, RS1Class>,
+          Sched<[!cast<SchedWrite>("WriteVC_V_" # NAME # "_" # m.MX)]>;
     }
-    def "PseudoVC_V_" # NAME # "_" # m.MX : VPseudoVC_V_X<OpClass, m.vrclass, RS1Class>;
+    def "PseudoVC_V_" # NAME # "_" # m.MX
+      : VPseudoVC_V_X<OpClass, m.vrclass, RS1Class>,
+        Sched<[!cast<SchedWrite>("WriteVC_V_" # NAME # "_" # m.MX)]>;
   }
 }
 
@@ -318,10 +324,16 @@ multiclass VPseudoVC_XV<LMULInfo m, DAGOperand RS1Class,
                         Operand OpClass = payload2> {
   let VLMul = m.value in {
     let Defs = [VCIX_STATE], Uses = [VCIX_STATE] in {
-      def "PseudoVC_" # NAME # "_SE_" # m.MX : VPseudoVC_XV<OpClass, m.vrclass, RS1Class>;
-      def "PseudoVC_V_" # NAME # "_SE_" # m.MX : VPseudoVC_V_XV<OpClass, m.vrclass, m.vrclass, RS1Class>;
+      def "PseudoVC_" # NAME # "_SE_" # m.MX
+        : VPseudoVC_XV<OpClass, m.vrclass, RS1Class>,
+          Sched<[!cast<SchedWrite>("WriteVC_" # NAME # "_" # m.MX)]>;
+      def "PseudoVC_V_" # NAME # "_SE_" # m.MX
+        : VPseudoVC_V_XV<OpClass, m.vrclass, m.vrclass, RS1Class>,
+          Sched<[!cast<SchedWrite>("WriteVC_V_" # NAME # "_" # m.MX)]>;
     }
-    def "PseudoVC_V_" # NAME # "_" # m.MX : VPseudoVC_V_XV<OpClass, m.vrclass, m.vrclass, RS1Class>;
+    def "PseudoVC_V_" # NAME # "_" # m.MX
+      : VPseudoVC_V_XV<OpClass, m.vrclass, m.vrclass, RS1Class>,
+        Sched<[!cast<SchedWrite>("WriteVC_V_" # NAME # "_" # m.MX)]>;
   }
 }
 
@@ -329,10 +341,16 @@ multiclass VPseudoVC_XVV<LMULInfo m, DAGOperand RS1Class,
                          Operand OpClass = payload2> {
   let VLMul = m.value in {
     let Defs = [VCIX_STATE], Uses = [VCIX_STATE] in {
-      def "PseudoVC_" # NAME # "_SE_" # m.MX : VPseudoVC_XVV<OpClass, m.vrclass, m.vrclass, RS1Class>;
-      def "PseudoVC_V_" # NAME # "_SE_" # m.MX : VPseudoVC_V_XVV<OpClass, m.vrclass, m.vrclass, RS1Class>;
+      def "PseudoVC_" # NAME # "_SE_" # m.MX
+        : VPseudoVC_XVV<OpClass, m.vrclass, m.vrclass, RS1Class>,
+          Sched<[!cast<SchedWrite>("WriteVC_" # NAME # "_" # m.MX)]>;
+      def "PseudoVC_V_" # NAME # "_SE_" # m.MX
+        : VPseudoVC_V_XVV<OpClass, m.vrclass, m.vrclass, RS1Class>,
+          Sched<[!cast<SchedWrite>("WriteVC_V_" # NAME # "_" # m.MX)]>;
     }
-    def "PseudoVC_V_" # NAME # "_" # m.MX : VPseudoVC_V_XVV<OpClass, m.vrclass, m.vrclass, RS1Class>;
+    def "PseudoVC_V_" # NAME # "_" # m.MX
+      : VPseudoVC_V_XVV<OpClass, m.vrclass, m.vrclass, RS1Class>,
+        Sched<[!cast<SchedWrite>("WriteVC_V_" # NAME # "_" # m.MX)]>;
   }
 }
 
@@ -340,11 +358,17 @@ multiclass VPseudoVC_XVW<LMULInfo m, DAGOperand RS1Class,
                          Operand OpClass = payload2> {
   let VLMul = m.value in {
     let Defs = [VCIX_STATE], Uses = [VCIX_STATE] in
-    def "PseudoVC_" # NAME # "_SE_" # m.MX : VPseudoVC_XVV<OpClass, m.wvrclass, m.vrclass, RS1Class>;
+    def "PseudoVC_" # NAME # "_SE_" # m.MX
+      : VPseudoVC_XVV<OpClass, m.wvrclass, m.vrclass, RS1Class>,
+        Sched<[!cast<SchedWrite>("WriteVC_" # NAME # "_" # m.MX)]>;
     let Constraints = "@earlyclobber $rd, $rd = $rs3" in {
       let Defs = [VCIX_STATE], Uses = [VCIX_STATE] in
-      def "PseudoVC_V_" # NAME # "_SE_" # m.MX : VPseudoVC_V_XVV<OpClass, m.wvrclass, m.vrclass, RS1Class>;
-      def "PseudoVC_V_" # NAME # "_" # m.MX : VPseudoVC_V_XVV<OpClass, m.wvrclass, m.vrclass, RS1Class>;
+      def "PseudoVC_V_" # NAME # "_SE_" # m.MX
+        : VPseudoVC_V_XVV<OpClass, m.wvrclass, m.vrclass, RS1Class>,
+          Sched<[!cast<SchedWrite>("WriteVC_V_" # NAME # "_" # m.MX)]>;
+      def "PseudoVC_V_" # NAME # "_" # m.MX
+        : VPseudoVC_V_XVV<OpClass, m.wvrclass, m.vrclass, RS1Class>,
+          Sched<[!cast<SchedWrite>("WriteVC_V_" # NAME # "_" # m.MX)]>;
     }
   }
 }
diff --git a/llvm/lib/Target/RISCV/RISCVSchedRocket.td b/llvm/lib/Target/RISCV/RISCVSchedRocket.td
index e74c7aab7474..65494e73758d 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedRocket.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedRocket.td
@@ -261,4 +261,5 @@ defm : UnsupportedSchedZbkx;
 defm : UnsupportedSchedZfa;
 defm : UnsupportedSchedZfh;
 defm : UnsupportedSchedSFB;
+defm : UnsupportedSchedXsfvcp;
 }
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
index 4dcec96df9d5..a532066b3a1c 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFive7.td
@@ -962,6 +962,54 @@ let Latency = 3 in
 
 def : InstRW<[WriteIALU], (instrs COPY)>;
 
+// VCIX
+//
+// In principle we don't know the latency of any VCIX instructions. But instead
+// of taking the default of 1, which can lead to issues [1], we assume that they
+// have a fairly high latency.
+//
+// [1] https://github.com/llvm/llvm-project/issues/83391
+foreach mx = SchedMxList in {
+  defvar Cycles = SiFive7GetCyclesDefault<mx>.c;
+  defvar IsWorstCase = SiFive7IsWorstCaseMX<mx, SchedMxList>.c;
+  let Latency = !mul(Cycles, 10),
+      AcquireAtCycles = [0, 1],
+      ReleaseAtCycles = [1, !add(1, Cycles)] in {
+    defm "" : LMULWriteResMX<"WriteVC_V_I",   [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVC_V_X",   [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVC_V_IV",  [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVC_V_VV",  [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVC_V_XV",  [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVC_V_IVV", [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVC_V_IVW", [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVC_V_VVV", [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVC_V_VVW", [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVC_V_XVV", [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVC_V_XVW", [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    foreach f = ["FPR16", "FPR32", "FPR64"] in {
+      defm "" : LMULWriteResMX<"WriteVC_V_" # f # "V",  [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+      defm "" : LMULWriteResMX<"WriteVC_V_" # f # "VV", [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+      defm "" : LMULWriteResMX<"WriteVC_V_" # f # "VW", [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    }
+    defm "" : LMULWriteResMX<"WriteVC_I",   [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVC_X",   [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVC_IV",  [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVC_VV",  [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVC_XV",  [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVC_IVV", [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVC_IVW", [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVC_VVV", [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVC_VVW", [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVC_XVV", [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    defm "" : LMULWriteResMX<"WriteVC_XVW", [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    foreach f = ["FPR16", "FPR32", "FPR64"] in {
+      defm "" : LMULWriteResMX<"WriteVC_" # f # "V",  [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+      defm "" : LMULWriteResMX<"WriteVC_" # f # "VV", [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+      defm "" : LMULWriteResMX<"WriteVC_" # f # "VW", [SiFive7VCQ, SiFive7VA], mx, IsWorstCase>;
+    }
+  }
+}
+
 //===----------------------------------------------------------------------===//
 
 // Bypass and advance
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP400.td b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP400.td
index 8ec2e4ff885e..fccdd7e4f3ec 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP400.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP400.td
@@ -366,4 +366,5 @@ defm : UnsupportedSchedZbkx;
 defm : UnsupportedSchedSFB;
 defm : UnsupportedSchedZfa;
 defm : UnsupportedSchedV;
+defm : UnsupportedSchedXsfvcp;
 }
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
index 0ced2efa3f7a..6e4fb19361f5 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSiFiveP600.td
@@ -1040,4 +1040,5 @@ defm : UnsupportedSchedZbkb;
 defm : UnsupportedSchedZbkx;
 defm : UnsupportedSchedSFB;
 defm : UnsupportedSchedZfa;
+defm : UnsupportedSchedXsfvcp;
 }
diff --git a/llvm/lib/Target/RISCV/RISCVSchedSyntacoreSCR1.td b/llvm/lib/Target/RISCV/RISCVSchedSyntacoreSCR1.td
index 9625d17e0b26..0885e325f24e 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSyntacoreSCR1.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSyntacoreSCR1.td
@@ -212,4 +212,5 @@ defm : UnsupportedSchedZbkb;
 defm : UnsupportedSchedZbkx;
 defm : UnsupportedSchedZfa;
 defm : UnsupportedSchedZfh;
+defm : UnsupportedSchedXsfvcp;
 }
diff --git a/llvm/lib/Target/RISCV/RISCVSchedXiangShanNanHu.td b/llvm/lib/Target/RISCV/RISCVSchedXiangShanNanHu.td
index 4fc7b0335af5..e0f1fab1d6b4 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedXiangShanNanHu.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedXiangShanNanHu.td
@@ -311,4 +311,5 @@ defm : UnsupportedSchedZfa;
 defm : UnsupportedSchedZfh;
 defm : UnsupportedSchedSFB;
 defm : UnsupportedSchedZabha;
+defm : UnsupportedSchedXsfvcp;
 }
diff --git a/llvm/lib/Target/RISCV/RISCVSchedule.td b/llvm/lib/Target/RISCV/RISCVSchedule.td
index 1d19624342d2..0086557a41fe 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedule.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedule.td
@@ -296,3 +296,4 @@ def : ReadAdvance<ReadAtomicHD, 0>;
 // Include the scheduler resources for other instruction extensions.
 include "RISCVScheduleZb.td"
 include "RISCVScheduleV.td"
+include "RISCVScheduleXSf.td"
diff --git a/llvm/lib/Target/RISCV/RISCVScheduleXSf.td b/llvm/lib/Target/RISCV/RISCVScheduleXSf.td
new file mode 100644
index 000000000000..58d508460f01
--- /dev/null
+++ b/llvm/lib/Target/RISCV/RISCVScheduleXSf.td
@@ -0,0 +1,59 @@
+//===-- RISCVScheduleXSf.td - Scheduling Definitions XSf ---*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file describes the scheduling information for SiFive extensions.
+//
+//===----------------------------------------------------------------------===//
+
+multiclass LMULSchedWritesVCIX<string id>{
+defm "" : LMULSchedWrites<"WriteVC_" # id>;
+defm "" : LMULSchedWrites<"WriteVC_V_" # id>;
+}
+
+defm "" : LMULSchedWritesVCIX<"I">;
+defm "" : LMULSchedWritesVCIX<"X">;
+defm "" : LMULSchedWritesVCIX<"IV">;
+defm "" : LMULSchedWritesVCIX<"VV">;
+defm "" : LMULSchedWritesVCIX<"XV">;
+defm "" : LMULSchedWritesVCIX<"IVV">;
+defm "" : LMULSchedWritesVCIX<"IVW">;
+defm "" : LMULSchedWritesVCIX<"VVV">;
+defm "" : LMULSchedWritesVCIX<"VVW">;
+defm "" : LMULSchedWritesVCIX<"XVV">;
+defm "" : LMULSchedWritesVCIX<"XVW">;
+foreach f = ["FPR16", "FPR32", "FPR64"] in {
+  defm "" : LMULSchedWritesVCIX<f # "V">;
+  defm "" : LMULSchedWritesVCIX<f # "VV">;
+  defm "" : LMULSchedWritesVCIX<f # "VW">;
+}
+
+multiclass LMULWriteResVCIX<string id, list<ProcResourceKind> resources>{
+defm : LMULWriteRes<"WriteVC_" # id, resources>;
+defm : LMULWriteRes<"WriteVC_V_" # id, resources>;
+}
+
+multiclass UnsupportedSchedXsfvcp {
+let Unsupported = true in {
+defm : LMULWriteResVCIX<"I", []>;
+defm : LMULWriteResVCIX<"X", []>;
+defm : LMULWriteResVCIX<"IV", []>;
+defm : LMULWriteResVCIX<"VV", []>;
+defm : LMULWriteResVCIX<"XV", []>;
+defm : LMULWriteResVCIX<"IVV", []>;
+defm : LMULWriteResVCIX<"IVW", []>;
+defm : LMULWriteResVCIX<"VVV", []>;
+defm : LMULWriteResVCIX<"VVW", []>;
+defm : LMULWriteResVCIX<"XVV", []>;
+defm : LMULWriteResVCIX<"XVW", []>;
+foreach f = ["FPR16", "FPR32", "FPR64"] in {
+  defm : LMULWriteResVCIX<f # "V", []>;
+  defm : LMULWriteResVCIX<f # "VV", []>;
+  defm : LMULWriteResVCIX<f # "VW", []>;
+}
+}
+}
-- 
GitLab


From e2c91091e537a54f1469610ab407e7c9561e7ffc Mon Sep 17 00:00:00 2001
From: darkbuck <michael.hliao@gmail.com>
Date: Tue, 16 Apr 2024 02:36:12 -0400
Subject: [PATCH 340/464] [GlobalISel] Handle more commutable instructions in
 `commute_constant_to_rhs`

Reviewers:
rupprecht, aartbik, cyndyishida, Pierre-vh, aemerson, ftynse, hanhanW, banach-space, jayfoad, nicolasvasilache, daniel-grumberg, arsenm, PeimingLiu, JDevlieghere, matthias-springer

Reviewed By: arsenm, Pierre-vh

Pull Request: https://github.com/llvm/llvm-project/pull/87424
---
 .../include/llvm/Target/GlobalISel/Combine.td |  10 +-
 .../lib/CodeGen/GlobalISel/CombinerHelper.cpp |  38 +-
 .../combine-commute-fp-const-lhs.mir          | 126 +++++
 .../combine-commute-int-const-lhs.mir         | 456 ++++++++++++++++++
 .../combine-const-fold-barrier-rhs.mir        | 160 ++++++
 ...relegalizer-combiner-select-to-fminmax.mir |  16 +-
 .../AArch64/GlobalISel/select-to-fmin-fmax.ll |  16 +-
 .../GlobalISel/combine-fcanonicalize.mir      |   6 +-
 .../GlobalISel/postlegalizercombiner-and.mir  |   2 +-
 llvm/test/CodeGen/AMDGPU/GlobalISel/smed3.ll  |   4 +-
 llvm/test/CodeGen/AMDGPU/GlobalISel/umed3.ll  |   4 +-
 .../CodeGen/AMDGPU/GlobalISel/urem.i64.ll     |  18 +-
 12 files changed, 815 insertions(+), 41 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/combine-commute-int-const-lhs.mir

diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index 778ff7e437eb..8568a7ae90e5 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -443,14 +443,20 @@ def select_constant_cmp: GICombineRule<
 // TODO: handle compares (currently not marked as isCommutable)
 def commute_int_constant_to_rhs : GICombineRule<
   (defs root:$root),
-  (match (wip_match_opcode G_ADD, G_MUL, G_AND, G_OR, G_XOR):$root,
+  (match (wip_match_opcode G_ADD, G_MUL, G_AND, G_OR, G_XOR,
+                           G_SMIN, G_SMAX, G_UMIN, G_UMAX, G_UADDO, G_SADDO,
+                           G_UMULO, G_SMULO, G_UMULH, G_SMULH,
+                           G_UADDSAT, G_SADDSAT, G_SMULFIX, G_UMULFIX,
+                           G_SMULFIXSAT, G_UMULFIXSAT):$root,
     [{ return Helper.matchCommuteConstantToRHS(*${root}); }]),
   (apply [{ Helper.applyCommuteBinOpOperands(*${root}); }])
 >;
 
 def commute_fp_constant_to_rhs : GICombineRule<
   (defs root:$root),
-  (match (wip_match_opcode G_FADD, G_FMUL):$root,
+  (match (wip_match_opcode G_FADD, G_FMUL, G_FMINNUM, G_FMAXNUM,
+                           G_FMINNUM_IEEE, G_FMAXNUM_IEEE,
+                           G_FMINIMUM, G_FMAXIMUM):$root,
     [{ return Helper.matchCommuteFPConstantToRHS(*${root}); }]),
   (apply [{ Helper.applyCommuteBinOpOperands(*${root}); }])
 >;
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 40c5119ee7fb..3829c33369b2 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -6273,8 +6273,21 @@ bool CombinerHelper::matchShiftsTooBig(MachineInstr &MI) {
 }
 
 bool CombinerHelper::matchCommuteConstantToRHS(MachineInstr &MI) {
-  Register LHS = MI.getOperand(1).getReg();
-  Register RHS = MI.getOperand(2).getReg();
+  unsigned LHSOpndIdx = 1;
+  unsigned RHSOpndIdx = 2;
+  switch (MI.getOpcode()) {
+  case TargetOpcode::G_UADDO:
+  case TargetOpcode::G_SADDO:
+  case TargetOpcode::G_UMULO:
+  case TargetOpcode::G_SMULO:
+    LHSOpndIdx = 2;
+    RHSOpndIdx = 3;
+    break;
+  default:
+    break;
+  }
+  Register LHS = MI.getOperand(LHSOpndIdx).getReg();
+  Register RHS = MI.getOperand(RHSOpndIdx).getReg();
   if (!getIConstantVRegVal(LHS, MRI)) {
     // Skip commuting if LHS is not a constant. But, LHS may be a
     // G_CONSTANT_FOLD_BARRIER. If so we commute as long as we don't already
@@ -6300,10 +6313,23 @@ bool CombinerHelper::matchCommuteFPConstantToRHS(MachineInstr &MI) {
 
 void CombinerHelper::applyCommuteBinOpOperands(MachineInstr &MI) {
   Observer.changingInstr(MI);
-  Register LHSReg = MI.getOperand(1).getReg();
-  Register RHSReg = MI.getOperand(2).getReg();
-  MI.getOperand(1).setReg(RHSReg);
-  MI.getOperand(2).setReg(LHSReg);
+  unsigned LHSOpndIdx = 1;
+  unsigned RHSOpndIdx = 2;
+  switch (MI.getOpcode()) {
+  case TargetOpcode::G_UADDO:
+  case TargetOpcode::G_SADDO:
+  case TargetOpcode::G_UMULO:
+  case TargetOpcode::G_SMULO:
+    LHSOpndIdx = 2;
+    RHSOpndIdx = 3;
+    break;
+  default:
+    break;
+  }
+  Register LHSReg = MI.getOperand(LHSOpndIdx).getReg();
+  Register RHSReg = MI.getOperand(RHSOpndIdx).getReg();
+  MI.getOperand(LHSOpndIdx).setReg(RHSReg);
+  MI.getOperand(RHSOpndIdx).setReg(LHSReg);
   Observer.changedInstr(MI);
 }
 
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-commute-fp-const-lhs.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-commute-fp-const-lhs.mir
index 76d82884a7b1..d791660b7a5e 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-commute-fp-const-lhs.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-commute-fp-const-lhs.mir
@@ -116,3 +116,129 @@ body:             |
     $q0 = COPY %mul
     RET_ReallyLR
 ...
+---
+name:            fminnum
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $s0
+
+    ; CHECK-LABEL: name: fminnum
+    ; CHECK: liveins: $s0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: %cst:_(s32) = G_FCONSTANT float 2.000000e+00
+    ; CHECK-NEXT: %min:_(s32) = G_FMINNUM [[COPY]], %cst
+    ; CHECK-NEXT: $s0 = COPY %min(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $s0
+    %cst:_(s32) = G_FCONSTANT float 2.000000e+00
+    %min:_(s32) = G_FMINNUM %cst, %0
+    $s0 = COPY %min
+    RET_ReallyLR
+...
+---
+name:            fmaxnum
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $s0
+
+    ; CHECK-LABEL: name: fmaxnum
+    ; CHECK: liveins: $s0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: %cst:_(s32) = G_FCONSTANT float 2.000000e+00
+    ; CHECK-NEXT: %max:_(s32) = G_FMAXNUM [[COPY]], %cst
+    ; CHECK-NEXT: $s0 = COPY %max(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $s0
+    %cst:_(s32) = G_FCONSTANT float 2.000000e+00
+    %max:_(s32) = G_FMAXNUM %cst, %0
+    $s0 = COPY %max
+    RET_ReallyLR
+...
+---
+name:            fminnum_ieee
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $s0
+
+    ; CHECK-LABEL: name: fminnum_ieee
+    ; CHECK: liveins: $s0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: %cst:_(s32) = G_FCONSTANT float 2.000000e+00
+    ; CHECK-NEXT: %min:_(s32) = G_FMINNUM_IEEE [[COPY]], %cst
+    ; CHECK-NEXT: $s0 = COPY %min(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $s0
+    %cst:_(s32) = G_FCONSTANT float 2.000000e+00
+    %min:_(s32) = G_FMINNUM_IEEE %cst, %0
+    $s0 = COPY %min
+    RET_ReallyLR
+...
+---
+name:            fmaxnum_ieee
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $s0
+
+    ; CHECK-LABEL: name: fmaxnum_ieee
+    ; CHECK: liveins: $s0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: %cst:_(s32) = G_FCONSTANT float 2.000000e+00
+    ; CHECK-NEXT: %max:_(s32) = G_FMAXNUM_IEEE [[COPY]], %cst
+    ; CHECK-NEXT: $s0 = COPY %max(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $s0
+    %cst:_(s32) = G_FCONSTANT float 2.000000e+00
+    %max:_(s32) = G_FMAXNUM_IEEE %cst, %0
+    $s0 = COPY %max
+    RET_ReallyLR
+...
+---
+name:            fminimum
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $s0
+
+    ; CHECK-LABEL: name: fminimum
+    ; CHECK: liveins: $s0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: %cst:_(s32) = G_FCONSTANT float 2.000000e+00
+    ; CHECK-NEXT: %min:_(s32) = G_FMINIMUM [[COPY]], %cst
+    ; CHECK-NEXT: $s0 = COPY %min(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $s0
+    %cst:_(s32) = G_FCONSTANT float 2.000000e+00
+    %min:_(s32) = G_FMINIMUM %cst, %0
+    $s0 = COPY %min
+    RET_ReallyLR
+...
+---
+name:            fmaximum
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $s0
+
+    ; CHECK-LABEL: name: fmaximum
+    ; CHECK: liveins: $s0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: %cst:_(s32) = G_FCONSTANT float 2.000000e+00
+    ; CHECK-NEXT: %max:_(s32) = G_FMAXIMUM [[COPY]], %cst
+    ; CHECK-NEXT: $s0 = COPY %max(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $s0
+    %cst:_(s32) = G_FCONSTANT float 2.000000e+00
+    %max:_(s32) = G_FMAXIMUM %cst, %0
+    $s0 = COPY %max
+    RET_ReallyLR
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-commute-int-const-lhs.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-commute-int-const-lhs.mir
new file mode 100644
index 000000000000..16365494f5f4
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-commute-int-const-lhs.mir
@@ -0,0 +1,456 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 2
+# RUN: llc -mtriple aarch64 -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+---
+name:            add
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $s0
+
+    ; CHECK-LABEL: name: add
+    ; CHECK: liveins: $s0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: %cst:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: %add:_(s32) = G_ADD [[COPY]], %cst
+    ; CHECK-NEXT: $s0 = COPY %add(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $s0
+    %cst:_(s32) = G_CONSTANT i32 1
+    %add:_(s32) = G_ADD %cst, %0
+    $s0 = COPY %add
+    RET_ReallyLR
+
+...
+---
+name:            mul
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $s0
+
+    ; CHECK-LABEL: name: mul
+    ; CHECK: liveins: $s0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: %cst:_(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: %mul:_(s32) = G_MUL [[COPY]], %cst
+    ; CHECK-NEXT: $s0 = COPY %mul(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $s0
+    %cst:_(s32) = G_CONSTANT i32 3
+    %mul:_(s32) = G_MUL %cst, %0
+    $s0 = COPY %mul
+    RET_ReallyLR
+...
+---
+name:            and
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $s0
+
+    ; CHECK-LABEL: name: and
+    ; CHECK: liveins: $s0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: %cst:_(s32) = G_CONSTANT i32 5
+    ; CHECK-NEXT: %and:_(s32) = G_AND [[COPY]], %cst
+    ; CHECK-NEXT: $s0 = COPY %and(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $s0
+    %cst:_(s32) = G_CONSTANT i32 5
+    %and:_(s32) = G_AND %cst, %0
+    $s0 = COPY %and
+    RET_ReallyLR
+...
+---
+name:            or
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $s0
+
+    ; CHECK-LABEL: name: or
+    ; CHECK: liveins: $s0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: %cst:_(s32) = G_CONSTANT i32 5
+    ; CHECK-NEXT: %or:_(s32) = G_OR [[COPY]], %cst
+    ; CHECK-NEXT: $s0 = COPY %or(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $s0
+    %cst:_(s32) = G_CONSTANT i32 5
+    %or:_(s32) = G_OR %cst, %0
+    $s0 = COPY %or
+    RET_ReallyLR
+...
+---
+name:            xor
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $s0
+
+    ; CHECK-LABEL: name: xor
+    ; CHECK: liveins: $s0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: %cst:_(s32) = G_CONSTANT i32 5
+    ; CHECK-NEXT: %xor:_(s32) = G_XOR [[COPY]], %cst
+    ; CHECK-NEXT: $s0 = COPY %xor(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $s0
+    %cst:_(s32) = G_CONSTANT i32 5
+    %xor:_(s32) = G_XOR %cst, %0
+    $s0 = COPY %xor
+    RET_ReallyLR
+...
+---
+name:            smin
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $s0
+
+    ; CHECK-LABEL: name: smin
+    ; CHECK: liveins: $s0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: %cst:_(s32) = G_CONSTANT i32 10
+    ; CHECK-NEXT: %min:_(s32) = G_SMIN [[COPY]], %cst
+    ; CHECK-NEXT: $s0 = COPY %min(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $s0
+    %cst:_(s32) = G_CONSTANT i32 10
+    %min:_(s32) = G_SMIN %cst, %0
+    $s0 = COPY %min
+    RET_ReallyLR
+...
+---
+name:            smax
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $s0
+
+    ; CHECK-LABEL: name: smax
+    ; CHECK: liveins: $s0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: %cst:_(s32) = G_CONSTANT i32 10
+    ; CHECK-NEXT: %max:_(s32) = G_SMAX [[COPY]], %cst
+    ; CHECK-NEXT: $s0 = COPY %max(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $s0
+    %cst:_(s32) = G_CONSTANT i32 10
+    %max:_(s32) = G_SMAX %cst, %0
+    $s0 = COPY %max
+    RET_ReallyLR
+...
+---
+name:            umin
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $s0
+
+    ; CHECK-LABEL: name: umin
+    ; CHECK: liveins: $s0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: %cst:_(s32) = G_CONSTANT i32 10
+    ; CHECK-NEXT: %min:_(s32) = G_UMIN [[COPY]], %cst
+    ; CHECK-NEXT: $s0 = COPY %min(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $s0
+    %cst:_(s32) = G_CONSTANT i32 10
+    %min:_(s32) = G_UMIN %cst, %0
+    $s0 = COPY %min
+    RET_ReallyLR
+...
+---
+name:            umax
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $s0
+
+    ; CHECK-LABEL: name: umax
+    ; CHECK: liveins: $s0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: %cst:_(s32) = G_CONSTANT i32 10
+    ; CHECK-NEXT: %max:_(s32) = G_UMAX [[COPY]], %cst
+    ; CHECK-NEXT: $s0 = COPY %max(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $s0
+    %cst:_(s32) = G_CONSTANT i32 10
+    %max:_(s32) = G_UMAX %cst, %0
+    $s0 = COPY %max
+    RET_ReallyLR
+...
+---
+name:            uaddo
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $s0
+
+    ; CHECK-LABEL: name: uaddo
+    ; CHECK: liveins: $s0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: %cst:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: %add:_(s32), %overflow:_(s1) = G_UADDO [[COPY]], %cst
+    ; CHECK-NEXT: %ret:_(s32) = G_ANYEXT %overflow(s1)
+    ; CHECK-NEXT: $s0 = COPY %ret(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $s0
+    %cst:_(s32) = G_CONSTANT i32 1
+    %add:_(s32), %overflow:_(s1) = G_UADDO %cst, %0
+    %ret:_(s32) = G_ANYEXT %overflow
+    $s0 = COPY %ret
+    RET_ReallyLR
+
+...
+---
+name:            saddo
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $s0
+
+    ; CHECK-LABEL: name: saddo
+    ; CHECK: liveins: $s0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: %cst:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: %add:_(s32), %overflow:_(s1) = G_SADDO [[COPY]], %cst
+    ; CHECK-NEXT: %ret:_(s32) = G_ANYEXT %overflow(s1)
+    ; CHECK-NEXT: $s0 = COPY %ret(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $s0
+    %cst:_(s32) = G_CONSTANT i32 1
+    %add:_(s32), %overflow:_(s1) = G_SADDO %cst, %0
+    %ret:_(s32) = G_ANYEXT %overflow
+    $s0 = COPY %ret
+    RET_ReallyLR
+
+...
+---
+name:            umulo
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $s0
+
+    ; CHECK-LABEL: name: umulo
+    ; CHECK: liveins: $s0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: %cst:_(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: %mul:_(s32), %overflow:_(s1) = G_UMULO [[COPY]], %cst
+    ; CHECK-NEXT: %ret:_(s32) = G_ANYEXT %overflow(s1)
+    ; CHECK-NEXT: $s0 = COPY %ret(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $s0
+    %cst:_(s32) = G_CONSTANT i32 3
+    %mul:_(s32), %overflow:_(s1) = G_UMULO %cst, %0
+    %ret:_(s32) = G_ANYEXT %overflow
+    $s0 = COPY %ret
+    RET_ReallyLR
+...
+---
+name:            smulo
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $s0
+
+    ; CHECK-LABEL: name: smulo
+    ; CHECK: liveins: $s0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: %cst:_(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: %mul:_(s32), %overflow:_(s1) = G_SMULO [[COPY]], %cst
+    ; CHECK-NEXT: %ret:_(s32) = G_ANYEXT %overflow(s1)
+    ; CHECK-NEXT: $s0 = COPY %ret(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $s0
+    %cst:_(s32) = G_CONSTANT i32 3
+    %mul:_(s32), %overflow:_(s1) = G_SMULO %cst, %0
+    %ret:_(s32) = G_ANYEXT %overflow
+    $s0 = COPY %ret
+    RET_ReallyLR
+...
+---
+name:            umulh
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $s0
+
+    ; CHECK-LABEL: name: umulh
+    ; CHECK: liveins: $s0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: %cst:_(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: %mul:_(s32) = G_UMULH [[COPY]], %cst
+    ; CHECK-NEXT: $s0 = COPY %mul(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $s0
+    %cst:_(s32) = G_CONSTANT i32 3
+    %mul:_(s32) = G_UMULH %cst, %0
+    $s0 = COPY %mul
+    RET_ReallyLR
+...
+---
+name:            smulh
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $s0
+
+    ; CHECK-LABEL: name: smulh
+    ; CHECK: liveins: $s0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: %cst:_(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: %mul:_(s32) = G_UMULH [[COPY]], %cst
+    ; CHECK-NEXT: $s0 = COPY %mul(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $s0
+    %cst:_(s32) = G_CONSTANT i32 3
+    %mul:_(s32) = G_UMULH %cst, %0
+    $s0 = COPY %mul
+    RET_ReallyLR
+...
+---
+name:            uaddsat
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $s0
+
+    ; CHECK-LABEL: name: uaddsat
+    ; CHECK: liveins: $s0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: %cst:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: %add:_(s32) = G_UADDSAT [[COPY]], %cst
+    ; CHECK-NEXT: $s0 = COPY %add(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $s0
+    %cst:_(s32) = G_CONSTANT i32 1
+    %add:_(s32) = G_UADDSAT %cst, %0
+    $s0 = COPY %add
+    RET_ReallyLR
+
+...
+---
+name:            saddsat
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $s0
+
+    ; CHECK-LABEL: name: saddsat
+    ; CHECK: liveins: $s0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: %cst:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: %add:_(s32) = G_SADDSAT [[COPY]], %cst
+    ; CHECK-NEXT: $s0 = COPY %add(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $s0
+    %cst:_(s32) = G_CONSTANT i32 1
+    %add:_(s32) = G_SADDSAT %cst, %0
+    $s0 = COPY %add
+    RET_ReallyLR
+
+...
+---
+name:            smulfix
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $s0
+
+    ; CHECK-LABEL: name: smulfix
+    ; CHECK: liveins: $s0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: %cst:_(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: %mul:_(s32) = G_SMULFIX [[COPY]], %cst, 7
+    ; CHECK-NEXT: $s0 = COPY %mul(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $s0
+    %cst:_(s32) = G_CONSTANT i32 3
+    %mul:_(s32) = G_SMULFIX %cst, %0, 7
+    $s0 = COPY %mul
+    RET_ReallyLR
+...
+---
+name:            umulfix
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $s0
+
+    ; CHECK-LABEL: name: umulfix
+    ; CHECK: liveins: $s0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: %cst:_(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: %mul:_(s32) = G_UMULFIX [[COPY]], %cst, 7
+    ; CHECK-NEXT: $s0 = COPY %mul(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $s0
+    %cst:_(s32) = G_CONSTANT i32 3
+    %mul:_(s32) = G_UMULFIX %cst, %0, 7
+    $s0 = COPY %mul
+    RET_ReallyLR
+...
+---
+name:            smulfixsat
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $s0
+
+    ; CHECK-LABEL: name: smulfixsat
+    ; CHECK: liveins: $s0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: %cst:_(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: %mul:_(s32) = G_SMULFIXSAT [[COPY]], %cst, 7
+    ; CHECK-NEXT: $s0 = COPY %mul(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $s0
+    %cst:_(s32) = G_CONSTANT i32 3
+    %mul:_(s32) = G_SMULFIXSAT %cst, %0, 7
+    $s0 = COPY %mul
+    RET_ReallyLR
+...
+---
+name:            umulfixsat
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $s0
+
+    ; CHECK-LABEL: name: umulfixsat
+    ; CHECK: liveins: $s0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: %cst:_(s32) = G_CONSTANT i32 3
+    ; CHECK-NEXT: %mul:_(s32) = G_UMULFIXSAT [[COPY]], %cst, 7
+    ; CHECK-NEXT: $s0 = COPY %mul(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $s0
+    %cst:_(s32) = G_CONSTANT i32 3
+    %mul:_(s32) = G_UMULFIXSAT %cst, %0, 7
+    $s0 = COPY %mul
+    RET_ReallyLR
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-const-fold-barrier-rhs.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-const-fold-barrier-rhs.mir
index 01e0dce5a661..c967e4f2ea5e 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-const-fold-barrier-rhs.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-const-fold-barrier-rhs.mir
@@ -78,3 +78,163 @@ body:             |
     RET_ReallyLR
 
 ...
+---
+name:            cfb_lhs_smulo
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $w0
+
+    ; CHECK-LABEL: name: cfb_lhs_smulo
+    ; CHECK: liveins: $w0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+    ; CHECK-NEXT: %cst:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: %cfb:_(s32) = G_CONSTANT_FOLD_BARRIER %cst
+    ; CHECK-NEXT: %mul:_(s32), %overflow:_(s1) = G_SMULO [[COPY]], %cfb
+    ; CHECK-NEXT: %ret:_(s32) = G_ANYEXT %overflow(s1)
+    ; CHECK-NEXT: $w0 = COPY %ret(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $w0
+    %cst:_(s32) = G_CONSTANT i32 1
+    %cfb:_(s32) = G_CONSTANT_FOLD_BARRIER %cst
+    %mul:_(s32), %overflow:_(s1) = G_SMULO %cfb, %0
+    %ret:_(s32) = G_ANYEXT %overflow
+    $w0 = COPY %ret
+    RET_ReallyLR
+
+...
+---
+name:            cfb_lhs_cfb_already_rhs_smulo
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $w0
+
+    ; CHECK-LABEL: name: cfb_lhs_cfb_already_rhs_smulo
+    ; CHECK: liveins: $w0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %cst:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: %cfb:_(s32) = G_CONSTANT_FOLD_BARRIER %cst
+    ; CHECK-NEXT: %cst2:_(s32) = G_CONSTANT i32 6
+    ; CHECK-NEXT: %cfb2:_(s32) = G_CONSTANT_FOLD_BARRIER %cst2
+    ; CHECK-NEXT: %mul:_(s32), %overflow:_(s1) = G_SMULO %cfb, %cfb2
+    ; CHECK-NEXT: %ret:_(s32) = G_ANYEXT %overflow(s1)
+    ; CHECK-NEXT: $w0 = COPY %ret(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $w0
+    %cst:_(s32) = G_CONSTANT i32 1
+    %cfb:_(s32) = G_CONSTANT_FOLD_BARRIER %cst
+    %cst2:_(s32) = G_CONSTANT i32 6
+    %cfb2:_(s32) = G_CONSTANT_FOLD_BARRIER %cst2
+    %mul:_(s32), %overflow:_(s1) = G_SMULO %cfb, %cfb2
+    %ret:_(s32) = G_ANYEXT %overflow
+    $w0 = COPY %ret
+    RET_ReallyLR
+
+...
+---
+name:            cfb_lhs_cst_on_rhs_smulo
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $w0
+
+    ; CHECK-LABEL: name: cfb_lhs_cst_on_rhs_smulo
+    ; CHECK: liveins: $w0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %cst:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: %cfb:_(s32) = G_CONSTANT_FOLD_BARRIER %cst
+    ; CHECK-NEXT: %cst2:_(s32) = G_CONSTANT i32 6
+    ; CHECK-NEXT: %mul:_(s32), %overflow:_(s1) = G_SMULO %cfb, %cst2
+    ; CHECK-NEXT: %ret:_(s32) = G_ANYEXT %overflow(s1)
+    ; CHECK-NEXT: $w0 = COPY %ret(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $w0
+    %cst:_(s32) = G_CONSTANT i32 1
+    %cfb:_(s32) = G_CONSTANT_FOLD_BARRIER %cst
+    %cst2:_(s32) = G_CONSTANT i32 6
+    %mul:_(s32), %overflow:_(s1) = G_SMULO %cfb, %cst2
+    %ret:_(s32) = G_ANYEXT %overflow
+    $w0 = COPY %ret
+    RET_ReallyLR
+
+...
+---
+name:            cfb_lhs_umulfixsat
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $w0
+
+    ; CHECK-LABEL: name: cfb_lhs_umulfixsat
+    ; CHECK: liveins: $w0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+    ; CHECK-NEXT: %cst:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: %cfb:_(s32) = G_CONSTANT_FOLD_BARRIER %cst
+    ; CHECK-NEXT: %mul:_(s32) = G_UMULFIXSAT [[COPY]], %cfb, 7
+    ; CHECK-NEXT: $w0 = COPY %mul(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $w0
+    %cst:_(s32) = G_CONSTANT i32 1
+    %cfb:_(s32) = G_CONSTANT_FOLD_BARRIER %cst
+    %mul:_(s32) = G_UMULFIXSAT %cfb, %0, 7
+    $w0 = COPY %mul
+    RET_ReallyLR
+
+...
+---
+name:            cfb_lhs_cfb_already_rhs_umulfixsat
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $w0
+
+    ; CHECK-LABEL: name: cfb_lhs_cfb_already_rhs_umulfixsat
+    ; CHECK: liveins: $w0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %cst:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: %cfb:_(s32) = G_CONSTANT_FOLD_BARRIER %cst
+    ; CHECK-NEXT: %cst2:_(s32) = G_CONSTANT i32 2
+    ; CHECK-NEXT: %cfb2:_(s32) = G_CONSTANT_FOLD_BARRIER %cst2
+    ; CHECK-NEXT: %add:_(s32) = G_UMULFIXSAT %cfb, %cfb2, 7
+    ; CHECK-NEXT: $w0 = COPY %add(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $w0
+    %cst:_(s32) = G_CONSTANT i32 1
+    %cfb:_(s32) = G_CONSTANT_FOLD_BARRIER %cst
+    %cst2:_(s32) = G_CONSTANT i32 2
+    %cfb2:_(s32) = G_CONSTANT_FOLD_BARRIER %cst2
+    %add:_(s32) = G_UMULFIXSAT %cfb, %cfb2, 7
+    $w0 = COPY %add
+    RET_ReallyLR
+
+...
+---
+name:            cfb_lhs_cst_on_rhs_umulfixsat
+alignment:       4
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $w0
+
+    ; CHECK-LABEL: name: cfb_lhs_cst_on_rhs_umulfixsat
+    ; CHECK: liveins: $w0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: %cst:_(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: %cfb:_(s32) = G_CONSTANT_FOLD_BARRIER %cst
+    ; CHECK-NEXT: %cst2:_(s32) = G_CONSTANT i32 2
+    ; CHECK-NEXT: %add:_(s32) = G_UMULFIXSAT %cfb, %cst2, 7
+    ; CHECK-NEXT: $w0 = COPY %add(s32)
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(s32) = COPY $w0
+    %cst:_(s32) = G_CONSTANT i32 1
+    %cfb:_(s32) = G_CONSTANT_FOLD_BARRIER %cst
+    %cst2:_(s32) = G_CONSTANT i32 2
+    %add:_(s32) = G_UMULFIXSAT %cfb, %cst2, 7
+    $w0 = COPY %add
+    RET_ReallyLR
+
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizer-combiner-select-to-fminmax.mir b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizer-combiner-select-to-fminmax.mir
index 8c4300d9e732..03e507f5eaa7 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizer-combiner-select-to-fminmax.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/prelegalizer-combiner-select-to-fminmax.mir
@@ -11,7 +11,7 @@ body:             |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s16) = COPY $h0
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s16) = G_FCONSTANT half 0xH0000
-    ; CHECK-NEXT: [[FMAXIMUM:%[0-9]+]]:_(s16) = G_FMAXIMUM [[C]], [[COPY]]
+    ; CHECK-NEXT: [[FMAXIMUM:%[0-9]+]]:_(s16) = G_FMAXIMUM [[COPY]], [[C]]
     ; CHECK-NEXT: $h0 = COPY [[FMAXIMUM]](s16)
     ; CHECK-NEXT: RET_ReallyLR implicit $h0
     %0:_(s16) = COPY $h0
@@ -33,7 +33,7 @@ body:             |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0.000000e+00
-    ; CHECK-NEXT: [[FMAXIMUM:%[0-9]+]]:_(s32) = G_FMAXIMUM [[C]], [[COPY]]
+    ; CHECK-NEXT: [[FMAXIMUM:%[0-9]+]]:_(s32) = G_FMAXIMUM [[COPY]], [[C]]
     ; CHECK-NEXT: $s0 = COPY [[FMAXIMUM]](s32)
     ; CHECK-NEXT: RET_ReallyLR implicit $s0
     %0:_(s32) = COPY $s0
@@ -55,7 +55,7 @@ body:             |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $d0
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_FCONSTANT double 0.000000e+00
-    ; CHECK-NEXT: [[FMAXIMUM:%[0-9]+]]:_(s64) = G_FMAXIMUM [[C]], [[COPY]]
+    ; CHECK-NEXT: [[FMAXIMUM:%[0-9]+]]:_(s64) = G_FMAXIMUM [[COPY]], [[C]]
     ; CHECK-NEXT: $d0 = COPY [[FMAXIMUM]](s64)
     ; CHECK-NEXT: RET_ReallyLR implicit $d0
     %0:_(s64) = COPY $d0
@@ -77,7 +77,7 @@ body:             |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $d0
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_FCONSTANT double 0.000000e+00
-    ; CHECK-NEXT: [[FMINIMUM:%[0-9]+]]:_(s64) = G_FMINIMUM [[C]], [[COPY]]
+    ; CHECK-NEXT: [[FMINIMUM:%[0-9]+]]:_(s64) = G_FMINIMUM [[COPY]], [[C]]
     ; CHECK-NEXT: $d0 = COPY [[FMINIMUM]](s64)
     ; CHECK-NEXT: RET_ReallyLR implicit $d0
     %0:_(s64) = COPY $d0
@@ -100,7 +100,7 @@ body:             |
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<8 x s16>) = COPY $q0
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s16) = G_FCONSTANT half 0xH0000
     ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s16>) = G_BUILD_VECTOR [[C]](s16), [[C]](s16), [[C]](s16), [[C]](s16), [[C]](s16), [[C]](s16), [[C]](s16), [[C]](s16)
-    ; CHECK-NEXT: [[FMAXIMUM:%[0-9]+]]:_(<8 x s16>) = G_FMAXIMUM [[BUILD_VECTOR]], [[COPY]]
+    ; CHECK-NEXT: [[FMAXIMUM:%[0-9]+]]:_(<8 x s16>) = G_FMAXIMUM [[COPY]], [[BUILD_VECTOR]]
     ; CHECK-NEXT: $q0 = COPY [[FMAXIMUM]](<8 x s16>)
     ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %0:_(<8 x s16>) = COPY $q0
@@ -125,7 +125,7 @@ body:             |
     ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<4 x s32>) = G_BITCAST [[COPY]](<2 x s64>)
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 0.000000e+00
     ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C]](s32), [[C]](s32), [[C]](s32)
-    ; CHECK-NEXT: [[FMAXIMUM:%[0-9]+]]:_(<4 x s32>) = G_FMAXIMUM [[BUILD_VECTOR]], [[BITCAST]]
+    ; CHECK-NEXT: [[FMAXIMUM:%[0-9]+]]:_(<4 x s32>) = G_FMAXIMUM [[BITCAST]], [[BUILD_VECTOR]]
     ; CHECK-NEXT: $q0 = COPY [[FMAXIMUM]](<4 x s32>)
     ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %1:_(<2 x s64>) = COPY $q0
@@ -150,7 +150,7 @@ body:             |
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_FCONSTANT double 0.000000e+00
     ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
-    ; CHECK-NEXT: [[FMAXIMUM:%[0-9]+]]:_(<2 x s64>) = G_FMAXIMUM [[BUILD_VECTOR]], [[COPY]]
+    ; CHECK-NEXT: [[FMAXIMUM:%[0-9]+]]:_(<2 x s64>) = G_FMAXIMUM [[COPY]], [[BUILD_VECTOR]]
     ; CHECK-NEXT: $q0 = COPY [[FMAXIMUM]](<2 x s64>)
     ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %0:_(<2 x s64>) = COPY $q0
@@ -174,7 +174,7 @@ body:             |
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
     ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_FCONSTANT double 0.000000e+00
     ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C]](s64)
-    ; CHECK-NEXT: [[FMINIMUM:%[0-9]+]]:_(<2 x s64>) = G_FMINIMUM [[BUILD_VECTOR]], [[COPY]]
+    ; CHECK-NEXT: [[FMINIMUM:%[0-9]+]]:_(<2 x s64>) = G_FMINIMUM [[COPY]], [[BUILD_VECTOR]]
     ; CHECK-NEXT: $q0 = COPY [[FMINIMUM]](<2 x s64>)
     ; CHECK-NEXT: RET_ReallyLR implicit $q0
     %0:_(<2 x s64>) = COPY $q0
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-to-fmin-fmax.ll b/llvm/test/CodeGen/AArch64/GlobalISel/select-to-fmin-fmax.ll
index 7badf4732fd0..ae0a9b1c7c4f 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/select-to-fmin-fmax.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-to-fmin-fmax.ll
@@ -4,7 +4,7 @@ define half @test_s16(half %a) #0 {
 ; CHECK-LABEL: test_s16:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    movi d1, #0000000000000000
-; CHECK-NEXT:    fmax h0, h1, h0
+; CHECK-NEXT:    fmax h0, h0, h1
 ; CHECK-NEXT:    ret
 entry:
   %fcmp = fcmp olt half %a, 0.0
@@ -16,7 +16,7 @@ define float @test_s32(float %a) #0 {
 ; CHECK-LABEL: test_s32:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    movi d1, #0000000000000000
-; CHECK-NEXT:    fmax s0, s1, s0
+; CHECK-NEXT:    fmax s0, s0, s1
 ; CHECK-NEXT:    ret
 entry:
   %fcmp = fcmp olt float %a, 0.0
@@ -28,7 +28,7 @@ define double @test_s64(double %a) #0 {
 ; CHECK-LABEL: test_s64:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    movi d1, #0000000000000000
-; CHECK-NEXT:    fmax d0, d1, d0
+; CHECK-NEXT:    fmax d0, d0, d1
 ; CHECK-NEXT:    ret
 entry:
   %fcmp = fcmp olt double %a, 0.0
@@ -40,7 +40,7 @@ define <4 x half> @test_v4s16(<4 x half> %a) #0 {
 ; CHECK-LABEL: test_v4s16:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    fmax v0.4h, v1.4h, v0.4h
+; CHECK-NEXT:    fmax v0.4h, v0.4h, v1.4h
 ; CHECK-NEXT:    ret
 entry:
   %fcmp = fcmp olt <4 x half> %a, zeroinitializer
@@ -52,7 +52,7 @@ define <8 x half> @test_v8s16(<8 x half> %a) #0 {
 ; CHECK-LABEL: test_v8s16:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    fmax v0.8h, v1.8h, v0.8h
+; CHECK-NEXT:    fmax v0.8h, v0.8h, v1.8h
 ; CHECK-NEXT:    ret
 entry:
   %fcmp = fcmp olt <8 x half> %a, zeroinitializer
@@ -64,7 +64,7 @@ define <2 x float> @test_v2s32(<2 x float> %a) #0 {
 ; CHECK-LABEL: test_v2s32:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    fmax v0.2s, v1.2s, v0.2s
+; CHECK-NEXT:    fmax v0.2s, v0.2s, v1.2s
 ; CHECK-NEXT:    ret
 entry:
   %fcmp = fcmp olt <2 x float> %a, zeroinitializer
@@ -76,7 +76,7 @@ define <4 x float> @test_v4s32(<4 x float> %a) #0 {
 ; CHECK-LABEL: test_v4s32:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    fmax v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    fmax v0.4s, v0.4s, v1.4s
 ; CHECK-NEXT:    ret
 entry:
   %fcmp = fcmp olt <4 x float> %a, zeroinitializer
@@ -88,7 +88,7 @@ define <2 x double> @test_v2s64(<2 x double> %a) #0 {
 ; CHECK-LABEL: test_v2s64:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    movi v1.2d, #0000000000000000
-; CHECK-NEXT:    fmax v0.2d, v1.2d, v0.2d
+; CHECK-NEXT:    fmax v0.2d, v0.2d, v1.2d
 ; CHECK-NEXT:    ret
 entry:
   %fcmp = fcmp olt <2 x double> %a, zeroinitializer
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fcanonicalize.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fcanonicalize.mir
index ee0e83c5e076..020761352148 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fcanonicalize.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fcanonicalize.mir
@@ -254,8 +254,8 @@ body: |
     ; CHECK-NEXT: %one_s32:_(s32) = G_ANYEXT %one(s16)
     ; CHECK-NEXT: %one_undef:_(<2 x s16>) = G_BUILD_VECTOR_TRUNC %one_s32(s32), %undef(s32)
     ; CHECK-NEXT: [[FMUL:%[0-9]+]]:_(<2 x s16>) = G_FMUL [[COPY]], %two_splat
-    ; CHECK-NEXT: [[FMAXNUM_IEEE:%[0-9]+]]:_(<2 x s16>) = G_FMAXNUM_IEEE %zero_undef, [[FMUL]]
-    ; CHECK-NEXT: [[FMINNUM_IEEE:%[0-9]+]]:_(<2 x s16>) = G_FMINNUM_IEEE %one_undef, [[FMAXNUM_IEEE]]
+    ; CHECK-NEXT: [[FMAXNUM_IEEE:%[0-9]+]]:_(<2 x s16>) = G_FMAXNUM_IEEE [[FMUL]], %zero_undef
+    ; CHECK-NEXT: [[FMINNUM_IEEE:%[0-9]+]]:_(<2 x s16>) = G_FMINNUM_IEEE [[FMAXNUM_IEEE]], %one_undef
     ; CHECK-NEXT: $vgpr0 = COPY [[FMINNUM_IEEE]](<2 x s16>)
     %0:_(<2 x s16>) = COPY $vgpr0
     %two:_(s16) = G_FCONSTANT half 0xH4000
@@ -306,7 +306,7 @@ body: |
     ; CHECK-NEXT: [[FMUL:%[0-9]+]]:_(<2 x s16>) = G_FMUL [[COPY]], %two_splat
     ; CHECK-NEXT: %snan_undef_fcan:_(<2 x s16>) = G_FCANONICALIZE %snan_undef
     ; CHECK-NEXT: [[FMAXNUM_IEEE:%[0-9]+]]:_(<2 x s16>) = G_FMAXNUM_IEEE %snan_undef_fcan, [[FMUL]]
-    ; CHECK-NEXT: [[FMINNUM_IEEE:%[0-9]+]]:_(<2 x s16>) = G_FMINNUM_IEEE %qnan_undef, [[FMAXNUM_IEEE]]
+    ; CHECK-NEXT: [[FMINNUM_IEEE:%[0-9]+]]:_(<2 x s16>) = G_FMINNUM_IEEE [[FMAXNUM_IEEE]], %qnan_undef
     ; CHECK-NEXT: $vgpr0 = COPY [[FMINNUM_IEEE]](<2 x s16>)
     %0:_(<2 x s16>) = COPY $vgpr0
     %two:_(s16) = G_FCONSTANT half 0xH4000
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-and.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-and.mir
index d6321dae3aa7..67e6de1ce764 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-and.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/postlegalizercombiner-and.mir
@@ -318,7 +318,7 @@ body:             |
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: %val:_(s32) = COPY $vgpr4
     ; CHECK-NEXT: %k255:_(s32) = G_CONSTANT i32 255
-    ; CHECK-NEXT: %umin0:_(s32) = G_UMIN %k255, %val
+    ; CHECK-NEXT: %umin0:_(s32) = G_UMIN %val, %k255
     ; CHECK-NEXT: $vgpr0 = COPY %umin0(s32)
     %ptr0:_(p1) = COPY $vgpr0_vgpr1
     %ptr1:_(p1) = COPY $vgpr2_vgpr3
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/smed3.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/smed3.ll
index dc13dee4f148..1d94d76da148 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/smed3.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/smed3.ll
@@ -145,10 +145,10 @@ define <2 x i16> @test_max_K0min_K1Val__v2i16(<2 x i16> %a) {
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v2, 17
 ; GFX8-NEXT:    v_min_i16_e32 v1, 17, v0
-; GFX8-NEXT:    v_min_i16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_min_i16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; GFX8-NEXT:    v_mov_b32_e32 v2, -12
 ; GFX8-NEXT:    v_max_i16_e32 v1, -12, v1
-; GFX8-NEXT:    v_max_i16_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_max_i16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/umed3.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/umed3.ll
index 7e38762e7b55..a8233054db9b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/umed3.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/umed3.ll
@@ -145,10 +145,10 @@ define <2 x i16> @test_max_K0min_K1Val__v2u16(<2 x i16> %a) {
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v2, 17
 ; GFX8-NEXT:    v_min_u16_e32 v1, 17, v0
-; GFX8-NEXT:    v_min_u16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_min_u16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; GFX8-NEXT:    v_mov_b32_e32 v2, 12
 ; GFX8-NEXT:    v_max_u16_e32 v1, 12, v1
-; GFX8-NEXT:    v_max_u16_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT:    v_max_u16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
index 07480a0ce0c2..cc0f7e2ca5a5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/urem.i64.ll
@@ -983,7 +983,7 @@ define i64 @v_urem_i64_oddk_denom(i64 %num) {
 ; CHECK-NEXT:    v_cvt_u32_f32_e32 v3, v3
 ; CHECK-NEXT:    v_mul_lo_u32 v6, v4, v5
 ; CHECK-NEXT:    v_mul_lo_u32 v7, v3, v5
-; CHECK-NEXT:    v_mul_hi_u32 v8, v5, v3
+; CHECK-NEXT:    v_mul_hi_u32 v8, v3, v5
 ; CHECK-NEXT:    v_sub_i32_e32 v6, vcc, v6, v3
 ; CHECK-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
 ; CHECK-NEXT:    v_mul_lo_u32 v8, v4, v7
@@ -1010,7 +1010,7 @@ define i64 @v_urem_i64_oddk_denom(i64 %num) {
 ; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v3, v7
 ; CHECK-NEXT:    v_addc_u32_e32 v4, vcc, v4, v6, vcc
 ; CHECK-NEXT:    v_mul_lo_u32 v6, v3, v5
-; CHECK-NEXT:    v_mul_hi_u32 v7, v5, v3
+; CHECK-NEXT:    v_mul_hi_u32 v7, v3, v5
 ; CHECK-NEXT:    v_mul_lo_u32 v5, v4, v5
 ; CHECK-NEXT:    v_mul_lo_u32 v8, v4, v6
 ; CHECK-NEXT:    v_mul_hi_u32 v9, v3, v6
@@ -1058,7 +1058,7 @@ define i64 @v_urem_i64_oddk_denom(i64 %num) {
 ; CHECK-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
 ; CHECK-NEXT:    v_add_i32_e32 v5, vcc, v6, v5
 ; CHECK-NEXT:    v_mul_lo_u32 v6, v3, v2
-; CHECK-NEXT:    v_mul_hi_u32 v3, v2, v3
+; CHECK-NEXT:    v_mul_hi_u32 v3, v3, v2
 ; CHECK-NEXT:    v_add_i32_e32 v4, vcc, v4, v5
 ; CHECK-NEXT:    v_mul_lo_u32 v4, v4, v2
 ; CHECK-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
@@ -1265,10 +1265,10 @@ define <2 x i64> @v_urem_v2i64_oddk_denom(<2 x i64> %num) {
 ; GISEL-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
 ; GISEL-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
 ; GISEL-NEXT:    v_mul_lo_u32 v11, v9, v4
-; GISEL-NEXT:    v_mul_hi_u32 v9, v4, v9
+; GISEL-NEXT:    v_mul_hi_u32 v9, v9, v4
 ; GISEL-NEXT:    v_add_i32_e32 v8, vcc, v12, v8
 ; GISEL-NEXT:    v_mul_lo_u32 v12, v5, v4
-; GISEL-NEXT:    v_mul_hi_u32 v5, v4, v5
+; GISEL-NEXT:    v_mul_hi_u32 v5, v5, v4
 ; GISEL-NEXT:    v_add_i32_e32 v7, vcc, v7, v10
 ; GISEL-NEXT:    v_add_i32_e32 v6, vcc, v6, v8
 ; GISEL-NEXT:    v_mul_lo_u32 v7, v7, v4
@@ -1339,7 +1339,7 @@ define <2 x i64> @v_urem_v2i64_oddk_denom(<2 x i64> %num) {
 ; CGP-NEXT:    v_cvt_u32_f32_e32 v5, v5
 ; CGP-NEXT:    v_mul_lo_u32 v8, v6, v7
 ; CGP-NEXT:    v_mul_lo_u32 v9, v5, v7
-; CGP-NEXT:    v_mul_hi_u32 v10, v7, v5
+; CGP-NEXT:    v_mul_hi_u32 v10, v5, v7
 ; CGP-NEXT:    v_sub_i32_e32 v8, vcc, v8, v5
 ; CGP-NEXT:    v_add_i32_e32 v8, vcc, v8, v10
 ; CGP-NEXT:    v_mul_lo_u32 v10, v6, v9
@@ -1366,7 +1366,7 @@ define <2 x i64> @v_urem_v2i64_oddk_denom(<2 x i64> %num) {
 ; CGP-NEXT:    v_add_i32_e32 v5, vcc, v5, v9
 ; CGP-NEXT:    v_addc_u32_e32 v6, vcc, v6, v8, vcc
 ; CGP-NEXT:    v_mul_lo_u32 v8, v5, v7
-; CGP-NEXT:    v_mul_hi_u32 v9, v7, v5
+; CGP-NEXT:    v_mul_hi_u32 v9, v5, v7
 ; CGP-NEXT:    v_mul_lo_u32 v7, v6, v7
 ; CGP-NEXT:    v_mul_lo_u32 v10, v6, v8
 ; CGP-NEXT:    v_mul_hi_u32 v11, v5, v8
@@ -1433,10 +1433,10 @@ define <2 x i64> @v_urem_v2i64_oddk_denom(<2 x i64> %num) {
 ; CGP-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
 ; CGP-NEXT:    v_add_i32_e32 v8, vcc, v9, v8
 ; CGP-NEXT:    v_mul_lo_u32 v9, v7, v4
-; CGP-NEXT:    v_mul_hi_u32 v7, v4, v7
+; CGP-NEXT:    v_mul_hi_u32 v7, v7, v4
 ; CGP-NEXT:    v_add_i32_e32 v10, vcc, v11, v10
 ; CGP-NEXT:    v_mul_lo_u32 v11, v5, v4
-; CGP-NEXT:    v_mul_hi_u32 v5, v4, v5
+; CGP-NEXT:    v_mul_hi_u32 v5, v5, v4
 ; CGP-NEXT:    v_add_i32_e32 v8, vcc, v15, v8
 ; CGP-NEXT:    v_add_i32_e32 v6, vcc, v6, v10
 ; CGP-NEXT:    v_mul_lo_u32 v8, v8, v4
-- 
GitLab


From 3c6f91e5b671321c95259dabecdbdfe4a6d69ce1 Mon Sep 17 00:00:00 2001
From: martinboehme <mboehme@google.com>
Date: Tue, 16 Apr 2024 08:49:45 +0200
Subject: [PATCH 341/464] [clang][dataflow] Fix result object location for
 builtin `<=>`. (#88726)

The newly added test causes an assertion failure in
`PropagateResultObject()`
without the fix added here.
---
 .../FlowSensitive/DataflowEnvironment.cpp     |  5 ++
 .../Analysis/FlowSensitive/TransferTest.cpp   | 52 +++++++++++++++++++
 2 files changed, 57 insertions(+)

diff --git a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
index bea15ce9bd24..ee2581143e11 100644
--- a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
+++ b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
@@ -508,6 +508,11 @@ public:
         isa<CXXStdInitializerListExpr>(E)) {
       return;
     }
+    if (auto *Op = dyn_cast<BinaryOperator>(E);
+        Op && Op->getOpcode() == BO_Cmp) {
+      // Builtin `<=>` returns a `std::strong_ordering` object.
+      return;
+    }
 
     if (auto *InitList = dyn_cast<InitListExpr>(E)) {
       if (!InitList->isSemanticForm())
diff --git a/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp b/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp
index 00dafb2988c6..d8bcc3da4b8b 100644
--- a/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp
+++ b/clang/unittests/Analysis/FlowSensitive/TransferTest.cpp
@@ -3098,6 +3098,58 @@ TEST(TransferTest, ResultObjectLocationForCXXOperatorCallExpr) {
       });
 }
 
+// Check that the `std::strong_ordering` object returned by builtin `<=>` has a
+// correctly modeled result object location.
+TEST(TransferTest, ResultObjectLocationForBuiltinSpaceshipOperator) {
+  std::string Code = R"(
+    namespace std {
+      // This is the minimal definition required to get
+      // `Sema::CheckComparisonCategoryType()` to accept this fake.
+      struct strong_ordering {
+        enum class ordering { less, equal, greater };
+        ordering o;
+        static const strong_ordering less;
+        static const strong_ordering equivalent;
+        static const strong_ordering equal;
+        static const strong_ordering greater;
+      };
+
+      inline constexpr strong_ordering strong_ordering::less =
+        { strong_ordering::ordering::less };
+      inline constexpr strong_ordering strong_ordering::equal =
+        { strong_ordering::ordering::equal };
+      inline constexpr strong_ordering strong_ordering::equivalent =
+        { strong_ordering::ordering::equal };
+      inline constexpr strong_ordering strong_ordering::greater =
+        { strong_ordering::ordering::greater };
+    }
+    void target(int i, int j) {
+      auto ordering = i <=> j;
+      // [[p]]
+    }
+  )";
+  using ast_matchers::binaryOperator;
+  using ast_matchers::hasOperatorName;
+  using ast_matchers::match;
+  using ast_matchers::selectFirst;
+  using ast_matchers::traverse;
+  runDataflow(
+      Code,
+      [](const llvm::StringMap<DataflowAnalysisState<NoopLattice>> &Results,
+         ASTContext &ASTCtx) {
+        const Environment &Env = getEnvironmentAtAnnotation(Results, "p");
+
+        auto *Spaceship = selectFirst<BinaryOperator>(
+            "op",
+            match(binaryOperator(hasOperatorName("<=>")).bind("op"), ASTCtx));
+
+        EXPECT_EQ(
+            &Env.getResultObjectLocation(*Spaceship),
+            &getLocForDecl<RecordStorageLocation>(ASTCtx, Env, "ordering"));
+      },
+      LangStandard::lang_cxx20);
+}
+
 TEST(TransferTest, ResultObjectLocationForStdInitializerListExpr) {
   std::string Code = R"(
     namespace std {
-- 
GitLab


From 7e49b0d5a67f212e84f8ec0ec2e39a6a8673bfaf Mon Sep 17 00:00:00 2001
From: Daniil Kovalev <dkovalev@accesssoftek.com>
Date: Tue, 16 Apr 2024 09:53:33 +0300
Subject: [PATCH 342/464] [lldb] Fix nullptr dereference on running x86 binary
 with x86-disabled llvm (#82603)

If `LLVM_TARGETS_TO_BUILD` does not contain `X86` and we try to run an
x86 binary in lldb, we get a `nullptr` dereference in
`LLVMDisasmInstruction(...)`. We try to call `getDisAsm()` method on a
`LLVMDisasmContext *DC` which is null. The pointer is passed from
`x86AssemblyInspectionEngine::instruction_length(...)` and is originally
`m_disasm_context` member of `x86AssemblyInspectionEngine`. This should
be filled by `LLVMCreateDisasm(...)` in the class constructor, but not
having X86 target enabled in llvm makes
`TargetRegistry::lookupTarget(...)` call return `nullptr`, which results
in `m_disasm_context` initialized with `nullptr` as well.

This patch adds if statements against `m_disasm_context` in
`x86AssemblyInspectionEngine::GetNonCallSiteUnwindPlanFromAssembly(...)`
and `x86AssemblyInspectionEngine::FindFirstNonPrologueInstruction(...)`
so subsequent calls to
`x86AssemblyInspectionEngine::instruction_length(...)` do not cause a
null pointer dereference.
---
 .../x86/x86AssemblyInspectionEngine.cpp       |   6 +
 lldb/unittests/UnwindAssembly/CMakeLists.txt  |   4 +
 .../x86-but-no-x86-target/CMakeLists.txt      |  10 ++
 .../Testx86AssemblyInspectionEngine.cpp       | 103 ++++++++++++++++++
 4 files changed, 123 insertions(+)
 create mode 100644 lldb/unittests/UnwindAssembly/x86-but-no-x86-target/CMakeLists.txt
 create mode 100644 lldb/unittests/UnwindAssembly/x86-but-no-x86-target/Testx86AssemblyInspectionEngine.cpp

diff --git a/lldb/source/Plugins/UnwindAssembly/x86/x86AssemblyInspectionEngine.cpp b/lldb/source/Plugins/UnwindAssembly/x86/x86AssemblyInspectionEngine.cpp
index 2032c5a68d05..6bfaa54135a9 100644
--- a/lldb/source/Plugins/UnwindAssembly/x86/x86AssemblyInspectionEngine.cpp
+++ b/lldb/source/Plugins/UnwindAssembly/x86/x86AssemblyInspectionEngine.cpp
@@ -909,6 +909,9 @@ bool x86AssemblyInspectionEngine::GetNonCallSiteUnwindPlanFromAssembly(
   if (!m_register_map_initialized)
     return false;
 
+  if (m_disasm_context == nullptr)
+    return false;
+
   addr_t current_func_text_offset = 0;
   int current_sp_bytes_offset_from_fa = 0;
   bool is_aligned = false;
@@ -1570,6 +1573,9 @@ bool x86AssemblyInspectionEngine::FindFirstNonPrologueInstruction(
   if (!m_register_map_initialized)
     return false;
 
+  if (m_disasm_context == nullptr)
+    return false;
+
   while (offset < size) {
     int regno;
     int insn_len;
diff --git a/lldb/unittests/UnwindAssembly/CMakeLists.txt b/lldb/unittests/UnwindAssembly/CMakeLists.txt
index 136fcd9ae979..d6e4471af4ec 100644
--- a/lldb/unittests/UnwindAssembly/CMakeLists.txt
+++ b/lldb/unittests/UnwindAssembly/CMakeLists.txt
@@ -9,3 +9,7 @@ endif()
 if ("X86" IN_LIST LLVM_TARGETS_TO_BUILD)
   add_subdirectory(x86)
 endif()
+
+if (NOT "X86" IN_LIST LLVM_TARGETS_TO_BUILD)
+  add_subdirectory(x86-but-no-x86-target)
+endif()
diff --git a/lldb/unittests/UnwindAssembly/x86-but-no-x86-target/CMakeLists.txt b/lldb/unittests/UnwindAssembly/x86-but-no-x86-target/CMakeLists.txt
new file mode 100644
index 000000000000..d28e9629a64c
--- /dev/null
+++ b/lldb/unittests/UnwindAssembly/x86-but-no-x86-target/CMakeLists.txt
@@ -0,0 +1,10 @@
+add_lldb_unittest(UnwindAssemblyX86ButNoX86TargetTests
+    Testx86AssemblyInspectionEngine.cpp
+    LINK_LIBS
+      lldbCore
+      lldbSymbol
+      lldbPluginUnwindAssemblyX86
+    LINK_COMPONENTS
+      Support
+      ${LLVM_TARGETS_TO_BUILD}
+  )
diff --git a/lldb/unittests/UnwindAssembly/x86-but-no-x86-target/Testx86AssemblyInspectionEngine.cpp b/lldb/unittests/UnwindAssembly/x86-but-no-x86-target/Testx86AssemblyInspectionEngine.cpp
new file mode 100644
index 000000000000..ed093d146440
--- /dev/null
+++ b/lldb/unittests/UnwindAssembly/x86-but-no-x86-target/Testx86AssemblyInspectionEngine.cpp
@@ -0,0 +1,103 @@
+//===-- Testx86AssemblyInspectionEngine.cpp -------------------------------===//
+
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "gtest/gtest.h"
+
+#include "Plugins/UnwindAssembly/x86/x86AssemblyInspectionEngine.h"
+#include "lldb/Core/AddressRange.h"
+#include "lldb/Symbol/UnwindPlan.h"
+#include "lldb/Utility/ArchSpec.h"
+
+#include "llvm/Support/TargetSelect.h"
+
+#include <memory>
+#include <vector>
+
+using namespace lldb;
+using namespace lldb_private;
+
+class Testx86AssemblyInspectionEngine : public testing::Test {
+public:
+  static void SetUpTestCase();
+};
+
+void Testx86AssemblyInspectionEngine::SetUpTestCase() {
+  llvm::InitializeAllTargets();
+  llvm::InitializeAllAsmPrinters();
+  llvm::InitializeAllTargetMCs();
+  llvm::InitializeAllDisassemblers();
+}
+
+// only defining the register names / numbers that the unwinder is actually
+// using today
+
+// names should match the constants below.  These will be the eRegisterKindLLDB
+// register numbers.
+
+const char *x86_64_reg_names[] = {"rax", "rbx", "rcx", "rdx", "rsp", "rbp",
+                                  "rsi", "rdi", "r8",  "r9",  "r10", "r11",
+                                  "r12", "r13", "r14", "r15", "rip"};
+
+enum x86_64_regs {
+  k_rax = 0,
+  k_rbx = 1,
+  k_rcx = 2,
+  k_rdx = 3,
+  k_rsp = 4,
+  k_rbp = 5,
+  k_rsi = 6,
+  k_rdi = 7,
+  k_r8 = 8,
+  k_r9 = 9,
+  k_r10 = 10,
+  k_r11 = 11,
+  k_r12 = 12,
+  k_r13 = 13,
+  k_r14 = 14,
+  k_r15 = 15,
+  k_rip = 16
+};
+
+std::unique_ptr<x86AssemblyInspectionEngine> Getx86_64Inspector() {
+
+  ArchSpec arch("x86_64-apple-macosx");
+  std::unique_ptr<x86AssemblyInspectionEngine> engine(
+      new x86AssemblyInspectionEngine(arch));
+
+  std::vector<x86AssemblyInspectionEngine::lldb_reg_info> lldb_regnums;
+  int i = 0;
+  for (const auto &name : x86_64_reg_names) {
+    x86AssemblyInspectionEngine::lldb_reg_info ri;
+    ri.name = name;
+    ri.lldb_regnum = i++;
+    lldb_regnums.push_back(ri);
+  }
+
+  engine->Initialize(lldb_regnums);
+  return engine;
+}
+
+TEST_F(Testx86AssemblyInspectionEngine, TestSimple64bitFrameFunction) {
+  std::unique_ptr<x86AssemblyInspectionEngine> engine = Getx86_64Inspector();
+
+  // 'int main() { }' compiled for x86_64-apple-macosx with clang
+  uint8_t data[] = {
+      0x55,             // offset 0 -- pushq %rbp
+      0x48, 0x89, 0xe5, // offset 1 -- movq %rsp, %rbp
+      0x31, 0xc0,       // offset 4 -- xorl %eax, %eax
+      0x5d,             // offset 6 -- popq %rbp
+      0xc3              // offset 7 -- retq
+  };
+
+  AddressRange sample_range(0x1000, sizeof(data));
+
+  UnwindPlan unwind_plan(eRegisterKindLLDB);
+  EXPECT_FALSE(engine->GetNonCallSiteUnwindPlanFromAssembly(
+      data, sizeof(data), sample_range, unwind_plan));
+}
-- 
GitLab


From 89071f35599ac58cde99923a376d6b75c0d49e4b Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Tue, 16 Apr 2024 00:09:41 -0700
Subject: [PATCH 343/464] [clang] Drop unaligned from calls to readNext (NFC)
 (#88842)

Now readNext defaults to unaligned accesses.  This patch drops
unaligned to improve readability.
---
 clang/lib/APINotes/APINotesReader.cpp         | 137 +++++++-----------
 clang/lib/Serialization/ASTReader.cpp         |  87 +++++------
 clang/lib/Serialization/GlobalModuleIndex.cpp |  12 +-
 .../lib/Serialization/MultiOnDiskHashTable.h  |   4 +-
 4 files changed, 98 insertions(+), 142 deletions(-)

diff --git a/clang/lib/APINotes/APINotesReader.cpp b/clang/lib/APINotes/APINotesReader.cpp
index fbbe9c32ce12..dfc3beb6fa13 100644
--- a/clang/lib/APINotes/APINotesReader.cpp
+++ b/clang/lib/APINotes/APINotesReader.cpp
@@ -30,23 +30,20 @@ namespace {
 llvm::VersionTuple ReadVersionTuple(const uint8_t *&Data) {
   uint8_t NumVersions = (*Data++) & 0x03;
 
-  unsigned Major =
-      endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
+  unsigned Major = endian::readNext<uint32_t, llvm::endianness::little>(Data);
   if (NumVersions == 0)
     return llvm::VersionTuple(Major);
 
-  unsigned Minor =
-      endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
+  unsigned Minor = endian::readNext<uint32_t, llvm::endianness::little>(Data);
   if (NumVersions == 1)
     return llvm::VersionTuple(Major, Minor);
 
   unsigned Subminor =
-      endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
+      endian::readNext<uint32_t, llvm::endianness::little>(Data);
   if (NumVersions == 2)
     return llvm::VersionTuple(Major, Minor, Subminor);
 
-  unsigned Build =
-      endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
+  unsigned Build = endian::readNext<uint32_t, llvm::endianness::little>(Data);
   return llvm::VersionTuple(Major, Minor, Subminor, Build);
 }
 
@@ -71,16 +68,16 @@ public:
 
   static std::pair<unsigned, unsigned> ReadKeyDataLength(const uint8_t *&Data) {
     unsigned KeyLength =
-        endian::readNext<uint16_t, llvm::endianness::little, unaligned>(Data);
+        endian::readNext<uint16_t, llvm::endianness::little>(Data);
     unsigned DataLength =
-        endian::readNext<uint16_t, llvm::endianness::little, unaligned>(Data);
+        endian::readNext<uint16_t, llvm::endianness::little>(Data);
     return {KeyLength, DataLength};
   }
 
   static data_type ReadData(internal_key_type Key, const uint8_t *Data,
                             unsigned Length) {
     unsigned NumElements =
-        endian::readNext<uint16_t, llvm::endianness::little, unaligned>(Data);
+        endian::readNext<uint16_t, llvm::endianness::little>(Data);
     data_type Result;
     Result.reserve(NumElements);
     for (unsigned i = 0; i != NumElements; ++i) {
@@ -105,14 +102,14 @@ void ReadCommonEntityInfo(const uint8_t *&Data, CommonEntityInfo &Info) {
     Info.setSwiftPrivate(static_cast<bool>((UnavailableBits >> 3) & 0x01));
 
   unsigned MsgLength =
-      endian::readNext<uint16_t, llvm::endianness::little, unaligned>(Data);
+      endian::readNext<uint16_t, llvm::endianness::little>(Data);
   Info.UnavailableMsg =
       std::string(reinterpret_cast<const char *>(Data),
                   reinterpret_cast<const char *>(Data) + MsgLength);
   Data += MsgLength;
 
   unsigned SwiftNameLength =
-      endian::readNext<uint16_t, llvm::endianness::little, unaligned>(Data);
+      endian::readNext<uint16_t, llvm::endianness::little>(Data);
   Info.SwiftName =
       std::string(reinterpret_cast<const char *>(Data),
                   reinterpret_cast<const char *>(Data) + SwiftNameLength);
@@ -124,7 +121,7 @@ void ReadCommonTypeInfo(const uint8_t *&Data, CommonTypeInfo &Info) {
   ReadCommonEntityInfo(Data, Info);
 
   unsigned SwiftBridgeLength =
-      endian::readNext<uint16_t, llvm::endianness::little, unaligned>(Data);
+      endian::readNext<uint16_t, llvm::endianness::little>(Data);
   if (SwiftBridgeLength > 0) {
     Info.setSwiftBridge(std::string(reinterpret_cast<const char *>(Data),
                                     SwiftBridgeLength - 1));
@@ -132,7 +129,7 @@ void ReadCommonTypeInfo(const uint8_t *&Data, CommonTypeInfo &Info) {
   }
 
   unsigned ErrorDomainLength =
-      endian::readNext<uint16_t, llvm::endianness::little, unaligned>(Data);
+      endian::readNext<uint16_t, llvm::endianness::little>(Data);
   if (ErrorDomainLength > 0) {
     Info.setNSErrorDomain(std::optional<std::string>(std::string(
         reinterpret_cast<const char *>(Data), ErrorDomainLength - 1)));
@@ -163,9 +160,9 @@ public:
 
   static std::pair<unsigned, unsigned> ReadKeyDataLength(const uint8_t *&Data) {
     unsigned KeyLength =
-        endian::readNext<uint16_t, llvm::endianness::little, unaligned>(Data);
+        endian::readNext<uint16_t, llvm::endianness::little>(Data);
     unsigned DataLength =
-        endian::readNext<uint16_t, llvm::endianness::little, unaligned>(Data);
+        endian::readNext<uint16_t, llvm::endianness::little>(Data);
     return {KeyLength, DataLength};
   }
 
@@ -175,8 +172,7 @@ public:
 
   static data_type ReadData(internal_key_type key, const uint8_t *Data,
                             unsigned Length) {
-    return endian::readNext<uint32_t, llvm::endianness::little, unaligned>(
-        Data);
+    return endian::readNext<uint32_t, llvm::endianness::little>(Data);
   }
 };
 
@@ -203,26 +199,24 @@ public:
 
   static std::pair<unsigned, unsigned> ReadKeyDataLength(const uint8_t *&Data) {
     unsigned KeyLength =
-        endian::readNext<uint16_t, llvm::endianness::little, unaligned>(Data);
+        endian::readNext<uint16_t, llvm::endianness::little>(Data);
     unsigned DataLength =
-        endian::readNext<uint16_t, llvm::endianness::little, unaligned>(Data);
+        endian::readNext<uint16_t, llvm::endianness::little>(Data);
     return {KeyLength, DataLength};
   }
 
   static internal_key_type ReadKey(const uint8_t *Data, unsigned Length) {
     auto ParentCtxID =
-        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
+        endian::readNext<uint32_t, llvm::endianness::little>(Data);
     auto ContextKind =
-        endian::readNext<uint8_t, llvm::endianness::little, unaligned>(Data);
-    auto NameID =
-        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
+        endian::readNext<uint8_t, llvm::endianness::little>(Data);
+    auto NameID = endian::readNext<uint32_t, llvm::endianness::little>(Data);
     return {ParentCtxID, ContextKind, NameID};
   }
 
   static data_type ReadData(internal_key_type Key, const uint8_t *Data,
                             unsigned Length) {
-    return endian::readNext<uint32_t, llvm::endianness::little, unaligned>(
-        Data);
+    return endian::readNext<uint32_t, llvm::endianness::little>(Data);
   }
 };
 
@@ -232,8 +226,7 @@ class ObjCContextInfoTableInfo
                                 ObjCContextInfo> {
 public:
   static internal_key_type ReadKey(const uint8_t *Data, unsigned Length) {
-    return endian::readNext<uint32_t, llvm::endianness::little, unaligned>(
-        Data);
+    return endian::readNext<uint32_t, llvm::endianness::little>(Data);
   }
 
   hash_value_type ComputeHash(internal_key_type Key) {
@@ -273,8 +266,7 @@ void ReadVariableInfo(const uint8_t *&Data, VariableInfo &Info) {
   }
   ++Data;
 
-  auto TypeLen =
-      endian::readNext<uint16_t, llvm::endianness::little, unaligned>(Data);
+  auto TypeLen = endian::readNext<uint16_t, llvm::endianness::little>(Data);
   Info.setType(std::string(Data, Data + TypeLen));
   Data += TypeLen;
 }
@@ -286,12 +278,9 @@ class ObjCPropertyTableInfo
                                 ObjCPropertyInfo> {
 public:
   static internal_key_type ReadKey(const uint8_t *Data, unsigned Length) {
-    auto ClassID =
-        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
-    auto NameID =
-        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
-    char IsInstance =
-        endian::readNext<uint8_t, llvm::endianness::little, unaligned>(Data);
+    auto ClassID = endian::readNext<uint32_t, llvm::endianness::little>(Data);
+    auto NameID = endian::readNext<uint32_t, llvm::endianness::little>(Data);
+    char IsInstance = endian::readNext<uint8_t, llvm::endianness::little>(Data);
     return {ClassID, NameID, IsInstance};
   }
 
@@ -314,8 +303,7 @@ public:
 void ReadParamInfo(const uint8_t *&Data, ParamInfo &Info) {
   ReadVariableInfo(Data, Info);
 
-  uint8_t Payload =
-      endian::readNext<uint8_t, llvm::endianness::little, unaligned>(Data);
+  uint8_t Payload = endian::readNext<uint8_t, llvm::endianness::little>(Data);
   if (auto RawConvention = Payload & 0x7) {
     auto Convention = static_cast<RetainCountConventionKind>(RawConvention - 1);
     Info.setRetainCountConvention(Convention);
@@ -331,8 +319,7 @@ void ReadParamInfo(const uint8_t *&Data, ParamInfo &Info) {
 void ReadFunctionInfo(const uint8_t *&Data, FunctionInfo &Info) {
   ReadCommonEntityInfo(Data, Info);
 
-  uint8_t Payload =
-      endian::readNext<uint8_t, llvm::endianness::little, unaligned>(Data);
+  uint8_t Payload = endian::readNext<uint8_t, llvm::endianness::little>(Data);
   if (auto RawConvention = Payload & 0x7) {
     auto Convention = static_cast<RetainCountConventionKind>(RawConvention - 1);
     Info.setRetainCountConvention(Convention);
@@ -343,12 +330,12 @@ void ReadFunctionInfo(const uint8_t *&Data, FunctionInfo &Info) {
   assert(Payload == 0 && "Bad API notes");
 
   Info.NumAdjustedNullable =
-      endian::readNext<uint8_t, llvm::endianness::little, unaligned>(Data);
+      endian::readNext<uint8_t, llvm::endianness::little>(Data);
   Info.NullabilityPayload =
-      endian::readNext<uint64_t, llvm::endianness::little, unaligned>(Data);
+      endian::readNext<uint64_t, llvm::endianness::little>(Data);
 
   unsigned NumParams =
-      endian::readNext<uint16_t, llvm::endianness::little, unaligned>(Data);
+      endian::readNext<uint16_t, llvm::endianness::little>(Data);
   while (NumParams > 0) {
     ParamInfo pi;
     ReadParamInfo(Data, pi);
@@ -357,7 +344,7 @@ void ReadFunctionInfo(const uint8_t *&Data, FunctionInfo &Info) {
   }
 
   unsigned ResultTypeLen =
-      endian::readNext<uint16_t, llvm::endianness::little, unaligned>(Data);
+      endian::readNext<uint16_t, llvm::endianness::little>(Data);
   Info.ResultType = std::string(Data, Data + ResultTypeLen);
   Data += ResultTypeLen;
 }
@@ -369,12 +356,10 @@ class ObjCMethodTableInfo
                                 ObjCMethodInfo> {
 public:
   static internal_key_type ReadKey(const uint8_t *Data, unsigned Length) {
-    auto ClassID =
-        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
+    auto ClassID = endian::readNext<uint32_t, llvm::endianness::little>(Data);
     auto SelectorID =
-        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
-    auto IsInstance =
-        endian::readNext<uint8_t, llvm::endianness::little, unaligned>(Data);
+        endian::readNext<uint32_t, llvm::endianness::little>(Data);
+    auto IsInstance = endian::readNext<uint8_t, llvm::endianness::little>(Data);
     return {ClassID, SelectorID, IsInstance};
   }
 
@@ -419,29 +404,26 @@ public:
 
   static std::pair<unsigned, unsigned> ReadKeyDataLength(const uint8_t *&Data) {
     unsigned KeyLength =
-        endian::readNext<uint16_t, llvm::endianness::little, unaligned>(Data);
+        endian::readNext<uint16_t, llvm::endianness::little>(Data);
     unsigned DataLength =
-        endian::readNext<uint16_t, llvm::endianness::little, unaligned>(Data);
+        endian::readNext<uint16_t, llvm::endianness::little>(Data);
     return {KeyLength, DataLength};
   }
 
   static internal_key_type ReadKey(const uint8_t *Data, unsigned Length) {
     internal_key_type Key;
-    Key.NumArgs =
-        endian::readNext<uint16_t, llvm::endianness::little, unaligned>(Data);
+    Key.NumArgs = endian::readNext<uint16_t, llvm::endianness::little>(Data);
     unsigned NumIdents = (Length - sizeof(uint16_t)) / sizeof(uint32_t);
     for (unsigned i = 0; i != NumIdents; ++i) {
       Key.Identifiers.push_back(
-          endian::readNext<uint32_t, llvm::endianness::little, unaligned>(
-              Data));
+          endian::readNext<uint32_t, llvm::endianness::little>(Data));
     }
     return Key;
   }
 
   static data_type ReadData(internal_key_type Key, const uint8_t *Data,
                             unsigned Length) {
-    return endian::readNext<uint32_t, llvm::endianness::little, unaligned>(
-        Data);
+    return endian::readNext<uint32_t, llvm::endianness::little>(Data);
   }
 };
 
@@ -451,12 +433,10 @@ class GlobalVariableTableInfo
                                 GlobalVariableInfo> {
 public:
   static internal_key_type ReadKey(const uint8_t *Data, unsigned Length) {
-    auto CtxID =
-        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
+    auto CtxID = endian::readNext<uint32_t, llvm::endianness::little>(Data);
     auto ContextKind =
-        endian::readNext<uint8_t, llvm::endianness::little, unaligned>(Data);
-    auto NameID =
-        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
+        endian::readNext<uint8_t, llvm::endianness::little>(Data);
+    auto NameID = endian::readNext<uint32_t, llvm::endianness::little>(Data);
     return {CtxID, ContextKind, NameID};
   }
 
@@ -478,12 +458,10 @@ class GlobalFunctionTableInfo
                                 GlobalFunctionInfo> {
 public:
   static internal_key_type ReadKey(const uint8_t *Data, unsigned Length) {
-    auto CtxID =
-        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
+    auto CtxID = endian::readNext<uint32_t, llvm::endianness::little>(Data);
     auto ContextKind =
-        endian::readNext<uint8_t, llvm::endianness::little, unaligned>(Data);
-    auto NameID =
-        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
+        endian::readNext<uint8_t, llvm::endianness::little>(Data);
+    auto NameID = endian::readNext<uint32_t, llvm::endianness::little>(Data);
     return {CtxID, ContextKind, NameID};
   }
 
@@ -505,8 +483,7 @@ class EnumConstantTableInfo
                                 EnumConstantInfo> {
 public:
   static internal_key_type ReadKey(const uint8_t *Data, unsigned Length) {
-    auto NameID =
-        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
+    auto NameID = endian::readNext<uint32_t, llvm::endianness::little>(Data);
     return NameID;
   }
 
@@ -527,13 +504,11 @@ class TagTableInfo
     : public VersionedTableInfo<TagTableInfo, ContextTableKey, TagInfo> {
 public:
   static internal_key_type ReadKey(const uint8_t *Data, unsigned Length) {
-    auto CtxID =
-        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
+    auto CtxID = endian::readNext<uint32_t, llvm::endianness::little>(Data);
     auto ContextKind =
-        endian::readNext<uint8_t, llvm::endianness::little, unaligned>(Data);
+        endian::readNext<uint8_t, llvm::endianness::little>(Data);
     auto NameID =
-        endian::readNext<IdentifierID, llvm::endianness::little, unaligned>(
-            Data);
+        endian::readNext<IdentifierID, llvm::endianness::little>(Data);
     return {CtxID, ContextKind, NameID};
   }
 
@@ -553,21 +528,21 @@ public:
           static_cast<EnumExtensibilityKind>((Payload & 0x3) - 1);
 
     unsigned ImportAsLength =
-        endian::readNext<uint16_t, llvm::endianness::little, unaligned>(Data);
+        endian::readNext<uint16_t, llvm::endianness::little>(Data);
     if (ImportAsLength > 0) {
       Info.SwiftImportAs =
           std::string(reinterpret_cast<const char *>(Data), ImportAsLength - 1);
       Data += ImportAsLength - 1;
     }
     unsigned RetainOpLength =
-        endian::readNext<uint16_t, llvm::endianness::little, unaligned>(Data);
+        endian::readNext<uint16_t, llvm::endianness::little>(Data);
     if (RetainOpLength > 0) {
       Info.SwiftRetainOp =
           std::string(reinterpret_cast<const char *>(Data), RetainOpLength - 1);
       Data += RetainOpLength - 1;
     }
     unsigned ReleaseOpLength =
-        endian::readNext<uint16_t, llvm::endianness::little, unaligned>(Data);
+        endian::readNext<uint16_t, llvm::endianness::little>(Data);
     if (ReleaseOpLength > 0) {
       Info.SwiftReleaseOp = std::string(reinterpret_cast<const char *>(Data),
                                         ReleaseOpLength - 1);
@@ -585,13 +560,11 @@ class TypedefTableInfo
                                 TypedefInfo> {
 public:
   static internal_key_type ReadKey(const uint8_t *Data, unsigned Length) {
-    auto CtxID =
-        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
+    auto CtxID = endian::readNext<uint32_t, llvm::endianness::little>(Data);
     auto ContextKind =
-        endian::readNext<uint8_t, llvm::endianness::little, unaligned>(Data);
+        endian::readNext<uint8_t, llvm::endianness::little>(Data);
     auto nameID =
-        endian::readNext<IdentifierID, llvm::endianness::little, unaligned>(
-            Data);
+        endian::readNext<IdentifierID, llvm::endianness::little>(Data);
     return {CtxID, ContextKind, nameID};
   }
 
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 8c4b460970ad..feb60bc54413 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -915,10 +915,9 @@ ASTSelectorLookupTrait::ReadKey(const unsigned char* d, unsigned) {
   using namespace llvm::support;
 
   SelectorTable &SelTable = Reader.getContext().Selectors;
-  unsigned N =
-      endian::readNext<uint16_t, llvm::endianness::little, unaligned>(d);
+  unsigned N = endian::readNext<uint16_t, llvm::endianness::little>(d);
   const IdentifierInfo *FirstII = Reader.getLocalIdentifier(
-      F, endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d));
+      F, endian::readNext<uint32_t, llvm::endianness::little>(d));
   if (N == 0)
     return SelTable.getNullarySelector(FirstII);
   else if (N == 1)
@@ -928,7 +927,7 @@ ASTSelectorLookupTrait::ReadKey(const unsigned char* d, unsigned) {
   Args.push_back(FirstII);
   for (unsigned I = 1; I != N; ++I)
     Args.push_back(Reader.getLocalIdentifier(
-        F, endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d)));
+        F, endian::readNext<uint32_t, llvm::endianness::little>(d)));
 
   return SelTable.getSelector(N, Args.data());
 }
@@ -941,11 +940,11 @@ ASTSelectorLookupTrait::ReadData(Selector, const unsigned char* d,
   data_type Result;
 
   Result.ID = Reader.getGlobalSelectorID(
-      F, endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d));
+      F, endian::readNext<uint32_t, llvm::endianness::little>(d));
   unsigned FullInstanceBits =
-      endian::readNext<uint16_t, llvm::endianness::little, unaligned>(d);
+      endian::readNext<uint16_t, llvm::endianness::little>(d);
   unsigned FullFactoryBits =
-      endian::readNext<uint16_t, llvm::endianness::little, unaligned>(d);
+      endian::readNext<uint16_t, llvm::endianness::little>(d);
   Result.InstanceBits = FullInstanceBits & 0x3;
   Result.InstanceHasMoreThanOneDecl = (FullInstanceBits >> 2) & 0x1;
   Result.FactoryBits = FullFactoryBits & 0x3;
@@ -956,16 +955,14 @@ ASTSelectorLookupTrait::ReadData(Selector, const unsigned char* d,
   // Load instance methods
   for (unsigned I = 0; I != NumInstanceMethods; ++I) {
     if (ObjCMethodDecl *Method = Reader.GetLocalDeclAs<ObjCMethodDecl>(
-            F,
-            endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d)))
+            F, endian::readNext<uint32_t, llvm::endianness::little>(d)))
       Result.Instance.push_back(Method);
   }
 
   // Load factory methods
   for (unsigned I = 0; I != NumFactoryMethods; ++I) {
     if (ObjCMethodDecl *Method = Reader.GetLocalDeclAs<ObjCMethodDecl>(
-            F,
-            endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d)))
+            F, endian::readNext<uint32_t, llvm::endianness::little>(d)))
       Result.Factory.push_back(Method);
   }
 
@@ -1009,8 +1006,7 @@ static bool readBit(unsigned &Bits) {
 IdentID ASTIdentifierLookupTrait::ReadIdentifierID(const unsigned char *d) {
   using namespace llvm::support;
 
-  unsigned RawID =
-      endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d);
+  unsigned RawID = endian::readNext<uint32_t, llvm::endianness::little>(d);
   return Reader.getGlobalIdentifierID(F, RawID >> 1);
 }
 
@@ -1028,8 +1024,7 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k,
                                                    unsigned DataLen) {
   using namespace llvm::support;
 
-  unsigned RawID =
-      endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d);
+  unsigned RawID = endian::readNext<uint32_t, llvm::endianness::little>(d);
   bool IsInteresting = RawID & 0x01;
 
   // Wipe out the "is interesting" bit.
@@ -1053,9 +1048,8 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k,
   }
 
   unsigned ObjCOrBuiltinID =
-      endian::readNext<uint16_t, llvm::endianness::little, unaligned>(d);
-  unsigned Bits =
-      endian::readNext<uint16_t, llvm::endianness::little, unaligned>(d);
+      endian::readNext<uint16_t, llvm::endianness::little>(d);
+  unsigned Bits = endian::readNext<uint16_t, llvm::endianness::little>(d);
   bool CPlusPlusOperatorKeyword = readBit(Bits);
   bool HasRevertedTokenIDToIdentifier = readBit(Bits);
   bool Poisoned = readBit(Bits);
@@ -1084,7 +1078,7 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k,
   // definition.
   if (HadMacroDefinition) {
     uint32_t MacroDirectivesOffset =
-        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d);
+        endian::readNext<uint32_t, llvm::endianness::little>(d);
     DataLen -= 4;
 
     Reader.addPendingMacro(II, &F, MacroDirectivesOffset);
@@ -1098,8 +1092,7 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k,
     SmallVector<uint32_t, 4> DeclIDs;
     for (; DataLen > 0; DataLen -= 4)
       DeclIDs.push_back(Reader.getGlobalDeclID(
-          F,
-          endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d)));
+          F, endian::readNext<uint32_t, llvm::endianness::little>(d)));
     Reader.SetGloballyVisibleDecls(II, DeclIDs);
   }
 
@@ -1169,7 +1162,7 @@ ASTDeclContextNameLookupTrait::ReadFileRef(const unsigned char *&d) {
   using namespace llvm::support;
 
   uint32_t ModuleFileID =
-      endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d);
+      endian::readNext<uint32_t, llvm::endianness::little>(d);
   return Reader.getLocalModuleFile(F, ModuleFileID);
 }
 
@@ -1189,18 +1182,15 @@ ASTDeclContextNameLookupTrait::ReadKey(const unsigned char *d, unsigned) {
   case DeclarationName::CXXLiteralOperatorName:
   case DeclarationName::CXXDeductionGuideName:
     Data = (uint64_t)Reader.getLocalIdentifier(
-        F, endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d));
+        F, endian::readNext<uint32_t, llvm::endianness::little>(d));
     break;
   case DeclarationName::ObjCZeroArgSelector:
   case DeclarationName::ObjCOneArgSelector:
   case DeclarationName::ObjCMultiArgSelector:
-    Data =
-        (uint64_t)Reader
-            .getLocalSelector(
-                F,
-                endian::readNext<uint32_t, llvm::endianness::little, unaligned>(
-                    d))
-            .getAsOpaquePtr();
+    Data = (uint64_t)Reader
+               .getLocalSelector(
+                   F, endian::readNext<uint32_t, llvm::endianness::little>(d))
+               .getAsOpaquePtr();
     break;
   case DeclarationName::CXXOperatorName:
     Data = *d++; // OverloadedOperatorKind
@@ -1223,8 +1213,7 @@ void ASTDeclContextNameLookupTrait::ReadDataInto(internal_key_type,
   using namespace llvm::support;
 
   for (unsigned NumDecls = DataLen / 4; NumDecls; --NumDecls) {
-    uint32_t LocalID =
-        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d);
+    uint32_t LocalID = endian::readNext<uint32_t, llvm::endianness::little>(d);
     Val.insert(Reader.getGlobalDeclID(F, LocalID));
   }
 }
@@ -2033,10 +2022,9 @@ HeaderFileInfoTrait::ReadKey(const unsigned char *d, unsigned) {
   using namespace llvm::support;
 
   internal_key_type ikey;
-  ikey.Size =
-      off_t(endian::readNext<uint64_t, llvm::endianness::little, unaligned>(d));
-  ikey.ModTime = time_t(
-      endian::readNext<uint64_t, llvm::endianness::little, unaligned>(d));
+  ikey.Size = off_t(endian::readNext<uint64_t, llvm::endianness::little>(d));
+  ikey.ModTime =
+      time_t(endian::readNext<uint64_t, llvm::endianness::little>(d));
   ikey.Filename = (const char *)d;
   ikey.Imported = true;
   return ikey;
@@ -2064,9 +2052,9 @@ HeaderFileInfoTrait::ReadData(internal_key_ref key, const unsigned char *d,
   HFI.DirInfo = (Flags >> 1) & 0x07;
   HFI.IndexHeaderMapHeader = Flags & 0x01;
   HFI.ControllingMacroID = Reader.getGlobalIdentifierID(
-      M, endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d));
+      M, endian::readNext<uint32_t, llvm::endianness::little>(d));
   if (unsigned FrameworkOffset =
-          endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d)) {
+          endian::readNext<uint32_t, llvm::endianness::little>(d)) {
     // The framework offset is 1 greater than the actual offset,
     // since 0 is used as an indicator for "no framework name".
     StringRef FrameworkName(FrameworkStrings + FrameworkOffset - 1);
@@ -2077,7 +2065,7 @@ HeaderFileInfoTrait::ReadData(internal_key_ref key, const unsigned char *d,
          "Wrong data length in HeaderFileInfo deserialization");
   while (d != End) {
     uint32_t LocalSMID =
-        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d);
+        endian::readNext<uint32_t, llvm::endianness::little>(d);
     auto HeaderRole = static_cast<ModuleMap::ModuleHeaderRole>(LocalSMID & 7);
     LocalSMID >>= 3;
 
@@ -4085,9 +4073,8 @@ void ASTReader::ReadModuleOffsetMap(ModuleFile &F) const {
     // how it goes...
     using namespace llvm::support;
     ModuleKind Kind = static_cast<ModuleKind>(
-        endian::readNext<uint8_t, llvm::endianness::little, unaligned>(Data));
-    uint16_t Len =
-        endian::readNext<uint16_t, llvm::endianness::little, unaligned>(Data);
+        endian::readNext<uint8_t, llvm::endianness::little>(Data));
+    uint16_t Len = endian::readNext<uint16_t, llvm::endianness::little>(Data);
     StringRef Name = StringRef((const char*)Data, Len);
     Data += Len;
     ModuleFile *OM = (Kind == MK_PrebuiltModule || Kind == MK_ExplicitModule ||
@@ -4103,21 +4090,21 @@ void ASTReader::ReadModuleOffsetMap(ModuleFile &F) const {
     }
 
     SourceLocation::UIntTy SLocOffset =
-        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
+        endian::readNext<uint32_t, llvm::endianness::little>(Data);
     uint32_t IdentifierIDOffset =
-        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
+        endian::readNext<uint32_t, llvm::endianness::little>(Data);
     uint32_t MacroIDOffset =
-        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
+        endian::readNext<uint32_t, llvm::endianness::little>(Data);
     uint32_t PreprocessedEntityIDOffset =
-        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
+        endian::readNext<uint32_t, llvm::endianness::little>(Data);
     uint32_t SubmoduleIDOffset =
-        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
+        endian::readNext<uint32_t, llvm::endianness::little>(Data);
     uint32_t SelectorIDOffset =
-        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
+        endian::readNext<uint32_t, llvm::endianness::little>(Data);
     uint32_t DeclIDOffset =
-        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
+        endian::readNext<uint32_t, llvm::endianness::little>(Data);
     uint32_t TypeIndexOffset =
-        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Data);
+        endian::readNext<uint32_t, llvm::endianness::little>(Data);
 
     auto mapOffset = [&](uint32_t Offset, uint32_t BaseOffset,
                          RemapBuilder &Remap) {
diff --git a/clang/lib/Serialization/GlobalModuleIndex.cpp b/clang/lib/Serialization/GlobalModuleIndex.cpp
index dd4fc3e00905..8ff10f6a8621 100644
--- a/clang/lib/Serialization/GlobalModuleIndex.cpp
+++ b/clang/lib/Serialization/GlobalModuleIndex.cpp
@@ -89,10 +89,8 @@ public:
   static std::pair<unsigned, unsigned>
   ReadKeyDataLength(const unsigned char*& d) {
     using namespace llvm::support;
-    unsigned KeyLen =
-        endian::readNext<uint16_t, llvm::endianness::little, unaligned>(d);
-    unsigned DataLen =
-        endian::readNext<uint16_t, llvm::endianness::little, unaligned>(d);
+    unsigned KeyLen = endian::readNext<uint16_t, llvm::endianness::little>(d);
+    unsigned DataLen = endian::readNext<uint16_t, llvm::endianness::little>(d);
     return std::make_pair(KeyLen, DataLen);
   }
 
@@ -113,8 +111,7 @@ public:
 
     data_type Result;
     while (DataLen > 0) {
-      unsigned ID =
-          endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d);
+      unsigned ID = endian::readNext<uint32_t, llvm::endianness::little>(d);
       Result.push_back(ID);
       DataLen -= 4;
     }
@@ -514,8 +511,7 @@ namespace {
       // The first bit indicates whether this identifier is interesting.
       // That's all we care about.
       using namespace llvm::support;
-      unsigned RawID =
-          endian::readNext<uint32_t, llvm::endianness::little, unaligned>(d);
+      unsigned RawID = endian::readNext<uint32_t, llvm::endianness::little>(d);
       bool IsInteresting = RawID & 0x01;
       return std::make_pair(k, IsInteresting);
     }
diff --git a/clang/lib/Serialization/MultiOnDiskHashTable.h b/clang/lib/Serialization/MultiOnDiskHashTable.h
index 2402a628b512..a0d75ec3a9e7 100644
--- a/clang/lib/Serialization/MultiOnDiskHashTable.h
+++ b/clang/lib/Serialization/MultiOnDiskHashTable.h
@@ -200,11 +200,11 @@ public:
     storage_type Ptr = Data;
 
     uint32_t BucketOffset =
-        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Ptr);
+        endian::readNext<uint32_t, llvm::endianness::little>(Ptr);
 
     // Read the list of overridden files.
     uint32_t NumFiles =
-        endian::readNext<uint32_t, llvm::endianness::little, unaligned>(Ptr);
+        endian::readNext<uint32_t, llvm::endianness::little>(Ptr);
     // FIXME: Add a reserve() to TinyPtrVector so that we don't need to make
     // an additional copy.
     llvm::SmallVector<file_type, 16> OverriddenFiles;
-- 
GitLab


From a169d4c2e974ceb20b86faea3fa4ac286a1d44e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martin=20Storsj=C3=B6?= <martin@martin.st>
Date: Tue, 16 Apr 2024 10:37:15 +0300
Subject: [PATCH 344/464] =?UTF-8?q?[LLD]=C2=A0[COFF]=20Error=20out=20if=20?=
 =?UTF-8?q?the=20runtime=20pseudo=20relocation=20function=20is=20missing?=
 =?UTF-8?q?=20(#88573)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When then linker creates runtime pseudo relocations, it places them in a
list with the assumption that the runtime will fix these relocations
later, when the image gets loaded. If the relevant runtime function
doesn't seem to be present in the linked image, error out.

Normally when linking the mingw-w64 runtime libraries, this function
always is available. However, if linking without including the mingw-w64
CRT startup files, and the image needs runtime pseudo relocations, make
it clear that this won't work as expected at runtime.

With ld.bfd, this situation is a hard error too; ld.bfd adds an
undefined reference to this symbol if runtime pseudo relocations are
needed.

A later alternative would be to actually try to pull in the symbol (if
seen in a static library, but not included yet). This would allow
decoupling the function from the main mingw-w64 CRT startup code (making
it optional, only running if the linker actually produced runtime pseudo
relocations).

Doing that would require restructuring the lld code (gathering pseudo
relocations earlier, then loading the relocator function, then pulling in
more object files to satisfy the dependencies of the relocator) though.

Also, ld.bfd doesn't currently successfully pull in more object files to
satisfy the dependency on _pei386_runtime_relocator, so with that in
mind, there's not much extra value in making LLD do it currently either;
we can't make such a change in mingw-w64's CRT until both linkers
handle it.

This fixes one issue brought up in
https://github.com/llvm/llvm-project/issues/84424.
---
 lld/COFF/Writer.cpp                     | 10 ++++++-
 lld/test/COFF/autoimport-arm-data.s     |  3 +++
 lld/test/COFF/autoimport-arm64-data.s   |  3 +++
 lld/test/COFF/autoimport-gnu-implib.s   |  3 +++
 lld/test/COFF/autoimport-handler-func.s | 36 +++++++++++++++++++++++++
 lld/test/COFF/autoimport-warn.s         |  3 +++
 lld/test/COFF/autoimport-x86.s          |  3 +++
 7 files changed, 60 insertions(+), 1 deletion(-)
 create mode 100644 lld/test/COFF/autoimport-handler-func.s

diff --git a/lld/COFF/Writer.cpp b/lld/COFF/Writer.cpp
index 9c20bbb83d86..7269d156752d 100644
--- a/lld/COFF/Writer.cpp
+++ b/lld/COFF/Writer.cpp
@@ -2072,8 +2072,16 @@ void Writer::createRuntimePseudoRelocs() {
     return;
   }
 
-  if (!rels.empty())
+  if (!rels.empty()) {
     log("Writing " + Twine(rels.size()) + " runtime pseudo relocations");
+    const char *symbolName = "_pei386_runtime_relocator";
+    Symbol *relocator = ctx.symtab.findUnderscore(symbolName);
+    if (!relocator)
+      error("output image has runtime pseudo relocations, but the function " +
+            Twine(symbolName) +
+            " is missing; it is needed for fixing the relocations at runtime");
+  }
+
   PseudoRelocTableChunk *table = make<PseudoRelocTableChunk>(rels);
   rdataSec->addChunk(table);
   EmptyChunk *endOfList = make<EmptyChunk>();
diff --git a/lld/test/COFF/autoimport-arm-data.s b/lld/test/COFF/autoimport-arm-data.s
index 74604aa5c823..82c66f0989d4 100644
--- a/lld/test/COFF/autoimport-arm-data.s
+++ b/lld/test/COFF/autoimport-arm-data.s
@@ -33,6 +33,9 @@
     .text
     .thumb
 main:
+    bx lr
+    .global _pei386_runtime_relocator
+_pei386_runtime_relocator:
     bx lr
     .data
 ptr:
diff --git a/lld/test/COFF/autoimport-arm64-data.s b/lld/test/COFF/autoimport-arm64-data.s
index fa3654be3a71..b49bd4f89c97 100644
--- a/lld/test/COFF/autoimport-arm64-data.s
+++ b/lld/test/COFF/autoimport-arm64-data.s
@@ -33,6 +33,9 @@
     .global main
     .text
 main:
+    ret
+    .global _pei386_runtime_relocator
+_pei386_runtime_relocator:
     ret
     .data
 ptr:
diff --git a/lld/test/COFF/autoimport-gnu-implib.s b/lld/test/COFF/autoimport-gnu-implib.s
index d7d4ed626e83..d9dc9d7a38fd 100644
--- a/lld/test/COFF/autoimport-gnu-implib.s
+++ b/lld/test/COFF/autoimport-gnu-implib.s
@@ -27,5 +27,8 @@
     .text
 main:
     movl data(%rip), %eax
+    ret
+    .global _pei386_runtime_relocator
+_pei386_runtime_relocator:
     ret
     .data
diff --git a/lld/test/COFF/autoimport-handler-func.s b/lld/test/COFF/autoimport-handler-func.s
new file mode 100644
index 000000000000..02d040bfa274
--- /dev/null
+++ b/lld/test/COFF/autoimport-handler-func.s
@@ -0,0 +1,36 @@
+# REQUIRES: x86
+# RUN: split-file %s %t.dir
+
+# RUN: llvm-dlltool -m i386:x86-64 -d %t.dir/lib.def -D lib.dll -l %t.dir/lib.lib
+
+# RUN: llvm-mc -triple=x86_64-windows-gnu %t.dir/main.s -filetype=obj -o %t.dir/main.obj
+# RUN: llvm-mc -triple=x86_64-windows-gnu %t.dir/func.s -filetype=obj -o %t.dir/func.obj
+# RUN: env LLD_IN_TEST=1 not lld-link -lldmingw -out:%t.dir/main.exe -entry:main %t.dir/main.obj %t.dir/lib.lib 2>&1 | FileCheck %s --check-prefix=ERR
+
+# RUN: lld-link -lldmingw -out:%t.dir/main.exe -entry:main %t.dir/main.obj %t.dir/func.obj %t.dir/lib.lib 2>&1 | FileCheck %s --check-prefix=NOERR --allow-empty
+
+# ERR: error: output image has runtime pseudo relocations, but the function _pei386_runtime_relocator is missing; it is needed for fixing the relocations at runtime
+
+# NOERR-NOT: error
+
+#--- main.s
+    .global main
+    .text
+main:
+    ret
+
+    .data
+    .long 1
+    .quad variable
+    .long 2
+
+#--- func.s
+    .global _pei386_runtime_relocator
+    .text
+_pei386_runtime_relocator:
+    ret
+
+#--- lib.def
+EXPORTS
+variable DATA
+
diff --git a/lld/test/COFF/autoimport-warn.s b/lld/test/COFF/autoimport-warn.s
index 9c363ed30f24..eead0fed861f 100644
--- a/lld/test/COFF/autoimport-warn.s
+++ b/lld/test/COFF/autoimport-warn.s
@@ -18,6 +18,9 @@ main:
     movl variable2(%rip), %ecx
     addl %ecx, %eax
     ret
+    .global _pei386_runtime_relocator
+_pei386_runtime_relocator:
+    ret
 
     .section .rdata$.refptr.variable1,"dr",discard,.refptr.variable1
     .global .refptr.variable1
diff --git a/lld/test/COFF/autoimport-x86.s b/lld/test/COFF/autoimport-x86.s
index fa36f10e9ca9..5d7c9c2c3fa5 100644
--- a/lld/test/COFF/autoimport-x86.s
+++ b/lld/test/COFF/autoimport-x86.s
@@ -55,6 +55,9 @@
     .text
 main:
     movl variable(%rip), %eax
+    ret
+    .global _pei386_runtime_relocator
+_pei386_runtime_relocator:
     ret
     .data
 ptr:
-- 
GitLab


From dbaa1893c9afe6a245860efb8d68875ba4fd6794 Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp@bytedance.com>
Date: Tue, 16 Apr 2024 15:37:31 +0800
Subject: [PATCH 345/464] [RISCV] Generate more W instructons

We rename `TuneNoStripWSuffix` to `TunePreferWInst`.

If all the users of an instruction just use the low 32 bits, we can
convert it to its W variant.

A quick test on Coremark (`-O3 -march=rv64gc`):

|        | W instructions | code size(.text) |
|--------|----------------|------------------|
| before | 302            | 12257            |
| after  | 343            | 12265            |
|        | +13.58%        | +0.065%          |

Reviewers: asb, dtcxzyw, preames, lukel97, michaelmaitland, topperc

Reviewed By: topperc, dtcxzyw

Pull Request: https://github.com/llvm/llvm-project/pull/87237
---
 llvm/lib/Target/RISCV/RISCVFeatures.td    |   6 +-
 llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp |  89 +++++++-
 llvm/test/CodeGen/RISCV/prefer-w-inst.ll  | 105 +++++++++
 llvm/test/CodeGen/RISCV/prefer-w-inst.mir | 262 ++++++++++++++++++++++
 llvm/test/CodeGen/RISCV/strip-w-suffix.ll |  74 ------
 5 files changed, 447 insertions(+), 89 deletions(-)
 create mode 100644 llvm/test/CodeGen/RISCV/prefer-w-inst.ll
 create mode 100644 llvm/test/CodeGen/RISCV/prefer-w-inst.mir
 delete mode 100644 llvm/test/CodeGen/RISCV/strip-w-suffix.ll

diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td
index 794455aa7304..59962216e0c0 100644
--- a/llvm/lib/Target/RISCV/RISCVFeatures.td
+++ b/llvm/lib/Target/RISCV/RISCVFeatures.td
@@ -1226,9 +1226,9 @@ def TuneNoSinkSplatOperands
                        "false", "Disable sink splat operands to enable .vx, .vf,"
                        ".wx, and .wf instructions">;
 
-def TuneNoStripWSuffix
-    : SubtargetFeature<"no-strip-w-suffix", "EnableStripWSuffix", "false",
-                       "Disable strip W suffix">;
+def TunePreferWInst
+    : SubtargetFeature<"prefer-w-inst", "PreferWInst", "true",
+                       "Prefer instructions with W suffix">;
 
 def TuneConditionalCompressedMoveFusion
     : SubtargetFeature<"conditional-cmv-fusion", "HasConditionalCompressedMoveFusion",
diff --git a/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp b/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp
index 39d420c2fbf0..ead91c5656be 100644
--- a/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp
+++ b/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp
@@ -12,15 +12,24 @@
 // extended bits aren't consumed or because the input was already sign extended
 // by an earlier instruction.
 //
-// Then it removes the -w suffix from opw instructions whenever all users are
-// dependent only on the lower word of the result of the instruction.
-// The cases handled are:
-// * addw because c.add has a larger register encoding than c.addw.
-// * addiw because it helps reduce test differences between RV32 and RV64
-//   w/o being a pessimization.
-// * mulw because c.mulw doesn't exist but c.mul does (w/ zcb)
-// * slliw because c.slliw doesn't exist and c.slli does
+// Then:
+// 1. Unless explicit disabled or the target prefers instructions with W suffix,
+//    it removes the -w suffix from opw instructions whenever all users are
+//    dependent only on the lower word of the result of the instruction.
+//    The cases handled are:
+//    * addw because c.add has a larger register encoding than c.addw.
+//    * addiw because it helps reduce test differences between RV32 and RV64
+//      w/o being a pessimization.
+//    * mulw because c.mulw doesn't exist but c.mul does (w/ zcb)
+//    * slliw because c.slliw doesn't exist and c.slli does
 //
+// 2. Or if explicit enabled or the target prefers instructions with W suffix,
+//    it adds the W suffix to the instruction whenever all users are dependent
+//    only on the lower word of the result of the instruction.
+//    The cases handled are:
+//    * add/addi/sub/mul.
+//    * slli with imm < 32.
+//    * ld/lwu.
 //===---------------------------------------------------------------------===//
 
 #include "RISCV.h"
@@ -60,6 +69,8 @@ public:
                          const RISCVSubtarget &ST, MachineRegisterInfo &MRI);
   bool stripWSuffixes(MachineFunction &MF, const RISCVInstrInfo &TII,
                       const RISCVSubtarget &ST, MachineRegisterInfo &MRI);
+  bool appendWSuffixes(MachineFunction &MF, const RISCVInstrInfo &TII,
+                       const RISCVSubtarget &ST, MachineRegisterInfo &MRI);
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.setPreservesCFG();
@@ -672,9 +683,6 @@ bool RISCVOptWInstrs::stripWSuffixes(MachineFunction &MF,
                                      const RISCVInstrInfo &TII,
                                      const RISCVSubtarget &ST,
                                      MachineRegisterInfo &MRI) {
-  if (DisableStripWSuffix || !ST.enableStripWSuffix())
-    return false;
-
   bool MadeChange = false;
   for (MachineBasicBlock &MBB : MF) {
     for (MachineInstr &MI : MBB) {
@@ -698,6 +706,58 @@ bool RISCVOptWInstrs::stripWSuffixes(MachineFunction &MF,
   return MadeChange;
 }
 
+bool RISCVOptWInstrs::appendWSuffixes(MachineFunction &MF,
+                                      const RISCVInstrInfo &TII,
+                                      const RISCVSubtarget &ST,
+                                      MachineRegisterInfo &MRI) {
+  bool MadeChange = false;
+  for (MachineBasicBlock &MBB : MF) {
+    for (MachineInstr &MI : MBB) {
+      unsigned WOpc;
+      // TODO: Add more?
+      switch (MI.getOpcode()) {
+      default:
+        continue;
+      case RISCV::ADD:
+        WOpc = RISCV::ADDW;
+        break;
+      case RISCV::ADDI:
+        WOpc = RISCV::ADDIW;
+        break;
+      case RISCV::SUB:
+        WOpc = RISCV::SUBW;
+        break;
+      case RISCV::MUL:
+        WOpc = RISCV::MULW;
+        break;
+      case RISCV::SLLI:
+        // SLLIW reads the lowest 5 bits, while SLLI reads lowest 6 bits
+        if (MI.getOperand(2).getImm() >= 32)
+          continue;
+        WOpc = RISCV::SLLIW;
+        break;
+      case RISCV::LD:
+      case RISCV::LWU:
+        WOpc = RISCV::LW;
+        break;
+      }
+
+      if (hasAllWUsers(MI, ST, MRI)) {
+        LLVM_DEBUG(dbgs() << "Replacing " << MI);
+        MI.setDesc(TII.get(WOpc));
+        MI.clearFlag(MachineInstr::MIFlag::NoSWrap);
+        MI.clearFlag(MachineInstr::MIFlag::NoUWrap);
+        MI.clearFlag(MachineInstr::MIFlag::IsExact);
+        LLVM_DEBUG(dbgs() << "     with " << MI);
+        ++NumTransformedToWInstrs;
+        MadeChange = true;
+      }
+    }
+  }
+
+  return MadeChange;
+}
+
 bool RISCVOptWInstrs::runOnMachineFunction(MachineFunction &MF) {
   if (skipFunction(MF.getFunction()))
     return false;
@@ -711,7 +771,12 @@ bool RISCVOptWInstrs::runOnMachineFunction(MachineFunction &MF) {
 
   bool MadeChange = false;
   MadeChange |= removeSExtWInstrs(MF, TII, ST, MRI);
-  MadeChange |= stripWSuffixes(MF, TII, ST, MRI);
+
+  if (!(DisableStripWSuffix || ST.preferWInst()))
+    MadeChange |= stripWSuffixes(MF, TII, ST, MRI);
+
+  if (ST.preferWInst())
+    MadeChange |= appendWSuffixes(MF, TII, ST, MRI);
 
   return MadeChange;
 }
diff --git a/llvm/test/CodeGen/RISCV/prefer-w-inst.ll b/llvm/test/CodeGen/RISCV/prefer-w-inst.ll
new file mode 100644
index 000000000000..34ab74d78a76
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/prefer-w-inst.ll
@@ -0,0 +1,105 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -mattr=+m -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefixes=NO-PREFER-W-INST %s
+; RUN: llc -mtriple=riscv64 -mattr=+m -riscv-disable-strip-w-suffix -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefixes=NO-STRIP %s
+; RUN: llc -mtriple=riscv64 -mattr=+m,+prefer-w-inst -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefixes=PREFER-W-INST %s
+
+define i32 @addiw(i32 %a) {
+; NO-PREFER-W-INST-LABEL: addiw:
+; NO-PREFER-W-INST:       # %bb.0:
+; NO-PREFER-W-INST-NEXT:    lui a1, 1
+; NO-PREFER-W-INST-NEXT:    addi a1, a1, -1
+; NO-PREFER-W-INST-NEXT:    addw a0, a0, a1
+; NO-PREFER-W-INST-NEXT:    ret
+;
+; NO-STRIP-LABEL: addiw:
+; NO-STRIP:       # %bb.0:
+; NO-STRIP-NEXT:    lui a1, 1
+; NO-STRIP-NEXT:    addiw a1, a1, -1
+; NO-STRIP-NEXT:    addw a0, a0, a1
+; NO-STRIP-NEXT:    ret
+;
+; PREFER-W-INST-LABEL: addiw:
+; PREFER-W-INST:       # %bb.0:
+; PREFER-W-INST-NEXT:    lui a1, 1
+; PREFER-W-INST-NEXT:    addiw a1, a1, -1
+; PREFER-W-INST-NEXT:    addw a0, a0, a1
+; PREFER-W-INST-NEXT:    ret
+  %ret = add i32 %a, 4095
+  ret i32 %ret
+}
+
+define i32 @addw(i32 %a, i32 %b) {
+; NO-PREFER-W-INST-LABEL: addw:
+; NO-PREFER-W-INST:       # %bb.0:
+; NO-PREFER-W-INST-NEXT:    add a0, a0, a1
+; NO-PREFER-W-INST-NEXT:    addiw a0, a0, 1024
+; NO-PREFER-W-INST-NEXT:    ret
+;
+; NO-STRIP-LABEL: addw:
+; NO-STRIP:       # %bb.0:
+; NO-STRIP-NEXT:    addw a0, a0, a1
+; NO-STRIP-NEXT:    addiw a0, a0, 1024
+; NO-STRIP-NEXT:    ret
+;
+; PREFER-W-INST-LABEL: addw:
+; PREFER-W-INST:       # %bb.0:
+; PREFER-W-INST-NEXT:    addw a0, a0, a1
+; PREFER-W-INST-NEXT:    addiw a0, a0, 1024
+; PREFER-W-INST-NEXT:    ret
+  %add = add i32 %a, %b
+  %ret = add i32 %add, 1024
+  ret i32 %ret
+}
+
+define i32 @mulw(i32 %a, i32 %b) {
+; NO-PREFER-W-INST-LABEL: mulw:
+; NO-PREFER-W-INST:       # %bb.0:
+; NO-PREFER-W-INST-NEXT:    mul a1, a0, a1
+; NO-PREFER-W-INST-NEXT:    mul a0, a0, a1
+; NO-PREFER-W-INST-NEXT:    addiw a0, a0, 1024
+; NO-PREFER-W-INST-NEXT:    ret
+;
+; NO-STRIP-LABEL: mulw:
+; NO-STRIP:       # %bb.0:
+; NO-STRIP-NEXT:    mulw a1, a0, a1
+; NO-STRIP-NEXT:    mulw a0, a0, a1
+; NO-STRIP-NEXT:    addiw a0, a0, 1024
+; NO-STRIP-NEXT:    ret
+;
+; PREFER-W-INST-LABEL: mulw:
+; PREFER-W-INST:       # %bb.0:
+; PREFER-W-INST-NEXT:    mulw a1, a0, a1
+; PREFER-W-INST-NEXT:    mulw a0, a0, a1
+; PREFER-W-INST-NEXT:    addiw a0, a0, 1024
+; PREFER-W-INST-NEXT:    ret
+  %mul1 = mul i32 %a, %b
+  %mul = mul i32 %a, %mul1
+  %ret = add i32 %mul, 1024
+  ret i32 %ret
+}
+
+define i32 @slliw(i32 %a) {
+; NO-PREFER-W-INST-LABEL: slliw:
+; NO-PREFER-W-INST:       # %bb.0:
+; NO-PREFER-W-INST-NEXT:    slli a0, a0, 1
+; NO-PREFER-W-INST-NEXT:    addiw a0, a0, 1024
+; NO-PREFER-W-INST-NEXT:    ret
+;
+; NO-STRIP-LABEL: slliw:
+; NO-STRIP:       # %bb.0:
+; NO-STRIP-NEXT:    slliw a0, a0, 1
+; NO-STRIP-NEXT:    addiw a0, a0, 1024
+; NO-STRIP-NEXT:    ret
+;
+; PREFER-W-INST-LABEL: slliw:
+; PREFER-W-INST:       # %bb.0:
+; PREFER-W-INST-NEXT:    slliw a0, a0, 1
+; PREFER-W-INST-NEXT:    addiw a0, a0, 1024
+; PREFER-W-INST-NEXT:    ret
+  %shl = shl i32 %a, 1
+  %ret = add i32 %shl, 1024
+  ret i32 %ret
+}
diff --git a/llvm/test/CodeGen/RISCV/prefer-w-inst.mir b/llvm/test/CodeGen/RISCV/prefer-w-inst.mir
new file mode 100644
index 000000000000..e05e27af4271
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/prefer-w-inst.mir
@@ -0,0 +1,262 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
+# RUN: llc %s -mtriple=riscv64 -run-pass=riscv-opt-w-instrs -verify-machineinstrs \
+# RUN:   -mattr=+m -o - | FileCheck %s -check-prefixes=NO-PREFER-W-INST
+# RUN: llc %s -mtriple=riscv64 -run-pass=riscv-opt-w-instrs -verify-machineinstrs \
+# RUN:   -mattr=+m,+prefer-w-inst -o - | FileCheck %s -check-prefixes=PREFER-W-INST
+
+---
+name: addi
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    ; NO-PREFER-W-INST-LABEL: name: addi
+    ; NO-PREFER-W-INST: liveins: $x10, $x11
+    ; NO-PREFER-W-INST-NEXT: {{  $}}
+    ; NO-PREFER-W-INST-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; NO-PREFER-W-INST-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; NO-PREFER-W-INST-NEXT: [[ADDI:%[0-9]+]]:gpr = ADDI [[COPY]], 1
+    ; NO-PREFER-W-INST-NEXT: [[ADDIW:%[0-9]+]]:gpr = ADDIW [[ADDI]], 1
+    ; NO-PREFER-W-INST-NEXT: $x10 = COPY [[ADDIW]]
+    ; NO-PREFER-W-INST-NEXT: PseudoRET
+    ;
+    ; PREFER-W-INST-LABEL: name: addi
+    ; PREFER-W-INST: liveins: $x10, $x11
+    ; PREFER-W-INST-NEXT: {{  $}}
+    ; PREFER-W-INST-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; PREFER-W-INST-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; PREFER-W-INST-NEXT: [[ADDIW:%[0-9]+]]:gpr = ADDIW [[COPY]], 1
+    ; PREFER-W-INST-NEXT: [[ADDIW1:%[0-9]+]]:gpr = ADDIW [[ADDIW]], 1
+    ; PREFER-W-INST-NEXT: $x10 = COPY [[ADDIW1]]
+    ; PREFER-W-INST-NEXT: PseudoRET
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = ADDI %1, 1
+    %4:gpr = ADDIW %3, 1
+    $x10 = COPY %4
+    PseudoRET
+...
+
+---
+name: add
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    ; NO-PREFER-W-INST-LABEL: name: add
+    ; NO-PREFER-W-INST: liveins: $x10, $x11
+    ; NO-PREFER-W-INST-NEXT: {{  $}}
+    ; NO-PREFER-W-INST-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; NO-PREFER-W-INST-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; NO-PREFER-W-INST-NEXT: [[ADD:%[0-9]+]]:gpr = ADD [[COPY]], [[COPY1]]
+    ; NO-PREFER-W-INST-NEXT: [[ADDIW:%[0-9]+]]:gpr = ADDIW [[ADD]], 1
+    ; NO-PREFER-W-INST-NEXT: $x10 = COPY [[ADDIW]]
+    ; NO-PREFER-W-INST-NEXT: PseudoRET
+    ;
+    ; PREFER-W-INST-LABEL: name: add
+    ; PREFER-W-INST: liveins: $x10, $x11
+    ; PREFER-W-INST-NEXT: {{  $}}
+    ; PREFER-W-INST-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; PREFER-W-INST-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; PREFER-W-INST-NEXT: [[ADDW:%[0-9]+]]:gpr = ADDW [[COPY]], [[COPY1]]
+    ; PREFER-W-INST-NEXT: [[ADDIW:%[0-9]+]]:gpr = ADDIW [[ADDW]], 1
+    ; PREFER-W-INST-NEXT: $x10 = COPY [[ADDIW]]
+    ; PREFER-W-INST-NEXT: PseudoRET
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = ADD %1, %2
+    %4:gpr = ADDIW %3, 1
+    $x10 = COPY %4
+    PseudoRET
+...
+
+---
+name: sub
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    ; NO-PREFER-W-INST-LABEL: name: sub
+    ; NO-PREFER-W-INST: liveins: $x10, $x11
+    ; NO-PREFER-W-INST-NEXT: {{  $}}
+    ; NO-PREFER-W-INST-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; NO-PREFER-W-INST-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; NO-PREFER-W-INST-NEXT: [[SUB:%[0-9]+]]:gpr = SUB [[COPY]], [[COPY1]]
+    ; NO-PREFER-W-INST-NEXT: [[ADDIW:%[0-9]+]]:gpr = ADDIW [[SUB]], 1
+    ; NO-PREFER-W-INST-NEXT: $x10 = COPY [[ADDIW]]
+    ; NO-PREFER-W-INST-NEXT: PseudoRET
+    ;
+    ; PREFER-W-INST-LABEL: name: sub
+    ; PREFER-W-INST: liveins: $x10, $x11
+    ; PREFER-W-INST-NEXT: {{  $}}
+    ; PREFER-W-INST-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; PREFER-W-INST-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; PREFER-W-INST-NEXT: [[SUBW:%[0-9]+]]:gpr = SUBW [[COPY]], [[COPY1]]
+    ; PREFER-W-INST-NEXT: [[ADDIW:%[0-9]+]]:gpr = ADDIW [[SUBW]], 1
+    ; PREFER-W-INST-NEXT: $x10 = COPY [[ADDIW]]
+    ; PREFER-W-INST-NEXT: PseudoRET
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SUB %1, %2
+    %4:gpr = ADDIW %3, 1
+    $x10 = COPY %4
+    PseudoRET
+...
+
+---
+name: mul
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    ; NO-PREFER-W-INST-LABEL: name: mul
+    ; NO-PREFER-W-INST: liveins: $x10, $x11
+    ; NO-PREFER-W-INST-NEXT: {{  $}}
+    ; NO-PREFER-W-INST-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; NO-PREFER-W-INST-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; NO-PREFER-W-INST-NEXT: [[MUL:%[0-9]+]]:gpr = MUL [[COPY]], [[COPY1]]
+    ; NO-PREFER-W-INST-NEXT: [[ADDIW:%[0-9]+]]:gpr = ADDIW [[MUL]], 1
+    ; NO-PREFER-W-INST-NEXT: $x10 = COPY [[ADDIW]]
+    ; NO-PREFER-W-INST-NEXT: PseudoRET
+    ;
+    ; PREFER-W-INST-LABEL: name: mul
+    ; PREFER-W-INST: liveins: $x10, $x11
+    ; PREFER-W-INST-NEXT: {{  $}}
+    ; PREFER-W-INST-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; PREFER-W-INST-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; PREFER-W-INST-NEXT: [[MULW:%[0-9]+]]:gpr = MULW [[COPY]], [[COPY1]]
+    ; PREFER-W-INST-NEXT: [[ADDIW:%[0-9]+]]:gpr = ADDIW [[MULW]], 1
+    ; PREFER-W-INST-NEXT: $x10 = COPY [[ADDIW]]
+    ; PREFER-W-INST-NEXT: PseudoRET
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = MUL %1, %2
+    %4:gpr = ADDIW %3, 1
+    $x10 = COPY %4
+    PseudoRET
+...
+
+
+---
+name: slli_31
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    ; NO-PREFER-W-INST-LABEL: name: slli_31
+    ; NO-PREFER-W-INST: liveins: $x10, $x11
+    ; NO-PREFER-W-INST-NEXT: {{  $}}
+    ; NO-PREFER-W-INST-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; NO-PREFER-W-INST-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; NO-PREFER-W-INST-NEXT: [[SLLI:%[0-9]+]]:gpr = SLLI [[COPY]], 31
+    ; NO-PREFER-W-INST-NEXT: [[ADDIW:%[0-9]+]]:gpr = ADDIW [[SLLI]], 1
+    ; NO-PREFER-W-INST-NEXT: $x10 = COPY [[ADDIW]]
+    ; NO-PREFER-W-INST-NEXT: PseudoRET
+    ;
+    ; PREFER-W-INST-LABEL: name: slli_31
+    ; PREFER-W-INST: liveins: $x10, $x11
+    ; PREFER-W-INST-NEXT: {{  $}}
+    ; PREFER-W-INST-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; PREFER-W-INST-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; PREFER-W-INST-NEXT: [[SLLIW:%[0-9]+]]:gpr = SLLIW [[COPY]], 31
+    ; PREFER-W-INST-NEXT: [[ADDIW:%[0-9]+]]:gpr = ADDIW [[SLLIW]], 1
+    ; PREFER-W-INST-NEXT: $x10 = COPY [[ADDIW]]
+    ; PREFER-W-INST-NEXT: PseudoRET
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SLLI %1, 31
+    %4:gpr = ADDIW %3, 1
+    $x10 = COPY %4
+    PseudoRET
+...
+
+---
+name: slli_32
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    ; NO-PREFER-W-INST-LABEL: name: slli_32
+    ; NO-PREFER-W-INST: liveins: $x10, $x11
+    ; NO-PREFER-W-INST-NEXT: {{  $}}
+    ; NO-PREFER-W-INST-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; NO-PREFER-W-INST-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; NO-PREFER-W-INST-NEXT: [[SLLI:%[0-9]+]]:gpr = SLLI [[COPY]], 32
+    ; NO-PREFER-W-INST-NEXT: [[ADDIW:%[0-9]+]]:gpr = ADDIW [[SLLI]], 1
+    ; NO-PREFER-W-INST-NEXT: $x10 = COPY [[ADDIW]]
+    ; NO-PREFER-W-INST-NEXT: PseudoRET
+    ;
+    ; PREFER-W-INST-LABEL: name: slli_32
+    ; PREFER-W-INST: liveins: $x10, $x11
+    ; PREFER-W-INST-NEXT: {{  $}}
+    ; PREFER-W-INST-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; PREFER-W-INST-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; PREFER-W-INST-NEXT: [[SLLI:%[0-9]+]]:gpr = SLLI [[COPY]], 32
+    ; PREFER-W-INST-NEXT: [[ADDIW:%[0-9]+]]:gpr = ADDIW [[SLLI]], 1
+    ; PREFER-W-INST-NEXT: $x10 = COPY [[ADDIW]]
+    ; PREFER-W-INST-NEXT: PseudoRET
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = SLLI %1, 32
+    %4:gpr = ADDIW %3, 1
+    $x10 = COPY %4
+    PseudoRET
+...
+
+---
+name: ld
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    ; NO-PREFER-W-INST-LABEL: name: ld
+    ; NO-PREFER-W-INST: liveins: $x10, $x11
+    ; NO-PREFER-W-INST-NEXT: {{  $}}
+    ; NO-PREFER-W-INST-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; NO-PREFER-W-INST-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; NO-PREFER-W-INST-NEXT: [[LD:%[0-9]+]]:gpr = LD [[COPY]], 0
+    ; NO-PREFER-W-INST-NEXT: [[ADDIW:%[0-9]+]]:gpr = ADDIW [[LD]], 1
+    ; NO-PREFER-W-INST-NEXT: $x10 = COPY [[ADDIW]]
+    ; NO-PREFER-W-INST-NEXT: PseudoRET
+    ;
+    ; PREFER-W-INST-LABEL: name: ld
+    ; PREFER-W-INST: liveins: $x10, $x11
+    ; PREFER-W-INST-NEXT: {{  $}}
+    ; PREFER-W-INST-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; PREFER-W-INST-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; PREFER-W-INST-NEXT: [[LW:%[0-9]+]]:gpr = LW [[COPY]], 0
+    ; PREFER-W-INST-NEXT: [[ADDIW:%[0-9]+]]:gpr = ADDIW [[LW]], 1
+    ; PREFER-W-INST-NEXT: $x10 = COPY [[ADDIW]]
+    ; PREFER-W-INST-NEXT: PseudoRET
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = LD %1, 0
+    %4:gpr = ADDIW %3, 1
+    $x10 = COPY %4
+    PseudoRET
+...
+
+---
+name: lwu
+body:             |
+  bb.0.entry:
+    liveins: $x10, $x11
+    ; NO-PREFER-W-INST-LABEL: name: lwu
+    ; NO-PREFER-W-INST: liveins: $x10, $x11
+    ; NO-PREFER-W-INST-NEXT: {{  $}}
+    ; NO-PREFER-W-INST-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; NO-PREFER-W-INST-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; NO-PREFER-W-INST-NEXT: [[LWU:%[0-9]+]]:gpr = LWU [[COPY]], 0
+    ; NO-PREFER-W-INST-NEXT: [[ADDIW:%[0-9]+]]:gpr = ADDIW [[LWU]], 1
+    ; NO-PREFER-W-INST-NEXT: $x10 = COPY [[ADDIW]]
+    ; NO-PREFER-W-INST-NEXT: PseudoRET
+    ;
+    ; PREFER-W-INST-LABEL: name: lwu
+    ; PREFER-W-INST: liveins: $x10, $x11
+    ; PREFER-W-INST-NEXT: {{  $}}
+    ; PREFER-W-INST-NEXT: [[COPY:%[0-9]+]]:gpr = COPY $x10
+    ; PREFER-W-INST-NEXT: [[COPY1:%[0-9]+]]:gpr = COPY $x11
+    ; PREFER-W-INST-NEXT: [[LW:%[0-9]+]]:gpr = LW [[COPY]], 0
+    ; PREFER-W-INST-NEXT: [[ADDIW:%[0-9]+]]:gpr = ADDIW [[LW]], 1
+    ; PREFER-W-INST-NEXT: $x10 = COPY [[ADDIW]]
+    ; PREFER-W-INST-NEXT: PseudoRET
+    %1:gpr = COPY $x10
+    %2:gpr = COPY $x11
+    %3:gpr = LWU %1, 0
+    %4:gpr = ADDIW %3, 1
+    $x10 = COPY %4
+    PseudoRET
+...
diff --git a/llvm/test/CodeGen/RISCV/strip-w-suffix.ll b/llvm/test/CodeGen/RISCV/strip-w-suffix.ll
deleted file mode 100644
index 4124b3d0d360..000000000000
--- a/llvm/test/CodeGen/RISCV/strip-w-suffix.ll
+++ /dev/null
@@ -1,74 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv64 -mattr=+m -verify-machineinstrs < %s \
-; RUN:   | FileCheck -check-prefixes=STRIP %s
-; RUN: llc -mtriple=riscv64 -mattr=+m,+no-strip-w-suffix -verify-machineinstrs < %s \
-; RUN:   | FileCheck -check-prefixes=NO-STRIP %s
-
-define i32 @addiw(i32 %a) {
-; STRIP-LABEL: addiw:
-; STRIP:       # %bb.0:
-; STRIP-NEXT:    lui a1, 1
-; STRIP-NEXT:    addi a1, a1, -1
-; STRIP-NEXT:    addw a0, a0, a1
-; STRIP-NEXT:    ret
-;
-; NO-STRIP-LABEL: addiw:
-; NO-STRIP:       # %bb.0:
-; NO-STRIP-NEXT:    lui a1, 1
-; NO-STRIP-NEXT:    addiw a1, a1, -1
-; NO-STRIP-NEXT:    addw a0, a0, a1
-; NO-STRIP-NEXT:    ret
-  %ret = add i32 %a, 4095
-  ret i32 %ret
-}
-
-define i32 @addw(i32 %a, i32 %b) {
-; STRIP-LABEL: addw:
-; STRIP:       # %bb.0:
-; STRIP-NEXT:    add a0, a0, a1
-; STRIP-NEXT:    addiw a0, a0, 1024
-; STRIP-NEXT:    ret
-;
-; NO-STRIP-LABEL: addw:
-; NO-STRIP:       # %bb.0:
-; NO-STRIP-NEXT:    addw a0, a0, a1
-; NO-STRIP-NEXT:    addiw a0, a0, 1024
-; NO-STRIP-NEXT:    ret
-  %add = add i32 %a, %b
-  %ret = add i32 %add, 1024
-  ret i32 %ret
-}
-
-define i32 @mulw(i32 %a, i32 %b) {
-; STRIP-LABEL: mulw:
-; STRIP:       # %bb.0:
-; STRIP-NEXT:    mul a0, a0, a1
-; STRIP-NEXT:    addiw a0, a0, 1024
-; STRIP-NEXT:    ret
-;
-; NO-STRIP-LABEL: mulw:
-; NO-STRIP:       # %bb.0:
-; NO-STRIP-NEXT:    mulw a0, a0, a1
-; NO-STRIP-NEXT:    addiw a0, a0, 1024
-; NO-STRIP-NEXT:    ret
-  %mul = mul i32 %a, %b
-  %ret = add i32 %mul, 1024
-  ret i32 %ret
-}
-
-define i32 @slliw(i32 %a) {
-; STRIP-LABEL: slliw:
-; STRIP:       # %bb.0:
-; STRIP-NEXT:    slli a0, a0, 1
-; STRIP-NEXT:    addiw a0, a0, 1024
-; STRIP-NEXT:    ret
-;
-; NO-STRIP-LABEL: slliw:
-; NO-STRIP:       # %bb.0:
-; NO-STRIP-NEXT:    slliw a0, a0, 1
-; NO-STRIP-NEXT:    addiw a0, a0, 1024
-; NO-STRIP-NEXT:    ret
-  %shl = shl i32 %a, 1
-  %ret = add i32 %shl, 1024
-  ret i32 %ret
-}
-- 
GitLab


From d26dd58ca5b59032eb371b8f51d9134acdd8d3ad Mon Sep 17 00:00:00 2001
From: Chuanqi Xu <yedeng.yd@linux.alibaba.com>
Date: Tue, 16 Apr 2024 14:37:38 +0800
Subject: [PATCH 346/464] [StmtProfile] Don't profile the body of lambda
 expressions

Close https://github.com/llvm/llvm-project/issues/87609

We tried to profile the body of the lambda expressions in
https://reviews.llvm.org/D153957. But as the original comments show,
it is indeed dangerous. After we tried to skip calculating the ODR
hash values recently, we have fall into this trap twice.

So in this patch, I choose to not profile the body of the lambda
expression. The signature of the lambda is still profiled.
---
 clang/include/clang/AST/DeclBase.h            | 10 -----
 clang/include/clang/Serialization/ASTReader.h |  6 +++
 clang/lib/AST/Decl.cpp                        |  2 +-
 clang/lib/AST/DeclBase.cpp                    |  5 ---
 clang/lib/AST/StmtProfile.cpp                 | 22 +++++++++-
 clang/lib/Serialization/ASTReader.cpp         |  2 +-
 clang/lib/Serialization/ASTReaderDecl.cpp     |  8 ++--
 clang/lib/Serialization/ASTWriter.cpp         |  2 +-
 clang/lib/Serialization/ASTWriterDecl.cpp     |  8 ++--
 .../hashing-decls-in-exprs-from-gmf-2.cppm    | 44 +++++++++++++++++++
 10 files changed, 81 insertions(+), 28 deletions(-)
 create mode 100644 clang/test/Modules/hashing-decls-in-exprs-from-gmf-2.cppm

diff --git a/clang/include/clang/AST/DeclBase.h b/clang/include/clang/AST/DeclBase.h
index 2194d268fa86..1079993f4969 100644
--- a/clang/include/clang/AST/DeclBase.h
+++ b/clang/include/clang/AST/DeclBase.h
@@ -672,16 +672,6 @@ public:
   /// Whether this declaration comes from explicit global module.
   bool isFromExplicitGlobalModule() const;
 
-  /// Check if we should skip checking ODRHash for declaration \param D.
-  ///
-  /// The existing ODRHash mechanism seems to be not stable enough and
-  /// the false positive ODR violation reports are annoying and we rarely see
-  /// true ODR violation reports. Also we learned that MSVC disabled ODR checks
-  /// for declarations in GMF. So we try to disable ODR checks in the GMF to
-  /// get better user experiences before we make the ODR violation checks stable
-  /// enough.
-  bool shouldSkipCheckingODR() const;
-
   /// Return true if this declaration has an attribute which acts as
   /// definition of the entity, such as 'alias' or 'ifunc'.
   bool hasDefiningAttr() const;
diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index e3fde887f99c..43ee06c524b3 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -2457,6 +2457,12 @@ private:
   uint32_t Value;
   uint32_t CurrentBitsIndex = ~0;
 };
+
+inline bool shouldSkipCheckingODR(const Decl *D) {
+  return D->getASTContext().getLangOpts().SkipODRCheckInGMF &&
+         D->isFromExplicitGlobalModule();
+}
+
 } // namespace clang
 
 #endif // LLVM_CLANG_SERIALIZATION_ASTREADER_H
diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp
index 2b2d5a2663a1..33b6f8611f21 100644
--- a/clang/lib/AST/Decl.cpp
+++ b/clang/lib/AST/Decl.cpp
@@ -4534,7 +4534,7 @@ unsigned FunctionDecl::getODRHash() {
   }
 
   class ODRHash Hash;
-  Hash.AddFunctionDecl(this, /*SkipBody=*/shouldSkipCheckingODR());
+  Hash.AddFunctionDecl(this);
   setHasODRHash(true);
   ODRHash = Hash.CalculateHash();
   return ODRHash;
diff --git a/clang/lib/AST/DeclBase.cpp b/clang/lib/AST/DeclBase.cpp
index 66a727d9dd0c..434926324c96 100644
--- a/clang/lib/AST/DeclBase.cpp
+++ b/clang/lib/AST/DeclBase.cpp
@@ -1106,11 +1106,6 @@ bool Decl::isFromExplicitGlobalModule() const {
   return getOwningModule() && getOwningModule()->isExplicitGlobalModule();
 }
 
-bool Decl::shouldSkipCheckingODR() const {
-  return getASTContext().getLangOpts().SkipODRCheckInGMF &&
-         isFromExplicitGlobalModule();
-}
-
 static Decl::Kind getKind(const Decl *D) { return D->getKind(); }
 static Decl::Kind getKind(const DeclContext *DC) { return DC->getDeclKind(); }
 
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index d2aac1e64038..789e4634bd29 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -2071,13 +2071,31 @@ StmtProfiler::VisitLambdaExpr(const LambdaExpr *S) {
   }
 
   CXXRecordDecl *Lambda = S->getLambdaClass();
-  ID.AddInteger(Lambda->getODRHash());
-
   for (const auto &Capture : Lambda->captures()) {
     ID.AddInteger(Capture.getCaptureKind());
     if (Capture.capturesVariable())
       VisitDecl(Capture.getCapturedVar());
   }
+
+  // Profiling the body of the lambda may be dangerous during deserialization.
+  // So we'd like only to profile the signature here.
+  ODRHash Hasher;
+  // FIXME: We can't get the operator call easily by
+  // `CXXRecordDecl::getLambdaCallOperator()` if we're in deserialization.
+  // So we have to do something raw here.
+  for (auto *SubDecl : Lambda->decls()) {
+    FunctionDecl *Call = nullptr;
+    if (auto *FTD = dyn_cast<FunctionTemplateDecl>(SubDecl))
+      Call = FTD->getTemplatedDecl();
+    else if (auto *FD = dyn_cast<FunctionDecl>(SubDecl))
+      Call = FD;
+
+    if (!Call)
+      continue;
+
+    Hasher.AddFunctionDecl(Call, /*SkipBody=*/true);
+  }
+  ID.AddInteger(Hasher.CalculateHash());
 }
 
 void
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index feb60bc54413..f47d540ea4b8 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -9785,7 +9785,7 @@ void ASTReader::finishPendingActions() {
             !NonConstDefn->isLateTemplateParsed() &&
             // We only perform ODR checks for decls not in the explicit
             // global module fragment.
-            !FD->shouldSkipCheckingODR() &&
+            !shouldSkipCheckingODR(FD) &&
             FD->getODRHash() != NonConstDefn->getODRHash()) {
           if (!isa<CXXMethodDecl>(FD)) {
             PendingFunctionOdrMergeFailures[FD].push_back(NonConstDefn);
diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index e4b6a75c118b..74d40f7da34c 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -826,7 +826,7 @@ void ASTDeclReader::VisitEnumDecl(EnumDecl *ED) {
       Reader.mergeDefinitionVisibility(OldDef, ED);
       // We don't want to check the ODR hash value for declarations from global
       // module fragment.
-      if (!ED->shouldSkipCheckingODR() &&
+      if (!shouldSkipCheckingODR(ED) &&
           OldDef->getODRHash() != ED->getODRHash())
         Reader.PendingEnumOdrMergeFailures[OldDef].push_back(ED);
     } else {
@@ -868,7 +868,7 @@ void ASTDeclReader::VisitRecordDecl(RecordDecl *RD) {
   VisitRecordDeclImpl(RD);
   // We should only reach here if we're in C/Objective-C. There is no
   // global module fragment.
-  assert(!RD->shouldSkipCheckingODR());
+  assert(!shouldSkipCheckingODR(RD));
   RD->setODRHash(Record.readInt());
 
   // Maintain the invariant of a redeclaration chain containing only
@@ -2155,7 +2155,7 @@ void ASTDeclReader::MergeDefinitionData(
   }
 
   // We don't want to check ODR for decls in the global module fragment.
-  if (MergeDD.Definition->shouldSkipCheckingODR())
+  if (shouldSkipCheckingODR(MergeDD.Definition))
     return;
 
   if (D->getODRHash() != MergeDD.ODRHash) {
@@ -3530,7 +3530,7 @@ ASTDeclReader::FindExistingResult ASTDeclReader::findExisting(NamedDecl *D) {
   // same template specialization into the same CXXRecordDecl.
   auto MergedDCIt = Reader.MergedDeclContexts.find(D->getLexicalDeclContext());
   if (MergedDCIt != Reader.MergedDeclContexts.end() &&
-      !D->shouldSkipCheckingODR() && MergedDCIt->second == D->getDeclContext())
+      !shouldSkipCheckingODR(D) && MergedDCIt->second == D->getDeclContext())
     Reader.PendingOdrMergeChecks.push_back(D);
 
   return FindExistingResult(Reader, D, /*Existing=*/nullptr,
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 85b7fd5535a1..ce6fa1feb1ee 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -6188,7 +6188,7 @@ void ASTRecordWriter::AddCXXDefinitionData(const CXXRecordDecl *D) {
 
   BitsPacker DefinitionBits;
 
-  bool ShouldSkipCheckingODR = D->shouldSkipCheckingODR();
+  bool ShouldSkipCheckingODR = shouldSkipCheckingODR(D);
   DefinitionBits.addBit(ShouldSkipCheckingODR);
 
 #define FIELD(Name, Width, Merge)                                              \
diff --git a/clang/lib/Serialization/ASTWriterDecl.cpp b/clang/lib/Serialization/ASTWriterDecl.cpp
index 276b6257f1d8..d0d49bcdf991 100644
--- a/clang/lib/Serialization/ASTWriterDecl.cpp
+++ b/clang/lib/Serialization/ASTWriterDecl.cpp
@@ -526,7 +526,7 @@ void ASTDeclWriter::VisitEnumDecl(EnumDecl *D) {
   BitsPacker EnumDeclBits;
   EnumDeclBits.addBits(D->getNumPositiveBits(), /*BitWidth=*/8);
   EnumDeclBits.addBits(D->getNumNegativeBits(), /*BitWidth=*/8);
-  bool ShouldSkipCheckingODR = D->shouldSkipCheckingODR();
+  bool ShouldSkipCheckingODR = shouldSkipCheckingODR(D);
   EnumDeclBits.addBit(ShouldSkipCheckingODR);
   EnumDeclBits.addBit(D->isScoped());
   EnumDeclBits.addBit(D->isScopedUsingClassTag());
@@ -552,7 +552,7 @@ void ASTDeclWriter::VisitEnumDecl(EnumDecl *D) {
       !D->isTopLevelDeclInObjCContainer() &&
       !CXXRecordDecl::classofKind(D->getKind()) &&
       !D->getIntegerTypeSourceInfo() && !D->getMemberSpecializationInfo() &&
-      !needsAnonymousDeclarationNumber(D) && !D->shouldSkipCheckingODR() &&
+      !needsAnonymousDeclarationNumber(D) && !shouldSkipCheckingODR(D) &&
       D->getDeclName().getNameKind() == DeclarationName::Identifier)
     AbbrevToUse = Writer.getDeclEnumAbbrev();
 
@@ -718,7 +718,7 @@ void ASTDeclWriter::VisitFunctionDecl(FunctionDecl *D) {
   // FIXME: stable encoding
   FunctionDeclBits.addBits(llvm::to_underlying(D->getLinkageInternal()), 3);
   FunctionDeclBits.addBits((uint32_t)D->getStorageClass(), /*BitWidth=*/3);
-  bool ShouldSkipCheckingODR = D->shouldSkipCheckingODR();
+  bool ShouldSkipCheckingODR = shouldSkipCheckingODR(D);
   FunctionDeclBits.addBit(ShouldSkipCheckingODR);
   FunctionDeclBits.addBit(D->isInlineSpecified());
   FunctionDeclBits.addBit(D->isInlined());
@@ -1559,7 +1559,7 @@ void ASTDeclWriter::VisitCXXMethodDecl(CXXMethodDecl *D) {
       D->getFirstDecl() == D->getMostRecentDecl() && !D->isInvalidDecl() &&
       !D->hasAttrs() && !D->isTopLevelDeclInObjCContainer() &&
       D->getDeclName().getNameKind() == DeclarationName::Identifier &&
-      !D->shouldSkipCheckingODR() && !D->hasExtInfo() &&
+      !shouldSkipCheckingODR(D) && !D->hasExtInfo() &&
       !D->isExplicitlyDefaulted()) {
     if (D->getTemplatedKind() == FunctionDecl::TK_NonTemplate ||
         D->getTemplatedKind() == FunctionDecl::TK_FunctionTemplate ||
diff --git a/clang/test/Modules/hashing-decls-in-exprs-from-gmf-2.cppm b/clang/test/Modules/hashing-decls-in-exprs-from-gmf-2.cppm
new file mode 100644
index 000000000000..66143102cb9e
--- /dev/null
+++ b/clang/test/Modules/hashing-decls-in-exprs-from-gmf-2.cppm
@@ -0,0 +1,44 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+//
+// RUN: %clang_cc1 -std=c++20 -fskip-odr-check-in-gmf %t/A.cppm -emit-module-interface -o %t/A.pcm
+// RUN: %clang_cc1 -std=c++20 -fskip-odr-check-in-gmf %t/test.cpp -fprebuilt-module-path=%t -fsyntax-only -verify
+
+//--- header.h
+#pragma once
+template <class _Tp>
+class Optional {};
+
+template <class _Tp>
+concept C = requires(const _Tp& __t) {
+    []<class _Up>(const Optional<_Up>&) {}(__t);
+};
+
+//--- func.h
+#include "header.h"
+template <C T>
+void func() {}
+
+//--- test_func.h
+#include "func.h"
+
+inline void test_func() {
+    func<Optional<int>>();
+}
+
+//--- A.cppm
+module;
+#include "header.h"
+#include "test_func.h"
+export module A;
+export using ::test_func;
+
+//--- test.cpp
+// expected-no-diagnostics
+import A;
+#include "test_func.h"
+
+void test() {
+    test_func();
+}
-- 
GitLab


From a29e85d6626aae7dba44c66ee5b703d3ab5de41c Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman@google.com>
Date: Tue, 16 Apr 2024 07:41:42 +0000
Subject: [PATCH 347/464] [llvm-exegesis] Change preprocessor directives for
 getCurrentTID

This patch changes the preprocessor directives surrounding
getCurrentTID, particularly moving it out of the block that is only
defined when not building for Android. The getCurrentTID function is
called in places that only require Linux definitions, so this function
should have the same preprocessor scoping around it to prevent link time
failures.
---
 llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp b/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp
index 0a947f6e206f..4699fbbea5de 100644
--- a/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp
+++ b/llvm/tools/llvm-exegesis/lib/SubprocessMemory.cpp
@@ -22,7 +22,7 @@
 namespace llvm {
 namespace exegesis {
 
-#if defined(__linux__) && !defined(__ANDROID__)
+#if defined(__linux__)
 
 long SubprocessMemory::getCurrentTID() {
   // We're using the raw syscall here rather than the gettid() function provided
@@ -31,6 +31,8 @@ long SubprocessMemory::getCurrentTID() {
   return syscall(SYS_gettid);
 }
 
+#if !defined(__ANDROID__)
+
 Error SubprocessMemory::initializeSubprocessMemory(pid_t ProcessID) {
   // Add the PID to the shared memory name so that if we're running multiple
   // processes at the same time, they won't interfere with each other.
@@ -157,7 +159,8 @@ Expected<int> SubprocessMemory::setupAuxiliaryMemoryInSubprocess(
 
 SubprocessMemory::~SubprocessMemory() {}
 
-#endif // defined(__linux__) && !defined(__ANDROID__)
+#endif // !defined(__ANDROID__)
+#endif // defined(__linux__)
 
 } // namespace exegesis
 } // namespace llvm
-- 
GitLab


From d34a2c2adb2a4f1dc262c5756d3725caa4ea2571 Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp@bytedance.com>
Date: Tue, 16 Apr 2024 15:55:14 +0800
Subject: [PATCH 348/464] [RISCV] Make more vector pseudos commutable

This PR includes:
* vadd.vv/vand.vv/vor.vv/vxor.vv
* vmseq.vv/vmsne.vv
* vmin.vv/vminu.vv/vmax.vv/vmaxu.vv
* vmul.vv/vmulh.vv/vmulhu.vv
* vwadd.vv/vwaddu.vv
* vwmul.vv/vwmulu
* vwmacc.vv/vwmaccu.vv
* vadc.vvm

There is no test change, I may add it later.

Fixes part of #64422

Reviewers: michaelmaitland, preames, lukel97, topperc, asb

Reviewed By: topperc, lukel97

Pull Request: https://github.com/llvm/llvm-project/pull/88379
---
 llvm/lib/Target/RISCV/RISCVInstrInfo.cpp      | 66 +++++++++++++
 .../Target/RISCV/RISCVInstrInfoVPseudos.td    | 92 ++++++++++---------
 2 files changed, 116 insertions(+), 42 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index b0fda040519a..668062c8d33f 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -2718,6 +2718,50 @@ std::string RISCVInstrInfo::createMIROperandComment(
   return Comment;
 }
 
+// clang-format off
+#define CASE_RVV_OPCODE_UNMASK_LMUL(OP, LMUL)                                 \
+  RISCV::Pseudo##OP##_##LMUL
+
+#define CASE_RVV_OPCODE_MASK_LMUL(OP, LMUL)                                   \
+  RISCV::Pseudo##OP##_##LMUL##_MASK
+
+#define CASE_RVV_OPCODE_LMUL(OP, LMUL)                                        \
+  CASE_RVV_OPCODE_UNMASK_LMUL(OP, LMUL):                                      \
+  case CASE_RVV_OPCODE_MASK_LMUL(OP, LMUL)
+
+#define CASE_RVV_OPCODE_UNMASK_WIDEN(OP)                                      \
+  CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF8):                                       \
+  case CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF4):                                  \
+  case CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF2):                                  \
+  case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M1):                                   \
+  case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M2):                                   \
+  case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M4)
+
+#define CASE_RVV_OPCODE_UNMASK(OP)                                            \
+  CASE_RVV_OPCODE_UNMASK_WIDEN(OP):                                           \
+  case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M8)
+
+#define CASE_RVV_OPCODE_MASK_WIDEN(OP)                                        \
+  CASE_RVV_OPCODE_MASK_LMUL(OP, MF8):                                         \
+  case CASE_RVV_OPCODE_MASK_LMUL(OP, MF4):                                    \
+  case CASE_RVV_OPCODE_MASK_LMUL(OP, MF2):                                    \
+  case CASE_RVV_OPCODE_MASK_LMUL(OP, M1):                                     \
+  case CASE_RVV_OPCODE_MASK_LMUL(OP, M2):                                     \
+  case CASE_RVV_OPCODE_MASK_LMUL(OP, M4)
+
+#define CASE_RVV_OPCODE_MASK(OP)                                              \
+  CASE_RVV_OPCODE_MASK_WIDEN(OP):                                             \
+  case CASE_RVV_OPCODE_MASK_LMUL(OP, M8)
+
+#define CASE_RVV_OPCODE_WIDEN(OP)                                             \
+  CASE_RVV_OPCODE_UNMASK_WIDEN(OP):                                           \
+  case CASE_RVV_OPCODE_MASK_WIDEN(OP)
+
+#define CASE_RVV_OPCODE(OP)                                                   \
+  CASE_RVV_OPCODE_UNMASK(OP):                                                 \
+  case CASE_RVV_OPCODE_MASK(OP)
+// clang-format on
+
 // clang-format off
 #define CASE_VMA_OPCODE_COMMON(OP, TYPE, LMUL)                                 \
   RISCV::PseudoV##OP##_##TYPE##_##LMUL
@@ -2798,6 +2842,28 @@ bool RISCVInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
   case RISCV::PseudoCCMOVGPR:
     // Operands 4 and 5 are commutable.
     return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 4, 5);
+  case CASE_RVV_OPCODE(VADD_VV):
+  case CASE_RVV_OPCODE(VAND_VV):
+  case CASE_RVV_OPCODE(VOR_VV):
+  case CASE_RVV_OPCODE(VXOR_VV):
+  case CASE_RVV_OPCODE_MASK(VMSEQ_VV):
+  case CASE_RVV_OPCODE_MASK(VMSNE_VV):
+  case CASE_RVV_OPCODE(VMIN_VV):
+  case CASE_RVV_OPCODE(VMINU_VV):
+  case CASE_RVV_OPCODE(VMAX_VV):
+  case CASE_RVV_OPCODE(VMAXU_VV):
+  case CASE_RVV_OPCODE(VMUL_VV):
+  case CASE_RVV_OPCODE(VMULH_VV):
+  case CASE_RVV_OPCODE(VMULHU_VV):
+  case CASE_RVV_OPCODE_WIDEN(VWADD_VV):
+  case CASE_RVV_OPCODE_WIDEN(VWADDU_VV):
+  case CASE_RVV_OPCODE_WIDEN(VWMUL_VV):
+  case CASE_RVV_OPCODE_WIDEN(VWMULU_VV):
+  case CASE_RVV_OPCODE_WIDEN(VWMACC_VV):
+  case CASE_RVV_OPCODE_WIDEN(VWMACCU_VV):
+  case CASE_RVV_OPCODE_UNMASK(VADC_VVM):
+    // Operands 2 and 3 are commutable.
+    return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2, 3);
   case CASE_VFMA_SPLATS(FMADD):
   case CASE_VFMA_SPLATS(FMSUB):
   case CASE_VFMA_SPLATS(FMACC):
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
index ad1821d57256..435cd7f84c61 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td
@@ -2127,8 +2127,9 @@ multiclass VPseudoBinary<VReg RetClass,
                          LMULInfo MInfo,
                          string Constraint = "",
                          int sew = 0,
-                         int TargetConstraintType = 1> {
-  let VLMul = MInfo.value, SEW=sew in {
+                         int TargetConstraintType = 1,
+                         bit Commutable = 0> {
+  let VLMul = MInfo.value, SEW=sew, isCommutable = Commutable in {
     defvar suffix = !if(sew, "_" # MInfo.MX # "_E" # sew, "_" # MInfo.MX);
     def suffix : VPseudoBinaryNoMaskTU<RetClass, Op1Class, Op2Class,
                                        Constraint, TargetConstraintType>;
@@ -2167,8 +2168,9 @@ multiclass VPseudoBinaryM<VReg RetClass,
                           DAGOperand Op2Class,
                           LMULInfo MInfo,
                           string Constraint = "",
-                          int TargetConstraintType = 1> {
-  let VLMul = MInfo.value in {
+                          int TargetConstraintType = 1,
+                          bit Commutable = 0> {
+  let VLMul = MInfo.value, isCommutable = Commutable in {
     def "_" # MInfo.MX : VPseudoBinaryMOutNoMask<RetClass, Op1Class, Op2Class,
                                                  Constraint, TargetConstraintType>;
     let ForceTailAgnostic = true in
@@ -2226,8 +2228,8 @@ multiclass VPseudoTiedBinaryRoundingMode<VReg RetClass,
 }
 
 
-multiclass VPseudoBinaryV_VV<LMULInfo m, string Constraint = "", int sew = 0> {
-  defm _VV : VPseudoBinary<m.vrclass, m.vrclass, m.vrclass, m, Constraint, sew>;
+multiclass VPseudoBinaryV_VV<LMULInfo m, string Constraint = "", int sew = 0, bit Commutable = 0> {
+  defm _VV : VPseudoBinary<m.vrclass, m.vrclass, m.vrclass, m, Constraint, sew, Commutable=Commutable>;
 }
 
 multiclass VPseudoBinaryV_VV_RM<LMULInfo m, string Constraint = ""> {
@@ -2331,9 +2333,10 @@ multiclass VPseudoVALU_MM<bit Commutable = 0> {
 // * The destination EEW is greater than the source EEW, the source EMUL is
 //   at least 1, and the overlap is in the highest-numbered part of the
 //   destination register group is legal. Otherwise, it is illegal.
-multiclass VPseudoBinaryW_VV<LMULInfo m> {
+multiclass VPseudoBinaryW_VV<LMULInfo m, bit Commutable = 0> {
   defm _VV : VPseudoBinary<m.wvrclass, m.vrclass, m.vrclass, m,
-                           "@earlyclobber $rd", TargetConstraintType=3>;
+                           "@earlyclobber $rd", TargetConstraintType=3,
+                           Commutable=Commutable>;
 }
 
 multiclass VPseudoBinaryW_VV_RM<LMULInfo m, int sew = 0> {
@@ -2453,7 +2456,9 @@ multiclass VPseudoBinaryV_VM<LMULInfo m, bit CarryOut = 0, bit CarryIn = 1,
                          m.vrclass, m.vrclass, m, CarryIn, Constraint, TargetConstraintType>;
 }
 
-multiclass VPseudoTiedBinaryV_VM<LMULInfo m, int TargetConstraintType = 1> {
+multiclass VPseudoTiedBinaryV_VM<LMULInfo m, int TargetConstraintType = 1,
+                                 bit Commutable = 0> {
+  let isCommutable = Commutable in
   def "_VVM" # "_" # m.MX:
     VPseudoTiedBinaryCarryIn<GetVRegNoV0<m.vrclass>.R,
                              m.vrclass, m.vrclass, m, 1, "",
@@ -2667,9 +2672,11 @@ multiclass PseudoVEXT_VF8 {
 //  lowest-numbered part of the source register group".
 // With LMUL<=1 the source and dest occupy a single register so any overlap
 // is in the lowest-numbered part.
-multiclass VPseudoBinaryM_VV<LMULInfo m, int TargetConstraintType = 1> {
+multiclass VPseudoBinaryM_VV<LMULInfo m, int TargetConstraintType = 1,
+                             bit Commutable = 0> {
   defm _VV : VPseudoBinaryM<VR, m.vrclass, m.vrclass, m,
-                            !if(!ge(m.octuple, 16), "@earlyclobber $rd", ""), TargetConstraintType>;
+                            !if(!ge(m.octuple, 16), "@earlyclobber $rd", ""),
+                            TargetConstraintType, Commutable=Commutable>;
 }
 
 multiclass VPseudoBinaryM_VX<LMULInfo m, int TargetConstraintType = 1> {
@@ -2751,10 +2758,11 @@ multiclass VPseudoVSSHT_VV_VX_VI_RM<Operand ImmType = simm5, string Constraint =
   }
 }
 
-multiclass VPseudoVALU_VV_VX_VI<Operand ImmType = simm5, string Constraint = ""> {
+multiclass VPseudoVALU_VV_VX_VI<Operand ImmType = simm5, string Constraint = "",
+                                bit Commutable = 0> {
   foreach m = MxList in {
     defvar mx = m.MX;
-    defm "" : VPseudoBinaryV_VV<m, Constraint>,
+    defm "" : VPseudoBinaryV_VV<m, Constraint, Commutable=Commutable>,
             SchedBinary<"WriteVIALUV", "ReadVIALUV", "ReadVIALUV", mx,
                         forceMergeOpRead=true>;
     defm "" : VPseudoBinaryV_VX<m, Constraint>,
@@ -2804,17 +2812,17 @@ multiclass VPseudoVAALU_VV_VX_RM {
 multiclass VPseudoVMINMAX_VV_VX {
   foreach m = MxList in {
     defvar mx = m.MX;
-    defm "" : VPseudoBinaryV_VV<m>,
+    defm "" : VPseudoBinaryV_VV<m, Commutable=1>,
               SchedBinary<"WriteVIMinMaxV", "ReadVIMinMaxV", "ReadVIMinMaxV", mx>;
     defm "" : VPseudoBinaryV_VX<m>,
               SchedBinary<"WriteVIMinMaxX", "ReadVIMinMaxV", "ReadVIMinMaxX", mx>;
   }
 }
 
-multiclass VPseudoVMUL_VV_VX {
+multiclass VPseudoVMUL_VV_VX<bit Commutable = 0> {
   foreach m = MxList in {
     defvar mx = m.MX;
-    defm "" : VPseudoBinaryV_VV<m>,
+    defm "" : VPseudoBinaryV_VV<m, Commutable=Commutable>,
               SchedBinary<"WriteVIMulV", "ReadVIMulV", "ReadVIMulV", mx>;
     defm "" : VPseudoBinaryV_VX<m>,
               SchedBinary<"WriteVIMulX", "ReadVIMulV", "ReadVIMulX", mx>;
@@ -2964,10 +2972,10 @@ multiclass VPseudoVALU_VX_VI<Operand ImmType = simm5> {
   }
 }
 
-multiclass VPseudoVWALU_VV_VX {
+multiclass VPseudoVWALU_VV_VX<bit Commutable = 0> {
   foreach m = MxListW in {
     defvar mx = m.MX;
-    defm "" : VPseudoBinaryW_VV<m>,
+    defm "" : VPseudoBinaryW_VV<m, Commutable=Commutable>,
               SchedBinary<"WriteVIWALUV", "ReadVIWALUV", "ReadVIWALUV", mx,
                           forceMergeOpRead=true>;
     defm "" : VPseudoBinaryW_VX<m>, 
@@ -2976,10 +2984,10 @@ multiclass VPseudoVWALU_VV_VX {
   }
 }
 
-multiclass VPseudoVWMUL_VV_VX {
+multiclass VPseudoVWMUL_VV_VX<bit Commutable = 0> {
   foreach m = MxListW in {
     defvar mx = m.MX;
-    defm "" : VPseudoBinaryW_VV<m>,
+    defm "" : VPseudoBinaryW_VV<m, Commutable=Commutable>,
               SchedBinary<"WriteVIWMulV", "ReadVIWMulV", "ReadVIWMulV", mx,
                           forceMergeOpRead=true>;
     defm "" : VPseudoBinaryW_VX<m>,
@@ -3074,7 +3082,7 @@ multiclass VPseudoVMRG_VM_XM_IM {
 multiclass VPseudoVCALU_VM_XM_IM {
   foreach m = MxList in {
     defvar mx = m.MX;
-    defm "" : VPseudoTiedBinaryV_VM<m>,
+    defm "" : VPseudoTiedBinaryV_VM<m, Commutable=1>,
               SchedBinary<"WriteVICALUV", "ReadVICALUV", "ReadVICALUV", mx,
                           forceMergeOpRead=true>;
     defm "" : VPseudoTiedBinaryV_XM<m>,
@@ -3287,10 +3295,10 @@ multiclass VPseudoTernaryV_VF_AAXA_RM<LMULInfo m, FPR_Info f,
                                                           sew, Commutable=1>;
 }
 
-multiclass VPseudoTernaryW_VV<LMULInfo m> {
+multiclass VPseudoTernaryW_VV<LMULInfo m, bit Commutable = 0> {
   defvar constraint = "@earlyclobber $rd";
   defm _VV : VPseudoTernaryWithPolicy<m.wvrclass, m.vrclass, m.vrclass, m,
-                                      constraint, /*Commutable*/ 0, TargetConstraintType=3>;
+                                      constraint, Commutable=Commutable, TargetConstraintType=3>;
 }
 
 multiclass VPseudoTernaryW_VV_RM<LMULInfo m, int sew = 0> {
@@ -3380,10 +3388,10 @@ multiclass VPseudoVSLD_VX_VI<Operand ImmType = simm5, string Constraint = ""> {
   }
 }
 
-multiclass VPseudoVWMAC_VV_VX {
+multiclass VPseudoVWMAC_VV_VX<bit Commutable = 0> {
   foreach m = MxListW in {
     defvar mx = m.MX;
-    defm "" : VPseudoTernaryW_VV<m>,
+    defm "" : VPseudoTernaryW_VV<m, Commutable=Commutable>,
               SchedTernary<"WriteVIWMulAddV", "ReadVIWMulAddV", "ReadVIWMulAddV",
                            "ReadVIWMulAddV", mx>;
     defm "" : VPseudoTernaryW_VX<m>,
@@ -3436,10 +3444,10 @@ multiclass VPseudoVWMAC_VV_VF_BF_RM {
   }
 }
 
-multiclass VPseudoVCMPM_VV_VX_VI {
+multiclass VPseudoVCMPM_VV_VX_VI<bit Commutable = 0> {
   foreach m = MxList in {
     defvar mx = m.MX;
-    defm "" : VPseudoBinaryM_VV<m, TargetConstraintType=2>,
+    defm "" : VPseudoBinaryM_VV<m, TargetConstraintType=2, Commutable=Commutable>,
               SchedBinary<"WriteVICmpV", "ReadVICmpV", "ReadVICmpV", mx>;
     defm "" : VPseudoBinaryM_VX<m, TargetConstraintType=2>,
               SchedBinary<"WriteVICmpX", "ReadVICmpV", "ReadVICmpX", mx>;
@@ -6248,7 +6256,7 @@ defm PseudoVLSEG : VPseudoUSSegLoadFF;
 //===----------------------------------------------------------------------===//
 // 11.1. Vector Single-Width Integer Add and Subtract
 //===----------------------------------------------------------------------===//
-defm PseudoVADD   : VPseudoVALU_VV_VX_VI;
+defm PseudoVADD   : VPseudoVALU_VV_VX_VI<Commutable=1>;
 defm PseudoVSUB   : VPseudoVALU_VV_VX;
 defm PseudoVRSUB  : VPseudoVALU_VX_VI;
 
@@ -6313,9 +6321,9 @@ foreach vti = AllIntegerVectors in {
 //===----------------------------------------------------------------------===//
 // 11.2. Vector Widening Integer Add/Subtract
 //===----------------------------------------------------------------------===//
-defm PseudoVWADDU : VPseudoVWALU_VV_VX;
+defm PseudoVWADDU : VPseudoVWALU_VV_VX<Commutable=1>;
 defm PseudoVWSUBU : VPseudoVWALU_VV_VX;
-defm PseudoVWADD  : VPseudoVWALU_VV_VX;
+defm PseudoVWADD  : VPseudoVWALU_VV_VX<Commutable=1>;
 defm PseudoVWSUB  : VPseudoVWALU_VV_VX;
 defm PseudoVWADDU : VPseudoVWALU_WV_WX;
 defm PseudoVWSUBU : VPseudoVWALU_WV_WX;
@@ -6346,9 +6354,9 @@ defm PseudoVMSBC : VPseudoVCALUM_V_X<"@earlyclobber $rd">;
 //===----------------------------------------------------------------------===//
 // 11.5. Vector Bitwise Logical Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVAND : VPseudoVALU_VV_VX_VI;
-defm PseudoVOR  : VPseudoVALU_VV_VX_VI;
-defm PseudoVXOR : VPseudoVALU_VV_VX_VI;
+defm PseudoVAND : VPseudoVALU_VV_VX_VI<Commutable=1>;
+defm PseudoVOR  : VPseudoVALU_VV_VX_VI<Commutable=1>;
+defm PseudoVXOR : VPseudoVALU_VV_VX_VI<Commutable=1>;
 
 //===----------------------------------------------------------------------===//
 // 11.6. Vector Single-Width Bit Shift Instructions
@@ -6366,8 +6374,8 @@ defm PseudoVNSRA : VPseudoVNSHT_WV_WX_WI;
 //===----------------------------------------------------------------------===//
 // 11.8. Vector Integer Comparison Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVMSEQ  : VPseudoVCMPM_VV_VX_VI;
-defm PseudoVMSNE  : VPseudoVCMPM_VV_VX_VI;
+defm PseudoVMSEQ  : VPseudoVCMPM_VV_VX_VI<Commutable=1>;
+defm PseudoVMSNE  : VPseudoVCMPM_VV_VX_VI<Commutable=1>;
 defm PseudoVMSLTU : VPseudoVCMPM_VV_VX;
 defm PseudoVMSLT  : VPseudoVCMPM_VV_VX;
 defm PseudoVMSLEU : VPseudoVCMPM_VV_VX_VI;
@@ -6386,9 +6394,9 @@ defm PseudoVMAX  : VPseudoVMINMAX_VV_VX;
 //===----------------------------------------------------------------------===//
 // 11.10. Vector Single-Width Integer Multiply Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVMUL    : VPseudoVMUL_VV_VX;
-defm PseudoVMULH   : VPseudoVMUL_VV_VX;
-defm PseudoVMULHU  : VPseudoVMUL_VV_VX;
+defm PseudoVMUL    : VPseudoVMUL_VV_VX<Commutable=1>;
+defm PseudoVMULH   : VPseudoVMUL_VV_VX<Commutable=1>;
+defm PseudoVMULHU  : VPseudoVMUL_VV_VX<Commutable=1>;
 defm PseudoVMULHSU : VPseudoVMUL_VV_VX;
 
 //===----------------------------------------------------------------------===//
@@ -6402,8 +6410,8 @@ defm PseudoVREM  : VPseudoVDIV_VV_VX;
 //===----------------------------------------------------------------------===//
 // 11.12. Vector Widening Integer Multiply Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVWMUL   : VPseudoVWMUL_VV_VX;
-defm PseudoVWMULU  : VPseudoVWMUL_VV_VX;
+defm PseudoVWMUL   : VPseudoVWMUL_VV_VX<Commutable=1>;
+defm PseudoVWMULU  : VPseudoVWMUL_VV_VX<Commutable=1>;
 defm PseudoVWMULSU : VPseudoVWMUL_VV_VX;
 
 //===----------------------------------------------------------------------===//
@@ -6417,8 +6425,8 @@ defm PseudoVNMSUB : VPseudoVMAC_VV_VX_AAXA;
 //===----------------------------------------------------------------------===//
 // 11.14. Vector Widening Integer Multiply-Add Instructions
 //===----------------------------------------------------------------------===//
-defm PseudoVWMACCU  : VPseudoVWMAC_VV_VX;
-defm PseudoVWMACC   : VPseudoVWMAC_VV_VX;
+defm PseudoVWMACCU  : VPseudoVWMAC_VV_VX<Commutable=1>;
+defm PseudoVWMACC   : VPseudoVWMAC_VV_VX<Commutable=1>;
 defm PseudoVWMACCSU : VPseudoVWMAC_VV_VX;
 defm PseudoVWMACCUS : VPseudoVWMAC_VX;
 
-- 
GitLab


From 5b811562a520a8a3cd164897f24dee7da3115bbe Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Tue, 16 Apr 2024 04:33:39 -0400
Subject: [PATCH 349/464] [libc++] Rename __cpu_traits functions (#88741)

Functions inside __cpu_traits were needlessly prefixed with __parallel,
which doesn't serve a real purpose anymore now that they are inside a
traits class.
---
 .../pstl_backends/cpu_backends/any_of.h       |  2 +-
 .../pstl_backends/cpu_backends/fill.h         |  2 +-
 .../pstl_backends/cpu_backends/find_if.h      |  2 +-
 .../pstl_backends/cpu_backends/for_each.h     |  2 +-
 .../pstl_backends/cpu_backends/libdispatch.h  | 24 ++++++++---------
 .../pstl_backends/cpu_backends/merge.h        |  2 +-
 .../pstl_backends/cpu_backends/serial.h       | 24 ++++++++---------
 .../pstl_backends/cpu_backends/stable_sort.h  |  2 +-
 .../pstl_backends/cpu_backends/thread.h       | 24 ++++++++---------
 .../pstl_backends/cpu_backends/transform.h    |  4 +--
 .../cpu_backends/transform_reduce.h           |  4 +--
 libcxx/include/__pstl/cpu_algos/cpu_traits.h  | 27 +++++++++----------
 12 files changed, 59 insertions(+), 60 deletions(-)

diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/any_of.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/any_of.h
index be5e54f3fa5c..3755d288047e 100644
--- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/any_of.h
+++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/any_of.h
@@ -34,7 +34,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 template <class _Backend, class _Index, class _Brick>
 _LIBCPP_HIDE_FROM_ABI optional<bool> __parallel_or(_Index __first, _Index __last, _Brick __f) {
   std::atomic<bool> __found(false);
-  auto __ret = __pstl::__cpu_traits<_Backend>::__parallel_for(__first, __last, [__f, &__found](_Index __i, _Index __j) {
+  auto __ret = __pstl::__cpu_traits<_Backend>::__for_each(__first, __last, [__f, &__found](_Index __i, _Index __j) {
     if (!__found.load(std::memory_order_relaxed) && __f(__i, __j)) {
       __found.store(true, std::memory_order_relaxed);
       __pstl::__cpu_traits<_Backend>::__cancel_execution();
diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/fill.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/fill.h
index 49a32f6c5ce5..0c20bdff6267 100644
--- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/fill.h
+++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/fill.h
@@ -40,7 +40,7 @@ _LIBCPP_HIDE_FROM_ABI optional<__empty>
 __pstl_fill(__cpu_backend_tag, _ForwardIterator __first, _ForwardIterator __last, const _Tp& __value) {
   if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> &&
                 __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) {
-    return __pstl::__cpu_traits<__cpu_backend_tag>::__parallel_for(
+    return __pstl::__cpu_traits<__cpu_backend_tag>::__for_each(
         __first, __last, [&__value](_ForwardIterator __brick_first, _ForwardIterator __brick_last) {
           [[maybe_unused]] auto __res = std::__pstl_fill<__remove_parallel_policy_t<_ExecutionPolicy>>(
               __cpu_backend_tag{}, __brick_first, __brick_last, __value);
diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/find_if.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/find_if.h
index 11a5668bf25a..626293faef69 100644
--- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/find_if.h
+++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/find_if.h
@@ -42,7 +42,7 @@ __parallel_find(_Index __first, _Index __last, _Brick __f, _Compare __comp, bool
   _DifferenceType __initial_dist = __b_first ? __n : -1;
   std::atomic<_DifferenceType> __extremum(__initial_dist);
   // TODO: find out what is better here: parallel_for or parallel_reduce
-  auto __res = __pstl::__cpu_traits<_Backend>::__parallel_for(
+  auto __res = __pstl::__cpu_traits<_Backend>::__for_each(
       __first, __last, [__comp, __f, __first, &__extremum](_Index __i, _Index __j) {
         // See "Reducing Contention Through Priority Updates", PPoPP '13, for discussion of
         // why using a shared variable scales fairly well in this situation.
diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/for_each.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/for_each.h
index 1667ec0f0c4f..d637084e151d 100644
--- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/for_each.h
+++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/for_each.h
@@ -40,7 +40,7 @@ _LIBCPP_HIDE_FROM_ABI optional<__empty>
 __pstl_for_each(__cpu_backend_tag, _ForwardIterator __first, _ForwardIterator __last, _Functor __func) {
   if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> &&
                 __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) {
-    return __pstl::__cpu_traits<__cpu_backend_tag>::__parallel_for(
+    return __pstl::__cpu_traits<__cpu_backend_tag>::__for_each(
         __first, __last, [__func](_ForwardIterator __brick_first, _ForwardIterator __brick_last) {
           [[maybe_unused]] auto __res = std::__pstl_for_each<__remove_parallel_policy_t<_ExecutionPolicy>>(
               __cpu_backend_tag{}, __brick_first, __brick_last, __func);
diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/libdispatch.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/libdispatch.h
index 8757f2496803..17faadf55dd4 100644
--- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/libdispatch.h
+++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/libdispatch.h
@@ -85,7 +85,7 @@ template <>
 struct __cpu_traits<__libdispatch_backend_tag> {
   template <class _RandomAccessIterator, class _Functor>
   _LIBCPP_HIDE_FROM_ABI static optional<__empty>
-  __parallel_for(_RandomAccessIterator __first, _RandomAccessIterator __last, _Functor __func) {
+  __for_each(_RandomAccessIterator __first, _RandomAccessIterator __last, _Functor __func) {
     return __libdispatch::__dispatch_parallel_for(
         __libdispatch::__partition_chunks(__last - __first), std::move(__first), std::move(__func));
   }
@@ -105,14 +105,14 @@ struct __cpu_traits<__libdispatch_backend_tag> {
             typename _RandomAccessIterator3,
             typename _Compare,
             typename _LeafMerge>
-  _LIBCPP_HIDE_FROM_ABI static optional<__empty> __parallel_merge(
-      _RandomAccessIterator1 __first1,
-      _RandomAccessIterator1 __last1,
-      _RandomAccessIterator2 __first2,
-      _RandomAccessIterator2 __last2,
-      _RandomAccessIterator3 __result,
-      _Compare __comp,
-      _LeafMerge __leaf_merge) noexcept {
+  _LIBCPP_HIDE_FROM_ABI static optional<__empty>
+  __merge(_RandomAccessIterator1 __first1,
+          _RandomAccessIterator1 __last1,
+          _RandomAccessIterator2 __first2,
+          _RandomAccessIterator2 __last2,
+          _RandomAccessIterator3 __result,
+          _Compare __comp,
+          _LeafMerge __leaf_merge) noexcept {
     __libdispatch::__chunk_partitions __partitions =
         __libdispatch::__partition_chunks(std::max<ptrdiff_t>(__last1 - __first1, __last2 - __first2));
 
@@ -201,7 +201,7 @@ struct __cpu_traits<__libdispatch_backend_tag> {
   }
 
   template <class _RandomAccessIterator, class _Transform, class _Value, class _Combiner, class _Reduction>
-  _LIBCPP_HIDE_FROM_ABI static optional<_Value> __parallel_transform_reduce(
+  _LIBCPP_HIDE_FROM_ABI static optional<_Value> __transform_reduce(
       _RandomAccessIterator __first,
       _RandomAccessIterator __last,
       _Transform __transform,
@@ -248,8 +248,8 @@ struct __cpu_traits<__libdispatch_backend_tag> {
   }
 
   template <class _RandomAccessIterator, class _Comp, class _LeafSort>
-  _LIBCPP_HIDE_FROM_ABI static optional<__empty> __parallel_stable_sort(
-      _RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp, _LeafSort __leaf_sort) {
+  _LIBCPP_HIDE_FROM_ABI static optional<__empty>
+  __stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp, _LeafSort __leaf_sort) {
     const auto __size = __last - __first;
     auto __partitions = __libdispatch::__partition_chunks(__size);
 
diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/merge.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/merge.h
index d03444790487..c93f4051c9d0 100644
--- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/merge.h
+++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/merge.h
@@ -46,7 +46,7 @@ _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator> __pstl_merge(
                 __has_random_access_iterator_category_or_concept<_ForwardIterator1>::value &&
                 __has_random_access_iterator_category_or_concept<_ForwardIterator2>::value &&
                 __has_random_access_iterator_category_or_concept<_ForwardOutIterator>::value) {
-    auto __res = __pstl::__cpu_traits<__cpu_backend_tag>::__parallel_merge(
+    auto __res = __pstl::__cpu_traits<__cpu_backend_tag>::__merge(
         __first1,
         __last1,
         __first2,
diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/serial.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/serial.h
index c3d2905daed1..7544619a8eef 100644
--- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/serial.h
+++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/serial.h
@@ -35,20 +35,20 @@ template <>
 struct __cpu_traits<__serial_backend_tag> {
   template <class _RandomAccessIterator, class _Fp>
   _LIBCPP_HIDE_FROM_ABI static optional<__empty>
-  __parallel_for(_RandomAccessIterator __first, _RandomAccessIterator __last, _Fp __f) {
+  __for_each(_RandomAccessIterator __first, _RandomAccessIterator __last, _Fp __f) {
     __f(__first, __last);
     return __empty{};
   }
 
   template <class _Index, class _UnaryOp, class _Tp, class _BinaryOp, class _Reduce>
   _LIBCPP_HIDE_FROM_ABI static optional<_Tp>
-  __parallel_transform_reduce(_Index __first, _Index __last, _UnaryOp, _Tp __init, _BinaryOp, _Reduce __reduce) {
+  __transform_reduce(_Index __first, _Index __last, _UnaryOp, _Tp __init, _BinaryOp, _Reduce __reduce) {
     return __reduce(std::move(__first), std::move(__last), std::move(__init));
   }
 
   template <class _RandomAccessIterator, class _Compare, class _LeafSort>
-  _LIBCPP_HIDE_FROM_ABI static optional<__empty> __parallel_stable_sort(
-      _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp, _LeafSort __leaf_sort) {
+  _LIBCPP_HIDE_FROM_ABI static optional<__empty>
+  __stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp, _LeafSort __leaf_sort) {
     __leaf_sort(__first, __last, __comp);
     return __empty{};
   }
@@ -60,14 +60,14 @@ struct __cpu_traits<__serial_backend_tag> {
             class _RandomAccessIterator3,
             class _Compare,
             class _LeafMerge>
-  _LIBCPP_HIDE_FROM_ABI static optional<__empty> __parallel_merge(
-      _RandomAccessIterator1 __first1,
-      _RandomAccessIterator1 __last1,
-      _RandomAccessIterator2 __first2,
-      _RandomAccessIterator2 __last2,
-      _RandomAccessIterator3 __outit,
-      _Compare __comp,
-      _LeafMerge __leaf_merge) {
+  _LIBCPP_HIDE_FROM_ABI static optional<__empty>
+  __merge(_RandomAccessIterator1 __first1,
+          _RandomAccessIterator1 __last1,
+          _RandomAccessIterator2 __first2,
+          _RandomAccessIterator2 __last2,
+          _RandomAccessIterator3 __outit,
+          _Compare __comp,
+          _LeafMerge __leaf_merge) {
     __leaf_merge(__first1, __last1, __first2, __last2, __outit, __comp);
     return __empty{};
   }
diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/stable_sort.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/stable_sort.h
index ebfa0fc69147..8c60cf897ff8 100644
--- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/stable_sort.h
+++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/stable_sort.h
@@ -29,7 +29,7 @@ template <class _ExecutionPolicy, class _RandomAccessIterator, class _Comp>
 _LIBCPP_HIDE_FROM_ABI optional<__empty>
 __pstl_stable_sort(__cpu_backend_tag, _RandomAccessIterator __first, _RandomAccessIterator __last, _Comp __comp) {
   if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy>) {
-    return __pstl::__cpu_traits<__cpu_backend_tag>::__parallel_stable_sort(
+    return __pstl::__cpu_traits<__cpu_backend_tag>::__stable_sort(
         __first, __last, __comp, [](_RandomAccessIterator __g_first, _RandomAccessIterator __g_last, _Comp __g_comp) {
           std::stable_sort(__g_first, __g_last, __g_comp);
         });
diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/thread.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/thread.h
index 8d1cb221c3d8..2acf912264a0 100644
--- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/thread.h
+++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/thread.h
@@ -38,20 +38,20 @@ template <>
 struct __cpu_traits<__std_thread_backend_tag> {
   template <class _RandomAccessIterator, class _Fp>
   _LIBCPP_HIDE_FROM_ABI static optional<__empty>
-  __parallel_for(_RandomAccessIterator __first, _RandomAccessIterator __last, _Fp __f) {
+  __for_each(_RandomAccessIterator __first, _RandomAccessIterator __last, _Fp __f) {
     __f(__first, __last);
     return __empty{};
   }
 
   template <class _Index, class _UnaryOp, class _Tp, class _BinaryOp, class _Reduce>
   _LIBCPP_HIDE_FROM_ABI static optional<_Tp>
-  __parallel_transform_reduce(_Index __first, _Index __last, _UnaryOp, _Tp __init, _BinaryOp, _Reduce __reduce) {
+  __transform_reduce(_Index __first, _Index __last, _UnaryOp, _Tp __init, _BinaryOp, _Reduce __reduce) {
     return __reduce(std::move(__first), std::move(__last), std::move(__init));
   }
 
   template <class _RandomAccessIterator, class _Compare, class _LeafSort>
-  _LIBCPP_HIDE_FROM_ABI static optional<__empty> __parallel_stable_sort(
-      _RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp, _LeafSort __leaf_sort) {
+  _LIBCPP_HIDE_FROM_ABI static optional<__empty>
+  __stable_sort(_RandomAccessIterator __first, _RandomAccessIterator __last, _Compare __comp, _LeafSort __leaf_sort) {
     __leaf_sort(__first, __last, __comp);
     return __empty{};
   }
@@ -63,14 +63,14 @@ struct __cpu_traits<__std_thread_backend_tag> {
             class _RandomAccessIterator3,
             class _Compare,
             class _LeafMerge>
-  _LIBCPP_HIDE_FROM_ABI static optional<__empty> __parallel_merge(
-      _RandomAccessIterator1 __first1,
-      _RandomAccessIterator1 __last1,
-      _RandomAccessIterator2 __first2,
-      _RandomAccessIterator2 __last2,
-      _RandomAccessIterator3 __outit,
-      _Compare __comp,
-      _LeafMerge __leaf_merge) {
+  _LIBCPP_HIDE_FROM_ABI static optional<__empty>
+  __merge(_RandomAccessIterator1 __first1,
+          _RandomAccessIterator1 __last1,
+          _RandomAccessIterator2 __first2,
+          _RandomAccessIterator2 __last2,
+          _RandomAccessIterator3 __outit,
+          _Compare __comp,
+          _LeafMerge __leaf_merge) {
     __leaf_merge(__first1, __last1, __first2, __last2, __outit, __comp);
     return __empty{};
   }
diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform.h
index d4c383997a67..4b9b29686683 100644
--- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform.h
+++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform.h
@@ -50,7 +50,7 @@ _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator> __pstl_transform(
   if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> &&
                 __has_random_access_iterator_category_or_concept<_ForwardIterator>::value &&
                 __has_random_access_iterator_category_or_concept<_ForwardOutIterator>::value) {
-    __pstl::__cpu_traits<__cpu_backend_tag>::__parallel_for(
+    __pstl::__cpu_traits<__cpu_backend_tag>::__for_each(
         __first, __last, [__op, __first, __result](_ForwardIterator __brick_first, _ForwardIterator __brick_last) {
           auto __res = std::__pstl_transform<__remove_parallel_policy_t<_ExecutionPolicy>>(
               __cpu_backend_tag{}, __brick_first, __brick_last, __result + (__brick_first - __first), __op);
@@ -98,7 +98,7 @@ _LIBCPP_HIDE_FROM_ABI optional<_ForwardOutIterator> __pstl_transform(
                 __has_random_access_iterator_category_or_concept<_ForwardIterator1>::value &&
                 __has_random_access_iterator_category_or_concept<_ForwardIterator2>::value &&
                 __has_random_access_iterator_category_or_concept<_ForwardOutIterator>::value) {
-    auto __res = __pstl::__cpu_traits<__cpu_backend_tag>::__parallel_for(
+    auto __res = __pstl::__cpu_traits<__cpu_backend_tag>::__for_each(
         __first1,
         __last1,
         [__op, __first1, __first2, __result](_ForwardIterator1 __brick_first, _ForwardIterator1 __brick_last) {
diff --git a/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform_reduce.h b/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform_reduce.h
index 956c7d6a88ce..c074eea9861c 100644
--- a/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform_reduce.h
+++ b/libcxx/include/__algorithm/pstl_backends/cpu_backends/transform_reduce.h
@@ -120,7 +120,7 @@ _LIBCPP_HIDE_FROM_ABI optional<_Tp> __pstl_transform_reduce(
   if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> &&
                 __has_random_access_iterator_category_or_concept<_ForwardIterator1>::value &&
                 __has_random_access_iterator_category_or_concept<_ForwardIterator2>::value) {
-    return __pstl::__cpu_traits<__cpu_backend_tag>::__parallel_transform_reduce(
+    return __pstl::__cpu_traits<__cpu_backend_tag>::__transform_reduce(
         __first1,
         std::move(__last1),
         [__first1, __first2, __transform](_ForwardIterator1 __iter) {
@@ -167,7 +167,7 @@ _LIBCPP_HIDE_FROM_ABI optional<_Tp> __pstl_transform_reduce(
     _UnaryOperation __transform) {
   if constexpr (__is_parallel_execution_policy_v<_ExecutionPolicy> &&
                 __has_random_access_iterator_category_or_concept<_ForwardIterator>::value) {
-    return __pstl::__cpu_traits<__cpu_backend_tag>::__parallel_transform_reduce(
+    return __pstl::__cpu_traits<__cpu_backend_tag>::__transform_reduce(
         std::move(__first),
         std::move(__last),
         [__transform](_ForwardIterator __iter) { return __transform(*__iter); },
diff --git a/libcxx/include/__pstl/cpu_algos/cpu_traits.h b/libcxx/include/__pstl/cpu_algos/cpu_traits.h
index 2f0db46e9be8..0483d6918fd0 100644
--- a/libcxx/include/__pstl/cpu_algos/cpu_traits.h
+++ b/libcxx/include/__pstl/cpu_algos/cpu_traits.h
@@ -32,31 +32,30 @@ namespace __pstl {
 // ================
 //
 //  template <class _RandomAccessIterator, class _Functor>
-//  optional<__empty> __parallel_for(_RandomAccessIterator __first, _RandomAccessIterator __last, _Functor __func);
+//  optional<__empty> __for_each(_RandomAccessIterator __first, _RandomAccessIterator __last, _Functor __func);
 //    - __func must take a subrange of [__first, __last) that should be executed in serial
 //
 //  template <class _Iterator, class _UnaryOp, class _Tp, class _BinaryOp, class _Reduction>
-//  optional<_Tp> __parallel_transform_reduce(_Iterator __first, _Iterator __last, _UnaryOp, _Tp __init, _BinaryOp,
-//  _Reduction);
+//  optional<_Tp> __transform_reduce(_Iterator __first, _Iterator __last, _UnaryOp, _Tp __init, _BinaryOp, _Reduction);
 //
 //  template <class _RandomAccessIterator1,
 //            class _RandomAccessIterator2,
 //            class _RandomAccessIterator3,
 //            class _Compare,
 //            class _LeafMerge>
-//  optional<_RandomAccessIterator3> __parallel_merge(_RandomAccessIterator1 __first1,
-//                                                    _RandomAccessIterator1 __last1,
-//                                                    _RandomAccessIterator2 __first2,
-//                                                    _RandomAccessIterator2 __last2,
-//                                                    _RandomAccessIterator3 __outit,
-//                                                    _Compare __comp,
-//                                                    _LeafMerge __leaf_merge);
+//  optional<_RandomAccessIterator3> __merge(_RandomAccessIterator1 __first1,
+//                                           _RandomAccessIterator1 __last1,
+//                                           _RandomAccessIterator2 __first2,
+//                                           _RandomAccessIterator2 __last2,
+//                                           _RandomAccessIterator3 __outit,
+//                                           _Compare __comp,
+//                                           _LeafMerge __leaf_merge);
 //
 //  template <class _RandomAccessIterator, class _Comp, class _LeafSort>
-//  optional<__empty> __parallel_stable_sort(_RandomAccessIterator __first,
-//                                           _RandomAccessIterator __last,
-//                                           _Comp __comp,
-//                                           _LeafSort __leaf_sort);
+//  optional<__empty> __stable_sort(_RandomAccessIterator __first,
+//                                  _RandomAccessIterator __last,
+//                                  _Comp __comp,
+//                                  _LeafSort __leaf_sort);
 //
 //   void __cancel_execution();
 //      Cancel the execution of other jobs - they aren't needed anymore. This is not a binding request,
-- 
GitLab


From 66cf995da76b9da3cfdee2f29eff6ea4d84305ef Mon Sep 17 00:00:00 2001
From: Carlos Alberto Enciso <carlos.alberto.enciso@gmail.com>
Date: Tue, 16 Apr 2024 09:37:51 +0100
Subject: [PATCH 350/464] [IPSCCP] Variable not visible at Og: (#77901)

https://bugs.llvm.org/show_bug.cgi?id=51559
https://github.com/llvm/llvm-project/issues/50901

IPSCCP pass removes the global variable and does not create a constant
expression for the initializer value.

Extend test coverage to include:
- half, bfloat types.
- checks for undef (int32 and ptr).

There is no support for:
- fp128, x86_fp80, ppc_fp128  types.
https://github.com/llvm/llvm-project/issues/88102
---
 llvm/lib/Transforms/Utils/Local.cpp           |  8 ++--
 llvm/test/Transforms/SCCP/pr50901.ll          | 41 ++++++++++++++++++-
 llvm/unittests/Transforms/Utils/LocalTest.cpp | 21 ++++++----
 3 files changed, 57 insertions(+), 13 deletions(-)

diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp
index 380bac9c6180..a42ef0c4e6ae 100644
--- a/llvm/lib/Transforms/Utils/Local.cpp
+++ b/llvm/lib/Transforms/Utils/Local.cpp
@@ -3627,10 +3627,12 @@ DIExpression *llvm::getExpressionForConstant(DIBuilder &DIB, const Constant &C,
     return createIntegerExpression(C);
 
   auto *FP = dyn_cast<ConstantFP>(&C);
-  if (FP && (Ty.isFloatTy() || Ty.isDoubleTy())) {
+  if (FP && Ty.isFloatingPointTy() && Ty.getScalarSizeInBits() <= 64) {
     const APFloat &APF = FP->getValueAPF();
-    return DIB.createConstantValueExpression(
-        APF.bitcastToAPInt().getZExtValue());
+    APInt const &API = APF.bitcastToAPInt();
+    if (auto Temp = API.getZExtValue())
+      return DIB.createConstantValueExpression(static_cast<uint64_t>(Temp));
+    return DIB.createConstantValueExpression(*API.getRawData());
   }
 
   if (!Ty.isPointerTy())
diff --git a/llvm/test/Transforms/SCCP/pr50901.ll b/llvm/test/Transforms/SCCP/pr50901.ll
index 11d6bba6f6a9..d48d67532d88 100644
--- a/llvm/test/Transforms/SCCP/pr50901.ll
+++ b/llvm/test/Transforms/SCCP/pr50901.ll
@@ -52,6 +52,16 @@
 ; CHECK:     = !DIGlobalVariableExpression(var: ![[DBG_FLOAT_UNDEF:.+]], expr: !DIExpression())
 ; CHECK-DAG: ![[DBG_FLOAT_UNDEF]]  = distinct !DIGlobalVariable(name: "g_float_undef"
 
+; CHECK: ![[G8:[0-9]+]] = !DIGlobalVariableExpression(var: ![[DBG8:[0-9]+]], expr: !DIExpression(DW_OP_constu, 22136, DW_OP_stack_value))
+; CHECK-DAG: ![[DBG8]] = distinct !DIGlobalVariable(name: "g_88", {{.*}}
+; CHECK: ![[G9:[0-9]+]] = !DIGlobalVariableExpression(var: ![[DBG9:[0-9]+]], expr: !DIExpression(DW_OP_constu, 23726, DW_OP_stack_value))
+; CHECK-DAG: ![[DBG9]] = distinct !DIGlobalVariable(name: "g_99", {{.*}}
+
+; CHECK-DAG: ![[DBGA:[0-9]+]] = distinct !DIGlobalVariable(name: "g_i32_undef"
+; CHECK-DAG: ![[GA:[0-9]+]] = !DIGlobalVariableExpression(var: ![[DBGA]], expr: !DIExpression())
+; CHECK-DAG: ![[DBGB:[0-9]+]] = distinct !DIGlobalVariable(name: "g_ptr_undef"
+; CHECK-DAG: ![[GB:[0-9]+]] = !DIGlobalVariableExpression(var: ![[DBGB]], expr: !DIExpression())
+
 @g_1 = dso_local global i32 -4, align 4, !dbg !0
 @g_2 = dso_local global float 0x4011C28F60000000, align 4, !dbg !8
 @g_3 = dso_local global i8 97, align 1, !dbg !10
@@ -59,6 +69,8 @@
 @g_5 = dso_local global i8 1, align 1, !dbg !16
 @g_6 = dso_local global ptr null, align 8, !dbg !19
 @g_7 = dso_local global ptr null, align 8, !dbg !23
+@g_8 = dso_local global half 0xH4321, align 4, !dbg !86
+@g_9 = dso_local global bfloat 0xR3F80, align 4, !dbg !90
 @_ZL4g_11 = internal global i32 -5, align 4, !dbg !25
 @_ZL4g_22 = internal global float 0x4016333340000000, align 4, !dbg !27
 @_ZL4g_33 = internal global i8 98, align 1, !dbg !29
@@ -67,6 +79,10 @@
 @_ZL4g_66 = internal global ptr null, align 8, !dbg !35
 @_ZL4g_77 = internal global ptr inttoptr (i64 70 to ptr), align 8, !dbg !37
 @g_float_undef = internal global float undef, align 4, !dbg !83
+@_ZL4g_88 = internal global half 0xH5678, align 4, !dbg !88
+@_ZL4g_99 = internal global bfloat 0xR5CAE, align 4, !dbg !92
+@g_i32_undef = internal global i32 undef, align 4, !dbg !95
+@g_ptr_undef = internal global ptr undef, align 8, !dbg !97
 
 define dso_local void @_Z3barv() !dbg !46 {
 entry:
@@ -88,6 +104,15 @@ entry:
   store ptr %6, ptr @g_7, align 8, !dbg !59
   %l = load float, ptr @g_float_undef, align 8, !dbg !59
   store float %l, ptr @g_2, align 8, !dbg !59
+  %7 = load half, ptr @_ZL4g_88, align 4, !dbg !59
+  store half %7, ptr @g_8, align 4, !dbg !59
+  %8 = load bfloat, ptr @_ZL4g_99, align 4, !dbg !59
+  store bfloat %8, ptr @g_9, align 4, !dbg !59
+  %9 = load i32, ptr @g_i32_undef, align 4, !dbg !59
+  store i32 %9, ptr @g_1, align 4, !dbg !59
+  %10 = load ptr, ptr @g_ptr_undef, align 8, !dbg !59
+  store ptr %10, ptr @g_6, align 8, !dbg !59
+
   ret void, !dbg !59
 }
 
@@ -108,7 +133,7 @@ entry:
 !4 = !{!5}
 !5 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !6, size: 64)
 !6 = !DIBasicType(name: "float", size: 32, encoding: DW_ATE_float)
-!7 = !{!0, !8, !10, !13, !16, !19, !23, !25, !27, !29, !31, !33, !35, !37, !83}
+!7 = !{!0, !8, !10, !13, !16, !19, !23, !25, !27, !29, !31, !33, !35, !37, !83, !86, !88, !90, !92, !95, !97}
 !8 = !DIGlobalVariableExpression(var: !9, expr: !DIExpression())
 !9 = distinct !DIGlobalVariable(name: "g_2", scope: !2, file: !3, line: 2, type: !6, isLocal: false, isDefinition: true)
 !10 = !DIGlobalVariableExpression(var: !11, expr: !DIExpression())
@@ -159,3 +184,17 @@ entry:
 !82 = !DILocation(line: 31, column: 1, scope: !77)
 !83 = !DIGlobalVariableExpression(var: !84, expr: !DIExpression())
 !84 = distinct !DIGlobalVariable(name: "g_float_undef", linkageName: "g_float_undef", scope: !2, file: !3, line: 15, type: !6, isLocal: true, isDefinition: true)
+!85 = !DIBasicType(name: "float", size: 16, encoding: DW_ATE_float)
+!86 = !DIGlobalVariableExpression(var: !87, expr: !DIExpression())
+!87 = distinct !DIGlobalVariable(name: "g_8", scope: !2, file: !3, line: 2, type: !85, isLocal: false, isDefinition: true)
+!88 = !DIGlobalVariableExpression(var: !89, expr: !DIExpression())
+!89 = distinct !DIGlobalVariable(name: "g_88", linkageName: "_ZL4g_88", scope: !2, file: !3, line: 10, type: !85, isLocal: true, isDefinition: true)
+!90 = !DIGlobalVariableExpression(var: !91, expr: !DIExpression())
+!91 = distinct !DIGlobalVariable(name: "g_9", scope: !2, file: !3, line: 2, type: !85, isLocal: false, isDefinition: true)
+!92 = !DIGlobalVariableExpression(var: !93, expr: !DIExpression())
+!93 = distinct !DIGlobalVariable(name: "g_99", linkageName: "_ZL4g_99", scope: !2, file: !3, line: 10, type: !85, isLocal: true, isDefinition: true)
+
+!95 = !DIGlobalVariableExpression(var: !96, expr: !DIExpression())
+!96 = distinct !DIGlobalVariable(name: "g_i32_undef", linkageName: "g_i32_undef", scope: !2, file: !3, line: 9, type: !22, isLocal: true, isDefinition: true)
+!97 = !DIGlobalVariableExpression(var: !98, expr: !DIExpression())
+!98 = distinct !DIGlobalVariable(name: "g_ptr_undef", linkageName: "g_ptr_undef", scope: !2, file: !3, line: 14, type: !21, isLocal: true, isDefinition: true)
diff --git a/llvm/unittests/Transforms/Utils/LocalTest.cpp b/llvm/unittests/Transforms/Utils/LocalTest.cpp
index a86775a1366b..d7d0ea2c6a6e 100644
--- a/llvm/unittests/Transforms/Utils/LocalTest.cpp
+++ b/llvm/unittests/Transforms/Utils/LocalTest.cpp
@@ -1241,6 +1241,18 @@ TEST(Local, ExpressionForConstant) {
   EXPECT_NE(Expr, nullptr);
   EXPECT_EQ(Expr->getElement(1), 13841306799765140275U);
 
+  // Half.
+  Type *HalfTy = Type::getHalfTy(Context);
+  Expr = createExpression(ConstantFP::get(HalfTy, 5.55), HalfTy);
+  EXPECT_NE(Expr, nullptr);
+  EXPECT_EQ(Expr->getElement(1), 17805U);
+
+  // BFloat.
+  Type *BFloatTy = Type::getBFloatTy(Context);
+  Expr = createExpression(ConstantFP::get(BFloatTy, -5.55), BFloatTy);
+  EXPECT_NE(Expr, nullptr);
+  EXPECT_EQ(Expr->getElement(1), 49330U);
+
   // Pointer.
   PointerType *PtrTy = PointerType::get(Context, 0);
   Expr = createExpression(ConstantPointerNull::get(PtrTy), PtrTy);
@@ -1257,15 +1269,6 @@ TEST(Local, ExpressionForConstant) {
   EXPECT_NE(Expr, nullptr);
   EXPECT_EQ(Expr->getElement(1), 5678U);
 
-  // Others.
-  Type *HalfTy = Type::getHalfTy(Context);
-  Expr = createExpression(ConstantFP::get(HalfTy, 32), HalfTy);
-  EXPECT_EQ(Expr, nullptr);
-
-  Type *BFloatTy = Type::getBFloatTy(Context);
-  Expr = createExpression(ConstantFP::get(BFloatTy, 32), BFloatTy);
-  EXPECT_EQ(Expr, nullptr);
-
   Type *FP128Ty = Type::getFP128Ty(Context);
   Expr = createExpression(ConstantFP::get(FP128Ty, 32), FP128Ty);
   EXPECT_EQ(Expr, nullptr);
-- 
GitLab


From d6d84b5d1448e4f2e24b467a0abcf42fe9d543e9 Mon Sep 17 00:00:00 2001
From: NagyDonat <donat.nagy@ericsson.com>
Date: Tue, 16 Apr 2024 10:41:26 +0200
Subject: [PATCH 351/464] [analyzer] Handle builtin functions in MallocChecker
 (#88416)

This commit ensures that the `CallDescription`s in `MallocChecker` are
matched with the mode `CDM::CLibrary`, so:
- they don't match methods or functions within user-defined namespaces;
- they also match builtin variants of these functions (if any), so the
checker can model `__builtin_alloca()` like `alloca()`.

This change fixes https://github.com/llvm/llvm-project/issues/81597. New
tests were added to verify that `std::malloc` and `std::free` (from
`<cstdlib>`) are modeled, but a method that's named e.g. `free` isn't
confused with the memory release function.

The responsibility for modeling `__builtin_alloca` and
`__builtin_alloca_with_align` was moved from `BuiltinFunctionChecker` to
`MallocChecker`, to avoid buggy interactions between the checkers and
ensure that the builtin and non-builtin variants are handled by exactly
the same logic.

This change might be a step backwards for the users who don't have
`unix.Malloc` enabled; but I suspect that `__builtin_alloca()` is so
rare that it would be a waste of time to implement backwards
compatibility for them.

There were several test files that relied on `__builtin_alloca()` calls
to get an `AllocaRegion`, these were modified to enable `unix.Malloc`.
One of these files (cxx-uninitialized-object-ptr-ref.cpp) had some tests
that relied on the fact that `malloc()` was treated as a "black box" in
them, these were updated to use `calloc()` (to get initialized memory)
and `free()` (to avoid memory leak reports).

While I was developing this change, I found a very suspicious assert in
`MallocChecker`. As it isn't blocking the goals of this commit, I just
marked it with a FIXME, but I'll try to investigate and fix it in a
follow-up change.
---
 .../Checkers/BuiltinFunctionChecker.cpp       | 25 ++----
 .../StaticAnalyzer/Checkers/MallocChecker.cpp | 89 ++++++++++---------
 .../Inputs/system-header-simulator-cxx.h      |  6 +-
 .../cxx-uninitialized-object-ptr-ref.cpp      | 24 +++--
 clang/test/Analysis/exercise-ps.c             |  2 +-
 clang/test/Analysis/explain-svals.cpp         |  2 +-
 clang/test/Analysis/malloc-std-namespace.cpp  | 24 +++++
 clang/test/Analysis/malloc.c                  | 11 +++
 clang/test/Analysis/malloc.cpp                | 11 +++
 clang/test/Analysis/stack-addr-ps.c           |  2 +-
 clang/test/Analysis/stackaddrleak.c           |  4 +-
 11 files changed, 125 insertions(+), 75 deletions(-)
 create mode 100644 clang/test/Analysis/malloc-std-namespace.cpp

diff --git a/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp
index 01e46fa8591c..1a75d7b52ad6 100644
--- a/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp
@@ -6,7 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This checker evaluates clang builtin functions.
+// This checker evaluates "standalone" clang builtin functions that are not
+// just special-cased variants of well-known non-builtin functions.
+// Builtin functions like __builtin_memcpy and __builtin_alloca should be
+// evaluated by the same checker that handles their non-builtin variant to
+// ensure that the two variants are handled consistently.
 //
 //===----------------------------------------------------------------------===//
 
@@ -80,25 +84,6 @@ bool BuiltinFunctionChecker::evalCall(const CallEvent &Call,
     return true;
   }
 
-  case Builtin::BI__builtin_alloca_with_align:
-  case Builtin::BI__builtin_alloca: {
-    SValBuilder &SVB = C.getSValBuilder();
-    const loc::MemRegionVal R =
-        SVB.getAllocaRegionVal(CE, C.getLocationContext(), C.blockCount());
-
-    // Set the extent of the region in bytes. This enables us to use the SVal
-    // of the argument directly. If we saved the extent in bits, it'd be more
-    // difficult to reason about values like symbol*8.
-    auto Size = Call.getArgSVal(0);
-    if (auto DefSize = Size.getAs<DefinedOrUnknownSVal>()) {
-      // This `getAs()` is mostly paranoia, because core.CallAndMessage reports
-      // undefined function arguments (unless it's disabled somehow).
-      state = setDynamicExtent(state, R.getRegion(), *DefSize, SVB);
-    }
-    C.addTransition(state->BindExpr(CE, LCtx, R));
-    return true;
-  }
-
   case Builtin::BI__builtin_dynamic_object_size:
   case Builtin::BI__builtin_object_size:
   case Builtin::BI__builtin_constant_p: {
diff --git a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
index 88fb42b6625a..11651fd491f7 100644
--- a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
@@ -401,10 +401,11 @@ private:
   };
 
   const CallDescriptionMap<CheckFn> FreeingMemFnMap{
-      {{{"free"}, 1}, &MallocChecker::checkFree},
-      {{{"if_freenameindex"}, 1}, &MallocChecker::checkIfFreeNameIndex},
-      {{{"kfree"}, 1}, &MallocChecker::checkFree},
-      {{{"g_free"}, 1}, &MallocChecker::checkFree},
+      {{CDM::CLibrary, {"free"}, 1}, &MallocChecker::checkFree},
+      {{CDM::CLibrary, {"if_freenameindex"}, 1},
+       &MallocChecker::checkIfFreeNameIndex},
+      {{CDM::CLibrary, {"kfree"}, 1}, &MallocChecker::checkFree},
+      {{CDM::CLibrary, {"g_free"}, 1}, &MallocChecker::checkFree},
   };
 
   bool isFreeingCall(const CallEvent &Call) const;
@@ -413,41 +414,46 @@ private:
   friend class NoOwnershipChangeVisitor;
 
   CallDescriptionMap<CheckFn> AllocatingMemFnMap{
-      {{{"alloca"}, 1}, &MallocChecker::checkAlloca},
-      {{{"_alloca"}, 1}, &MallocChecker::checkAlloca},
-      {{{"malloc"}, 1}, &MallocChecker::checkBasicAlloc},
-      {{{"malloc"}, 3}, &MallocChecker::checkKernelMalloc},
-      {{{"calloc"}, 2}, &MallocChecker::checkCalloc},
-      {{{"valloc"}, 1}, &MallocChecker::checkBasicAlloc},
+      {{CDM::CLibrary, {"alloca"}, 1}, &MallocChecker::checkAlloca},
+      {{CDM::CLibrary, {"_alloca"}, 1}, &MallocChecker::checkAlloca},
+      // The line for "alloca" also covers "__builtin_alloca", but the
+      // _with_align variant must be listed separately because it takes an
+      // extra argument:
+      {{CDM::CLibrary, {"__builtin_alloca_with_align"}, 2},
+       &MallocChecker::checkAlloca},
+      {{CDM::CLibrary, {"malloc"}, 1}, &MallocChecker::checkBasicAlloc},
+      {{CDM::CLibrary, {"malloc"}, 3}, &MallocChecker::checkKernelMalloc},
+      {{CDM::CLibrary, {"calloc"}, 2}, &MallocChecker::checkCalloc},
+      {{CDM::CLibrary, {"valloc"}, 1}, &MallocChecker::checkBasicAlloc},
       {{CDM::CLibrary, {"strndup"}, 2}, &MallocChecker::checkStrdup},
       {{CDM::CLibrary, {"strdup"}, 1}, &MallocChecker::checkStrdup},
-      {{{"_strdup"}, 1}, &MallocChecker::checkStrdup},
-      {{{"kmalloc"}, 2}, &MallocChecker::checkKernelMalloc},
-      {{{"if_nameindex"}, 1}, &MallocChecker::checkIfNameIndex},
+      {{CDM::CLibrary, {"_strdup"}, 1}, &MallocChecker::checkStrdup},
+      {{CDM::CLibrary, {"kmalloc"}, 2}, &MallocChecker::checkKernelMalloc},
+      {{CDM::CLibrary, {"if_nameindex"}, 1}, &MallocChecker::checkIfNameIndex},
       {{CDM::CLibrary, {"wcsdup"}, 1}, &MallocChecker::checkStrdup},
       {{CDM::CLibrary, {"_wcsdup"}, 1}, &MallocChecker::checkStrdup},
-      {{{"g_malloc"}, 1}, &MallocChecker::checkBasicAlloc},
-      {{{"g_malloc0"}, 1}, &MallocChecker::checkGMalloc0},
-      {{{"g_try_malloc"}, 1}, &MallocChecker::checkBasicAlloc},
-      {{{"g_try_malloc0"}, 1}, &MallocChecker::checkGMalloc0},
-      {{{"g_memdup"}, 2}, &MallocChecker::checkGMemdup},
-      {{{"g_malloc_n"}, 2}, &MallocChecker::checkGMallocN},
-      {{{"g_malloc0_n"}, 2}, &MallocChecker::checkGMallocN0},
-      {{{"g_try_malloc_n"}, 2}, &MallocChecker::checkGMallocN},
-      {{{"g_try_malloc0_n"}, 2}, &MallocChecker::checkGMallocN0},
+      {{CDM::CLibrary, {"g_malloc"}, 1}, &MallocChecker::checkBasicAlloc},
+      {{CDM::CLibrary, {"g_malloc0"}, 1}, &MallocChecker::checkGMalloc0},
+      {{CDM::CLibrary, {"g_try_malloc"}, 1}, &MallocChecker::checkBasicAlloc},
+      {{CDM::CLibrary, {"g_try_malloc0"}, 1}, &MallocChecker::checkGMalloc0},
+      {{CDM::CLibrary, {"g_memdup"}, 2}, &MallocChecker::checkGMemdup},
+      {{CDM::CLibrary, {"g_malloc_n"}, 2}, &MallocChecker::checkGMallocN},
+      {{CDM::CLibrary, {"g_malloc0_n"}, 2}, &MallocChecker::checkGMallocN0},
+      {{CDM::CLibrary, {"g_try_malloc_n"}, 2}, &MallocChecker::checkGMallocN},
+      {{CDM::CLibrary, {"g_try_malloc0_n"}, 2}, &MallocChecker::checkGMallocN0},
   };
 
   CallDescriptionMap<CheckFn> ReallocatingMemFnMap{
-      {{{"realloc"}, 2},
+      {{CDM::CLibrary, {"realloc"}, 2},
        std::bind(&MallocChecker::checkRealloc, _1, _2, _3, false)},
-      {{{"reallocf"}, 2},
+      {{CDM::CLibrary, {"reallocf"}, 2},
        std::bind(&MallocChecker::checkRealloc, _1, _2, _3, true)},
-      {{{"g_realloc"}, 2},
+      {{CDM::CLibrary, {"g_realloc"}, 2},
        std::bind(&MallocChecker::checkRealloc, _1, _2, _3, false)},
-      {{{"g_try_realloc"}, 2},
+      {{CDM::CLibrary, {"g_try_realloc"}, 2},
        std::bind(&MallocChecker::checkRealloc, _1, _2, _3, false)},
-      {{{"g_realloc_n"}, 3}, &MallocChecker::checkReallocN},
-      {{{"g_try_realloc_n"}, 3}, &MallocChecker::checkReallocN},
+      {{CDM::CLibrary, {"g_realloc_n"}, 3}, &MallocChecker::checkReallocN},
+      {{CDM::CLibrary, {"g_try_realloc_n"}, 3}, &MallocChecker::checkReallocN},
 
       // NOTE: the following CallDescription also matches the C++ standard
       // library function std::getline(); the callback will filter it out.
@@ -1259,9 +1265,6 @@ static bool isStandardRealloc(const CallEvent &Call) {
   assert(FD);
   ASTContext &AC = FD->getASTContext();
 
-  if (isa<CXXMethodDecl>(FD))
-    return false;
-
   return FD->getDeclaredReturnType().getDesugaredType(AC) == AC.VoidPtrTy &&
          FD->getParamDecl(0)->getType().getDesugaredType(AC) == AC.VoidPtrTy &&
          FD->getParamDecl(1)->getType().getDesugaredType(AC) ==
@@ -1273,9 +1276,6 @@ static bool isGRealloc(const CallEvent &Call) {
   assert(FD);
   ASTContext &AC = FD->getASTContext();
 
-  if (isa<CXXMethodDecl>(FD))
-    return false;
-
   return FD->getDeclaredReturnType().getDesugaredType(AC) == AC.VoidPtrTy &&
          FD->getParamDecl(0)->getType().getDesugaredType(AC) == AC.VoidPtrTy &&
          FD->getParamDecl(1)->getType().getDesugaredType(AC) ==
@@ -1284,14 +1284,14 @@ static bool isGRealloc(const CallEvent &Call) {
 
 void MallocChecker::checkRealloc(const CallEvent &Call, CheckerContext &C,
                                  bool ShouldFreeOnFail) const {
-  // HACK: CallDescription currently recognizes non-standard realloc functions
-  // as standard because it doesn't check the type, or wether its a non-method
-  // function. This should be solved by making CallDescription smarter.
-  // Mind that this came from a bug report, and all other functions suffer from
-  // this.
-  // https://bugs.llvm.org/show_bug.cgi?id=46253
+  // Ignore calls to functions whose type does not match the expected type of
+  // either the standard realloc or g_realloc from GLib.
+  // FIXME: Should we perform this kind of checking consistently for each
+  // function? If yes, then perhaps extend the `CallDescription` interface to
+  // handle this.
   if (!isStandardRealloc(Call) && !isGRealloc(Call))
     return;
+
   ProgramStateRef State = C.getState();
   State = ReallocMemAux(C, Call, ShouldFreeOnFail, State, AF_Malloc);
   State = ProcessZeroAllocCheck(Call, 1, State);
@@ -1842,9 +1842,18 @@ static ProgramStateRef MallocUpdateRefState(CheckerContext &C, const Expr *E,
     return nullptr;
 
   SymbolRef Sym = RetVal->getAsLocSymbol();
+
   // This is a return value of a function that was not inlined, such as malloc()
   // or new(). We've checked that in the caller. Therefore, it must be a symbol.
   assert(Sym);
+  // FIXME: In theory this assertion should fail for `alloca()` calls (because
+  // `AllocaRegion`s are not symbolic); but in practice this does not happen.
+  // As the current code appears to work correctly, I'm not touching this issue
+  // now, but it would be good to investigate and clarify this.
+  // Also note that perhaps the special `AllocaRegion` should be replaced by
+  // `SymbolicRegion` (or turned into a subclass of `SymbolicRegion`) to enable
+  // proper tracking of memory allocated by `alloca()` -- and after that change
+  // this assertion would become valid again.
 
   // Set the symbol's state to Allocated.
   return State->set<RegionState>(Sym, RefState::getAllocated(Family, E));
diff --git a/clang/test/Analysis/Inputs/system-header-simulator-cxx.h b/clang/test/Analysis/Inputs/system-header-simulator-cxx.h
index 85db68d41a6c..1c2be322f83c 100644
--- a/clang/test/Analysis/Inputs/system-header-simulator-cxx.h
+++ b/clang/test/Analysis/Inputs/system-header-simulator-cxx.h
@@ -1106,6 +1106,7 @@ using ostream = basic_ostream<char>;
 extern std::ostream cout;
 
 ostream &operator<<(ostream &, const string &);
+
 #if __cplusplus >= 202002L
 template <class T>
 ostream &operator<<(ostream &, const std::unique_ptr<T> &);
@@ -1122,11 +1123,12 @@ istream &getline(istream &, string &, char);
 istream &getline(istream &, string &);
 } // namespace std
 
-#ifdef TEST_INLINABLE_ALLOCATORS
 namespace std {
   void *malloc(size_t);
   void free(void *);
-}
+} // namespace std
+
+#ifdef TEST_INLINABLE_ALLOCATORS
 void* operator new(std::size_t size, const std::nothrow_t&) throw() { return std::malloc(size); }
 void* operator new[](std::size_t size, const std::nothrow_t&) throw() { return std::malloc(size); }
 void operator delete(void* ptr, const std::nothrow_t&) throw() { std::free(ptr); }
diff --git a/clang/test/Analysis/cxx-uninitialized-object-ptr-ref.cpp b/clang/test/Analysis/cxx-uninitialized-object-ptr-ref.cpp
index fc067dd04428..f46a2c9bc368 100644
--- a/clang/test/Analysis/cxx-uninitialized-object-ptr-ref.cpp
+++ b/clang/test/Analysis/cxx-uninitialized-object-ptr-ref.cpp
@@ -1,9 +1,9 @@
-// RUN: %clang_analyze_cc1 -analyzer-checker=core,optin.cplusplus.UninitializedObject \
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix.Malloc,optin.cplusplus.UninitializedObject \
 // RUN:   -analyzer-config optin.cplusplus.UninitializedObject:Pedantic=true -DPEDANTIC \
 // RUN:   -analyzer-config optin.cplusplus.UninitializedObject:CheckPointeeInitialization=true \
 // RUN:   -std=c++11 -verify  %s
 
-// RUN: %clang_analyze_cc1 -analyzer-checker=core,optin.cplusplus.UninitializedObject \
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix.Malloc,optin.cplusplus.UninitializedObject \
 // RUN:   -analyzer-config optin.cplusplus.UninitializedObject:CheckPointeeInitialization=true \
 // RUN:   -std=c++11 -verify  %s
 
@@ -316,7 +316,10 @@ void fCyclicPointerTest2() {
 
 // Void pointer tests are mainly no-crash tests.
 
-void *malloc(int size);
+typedef __typeof(sizeof(int)) size_t;
+
+void *calloc(size_t nmemb, size_t size);
+void free(void *p);
 
 class VoidPointerTest1 {
   void *vptr;
@@ -328,8 +331,9 @@ public:
 };
 
 void fVoidPointerTest1() {
-  void *vptr = malloc(sizeof(int));
+  void *vptr = calloc(1, sizeof(int));
   VoidPointerTest1(vptr, char());
+  free(vptr);
 }
 
 class VoidPointerTest2 {
@@ -342,8 +346,9 @@ public:
 };
 
 void fVoidPointerTest2() {
-  void *vptr = malloc(sizeof(int));
+  void *vptr = calloc(1, sizeof(int));
   VoidPointerTest2(&vptr, char());
+  free(vptr);
 }
 
 class VoidPointerRRefTest1 {
@@ -359,8 +364,9 @@ upon returning to the caller.  This will be a dangling reference}}
 };
 
 void fVoidPointerRRefTest1() {
-  void *vptr = malloc(sizeof(int));
+  void *vptr = calloc(1, sizeof(int));
   VoidPointerRRefTest1(vptr, char());
+  free(vptr);
 }
 
 class VoidPointerRRefTest2 {
@@ -376,8 +382,9 @@ upon returning to the caller.  This will be a dangling reference}}
 };
 
 void fVoidPointerRRefTest2() {
-  void *vptr = malloc(sizeof(int));
+  void *vptr = calloc(1, sizeof(int));
   VoidPointerRRefTest2(&vptr, char());
+  free(vptr);
 }
 
 class VoidPointerLRefTest {
@@ -393,8 +400,9 @@ upon returning to the caller.  This will be a dangling reference}}
 };
 
 void fVoidPointerLRefTest() {
-  void *vptr = malloc(sizeof(int));
+  void *vptr = calloc(1, sizeof(int));
   VoidPointerLRefTest(vptr, char());
+  free(vptr);
 }
 
 struct CyclicVoidPointerTest {
diff --git a/clang/test/Analysis/exercise-ps.c b/clang/test/Analysis/exercise-ps.c
index d214c3959b20..d1e1771afddb 100644
--- a/clang/test/Analysis/exercise-ps.c
+++ b/clang/test/Analysis/exercise-ps.c
@@ -1,5 +1,5 @@
 // RUN: %clang_analyze_cc1 %s -verify -Wno-error=implicit-function-declaration \
-// RUN:   -analyzer-checker=core \
+// RUN:   -analyzer-checker=core,unix.Malloc \
 // RUN:   -analyzer-config core.CallAndMessage:ArgPointeeInitializedness=true
 //
 // Just exercise the analyzer on code that has at one point caused issues
diff --git a/clang/test/Analysis/explain-svals.cpp b/clang/test/Analysis/explain-svals.cpp
index 30368b6976cc..33fce10c4e2b 100644
--- a/clang/test/Analysis/explain-svals.cpp
+++ b/clang/test/Analysis/explain-svals.cpp
@@ -1,7 +1,7 @@
 // RUN: %clang_analyze_cc1 -triple i386-apple-darwin10 -verify %s \
-// RUN:   -analyzer-checker=core.builtin \
 // RUN:   -analyzer-checker=debug.ExprInspection \
 // RUN:   -analyzer-checker=unix.cstring \
+// RUN:   -analyzer-checker=unix.Malloc \
 // RUN:   -analyzer-config display-checker-name=false
 
 typedef unsigned long size_t;
diff --git a/clang/test/Analysis/malloc-std-namespace.cpp b/clang/test/Analysis/malloc-std-namespace.cpp
new file mode 100644
index 000000000000..d4e397bb812a
--- /dev/null
+++ b/clang/test/Analysis/malloc-std-namespace.cpp
@@ -0,0 +1,24 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix.Malloc -verify -analyzer-output=text %s
+
+// This file tests that unix.Malloc can handle C++ code where e.g. malloc and
+// free are declared within the namespace 'std' by the header <cstdlib>.
+
+#include "Inputs/system-header-simulator-cxx.h"
+
+void leak() {
+  int *p = static_cast<int*>(std::malloc(sizeof(int))); // expected-note{{Memory is allocated}}
+} // expected-warning{{Potential leak of memory pointed to by 'p'}}
+  // expected-note@-1{{Potential leak of memory pointed to by 'p'}}
+
+void no_leak() {
+  int *p = static_cast<int*>(std::malloc(sizeof(int)));
+  std::free(p); // no-warning
+}
+
+void invalid_free() {
+  int i;
+  int *p = &i;
+  //expected-note@+2{{Argument to free() is the address of the local variable 'i', which is not memory allocated by malloc()}}
+  //expected-warning@+1{{Argument to free() is the address of the local variable 'i', which is not memory allocated by malloc()}}
+  std::free(p);
+}
diff --git a/clang/test/Analysis/malloc.c b/clang/test/Analysis/malloc.c
index 09cd4b0bfce6..e5cb45ba7335 100644
--- a/clang/test/Analysis/malloc.c
+++ b/clang/test/Analysis/malloc.c
@@ -740,6 +740,17 @@ void allocaFree(void) {
   free(p); // expected-warning {{Memory allocated by alloca() should not be deallocated}}
 }
 
+void allocaFreeBuiltin(void) {
+  int *p = __builtin_alloca(sizeof(int));
+  free(p); // expected-warning {{Memory allocated by alloca() should not be deallocated}}
+}
+
+void allocaFreeBuiltinAlign(void) {
+  int *p = __builtin_alloca_with_align(sizeof(int), 64);
+  free(p); // expected-warning {{Memory allocated by alloca() should not be deallocated}}
+}
+
+
 int* mallocEscapeRet(void) {
   int *p = malloc(12);
   return p; // no warning
diff --git a/clang/test/Analysis/malloc.cpp b/clang/test/Analysis/malloc.cpp
index 14b4c0576384..300b344ab25d 100644
--- a/clang/test/Analysis/malloc.cpp
+++ b/clang/test/Analysis/malloc.cpp
@@ -214,3 +214,14 @@ void *realloc(void **ptr, size_t size) { realloc(ptr, size); } // no-crash
 namespace pr46253_paramty2{
 void *realloc(void *ptr, int size) { realloc(ptr, size); } // no-crash
 } // namespace pr46253_paramty2
+
+namespace pr81597 {
+struct S {};
+struct T {
+  void free(const S& s);
+};
+void f(T& t) {
+  S s;
+  t.free(s); // no-warning: This is not the free you are looking for...
+}
+} // namespace pr81597
diff --git a/clang/test/Analysis/stack-addr-ps.c b/clang/test/Analysis/stack-addr-ps.c
index e469396e1bb2..e69ab4189b52 100644
--- a/clang/test/Analysis/stack-addr-ps.c
+++ b/clang/test/Analysis/stack-addr-ps.c
@@ -1,4 +1,4 @@
-// RUN: %clang_analyze_cc1 -analyzer-checker=core -fblocks -verify %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix.Malloc -fblocks -verify %s
 
 int* f1(void) {
   int x = 0;
diff --git a/clang/test/Analysis/stackaddrleak.c b/clang/test/Analysis/stackaddrleak.c
index 0583bfc18711..39c29f2a2635 100644
--- a/clang/test/Analysis/stackaddrleak.c
+++ b/clang/test/Analysis/stackaddrleak.c
@@ -1,5 +1,5 @@
-// RUN: %clang_analyze_cc1 -analyzer-checker=core -verify -std=c99 -Dbool=_Bool -Wno-bool-conversion %s
-// RUN: %clang_analyze_cc1 -analyzer-checker=core -verify -x c++ -Wno-bool-conversion %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix.Malloc -verify -std=c99 -Dbool=_Bool -Wno-bool-conversion %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix.Malloc -verify -x c++ -Wno-bool-conversion %s
 
 typedef __INTPTR_TYPE__ intptr_t;
 char const *p;
-- 
GitLab


From 40dd3aa91d3f73184e34e45e597b84bec059c572 Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Tue, 16 Apr 2024 10:59:02 +0200
Subject: [PATCH 352/464] [mlir][Interfaces] `Variable` abstraction for
 `ValueBoundsOpInterface` (#87980)

This commit generalizes and cleans up the `ValueBoundsConstraintSet`
API. The API used to provide function overloads for comparing/computing
bounds of:
- index-typed SSA value
- dimension of shaped value
- affine map + operands

This commit removes all overloads. There is now a single entry point for
each `compare` variant and each `computeBound` variant. These functions
now take a `Variable`, which is internally represented as an affine map
and map operands.

This commit also adds support for computing bounds for an affine map +
operands. There was previously no public API for that.
---
 .../Dialect/Affine/Transforms/Transforms.h    |  11 +
 .../Dialect/Arith/Transforms/Transforms.h     |  11 +
 .../mlir/Interfaces/ValueBoundsOpInterface.h  | 119 +++---
 .../Affine/IR/ValueBoundsOpInterfaceImpl.cpp  |   6 +-
 .../Affine/Transforms/ReifyValueBounds.cpp    |  15 +-
 .../Arith/IR/ValueBoundsOpInterfaceImpl.cpp   |   8 +-
 .../Dialect/Arith/Transforms/IntNarrowing.cpp |   2 +-
 .../Arith/Transforms/ReifyValueBounds.cpp     |  15 +-
 .../lib/Dialect/Linalg/Transforms/Padding.cpp |   6 +-
 .../Dialect/Linalg/Transforms/Promotion.cpp   |   6 +-
 .../Transforms/IndependenceTransforms.cpp     |   5 +-
 .../SCF/IR/ValueBoundsOpInterfaceImpl.cpp     |  17 +-
 .../Tensor/IR/TensorTilingInterfaceImpl.cpp   |   3 +-
 .../Transforms/IndependenceTransforms.cpp     |   3 +-
 mlir/lib/Dialect/Tensor/Utils/Utils.cpp       |   4 +-
 .../lib/Interfaces/ValueBoundsOpInterface.cpp | 338 ++++++++----------
 .../value-bounds-op-interface-impl.mlir       |  24 ++
 .../Dialect/Affine/TestReifyValueBounds.cpp   |  26 +-
 mlir/test/lib/Dialect/Test/TestDialect.cpp    |  37 ++
 mlir/test/lib/Dialect/Test/TestOps.td         |  16 +-
 20 files changed, 361 insertions(+), 311 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Affine/Transforms/Transforms.h b/mlir/include/mlir/Dialect/Affine/Transforms/Transforms.h
index 8e840e744064..1ea737522081 100644
--- a/mlir/include/mlir/Dialect/Affine/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/Affine/Transforms/Transforms.h
@@ -53,6 +53,17 @@ void reorderOperandsByHoistability(RewriterBase &rewriter, AffineApplyOp op);
 /// maximally compose chains of AffineApplyOps.
 FailureOr<AffineApplyOp> decompose(RewriterBase &rewriter, AffineApplyOp op);
 
+/// Reify a bound for the given variable in terms of SSA values for which
+/// `stopCondition` is met.
+///
+/// By default, lower/equal bounds are closed and upper bounds are open. If
+/// `closedUB` is set to "true", upper bounds are also closed.
+FailureOr<OpFoldResult>
+reifyValueBound(OpBuilder &b, Location loc, presburger::BoundType type,
+                const ValueBoundsConstraintSet::Variable &var,
+                ValueBoundsConstraintSet::StopConditionFn stopCondition,
+                bool closedUB = false);
+
 /// Reify a bound for the given index-typed value in terms of SSA values for
 /// which `stopCondition` is met. If no stop condition is specified, reify in
 /// terms of the operands of the owner op.
diff --git a/mlir/include/mlir/Dialect/Arith/Transforms/Transforms.h b/mlir/include/mlir/Dialect/Arith/Transforms/Transforms.h
index 970a52a06a11..bbc7e5d3e0dd 100644
--- a/mlir/include/mlir/Dialect/Arith/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/Arith/Transforms/Transforms.h
@@ -24,6 +24,17 @@ enum class BoundType;
 
 namespace arith {
 
+/// Reify a bound for the given variable in terms of SSA values for which
+/// `stopCondition` is met.
+///
+/// By default, lower/equal bounds are closed and upper bounds are open. If
+/// `closedUB` is set to "true", upper bounds are also closed.
+FailureOr<OpFoldResult>
+reifyValueBound(OpBuilder &b, Location loc, presburger::BoundType type,
+                const ValueBoundsConstraintSet::Variable &var,
+                ValueBoundsConstraintSet::StopConditionFn stopCondition,
+                bool closedUB = false);
+
 /// Reify a bound for the given index-typed value in terms of SSA values for
 /// which `stopCondition` is met. If no stop condition is specified, reify in
 /// terms of the operands of the owner op.
diff --git a/mlir/include/mlir/Interfaces/ValueBoundsOpInterface.h b/mlir/include/mlir/Interfaces/ValueBoundsOpInterface.h
index 1d7bc6ea961c..ac17ace5a976 100644
--- a/mlir/include/mlir/Interfaces/ValueBoundsOpInterface.h
+++ b/mlir/include/mlir/Interfaces/ValueBoundsOpInterface.h
@@ -15,6 +15,7 @@
 #include "mlir/IR/Value.h"
 #include "mlir/Interfaces/DestinationStyleOpInterface.h"
 #include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/ExtensibleRTTI.h"
 
 #include <queue>
@@ -111,6 +112,39 @@ protected:
 public:
   static char ID;
 
+  /// A variable that can be added to the constraint set as a "column". The
+  /// value bounds infrastructure can compute bounds for variables and compare
+  /// two variables.
+  ///
+  /// Internally, a variable is represented as an affine map and operands.
+  class Variable {
+  public:
+    /// Construct a variable for an index-typed attribute or SSA value.
+    Variable(OpFoldResult ofr);
+
+    /// Construct a variable for an index-typed SSA value.
+    Variable(Value indexValue);
+
+    /// Construct a variable for a dimension of a shaped value.
+    Variable(Value shapedValue, int64_t dim);
+
+    /// Construct a variable for an index-typed attribute/SSA value or for a
+    /// dimension of a shaped value. A non-null dimension must be provided if
+    /// and only if `ofr` is a shaped value.
+    Variable(OpFoldResult ofr, std::optional<int64_t> dim);
+
+    /// Construct a variable for a map and its operands.
+    Variable(AffineMap map, ArrayRef<Variable> mapOperands);
+    Variable(AffineMap map, ArrayRef<Value> mapOperands);
+
+    MLIRContext *getContext() const { return map.getContext(); }
+
+  private:
+    friend class ValueBoundsConstraintSet;
+    AffineMap map;
+    ValueDimList mapOperands;
+  };
+
   /// The stop condition when traversing the backward slice of a shaped value/
   /// index-type value. The traversal continues until the stop condition
   /// evaluates to "true" for a value.
@@ -121,35 +155,31 @@ public:
   using StopConditionFn = std::function<bool(
       Value, std::optional<int64_t> /*dim*/, ValueBoundsConstraintSet &cstr)>;
 
-  /// Compute a bound for the given index-typed value or shape dimension size.
-  /// The computed bound is stored in `resultMap`. The operands of the bound are
-  /// stored in `mapOperands`. An operand is either an index-type SSA value
-  /// or a shaped value and a dimension.
+  /// Compute a bound for the given variable. The computed bound is stored in
+  /// `resultMap`. The operands of the bound are stored in `mapOperands`. An
+  /// operand is either an index-type SSA value or a shaped value and a
+  /// dimension.
   ///
-  /// `dim` must be `nullopt` if and only if `value` is index-typed. The bound
-  /// is computed in terms of values/dimensions for which `stopCondition`
-  /// evaluates to "true". To that end, the backward slice (reverse use-def
-  /// chain) of the given value is visited in a worklist-driven manner and the
-  /// constraint set is populated according to `ValueBoundsOpInterface` for each
-  /// visited value.
+  /// The bound is computed in terms of values/dimensions for which
+  /// `stopCondition` evaluates to "true". To that end, the backward slice
+  /// (reverse use-def chain) of the given value is visited in a worklist-driven
+  /// manner and the constraint set is populated according to
+  /// `ValueBoundsOpInterface` for each visited value.
   ///
   /// By default, lower/equal bounds are closed and upper bounds are open. If
   /// `closedUB` is set to "true", upper bounds are also closed.
-  static LogicalResult computeBound(AffineMap &resultMap,
-                                    ValueDimList &mapOperands,
-                                    presburger::BoundType type, Value value,
-                                    std::optional<int64_t> dim,
-                                    StopConditionFn stopCondition,
-                                    bool closedUB = false);
+  static LogicalResult
+  computeBound(AffineMap &resultMap, ValueDimList &mapOperands,
+               presburger::BoundType type, const Variable &var,
+               StopConditionFn stopCondition, bool closedUB = false);
 
   /// Compute a bound in terms of the values/dimensions in `dependencies`. The
   /// computed bound consists of only constant terms and dependent values (or
   /// dimension sizes thereof).
   static LogicalResult
   computeDependentBound(AffineMap &resultMap, ValueDimList &mapOperands,
-                        presburger::BoundType type, Value value,
-                        std::optional<int64_t> dim, ValueDimList dependencies,
-                        bool closedUB = false);
+                        presburger::BoundType type, const Variable &var,
+                        ValueDimList dependencies, bool closedUB = false);
 
   /// Compute a bound in that is independent of all values in `independencies`.
   ///
@@ -161,13 +191,10 @@ public:
   /// appear in the computed bound.
   static LogicalResult
   computeIndependentBound(AffineMap &resultMap, ValueDimList &mapOperands,
-                          presburger::BoundType type, Value value,
-                          std::optional<int64_t> dim, ValueRange independencies,
-                          bool closedUB = false);
+                          presburger::BoundType type, const Variable &var,
+                          ValueRange independencies, bool closedUB = false);
 
-  /// Compute a constant bound for the given affine map, where dims and symbols
-  /// are bound to the given operands. The affine map must have exactly one
-  /// result.
+  /// Compute a constant bound for the given variable.
   ///
   /// This function traverses the backward slice of the given operands in a
   /// worklist-driven manner until `stopCondition` evaluates to "true". The
@@ -182,16 +209,9 @@ public:
   /// By default, lower/equal bounds are closed and upper bounds are open. If
   /// `closedUB` is set to "true", upper bounds are also closed.
   static FailureOr<int64_t>
-  computeConstantBound(presburger::BoundType type, Value value,
-                       std::optional<int64_t> dim = std::nullopt,
+  computeConstantBound(presburger::BoundType type, const Variable &var,
                        StopConditionFn stopCondition = nullptr,
                        bool closedUB = false);
-  static FailureOr<int64_t> computeConstantBound(
-      presburger::BoundType type, AffineMap map, ValueDimList mapOperands,
-      StopConditionFn stopCondition = nullptr, bool closedUB = false);
-  static FailureOr<int64_t> computeConstantBound(
-      presburger::BoundType type, AffineMap map, ArrayRef<Value> mapOperands,
-      StopConditionFn stopCondition = nullptr, bool closedUB = false);
 
   /// Compute a constant delta between the given two values. Return "failure"
   /// if a constant delta could not be determined.
@@ -221,9 +241,8 @@ public:
   /// proven. This could be because the specified relation does in fact not hold
   /// or because there is not enough information in the constraint set. In other
   /// words, if we do not know for sure, this function returns "false".
-  bool populateAndCompare(OpFoldResult lhs, std::optional<int64_t> lhsDim,
-                          ComparisonOperator cmp, OpFoldResult rhs,
-                          std::optional<int64_t> rhsDim);
+  bool populateAndCompare(const Variable &lhs, ComparisonOperator cmp,
+                          const Variable &rhs);
 
   /// Return "true" if "lhs cmp rhs" was proven to hold. Return "false" if the
   /// specified relation could not be proven. This could be because the
@@ -233,24 +252,12 @@ public:
   ///
   /// This function keeps traversing the backward slice of lhs/rhs until could
   /// prove the relation or until it ran out of IR.
-  static bool compare(OpFoldResult lhs, std::optional<int64_t> lhsDim,
-                      ComparisonOperator cmp, OpFoldResult rhs,
-                      std::optional<int64_t> rhsDim);
-  static bool compare(AffineMap lhs, ValueDimList lhsOperands,
-                      ComparisonOperator cmp, AffineMap rhs,
-                      ValueDimList rhsOperands);
-  static bool compare(AffineMap lhs, ArrayRef<Value> lhsOperands,
-                      ComparisonOperator cmp, AffineMap rhs,
-                      ArrayRef<Value> rhsOperands);
-
-  /// Compute whether the given values/dimensions are equal. Return "failure" if
+  static bool compare(const Variable &lhs, ComparisonOperator cmp,
+                      const Variable &rhs);
+
+  /// Compute whether the given variables are equal. Return "failure" if
   /// equality could not be determined.
-  ///
-  /// `dim1`/`dim2` must be `nullopt` if and only if `value1`/`value2` are
-  /// index-typed.
-  static FailureOr<bool> areEqual(OpFoldResult value1, OpFoldResult value2,
-                                  std::optional<int64_t> dim1 = std::nullopt,
-                                  std::optional<int64_t> dim2 = std::nullopt);
+  static FailureOr<bool> areEqual(const Variable &var1, const Variable &var2);
 
   /// Return "true" if the given slices are guaranteed to be overlapping.
   /// Return "false" if the given slices are guaranteed to be non-overlapping.
@@ -317,9 +324,6 @@ protected:
   ///
   /// This function does not analyze any IR and does not populate any additional
   /// constraints.
-  bool compareValueDims(OpFoldResult lhs, std::optional<int64_t> lhsDim,
-                        ComparisonOperator cmp, OpFoldResult rhs,
-                        std::optional<int64_t> rhsDim);
   bool comparePos(int64_t lhsPos, ComparisonOperator cmp, int64_t rhsPos);
 
   /// Given an affine map with a single result (and map operands), add a new
@@ -374,6 +378,7 @@ protected:
   /// constraint system. Return the position of the new column. Any operands
   /// that were not analyzed yet are put on the worklist.
   int64_t insert(AffineMap map, ValueDimList operands, bool isSymbol = true);
+  int64_t insert(const Variable &var, bool isSymbol = true);
 
   /// Project out the given column in the constraint set.
   void projectOut(int64_t pos);
@@ -381,6 +386,8 @@ protected:
   /// Project out all columns for which the condition holds.
   void projectOut(function_ref<bool(ValueDim)> condition);
 
+  void projectOutAnonymous(std::optional<int64_t> except = std::nullopt);
+
   /// Mapping of columns to values/shape dimensions.
   SmallVector<std::optional<ValueDim>> positionToValueDim;
   /// Reverse mapping of values/shape dimensions to columns.
diff --git a/mlir/lib/Dialect/Affine/IR/ValueBoundsOpInterfaceImpl.cpp b/mlir/lib/Dialect/Affine/IR/ValueBoundsOpInterfaceImpl.cpp
index e0c3abe7a0f7..82a9fb0d4908 100644
--- a/mlir/lib/Dialect/Affine/IR/ValueBoundsOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Affine/IR/ValueBoundsOpInterfaceImpl.cpp
@@ -120,9 +120,7 @@ mlir::affine::fullyComposeAndComputeConstantDelta(Value value1, Value value2) {
   mapOperands.push_back(value1);
   mapOperands.push_back(value2);
   affine::fullyComposeAffineMapAndOperands(&map, &mapOperands);
-  ValueDimList valueDims;
-  for (Value v : mapOperands)
-    valueDims.push_back({v, std::nullopt});
   return ValueBoundsConstraintSet::computeConstantBound(
-      presburger::BoundType::EQ, map, valueDims);
+      presburger::BoundType::EQ,
+      ValueBoundsConstraintSet::Variable(map, mapOperands));
 }
diff --git a/mlir/lib/Dialect/Affine/Transforms/ReifyValueBounds.cpp b/mlir/lib/Dialect/Affine/Transforms/ReifyValueBounds.cpp
index 117ee8e8701a..1a266b72d1f8 100644
--- a/mlir/lib/Dialect/Affine/Transforms/ReifyValueBounds.cpp
+++ b/mlir/lib/Dialect/Affine/Transforms/ReifyValueBounds.cpp
@@ -16,16 +16,15 @@
 using namespace mlir;
 using namespace mlir::affine;
 
-static FailureOr<OpFoldResult>
-reifyValueBound(OpBuilder &b, Location loc, presburger::BoundType type,
-                Value value, std::optional<int64_t> dim,
-                ValueBoundsConstraintSet::StopConditionFn stopCondition,
-                bool closedUB) {
+FailureOr<OpFoldResult> mlir::affine::reifyValueBound(
+    OpBuilder &b, Location loc, presburger::BoundType type,
+    const ValueBoundsConstraintSet::Variable &var,
+    ValueBoundsConstraintSet::StopConditionFn stopCondition, bool closedUB) {
   // Compute bound.
   AffineMap boundMap;
   ValueDimList mapOperands;
   if (failed(ValueBoundsConstraintSet::computeBound(
-          boundMap, mapOperands, type, value, dim, stopCondition, closedUB)))
+          boundMap, mapOperands, type, var, stopCondition, closedUB)))
     return failure();
 
   // Reify bound.
@@ -93,7 +92,7 @@ FailureOr<OpFoldResult> mlir::affine::reifyShapedValueDimBound(
     // the owner of `value`.
     return v != value;
   };
-  return reifyValueBound(b, loc, type, value, dim,
+  return reifyValueBound(b, loc, type, {value, dim},
                          stopCondition ? stopCondition : reifyToOperands,
                          closedUB);
 }
@@ -105,7 +104,7 @@ FailureOr<OpFoldResult> mlir::affine::reifyIndexValueBound(
                              ValueBoundsConstraintSet &cstr) {
     return v != value;
   };
-  return reifyValueBound(b, loc, type, value, /*dim=*/std::nullopt,
+  return reifyValueBound(b, loc, type, value,
                          stopCondition ? stopCondition : reifyToOperands,
                          closedUB);
 }
diff --git a/mlir/lib/Dialect/Arith/IR/ValueBoundsOpInterfaceImpl.cpp b/mlir/lib/Dialect/Arith/IR/ValueBoundsOpInterfaceImpl.cpp
index f0d43808bc45..7cfcc4180539 100644
--- a/mlir/lib/Dialect/Arith/IR/ValueBoundsOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ValueBoundsOpInterfaceImpl.cpp
@@ -107,9 +107,9 @@ struct SelectOpInterface
     // If trueValue <= falseValue:
     // * result <= falseValue
     // * result >= trueValue
-    if (cstr.compare(trueValue, dim,
+    if (cstr.compare(/*lhs=*/{trueValue, dim},
                      ValueBoundsConstraintSet::ComparisonOperator::LE,
-                     falseValue, dim)) {
+                     /*rhs=*/{falseValue, dim})) {
       if (dim) {
         cstr.bound(value)[*dim] >= cstr.getExpr(trueValue, dim);
         cstr.bound(value)[*dim] <= cstr.getExpr(falseValue, dim);
@@ -121,9 +121,9 @@ struct SelectOpInterface
     // If falseValue <= trueValue:
     // * result <= trueValue
     // * result >= falseValue
-    if (cstr.compare(falseValue, dim,
+    if (cstr.compare(/*lhs=*/{falseValue, dim},
                      ValueBoundsConstraintSet::ComparisonOperator::LE,
-                     trueValue, dim)) {
+                     /*rhs=*/{trueValue, dim})) {
       if (dim) {
         cstr.bound(value)[*dim] >= cstr.getExpr(falseValue, dim);
         cstr.bound(value)[*dim] <= cstr.getExpr(trueValue, dim);
diff --git a/mlir/lib/Dialect/Arith/Transforms/IntNarrowing.cpp b/mlir/lib/Dialect/Arith/Transforms/IntNarrowing.cpp
index 79fabd6ed2e9..f87f3d6350c0 100644
--- a/mlir/lib/Dialect/Arith/Transforms/IntNarrowing.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/IntNarrowing.cpp
@@ -449,7 +449,7 @@ struct IndexCastPattern final : NarrowingPattern<CastOp> {
       return failure();
 
     FailureOr<int64_t> ub = ValueBoundsConstraintSet::computeConstantBound(
-        presburger::BoundType::UB, in, /*dim=*/std::nullopt,
+        presburger::BoundType::UB, in,
         /*stopCondition=*/nullptr, /*closedUB=*/true);
     if (failed(ub))
       return failure();
diff --git a/mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp b/mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp
index fad221288f19..5fb7953f9370 100644
--- a/mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp
@@ -61,16 +61,15 @@ static Value buildArithValue(OpBuilder &b, Location loc, AffineMap map,
   return buildExpr(map.getResult(0));
 }
 
-static FailureOr<OpFoldResult>
-reifyValueBound(OpBuilder &b, Location loc, presburger::BoundType type,
-                Value value, std::optional<int64_t> dim,
-                ValueBoundsConstraintSet::StopConditionFn stopCondition,
-                bool closedUB) {
+FailureOr<OpFoldResult> mlir::arith::reifyValueBound(
+    OpBuilder &b, Location loc, presburger::BoundType type,
+    const ValueBoundsConstraintSet::Variable &var,
+    ValueBoundsConstraintSet::StopConditionFn stopCondition, bool closedUB) {
   // Compute bound.
   AffineMap boundMap;
   ValueDimList mapOperands;
   if (failed(ValueBoundsConstraintSet::computeBound(
-          boundMap, mapOperands, type, value, dim, stopCondition, closedUB)))
+          boundMap, mapOperands, type, var, stopCondition, closedUB)))
     return failure();
 
   // Materialize tensor.dim/memref.dim ops.
@@ -128,7 +127,7 @@ FailureOr<OpFoldResult> mlir::arith::reifyShapedValueDimBound(
     // the owner of `value`.
     return v != value;
   };
-  return reifyValueBound(b, loc, type, value, dim,
+  return reifyValueBound(b, loc, type, {value, dim},
                          stopCondition ? stopCondition : reifyToOperands,
                          closedUB);
 }
@@ -140,7 +139,7 @@ FailureOr<OpFoldResult> mlir::arith::reifyIndexValueBound(
                              ValueBoundsConstraintSet &cstr) {
     return v != value;
   };
-  return reifyValueBound(b, loc, type, value, /*dim=*/std::nullopt,
+  return reifyValueBound(b, loc, type, value,
                          stopCondition ? stopCondition : reifyToOperands,
                          closedUB);
 }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp b/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp
index 8c4b70db2489..518d2e138c02 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp
@@ -72,8 +72,10 @@ static LogicalResult computePaddedShape(linalg::LinalgOp opToPad,
     // Otherwise, try to compute a constant upper bound for the size value.
     FailureOr<int64_t> upperBound =
         ValueBoundsConstraintSet::computeConstantBound(
-            presburger::BoundType::UB, opOperand->get(),
-            /*dim=*/i, /*stopCondition=*/nullptr, /*closedUB=*/true);
+            presburger::BoundType::UB,
+            {opOperand->get(),
+             /*dim=*/i},
+            /*stopCondition=*/nullptr, /*closedUB=*/true);
     if (failed(upperBound)) {
       LLVM_DEBUG(DBGS() << "----could not compute a bounding box for padding");
       return failure();
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp b/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
index ac896d6c30d0..71eb59d40836 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Promotion.cpp
@@ -257,14 +257,12 @@ FailureOr<PromotionInfo> mlir::linalg::promoteSubviewAsNewBuffer(
     if (auto attr = llvm::dyn_cast_if_present<Attribute>(rangeValue.size)) {
       size = getValueOrCreateConstantIndexOp(b, loc, rangeValue.size);
     } else {
-      Value materializedSize =
-          getValueOrCreateConstantIndexOp(b, loc, rangeValue.size);
       FailureOr<int64_t> upperBound =
           ValueBoundsConstraintSet::computeConstantBound(
-              presburger::BoundType::UB, materializedSize, /*dim=*/std::nullopt,
+              presburger::BoundType::UB, rangeValue.size,
               /*stopCondition=*/nullptr, /*closedUB=*/true);
       size = failed(upperBound)
-                 ? materializedSize
+                 ? getValueOrCreateConstantIndexOp(b, loc, rangeValue.size)
                  : b.create<arith::ConstantIndexOp>(loc, *upperBound);
     }
     LLVM_DEBUG(llvm::dbgs() << "Extracted tightest: " << size << "\n");
diff --git a/mlir/lib/Dialect/MemRef/Transforms/IndependenceTransforms.cpp b/mlir/lib/Dialect/MemRef/Transforms/IndependenceTransforms.cpp
index 10ba508265e7..1f06318cbd60 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/IndependenceTransforms.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/IndependenceTransforms.cpp
@@ -23,12 +23,11 @@ static FailureOr<OpFoldResult> makeIndependent(OpBuilder &b, Location loc,
                                                ValueRange independencies) {
   if (ofr.is<Attribute>())
     return ofr;
-  Value value = ofr.get<Value>();
   AffineMap boundMap;
   ValueDimList mapOperands;
   if (failed(ValueBoundsConstraintSet::computeIndependentBound(
-          boundMap, mapOperands, presburger::BoundType::UB, value,
-          /*dim=*/std::nullopt, independencies, /*closedUB=*/true)))
+          boundMap, mapOperands, presburger::BoundType::UB, ofr, independencies,
+          /*closedUB=*/true)))
     return failure();
   return affine::materializeComputedBound(b, loc, boundMap, mapOperands);
 }
diff --git a/mlir/lib/Dialect/SCF/IR/ValueBoundsOpInterfaceImpl.cpp b/mlir/lib/Dialect/SCF/IR/ValueBoundsOpInterfaceImpl.cpp
index 087ffc438a83..17a1c016ea16 100644
--- a/mlir/lib/Dialect/SCF/IR/ValueBoundsOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/SCF/IR/ValueBoundsOpInterfaceImpl.cpp
@@ -61,12 +61,13 @@ struct ForOpInterface
     // An EQ constraint can be added if the yielded value (dimension size)
     // equals the corresponding block argument (dimension size).
     if (cstr.populateAndCompare(
-            yieldedValue, dim, ValueBoundsConstraintSet::ComparisonOperator::EQ,
-            iterArg, dim)) {
+            /*lhs=*/{yieldedValue, dim},
+            ValueBoundsConstraintSet::ComparisonOperator::EQ,
+            /*rhs=*/{iterArg, dim})) {
       if (dim.has_value()) {
         cstr.bound(value)[*dim] == cstr.getExpr(initArg, dim);
       } else {
-        cstr.bound(value) == initArg;
+        cstr.bound(value) == cstr.getExpr(initArg);
       }
     }
   }
@@ -113,8 +114,9 @@ struct IfOpInterface
     // * result <= elseValue
     // * result >= thenValue
     if (cstr.populateAndCompare(
-            thenValue, dim, ValueBoundsConstraintSet::ComparisonOperator::LE,
-            elseValue, dim)) {
+            /*lhs=*/{thenValue, dim},
+            ValueBoundsConstraintSet::ComparisonOperator::LE,
+            /*rhs=*/{elseValue, dim})) {
       if (dim) {
         cstr.bound(value)[*dim] >= cstr.getExpr(thenValue, dim);
         cstr.bound(value)[*dim] <= cstr.getExpr(elseValue, dim);
@@ -127,8 +129,9 @@ struct IfOpInterface
     // * result <= thenValue
     // * result >= elseValue
     if (cstr.populateAndCompare(
-            elseValue, dim, ValueBoundsConstraintSet::ComparisonOperator::LE,
-            thenValue, dim)) {
+            /*lhs=*/{elseValue, dim},
+            ValueBoundsConstraintSet::ComparisonOperator::LE,
+            /*rhs=*/{thenValue, dim})) {
       if (dim) {
         cstr.bound(value)[*dim] >= cstr.getExpr(elseValue, dim);
         cstr.bound(value)[*dim] <= cstr.getExpr(thenValue, dim);
diff --git a/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp b/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp
index 67080d8e301c..d25efcf50ec5 100644
--- a/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp
@@ -289,8 +289,7 @@ static UnpackTileDimInfo getUnpackTileDimInfo(OpBuilder &b, UnPackOp unpackOp,
 
   info.isAlignedToInnerTileSize = false;
   FailureOr<int64_t> cstSize = ValueBoundsConstraintSet::computeConstantBound(
-      presburger::BoundType::UB,
-      getValueOrCreateConstantIndexOp(b, loc, tileSize), /*dim=*/std::nullopt,
+      presburger::BoundType::UB, tileSize,
       /*stopCondition=*/nullptr, /*closedUB=*/true);
   std::optional<int64_t> cstInnerSize = getConstantIntValue(innerTileSize);
   if (!failed(cstSize) && cstInnerSize) {
diff --git a/mlir/lib/Dialect/Tensor/Transforms/IndependenceTransforms.cpp b/mlir/lib/Dialect/Tensor/Transforms/IndependenceTransforms.cpp
index 721730862d49..a89ce20048df 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/IndependenceTransforms.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/IndependenceTransforms.cpp
@@ -28,7 +28,8 @@ static FailureOr<OpFoldResult> makeIndependent(OpBuilder &b, Location loc,
   ValueDimList mapOperands;
   if (failed(ValueBoundsConstraintSet::computeIndependentBound(
           boundMap, mapOperands, presburger::BoundType::UB, value,
-          /*dim=*/std::nullopt, independencies, /*closedUB=*/true)))
+          independencies,
+          /*closedUB=*/true)))
     return failure();
   return mlir::affine::materializeComputedBound(b, loc, boundMap, mapOperands);
 }
diff --git a/mlir/lib/Dialect/Tensor/Utils/Utils.cpp b/mlir/lib/Dialect/Tensor/Utils/Utils.cpp
index 2dd91e2f7a17..15381ec520e2 100644
--- a/mlir/lib/Dialect/Tensor/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Tensor/Utils/Utils.cpp
@@ -154,7 +154,7 @@ bool mlir::tensor::isCastLikeInsertSliceOp(InsertSliceOp op) {
       continue;
     }
     FailureOr<bool> equalDimSize = ValueBoundsConstraintSet::areEqual(
-        op.getSource(), op.getResult(), srcDim, resultDim);
+        {op.getSource(), srcDim}, {op.getResult(), resultDim});
     if (failed(equalDimSize) || !*equalDimSize)
       return false;
     ++srcDim;
@@ -178,7 +178,7 @@ bool mlir::tensor::isCastLikeExtractSliceOp(ExtractSliceOp op) {
       continue;
     }
     FailureOr<bool> equalDimSize = ValueBoundsConstraintSet::areEqual(
-        op.getSource(), op.getResult(), dim, resultDim);
+        {op.getSource(), dim}, {op.getResult(), resultDim});
     if (failed(equalDimSize) || !*equalDimSize)
       return false;
     ++resultDim;
diff --git a/mlir/lib/Interfaces/ValueBoundsOpInterface.cpp b/mlir/lib/Interfaces/ValueBoundsOpInterface.cpp
index ffa4c0b55cad..87937591e60a 100644
--- a/mlir/lib/Interfaces/ValueBoundsOpInterface.cpp
+++ b/mlir/lib/Interfaces/ValueBoundsOpInterface.cpp
@@ -25,6 +25,12 @@ namespace mlir {
 #include "mlir/Interfaces/ValueBoundsOpInterface.cpp.inc"
 } // namespace mlir
 
+static Operation *getOwnerOfValue(Value value) {
+  if (auto bbArg = dyn_cast<BlockArgument>(value))
+    return bbArg.getOwner()->getParentOp();
+  return value.getDefiningOp();
+}
+
 HyperrectangularSlice::HyperrectangularSlice(ArrayRef<OpFoldResult> offsets,
                                              ArrayRef<OpFoldResult> sizes,
                                              ArrayRef<OpFoldResult> strides)
@@ -67,6 +73,83 @@ static std::optional<int64_t> getConstantIntValue(OpFoldResult ofr) {
   return std::nullopt;
 }
 
+ValueBoundsConstraintSet::Variable::Variable(OpFoldResult ofr)
+    : Variable(ofr, std::nullopt) {}
+
+ValueBoundsConstraintSet::Variable::Variable(Value indexValue)
+    : Variable(static_cast<OpFoldResult>(indexValue)) {}
+
+ValueBoundsConstraintSet::Variable::Variable(Value shapedValue, int64_t dim)
+    : Variable(static_cast<OpFoldResult>(shapedValue), std::optional(dim)) {}
+
+ValueBoundsConstraintSet::Variable::Variable(OpFoldResult ofr,
+                                             std::optional<int64_t> dim) {
+  Builder b(ofr.getContext());
+  if (auto constInt = ::getConstantIntValue(ofr)) {
+    assert(!dim && "expected no dim for index-typed values");
+    map = AffineMap::get(/*dimCount=*/0, /*symbolCount=*/0,
+                         b.getAffineConstantExpr(*constInt));
+    return;
+  }
+  Value value = cast<Value>(ofr);
+#ifndef NDEBUG
+  if (dim) {
+    assert(isa<ShapedType>(value.getType()) && "expected shaped type");
+  } else {
+    assert(value.getType().isIndex() && "expected index type");
+  }
+#endif // NDEBUG
+  map = AffineMap::get(/*dimCount=*/0, /*symbolCount=*/1,
+                       b.getAffineSymbolExpr(0));
+  mapOperands.emplace_back(value, dim);
+}
+
+ValueBoundsConstraintSet::Variable::Variable(AffineMap map,
+                                             ArrayRef<Variable> mapOperands) {
+  assert(map.getNumResults() == 1 && "expected single result");
+
+  // Turn all dims into symbols.
+  Builder b(map.getContext());
+  SmallVector<AffineExpr> dimReplacements, symReplacements;
+  for (int64_t i = 0, e = map.getNumDims(); i < e; ++i)
+    dimReplacements.push_back(b.getAffineSymbolExpr(i));
+  for (int64_t i = 0, e = map.getNumSymbols(); i < e; ++i)
+    symReplacements.push_back(b.getAffineSymbolExpr(i + map.getNumDims()));
+  AffineMap tmpMap = map.replaceDimsAndSymbols(
+      dimReplacements, symReplacements, /*numResultDims=*/0,
+      /*numResultSyms=*/map.getNumSymbols() + map.getNumDims());
+
+  // Inline operands.
+  DenseMap<AffineExpr, AffineExpr> replacements;
+  for (auto [index, var] : llvm::enumerate(mapOperands)) {
+    assert(var.map.getNumResults() == 1 && "expected single result");
+    assert(var.map.getNumDims() == 0 && "expected only symbols");
+    SmallVector<AffineExpr> symReplacements;
+    for (auto valueDim : var.mapOperands) {
+      auto it = llvm::find(this->mapOperands, valueDim);
+      if (it != this->mapOperands.end()) {
+        // There is already a symbol for this operand.
+        symReplacements.push_back(b.getAffineSymbolExpr(
+            std::distance(this->mapOperands.begin(), it)));
+      } else {
+        // This is a new operand: add a new symbol.
+        symReplacements.push_back(
+            b.getAffineSymbolExpr(this->mapOperands.size()));
+        this->mapOperands.push_back(valueDim);
+      }
+    }
+    replacements[b.getAffineSymbolExpr(index)] =
+        var.map.getResult(0).replaceSymbols(symReplacements);
+  }
+  this->map = tmpMap.replace(replacements, /*numResultDims=*/0,
+                             /*numResultSyms=*/this->mapOperands.size());
+}
+
+ValueBoundsConstraintSet::Variable::Variable(AffineMap map,
+                                             ArrayRef<Value> mapOperands)
+    : Variable(map, llvm::map_to_vector(mapOperands,
+                                        [](Value v) { return Variable(v); })) {}
+
 ValueBoundsConstraintSet::ValueBoundsConstraintSet(
     MLIRContext *ctx, StopConditionFn stopCondition)
     : builder(ctx), stopCondition(stopCondition) {
@@ -176,6 +259,11 @@ int64_t ValueBoundsConstraintSet::insert(Value value,
   assert(!valueDimToPosition.contains(valueDim) && "already mapped");
   int64_t pos = isSymbol ? cstr.appendVar(VarKind::Symbol)
                          : cstr.appendVar(VarKind::SetDim);
+  LLVM_DEBUG(llvm::dbgs() << "Inserting constraint set column " << pos
+                          << " for: " << value
+                          << " (dim: " << dim.value_or(kIndexValue)
+                          << ", owner: " << getOwnerOfValue(value)->getName()
+                          << ")\n");
   positionToValueDim.insert(positionToValueDim.begin() + pos, valueDim);
   // Update reverse mapping.
   for (int64_t i = pos, e = positionToValueDim.size(); i < e; ++i)
@@ -194,6 +282,8 @@ int64_t ValueBoundsConstraintSet::insert(Value value,
 int64_t ValueBoundsConstraintSet::insert(bool isSymbol) {
   int64_t pos = isSymbol ? cstr.appendVar(VarKind::Symbol)
                          : cstr.appendVar(VarKind::SetDim);
+  LLVM_DEBUG(llvm::dbgs() << "Inserting anonymous constraint set column " << pos
+                          << "\n");
   positionToValueDim.insert(positionToValueDim.begin() + pos, std::nullopt);
   // Update reverse mapping.
   for (int64_t i = pos, e = positionToValueDim.size(); i < e; ++i)
@@ -224,6 +314,10 @@ int64_t ValueBoundsConstraintSet::insert(AffineMap map, ValueDimList operands,
   return pos;
 }
 
+int64_t ValueBoundsConstraintSet::insert(const Variable &var, bool isSymbol) {
+  return insert(var.map, var.mapOperands, isSymbol);
+}
+
 int64_t ValueBoundsConstraintSet::getPos(Value value,
                                          std::optional<int64_t> dim) const {
 #ifndef NDEBUG
@@ -232,7 +326,10 @@ int64_t ValueBoundsConstraintSet::getPos(Value value,
           cast<BlockArgument>(value).getOwner()->isEntryBlock()) &&
          "unstructured control flow is not supported");
 #endif // NDEBUG
-
+  LLVM_DEBUG(llvm::dbgs() << "Getting pos for: " << value
+                          << " (dim: " << dim.value_or(kIndexValue)
+                          << ", owner: " << getOwnerOfValue(value)->getName()
+                          << ")\n");
   auto it =
       valueDimToPosition.find(std::make_pair(value, dim.value_or(kIndexValue)));
   assert(it != valueDimToPosition.end() && "expected mapped entry");
@@ -253,12 +350,6 @@ bool ValueBoundsConstraintSet::isMapped(Value value,
   return it != valueDimToPosition.end();
 }
 
-static Operation *getOwnerOfValue(Value value) {
-  if (auto bbArg = dyn_cast<BlockArgument>(value))
-    return bbArg.getOwner()->getParentOp();
-  return value.getDefiningOp();
-}
-
 void ValueBoundsConstraintSet::processWorklist() {
   LLVM_DEBUG(llvm::dbgs() << "Processing value bounds worklist...\n");
   while (!worklist.empty()) {
@@ -346,41 +437,47 @@ void ValueBoundsConstraintSet::projectOut(
   }
 }
 
+void ValueBoundsConstraintSet::projectOutAnonymous(
+    std::optional<int64_t> except) {
+  int64_t nextPos = 0;
+  while (nextPos < static_cast<int64_t>(positionToValueDim.size())) {
+    if (positionToValueDim[nextPos].has_value() || except == nextPos) {
+      ++nextPos;
+    } else {
+      projectOut(nextPos);
+      // The column was projected out so another column is now at that position.
+      // Do not increase the counter.
+    }
+  }
+}
+
 LogicalResult ValueBoundsConstraintSet::computeBound(
     AffineMap &resultMap, ValueDimList &mapOperands, presburger::BoundType type,
-    Value value, std::optional<int64_t> dim, StopConditionFn stopCondition,
-    bool closedUB) {
-#ifndef NDEBUG
-  assertValidValueDim(value, dim);
-#endif // NDEBUG
-
+    const Variable &var, StopConditionFn stopCondition, bool closedUB) {
+  MLIRContext *ctx = var.getContext();
   int64_t ubAdjustment = closedUB ? 0 : 1;
-  Builder b(value.getContext());
+  Builder b(ctx);
   mapOperands.clear();
 
   // Process the backward slice of `value` (i.e., reverse use-def chain) until
   // `stopCondition` is met.
-  ValueDim valueDim = std::make_pair(value, dim.value_or(kIndexValue));
-  ValueBoundsConstraintSet cstr(value.getContext(), stopCondition);
-  assert(!stopCondition(value, dim, cstr) &&
-         "stop condition should not be satisfied for starting point");
-  int64_t pos = cstr.insert(value, dim, /*isSymbol=*/false);
+  ValueBoundsConstraintSet cstr(ctx, stopCondition);
+  int64_t pos = cstr.insert(var, /*isSymbol=*/false);
+  assert(pos == 0 && "expected first column");
   cstr.processWorklist();
 
   // Project out all variables (apart from `valueDim`) that do not match the
   // stop condition.
   cstr.projectOut([&](ValueDim p) {
-    // Do not project out `valueDim`.
-    if (valueDim == p)
-      return false;
     auto maybeDim =
         p.second == kIndexValue ? std::nullopt : std::make_optional(p.second);
     return !stopCondition(p.first, maybeDim, cstr);
   });
+  cstr.projectOutAnonymous(/*except=*/pos);
 
   // Compute lower and upper bounds for `valueDim`.
   SmallVector<AffineMap> lb(1), ub(1);
-  cstr.cstr.getSliceBounds(pos, 1, value.getContext(), &lb, &ub,
+  cstr.cstr.getSliceBounds(pos, 1, ctx, &lb, &ub,
                            /*closedUB=*/true);
 
   // Note: There are TODOs in the implementation of `getSliceBounds`. In such a
@@ -477,10 +574,9 @@ LogicalResult ValueBoundsConstraintSet::computeBound(
 
 LogicalResult ValueBoundsConstraintSet::computeDependentBound(
     AffineMap &resultMap, ValueDimList &mapOperands, presburger::BoundType type,
-    Value value, std::optional<int64_t> dim, ValueDimList dependencies,
-    bool closedUB) {
+    const Variable &var, ValueDimList dependencies, bool closedUB) {
   return computeBound(
-      resultMap, mapOperands, type, value, dim,
+      resultMap, mapOperands, type, var,
       [&](Value v, std::optional<int64_t> d, ValueBoundsConstraintSet &cstr) {
         return llvm::is_contained(dependencies, std::make_pair(v, d));
       },
@@ -489,8 +585,7 @@ LogicalResult ValueBoundsConstraintSet::computeDependentBound(
 
 LogicalResult ValueBoundsConstraintSet::computeIndependentBound(
     AffineMap &resultMap, ValueDimList &mapOperands, presburger::BoundType type,
-    Value value, std::optional<int64_t> dim, ValueRange independencies,
-    bool closedUB) {
+    const Variable &var, ValueRange independencies, bool closedUB) {
   // Return "true" if the given value is independent of all values in
   // `independencies`. I.e., neither the value itself nor any value in the
   // backward slice (reverse use-def chain) is contained in `independencies`.
@@ -516,7 +611,7 @@ LogicalResult ValueBoundsConstraintSet::computeIndependentBound(
 
   // Reify bounds in terms of any independent values.
   return computeBound(
-      resultMap, mapOperands, type, value, dim,
+      resultMap, mapOperands, type, var,
       [&](Value v, std::optional<int64_t> d, ValueBoundsConstraintSet &cstr) {
         return isIndependent(v);
       },
@@ -524,35 +619,8 @@ LogicalResult ValueBoundsConstraintSet::computeIndependentBound(
 }
 
 FailureOr<int64_t> ValueBoundsConstraintSet::computeConstantBound(
-    presburger::BoundType type, Value value, std::optional<int64_t> dim,
-    StopConditionFn stopCondition, bool closedUB) {
-#ifndef NDEBUG
-  assertValidValueDim(value, dim);
-#endif // NDEBUG
-
-  AffineMap map =
-      AffineMap::get(/*dimCount=*/1, /*symbolCount=*/0,
-                     Builder(value.getContext()).getAffineDimExpr(0));
-  return computeConstantBound(type, map, {{value, dim}}, stopCondition,
-                              closedUB);
-}
-
-FailureOr<int64_t> ValueBoundsConstraintSet::computeConstantBound(
-    presburger::BoundType type, AffineMap map, ArrayRef<Value> operands,
+    presburger::BoundType type, const Variable &var,
     StopConditionFn stopCondition, bool closedUB) {
-  ValueDimList valueDims;
-  for (Value v : operands) {
-    assert(v.getType().isIndex() && "expected index type");
-    valueDims.emplace_back(v, std::nullopt);
-  }
-  return computeConstantBound(type, map, valueDims, stopCondition, closedUB);
-}
-
-FailureOr<int64_t> ValueBoundsConstraintSet::computeConstantBound(
-    presburger::BoundType type, AffineMap map, ValueDimList operands,
-    StopConditionFn stopCondition, bool closedUB) {
-  assert(map.getNumResults() == 1 && "expected affine map with one result");
-
   // Default stop condition if none was specified: Keep adding constraints until
   // a bound could be computed.
   int64_t pos = 0;
@@ -562,8 +630,8 @@ FailureOr<int64_t> ValueBoundsConstraintSet::computeConstantBound(
   };
 
   ValueBoundsConstraintSet cstr(
-      map.getContext(), stopCondition ? stopCondition : defaultStopCondition);
-  pos = cstr.populateConstraints(map, operands);
+      var.getContext(), stopCondition ? stopCondition : defaultStopCondition);
+  pos = cstr.populateConstraints(var.map, var.mapOperands);
   assert(pos == 0 && "expected `map` is the first column");
 
   // Compute constant bound for `valueDim`.
@@ -608,22 +676,13 @@ ValueBoundsConstraintSet::computeConstantDelta(Value value1, Value value2,
   Builder b(value1.getContext());
   AffineMap map = AffineMap::get(/*dimCount=*/2, /*symbolCount=*/0,
                                  b.getAffineDimExpr(0) - b.getAffineDimExpr(1));
-  return computeConstantBound(presburger::BoundType::EQ, map,
-                              {{value1, dim1}, {value2, dim2}});
+  return computeConstantBound(presburger::BoundType::EQ,
+                              Variable(map, {{value1, dim1}, {value2, dim2}}));
 }
 
-bool ValueBoundsConstraintSet::compareValueDims(OpFoldResult lhs,
-                                                std::optional<int64_t> lhsDim,
-                                                ComparisonOperator cmp,
-                                                OpFoldResult rhs,
-                                                std::optional<int64_t> rhsDim) {
-#ifndef NDEBUG
-  if (auto lhsVal = dyn_cast<Value>(lhs))
-    assertValidValueDim(lhsVal, lhsDim);
-  if (auto rhsVal = dyn_cast<Value>(rhs))
-    assertValidValueDim(rhsVal, rhsDim);
-#endif // NDEBUG
-
+bool ValueBoundsConstraintSet::comparePos(int64_t lhsPos,
+                                          ComparisonOperator cmp,
+                                          int64_t rhsPos) {
   // This function returns "true" if "lhs CMP rhs" is proven to hold.
   //
   // Example for ComparisonOperator::LE and index-typed values: We would like to
@@ -640,50 +699,6 @@ bool ValueBoundsConstraintSet::compareValueDims(OpFoldResult lhs,
     return false;
   }
 
-  // EQ can be expressed as LE and GE.
-  if (cmp == EQ)
-    return compareValueDims(lhs, lhsDim, ComparisonOperator::LE, rhs, rhsDim) &&
-           compareValueDims(lhs, lhsDim, ComparisonOperator::GE, rhs, rhsDim);
-
-  // Construct inequality. For the above example: lhs > rhs.
-  // `IntegerRelation` inequalities are expressed in the "flattened" form and
-  // with ">= 0". I.e., lhs - rhs - 1 >= 0.
-  SmallVector<int64_t> eq(cstr.getNumCols(), 0);
-  auto addToEq = [&](OpFoldResult ofr, std::optional<int64_t> dim,
-                     int64_t factor) {
-    if (auto constVal = ::getConstantIntValue(ofr)) {
-      eq[cstr.getNumCols() - 1] += *constVal * factor;
-    } else {
-      eq[getPos(cast<Value>(ofr), dim)] += factor;
-    }
-  };
-  if (cmp == LT || cmp == LE) {
-    addToEq(lhs, lhsDim, 1);
-    addToEq(rhs, rhsDim, -1);
-  } else if (cmp == GT || cmp == GE) {
-    addToEq(lhs, lhsDim, -1);
-    addToEq(rhs, rhsDim, 1);
-  } else {
-    llvm_unreachable("unsupported comparison operator");
-  }
-  if (cmp == LE || cmp == GE)
-    eq[cstr.getNumCols() - 1] -= 1;
-
-  // Add inequality to the constraint set and check if it made the constraint
-  // set empty.
-  int64_t ineqPos = cstr.getNumInequalities();
-  cstr.addInequality(eq);
-  bool isEmpty = cstr.isEmpty();
-  cstr.removeInequality(ineqPos);
-  return isEmpty;
-}
-
-bool ValueBoundsConstraintSet::comparePos(int64_t lhsPos,
-                                          ComparisonOperator cmp,
-                                          int64_t rhsPos) {
-  // This function returns "true" if "lhs CMP rhs" is proven to hold. For
-  // detailed documentation, see `compareValueDims`.
-
   // EQ can be expressed as LE and GE.
   if (cmp == EQ)
     return comparePos(lhsPos, ComparisonOperator::LE, rhsPos) &&
@@ -712,48 +727,17 @@ bool ValueBoundsConstraintSet::comparePos(int64_t lhsPos,
   return isEmpty;
 }
 
-bool ValueBoundsConstraintSet::populateAndCompare(
-    OpFoldResult lhs, std::optional<int64_t> lhsDim, ComparisonOperator cmp,
-    OpFoldResult rhs, std::optional<int64_t> rhsDim) {
-#ifndef NDEBUG
-  if (auto lhsVal = dyn_cast<Value>(lhs))
-    assertValidValueDim(lhsVal, lhsDim);
-  if (auto rhsVal = dyn_cast<Value>(rhs))
-    assertValidValueDim(rhsVal, rhsDim);
-#endif // NDEBUG
-
-  if (auto lhsVal = dyn_cast<Value>(lhs))
-    populateConstraints(lhsVal, lhsDim);
-  if (auto rhsVal = dyn_cast<Value>(rhs))
-    populateConstraints(rhsVal, rhsDim);
-
-  return compareValueDims(lhs, lhsDim, cmp, rhs, rhsDim);
+bool ValueBoundsConstraintSet::populateAndCompare(const Variable &lhs,
+                                                  ComparisonOperator cmp,
+                                                  const Variable &rhs) {
+  int64_t lhsPos = populateConstraints(lhs.map, lhs.mapOperands);
+  int64_t rhsPos = populateConstraints(rhs.map, rhs.mapOperands);
+  return comparePos(lhsPos, cmp, rhsPos);
 }
 
-bool ValueBoundsConstraintSet::compare(OpFoldResult lhs,
-                                       std::optional<int64_t> lhsDim,
-                                       ComparisonOperator cmp, OpFoldResult rhs,
-                                       std::optional<int64_t> rhsDim) {
-  auto stopCondition = [&](Value v, std::optional<int64_t> dim,
-                           ValueBoundsConstraintSet &cstr) {
-    // Keep processing as long as lhs/rhs are not mapped.
-    if (auto lhsVal = dyn_cast<Value>(lhs))
-      if (!cstr.isMapped(lhsVal, dim))
-        return false;
-    if (auto rhsVal = dyn_cast<Value>(rhs))
-      if (!cstr.isMapped(rhsVal, dim))
-        return false;
-    // Keep processing as long as the relation cannot be proven.
-    return cstr.compareValueDims(lhs, lhsDim, cmp, rhs, rhsDim);
-  };
-
-  ValueBoundsConstraintSet cstr(lhs.getContext(), stopCondition);
-  return cstr.populateAndCompare(lhs, lhsDim, cmp, rhs, rhsDim);
-}
-
-bool ValueBoundsConstraintSet::compare(AffineMap lhs, ValueDimList lhsOperands,
-                                       ComparisonOperator cmp, AffineMap rhs,
-                                       ValueDimList rhsOperands) {
+bool ValueBoundsConstraintSet::compare(const Variable &lhs,
+                                       ComparisonOperator cmp,
+                                       const Variable &rhs) {
   int64_t lhsPos = -1, rhsPos = -1;
   auto stopCondition = [&](Value v, std::optional<int64_t> dim,
                            ValueBoundsConstraintSet &cstr) {
@@ -765,39 +749,17 @@ bool ValueBoundsConstraintSet::compare(AffineMap lhs, ValueDimList lhsOperands,
     return cstr.comparePos(lhsPos, cmp, rhsPos);
   };
   ValueBoundsConstraintSet cstr(lhs.getContext(), stopCondition);
-  lhsPos = cstr.insert(lhs, lhsOperands);
-  rhsPos = cstr.insert(rhs, rhsOperands);
-  cstr.processWorklist();
+  lhsPos = cstr.populateConstraints(lhs.map, lhs.mapOperands);
+  rhsPos = cstr.populateConstraints(rhs.map, rhs.mapOperands);
   return cstr.comparePos(lhsPos, cmp, rhsPos);
 }
 
-bool ValueBoundsConstraintSet::compare(AffineMap lhs,
-                                       ArrayRef<Value> lhsOperands,
-                                       ComparisonOperator cmp, AffineMap rhs,
-                                       ArrayRef<Value> rhsOperands) {
-  ValueDimList lhsValueDimOperands =
-      llvm::map_to_vector(lhsOperands, [](Value v) {
-        return std::make_pair(v, std::optional<int64_t>());
-      });
-  ValueDimList rhsValueDimOperands =
-      llvm::map_to_vector(rhsOperands, [](Value v) {
-        return std::make_pair(v, std::optional<int64_t>());
-      });
-  return ValueBoundsConstraintSet::compare(lhs, lhsValueDimOperands, cmp, rhs,
-                                           rhsValueDimOperands);
-}
-
-FailureOr<bool>
-ValueBoundsConstraintSet::areEqual(OpFoldResult value1, OpFoldResult value2,
-                                   std::optional<int64_t> dim1,
-                                   std::optional<int64_t> dim2) {
-  if (ValueBoundsConstraintSet::compare(value1, dim1, ComparisonOperator::EQ,
-                                        value2, dim2))
+FailureOr<bool> ValueBoundsConstraintSet::areEqual(const Variable &var1,
+                                                   const Variable &var2) {
+  if (ValueBoundsConstraintSet::compare(var1, ComparisonOperator::EQ, var2))
     return true;
-  if (ValueBoundsConstraintSet::compare(value1, dim1, ComparisonOperator::LT,
-                                        value2, dim2) ||
-      ValueBoundsConstraintSet::compare(value1, dim1, ComparisonOperator::GT,
-                                        value2, dim2))
+  if (ValueBoundsConstraintSet::compare(var1, ComparisonOperator::LT, var2) ||
+      ValueBoundsConstraintSet::compare(var1, ComparisonOperator::GT, var2))
     return false;
   return failure();
 }
@@ -833,7 +795,7 @@ ValueBoundsConstraintSet::areOverlappingSlices(MLIRContext *ctx,
       AffineMap foldedMap =
           foldAttributesIntoMap(b, map, ofrOperands, valueOperands);
       FailureOr<int64_t> constBound = computeConstantBound(
-          presburger::BoundType::EQ, foldedMap, valueOperands);
+          presburger::BoundType::EQ, Variable(foldedMap, valueOperands));
       foundUnknownBound |= failed(constBound);
       if (succeeded(constBound) && *constBound <= 0)
         return false;
@@ -850,7 +812,7 @@ ValueBoundsConstraintSet::areOverlappingSlices(MLIRContext *ctx,
       AffineMap foldedMap =
           foldAttributesIntoMap(b, map, ofrOperands, valueOperands);
       FailureOr<int64_t> constBound = computeConstantBound(
-          presburger::BoundType::EQ, foldedMap, valueOperands);
+          presburger::BoundType::EQ, Variable(foldedMap, valueOperands));
       foundUnknownBound |= failed(constBound);
       if (succeeded(constBound) && *constBound <= 0)
         return false;
diff --git a/mlir/test/Dialect/Affine/value-bounds-op-interface-impl.mlir b/mlir/test/Dialect/Affine/value-bounds-op-interface-impl.mlir
index 23c6872dcebe..935c08aceff5 100644
--- a/mlir/test/Dialect/Affine/value-bounds-op-interface-impl.mlir
+++ b/mlir/test/Dialect/Affine/value-bounds-op-interface-impl.mlir
@@ -131,3 +131,27 @@ func.func @compare_affine_min(%a: index, %b: index) {
   "test.compare"(%0, %a) {cmp = "LE"} : (index, index) -> ()
   return
 }
+
+// -----
+
+func.func @compare_const_map() {
+  %c5 = arith.constant 5 : index
+  // expected-remark @below{{true}}
+  "test.compare"(%c5) {cmp = "GT", rhs_map = affine_map<() -> (4)>}
+      : (index) -> ()
+  // expected-remark @below{{true}}
+  "test.compare"(%c5) {cmp = "LT", lhs_map = affine_map<() -> (4)>}
+      : (index) -> ()
+  return
+}
+
+// -----
+
+func.func @compare_maps(%a: index, %b: index) {
+  // expected-remark @below{{true}}
+  "test.compare"(%a, %b, %b, %a)
+      {cmp = "GT", lhs_map = affine_map<(d0, d1) -> (1 + d0 + d1)>,
+       rhs_map = affine_map<(d0, d1) -> (d0 + d1)>}
+      : (index, index, index, index) -> ()
+  return
+}
diff --git a/mlir/test/lib/Dialect/Affine/TestReifyValueBounds.cpp b/mlir/test/lib/Dialect/Affine/TestReifyValueBounds.cpp
index 6730f9b292ad..b098a5a23fd3 100644
--- a/mlir/test/lib/Dialect/Affine/TestReifyValueBounds.cpp
+++ b/mlir/test/lib/Dialect/Affine/TestReifyValueBounds.cpp
@@ -109,7 +109,7 @@ static LogicalResult testReifyValueBounds(func::FuncOp funcOp,
     FailureOr<OpFoldResult> reified = failure();
     if (constant) {
       auto reifiedConst = ValueBoundsConstraintSet::computeConstantBound(
-          boundType, value, dim, /*stopCondition=*/nullptr);
+          boundType, {value, dim}, /*stopCondition=*/nullptr);
       if (succeeded(reifiedConst))
         reified = FailureOr<OpFoldResult>(rewriter.getIndexAttr(*reifiedConst));
     } else if (scalable) {
@@ -128,22 +128,12 @@ static LogicalResult testReifyValueBounds(func::FuncOp funcOp,
             rewriter, loc, reifiedScalable->map, vscaleOperand);
       }
     } else {
-      if (dim) {
-        if (useArithOps) {
-          reified = arith::reifyShapedValueDimBound(
-              rewriter, op->getLoc(), boundType, value, *dim, stopCondition);
-        } else {
-          reified = reifyShapedValueDimBound(rewriter, op->getLoc(), boundType,
-                                             value, *dim, stopCondition);
-        }
+      if (useArithOps) {
+        reified = arith::reifyValueBound(rewriter, op->getLoc(), boundType,
+                                         op.getVariable(), stopCondition);
       } else {
-        if (useArithOps) {
-          reified = arith::reifyIndexValueBound(
-              rewriter, op->getLoc(), boundType, value, stopCondition);
-        } else {
-          reified = reifyIndexValueBound(rewriter, op->getLoc(), boundType,
-                                         value, stopCondition);
-        }
+        reified = reifyValueBound(rewriter, op->getLoc(), boundType,
+                                  op.getVariable(), stopCondition);
       }
     }
     if (failed(reified)) {
@@ -188,9 +178,7 @@ static LogicalResult testEquality(func::FuncOp funcOp) {
     }
 
     auto compare = [&](ValueBoundsConstraintSet::ComparisonOperator cmp) {
-      return ValueBoundsConstraintSet::compare(
-          /*lhs=*/op.getLhs(), /*lhsDim=*/std::nullopt, cmp,
-          /*rhs=*/op.getRhs(), /*rhsDim=*/std::nullopt);
+      return ValueBoundsConstraintSet::compare(op.getLhs(), cmp, op.getRhs());
     };
     if (compare(cmpType)) {
       op->emitRemark("true");
diff --git a/mlir/test/lib/Dialect/Test/TestDialect.cpp b/mlir/test/lib/Dialect/Test/TestDialect.cpp
index 25c5190ca0ef..36d7606fe134 100644
--- a/mlir/test/lib/Dialect/Test/TestDialect.cpp
+++ b/mlir/test/lib/Dialect/Test/TestDialect.cpp
@@ -549,6 +549,12 @@ LogicalResult ReifyBoundOp::verify() {
   return success();
 }
 
+::mlir::ValueBoundsConstraintSet::Variable ReifyBoundOp::getVariable() {
+  if (getDim().has_value())
+    return ValueBoundsConstraintSet::Variable(getVar(), *getDim());
+  return ValueBoundsConstraintSet::Variable(getVar());
+}
+
 ::mlir::ValueBoundsConstraintSet::ComparisonOperator
 CompareOp::getComparisonOperator() {
   if (getCmp() == "EQ")
@@ -564,6 +570,37 @@ CompareOp::getComparisonOperator() {
   llvm_unreachable("invalid comparison operator");
 }
 
+::mlir::ValueBoundsConstraintSet::Variable CompareOp::getLhs() {
+  if (!getLhsMap())
+    return ValueBoundsConstraintSet::Variable(getVarOperands()[0]);
+  SmallVector<Value> mapOperands(
+      getVarOperands().slice(0, getLhsMap()->getNumInputs()));
+  return ValueBoundsConstraintSet::Variable(*getLhsMap(), mapOperands);
+}
+
+::mlir::ValueBoundsConstraintSet::Variable CompareOp::getRhs() {
+  int64_t rhsOperandsBegin = getLhsMap() ? getLhsMap()->getNumInputs() : 1;
+  if (!getRhsMap())
+    return ValueBoundsConstraintSet::Variable(
+        getVarOperands()[rhsOperandsBegin]);
+  SmallVector<Value> mapOperands(
+      getVarOperands().slice(rhsOperandsBegin, getRhsMap()->getNumInputs()));
+  return ValueBoundsConstraintSet::Variable(*getRhsMap(), mapOperands);
+}
+
+LogicalResult CompareOp::verify() {
+  if (getCompose() && (getLhsMap() || getRhsMap()))
+    return emitOpError(
+        "'compose' not supported when 'lhs_map' or 'rhs_map' is present");
+  int64_t expectedNumOperands = getLhsMap() ? getLhsMap()->getNumInputs() : 1;
+  expectedNumOperands += getRhsMap() ? getRhsMap()->getNumInputs() : 1;
+  if (getVarOperands().size() != expectedNumOperands)
+    return emitOpError("expected ")
+           << expectedNumOperands << " operands, but got "
+           << getVarOperands().size();
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // Test removing op with inner ops.
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/lib/Dialect/Test/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td
index ebf158b8bb82..b641b3da719c 100644
--- a/mlir/test/lib/Dialect/Test/TestOps.td
+++ b/mlir/test/lib/Dialect/Test/TestOps.td
@@ -2207,6 +2207,7 @@ def ReifyBoundOp : TEST_Op<"reify_bound", [Pure]> {
 
   let extraClassDeclaration = [{
     ::mlir::presburger::BoundType getBoundType();
+    ::mlir::ValueBoundsConstraintSet::Variable getVariable();
   }];
 
   let hasVerifier = 1;
@@ -2217,18 +2218,29 @@ def CompareOp : TEST_Op<"compare"> {
     Compare `lhs` and `rhs`. A remark is emitted which indicates whether the
     specified comparison operator was proven to hold. The remark also indicates
     whether the opposite comparison operator was proven to hold.
+
+    `var_operands` must have exactly two operands: one for the LHS operand and
+    one for the RHS operand. If `lhs_map` is specified, as many operands as
+    `lhs_map` has inputs are expected instead of the first operand. If `rhs_map`
+    is specified, as many operands as `rhs_map` has inputs are expected instead
+    of the second operand.
   }];
 
-  let arguments = (ins Index:$lhs,
-                       Index:$rhs,
+  let arguments = (ins Variadic<Index>:$var_operands,
                        DefaultValuedAttr<StrAttr, "\"EQ\"">:$cmp,
+                       OptionalAttr<AffineMapAttr>:$lhs_map,
+                       OptionalAttr<AffineMapAttr>:$rhs_map,
                        UnitAttr:$compose);
   let results = (outs);
 
   let extraClassDeclaration = [{
     ::mlir::ValueBoundsConstraintSet::ComparisonOperator
         getComparisonOperator();
+    ::mlir::ValueBoundsConstraintSet::Variable getLhs();
+    ::mlir::ValueBoundsConstraintSet::Variable getRhs();
   }];
+
+  let hasVerifier = 1;
 }
 
 //===----------------------------------------------------------------------===//
-- 
GitLab


From 5a46123ddf62900d3dc73330f699c73038645198 Mon Sep 17 00:00:00 2001
From: Utkarsh Saxena <usx@google.com>
Date: Tue, 16 Apr 2024 11:01:03 +0200
Subject: [PATCH 353/464] Fix missing dtor in function calls accepting trivial
 ABI structs (#88751)

Fixes https://github.com/llvm/llvm-project/issues/88478

Promoting the `EHCleanup` to `NormalAndEHCleanup` in `EmitCallArgs`
surfaced another bug with deactivation of normal cleanups. Here we
missed emitting CPP scope ends for deactivated normal cleanups. This
patch also fixes that bug.

We missed emitting CPP scope ends because we remove the `fallthrough`
(clears the insertion point) before deactivating normal cleanups. This
is to make the emitted "normal" cleanup code unreachable. But we still
need to emit CPP scope ends in the original basic block even for a
deactivated normal cleanup.
(This worked correctly before we did not remove `fallthrough` for
`EHCleanup`s).
---
 clang/lib/CodeGen/CGCall.cpp                  | 13 ++++---
 clang/lib/CodeGen/CGCleanup.cpp               | 37 ++++++++++++-------
 clang/lib/CodeGen/CodeGenFunction.h           |  3 +-
 .../CodeGenCXX/control-flow-in-stmt-expr.cpp  | 16 ++++++++
 4 files changed, 48 insertions(+), 21 deletions(-)

diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index 7a0bc6fa77b8..0c860a3ccbd2 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -4694,11 +4694,11 @@ void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
     AggValueSlot Slot = args.isUsingInAlloca()
         ? createPlaceholderSlot(*this, type) : CreateAggTemp(type, "agg.tmp");
 
-    bool DestroyedInCallee = true, NeedsEHCleanup = true;
+    bool DestroyedInCallee = true, NeedsCleanup = true;
     if (const auto *RD = type->getAsCXXRecordDecl())
       DestroyedInCallee = RD->hasNonTrivialDestructor();
     else
-      NeedsEHCleanup = needsEHCleanup(type.isDestructedType());
+      NeedsCleanup = type.isDestructedType();
 
     if (DestroyedInCallee)
       Slot.setExternallyDestructed();
@@ -4707,14 +4707,15 @@ void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
     RValue RV = Slot.asRValue();
     args.add(RV, type);
 
-    if (DestroyedInCallee && NeedsEHCleanup) {
+    if (DestroyedInCallee && NeedsCleanup) {
       // Create a no-op GEP between the placeholder and the cleanup so we can
       // RAUW it successfully.  It also serves as a marker of the first
       // instruction where the cleanup is active.
-      pushFullExprCleanup<DestroyUnpassedArg>(EHCleanup, Slot.getAddress(),
-                                              type);
+      pushFullExprCleanup<DestroyUnpassedArg>(NormalAndEHCleanup,
+                                              Slot.getAddress(), type);
       // This unreachable is a temporary marker which will be removed later.
-      llvm::Instruction *IsActive = Builder.CreateUnreachable();
+      llvm::Instruction *IsActive =
+          Builder.CreateFlagLoad(llvm::Constant::getNullValue(Int8PtrTy));
       args.addArgCleanupDeactivation(EHStack.stable_begin(), IsActive);
     }
     return;
diff --git a/clang/lib/CodeGen/CGCleanup.cpp b/clang/lib/CodeGen/CGCleanup.cpp
index 5bf48bc22a54..8683f19d9da2 100644
--- a/clang/lib/CodeGen/CGCleanup.cpp
+++ b/clang/lib/CodeGen/CGCleanup.cpp
@@ -634,12 +634,19 @@ static void destroyOptimisticNormalEntry(CodeGenFunction &CGF,
 /// Pops a cleanup block.  If the block includes a normal cleanup, the
 /// current insertion point is threaded through the cleanup, as are
 /// any branch fixups on the cleanup.
-void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) {
+void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough,
+                                      bool ForDeactivation) {
   assert(!EHStack.empty() && "cleanup stack is empty!");
   assert(isa<EHCleanupScope>(*EHStack.begin()) && "top not a cleanup!");
   EHCleanupScope &Scope = cast<EHCleanupScope>(*EHStack.begin());
   assert(Scope.getFixupDepth() <= EHStack.getNumBranchFixups());
 
+  // If we are deactivating a normal cleanup, we need to pretend that the
+  // fallthrough is unreachable. We restore this IP before returning.
+  CGBuilderTy::InsertPoint NormalDeactivateOrigIP;
+  if (ForDeactivation && (Scope.isNormalCleanup() || !getLangOpts().EHAsynch)) {
+    NormalDeactivateOrigIP = Builder.saveAndClearIP();
+  }
   // Remember activation information.
   bool IsActive = Scope.isActive();
   Address NormalActiveFlag =
@@ -729,6 +736,8 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) {
     EHStack.popCleanup(); // safe because there are no fixups
     assert(EHStack.getNumBranchFixups() == 0 ||
            EHStack.hasNormalCleanups());
+    if (NormalDeactivateOrigIP.isSet())
+      Builder.restoreIP(NormalDeactivateOrigIP);
     return;
   }
 
@@ -765,9 +774,16 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) {
   if (!RequiresNormalCleanup) {
     // Mark CPP scope end for passed-by-value Arg temp
     //   per Windows ABI which is "normally" Cleanup in callee
-    if (IsEHa && getInvokeDest() && Builder.GetInsertBlock()) {
-      if (Personality.isMSVCXXPersonality())
+    if (IsEHa && getInvokeDest()) {
+      // If we are deactivating a normal cleanup then we don't have a
+      // fallthrough. Restore original IP to emit CPP scope ends in the correct
+      // block.
+      if (NormalDeactivateOrigIP.isSet())
+        Builder.restoreIP(NormalDeactivateOrigIP);
+      if (Personality.isMSVCXXPersonality() && Builder.GetInsertBlock())
         EmitSehCppScopeEnd();
+      if (NormalDeactivateOrigIP.isSet())
+        NormalDeactivateOrigIP = Builder.saveAndClearIP();
     }
     destroyOptimisticNormalEntry(*this, Scope);
     Scope.MarkEmitted();
@@ -992,6 +1008,8 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) {
     }
   }
 
+  if (NormalDeactivateOrigIP.isSet())
+    Builder.restoreIP(NormalDeactivateOrigIP);
   assert(EHStack.hasNormalCleanups() || EHStack.getNumBranchFixups() == 0);
 
   // Emit the EH cleanup if required.
@@ -1281,17 +1299,8 @@ void CodeGenFunction::DeactivateCleanupBlock(EHScopeStack::stable_iterator C,
   // to the current RunCleanupsScope.
   if (C == EHStack.stable_begin() &&
       CurrentCleanupScopeDepth.strictlyEncloses(C)) {
-    // Per comment below, checking EHAsynch is not really necessary
-    // it's there to assure zero-impact w/o EHAsynch option
-    if (!Scope.isNormalCleanup() && getLangOpts().EHAsynch) {
-      PopCleanupBlock();
-    } else {
-      // If it's a normal cleanup, we need to pretend that the
-      // fallthrough is unreachable.
-      CGBuilderTy::InsertPoint SavedIP = Builder.saveAndClearIP();
-      PopCleanupBlock();
-      Builder.restoreIP(SavedIP);
-    }
+    PopCleanupBlock(/*FallthroughIsBranchThrough=*/false,
+                    /*ForDeactivation=*/true);
     return;
   }
 
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index c49e9fd00c8d..d99188671f1f 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -957,7 +957,8 @@ public:
 
   /// PopCleanupBlock - Will pop the cleanup entry on the stack and
   /// process all branch fixups.
-  void PopCleanupBlock(bool FallThroughIsBranchThrough = false);
+  void PopCleanupBlock(bool FallThroughIsBranchThrough = false,
+                       bool ForDeactivation = false);
 
   /// DeactivateCleanupBlock - Deactivates the given cleanup block.
   /// The block cannot be reactivated.  Pops it if it's the top of the
diff --git a/clang/test/CodeGenCXX/control-flow-in-stmt-expr.cpp b/clang/test/CodeGenCXX/control-flow-in-stmt-expr.cpp
index 95deee8bb1f1..0a51b0e4121c 100644
--- a/clang/test/CodeGenCXX/control-flow-in-stmt-expr.cpp
+++ b/clang/test/CodeGenCXX/control-flow-in-stmt-expr.cpp
@@ -391,3 +391,19 @@ void ArrayInitWithContinue() {
                      })};
   }
 }
+
+struct [[clang::trivial_abi]] HasTrivialABI {
+  HasTrivialABI();
+  ~HasTrivialABI();
+};
+void AcceptTrivialABI(HasTrivialABI, int);
+void TrivialABI() {
+  // CHECK-LABEL: define dso_local void @_Z10TrivialABIv()
+  AcceptTrivialABI(HasTrivialABI(), ({
+                     if (foo()) return;
+                     // CHECK:      if.then:
+                     // CHECK-NEXT:   call void @_ZN13HasTrivialABID1Ev
+                     // CHECK-NEXT:   br label %return
+                     0;
+                   }));
+}
-- 
GitLab


From 9141e1c24f87e5735bc4178a018eba4bdf2750aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Fri, 12 Apr 2024 10:43:14 +0200
Subject: [PATCH 354/464] [clang][Interp] Gracefully handle bitcasts to
 non-primitive types

We were calling classfiyPrim() instead of classify().
---
 clang/lib/AST/Interp/ByteCodeExprGen.cpp | 2 +-
 clang/test/AST/Interp/vectors.cpp        | 8 +++++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
index 01ec31e4077f..5866228663dc 100644
--- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp
+++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
@@ -262,7 +262,7 @@ bool ByteCodeExprGen<Emitter>::VisitCastExpr(const CastExpr *CE) {
       return this->discard(SubExpr);
 
     std::optional<PrimType> FromT = classify(SubExpr->getType());
-    std::optional<PrimType> ToT = classifyPrim(CE->getType());
+    std::optional<PrimType> ToT = classify(CE->getType());
     if (!FromT || !ToT)
       return false;
 
diff --git a/clang/test/AST/Interp/vectors.cpp b/clang/test/AST/Interp/vectors.cpp
index 8afef3c897bf..fb5787a9eda9 100644
--- a/clang/test/AST/Interp/vectors.cpp
+++ b/clang/test/AST/Interp/vectors.cpp
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -fexperimental-new-constant-interpreter -verify=expected,both %s
 // RUN: %clang_cc1 -verify=ref,both %s
 
-// both-no-diagnostics
+// ref-no-diagnostics
 
 typedef int __attribute__((vector_size(16))) VI4;
 constexpr VI4 A = {1,2,3,4};
@@ -20,3 +20,9 @@ namespace Vector {
   }
   constexpr auto v2 = g(4);
 }
+
+/// FIXME: We need to support BitCasts between vector types.
+namespace {
+  typedef float __attribute__((vector_size(16))) VI42;
+  constexpr VI42 A2 = A; // expected-error {{must be initialized by a constant expression}}
+}
-- 
GitLab


From 4fc0a99b8f220b6b41648da491bcc81a067f1600 Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad@amd.com>
Date: Tue, 16 Apr 2024 10:14:15 +0100
Subject: [PATCH 355/464] [AMDGPU] Fix implicit operands of VOPD cndmask
 instructions (#87788)

---
 llvm/lib/Target/AMDGPU/GCNCreateVOPD.cpp  |  1 +
 llvm/test/CodeGen/AMDGPU/vopd-combine.mir | 32 +++++++++++------------
 2 files changed, 17 insertions(+), 16 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/GCNCreateVOPD.cpp b/llvm/lib/Target/AMDGPU/GCNCreateVOPD.cpp
index 05e10a95b157..1dda1b89b2d3 100644
--- a/llvm/lib/Target/AMDGPU/GCNCreateVOPD.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNCreateVOPD.cpp
@@ -101,6 +101,7 @@ public:
       }
     }
 
+    SII->fixImplicitOperands(*VOPDInst);
     for (auto CompIdx : VOPD::COMPONENTS)
       VOPDInst.copyImplicitOps(*MI[CompIdx]);
 
diff --git a/llvm/test/CodeGen/AMDGPU/vopd-combine.mir b/llvm/test/CodeGen/AMDGPU/vopd-combine.mir
index 63bef40c3474..b8ac50c3aeb5 100644
--- a/llvm/test/CodeGen/AMDGPU/vopd-combine.mir
+++ b/llvm/test/CodeGen/AMDGPU/vopd-combine.mir
@@ -160,7 +160,7 @@ body:             |
     ; PAIR-GFX11-NEXT: $vgpr3 = IMPLICIT_DEF
     ; PAIR-GFX11-NEXT: $sgpr20 = IMPLICIT_DEF
     ; PAIR-GFX11-NEXT: $vgpr4 = V_FMAMK_F32 $sgpr20, 12345, $vgpr3, implicit $mode, implicit $exec
-    ; PAIR-GFX11-NEXT: $vgpr2, $vgpr5 = V_DUAL_FMAC_F32_e32_X_CNDMASK_B32_e32_gfx11 $sgpr20, killed $vgpr1, killed $vgpr2, $vgpr0, $vgpr3, implicit $mode, implicit $exec, implicit $vcc, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $vcc_lo
+    ; PAIR-GFX11-NEXT: $vgpr2, $vgpr5 = V_DUAL_FMAC_F32_e32_X_CNDMASK_B32_e32_gfx11 $sgpr20, killed $vgpr1, killed $vgpr2, $vgpr0, $vgpr3, implicit $mode, implicit $exec, implicit $vcc_lo, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $vcc_lo
     ; PAIR-GFX11-NEXT: $vgpr7 = V_CNDMASK_B32_e32 killed $vgpr0, $vgpr3, implicit $mode, implicit $exec, implicit $vcc_lo
     ; PAIR-GFX11-NEXT: $vgpr6 = V_ADD_F32_e32 $sgpr20, $vgpr3, implicit $mode, implicit $exec
     ; PAIR-GFX11-NEXT: $vgpr9 = V_CNDMASK_B32_e32 killed $sgpr20, killed $vgpr3, implicit $mode, implicit $exec, implicit killed $vcc_lo
@@ -174,7 +174,7 @@ body:             |
     ; PAIR-GFX12-NEXT: $vgpr3 = IMPLICIT_DEF
     ; PAIR-GFX12-NEXT: $sgpr20 = IMPLICIT_DEF
     ; PAIR-GFX12-NEXT: $vgpr4 = V_FMAMK_F32 $sgpr20, 12345, $vgpr3, implicit $mode, implicit $exec
-    ; PAIR-GFX12-NEXT: $vgpr2, $vgpr5 = V_DUAL_FMAC_F32_e32_X_CNDMASK_B32_e32_gfx12 $sgpr20, killed $vgpr1, killed $vgpr2, $vgpr0, $vgpr3, implicit $mode, implicit $exec, implicit $vcc, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $vcc_lo
+    ; PAIR-GFX12-NEXT: $vgpr2, $vgpr5 = V_DUAL_FMAC_F32_e32_X_CNDMASK_B32_e32_gfx12 $sgpr20, killed $vgpr1, killed $vgpr2, $vgpr0, $vgpr3, implicit $mode, implicit $exec, implicit $vcc_lo, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $vcc_lo
     ; PAIR-GFX12-NEXT: $vgpr7 = V_CNDMASK_B32_e32 killed $vgpr0, $vgpr3, implicit $mode, implicit $exec, implicit $vcc_lo
     ; PAIR-GFX12-NEXT: $vgpr6 = V_ADD_F32_e32 $sgpr20, $vgpr3, implicit $mode, implicit $exec
     ; PAIR-GFX12-NEXT: $vgpr9 = V_CNDMASK_B32_e32 killed $sgpr20, killed $vgpr3, implicit $mode, implicit $exec, implicit killed $vcc_lo
@@ -458,9 +458,9 @@ body:             |
     ; PAIR-GFX11-NEXT: $vgpr3, $vgpr6 = V_DUAL_SUB_F32_e32_X_MUL_F32_e32_gfx11 $vgpr1, $vgpr1, $vgpr0, $vgpr0, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
     ; PAIR-GFX11-NEXT: $vgpr2 = V_FMAC_F32_e32 10, $vgpr1, killed $vgpr2, implicit $mode, implicit $exec
     ; PAIR-GFX11-NEXT: $vgpr2 = V_ADD_F32_e32 $vgpr1, $vgpr1, implicit $mode, implicit $exec
-    ; PAIR-GFX11-NEXT: $vgpr12, $vgpr19 = V_DUAL_ADD_F32_e32_X_CNDMASK_B32_e32_gfx11 $vgpr1, $vgpr1, $vgpr0, $vgpr3, implicit $mode, implicit $exec, implicit $vcc, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $vcc_lo
+    ; PAIR-GFX11-NEXT: $vgpr12, $vgpr19 = V_DUAL_ADD_F32_e32_X_CNDMASK_B32_e32_gfx11 $vgpr1, $vgpr1, $vgpr0, $vgpr3, implicit $mode, implicit $exec, implicit $vcc_lo, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $vcc_lo
     ; PAIR-GFX11-NEXT: $vgpr11 = V_CNDMASK_B32_e32 $vgpr0, killed $vgpr3, implicit $mode, implicit $exec, implicit $vcc_lo
-    ; PAIR-GFX11-NEXT: $vgpr17, $vgpr10 = V_DUAL_MUL_F32_e32_X_CNDMASK_B32_e32_gfx11 killed $vgpr0, $vgpr0, $vgpr1, $vgpr2, implicit $mode, implicit $exec, implicit $vcc, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $vcc_lo
+    ; PAIR-GFX11-NEXT: $vgpr17, $vgpr10 = V_DUAL_MUL_F32_e32_X_CNDMASK_B32_e32_gfx11 killed $vgpr0, $vgpr0, $vgpr1, $vgpr2, implicit $mode, implicit $exec, implicit $vcc_lo, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $vcc_lo
     ; PAIR-GFX11-NEXT: $vgpr15 = V_CNDMASK_B32_e32 $vgpr1, killed $vgpr2, implicit $mode, implicit $exec, implicit killed $vcc_lo
     ; PAIR-GFX11-NEXT: $vgpr16 = V_SUB_F32_e32 $vgpr1, $vgpr1, implicit $mode, implicit $exec
     ; PAIR-GFX11-NEXT: $vgpr14 = V_SUB_F32_e32 killed $vgpr1, $vgpr1, implicit $mode, implicit $exec
@@ -476,9 +476,9 @@ body:             |
     ; PAIR-GFX12-NEXT: $vgpr3, $vgpr6 = V_DUAL_SUB_F32_e32_X_MUL_F32_e32_gfx12 $vgpr1, $vgpr1, $vgpr0, $vgpr0, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
     ; PAIR-GFX12-NEXT: $vgpr2 = V_FMAC_F32_e32 10, $vgpr1, killed $vgpr2, implicit $mode, implicit $exec
     ; PAIR-GFX12-NEXT: $vgpr2 = V_ADD_F32_e32 $vgpr1, $vgpr1, implicit $mode, implicit $exec
-    ; PAIR-GFX12-NEXT: $vgpr12, $vgpr19 = V_DUAL_ADD_F32_e32_X_CNDMASK_B32_e32_gfx12 $vgpr1, $vgpr1, $vgpr0, $vgpr3, implicit $mode, implicit $exec, implicit $vcc, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $vcc_lo
+    ; PAIR-GFX12-NEXT: $vgpr12, $vgpr19 = V_DUAL_ADD_F32_e32_X_CNDMASK_B32_e32_gfx12 $vgpr1, $vgpr1, $vgpr0, $vgpr3, implicit $mode, implicit $exec, implicit $vcc_lo, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $vcc_lo
     ; PAIR-GFX12-NEXT: $vgpr11 = V_CNDMASK_B32_e32 $vgpr0, killed $vgpr3, implicit $mode, implicit $exec, implicit $vcc_lo
-    ; PAIR-GFX12-NEXT: $vgpr17, $vgpr10 = V_DUAL_MUL_F32_e32_X_CNDMASK_B32_e32_gfx12 killed $vgpr0, $vgpr0, $vgpr1, $vgpr2, implicit $mode, implicit $exec, implicit $vcc, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $vcc_lo
+    ; PAIR-GFX12-NEXT: $vgpr17, $vgpr10 = V_DUAL_MUL_F32_e32_X_CNDMASK_B32_e32_gfx12 killed $vgpr0, $vgpr0, $vgpr1, $vgpr2, implicit $mode, implicit $exec, implicit $vcc_lo, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $vcc_lo
     ; PAIR-GFX12-NEXT: $vgpr15 = V_CNDMASK_B32_e32 $vgpr1, killed $vgpr2, implicit $mode, implicit $exec, implicit killed $vcc_lo
     ; PAIR-GFX12-NEXT: $vgpr16 = V_SUB_F32_e32 $vgpr1, $vgpr1, implicit $mode, implicit $exec
     ; PAIR-GFX12-NEXT: $vgpr14 = V_SUB_F32_e32 killed $vgpr1, $vgpr1, implicit $mode, implicit $exec
@@ -559,12 +559,12 @@ body:             |
     ; PAIR-GFX11-NEXT: $vgpr3, $vgpr6 = V_DUAL_SUB_F32_e32_X_MUL_F32_e32_gfx11 $vgpr1, $vgpr1, $vgpr0, $vgpr0, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
     ; PAIR-GFX11-NEXT: $vgpr2 = V_FMAC_F32_e32 10, $vgpr1, killed $vgpr2, implicit $mode, implicit $exec
     ; PAIR-GFX11-NEXT: $vgpr2 = V_ADD_F32_e32 $vgpr1, $vgpr1, implicit $mode, implicit $exec
-    ; PAIR-GFX11-NEXT: $vgpr4, $vgpr29 = V_DUAL_SUB_F32_e32_X_CNDMASK_B32_e32_gfx11 $vgpr1, $vgpr1, $vgpr0, $vgpr3, implicit $mode, implicit $exec, implicit $vcc, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $vcc_lo
-    ; PAIR-GFX11-NEXT: $vgpr19, $vgpr20 = V_DUAL_CNDMASK_B32_e32_X_FMAC_F32_e32_gfx11 $vgpr0, $vgpr3, 10, $vgpr1, killed $vgpr20, implicit $vcc, implicit $exec, implicit $mode, implicit $mode, implicit $exec, implicit $vcc_lo, implicit $mode, implicit $exec
+    ; PAIR-GFX11-NEXT: $vgpr4, $vgpr29 = V_DUAL_SUB_F32_e32_X_CNDMASK_B32_e32_gfx11 $vgpr1, $vgpr1, $vgpr0, $vgpr3, implicit $mode, implicit $exec, implicit $vcc_lo, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $vcc_lo
+    ; PAIR-GFX11-NEXT: $vgpr19, $vgpr20 = V_DUAL_CNDMASK_B32_e32_X_FMAC_F32_e32_gfx11 $vgpr0, $vgpr3, 10, $vgpr1, killed $vgpr20, implicit $vcc_lo, implicit $exec, implicit $mode, implicit $mode, implicit $exec, implicit $vcc_lo, implicit $mode, implicit $exec
     ; PAIR-GFX11-NEXT: $vgpr15 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $mode, implicit $exec, implicit $vcc_lo
-    ; PAIR-GFX11-NEXT: $vgpr10, $vgpr17 = V_DUAL_CNDMASK_B32_e32_X_MUL_F32_e32_gfx11 $vgpr1, $vgpr2, $vgpr0, $vgpr0, implicit $vcc, implicit $exec, implicit $mode, implicit $mode, implicit $exec, implicit $vcc_lo, implicit $mode, implicit $exec
-    ; PAIR-GFX11-NEXT: $vgpr11, $vgpr12 = V_DUAL_CNDMASK_B32_e32_X_ADD_F32_e32_gfx11 $vgpr0, $vgpr3, $vgpr1, $vgpr1, implicit $vcc, implicit $exec, implicit $mode, implicit $mode, implicit $exec, implicit $vcc_lo, implicit $mode, implicit $exec
-    ; PAIR-GFX11-NEXT: $vgpr37, $vgpr14 = V_DUAL_CNDMASK_B32_e32_X_SUB_F32_e32_gfx11 $vgpr0, killed $vgpr3, $vgpr1, $vgpr1, implicit $vcc, implicit $exec, implicit $mode, implicit $mode, implicit $exec, implicit $vcc_lo, implicit $mode, implicit $exec
+    ; PAIR-GFX11-NEXT: $vgpr10, $vgpr17 = V_DUAL_CNDMASK_B32_e32_X_MUL_F32_e32_gfx11 $vgpr1, $vgpr2, $vgpr0, $vgpr0, implicit $vcc_lo, implicit $exec, implicit $mode, implicit $mode, implicit $exec, implicit $vcc_lo, implicit $mode, implicit $exec
+    ; PAIR-GFX11-NEXT: $vgpr11, $vgpr12 = V_DUAL_CNDMASK_B32_e32_X_ADD_F32_e32_gfx11 $vgpr0, $vgpr3, $vgpr1, $vgpr1, implicit $vcc_lo, implicit $exec, implicit $mode, implicit $mode, implicit $exec, implicit $vcc_lo, implicit $mode, implicit $exec
+    ; PAIR-GFX11-NEXT: $vgpr37, $vgpr14 = V_DUAL_CNDMASK_B32_e32_X_SUB_F32_e32_gfx11 $vgpr0, killed $vgpr3, $vgpr1, $vgpr1, implicit $vcc_lo, implicit $exec, implicit $mode, implicit $mode, implicit $exec, implicit $vcc_lo, implicit $mode, implicit $exec
     ; PAIR-GFX11-NEXT: $vgpr20 = V_ADD_F32_e32 $vgpr1, $vgpr1, implicit $mode, implicit $exec
     ; PAIR-GFX11-NEXT: $vgpr21, $vgpr24 = V_DUAL_SUB_F32_e32_X_MUL_F32_e32_gfx11 $vgpr1, $vgpr1, killed $vgpr0, $vgpr0, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
     ; PAIR-GFX11-NEXT: $vgpr28 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $mode, implicit $exec, implicit $vcc_lo
@@ -586,12 +586,12 @@ body:             |
     ; PAIR-GFX12-NEXT: $vgpr3, $vgpr6 = V_DUAL_SUB_F32_e32_X_MUL_F32_e32_gfx12 $vgpr1, $vgpr1, $vgpr0, $vgpr0, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
     ; PAIR-GFX12-NEXT: $vgpr2 = V_FMAC_F32_e32 10, $vgpr1, killed $vgpr2, implicit $mode, implicit $exec
     ; PAIR-GFX12-NEXT: $vgpr2 = V_ADD_F32_e32 $vgpr1, $vgpr1, implicit $mode, implicit $exec
-    ; PAIR-GFX12-NEXT: $vgpr4, $vgpr29 = V_DUAL_SUB_F32_e32_X_CNDMASK_B32_e32_gfx12 $vgpr1, $vgpr1, $vgpr0, $vgpr3, implicit $mode, implicit $exec, implicit $vcc, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $vcc_lo
-    ; PAIR-GFX12-NEXT: $vgpr19, $vgpr20 = V_DUAL_CNDMASK_B32_e32_X_FMAC_F32_e32_gfx12 $vgpr0, $vgpr3, 10, $vgpr1, killed $vgpr20, implicit $vcc, implicit $exec, implicit $mode, implicit $mode, implicit $exec, implicit $vcc_lo, implicit $mode, implicit $exec
+    ; PAIR-GFX12-NEXT: $vgpr4, $vgpr29 = V_DUAL_SUB_F32_e32_X_CNDMASK_B32_e32_gfx12 $vgpr1, $vgpr1, $vgpr0, $vgpr3, implicit $mode, implicit $exec, implicit $vcc_lo, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $vcc_lo
+    ; PAIR-GFX12-NEXT: $vgpr19, $vgpr20 = V_DUAL_CNDMASK_B32_e32_X_FMAC_F32_e32_gfx12 $vgpr0, $vgpr3, 10, $vgpr1, killed $vgpr20, implicit $vcc_lo, implicit $exec, implicit $mode, implicit $mode, implicit $exec, implicit $vcc_lo, implicit $mode, implicit $exec
     ; PAIR-GFX12-NEXT: $vgpr15 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $mode, implicit $exec, implicit $vcc_lo
-    ; PAIR-GFX12-NEXT: $vgpr10, $vgpr17 = V_DUAL_CNDMASK_B32_e32_X_MUL_F32_e32_gfx12 $vgpr1, $vgpr2, $vgpr0, $vgpr0, implicit $vcc, implicit $exec, implicit $mode, implicit $mode, implicit $exec, implicit $vcc_lo, implicit $mode, implicit $exec
-    ; PAIR-GFX12-NEXT: $vgpr11, $vgpr12 = V_DUAL_CNDMASK_B32_e32_X_ADD_F32_e32_gfx12 $vgpr0, $vgpr3, $vgpr1, $vgpr1, implicit $vcc, implicit $exec, implicit $mode, implicit $mode, implicit $exec, implicit $vcc_lo, implicit $mode, implicit $exec
-    ; PAIR-GFX12-NEXT: $vgpr37, $vgpr14 = V_DUAL_CNDMASK_B32_e32_X_SUB_F32_e32_gfx12 $vgpr0, killed $vgpr3, $vgpr1, $vgpr1, implicit $vcc, implicit $exec, implicit $mode, implicit $mode, implicit $exec, implicit $vcc_lo, implicit $mode, implicit $exec
+    ; PAIR-GFX12-NEXT: $vgpr10, $vgpr17 = V_DUAL_CNDMASK_B32_e32_X_MUL_F32_e32_gfx12 $vgpr1, $vgpr2, $vgpr0, $vgpr0, implicit $vcc_lo, implicit $exec, implicit $mode, implicit $mode, implicit $exec, implicit $vcc_lo, implicit $mode, implicit $exec
+    ; PAIR-GFX12-NEXT: $vgpr11, $vgpr12 = V_DUAL_CNDMASK_B32_e32_X_ADD_F32_e32_gfx12 $vgpr0, $vgpr3, $vgpr1, $vgpr1, implicit $vcc_lo, implicit $exec, implicit $mode, implicit $mode, implicit $exec, implicit $vcc_lo, implicit $mode, implicit $exec
+    ; PAIR-GFX12-NEXT: $vgpr37, $vgpr14 = V_DUAL_CNDMASK_B32_e32_X_SUB_F32_e32_gfx12 $vgpr0, killed $vgpr3, $vgpr1, $vgpr1, implicit $vcc_lo, implicit $exec, implicit $mode, implicit $mode, implicit $exec, implicit $vcc_lo, implicit $mode, implicit $exec
     ; PAIR-GFX12-NEXT: $vgpr20 = V_ADD_F32_e32 $vgpr1, $vgpr1, implicit $mode, implicit $exec
     ; PAIR-GFX12-NEXT: $vgpr21, $vgpr24 = V_DUAL_SUB_F32_e32_X_MUL_F32_e32_gfx12 $vgpr1, $vgpr1, killed $vgpr0, $vgpr0, implicit $mode, implicit $exec, implicit $mode, implicit $exec, implicit $mode, implicit $exec
     ; PAIR-GFX12-NEXT: $vgpr28 = V_CNDMASK_B32_e32 $vgpr1, $vgpr2, implicit $mode, implicit $exec, implicit $vcc_lo
-- 
GitLab


From f4f772ceef379bd434d266b6e0d2bbdf796f81cb Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad@amd.com>
Date: Tue, 16 Apr 2024 10:15:00 +0100
Subject: [PATCH 356/464] [AMDGPU] Stop reserving $vcc_hi in wave32 mode
 (#87783)

This gives us one extra SGPR to play with. The comment suggested that it
could cause bugs, but I have tested it with Vulkan CTS with the default
wave size for compute shaders set to 32 and did not find any problems.
---
 llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp     |  7 --
 llvm/test/CodeGen/AMDGPU/bf16.ll              | 22 +++--
 .../CodeGen/AMDGPU/llvm.is.fpclass.f16.ll     | 20 ++---
 llvm/test/CodeGen/AMDGPU/load-constant-i16.ll | 22 ++---
 llvm/test/CodeGen/AMDGPU/load-constant-i8.ll  | 90 +++++++++----------
 .../AMDGPU/sgpr-spill-overlap-wwm-reserve.mir | 26 +++---
 6 files changed, 88 insertions(+), 99 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index 245731ad5fc7..acb54fd10b90 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -612,13 +612,6 @@ BitVector SIRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
   // Reserve null register - it shall never be allocated
   reserveRegisterTuples(Reserved, AMDGPU::SGPR_NULL64);
 
-  // Disallow vcc_hi allocation in wave32. It may be allocated but most likely
-  // will result in bugs.
-  if (isWave32) {
-    Reserved.set(AMDGPU::VCC);
-    Reserved.set(AMDGPU::VCC_HI);
-  }
-
   // Reserve SGPRs.
   //
   unsigned MaxNumSGPRs = ST.getMaxNumSGPRs(MF);
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index bf4302c156d8..4c9c34de7194 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -38342,12 +38342,11 @@ define <32 x bfloat> @v_vselect_v32bf16(<32 x i1> %cond, <32 x bfloat> %a, <32 x
 ; GFX10-NEXT:    v_and_b32_e32 v2, 1, v2
 ; GFX10-NEXT:    v_and_b32_e32 v4, 1, v4
 ; GFX10-NEXT:    v_and_b32_e32 v6, 1, v6
-; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    v_and_b32_e32 v8, 1, v8
 ; GFX10-NEXT:    v_and_b32_e32 v10, 1, v10
+; GFX10-NEXT:    v_writelane_b32 v40, s31, 1
 ; GFX10-NEXT:    v_and_b32_e32 v1, 1, v1
 ; GFX10-NEXT:    v_and_b32_e32 v3, 1, v3
-; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
 ; GFX10-NEXT:    v_and_b32_e32 v5, 1, v5
 ; GFX10-NEXT:    v_and_b32_e32 v7, 1, v7
 ; GFX10-NEXT:    v_and_b32_e32 v9, 1, v9
@@ -38366,7 +38365,7 @@ define <32 x bfloat> @v_vselect_v32bf16(<32 x i1> %cond, <32 x bfloat> %a, <32 x
 ; GFX10-NEXT:    v_cmp_eq_u32_e64 s17, 1, v4
 ; GFX10-NEXT:    v_cmp_eq_u32_e64 s18, 1, v2
 ; GFX10-NEXT:    v_cmp_eq_u32_e64 s19, 1, v0
-; GFX10-NEXT:    v_writelane_b32 v40, s35, 3
+; GFX10-NEXT:    v_writelane_b32 v40, s34, 2
 ; GFX10-NEXT:    v_cmp_eq_u32_e64 s20, 1, v27
 ; GFX10-NEXT:    v_cmp_eq_u32_e64 s21, 1, v25
 ; GFX10-NEXT:    v_cmp_eq_u32_e64 s22, 1, v23
@@ -38377,10 +38376,10 @@ define <32 x bfloat> @v_vselect_v32bf16(<32 x i1> %cond, <32 x bfloat> %a, <32 x
 ; GFX10-NEXT:    v_cmp_eq_u32_e64 s27, 1, v13
 ; GFX10-NEXT:    v_cmp_eq_u32_e64 s28, 1, v11
 ; GFX10-NEXT:    v_cmp_eq_u32_e64 s29, 1, v7
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s30, 1, v3
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s31, 1, v1
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s34, 1, v5
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s35, 1, v9
+; GFX10-NEXT:    v_cmp_eq_u32_e64 vcc_hi, 1, v3
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s30, 1, v1
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s31, 1, v5
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s34, 1, v9
 ; GFX10-NEXT:    s_waitcnt vmcnt(32)
 ; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 16, v31
 ; GFX10-NEXT:    s_waitcnt vmcnt(31)
@@ -38460,10 +38459,10 @@ define <32 x bfloat> @v_vselect_v32bf16(<32 x i1> %cond, <32 x bfloat> %a, <32 x
 ; GFX10-NEXT:    v_cndmask_b32_e64 v6, v29, v39, s27
 ; GFX10-NEXT:    v_cndmask_b32_e64 v5, v28, v26, s28
 ; GFX10-NEXT:    v_cndmask_b32_e64 v20, v51, v20, s29
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v14, v12, s31
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v55, v16, s30
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v53, v18, s34
-; GFX10-NEXT:    v_cndmask_b32_e64 v12, v24, v22, s35
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v14, v12, s30
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v55, v16, vcc_hi
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v53, v18, s31
+; GFX10-NEXT:    v_cndmask_b32_e64 v12, v24, v22, s34
 ; GFX10-NEXT:    v_cndmask_b32_e64 v16, v4, v3, s4
 ; GFX10-NEXT:    v_perm_b32 v0, v0, v64, 0x5040100
 ; GFX10-NEXT:    v_perm_b32 v1, v1, v54, 0x5040100
@@ -38481,7 +38480,6 @@ define <32 x bfloat> @v_vselect_v32bf16(<32 x i1> %cond, <32 x bfloat> %a, <32 x
 ; GFX10-NEXT:    v_perm_b32 v13, v66, v13, 0x5040100
 ; GFX10-NEXT:    v_perm_b32 v14, v65, v17, 0x5040100
 ; GFX10-NEXT:    v_perm_b32 v15, v16, v15, 0x5040100
-; GFX10-NEXT:    v_readlane_b32 s35, v40, 3
 ; GFX10-NEXT:    v_readlane_b32 s34, v40, 2
 ; GFX10-NEXT:    v_readlane_b32 s31, v40, 1
 ; GFX10-NEXT:    v_readlane_b32 s30, v40, 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll
index ec3c08ec7952..da64c379672e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll
@@ -1259,17 +1259,17 @@ define <4 x i1> @isnan_v4f16(<4 x half> %x) nounwind {
 ; GFX10SELDAG-LABEL: isnan_v4f16:
 ; GFX10SELDAG:       ; %bb.0:
 ; GFX10SELDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10SELDAG-NEXT:    v_mov_b32_e32 v2, 3
-; GFX10SELDAG-NEXT:    v_cmp_class_f16_e64 s5, v0, 3
-; GFX10SELDAG-NEXT:    v_cmp_class_f16_sdwa s4, v1, v2 src0_sel:WORD_1 src1_sel:DWORD
-; GFX10SELDAG-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s5
-; GFX10SELDAG-NEXT:    v_cmp_class_f16_sdwa s5, v0, v2 src0_sel:WORD_1 src1_sel:DWORD
+; GFX10SELDAG-NEXT:    v_cmp_class_f16_e64 s4, v0, 3
+; GFX10SELDAG-NEXT:    v_mov_b32_e32 v3, 3
+; GFX10SELDAG-NEXT:    v_cndmask_b32_e64 v5, 0, 1, s4
+; GFX10SELDAG-NEXT:    v_cmp_class_f16_e64 s4, v1, 3
+; GFX10SELDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s4
+; GFX10SELDAG-NEXT:    v_cmp_class_f16_sdwa s4, v0, v3 src0_sel:WORD_1 src1_sel:DWORD
+; GFX10SELDAG-NEXT:    v_mov_b32_e32 v0, v5
+; GFX10SELDAG-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s4
+; GFX10SELDAG-NEXT:    v_cmp_class_f16_sdwa s4, v1, v3 src0_sel:WORD_1 src1_sel:DWORD
+; GFX10SELDAG-NEXT:    v_mov_b32_e32 v1, v4
 ; GFX10SELDAG-NEXT:    v_cndmask_b32_e64 v3, 0, 1, s4
-; GFX10SELDAG-NEXT:    v_mov_b32_e32 v0, v4
-; GFX10SELDAG-NEXT:    v_cndmask_b32_e64 v5, 0, 1, s5
-; GFX10SELDAG-NEXT:    v_cmp_class_f16_e64 s5, v1, 3
-; GFX10SELDAG-NEXT:    v_mov_b32_e32 v1, v5
-; GFX10SELDAG-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s5
 ; GFX10SELDAG-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10GLISEL-LABEL: isnan_v4f16:
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
index ab6a9dcf71ac..a87fa8bf36d9 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
@@ -7404,35 +7404,35 @@ define amdgpu_kernel void @constant_sextload_v16i16_to_v16i64(ptr addrspace(1) %
 ; GFX12-NEXT:    v_dual_mov_b32 v4, s22 :: v_dual_mov_b32 v9, s31
 ; GFX12-NEXT:    v_dual_mov_b32 v8, s30 :: v_dual_mov_b32 v11, s35
 ; GFX12-NEXT:    v_dual_mov_b32 v10, s34 :: v_dual_mov_b32 v3, s5
-; GFX12-NEXT:    s_bfe_i64 s[10:11], s[0:1], 0x100000
-; GFX12-NEXT:    s_lshr_b32 s12, s0, 16
-; GFX12-NEXT:    s_mov_b32 s14, s1
-; GFX12-NEXT:    s_lshr_b32 s16, s1, 16
-; GFX12-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x100000
+; GFX12-NEXT:    s_bfe_i64 s[16:17], s[2:3], 0x100000
 ; GFX12-NEXT:    s_lshr_b32 s2, s2, 16
 ; GFX12-NEXT:    s_bfe_i64 s[6:7], s[6:7], 0x100000
 ; GFX12-NEXT:    v_dual_mov_b32 v0, s28 :: v_dual_mov_b32 v5, s23
 ; GFX12-NEXT:    v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v13, s25
+; GFX12-NEXT:    s_mov_b32 s12, s1
+; GFX12-NEXT:    s_lshr_b32 s14, s1, 16
 ; GFX12-NEXT:    s_bfe_i64 s[18:19], s[18:19], 0x100000
 ; GFX12-NEXT:    s_bfe_i64 s[20:21], s[20:21], 0x100000
 ; GFX12-NEXT:    v_dual_mov_b32 v12, s24 :: v_dual_mov_b32 v15, s27
 ; GFX12-NEXT:    v_dual_mov_b32 v14, s26 :: v_dual_mov_b32 v7, s7
+; GFX12-NEXT:    s_bfe_i64 s[10:11], s[0:1], 0x100000
+; GFX12-NEXT:    s_lshr_b32 s0, s0, 16
 ; GFX12-NEXT:    s_bfe_i64 s[2:3], s[2:3], 0x100000
 ; GFX12-NEXT:    v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v17, s19
+; GFX12-NEXT:    s_bfe_i64 s[12:13], s[12:13], 0x100000
 ; GFX12-NEXT:    s_bfe_i64 s[14:15], s[14:15], 0x100000
-; GFX12-NEXT:    s_bfe_i64 s[16:17], s[16:17], 0x100000
 ; GFX12-NEXT:    v_dual_mov_b32 v16, s18 :: v_dual_mov_b32 v19, s21
 ; GFX12-NEXT:    v_mov_b32_e32 v18, s20
-; GFX12-NEXT:    s_bfe_i64 s[12:13], s[12:13], 0x100000
+; GFX12-NEXT:    s_bfe_i64 s[0:1], s[0:1], 0x100000
 ; GFX12-NEXT:    s_clause 0x1
 ; GFX12-NEXT:    global_store_b128 v24, v[8:11], s[8:9] offset:80
 ; GFX12-NEXT:    global_store_b128 v24, v[0:3], s[8:9] offset:64
-; GFX12-NEXT:    v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v0, s0
+; GFX12-NEXT:    v_dual_mov_b32 v1, s17 :: v_dual_mov_b32 v0, s16
 ; GFX12-NEXT:    v_dual_mov_b32 v3, s3 :: v_dual_mov_b32 v2, s2
-; GFX12-NEXT:    v_dual_mov_b32 v9, s15 :: v_dual_mov_b32 v8, s14
-; GFX12-NEXT:    v_dual_mov_b32 v11, s17 :: v_dual_mov_b32 v10, s16
+; GFX12-NEXT:    v_dual_mov_b32 v9, s13 :: v_dual_mov_b32 v8, s12
+; GFX12-NEXT:    v_dual_mov_b32 v11, s15 :: v_dual_mov_b32 v10, s14
 ; GFX12-NEXT:    v_dual_mov_b32 v21, s11 :: v_dual_mov_b32 v20, s10
-; GFX12-NEXT:    v_dual_mov_b32 v23, s13 :: v_dual_mov_b32 v22, s12
+; GFX12-NEXT:    v_dual_mov_b32 v23, s1 :: v_dual_mov_b32 v22, s0
 ; GFX12-NEXT:    s_clause 0x5
 ; GFX12-NEXT:    global_store_b128 v24, v[12:15], s[8:9] offset:112
 ; GFX12-NEXT:    global_store_b128 v24, v[4:7], s[8:9] offset:96
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll
index 952827b8cd0e..889755c23bbc 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll
@@ -8808,73 +8808,73 @@ define amdgpu_kernel void @constant_sextload_v32i8_to_v32i64(ptr addrspace(1) %o
 ; GFX12-NEXT:    v_lshrrev_b16 v2, 8, s6
 ; GFX12-NEXT:    v_lshrrev_b16 v4, 8, s5
 ; GFX12-NEXT:    v_lshrrev_b16 v8, 8, s2
-; GFX12-NEXT:    s_lshr_b32 s24, s7, 16
+; GFX12-NEXT:    s_lshr_b32 s22, s7, 16
 ; GFX12-NEXT:    v_bfe_i32 v31, v1, 0, 8
-; GFX12-NEXT:    s_lshr_b32 s42, s2, 24
-; GFX12-NEXT:    s_mov_b32 s48, s7
+; GFX12-NEXT:    s_lshr_b32 s40, s2, 24
+; GFX12-NEXT:    s_mov_b32 s46, s7
 ; GFX12-NEXT:    v_lshrrev_b16 v5, 8, s4
 ; GFX12-NEXT:    v_lshrrev_b16 v7, 8, s1
-; GFX12-NEXT:    s_lshr_b32 s26, s6, 16
-; GFX12-NEXT:    s_lshr_b32 s44, s1, 16
+; GFX12-NEXT:    s_lshr_b32 s24, s6, 16
+; GFX12-NEXT:    s_lshr_b32 s42, s1, 16
 ; GFX12-NEXT:    s_ashr_i64 s[58:59], s[6:7], 56
-; GFX12-NEXT:    s_bfe_i64 s[48:49], s[48:49], 0x80000
-; GFX12-NEXT:    s_bfe_i64 s[42:43], s[42:43], 0x80000
-; GFX12-NEXT:    s_bfe_i64 s[24:25], s[24:25], 0x80000
+; GFX12-NEXT:    s_bfe_i64 s[46:47], s[46:47], 0x80000
+; GFX12-NEXT:    s_bfe_i64 s[40:41], s[40:41], 0x80000
+; GFX12-NEXT:    s_bfe_i64 s[22:23], s[22:23], 0x80000
 ; GFX12-NEXT:    v_lshrrev_b16 v6, 8, s3
 ; GFX12-NEXT:    v_lshrrev_b16 v3, 8, s0
-; GFX12-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v33, s24
-; GFX12-NEXT:    s_lshr_b32 s28, s6, 24
-; GFX12-NEXT:    s_lshr_b32 s30, s5, 16
-; GFX12-NEXT:    s_lshr_b32 s40, s2, 16
+; GFX12-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v33, s22
+; GFX12-NEXT:    s_lshr_b32 s26, s6, 24
+; GFX12-NEXT:    s_lshr_b32 s28, s5, 16
+; GFX12-NEXT:    s_lshr_b32 s38, s2, 16
 ; GFX12-NEXT:    v_bfe_i32 v11, v8, 0, 8
 ; GFX12-NEXT:    v_bfe_i32 v23, v4, 0, 8
 ; GFX12-NEXT:    v_bfe_i32 v27, v2, 0, 8
 ; GFX12-NEXT:    v_ashrrev_i32_e32 v32, 31, v31
-; GFX12-NEXT:    s_bfe_i64 s[44:45], s[44:45], 0x80000
-; GFX12-NEXT:    s_bfe_i64 s[26:27], s[26:27], 0x80000
-; GFX12-NEXT:    v_dual_mov_b32 v34, s25 :: v_dual_mov_b32 v35, s58
-; GFX12-NEXT:    v_dual_mov_b32 v36, s59 :: v_dual_mov_b32 v37, s26
-; GFX12-NEXT:    v_dual_mov_b32 v56, s43 :: v_dual_mov_b32 v29, s48
-; GFX12-NEXT:    v_mov_b32_e32 v30, s49
-; GFX12-NEXT:    s_lshr_b32 s46, s0, 24
-; GFX12-NEXT:    s_mov_b32 s50, s5
-; GFX12-NEXT:    s_mov_b32 s52, s3
-; GFX12-NEXT:    s_lshr_b32 s34, s4, 16
-; GFX12-NEXT:    s_lshr_b32 s36, s4, 24
-; GFX12-NEXT:    s_ashr_i64 s[22:23], s[2:3], 56
+; GFX12-NEXT:    s_bfe_i64 s[42:43], s[42:43], 0x80000
+; GFX12-NEXT:    s_bfe_i64 s[24:25], s[24:25], 0x80000
+; GFX12-NEXT:    v_dual_mov_b32 v34, s23 :: v_dual_mov_b32 v35, s58
+; GFX12-NEXT:    v_dual_mov_b32 v36, s59 :: v_dual_mov_b32 v37, s24
+; GFX12-NEXT:    v_dual_mov_b32 v56, s41 :: v_dual_mov_b32 v29, s46
+; GFX12-NEXT:    v_mov_b32_e32 v30, s47
+; GFX12-NEXT:    s_lshr_b32 s44, s0, 24
+; GFX12-NEXT:    s_mov_b32 s48, s5
+; GFX12-NEXT:    s_mov_b32 s50, s3
+; GFX12-NEXT:    s_lshr_b32 s30, s4, 16
+; GFX12-NEXT:    s_lshr_b32 s34, s4, 24
+; GFX12-NEXT:    s_ashr_i64 s[54:55], s[2:3], 56
 ; GFX12-NEXT:    s_ashr_i64 s[56:57], s[4:5], 56
 ; GFX12-NEXT:    v_bfe_i32 v7, v7, 0, 8
 ; GFX12-NEXT:    v_bfe_i32 v19, v5, 0, 8
-; GFX12-NEXT:    s_bfe_i64 s[40:41], s[40:41], 0x80000
-; GFX12-NEXT:    s_bfe_i64 s[30:31], s[30:31], 0x80000
+; GFX12-NEXT:    s_bfe_i64 s[38:39], s[38:39], 0x80000
 ; GFX12-NEXT:    s_bfe_i64 s[28:29], s[28:29], 0x80000
-; GFX12-NEXT:    s_lshr_b32 s38, s3, 16
-; GFX12-NEXT:    s_mov_b32 s54, s1
+; GFX12-NEXT:    s_bfe_i64 s[26:27], s[26:27], 0x80000
+; GFX12-NEXT:    s_lshr_b32 s36, s3, 16
+; GFX12-NEXT:    s_mov_b32 s52, s1
 ; GFX12-NEXT:    s_bfe_i64 s[12:13], s[2:3], 0x80000
 ; GFX12-NEXT:    s_bfe_i64 s[14:15], s[4:5], 0x80000
 ; GFX12-NEXT:    s_bfe_i64 s[16:17], s[6:7], 0x80000
-; GFX12-NEXT:    s_bfe_i64 s[2:3], s[52:53], 0x80000
-; GFX12-NEXT:    s_bfe_i64 s[4:5], s[50:51], 0x80000
-; GFX12-NEXT:    s_bfe_i64 s[6:7], s[46:47], 0x80000
+; GFX12-NEXT:    s_bfe_i64 s[2:3], s[50:51], 0x80000
+; GFX12-NEXT:    s_bfe_i64 s[4:5], s[48:49], 0x80000
+; GFX12-NEXT:    s_bfe_i64 s[6:7], s[44:45], 0x80000
 ; GFX12-NEXT:    s_lshr_b32 s20, s0, 16
 ; GFX12-NEXT:    s_ashr_i64 s[18:19], s[0:1], 56
 ; GFX12-NEXT:    v_bfe_i32 v3, v3, 0, 8
 ; GFX12-NEXT:    v_bfe_i32 v15, v6, 0, 8
-; GFX12-NEXT:    s_bfe_i64 s[36:37], s[36:37], 0x80000
 ; GFX12-NEXT:    s_bfe_i64 s[34:35], s[34:35], 0x80000
-; GFX12-NEXT:    v_dual_mov_b32 v38, s27 :: v_dual_mov_b32 v39, s28
-; GFX12-NEXT:    v_dual_mov_b32 v40, s29 :: v_dual_mov_b32 v41, s30
-; GFX12-NEXT:    v_dual_mov_b32 v42, s31 :: v_dual_mov_b32 v43, s56
-; GFX12-NEXT:    v_dual_mov_b32 v44, s57 :: v_dual_mov_b32 v45, s34
-; GFX12-NEXT:    v_dual_mov_b32 v52, s23 :: v_dual_mov_b32 v53, s40
-; GFX12-NEXT:    v_dual_mov_b32 v54, s41 :: v_dual_mov_b32 v55, s42
+; GFX12-NEXT:    s_bfe_i64 s[30:31], s[30:31], 0x80000
+; GFX12-NEXT:    v_dual_mov_b32 v38, s25 :: v_dual_mov_b32 v39, s26
+; GFX12-NEXT:    v_dual_mov_b32 v40, s27 :: v_dual_mov_b32 v41, s28
+; GFX12-NEXT:    v_dual_mov_b32 v42, s29 :: v_dual_mov_b32 v43, s56
+; GFX12-NEXT:    v_dual_mov_b32 v44, s57 :: v_dual_mov_b32 v45, s30
+; GFX12-NEXT:    v_dual_mov_b32 v52, s55 :: v_dual_mov_b32 v53, s38
+; GFX12-NEXT:    v_dual_mov_b32 v54, s39 :: v_dual_mov_b32 v55, s40
 ; GFX12-NEXT:    s_bfe_i64 s[10:11], s[0:1], 0x80000
-; GFX12-NEXT:    s_bfe_i64 s[0:1], s[54:55], 0x80000
+; GFX12-NEXT:    s_bfe_i64 s[0:1], s[52:53], 0x80000
 ; GFX12-NEXT:    v_ashrrev_i32_e32 v12, 31, v11
 ; GFX12-NEXT:    v_ashrrev_i32_e32 v24, 31, v23
 ; GFX12-NEXT:    v_ashrrev_i32_e32 v28, 31, v27
 ; GFX12-NEXT:    global_store_b128 v0, v[33:36], s[8:9] offset:240
-; GFX12-NEXT:    v_mov_b32_e32 v33, s44
+; GFX12-NEXT:    v_mov_b32_e32 v33, s42
 ; GFX12-NEXT:    global_store_b128 v0, v[29:32], s[8:9] offset:224
 ; GFX12-NEXT:    v_dual_mov_b32 v25, s16 :: v_dual_mov_b32 v26, s17
 ; GFX12-NEXT:    v_dual_mov_b32 v32, s7 :: v_dual_mov_b32 v21, s4
@@ -8882,16 +8882,16 @@ define amdgpu_kernel void @constant_sextload_v32i8_to_v32i64(ptr addrspace(1) %o
 ; GFX12-NEXT:    v_dual_mov_b32 v14, s3 :: v_dual_mov_b32 v9, s12
 ; GFX12-NEXT:    v_dual_mov_b32 v10, s13 :: v_dual_mov_b32 v5, s0
 ; GFX12-NEXT:    s_bfe_i64 s[20:21], s[20:21], 0x80000
-; GFX12-NEXT:    s_bfe_i64 s[38:39], s[38:39], 0x80000
-; GFX12-NEXT:    v_dual_mov_b32 v46, s35 :: v_dual_mov_b32 v47, s36
-; GFX12-NEXT:    v_dual_mov_b32 v48, s37 :: v_dual_mov_b32 v49, s38
-; GFX12-NEXT:    v_dual_mov_b32 v34, s45 :: v_dual_mov_b32 v35, s18
+; GFX12-NEXT:    s_bfe_i64 s[36:37], s[36:37], 0x80000
+; GFX12-NEXT:    v_dual_mov_b32 v46, s31 :: v_dual_mov_b32 v47, s34
+; GFX12-NEXT:    v_dual_mov_b32 v48, s35 :: v_dual_mov_b32 v49, s36
+; GFX12-NEXT:    v_dual_mov_b32 v34, s43 :: v_dual_mov_b32 v35, s18
 ; GFX12-NEXT:    v_dual_mov_b32 v36, s19 :: v_dual_mov_b32 v29, s20
 ; GFX12-NEXT:    v_ashrrev_i32_e32 v8, 31, v7
 ; GFX12-NEXT:    v_ashrrev_i32_e32 v20, 31, v19
 ; GFX12-NEXT:    v_dual_mov_b32 v18, s15 :: v_dual_mov_b32 v13, s2
 ; GFX12-NEXT:    v_dual_mov_b32 v6, s1 :: v_dual_mov_b32 v1, s10
-; GFX12-NEXT:    v_dual_mov_b32 v50, s39 :: v_dual_mov_b32 v51, s22
+; GFX12-NEXT:    v_dual_mov_b32 v50, s37 :: v_dual_mov_b32 v51, s54
 ; GFX12-NEXT:    v_dual_mov_b32 v30, s21 :: v_dual_mov_b32 v31, s6
 ; GFX12-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
 ; GFX12-NEXT:    v_ashrrev_i32_e32 v16, 31, v15
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-spill-overlap-wwm-reserve.mir b/llvm/test/CodeGen/AMDGPU/sgpr-spill-overlap-wwm-reserve.mir
index f8e7cb397b47..8a5f75332557 100644
--- a/llvm/test/CodeGen/AMDGPU/sgpr-spill-overlap-wwm-reserve.mir
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-spill-overlap-wwm-reserve.mir
@@ -28,18 +28,17 @@ body:             |
   ; GCN-LABEL: name: test_main
   ; GCN: bb.0:
   ; GCN-NEXT:   successors: %bb.1(0x80000000)
-  ; GCN-NEXT:   liveins: $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr18, $sgpr19, $sgpr20, $sgpr21, $sgpr22, $sgpr23, $sgpr24, $sgpr25, $sgpr26, $sgpr27, $sgpr28, $sgpr29, $sgpr30, $sgpr31, $sgpr64, $sgpr65, $sgpr66, $sgpr67, $sgpr68, $sgpr69, $sgpr70, $sgpr71, $sgpr72, $sgpr73, $sgpr74, $sgpr75, $sgpr76, $sgpr77, $sgpr78, $sgpr79, $sgpr80, $sgpr81, $sgpr82, $sgpr83, $sgpr84, $sgpr85, $sgpr86, $sgpr87, $sgpr88, $sgpr89, $sgpr90, $sgpr91, $sgpr92, $sgpr93, $sgpr94, $sgpr95, $sgpr96, $sgpr97, $sgpr98, $sgpr99, $sgpr100, $sgpr101, $sgpr102, $sgpr103, $vgpr0, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GCN-NEXT:   liveins: $vcc_hi, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $sgpr14, $sgpr15, $sgpr16, $sgpr17, $sgpr18, $sgpr19, $sgpr20, $sgpr21, $sgpr22, $sgpr23, $sgpr24, $sgpr25, $sgpr26, $sgpr27, $sgpr28, $sgpr29, $sgpr30, $sgpr31, $sgpr64, $sgpr65, $sgpr66, $sgpr67, $sgpr68, $sgpr69, $sgpr70, $sgpr71, $sgpr72, $sgpr73, $sgpr74, $sgpr75, $sgpr76, $sgpr77, $sgpr78, $sgpr79, $sgpr80, $sgpr81, $sgpr82, $sgpr83, $sgpr84, $sgpr85, $sgpr86, $sgpr87, $sgpr88, $sgpr89, $sgpr90, $sgpr91, $sgpr92, $sgpr93, $sgpr94, $sgpr95, $sgpr96, $sgpr97, $sgpr98, $sgpr99, $sgpr100, $sgpr101, $sgpr102, $sgpr103, $vgpr0, $vgpr2, $vgpr3, $vgpr4, $vgpr5
   ; GCN-NEXT: {{  $}}
-  ; GCN-NEXT:   $sgpr0 = COPY $sgpr33
+  ; GCN-NEXT:   $vcc_hi = frame-setup COPY $sgpr33
   ; GCN-NEXT:   $sgpr33 = frame-setup COPY $sgpr32
-  ; GCN-NEXT:   $sgpr1 = S_XOR_SAVEEXEC_B32 -1, implicit-def $exec, implicit-def dead $scc, implicit $exec
+  ; GCN-NEXT:   $sgpr0 = S_XOR_SAVEEXEC_B32 -1, implicit-def $exec, implicit-def dead $scc, implicit $exec
   ; GCN-NEXT:   SCRATCH_STORE_DWORD_SADDR $vgpr3, $sgpr33, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32) into %stack.69, addrspace 5)
   ; GCN-NEXT:   SCRATCH_STORE_DWORD_SADDR $vgpr4, $sgpr33, 4, 0, implicit $exec, implicit $flat_scr :: (store (s32) into %stack.70, addrspace 5)
   ; GCN-NEXT:   SCRATCH_STORE_DWORD_SADDR $vgpr5, $sgpr33, 8, 0, implicit $exec, implicit $flat_scr :: (store (s32) into %stack.71, addrspace 5)
   ; GCN-NEXT:   SCRATCH_STORE_DWORD_SADDR $vgpr2, $sgpr33, 12, 0, implicit $exec, implicit $flat_scr :: (store (s32) into %stack.72, addrspace 5)
-  ; GCN-NEXT:   SCRATCH_STORE_DWORD_SADDR killed $vgpr1, $sgpr33, 16, 0, implicit $exec, implicit $flat_scr :: (store (s32) into %stack.74, addrspace 5)
-  ; GCN-NEXT:   $exec_lo = S_MOV_B32 killed $sgpr1
-  ; GCN-NEXT:   $vgpr5 = SI_SPILL_S32_TO_VGPR $sgpr0, 4, undef $vgpr5
+  ; GCN-NEXT:   SCRATCH_STORE_DWORD_SADDR killed $vgpr1, $sgpr33, 16, 0, implicit $exec, implicit $flat_scr :: (store (s32) into %stack.73, addrspace 5)
+  ; GCN-NEXT:   $exec_lo = S_MOV_B32 killed $sgpr0
   ; GCN-NEXT:   $sgpr32 = frame-setup S_ADD_I32 $sgpr32, 24, implicit-def dead $scc
   ; GCN-NEXT:   renamable $vgpr2 = IMPLICIT_DEF
   ; GCN-NEXT:   $vgpr3 = SI_SPILL_S32_TO_VGPR $sgpr4, 0, $vgpr3
@@ -116,18 +115,18 @@ body:             |
   ; GCN-NEXT: {{  $}}
   ; GCN-NEXT: bb.1:
   ; GCN-NEXT:   successors: %bb.2(0x80000000)
-  ; GCN-NEXT:   liveins: $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GCN-NEXT:   liveins: $vcc_hi, $vgpr2, $vgpr3, $vgpr4, $vgpr5
   ; GCN-NEXT: {{  $}}
   ; GCN-NEXT:   KILL implicit-def $vcc_lo, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31, implicit-def $sgpr32_sgpr33_sgpr34_sgpr35_sgpr36_sgpr37_sgpr38_sgpr39_sgpr40_sgpr41_sgpr42_sgpr43_sgpr44_sgpr45_sgpr46_sgpr47_sgpr48_sgpr49_sgpr50_sgpr51_sgpr52_sgpr53_sgpr54_sgpr55_sgpr56_sgpr57_sgpr58_sgpr59_sgpr60_sgpr61_sgpr62_sgpr63, implicit-def $sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75_sgpr76_sgpr77_sgpr78_sgpr79_sgpr80_sgpr81_sgpr82_sgpr83_sgpr84_sgpr85_sgpr86_sgpr87_sgpr88_sgpr89_sgpr90_sgpr91_sgpr92_sgpr93_sgpr94_sgpr95, implicit-def $sgpr96_sgpr97_sgpr98_sgpr99_sgpr100_sgpr101_sgpr102_sgpr103
   ; GCN-NEXT: {{  $}}
   ; GCN-NEXT: bb.2:
   ; GCN-NEXT:   successors: %bb.3(0x80000000)
-  ; GCN-NEXT:   liveins: $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GCN-NEXT:   liveins: $vcc_hi, $vgpr2, $vgpr3, $vgpr4, $vgpr5
   ; GCN-NEXT: {{  $}}
   ; GCN-NEXT:   $sgpr22 = SI_RESTORE_S32_FROM_VGPR $vgpr2, 0
   ; GCN-NEXT: {{  $}}
   ; GCN-NEXT: bb.3:
-  ; GCN-NEXT:   liveins: $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GCN-NEXT:   liveins: $vcc_hi, $vgpr2, $vgpr3, $vgpr4, $vgpr5
   ; GCN-NEXT: {{  $}}
   ; GCN-NEXT:   $sgpr103 = SI_RESTORE_S32_FROM_VGPR $vgpr5, 3
   ; GCN-NEXT:   $sgpr102 = SI_RESTORE_S32_FROM_VGPR $vgpr5, 2
@@ -198,16 +197,15 @@ body:             |
   ; GCN-NEXT:   $sgpr5 = SI_RESTORE_S32_FROM_VGPR $vgpr3, 1
   ; GCN-NEXT:   $sgpr4 = SI_RESTORE_S32_FROM_VGPR $vgpr3, 0
   ; GCN-NEXT:   KILL killed renamable $vgpr2
-  ; GCN-NEXT:   $sgpr0 = SI_RESTORE_S32_FROM_VGPR $vgpr5, 4
-  ; GCN-NEXT:   $sgpr1 = S_XOR_SAVEEXEC_B32 -1, implicit-def $exec, implicit-def dead $scc, implicit $exec
+  ; GCN-NEXT:   $sgpr0 = S_XOR_SAVEEXEC_B32 -1, implicit-def $exec, implicit-def dead $scc, implicit $exec
   ; GCN-NEXT:   $vgpr3 = SCRATCH_LOAD_DWORD_SADDR $sgpr33, 0, 0, implicit $exec, implicit $flat_scr :: (load (s32) from %stack.69, addrspace 5)
   ; GCN-NEXT:   $vgpr4 = SCRATCH_LOAD_DWORD_SADDR $sgpr33, 4, 0, implicit $exec, implicit $flat_scr :: (load (s32) from %stack.70, addrspace 5)
   ; GCN-NEXT:   $vgpr5 = SCRATCH_LOAD_DWORD_SADDR $sgpr33, 8, 0, implicit $exec, implicit $flat_scr :: (load (s32) from %stack.71, addrspace 5)
   ; GCN-NEXT:   $vgpr2 = SCRATCH_LOAD_DWORD_SADDR $sgpr33, 12, 0, implicit $exec, implicit $flat_scr :: (load (s32) from %stack.72, addrspace 5)
-  ; GCN-NEXT:   $vgpr1 = SCRATCH_LOAD_DWORD_SADDR $sgpr33, 16, 0, implicit $exec, implicit $flat_scr :: (load (s32) from %stack.74, addrspace 5)
-  ; GCN-NEXT:   $exec_lo = S_MOV_B32 killed $sgpr1
+  ; GCN-NEXT:   $vgpr1 = SCRATCH_LOAD_DWORD_SADDR $sgpr33, 16, 0, implicit $exec, implicit $flat_scr :: (load (s32) from %stack.73, addrspace 5)
+  ; GCN-NEXT:   $exec_lo = S_MOV_B32 killed $sgpr0
   ; GCN-NEXT:   $sgpr32 = frame-destroy S_ADD_I32 $sgpr32, -24, implicit-def dead $scc
-  ; GCN-NEXT:   $sgpr33 = COPY $sgpr0
+  ; GCN-NEXT:   $sgpr33 = frame-destroy COPY $vcc_hi
   ; GCN-NEXT:   S_ENDPGM 0
   bb.0:
     liveins: $vgpr0
-- 
GitLab


From 485d556d8c23b54da952e75c3cadc9db3050fd9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Sat, 13 Apr 2024 06:10:38 +0200
Subject: [PATCH 357/464] [clang][Interp][NFC] Add Block::dump()

---
 clang/lib/AST/Interp/Disasm.cpp    | 16 ++++++++++++++++
 clang/lib/AST/Interp/InterpBlock.h |  3 +++
 2 files changed, 19 insertions(+)

diff --git a/clang/lib/AST/Interp/Disasm.cpp b/clang/lib/AST/Interp/Disasm.cpp
index 022b394e58e6..ebc4e4f195ba 100644
--- a/clang/lib/AST/Interp/Disasm.cpp
+++ b/clang/lib/AST/Interp/Disasm.cpp
@@ -264,3 +264,19 @@ LLVM_DUMP_METHOD void Record::dump(llvm::raw_ostream &OS, unsigned Indentation,
     ++I;
   }
 }
+
+LLVM_DUMP_METHOD void Block::dump(llvm::raw_ostream &OS) const {
+  {
+    ColorScope SC(OS, true, {llvm::raw_ostream::BRIGHT_BLUE, true});
+    OS << "Block " << (void *)this << "\n";
+  }
+  unsigned NPointers = 0;
+  for (const Pointer *P = Pointers; P; P = P->Next) {
+    ++NPointers;
+  }
+  OS << "  Pointers: " << NPointers << "\n";
+  OS << "  Dead: " << IsDead << "\n";
+  OS << "  Static: " << IsStatic << "\n";
+  OS << "  Extern: " << IsExtern << "\n";
+  OS << "  Initialized: " << IsInitialized << "\n";
+}
diff --git a/clang/lib/AST/Interp/InterpBlock.h b/clang/lib/AST/Interp/InterpBlock.h
index 9db82567d2d5..6d5856fbd4ea 100644
--- a/clang/lib/AST/Interp/InterpBlock.h
+++ b/clang/lib/AST/Interp/InterpBlock.h
@@ -118,6 +118,9 @@ public:
     IsInitialized = false;
   }
 
+  void dump() const { dump(llvm::errs()); }
+  void dump(llvm::raw_ostream &OS) const;
+
 protected:
   friend class Pointer;
   friend class DeadBlock;
-- 
GitLab


From 80fce05f2104d1c42db814276130536b014fcca2 Mon Sep 17 00:00:00 2001
From: Yingwei Zheng <dtcxzyw2333@gmail.com>
Date: Tue, 16 Apr 2024 17:16:35 +0800
Subject: [PATCH 358/464] [InstCombine] Fold `minmax (X & NegPow2C, Y &
 NegPow2C) -> minmax(X, Y) & NegPow2C` (#88859)

Alive2: https://alive2.llvm.org/ce/z/NFtkSX

This optimization will be beneficial to jemalloc users.
---
 .../InstCombine/InstCombineCalls.cpp          |  8 +-
 .../InstCombine/minmax-intrinsics.ll          | 89 +++++++++++++++++++
 2 files changed, 96 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index bae8579fc365..ba5db854647a 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -1774,6 +1774,13 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
     if (Instruction *I = moveAddAfterMinMax(II, Builder))
       return I;
 
+    // minmax (X & NegPow2C, Y & NegPow2C) --> minmax(X, Y) & NegPow2C
+    const APInt *RHSC;
+    if (match(I0, m_OneUse(m_And(m_Value(X), m_NegatedPower2(RHSC)))) &&
+        match(I1, m_OneUse(m_And(m_Value(Y), m_SpecificInt(*RHSC)))))
+      return BinaryOperator::CreateAnd(Builder.CreateBinaryIntrinsic(IID, X, Y),
+                                       ConstantInt::get(II->getType(), *RHSC));
+
     // smax(X, -X) --> abs(X)
     // smin(X, -X) --> -abs(X)
     // umax(X, -X) --> -abs(X)
@@ -1815,7 +1822,6 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
        return NewMinMax;
 
     // Try to fold minmax with constant RHS based on range information
-    const APInt *RHSC;
     if (match(I1, m_APIntAllowUndef(RHSC))) {
       ICmpInst::Predicate Pred =
           ICmpInst::getNonStrictPredicate(MinMaxIntrinsic::getPredicate(IID));
diff --git a/llvm/test/Transforms/InstCombine/minmax-intrinsics.ll b/llvm/test/Transforms/InstCombine/minmax-intrinsics.ll
index ae2e115b1dd9..bd1a47bbfcc1 100644
--- a/llvm/test/Transforms/InstCombine/minmax-intrinsics.ll
+++ b/llvm/test/Transforms/InstCombine/minmax-intrinsics.ll
@@ -2581,3 +2581,92 @@ entry:
   %val = call i8 @llvm.umin.i8(i8 %sub, i8 3)
   ret i8 %val
 }
+
+define i8 @test_umax_and(i8 %x, i8 %y) {
+; CHECK-LABEL: @test_umax_and(
+; CHECK-NEXT:    [[RES:%.*]] = call i8 @llvm.umax.i8(i8 [[X1:%.*]], i8 [[Y1:%.*]])
+; CHECK-NEXT:    [[RES1:%.*]] = and i8 [[RES]], -64
+; CHECK-NEXT:    ret i8 [[RES1]]
+;
+  %x1 = and i8 %x, -64
+  %y1 = and i8 %y, -64
+  %res = call i8 @llvm.umax.i8(i8 %x1, i8 %y1)
+  ret i8 %res
+}
+
+define i8 @test_umin_and(i8 %x, i8 %y) {
+; CHECK-LABEL: @test_umin_and(
+; CHECK-NEXT:    [[RES:%.*]] = call i8 @llvm.umin.i8(i8 [[X1:%.*]], i8 [[Y1:%.*]])
+; CHECK-NEXT:    [[RES1:%.*]] = and i8 [[RES]], -64
+; CHECK-NEXT:    ret i8 [[RES1]]
+;
+  %x1 = and i8 %x, -64
+  %y1 = and i8 %y, -64
+  %res = call i8 @llvm.umin.i8(i8 %x1, i8 %y1)
+  ret i8 %res
+}
+
+define i8 @test_smax_and(i8 %x, i8 %y) {
+; CHECK-LABEL: @test_smax_and(
+; CHECK-NEXT:    [[RES:%.*]] = call i8 @llvm.smax.i8(i8 [[X1:%.*]], i8 [[Y1:%.*]])
+; CHECK-NEXT:    [[RES1:%.*]] = and i8 [[RES]], -64
+; CHECK-NEXT:    ret i8 [[RES1]]
+;
+  %x1 = and i8 %x, -64
+  %y1 = and i8 %y, -64
+  %res = call i8 @llvm.smax.i8(i8 %x1, i8 %y1)
+  ret i8 %res
+}
+
+define i8 @test_smin_and(i8 %x, i8 %y) {
+; CHECK-LABEL: @test_smin_and(
+; CHECK-NEXT:    [[RES:%.*]] = call i8 @llvm.smin.i8(i8 [[X1:%.*]], i8 [[Y1:%.*]])
+; CHECK-NEXT:    [[RES1:%.*]] = and i8 [[RES]], -64
+; CHECK-NEXT:    ret i8 [[RES1]]
+;
+  %x1 = and i8 %x, -64
+  %y1 = and i8 %y, -64
+  %res = call i8 @llvm.smin.i8(i8 %x1, i8 %y1)
+  ret i8 %res
+}
+
+define i8 @test_smin_and_mismatch(i8 %x, i8 %y) {
+; CHECK-LABEL: @test_smin_and_mismatch(
+; CHECK-NEXT:    [[X1:%.*]] = and i8 [[X:%.*]], -64
+; CHECK-NEXT:    [[Y1:%.*]] = and i8 [[Y:%.*]], -32
+; CHECK-NEXT:    [[RES:%.*]] = call i8 @llvm.smin.i8(i8 [[X1]], i8 [[Y1]])
+; CHECK-NEXT:    ret i8 [[RES]]
+;
+  %x1 = and i8 %x, -64
+  %y1 = and i8 %y, -32
+  %res = call i8 @llvm.smin.i8(i8 %x1, i8 %y1)
+  ret i8 %res
+}
+
+define i8 @test_smin_and_non_negated_pow2(i8 %x, i8 %y) {
+; CHECK-LABEL: @test_smin_and_non_negated_pow2(
+; CHECK-NEXT:    [[X1:%.*]] = and i8 [[X:%.*]], 31
+; CHECK-NEXT:    [[Y1:%.*]] = and i8 [[Y:%.*]], 31
+; CHECK-NEXT:    [[RES:%.*]] = call i8 @llvm.smin.i8(i8 [[X1]], i8 [[Y1]])
+; CHECK-NEXT:    ret i8 [[RES]]
+;
+  %x1 = and i8 %x, 31
+  %y1 = and i8 %y, 31
+  %res = call i8 @llvm.smin.i8(i8 %x1, i8 %y1)
+  ret i8 %res
+}
+
+define i8 @test_smin_and_multiuse(i8 %x, i8 %y) {
+; CHECK-LABEL: @test_smin_and_multiuse(
+; CHECK-NEXT:    [[X1:%.*]] = and i8 [[X:%.*]], 31
+; CHECK-NEXT:    [[Y1:%.*]] = and i8 [[Y:%.*]], 31
+; CHECK-NEXT:    call void @use(i8 [[Y1]])
+; CHECK-NEXT:    [[RES:%.*]] = call i8 @llvm.smin.i8(i8 [[X1]], i8 [[Y1]])
+; CHECK-NEXT:    ret i8 [[RES]]
+;
+  %x1 = and i8 %x, 31
+  %y1 = and i8 %y, 31
+  call void @use(i8 %y1)
+  %res = call i8 @llvm.smin.i8(i8 %x1, i8 %y1)
+  ret i8 %res
+}
-- 
GitLab


From cce026bf8f7dcf5aa402a6da20f0d4da56aee8b5 Mon Sep 17 00:00:00 2001
From: Jie Fu <jiefu@tencent.com>
Date: Tue, 16 Apr 2024 17:19:45 +0800
Subject: [PATCH 359/464] [mlir][test] Fix -Wsign-compare in TestDialect.cpp
 (NFC)

llvm-project/mlir/test/lib/Dialect/Test/TestDialect.cpp:597:31:
error: comparison of integers of different signs: 'size_t' (aka 'unsigned long') and 'int64_t' (aka 'long') [-Werror,-Wsign-compare]
  if (getVarOperands().size() != expectedNumOperands)
      ~~~~~~~~~~~~~~~~~~~~~~~ ^  ~~~~~~~~~~~~~~~~~~~
1 error generated.
---
 mlir/test/lib/Dialect/Test/TestDialect.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlir/test/lib/Dialect/Test/TestDialect.cpp b/mlir/test/lib/Dialect/Test/TestDialect.cpp
index 36d7606fe134..a23ed89c4b04 100644
--- a/mlir/test/lib/Dialect/Test/TestDialect.cpp
+++ b/mlir/test/lib/Dialect/Test/TestDialect.cpp
@@ -594,7 +594,7 @@ LogicalResult CompareOp::verify() {
         "'compose' not supported when 'lhs_map' or 'rhs_map' is present");
   int64_t expectedNumOperands = getLhsMap() ? getLhsMap()->getNumInputs() : 1;
   expectedNumOperands += getRhsMap() ? getRhsMap()->getNumInputs() : 1;
-  if (getVarOperands().size() != expectedNumOperands)
+  if (getVarOperands().size() != size_t(expectedNumOperands))
     return emitOpError("expected ")
            << expectedNumOperands << " operands, but got "
            << getVarOperands().size();
-- 
GitLab


From 58b49cef1d772a922a433fd4a42e41db3f18d34b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Sat, 13 Apr 2024 15:06:22 +0200
Subject: [PATCH 360/464] [clang][Interp] Support __builtin_vectorelements

---
 clang/lib/AST/Interp/ByteCodeExprGen.cpp | 9 +++++++++
 clang/test/AST/Interp/vectors.cpp        | 2 ++
 2 files changed, 11 insertions(+)

diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
index 5866228663dc..93059edc4622 100644
--- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp
+++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
@@ -1251,6 +1251,15 @@ bool ByteCodeExprGen<Emitter>::VisitUnaryExprOrTypeTraitExpr(
     return this->emitConst(Size.getQuantity(), E);
   }
 
+  if (Kind == UETT_VectorElements) {
+    if (const auto *VT = E->getTypeOfArgument()->getAs<VectorType>())
+      return this->emitConst(VT->getNumElements(), E);
+
+    // FIXME: Apparently we need to catch the fact that a sizeless vector type
+    // has been passed and diagnose that (at run time).
+    assert(E->getTypeOfArgument()->isSizelessVectorType());
+  }
+
   return false;
 }
 
diff --git a/clang/test/AST/Interp/vectors.cpp b/clang/test/AST/Interp/vectors.cpp
index fb5787a9eda9..6c5d916f51f5 100644
--- a/clang/test/AST/Interp/vectors.cpp
+++ b/clang/test/AST/Interp/vectors.cpp
@@ -13,12 +13,14 @@ namespace Vector {
     return VI4 { n * 3, n + 4, n - 5, n / 6 };
   }
   constexpr auto v1 = f(10);
+  static_assert(__builtin_vectorelements(v1) == (16 / sizeof(int)), "");
 
   typedef double __attribute__((vector_size(32))) VD4;
   constexpr VD4 g(int n) {
     return (VD4) { n / 2.0, n + 1.5, n - 5.4, n * 0.9 };
   }
   constexpr auto v2 = g(4);
+  static_assert(__builtin_vectorelements(v2) == (32 / sizeof(double)), "");
 }
 
 /// FIXME: We need to support BitCasts between vector types.
-- 
GitLab


From 70fe6ad535365de20c3a960dcfe97bd5cf68abb5 Mon Sep 17 00:00:00 2001
From: Sergio Afonso <safonsof@amd.com>
Date: Tue, 16 Apr 2024 10:24:23 +0100
Subject: [PATCH 361/464] [MLIR][OpenMP] Make omp.distribute into a loop
 wrapper (#87239)

This patch updates the definition of `omp.distribute` to enforce the
restrictions of a wrapper operation.
---
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td | 22 +++++++++--
 mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp  | 11 ++++++
 mlir/test/Dialect/OpenMP/invalid.mlir         | 34 ++++++++++++++++-
 mlir/test/Dialect/OpenMP/ops.mlir             | 38 +++++++++++++++----
 4 files changed, 93 insertions(+), 12 deletions(-)

diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index 3abdbe3adfd0..7e677c9839f6 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -840,7 +840,8 @@ def YieldOp : OpenMP_Op<"yield",
 //===----------------------------------------------------------------------===//
 def DistributeOp : OpenMP_Op<"distribute", [AttrSizedOperandSegments,
                              DeclareOpInterfaceMethods<LoopWrapperInterface>,
-                             RecursiveMemoryEffects]> {
+                             RecursiveMemoryEffects,
+                             SingleBlockImplicitTerminator<"TerminatorOp">]> {
   let summary = "distribute construct";
   let description = [{
     The distribute construct specifies that the iterations of one or more loops
@@ -855,15 +856,28 @@ def DistributeOp : OpenMP_Op<"distribute", [AttrSizedOperandSegments,
     The distribute loop construct specifies that the iterations of the loop(s)
     will be executed in parallel by threads in the current context. These
     iterations are spread across threads that already exist in the enclosing
-    region. The lower and upper bounds specify a half-open range: the
-    range includes the lower bound but does not include the upper bound. If the
-    `inclusive` attribute is specified then the upper bound is also included.
+    region.
+    
+    The body region can contain a single block which must contain a single
+    operation and a terminator. The operation must be another compatible loop
+    wrapper or an `omp.loop_nest`.
 
     The `dist_schedule_static` attribute specifies the  schedule for this
     loop, determining how the loop is distributed across the parallel threads.
     The optional `schedule_chunk` associated with this determines further
     controls this distribution.
 
+    ```mlir
+    omp.distribute <clauses> {
+      omp.loop_nest (%i1, %i2) : index = (%c0, %c0) to (%c10, %c10) step (%c1, %c1) {
+        %a = load %arrA[%i1, %i2] : memref<?x?xf32>
+        %b = load %arrB[%i1, %i2] : memref<?x?xf32>
+        %sum = arith.addf %a, %b : f32
+        store %sum, %arrC[%i1, %i2] : memref<?x?xf32>
+        omp.yield
+      }
+    }
+    ```
     // TODO: private_var, firstprivate_var, lastprivate_var, collapse
   }];
   let arguments = (ins
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index 90b49b2528b7..365b3c4246bf 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -1656,6 +1656,17 @@ LogicalResult DistributeOp::verify() {
     return emitError(
         "expected equal sizes for allocate and allocator variables");
 
+  if (!isWrapper())
+    return emitOpError() << "must be a loop wrapper";
+
+  if (LoopWrapperInterface nested = getNestedWrapper()) {
+    // Check for the allowed leaf constructs that may appear in a composite
+    // construct directly after DISTRIBUTE.
+    if (!isa<ParallelOp, SimdLoopOp>(nested))
+      return emitError() << "only supported nested wrappers are 'omp.parallel' "
+                            "and 'omp.simdloop'";
+  }
+
   return success();
 }
 
diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir
index 88dca1b85ee5..f8739887214b 100644
--- a/mlir/test/Dialect/OpenMP/invalid.mlir
+++ b/mlir/test/Dialect/OpenMP/invalid.mlir
@@ -1866,7 +1866,16 @@ func.func @omp_target_depend(%data_var: memref<i32>) {
 
 // -----
 
-func.func @omp_distribute(%data_var : memref<i32>) -> () {
+func.func @omp_distribute_schedule(%chunk_size : i32) -> () {
+  // expected-error @below {{op chunk size set without dist_schedule_static being present}}
+  "omp.distribute"(%chunk_size) <{operandSegmentSizes = array<i32: 1, 0, 0>}> ({
+      "omp.terminator"() : () -> ()
+    }) : (i32) -> ()
+}
+
+// -----
+
+func.func @omp_distribute_allocate(%data_var : memref<i32>) -> () {
   // expected-error @below {{expected equal sizes for allocate and allocator variables}}
   "omp.distribute"(%data_var) <{operandSegmentSizes = array<i32: 0, 1, 0>}> ({
       "omp.terminator"() : () -> ()
@@ -1875,6 +1884,29 @@ func.func @omp_distribute(%data_var : memref<i32>) -> () {
 
 // -----
 
+func.func @omp_distribute_wrapper() -> () {
+  // expected-error @below {{op must be a loop wrapper}}
+  "omp.distribute"() ({
+      %0 = arith.constant 0 : i32
+      "omp.terminator"() : () -> ()
+    }) : () -> ()
+}
+
+// -----
+
+func.func @omp_distribute_nested_wrapper(%data_var : memref<i32>) -> () {
+  // expected-error @below {{only supported nested wrappers are 'omp.parallel' and 'omp.simdloop'}}
+  "omp.distribute"() ({
+      "omp.wsloop"() ({
+        %0 = arith.constant 0 : i32
+        "omp.terminator"() : () -> ()
+      }) : () -> ()
+      "omp.terminator"() : () -> ()
+    }) : () -> ()
+}
+
+// -----
+
 omp.private {type = private} @x.privatizer : i32 alloc {
 ^bb0(%arg0: i32):
   %0 = arith.constant 0.0 : f32
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 851d44ad984e..018b82e995d7 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -559,30 +559,54 @@ func.func @omp_simdloop_pretty_multiple(%lb1 : index, %ub1 : index, %step1 : ind
 }
 
 // CHECK-LABEL: omp_distribute
-func.func @omp_distribute(%chunk_size : i32, %data_var : memref<i32>) -> () {
+func.func @omp_distribute(%chunk_size : i32, %data_var : memref<i32>, %arg0 : i32) -> () {
   // CHECK: omp.distribute
   "omp.distribute" () ({
-    omp.terminator
+    "omp.loop_nest" (%arg0, %arg0, %arg0) ({
+    ^bb0(%iv: i32):
+      "omp.yield"() : () -> ()
+    }) : (i32, i32, i32) -> ()
+    "omp.terminator"() : () -> ()
   }) {} : () -> ()
   // CHECK: omp.distribute
   omp.distribute {
-    omp.terminator
+    omp.loop_nest (%iv) : i32 = (%arg0) to (%arg0) step (%arg0) {
+      omp.yield
+    }
   }
   // CHECK: omp.distribute dist_schedule_static
   omp.distribute dist_schedule_static {
-    omp.terminator
+    omp.loop_nest (%iv) : i32 = (%arg0) to (%arg0) step (%arg0) {
+      omp.yield
+    }
   }
   // CHECK: omp.distribute dist_schedule_static chunk_size(%{{.+}} : i32)
   omp.distribute dist_schedule_static chunk_size(%chunk_size : i32) {
-    omp.terminator
+    omp.loop_nest (%iv) : i32 = (%arg0) to (%arg0) step (%arg0) {
+      omp.yield
+    }
   }
   // CHECK: omp.distribute order(concurrent)
   omp.distribute order(concurrent) {
-    omp.terminator
+    omp.loop_nest (%iv) : i32 = (%arg0) to (%arg0) step (%arg0) {
+      omp.yield
+    }
   }
   // CHECK: omp.distribute allocate(%{{.+}} : memref<i32> -> %{{.+}} : memref<i32>)
   omp.distribute allocate(%data_var : memref<i32> -> %data_var : memref<i32>) {
-    omp.terminator
+    omp.loop_nest (%iv) : i32 = (%arg0) to (%arg0) step (%arg0) {
+      omp.yield
+    }
+  }
+  // CHECK: omp.distribute
+  omp.distribute {
+    // TODO Remove induction variables from omp.simdloop.
+    omp.simdloop for (%iv) : i32 = (%arg0) to (%arg0) step (%arg0) {
+      omp.loop_nest (%iv2) : i32 = (%arg0) to (%arg0) step (%arg0) {
+        omp.yield
+      }
+      omp.yield
+    }
   }
 return
 }
-- 
GitLab


From 668a58b8926473d731c41c55007f1fe4571ada86 Mon Sep 17 00:00:00 2001
From: Tom Eccles <tom.eccles@arm.com>
Date: Tue, 16 Apr 2024 10:25:26 +0100
Subject: [PATCH 362/464] [flang][runtime] Add ACCESS library procedure
 (#88517)

Re-land https://github.com/llvm/llvm-project/pull/88395

Two build-bots were broken by the old version:
 - https://lab.llvm.org/buildbot/#/builders/285/builds/245
 - https://lab.llvm.org/buildbot/#/builders/21/builds/96988

The problem in both cases was that the compiler did not support
`std::filesystem` (which I use in the unit test).

I have removed the dependency upon std::filesystem because there isn't
an easy way to add the right linker options so that this is supported
correctly in all build environments [1]

[1] https://gitlab.kitware.com/cmake/cmake/-/issues/17834

---

This is a GNU extension:
https://gcc.gnu.org/onlinedocs/gfortran/ACCESS.html

Used in SALMON:
https://salmon-tddft.jp/download.html

Unfortunately the intrinsic takes a file path to operate on so there
isn't an easy way to make the test robust. The unit test expects to be
able to create, set read write and execute permissions, and delete files
called
std::filesystem::temp_directory_path() / <test_name>.<pid>

The test will fail if a file already exists with that name.

I have not implemented the intrinsic on Windows because this is wrapping
a POSIX system call and Windows doesn't support all of the permission
bits tested by the intrinsic. I don't have a Windows machine easily
available to check if Gfortran implements this intrinsic on Windows.
---
 flang/docs/Intrinsics.md                 |   8 +
 flang/include/flang/Runtime/extensions.h |   7 +
 flang/runtime/extensions.cpp             |  73 ++++
 flang/unittests/Runtime/AccessTest.cpp   | 422 +++++++++++++++++++++++
 flang/unittests/Runtime/CMakeLists.txt   |   1 +
 5 files changed, 511 insertions(+)
 create mode 100644 flang/unittests/Runtime/AccessTest.cpp

diff --git a/flang/docs/Intrinsics.md b/flang/docs/Intrinsics.md
index ccb93e104dab..848619cb65d9 100644
--- a/flang/docs/Intrinsics.md
+++ b/flang/docs/Intrinsics.md
@@ -657,6 +657,14 @@ CALL CO_REDUCE
 CALL CO_SUM
 ```
 
+### Inquiry Functions
+ACCESS (GNU extension) is not supported on Windows. Otherwise:
+```
+CHARACTER(LEN=*) :: path = 'path/to/file'
+IF (ACCESS(path, 'rwx')) &
+  ...
+```
+
 ## Non-standard intrinsics
 ### PGI
 ```
diff --git a/flang/include/flang/Runtime/extensions.h b/flang/include/flang/Runtime/extensions.h
index 7d0952206fc1..fef651f3b2ee 100644
--- a/flang/include/flang/Runtime/extensions.h
+++ b/flang/include/flang/Runtime/extensions.h
@@ -44,5 +44,12 @@ std::int64_t RTNAME(Signal)(std::int64_t number, void (*handler)(int));
 // GNU extension subroutine SLEEP(SECONDS)
 void RTNAME(Sleep)(std::int64_t seconds);
 
+// GNU extension function ACCESS(NAME, MODE)
+// TODO: not supported on Windows
+#ifndef _WIN32
+std::int64_t FORTRAN_PROCEDURE_NAME(access)(const char *name,
+    std::int64_t nameLength, const char *mode, std::int64_t modeLength);
+#endif
+
 } // extern "C"
 #endif // FORTRAN_RUNTIME_EXTENSIONS_H_
diff --git a/flang/runtime/extensions.cpp b/flang/runtime/extensions.cpp
index 3ac98000335d..12498b502ae1 100644
--- a/flang/runtime/extensions.cpp
+++ b/flang/runtime/extensions.cpp
@@ -17,6 +17,7 @@
 #include "flang/Runtime/entry-names.h"
 #include "flang/Runtime/io-api.h"
 #include <chrono>
+#include <cstring>
 #include <ctime>
 #include <signal.h>
 #include <thread>
@@ -138,5 +139,77 @@ void RTNAME(Sleep)(std::int64_t seconds) {
   std::this_thread::sleep_for(std::chrono::seconds(seconds));
 }
 
+// TODO: not supported on Windows
+#ifndef _WIN32
+std::int64_t FORTRAN_PROCEDURE_NAME(access)(const char *name,
+    std::int64_t nameLength, const char *mode, std::int64_t modeLength) {
+  std::int64_t ret{-1};
+  if (nameLength <= 0 || modeLength <= 0 || !name || !mode) {
+    return ret;
+  }
+
+  // ensure name is null terminated
+  char *newName{nullptr};
+  if (name[nameLength - 1] != '\0') {
+    newName = static_cast<char *>(std::malloc(nameLength + 1));
+    std::memcpy(newName, name, nameLength);
+    newName[nameLength] = '\0';
+    name = newName;
+  }
+
+  // calculate mode
+  bool read{false};
+  bool write{false};
+  bool execute{false};
+  bool exists{false};
+  int imode{0};
+
+  for (std::int64_t i = 0; i < modeLength; ++i) {
+    switch (mode[i]) {
+    case 'r':
+      read = true;
+      break;
+    case 'w':
+      write = true;
+      break;
+    case 'x':
+      execute = true;
+      break;
+    case ' ':
+      exists = true;
+      break;
+    default:
+      // invalid mode
+      goto cleanup;
+    }
+  }
+  if (!read && !write && !execute && !exists) {
+    // invalid mode
+    goto cleanup;
+  }
+
+  if (!read && !write && !execute) {
+    imode = F_OK;
+  } else {
+    if (read) {
+      imode |= R_OK;
+    }
+    if (write) {
+      imode |= W_OK;
+    }
+    if (execute) {
+      imode |= X_OK;
+    }
+  }
+  ret = access(name, imode);
+
+cleanup:
+  if (newName) {
+    free(newName);
+  }
+  return ret;
+}
+#endif
+
 } // namespace Fortran::runtime
 } // extern "C"
diff --git a/flang/unittests/Runtime/AccessTest.cpp b/flang/unittests/Runtime/AccessTest.cpp
new file mode 100644
index 000000000000..66f19f78c7cf
--- /dev/null
+++ b/flang/unittests/Runtime/AccessTest.cpp
@@ -0,0 +1,422 @@
+//===-- flang/unittests/Runtime/AccessTest.cpp ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// TODO: ACCESS is not yet implemented on Windows
+#ifndef _WIN32
+
+#include "CrashHandlerFixture.h"
+#include "gtest/gtest.h"
+#include "flang/Runtime/extensions.h"
+#include "llvm/ADT/Twine.h"
+
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+namespace {
+
+struct AccessTests : public CrashHandlerFixture {};
+
+struct AccessType {
+  bool read{false};
+  bool write{false};
+  bool execute{false};
+  bool exists{false};
+};
+
+} // namespace
+
+static std::string addPIDSuffix(const char *name) {
+  std::stringstream ss;
+  ss << name;
+  ss << '.';
+
+  ss << getpid();
+
+  return ss.str();
+}
+
+static bool exists(const std::string &path) {
+  return access(path.c_str(), F_OK) == 0;
+}
+
+// Implementation of std::filesystem::temp_directory_path adapted from libcxx
+// See llvm-project/libcxx/src/filesystem/operations.cpp
+// Using std::filesystem is inconvenient because the required flags are not
+// consistent accross compilers and CMake doesn't have built in support to
+// determine the correct flags.
+static const char *temp_directory_path() {
+  // TODO: Windows
+  const char *env_paths[] = {"TMPDIR", "TMP", "TEMP", "TEMPDIR"};
+  const char *ret = nullptr;
+
+  for (auto &ep : env_paths) {
+    if ((ret = getenv(ep))) {
+      break;
+    }
+  }
+
+  if (ret == nullptr) {
+#if defined(__ANDROID__)
+    ret = "/data/local/tmp";
+#else
+    ret = "/tmp";
+#endif
+  }
+
+  assert(exists(ret));
+  return ret;
+}
+
+static std::string createTemporaryFile(
+    const char *name, const AccessType &accessType) {
+  std::string path =
+      (llvm::Twine{temp_directory_path()} + "/" + addPIDSuffix(name)).str();
+
+  // O_CREAT | O_EXCL enforces that this file is newly created by this call.
+  // This feels risky. If we don't have permission to create files in the
+  // temporary directory or if the files already exist, the test will fail.
+  // But we can't use std::tmpfile() because we need a path to the file and
+  // to control the filesystem permissions
+  mode_t mode{0};
+  if (accessType.read) {
+    mode |= S_IRUSR;
+  }
+  if (accessType.write) {
+    mode |= S_IWUSR;
+  }
+  if (accessType.execute) {
+    mode |= S_IXUSR;
+  }
+
+  int file = open(path.c_str(), O_CREAT | O_EXCL, mode);
+  if (file == -1) {
+    return {};
+  }
+
+  close(file);
+
+  return path;
+}
+
+static std::int64_t callAccess(
+    const std::string &path, const AccessType &accessType) {
+  const char *cpath{path.c_str()};
+  std::int64_t pathlen = std::strlen(cpath);
+
+  std::string mode;
+  if (accessType.read) {
+    mode += 'r';
+  }
+  if (accessType.write) {
+    mode += 'w';
+  }
+  if (accessType.execute) {
+    mode += 'x';
+  }
+  if (accessType.exists) {
+    mode += ' ';
+  }
+
+  const char *cmode = mode.c_str();
+  std::int64_t modelen = std::strlen(cmode);
+
+  return FORTRAN_PROCEDURE_NAME(access)(cpath, pathlen, cmode, modelen);
+}
+
+TEST(AccessTests, TestExists) {
+  AccessType accessType;
+  accessType.exists = true;
+
+  std::string path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  std::int64_t res = callAccess(path, accessType);
+
+  ASSERT_EQ(unlink(path.c_str()), 0);
+
+  ASSERT_EQ(res, 0);
+}
+
+TEST(AccessTests, TestNotExists) {
+  std::string nonExistant{addPIDSuffix(__func__)};
+  ASSERT_FALSE(exists(nonExistant));
+
+  AccessType accessType;
+  accessType.exists = true;
+  std::int64_t res = callAccess(nonExistant, accessType);
+
+  ASSERT_NE(res, 0);
+}
+
+TEST(AccessTests, TestRead) {
+  AccessType accessType;
+  accessType.read = true;
+
+  std::string path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  std::int64_t res = callAccess(path, accessType);
+
+  ASSERT_EQ(unlink(path.c_str()), 0);
+
+  ASSERT_EQ(res, 0);
+}
+
+TEST(AccessTests, TestNotRead) {
+  AccessType accessType;
+  accessType.read = false;
+
+  std::string path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  accessType.read = true;
+  std::int64_t res = callAccess(path, accessType);
+
+  ASSERT_EQ(unlink(path.c_str()), 0);
+
+  ASSERT_NE(res, 0);
+}
+
+TEST(AccessTests, TestWrite) {
+  AccessType accessType;
+  accessType.write = true;
+
+  std::string path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  std::int64_t res = callAccess(path, accessType);
+
+  ASSERT_EQ(unlink(path.c_str()), 0);
+
+  ASSERT_EQ(res, 0);
+}
+
+TEST(AccessTests, TestNotWrite) {
+  AccessType accessType;
+  accessType.write = false;
+
+  std::string path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  accessType.write = true;
+  std::int64_t res = callAccess(path, accessType);
+
+  ASSERT_EQ(unlink(path.c_str()), 0);
+
+  ASSERT_NE(res, 0);
+}
+
+TEST(AccessTests, TestReadWrite) {
+  AccessType accessType;
+  accessType.read = true;
+  accessType.write = true;
+
+  std::string path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  std::int64_t res = callAccess(path, accessType);
+
+  ASSERT_EQ(unlink(path.c_str()), 0);
+
+  ASSERT_EQ(res, 0);
+}
+
+TEST(AccessTests, TestNotReadWrite0) {
+  AccessType accessType;
+  accessType.read = false;
+  accessType.write = false;
+
+  std::string path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  accessType.read = true;
+  accessType.write = true;
+  std::int64_t res = callAccess(path, accessType);
+
+  ASSERT_EQ(unlink(path.c_str()), 0);
+
+  ASSERT_NE(res, 0);
+}
+
+TEST(AccessTests, TestNotReadWrite1) {
+  AccessType accessType;
+  accessType.read = true;
+  accessType.write = false;
+
+  std::string path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  accessType.read = true;
+  accessType.write = true;
+  std::int64_t res = callAccess(path, accessType);
+
+  ASSERT_EQ(unlink(path.c_str()), 0);
+
+  ASSERT_NE(res, 0);
+}
+
+TEST(AccessTests, TestNotReadWrite2) {
+  AccessType accessType;
+  accessType.read = false;
+  accessType.write = true;
+
+  std::string path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  accessType.read = true;
+  accessType.write = true;
+  std::int64_t res = callAccess(path, accessType);
+
+  ASSERT_EQ(unlink(path.c_str()), 0);
+
+  ASSERT_NE(res, 0);
+}
+
+TEST(AccessTests, TestExecute) {
+  AccessType accessType;
+  accessType.execute = true;
+
+  std::string path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  std::int64_t res = callAccess(path, accessType);
+
+  ASSERT_EQ(unlink(path.c_str()), 0);
+
+  ASSERT_EQ(res, 0);
+}
+
+TEST(AccessTests, TestNotExecute) {
+  AccessType accessType;
+  accessType.execute = false;
+
+  std::string path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  accessType.execute = true;
+  std::int64_t res = callAccess(path, accessType);
+
+  ASSERT_EQ(unlink(path.c_str()), 0);
+
+  ASSERT_NE(res, 0);
+}
+
+TEST(AccessTests, TestRWX) {
+  AccessType accessType;
+  accessType.read = true;
+  accessType.write = true;
+  accessType.execute = true;
+
+  std::string path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  std::int64_t res = callAccess(path, accessType);
+
+  ASSERT_EQ(unlink(path.c_str()), 0);
+
+  ASSERT_EQ(res, 0);
+}
+
+TEST(AccessTests, TestNotRWX0) {
+  AccessType accessType;
+  accessType.read = false;
+  accessType.write = false;
+  accessType.execute = false;
+
+  std::string path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  accessType.read = true;
+  accessType.write = true;
+  accessType.execute = true;
+  std::int64_t res = callAccess(path, accessType);
+
+  ASSERT_EQ(unlink(path.c_str()), 0);
+
+  ASSERT_NE(res, 0);
+}
+
+TEST(AccessTests, TestNotRWX1) {
+  AccessType accessType;
+  accessType.read = true;
+  accessType.write = false;
+  accessType.execute = false;
+
+  std::string path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  accessType.read = true;
+  accessType.write = true;
+  accessType.execute = true;
+  std::int64_t res = callAccess(path, accessType);
+
+  ASSERT_EQ(unlink(path.c_str()), 0);
+
+  ASSERT_NE(res, 0);
+}
+
+TEST(AccessTests, TestNotRWX2) {
+  AccessType accessType;
+  accessType.read = true;
+  accessType.write = true;
+  accessType.execute = false;
+
+  std::string path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  accessType.read = true;
+  accessType.write = true;
+  accessType.execute = true;
+  std::int64_t res = callAccess(path, accessType);
+
+  ASSERT_EQ(unlink(path.c_str()), 0);
+
+  ASSERT_NE(res, 0);
+}
+
+TEST(AccessTests, TestNotRWX3) {
+  AccessType accessType;
+  accessType.read = true;
+  accessType.write = false;
+  accessType.execute = true;
+
+  std::string path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  accessType.read = true;
+  accessType.write = true;
+  accessType.execute = true;
+  std::int64_t res = callAccess(path, accessType);
+
+  ASSERT_EQ(unlink(path.c_str()), 0);
+
+  ASSERT_NE(res, 0);
+}
+
+TEST(AccessTests, TestNotRWX4) {
+  AccessType accessType;
+  accessType.read = false;
+  accessType.write = true;
+  accessType.execute = true;
+
+  std::string path = createTemporaryFile(__func__, accessType);
+  ASSERT_FALSE(path.empty());
+
+  accessType.read = true;
+  accessType.write = true;
+  accessType.execute = true;
+  std::int64_t res = callAccess(path, accessType);
+
+  ASSERT_EQ(unlink(path.c_str()), 0);
+
+  ASSERT_NE(res, 0);
+}
+
+#endif // !_WIN32
diff --git a/flang/unittests/Runtime/CMakeLists.txt b/flang/unittests/Runtime/CMakeLists.txt
index 23f02aa75124..f7caacad3a59 100644
--- a/flang/unittests/Runtime/CMakeLists.txt
+++ b/flang/unittests/Runtime/CMakeLists.txt
@@ -1,4 +1,5 @@
 add_flang_unittest(FlangRuntimeTests
+  AccessTest.cpp
   Allocatable.cpp
   ArrayConstructor.cpp
   BufferTest.cpp
-- 
GitLab


From 76782e28869abf93716f72f195d55c28eaf263ed Mon Sep 17 00:00:00 2001
From: Kiran Chandramohan <kiran.chandramohan@arm.com>
Date: Tue, 16 Apr 2024 10:29:26 +0100
Subject: [PATCH 363/464] [Flang][OpenMP] NFC: Remove old reduction lowering
 code (#88798)

The old code was replaced by
https://github.com/llvm/llvm-project/pull/80019.
---
 flang/lib/Lower/OpenMP/OpenMP.cpp | 223 ------------------------------
 1 file changed, 223 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 3dcfe0fd775d..352ca66e8735 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -103,21 +103,6 @@ static fir::GlobalOp globalInitialization(
   return global;
 }
 
-static mlir::Operation *getCompareFromReductionOp(mlir::Operation *reductionOp,
-                                                  mlir::Value loadVal) {
-  for (mlir::Value reductionOperand : reductionOp->getOperands()) {
-    if (mlir::Operation *compareOp = reductionOperand.getDefiningOp()) {
-      if (compareOp->getOperand(0) == loadVal ||
-          compareOp->getOperand(1) == loadVal)
-        assert((mlir::isa<mlir::arith::CmpIOp>(compareOp) ||
-                mlir::isa<mlir::arith::CmpFOp>(compareOp)) &&
-               "Expected comparison not found in reduction intrinsic");
-      return compareOp;
-    }
-  }
-  return nullptr;
-}
-
 // Get the extended value for \p val by extracting additional variable
 // information from \p base.
 static fir::ExtendedValue getExtendedValue(fir::ExtendedValue base,
@@ -237,213 +222,6 @@ createAndSetPrivatizedLoopVar(Fortran::lower::AbstractConverter &converter,
   return storeOp;
 }
 
-static mlir::Operation *
-findReductionChain(mlir::Value loadVal, mlir::Value *reductionVal = nullptr) {
-  for (mlir::OpOperand &loadOperand : loadVal.getUses()) {
-    if (mlir::Operation *reductionOp = loadOperand.getOwner()) {
-      if (auto convertOp = mlir::dyn_cast<fir::ConvertOp>(reductionOp)) {
-        for (mlir::OpOperand &convertOperand : convertOp.getRes().getUses()) {
-          if (mlir::Operation *reductionOp = convertOperand.getOwner())
-            return reductionOp;
-        }
-      }
-      for (mlir::OpOperand &reductionOperand : reductionOp->getUses()) {
-        if (auto store =
-                mlir::dyn_cast<fir::StoreOp>(reductionOperand.getOwner())) {
-          if (store.getMemref() == *reductionVal) {
-            store.erase();
-            return reductionOp;
-          }
-        }
-        if (auto assign =
-                mlir::dyn_cast<hlfir::AssignOp>(reductionOperand.getOwner())) {
-          if (assign.getLhs() == *reductionVal) {
-            assign.erase();
-            return reductionOp;
-          }
-        }
-      }
-    }
-  }
-  return nullptr;
-}
-
-// for a logical operator 'op' reduction X = X op Y
-// This function returns the operation responsible for converting Y from
-// fir.logical<4> to i1
-static fir::ConvertOp getConvertFromReductionOp(mlir::Operation *reductionOp,
-                                                mlir::Value loadVal) {
-  for (mlir::Value reductionOperand : reductionOp->getOperands()) {
-    if (auto convertOp =
-            mlir::dyn_cast<fir::ConvertOp>(reductionOperand.getDefiningOp())) {
-      if (convertOp.getOperand() == loadVal)
-        continue;
-      return convertOp;
-    }
-  }
-  return nullptr;
-}
-
-static void updateReduction(mlir::Operation *op,
-                            fir::FirOpBuilder &firOpBuilder,
-                            mlir::Value loadVal, mlir::Value reductionVal,
-                            fir::ConvertOp *convertOp = nullptr) {
-  mlir::OpBuilder::InsertPoint insertPtDel = firOpBuilder.saveInsertionPoint();
-  firOpBuilder.setInsertionPoint(op);
-
-  mlir::Value reductionOp;
-  if (convertOp)
-    reductionOp = convertOp->getOperand();
-  else if (op->getOperand(0) == loadVal)
-    reductionOp = op->getOperand(1);
-  else
-    reductionOp = op->getOperand(0);
-
-  firOpBuilder.create<mlir::omp::ReductionOp>(op->getLoc(), reductionOp,
-                                              reductionVal);
-  firOpBuilder.restoreInsertionPoint(insertPtDel);
-}
-
-static void removeStoreOp(mlir::Operation *reductionOp, mlir::Value symVal) {
-  for (mlir::Operation *reductionOpUse : reductionOp->getUsers()) {
-    if (auto convertReduction =
-            mlir::dyn_cast<fir::ConvertOp>(reductionOpUse)) {
-      for (mlir::Operation *convertReductionUse :
-           convertReduction.getRes().getUsers()) {
-        if (auto storeOp = mlir::dyn_cast<fir::StoreOp>(convertReductionUse)) {
-          if (storeOp.getMemref() == symVal)
-            storeOp.erase();
-        }
-        if (auto assignOp =
-                mlir::dyn_cast<hlfir::AssignOp>(convertReductionUse)) {
-          if (assignOp.getLhs() == symVal)
-            assignOp.erase();
-        }
-      }
-    }
-  }
-}
-
-// Generate an OpenMP reduction operation.
-// TODO: Currently assumes it is either an integer addition/multiplication
-// reduction, or a logical and reduction. Generalize this for various reduction
-// operation types.
-// TODO: Generate the reduction operation during lowering instead of creating
-// and removing operations since this is not a robust approach. Also, removing
-// ops in the builder (instead of a rewriter) is probably not the best approach.
-static void
-genOpenMPReduction(Fortran::lower::AbstractConverter &converter,
-                   Fortran::semantics::SemanticsContext &semaCtx,
-                   const Fortran::parser::OmpClauseList &clauseList) {
-  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-
-  List<Clause> clauses{makeClauses(clauseList, semaCtx)};
-
-  for (const Clause &clause : clauses) {
-    if (const auto &reductionClause =
-            std::get_if<clause::Reduction>(&clause.u)) {
-      const auto &redOperatorList{
-          std::get<clause::Reduction::ReductionIdentifiers>(
-              reductionClause->t)};
-      assert(redOperatorList.size() == 1 && "Expecting single operator");
-      const auto &redOperator = redOperatorList.front();
-      const auto &objects{std::get<ObjectList>(reductionClause->t)};
-      if (const auto *reductionOp =
-              std::get_if<clause::DefinedOperator>(&redOperator.u)) {
-        const auto &intrinsicOp{
-            std::get<clause::DefinedOperator::IntrinsicOperator>(
-                reductionOp->u)};
-
-        switch (intrinsicOp) {
-        case clause::DefinedOperator::IntrinsicOperator::Add:
-        case clause::DefinedOperator::IntrinsicOperator::Multiply:
-        case clause::DefinedOperator::IntrinsicOperator::AND:
-        case clause::DefinedOperator::IntrinsicOperator::EQV:
-        case clause::DefinedOperator::IntrinsicOperator::OR:
-        case clause::DefinedOperator::IntrinsicOperator::NEQV:
-          break;
-        default:
-          continue;
-        }
-        for (const Object &object : objects) {
-          if (const Fortran::semantics::Symbol *symbol = object.id()) {
-            mlir::Value reductionVal = converter.getSymbolAddress(*symbol);
-            if (auto declOp = reductionVal.getDefiningOp<hlfir::DeclareOp>())
-              reductionVal = declOp.getBase();
-            mlir::Type reductionType =
-                reductionVal.getType().cast<fir::ReferenceType>().getEleTy();
-            if (!reductionType.isa<fir::LogicalType>()) {
-              if (!reductionType.isIntOrIndexOrFloat())
-                continue;
-            }
-            for (mlir::OpOperand &reductionValUse : reductionVal.getUses()) {
-              if (auto loadOp =
-                      mlir::dyn_cast<fir::LoadOp>(reductionValUse.getOwner())) {
-                mlir::Value loadVal = loadOp.getRes();
-                if (reductionType.isa<fir::LogicalType>()) {
-                  mlir::Operation *reductionOp = findReductionChain(loadVal);
-                  fir::ConvertOp convertOp =
-                      getConvertFromReductionOp(reductionOp, loadVal);
-                  updateReduction(reductionOp, firOpBuilder, loadVal,
-                                  reductionVal, &convertOp);
-                  removeStoreOp(reductionOp, reductionVal);
-                } else if (mlir::Operation *reductionOp =
-                               findReductionChain(loadVal, &reductionVal)) {
-                  updateReduction(reductionOp, firOpBuilder, loadVal,
-                                  reductionVal);
-                }
-              }
-            }
-          }
-        }
-      } else if (const auto *reductionIntrinsic =
-                     std::get_if<clause::ProcedureDesignator>(&redOperator.u)) {
-        if (!ReductionProcessor::supportedIntrinsicProcReduction(
-                *reductionIntrinsic))
-          continue;
-        ReductionProcessor::ReductionIdentifier redId =
-            ReductionProcessor::getReductionType(*reductionIntrinsic);
-        for (const Object &object : objects) {
-          if (const Fortran::semantics::Symbol *symbol = object.id()) {
-            mlir::Value reductionVal = converter.getSymbolAddress(*symbol);
-            if (auto declOp = reductionVal.getDefiningOp<hlfir::DeclareOp>())
-              reductionVal = declOp.getBase();
-            for (const mlir::OpOperand &reductionValUse :
-                 reductionVal.getUses()) {
-              if (auto loadOp =
-                      mlir::dyn_cast<fir::LoadOp>(reductionValUse.getOwner())) {
-                mlir::Value loadVal = loadOp.getRes();
-                // Max is lowered as a compare -> select.
-                // Match the pattern here.
-                mlir::Operation *reductionOp =
-                    findReductionChain(loadVal, &reductionVal);
-                if (reductionOp == nullptr)
-                  continue;
-
-                if (redId == ReductionProcessor::ReductionIdentifier::MAX ||
-                    redId == ReductionProcessor::ReductionIdentifier::MIN) {
-                  assert(mlir::isa<mlir::arith::SelectOp>(reductionOp) &&
-                         "Selection Op not found in reduction intrinsic");
-                  mlir::Operation *compareOp =
-                      getCompareFromReductionOp(reductionOp, loadVal);
-                  updateReduction(compareOp, firOpBuilder, loadVal,
-                                  reductionVal);
-                }
-                if (redId == ReductionProcessor::ReductionIdentifier::IOR ||
-                    redId == ReductionProcessor::ReductionIdentifier::IEOR ||
-                    redId == ReductionProcessor::ReductionIdentifier::IAND) {
-                  updateReduction(reductionOp, firOpBuilder, loadVal,
-                                  reductionVal);
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  }
-}
-
 struct OpWithBodyGenInfo {
   /// A type for a code-gen callback function. This takes as argument the op for
   /// which the code is being generated and returns the arguments of the op's
@@ -2197,7 +1975,6 @@ static void genOMP(Fortran::lower::AbstractConverter &converter,
     // 2.9.3.1 SIMD construct
     createSimdLoop(converter, semaCtx, eval, ompDirective, loopOpClauseList,
                    currentLocation);
-    genOpenMPReduction(converter, semaCtx, loopOpClauseList);
   } else {
     createWsloop(converter, semaCtx, eval, ompDirective, loopOpClauseList,
                  endClauseList, currentLocation);
-- 
GitLab


From 61717c1aa1f08eb57839a21fb2d9004739022e0d Mon Sep 17 00:00:00 2001
From: Jon Chesterfield <jonathanchesterfield@gmail.com>
Date: Tue, 16 Apr 2024 10:33:54 +0100
Subject: [PATCH 364/464] [Verifier] Reject va_start in non-variadic function
 (#88809)

A va_start intrinsic lowers to something derived from the variadic
parameter to the function. If there is no such parameter, it can't lower
meaningfully. Clang sema rejects the same with `error: 'va_start' used
in function with fixed args`.

Moves the existing lint warning into a verifier error. Updates the one
lit test that had a va_start in a non-variadic function.
---
 llvm/lib/Analysis/Lint.cpp                      | 5 +----
 llvm/lib/IR/Verifier.cpp                        | 5 +++++
 llvm/test/CodeGen/AArch64/GlobalISel/vastart.ll | 2 +-
 llvm/test/Other/lint.ll                         | 7 -------
 llvm/test/Verifier/variadic.ll                  | 8 ++++++++
 5 files changed, 15 insertions(+), 12 deletions(-)
 create mode 100644 llvm/test/Verifier/variadic.ll

diff --git a/llvm/lib/Analysis/Lint.cpp b/llvm/lib/Analysis/Lint.cpp
index 0694c2995dfc..1ab856ac8830 100644
--- a/llvm/lib/Analysis/Lint.cpp
+++ b/llvm/lib/Analysis/Lint.cpp
@@ -350,10 +350,7 @@ void Lint::visitCallBase(CallBase &I) {
     }
 
     case Intrinsic::vastart:
-      Check(I.getParent()->getParent()->isVarArg(),
-            "Undefined behavior: va_start called in a non-varargs function",
-            &I);
-
+      // vastart in non-varargs function is rejected by the verifier
       visitMemoryReference(I, MemoryLocation::getForArgument(&I, 0, TLI),
                            std::nullopt, nullptr, MemRef::Read | MemRef::Write);
       break;
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 516d4a051556..4cd61e6e531b 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -5798,6 +5798,11 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
 
     break;
   }
+  case Intrinsic::vastart: {
+    Check(Call.getFunction()->isVarArg(),
+          "va_start called in a non-varargs function");
+    break;
+  }
   case Intrinsic::vector_reduce_and:
   case Intrinsic::vector_reduce_or:
   case Intrinsic::vector_reduce_xor:
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/vastart.ll b/llvm/test/CodeGen/AArch64/GlobalISel/vastart.ll
index bd576d0f70e9..8c6e01d934c2 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/vastart.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/vastart.ll
@@ -3,7 +3,7 @@
 
 
 declare void @llvm.va_start(ptr)
-define void @test_va_start(ptr %list) {
+define void @test_va_start(ptr %list, ...) {
 ; CHECK-LABEL: name: test_va_start
 ; CHECK: [[LIST:%[0-9]+]]:_(p0) = COPY $x0
 ; CHECK-IOS: G_VASTART [[LIST]](p0) :: (store (s64) into %ir.list, align 1)
diff --git a/llvm/test/Other/lint.ll b/llvm/test/Other/lint.ll
index 6b31b31a78c9..6fd2d40cd2f2 100644
--- a/llvm/test/Other/lint.ll
+++ b/llvm/test/Other/lint.ll
@@ -124,13 +124,6 @@ define void @0() nounwind {
   ret void
 }
 
-; CHECK: va_start called in a non-varargs function
-declare void @llvm.va_start(ptr)
-define void @not_vararg(ptr %p) nounwind {
-  call void @llvm.va_start(ptr %p)
-  ret void
-}
-
 ; CHECK: Undefined behavior: Branch to non-blockaddress
 define void @use_indbr() {
   indirectbr ptr @foo, [label %block]
diff --git a/llvm/test/Verifier/variadic.ll b/llvm/test/Verifier/variadic.ll
new file mode 100644
index 000000000000..55e4a4da0a92
--- /dev/null
+++ b/llvm/test/Verifier/variadic.ll
@@ -0,0 +1,8 @@
+; RUN: not opt -S -passes=verify 2>&1 < %s | FileCheck %s
+
+; CHECK: va_start called in a non-varargs function
+declare void @llvm.va_start(ptr)
+define void @not_vararg(ptr %p) nounwind {
+  call void @llvm.va_start(ptr %p)
+  ret void
+}
-- 
GitLab


From 1ca6b4475c02e5d022ec6b35dbb65d0f11409a88 Mon Sep 17 00:00:00 2001
From: Ivan Butygin <ivan.butygin@gmail.com>
Date: Tue, 16 Apr 2024 12:39:57 +0300
Subject: [PATCH 365/464] [mlir][scf] `scf.while` uplifting: optimize op
 matching (#88813)

Instead of iterating over potential induction var uses looking for
suitable `arith.addi`, try to trace it back from yield argument.
---
 .../SCF/Transforms/UpliftWhileToFor.cpp       | 36 ++++++++-----------
 1 file changed, 14 insertions(+), 22 deletions(-)

diff --git a/mlir/lib/Dialect/SCF/Transforms/UpliftWhileToFor.cpp b/mlir/lib/Dialect/SCF/Transforms/UpliftWhileToFor.cpp
index fea2f659535b..7b4024b6861a 100644
--- a/mlir/lib/Dialect/SCF/Transforms/UpliftWhileToFor.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/UpliftWhileToFor.cpp
@@ -101,38 +101,30 @@ FailureOr<scf::ForOp> mlir::scf::upliftWhileToForLoop(RewriterBase &rewriter,
 
   Block *afterBody = loop.getAfterBody();
   scf::YieldOp afterTerm = loop.getYieldOp();
-  auto argNumber = inductionVar.getArgNumber();
-  auto afterTermIndArg = afterTerm.getResults()[argNumber];
+  unsigned argNumber = inductionVar.getArgNumber();
+  Value afterTermIndArg = afterTerm.getResults()[argNumber];
 
-  auto inductionVarAfter = afterBody->getArgument(argNumber);
-
-  Value step;
+  Value inductionVarAfter = afterBody->getArgument(argNumber);
 
   // Find suitable `addi` op inside `after` block, one of the args must be an
   // Induction var passed from `before` block and second arg must be defined
   // outside of the loop and will be considered step value.
   // TODO: Add `subi` support?
-  for (auto &use : inductionVarAfter.getUses()) {
-    auto owner = dyn_cast<arith::AddIOp>(use.getOwner());
-    if (!owner)
-      continue;
-
-    auto other =
-        (inductionVarAfter == owner.getLhs() ? owner.getRhs() : owner.getLhs());
-    if (!dom.properlyDominates(other, loop))
-      continue;
-
-    if (afterTermIndArg != owner.getResult())
-      continue;
+  auto addOp = afterTermIndArg.getDefiningOp<arith::AddIOp>();
+  if (!addOp)
+    return rewriter.notifyMatchFailure(loop, "Didn't found suitable 'addi' op");
 
-    step = other;
-    break;
+  Value step;
+  if (addOp.getLhs() == inductionVarAfter) {
+    step = addOp.getRhs();
+  } else if (addOp.getRhs() == inductionVarAfter) {
+    step = addOp.getLhs();
   }
 
-  if (!step)
-    return rewriter.notifyMatchFailure(loop, "Didn't found suitable 'addi' op");
+  if (!step || !dom.properlyDominates(step, loop))
+    return rewriter.notifyMatchFailure(loop, "Invalid 'addi' form");
 
-  auto lb = loop.getInits()[argNumber];
+  Value lb = loop.getInits()[argNumber];
 
   assert(lb.getType().isIntOrIndex());
   assert(lb.getType() == ub.getType());
-- 
GitLab


From aae08f4f8ef7bc9c35d263ed974679130c49f5fc Mon Sep 17 00:00:00 2001
From: Sergio Afonso <safonsof@amd.com>
Date: Tue, 16 Apr 2024 10:40:46 +0100
Subject: [PATCH 366/464] [MLIR][OpenMP] Make omp.taskloop into a loop wrapper
 (#87253)

This patch updates the definition of `omp.taskloop` to enforce the
restrictions of a wrapper operation.
---
 .../Dialect/OpenMP/OpenMPClauseOperands.h     |   9 +-
 mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td |  39 ++-
 mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp  |  15 +-
 mlir/test/Dialect/OpenMP/invalid.mlir         |  92 +++++---
 mlir/test/Dialect/OpenMP/ops.mlir             | 222 ++++++++++--------
 5 files changed, 218 insertions(+), 159 deletions(-)

diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPClauseOperands.h b/mlir/include/mlir/Dialect/OpenMP/OpenMPClauseOperands.h
index 304a9740d91e..27a766aceb31 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPClauseOperands.h
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPClauseOperands.h
@@ -284,11 +284,10 @@ using TaskgroupClauseOps =
     detail::Clauses<AllocateClauseOps, TaskReductionClauseOps>;
 
 using TaskloopClauseOps =
-    detail::Clauses<AllocateClauseOps, CollapseClauseOps, FinalClauseOps,
-                    GrainsizeClauseOps, IfClauseOps, InReductionClauseOps,
-                    LoopRelatedOps, MergeableClauseOps, NogroupClauseOps,
-                    NumTasksClauseOps, PriorityClauseOps, PrivateClauseOps,
-                    ReductionClauseOps, UntiedClauseOps>;
+    detail::Clauses<AllocateClauseOps, FinalClauseOps, GrainsizeClauseOps,
+                    IfClauseOps, InReductionClauseOps, MergeableClauseOps,
+                    NogroupClauseOps, NumTasksClauseOps, PriorityClauseOps,
+                    PrivateClauseOps, ReductionClauseOps, UntiedClauseOps>;
 
 using TaskwaitClauseOps = detail::Clauses<DependClauseOps, NowaitClauseOps>;
 
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index 7e677c9839f6..82be7ad31a15 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -1030,10 +1030,10 @@ def TaskOp : OpenMP_Op<"task", [AttrSizedOperandSegments,
 }
 
 def TaskloopOp : OpenMP_Op<"taskloop", [AttrSizedOperandSegments,
-                           AutomaticAllocationScope, RecursiveMemoryEffects,
-                           AllTypesMatch<["lowerBound", "upperBound", "step"]>,
+                           AutomaticAllocationScope,
                            DeclareOpInterfaceMethods<LoopWrapperInterface>,
-                           ReductionClauseInterface]> {
+                           RecursiveMemoryEffects, ReductionClauseInterface,
+                           SingleBlockImplicitTerminator<"TerminatorOp">]> {
   let summary = "taskloop construct";
   let description = [{
     The taskloop construct specifies that the iterations of one or more
@@ -1041,21 +1041,19 @@ def TaskloopOp : OpenMP_Op<"taskloop", [AttrSizedOperandSegments,
     iterations are distributed across tasks generated by the construct and
     scheduled to be executed.
 
-    The `lowerBound` and `upperBound` specify a half-open range: the range
-    includes the lower bound but does not include the upper bound. If the
-    `inclusive` attribute is specified then the upper bound is also included.
-    The `step` specifies the loop step.
-
-    The body region can contain any number of blocks.
+    The body region can contain a single block which must contain a single
+    operation and a terminator. The operation must be another compatible loop
+    wrapper or an `omp.loop_nest`.
 
     ```
-    omp.taskloop <clauses>
-    for (%i1, %i2) : index = (%c0, %c0) to (%c10, %c10) step (%c1, %c1) {
-      %a = load %arrA[%i1, %i2] : memref<?x?xf32>
-      %b = load %arrB[%i1, %i2] : memref<?x?xf32>
-      %sum = arith.addf %a, %b : f32
-      store %sum, %arrC[%i1, %i2] : memref<?x?xf32>
-      omp.terminator
+    omp.taskloop <clauses> {
+      omp.loop_nest (%i1, %i2) : index = (%c0, %c0) to (%c10, %c10) step (%c1, %c1) {
+        %a = load %arrA[%i1, %i2] : memref<?x?xf32>
+        %b = load %arrB[%i1, %i2] : memref<?x?xf32>
+        %sum = arith.addf %a, %b : f32
+        store %sum, %arrC[%i1, %i2] : memref<?x?xf32>
+        omp.yield
+      }
     }
     ```
 
@@ -1132,11 +1130,7 @@ def TaskloopOp : OpenMP_Op<"taskloop", [AttrSizedOperandSegments,
     created.
   }];
 
-  let arguments = (ins Variadic<IntLikeType>:$lowerBound,
-                       Variadic<IntLikeType>:$upperBound,
-                       Variadic<IntLikeType>:$step,
-                       UnitAttr:$inclusive,
-                       Optional<I1>:$if_expr,
+  let arguments = (ins Optional<I1>:$if_expr,
                        Optional<I1>:$final_expr,
                        UnitAttr:$untied,
                        UnitAttr:$mergeable,
@@ -1179,8 +1173,7 @@ def TaskloopOp : OpenMP_Op<"taskloop", [AttrSizedOperandSegments,
           |`grain_size` `(` $grain_size `:` type($grain_size) `)`
           |`num_tasks` `(` $num_tasks `:` type($num_tasks) `)`
           |`nogroup` $nogroup
-    ) `for` custom<LoopControl>($region, $lowerBound, $upperBound, $step,
-                                  type($step), $inclusive) attr-dict
+    ) $region attr-dict
   }];
 
   let extraClassDeclaration = [{
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index 365b3c4246bf..e500d0fca741 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -1829,9 +1829,8 @@ void TaskloopOp::build(OpBuilder &builder, OperationState &state,
   MLIRContext *ctx = builder.getContext();
   // TODO Store clauses in op: reductionByRefAttr, privateVars, privatizers.
   TaskloopOp::build(
-      builder, state, clauses.loopLBVar, clauses.loopUBVar, clauses.loopStepVar,
-      clauses.loopInclusiveAttr, clauses.ifVar, clauses.finalVar,
-      clauses.untiedAttr, clauses.mergeableAttr, clauses.inReductionVars,
+      builder, state, clauses.ifVar, clauses.finalVar, clauses.untiedAttr,
+      clauses.mergeableAttr, clauses.inReductionVars,
       makeArrayAttr(ctx, clauses.inReductionDeclSymbols), clauses.reductionVars,
       makeArrayAttr(ctx, clauses.reductionDeclSymbols), clauses.priorityVar,
       clauses.allocateVars, clauses.allocatorVars, clauses.grainsizeVar,
@@ -1870,6 +1869,16 @@ LogicalResult TaskloopOp::verify() {
         "the grainsize clause and num_tasks clause are mutually exclusive and "
         "may not appear on the same taskloop directive");
   }
+
+  if (!isWrapper())
+    return emitOpError() << "must be a loop wrapper";
+
+  if (LoopWrapperInterface nested = getNestedWrapper()) {
+    // Check for the allowed leaf constructs that may appear in a composite
+    // construct directly after TASKLOOP.
+    if (!isa<SimdLoopOp>(nested))
+      return emitError() << "only supported nested wrapper is 'omp.simdloop'";
+  }
   return success();
 }
 
diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir
index f8739887214b..7f86a7f5b318 100644
--- a/mlir/test/Dialect/OpenMP/invalid.mlir
+++ b/mlir/test/Dialect/OpenMP/invalid.mlir
@@ -1580,10 +1580,11 @@ func.func @omp_cancellationpoint2() {
 func.func @taskloop(%lb: i32, %ub: i32, %step: i32) {
   %testmemref = "test.memref"() : () -> (memref<i32>)
   // expected-error @below {{expected equal sizes for allocate and allocator variables}}
-  "omp.taskloop"(%lb, %ub, %ub, %lb, %step, %step, %testmemref) ({
-  ^bb0(%arg3: i32, %arg4: i32):
-    "omp.terminator"() : () -> ()
-  }) {operandSegmentSizes = array<i32: 2, 2, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0>} : (i32, i32, i32, i32, i32, i32, memref<i32>) -> ()
+  "omp.taskloop"(%testmemref) ({
+    omp.loop_nest (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
+      omp.yield
+    }
+  }) {operandSegmentSizes = array<i32: 0, 0, 0, 0, 0, 1, 0, 0, 0>} : (memref<i32>) -> ()
   return
 }
 
@@ -1593,10 +1594,11 @@ func.func @taskloop(%lb: i32, %ub: i32, %step: i32) {
   %testf32 = "test.f32"() : () -> (!llvm.ptr)
   %testf32_2 = "test.f32"() : () -> (!llvm.ptr)
   // expected-error @below {{expected as many reduction symbol references as reduction variables}}
-  "omp.taskloop"(%lb, %ub, %ub, %lb, %step, %step, %testf32, %testf32_2) ({
-  ^bb0(%arg3: i32, %arg4: i32):
-    "omp.terminator"() : () -> ()
-  }) {operandSegmentSizes = array<i32: 2, 2, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0>, reductions = [@add_f32]} : (i32, i32, i32, i32, i32, i32, !llvm.ptr, !llvm.ptr) -> ()
+  "omp.taskloop"(%testf32, %testf32_2) ({
+    omp.loop_nest (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
+      omp.yield
+    }
+  }) {operandSegmentSizes = array<i32: 0, 0, 0, 2, 0, 0, 0, 0, 0>, reductions = [@add_f32]} : (!llvm.ptr, !llvm.ptr) -> ()
   return
 }
 
@@ -1604,12 +1606,12 @@ func.func @taskloop(%lb: i32, %ub: i32, %step: i32) {
 
 func.func @taskloop(%lb: i32, %ub: i32, %step: i32) {
   %testf32 = "test.f32"() : () -> (!llvm.ptr)
-  %testf32_2 = "test.f32"() : () -> (!llvm.ptr)
   // expected-error @below {{expected as many reduction symbol references as reduction variables}}
-  "omp.taskloop"(%lb, %ub, %ub, %lb, %step, %step, %testf32) ({
-  ^bb0(%arg3: i32, %arg4: i32):
-    "omp.terminator"() : () -> ()
-  }) {operandSegmentSizes = array<i32: 2, 2, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0>, reductions = [@add_f32, @add_f32]} : (i32, i32, i32, i32, i32, i32, !llvm.ptr) -> ()
+  "omp.taskloop"(%testf32) ({
+    omp.loop_nest (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
+      omp.yield
+    }
+  }) {operandSegmentSizes = array<i32: 0, 0, 0, 1, 0, 0, 0, 0, 0>, reductions = [@add_f32, @add_f32]} : (!llvm.ptr) -> ()
   return
 }
 
@@ -1619,10 +1621,11 @@ func.func @taskloop(%lb: i32, %ub: i32, %step: i32) {
   %testf32 = "test.f32"() : () -> (!llvm.ptr)
   %testf32_2 = "test.f32"() : () -> (!llvm.ptr)
   // expected-error @below {{expected as many reduction symbol references as reduction variables}}
-  "omp.taskloop"(%lb, %ub, %ub, %lb, %step, %step, %testf32, %testf32_2) ({
-  ^bb0(%arg3: i32, %arg4: i32):
-    "omp.terminator"() : () -> ()
-  }) {in_reductions = [@add_f32], operandSegmentSizes = array<i32: 2, 2, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0>} : (i32, i32, i32, i32, i32, i32, !llvm.ptr, !llvm.ptr) -> ()
+  "omp.taskloop"(%testf32, %testf32_2) ({
+    omp.loop_nest (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
+      omp.yield
+    }
+  }) {in_reductions = [@add_f32], operandSegmentSizes = array<i32: 0, 0, 2, 0, 0, 0, 0, 0, 0>} : (!llvm.ptr, !llvm.ptr) -> ()
   return
 }
 
@@ -1630,12 +1633,12 @@ func.func @taskloop(%lb: i32, %ub: i32, %step: i32) {
 
 func.func @taskloop(%lb: i32, %ub: i32, %step: i32) {
   %testf32 = "test.f32"() : () -> (!llvm.ptr)
-  %testf32_2 = "test.f32"() : () -> (!llvm.ptr)
   // expected-error @below {{expected as many reduction symbol references as reduction variables}}
-  "omp.taskloop"(%lb, %ub, %ub, %lb, %step, %step, %testf32_2) ({
-  ^bb0(%arg3: i32, %arg4: i32):
-    "omp.terminator"() : () -> ()
-  }) {in_reductions = [@add_f32, @add_f32], operandSegmentSizes = array<i32: 2, 2, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0>} : (i32, i32, i32, i32, i32, i32, !llvm.ptr) -> ()
+  "omp.taskloop"(%testf32) ({
+    omp.loop_nest (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
+      omp.yield
+    }
+  }) {in_reductions = [@add_f32, @add_f32], operandSegmentSizes = array<i32: 0, 0, 1, 0, 0, 0, 0, 0, 0>} : (!llvm.ptr) -> ()
   return
 }
 
@@ -1657,9 +1660,10 @@ func.func @taskloop(%lb: i32, %ub: i32, %step: i32) {
   %testf32 = "test.f32"() : () -> (!llvm.ptr)
   %testf32_2 = "test.f32"() : () -> (!llvm.ptr)
   // expected-error @below {{if a reduction clause is present on the taskloop directive, the nogroup clause must not be specified}}
-  omp.taskloop reduction(@add_f32 -> %testf32 : !llvm.ptr, @add_f32 -> %testf32_2 : !llvm.ptr) nogroup
-  for (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
-    omp.terminator
+  omp.taskloop reduction(@add_f32 -> %testf32 : !llvm.ptr, @add_f32 -> %testf32_2 : !llvm.ptr) nogroup {
+    omp.loop_nest (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
+      omp.yield
+    }
   }
   return
 }
@@ -1681,9 +1685,10 @@ combiner {
 func.func @taskloop(%lb: i32, %ub: i32, %step: i32) {
   %testf32 = "test.f32"() : () -> (!llvm.ptr)
   // expected-error @below {{the same list item cannot appear in both a reduction and an in_reduction clause}}
-  omp.taskloop reduction(@add_f32 -> %testf32 : !llvm.ptr) in_reduction(@add_f32 -> %testf32 : !llvm.ptr)
-  for (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
-    omp.terminator
+  omp.taskloop reduction(@add_f32 -> %testf32 : !llvm.ptr) in_reduction(@add_f32 -> %testf32 : !llvm.ptr) {
+    omp.loop_nest (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
+      omp.yield
+    }
   }
   return
 }
@@ -1693,8 +1698,20 @@ func.func @taskloop(%lb: i32, %ub: i32, %step: i32) {
 func.func @taskloop(%lb: i32, %ub: i32, %step: i32) {
   %testi64 = "test.i64"() : () -> (i64)
   // expected-error @below {{the grainsize clause and num_tasks clause are mutually exclusive and may not appear on the same taskloop directive}}
-  omp.taskloop grain_size(%testi64: i64) num_tasks(%testi64: i64)
-  for (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
+  omp.taskloop grain_size(%testi64: i64) num_tasks(%testi64: i64) {
+    omp.loop_nest (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
+      omp.yield
+    }
+  }
+  return
+}
+
+// -----
+
+func.func @taskloop(%lb: i32, %ub: i32, %step: i32) {
+  // expected-error @below {{op must be a loop wrapper}}
+  omp.taskloop {
+    %0 = arith.constant 0 : i32
     omp.terminator
   }
   return
@@ -1702,6 +1719,21 @@ func.func @taskloop(%lb: i32, %ub: i32, %step: i32) {
 
 // -----
 
+func.func @taskloop(%lb: i32, %ub: i32, %step: i32) {
+  // expected-error @below {{only supported nested wrapper is 'omp.simdloop'}}
+  omp.taskloop {
+    omp.distribute {
+      omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
+        omp.yield
+      }
+      omp.terminator
+    }
+  }
+  return
+}
+
+// -----
+
 func.func @omp_threadprivate() {
   %1 = llvm.mlir.addressof @_QFsubEx : !llvm.ptr
   // expected-error @below {{op failed to verify that all of {sym_addr, tls_addr} have same type}}
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 018b82e995d7..802e1795b3ff 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -171,6 +171,23 @@ func.func @omp_loop_nest(%lb : index, %ub : index, %step : index) -> () {
     omp.yield
   }
 
+  // TODO Remove induction variables from omp.wsloop.
+  omp.wsloop for (%iv) : index = (%lb) to (%ub) step (%step) {
+    // CHECK: omp.loop_nest
+    // CHECK-SAME: (%{{.*}}) : index =
+    // CHECK-SAME: (%{{.*}}) to (%{{.*}}) step (%{{.*}})
+    "omp.loop_nest" (%lb, %ub, %step) ({
+    ^bb0(%iv2: index):
+      // CHECK: test.op1
+      "test.op1"(%lb) : (index) -> ()
+      // CHECK: test.op2
+      "test.op2"() : () -> ()
+      // CHECK: omp.yield
+      omp.yield
+    }) : (index, index, index) -> ()
+    omp.yield
+  }
+
   return
 }
 
@@ -209,6 +226,22 @@ func.func @omp_loop_nest_pretty(%lb : index, %ub : index, %step : index) -> () {
     omp.yield
   }
 
+  // TODO Remove induction variables from omp.wsloop.
+  omp.wsloop for (%iv) : index = (%lb) to (%ub) step (%step) {
+    // CHECK: omp.loop_nest
+    // CHECK-SAME: (%{{.*}}) : index =
+    // CHECK-SAME: (%{{.*}}) to (%{{.*}}) step (%{{.*}})
+    omp.loop_nest (%iv2) : index = (%lb) to (%ub) step (%step)  {
+      // CHECK: test.op1
+      "test.op1"(%lb) : (index) -> ()
+      // CHECK: test.op2
+      "test.op2"() : () -> ()
+      // CHECK: omp.yield
+      omp.yield
+    }
+    omp.yield
+  }
+
   return
 }
 
@@ -2024,135 +2057,128 @@ func.func @omp_taskgroup_clauses() -> () {
 // CHECK-LABEL: @omp_taskloop
 func.func @omp_taskloop(%lb: i32, %ub: i32, %step: i32) -> () {
 
-  // CHECK: omp.taskloop for (%{{.+}}) : i32 = (%{{.+}}) to (%{{.+}}) step (%{{.+}}) {
-  omp.taskloop for (%i) : i32 = (%lb) to (%ub) step (%step)  {
-    // CHECK: omp.terminator
-    omp.terminator
-  }
-
-  // CHECK: omp.taskloop for (%{{.+}}) : i32 = (%{{.+}}) to (%{{.+}}) step (%{{.+}}) {
-  omp.taskloop for (%i) : i32 = (%lb) to (%ub) step (%step)  {
-    // CHECK: test.op1
-    "test.op1"(%lb) : (i32) -> ()
-    // CHECK: test.op2
-    "test.op2"() : () -> ()
-    // CHECK: omp.terminator
-    omp.terminator
-  }
-
-  // CHECK: omp.taskloop for (%{{.+}}, %{{.+}}) : i32 = (%{{.+}}, %{{.+}}) to (%{{.+}}, %{{.+}}) step (%{{.+}}, %{{.+}}) {
-  omp.taskloop for (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
-    // CHECK: omp.terminator
-    omp.terminator
-  }
-
-  // CHECK: omp.taskloop for (%{{.+}}, %{{.+}}) : i32 = (%{{.+}}, %{{.+}}) to (%{{.+}}, %{{.+}}) inclusive step (%{{.+}}, %{{.+}}) {
-  omp.taskloop for (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) inclusive step (%step, %step) {
-    // CHECK: omp.terminator
-    omp.terminator
+  // CHECK: omp.taskloop {
+  omp.taskloop {
+    omp.loop_nest (%i) : i32 = (%lb) to (%ub) step (%step)  {
+      // CHECK: omp.yield
+      omp.yield
+    }
   }
 
   %testbool = "test.bool"() : () -> (i1)
 
-  // CHECK: omp.taskloop if(%{{[^)]+}})
-  // CHECK-SAME: for (%{{.+}}, %{{.+}}) : i32 = (%{{.+}}, %{{.+}}) to (%{{.+}}, %{{.+}}) step (%{{.+}}, %{{.+}}) {
-  omp.taskloop if(%testbool)
-  for (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
-    // CHECK: omp.terminator
-    omp.terminator
+  // CHECK: omp.taskloop if(%{{[^)]+}}) {
+  omp.taskloop if(%testbool) {
+    omp.loop_nest (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
+      // CHECK: omp.yield
+      omp.yield
+    }
   }
 
-  // CHECK: omp.taskloop final(%{{[^)]+}})
-  // CHECK-SAME: for (%{{.+}}, %{{.+}}) : i32 = (%{{.+}}, %{{.+}}) to (%{{.+}}, %{{.+}}) step (%{{.+}}, %{{.+}}) {
-  omp.taskloop final(%testbool)
-  for (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
-    // CHECK: omp.terminator
-    omp.terminator
+  // CHECK: omp.taskloop final(%{{[^)]+}}) {
+  omp.taskloop final(%testbool) {
+    omp.loop_nest (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
+      // CHECK: omp.yield
+      omp.yield
+    }
   }
 
-  // CHECK: omp.taskloop untied
-  // CHECK-SAME: for (%{{.+}}, %{{.+}}) : i32 = (%{{.+}}, %{{.+}}) to (%{{.+}}, %{{.+}}) step (%{{.+}}, %{{.+}}) {
-  omp.taskloop untied
-  for (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
-    // CHECK: omp.terminator
-    omp.terminator
+  // CHECK: omp.taskloop untied {
+  omp.taskloop untied {
+    omp.loop_nest (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
+      // CHECK: omp.yield
+      omp.yield
+    }
   }
 
-  // CHECK: omp.taskloop mergeable
-  // CHECK-SAME: for (%{{.+}}, %{{.+}}) : i32 = (%{{.+}}, %{{.+}}) to (%{{.+}}, %{{.+}}) step (%{{.+}}, %{{.+}}) {
-  omp.taskloop mergeable
-  for (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
-    // CHECK: omp.terminator
-    omp.terminator
+  // CHECK: omp.taskloop mergeable {
+  omp.taskloop mergeable {
+    omp.loop_nest (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
+      // CHECK: omp.yield
+      omp.yield
+    }
   }
 
   %testf32 = "test.f32"() : () -> (!llvm.ptr)
   %testf32_2 = "test.f32"() : () -> (!llvm.ptr)
-  // CHECK: omp.taskloop in_reduction(@add_f32 -> %{{.+}} : !llvm.ptr, @add_f32 -> %{{.+}} : !llvm.ptr)
-  // CHECK-SAME: for (%{{.+}}, %{{.+}}) : i32 = (%{{.+}}, %{{.+}}) to (%{{.+}}, %{{.+}}) step (%{{.+}}, %{{.+}}) {
-  omp.taskloop in_reduction(@add_f32 -> %testf32 : !llvm.ptr, @add_f32 -> %testf32_2 : !llvm.ptr)
-  for (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
-    // CHECK: omp.terminator
-    omp.terminator
+  // CHECK: omp.taskloop in_reduction(@add_f32 -> %{{.+}} : !llvm.ptr, @add_f32 -> %{{.+}} : !llvm.ptr) {
+  omp.taskloop in_reduction(@add_f32 -> %testf32 : !llvm.ptr, @add_f32 -> %testf32_2 : !llvm.ptr) {
+    omp.loop_nest (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
+      // CHECK: omp.yield
+      omp.yield
+    }
   }
 
-  // CHECK: omp.taskloop reduction(@add_f32 -> %{{.+}} : !llvm.ptr, @add_f32 -> %{{.+}} : !llvm.ptr)
-  // CHECK-SAME: for (%{{.+}}, %{{.+}}) : i32 = (%{{.+}}, %{{.+}}) to (%{{.+}}, %{{.+}}) step (%{{.+}}, %{{.+}}) {
-  omp.taskloop reduction(@add_f32 -> %testf32 : !llvm.ptr, @add_f32 -> %testf32_2 : !llvm.ptr)
-  for (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
-    // CHECK: omp.terminator
-    omp.terminator
+  // CHECK: omp.taskloop reduction(@add_f32 -> %{{.+}} : !llvm.ptr, @add_f32 -> %{{.+}} : !llvm.ptr) {
+  omp.taskloop reduction(@add_f32 -> %testf32 : !llvm.ptr, @add_f32 -> %testf32_2 : !llvm.ptr) {
+    omp.loop_nest (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
+      // CHECK: omp.yield
+      omp.yield
+    }
   }
 
-  // CHECK: omp.taskloop in_reduction(@add_f32 -> %{{.+}} : !llvm.ptr) reduction(@add_f32 -> %{{.+}} : !llvm.ptr)
-  // CHECK-SAME: for (%{{.+}}, %{{.+}}) : i32 = (%{{.+}}, %{{.+}}) to (%{{.+}}, %{{.+}}) step (%{{.+}}, %{{.+}}) {
-  omp.taskloop in_reduction(@add_f32 -> %testf32 : !llvm.ptr) reduction(@add_f32 -> %testf32_2 : !llvm.ptr)
-  for (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
-    // CHECK: omp.terminator
-    omp.terminator
+  // CHECK: omp.taskloop in_reduction(@add_f32 -> %{{.+}} : !llvm.ptr) reduction(@add_f32 -> %{{.+}} : !llvm.ptr) {
+  omp.taskloop in_reduction(@add_f32 -> %testf32 : !llvm.ptr) reduction(@add_f32 -> %testf32_2 : !llvm.ptr) {
+    omp.loop_nest (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
+      // CHECK: omp.yield
+      omp.yield
+    }
   }
 
   %testi32 = "test.i32"() : () -> (i32)
-  // CHECK: omp.taskloop priority(%{{[^:]+}}: i32)
-  // CHECK-SAME: for (%{{.+}}, %{{.+}}) : i32 = (%{{.+}}, %{{.+}}) to (%{{.+}}, %{{.+}}) step (%{{.+}}, %{{.+}}) {
-  omp.taskloop priority(%testi32: i32)
-  for (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
-    // CHECK: omp.terminator
-    omp.terminator
+  // CHECK: omp.taskloop priority(%{{[^:]+}}: i32) {
+  omp.taskloop priority(%testi32: i32) {
+    omp.loop_nest (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
+      // CHECK: omp.yield
+      omp.yield
+    }
   }
 
   %testmemref = "test.memref"() : () -> (memref<i32>)
-  // CHECK: omp.taskloop allocate(%{{.+}} : memref<i32> -> %{{.+}} : memref<i32>)
-  omp.taskloop allocate(%testmemref : memref<i32> -> %testmemref : memref<i32>)
-  // CHECK-SAME: for (%{{.+}}, %{{.+}}) : i32 = (%{{.+}}, %{{.+}}) to (%{{.+}}, %{{.+}}) step (%{{.+}}, %{{.+}}) {
-  for (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
-    // CHECK: omp.terminator
-    omp.terminator
+  // CHECK: omp.taskloop allocate(%{{.+}} : memref<i32> -> %{{.+}} : memref<i32>) {
+  omp.taskloop allocate(%testmemref : memref<i32> -> %testmemref : memref<i32>) {
+    omp.loop_nest (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
+      // CHECK: omp.yield
+      omp.yield
+    }
   }
 
   %testi64 = "test.i64"() : () -> (i64)
-  // CHECK: omp.taskloop grain_size(%{{[^:]+}}: i64)
-  // CHECK-SAME: for (%{{.+}}, %{{.+}}) : i32 = (%{{.+}}, %{{.+}}) to (%{{.+}}, %{{.+}}) step (%{{.+}}, %{{.+}}) {
-  omp.taskloop grain_size(%testi64: i64)
-  for (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
-    // CHECK: omp.terminator
-    omp.terminator
+  // CHECK: omp.taskloop grain_size(%{{[^:]+}}: i64) {
+  omp.taskloop grain_size(%testi64: i64) {
+    omp.loop_nest (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
+      // CHECK: omp.yield
+      omp.yield
+    }
   }
 
-  // CHECK: omp.taskloop num_tasks(%{{[^:]+}}: i64)
-  // CHECK-SAME: for (%{{.+}}, %{{.+}}) : i32 = (%{{.+}}, %{{.+}}) to (%{{.+}}, %{{.+}}) step (%{{.+}}, %{{.+}}) {
-  omp.taskloop num_tasks(%testi64: i64)
-  for (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
-    // CHECK: omp.terminator
-    omp.terminator
+  // CHECK: omp.taskloop num_tasks(%{{[^:]+}}: i64) {
+  omp.taskloop num_tasks(%testi64: i64) {
+    omp.loop_nest (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
+      // CHECK: omp.yield
+      omp.yield
+    }
   }
 
-  // CHECK: omp.taskloop nogroup
-  // CHECK-SAME: for (%{{.+}}, %{{.+}}) : i32 = (%{{.+}}, %{{.+}}) to (%{{.+}}, %{{.+}}) step (%{{.+}}, %{{.+}}) {
-  omp.taskloop nogroup
-  for (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
-    // CHECK: omp.terminator
-    omp.terminator
+  // CHECK: omp.taskloop nogroup {
+  omp.taskloop nogroup {
+    omp.loop_nest (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
+      // CHECK: omp.yield
+      omp.yield
+    }
+  }
+
+  // CHECK: omp.taskloop {
+  omp.taskloop {
+    // TODO Remove induction variables from omp.simdloop.
+    omp.simdloop for (%iv) : i32 = (%lb) to (%ub) step (%step) {
+      omp.loop_nest (%i, %j) : i32 = (%lb, %ub) to (%ub, %lb) step (%step, %step) {
+        // CHECK: omp.yield
+        omp.yield
+      }
+      // CHECK: omp.yield
+      omp.yield
+    }
   }
 
   // CHECK: return
-- 
GitLab


From ca4cf973279a3991248056a73bcb2bac8b37d035 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Mon, 15 Apr 2024 12:16:55 +0200
Subject: [PATCH 367/464] [clang][Interp][NFC] Fix Pointer::isZero() for block
 pointers

We don't need to consider the offset here anymore since we now
have proper integral pointers.
---
 clang/lib/AST/Interp/Pointer.h | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/clang/lib/AST/Interp/Pointer.h b/clang/lib/AST/Interp/Pointer.h
index fcd00aac62f9..b4475577b746 100644
--- a/clang/lib/AST/Interp/Pointer.h
+++ b/clang/lib/AST/Interp/Pointer.h
@@ -241,13 +241,10 @@ public:
 
   /// Checks if the pointer is null.
   bool isZero() const {
-    if (Offset != 0)
-      return false;
-
     if (isBlockPointer())
       return asBlockPointer().Pointee == nullptr;
     assert(isIntegralPointer());
-    return asIntPointer().Value == 0;
+    return asIntPointer().Value == 0 && Offset == 0;
   }
   /// Checks if the pointer is live.
   bool isLive() const {
-- 
GitLab


From 422bf13f336923da89055f8e70e49e7e9ced2c70 Mon Sep 17 00:00:00 2001
From: choikwa <5455710+choikwa@users.noreply.github.com>
Date: Tue, 16 Apr 2024 06:04:37 -0400
Subject: [PATCH 368/464] =?UTF-8?q?[AMDGPU]=20In=20VectorLegalizer::Expand?=
 =?UTF-8?q?,=20if=20UnrollVectorOp=20returns=20Load,=20=E2=80=A6=20(#88475?=
 =?UTF-8?q?)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…return only Load since other output is chain.

Added testcase that showed mismatched expected arity when Load and chain
were returned as separate items after
003b58f65bdd5d9c7d0c1b355566c9ef430c0e7d
---
 .../SelectionDAG/LegalizeVectorOps.cpp        | 10 +-
 llvm/test/CodeGen/AMDGPU/build_vector.ll      | 97 +++++++++++++++++++
 2 files changed, 105 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index 57a3f6a65e00..7a9cfdf5c3fd 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -1159,8 +1159,14 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
   }
 
   SDValue Unrolled = DAG.UnrollVectorOp(Node);
-  for (unsigned I = 0, E = Unrolled->getNumValues(); I != E; ++I)
-    Results.push_back(Unrolled.getValue(I));
+  if (Node->getNumValues() == 1) {
+    Results.push_back(Unrolled);
+  } else {
+    assert(Node->getNumValues() == Unrolled->getNumValues() &&
+      "VectorLegalizer Expand returned wrong number of results!");
+    for (unsigned I = 0, E = Unrolled->getNumValues(); I != E; ++I)
+      Results.push_back(Unrolled.getValue(I));
+  }
 }
 
 SDValue VectorLegalizer::ExpandSELECT(SDNode *Node) {
diff --git a/llvm/test/CodeGen/AMDGPU/build_vector.ll b/llvm/test/CodeGen/AMDGPU/build_vector.ll
index 37412ac3aa54..99755133f36d 100644
--- a/llvm/test/CodeGen/AMDGPU/build_vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/build_vector.ll
@@ -3,6 +3,7 @@
 ; RUN: llc < %s -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs | FileCheck %s --check-prefixes=GFX8,GFX678,ALL
 ; RUN: llc < %s -mtriple=amdgcn-amd-amdpal -mcpu=gfx1030 -verify-machineinstrs | FileCheck %s --check-prefixes=GFX10,GFX1011,ALL
 ; RUN: llc < %s -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 -amdgpu-enable-vopd=0 -verify-machineinstrs | FileCheck %s --check-prefixes=GFX11,GFX1011,ALL
+; RUN: llc < %s -mtriple=amdgcn -mcpu=gfx940 | FileCheck %s --check-prefixes=GFX940,ALL
 
 ; ALL-LABEL: {{^}}build_vector2:
 ; R600: MOV
@@ -96,3 +97,99 @@ define amdgpu_kernel void @build_vector_v2i16_trunc (ptr addrspace(1) %out, i32
   store <2 x i16> %ins.1, ptr addrspace(1) %out
   ret void
 }
+
+; R600-LABEL: build_v2i32_from_v4i16_shuffle:
+; R600:       ; %bb.0: ; %entry
+; R600-NEXT:    ALU 0, @10, KC0[], KC1[]
+; R600-NEXT:    TEX 1 @6
+; R600-NEXT:    ALU 4, @11, KC0[CB0:0-32], KC1[]
+; R600-NEXT:    MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1
+; R600-NEXT:    CF_END
+; R600-NEXT:    PAD
+; R600-NEXT:    Fetch clause starting at 6:
+; R600-NEXT:     VTX_READ_16 T1.X, T0.X, 48, #3
+; R600-NEXT:     VTX_READ_16 T0.X, T0.X, 44, #3
+; R600-NEXT:    ALU clause starting at 10:
+; R600-NEXT:     MOV * T0.X, 0.0,
+; R600-NEXT:    ALU clause starting at 11:
+; R600-NEXT:     LSHL * T0.Y, T1.X, literal.x,
+; R600-NEXT:    16(2.242078e-44), 0(0.000000e+00)
+; R600-NEXT:     LSHL T0.X, T0.X, literal.x,
+; R600-NEXT:     LSHR * T1.X, KC0[2].Y, literal.y,
+; R600-NEXT:    16(2.242078e-44), 2(2.802597e-45)
+;
+; GFX6-LABEL: build_v2i32_from_v4i16_shuffle:
+; GFX6:       ; %bb.0: ; %entry
+; GFX6-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; GFX6-NEXT:    s_mov_b32 s7, 0xf000
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX6-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX6-NEXT:    s_mov_b32 s6, -1
+; GFX6-NEXT:    s_mov_b32 s4, s0
+; GFX6-NEXT:    s_mov_b32 s5, s1
+; GFX6-NEXT:    v_mov_b32_e32 v0, s2
+; GFX6-NEXT:    v_mov_b32_e32 v1, s3
+; GFX6-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX6-NEXT:    s_endpgm
+;
+; GFX8-LABEL: build_v2i32_from_v4i16_shuffle:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX8-NEXT:    s_mov_b32 s7, 0xf000
+; GFX8-NEXT:    s_mov_b32 s6, -1
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s4, s0
+; GFX8-NEXT:    s_mov_b32 s5, s1
+; GFX8-NEXT:    s_lshl_b32 s0, s3, 16
+; GFX8-NEXT:    s_lshl_b32 s1, s2, 16
+; GFX8-NEXT:    v_mov_b32_e32 v0, s1
+; GFX8-NEXT:    v_mov_b32_e32 v1, s0
+; GFX8-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX8-NEXT:    s_endpgm
+;
+; GFX10-LABEL: build_v2i32_from_v4i16_shuffle:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x0
+; GFX10-NEXT:    v_mov_b32_e32 v2, 0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX10-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX10-NEXT:    v_mov_b32_e32 v0, s2
+; GFX10-NEXT:    v_mov_b32_e32 v1, s3
+; GFX10-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: build_v2i32_from_v4i16_shuffle:
+; GFX11:       ; %bb.0: ; %entry
+; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x0
+; GFX11-NEXT:    v_mov_b32_e32 v2, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX11-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX11-NEXT:    v_mov_b32_e32 v0, s2
+; GFX11-NEXT:    v_mov_b32_e32 v1, s3
+; GFX11-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-NEXT:    s_nop 0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+;
+; GFX940-LABEL: build_v2i32_from_v4i16_shuffle:
+; GFX940:       ; %bb.0: ; %entry
+; GFX940-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX940-NEXT:    v_mov_b32_e32 v2, 0
+; GFX940-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX940-NEXT:    s_lshl_b32 s3, s3, 16
+; GFX940-NEXT:    s_lshl_b32 s2, s2, 16
+; GFX940-NEXT:    v_mov_b32_e32 v0, s2
+; GFX940-NEXT:    v_mov_b32_e32 v1, s3
+; GFX940-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1] sc0 sc1
+; GFX940-NEXT:    s_endpgm
+define amdgpu_kernel void @build_v2i32_from_v4i16_shuffle(ptr addrspace(1) %out, <4 x i16> %in) {
+entry:
+  %shuf = shufflevector <4 x i16> %in, <4 x i16> zeroinitializer, <2 x i32> <i32 0, i32 2>
+  %zextended = zext <2 x i16> %shuf to <2 x i32>
+  %shifted = shl <2 x i32> %zextended, <i32 16, i32 16>
+  store <2 x i32> %shifted, ptr addrspace(1) %out
+  ret void
+}
-- 
GitLab


From 4dd5180a2d43b088d7637c30c2654f3c01c46987 Mon Sep 17 00:00:00 2001
From: Sergio Afonso <safonsof@amd.com>
Date: Tue, 16 Apr 2024 11:08:25 +0100
Subject: [PATCH 369/464] [Flang][OpenMP][Lower] Split MLIR codegen for clauses
 and constructs (#86963)

This patch performs several cleanups with the main purpose of
normalizing the code patterns used to trigger codegen for MLIR OpenMP
operations and making the processing of clauses and constructs
independent. The following changes are made:

- Clean up unused `directive` argument to
`ClauseProcessor::processMap()`.
- Move general helper functions in OpenMP.cpp to the appropriate section
of the file.
- Create `gen<OpName>Clauses()` functions containing the clause
processing code specific for the associated OpenMP construct.
- Update `gen<OpName>Op()` functions to call the corresponding
`gen<OpName>Clauses()` function.
- Sort calls to `ClauseProcessor::process<ClauseName>()` alphabetically,
to avoid inadvertently relying on some arbitrary order. Update some
tests that broke due to the order change.
- Normalize `genOMP()` functions so they all delegate the generation of
MLIR to `gen<OpName>Op()` functions following the same pattern.
- Only process `nowait` clause on `TARGET` constructs if not compiling
for the target device.

A later patch can move the calls to `gen<OpName>Clauses()` out of
`gen<OpName>Op()` functions and passing completed clause structures
instead, in preparation to supporting composite constructs. That will
make it possible to reuse clause processing for a given leaf construct
when appearing alone or in a combined or composite construct, while
controlling where the associated code is produced.
---
 flang/lib/Lower/OpenMP/ClauseProcessor.cpp    |    4 +-
 flang/lib/Lower/OpenMP/ClauseProcessor.h      |    3 +-
 flang/lib/Lower/OpenMP/OpenMP.cpp             | 2136 +++++++++--------
 flang/test/Lower/OpenMP/FIR/target.f90        |    2 +-
 flang/test/Lower/OpenMP/target.f90            |    2 +-
 .../use-device-ptr-to-use-device-addr.f90     |    4 +-
 6 files changed, 1198 insertions(+), 953 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index ae0d8bd37228..4c51b61f6bf0 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -832,8 +832,8 @@ createMapInfoOp(fir::FirOpBuilder &builder, mlir::Location loc,
 }
 
 bool ClauseProcessor::processMap(
-    mlir::Location currentLocation, const llvm::omp::Directive &directive,
-    Fortran::lower::StatementContext &stmtCtx, mlir::omp::MapClauseOps &result,
+    mlir::Location currentLocation, Fortran::lower::StatementContext &stmtCtx,
+    mlir::omp::MapClauseOps &result,
     llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *mapSyms,
     llvm::SmallVectorImpl<mlir::Location> *mapSymLocs,
     llvm::SmallVectorImpl<mlir::Type> *mapSymTypes) const {
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.h b/flang/lib/Lower/OpenMP/ClauseProcessor.h
index aa2c14b61e75..3f9701310eba 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.h
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.h
@@ -114,8 +114,7 @@ public:
   // They may be used later on to create the block_arguments for some of the
   // target directives that require it.
   bool processMap(
-      mlir::Location currentLocation, const llvm::omp::Directive &directive,
-      Fortran::lower::StatementContext &stmtCtx,
+      mlir::Location currentLocation, Fortran::lower::StatementContext &stmtCtx,
       mlir::omp::MapClauseOps &result,
       llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> *mapSyms =
           nullptr,
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 352ca66e8735..9b9975223666 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -222,6 +222,276 @@ createAndSetPrivatizedLoopVar(Fortran::lower::AbstractConverter &converter,
   return storeOp;
 }
 
+// This helper function implements the functionality of "promoting"
+// non-CPTR arguments of use_device_ptr to use_device_addr
+// arguments (automagic conversion of use_device_ptr ->
+// use_device_addr in these cases). The way we do so currently is
+// through the shuffling of operands from the devicePtrOperands to
+// deviceAddrOperands where neccesary and re-organizing the types,
+// locations and symbols to maintain the correct ordering of ptr/addr
+// input -> BlockArg.
+//
+// This effectively implements some deprecated OpenMP functionality
+// that some legacy applications unfortunately depend on
+// (deprecated in specification version 5.2):
+//
+// "If a list item in a use_device_ptr clause is not of type C_PTR,
+//  the behavior is as if the list item appeared in a use_device_addr
+//  clause. Support for such list items in a use_device_ptr clause
+//  is deprecated."
+static void promoteNonCPtrUseDevicePtrArgsToUseDeviceAddr(
+    mlir::omp::UseDeviceClauseOps &clauseOps,
+    llvm::SmallVectorImpl<mlir::Type> &useDeviceTypes,
+    llvm::SmallVectorImpl<mlir::Location> &useDeviceLocs,
+    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *>
+        &useDeviceSymbols) {
+  auto moveElementToBack = [](size_t idx, auto &vector) {
+    auto *iter = std::next(vector.begin(), idx);
+    vector.push_back(*iter);
+    vector.erase(iter);
+  };
+
+  // Iterate over our use_device_ptr list and shift all non-cptr arguments into
+  // use_device_addr.
+  for (auto *it = clauseOps.useDevicePtrVars.begin();
+       it != clauseOps.useDevicePtrVars.end();) {
+    if (!fir::isa_builtin_cptr_type(fir::unwrapRefType(it->getType()))) {
+      clauseOps.useDeviceAddrVars.push_back(*it);
+      // We have to shuffle the symbols around as well, to maintain
+      // the correct Input -> BlockArg for use_device_ptr/use_device_addr.
+      // NOTE: However, as map's do not seem to be included currently
+      // this isn't as pertinent, but we must try to maintain for
+      // future alterations. I believe the reason they are not currently
+      // is that the BlockArg assign/lowering needs to be extended
+      // to a greater set of types.
+      auto idx = std::distance(clauseOps.useDevicePtrVars.begin(), it);
+      moveElementToBack(idx, useDeviceTypes);
+      moveElementToBack(idx, useDeviceLocs);
+      moveElementToBack(idx, useDeviceSymbols);
+      it = clauseOps.useDevicePtrVars.erase(it);
+      continue;
+    }
+    ++it;
+  }
+}
+
+/// Extract the list of function and variable symbols affected by the given
+/// 'declare target' directive and return the intended device type for them.
+static void getDeclareTargetInfo(
+    Fortran::lower::AbstractConverter &converter,
+    Fortran::semantics::SemanticsContext &semaCtx,
+    Fortran::lower::pft::Evaluation &eval,
+    const Fortran::parser::OpenMPDeclareTargetConstruct &declareTargetConstruct,
+    mlir::omp::DeclareTargetClauseOps &clauseOps,
+    llvm::SmallVectorImpl<DeclareTargetCapturePair> &symbolAndClause) {
+  const auto &spec = std::get<Fortran::parser::OmpDeclareTargetSpecifier>(
+      declareTargetConstruct.t);
+  if (const auto *objectList{
+          Fortran::parser::Unwrap<Fortran::parser::OmpObjectList>(spec.u)}) {
+    ObjectList objects{makeObjects(*objectList, semaCtx)};
+    // Case: declare target(func, var1, var2)
+    gatherFuncAndVarSyms(objects, mlir::omp::DeclareTargetCaptureClause::to,
+                         symbolAndClause);
+  } else if (const auto *clauseList{
+                 Fortran::parser::Unwrap<Fortran::parser::OmpClauseList>(
+                     spec.u)}) {
+    if (clauseList->v.empty()) {
+      // Case: declare target, implicit capture of function
+      symbolAndClause.emplace_back(
+          mlir::omp::DeclareTargetCaptureClause::to,
+          eval.getOwningProcedure()->getSubprogramSymbol());
+    }
+
+    ClauseProcessor cp(converter, semaCtx, *clauseList);
+    cp.processDeviceType(clauseOps);
+    cp.processEnter(symbolAndClause);
+    cp.processLink(symbolAndClause);
+    cp.processTo(symbolAndClause);
+
+    cp.processTODO<clause::Indirect>(converter.getCurrentLocation(),
+                                     llvm::omp::Directive::OMPD_declare_target);
+  }
+}
+
+static void collectDeferredDeclareTargets(
+    Fortran::lower::AbstractConverter &converter,
+    Fortran::semantics::SemanticsContext &semaCtx,
+    Fortran::lower::pft::Evaluation &eval,
+    const Fortran::parser::OpenMPDeclareTargetConstruct &declareTargetConstruct,
+    llvm::SmallVectorImpl<Fortran::lower::OMPDeferredDeclareTargetInfo>
+        &deferredDeclareTarget) {
+  mlir::omp::DeclareTargetClauseOps clauseOps;
+  llvm::SmallVector<DeclareTargetCapturePair> symbolAndClause;
+  getDeclareTargetInfo(converter, semaCtx, eval, declareTargetConstruct,
+                       clauseOps, symbolAndClause);
+  // Return the device type only if at least one of the targets for the
+  // directive is a function or subroutine
+  mlir::ModuleOp mod = converter.getFirOpBuilder().getModule();
+
+  for (const DeclareTargetCapturePair &symClause : symbolAndClause) {
+    mlir::Operation *op = mod.lookupSymbol(converter.mangleName(
+        std::get<const Fortran::semantics::Symbol &>(symClause)));
+
+    if (!op) {
+      deferredDeclareTarget.push_back({std::get<0>(symClause),
+                                       clauseOps.deviceType,
+                                       std::get<1>(symClause)});
+    }
+  }
+}
+
+static std::optional<mlir::omp::DeclareTargetDeviceType>
+getDeclareTargetFunctionDevice(
+    Fortran::lower::AbstractConverter &converter,
+    Fortran::semantics::SemanticsContext &semaCtx,
+    Fortran::lower::pft::Evaluation &eval,
+    const Fortran::parser::OpenMPDeclareTargetConstruct
+        &declareTargetConstruct) {
+  mlir::omp::DeclareTargetClauseOps clauseOps;
+  llvm::SmallVector<DeclareTargetCapturePair> symbolAndClause;
+  getDeclareTargetInfo(converter, semaCtx, eval, declareTargetConstruct,
+                       clauseOps, symbolAndClause);
+
+  // Return the device type only if at least one of the targets for the
+  // directive is a function or subroutine
+  mlir::ModuleOp mod = converter.getFirOpBuilder().getModule();
+  for (const DeclareTargetCapturePair &symClause : symbolAndClause) {
+    mlir::Operation *op = mod.lookupSymbol(converter.mangleName(
+        std::get<const Fortran::semantics::Symbol &>(symClause)));
+
+    if (mlir::isa_and_nonnull<mlir::func::FuncOp>(op))
+      return clauseOps.deviceType;
+  }
+
+  return std::nullopt;
+}
+
+static llvm::SmallVector<const Fortran::semantics::Symbol *>
+genLoopVars(mlir::Operation *op, Fortran::lower::AbstractConverter &converter,
+            mlir::Location &loc,
+            llvm::ArrayRef<const Fortran::semantics::Symbol *> args) {
+  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+  auto &region = op->getRegion(0);
+
+  std::size_t loopVarTypeSize = 0;
+  for (const Fortran::semantics::Symbol *arg : args)
+    loopVarTypeSize = std::max(loopVarTypeSize, arg->GetUltimate().size());
+  mlir::Type loopVarType = getLoopVarType(converter, loopVarTypeSize);
+  llvm::SmallVector<mlir::Type> tiv(args.size(), loopVarType);
+  llvm::SmallVector<mlir::Location> locs(args.size(), loc);
+  firOpBuilder.createBlock(&region, {}, tiv, locs);
+  // The argument is not currently in memory, so make a temporary for the
+  // argument, and store it there, then bind that location to the argument.
+  mlir::Operation *storeOp = nullptr;
+  for (auto [argIndex, argSymbol] : llvm::enumerate(args)) {
+    mlir::Value indexVal = fir::getBase(region.front().getArgument(argIndex));
+    storeOp =
+        createAndSetPrivatizedLoopVar(converter, loc, indexVal, argSymbol);
+  }
+  firOpBuilder.setInsertionPointAfter(storeOp);
+  return llvm::SmallVector<const Fortran::semantics::Symbol *>(args);
+}
+
+static void genReductionVars(
+    mlir::Operation *op, Fortran::lower::AbstractConverter &converter,
+    mlir::Location &loc,
+    llvm::ArrayRef<const Fortran::semantics::Symbol *> reductionArgs,
+    llvm::ArrayRef<mlir::Type> reductionTypes) {
+  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+  llvm::SmallVector<mlir::Location> blockArgLocs(reductionArgs.size(), loc);
+
+  mlir::Block *entryBlock = firOpBuilder.createBlock(
+      &op->getRegion(0), {}, reductionTypes, blockArgLocs);
+
+  // Bind the reduction arguments to their block arguments.
+  for (auto [arg, prv] :
+       llvm::zip_equal(reductionArgs, entryBlock->getArguments())) {
+    converter.bindSymbol(*arg, prv);
+  }
+}
+
+static llvm::SmallVector<const Fortran::semantics::Symbol *>
+genLoopAndReductionVars(
+    mlir::Operation *op, Fortran::lower::AbstractConverter &converter,
+    mlir::Location &loc,
+    llvm::ArrayRef<const Fortran::semantics::Symbol *> loopArgs,
+    llvm::ArrayRef<const Fortran::semantics::Symbol *> reductionArgs,
+    llvm::ArrayRef<mlir::Type> reductionTypes) {
+  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+
+  llvm::SmallVector<mlir::Type> blockArgTypes;
+  llvm::SmallVector<mlir::Location> blockArgLocs;
+  blockArgTypes.reserve(loopArgs.size() + reductionArgs.size());
+  blockArgLocs.reserve(blockArgTypes.size());
+  mlir::Block *entryBlock;
+
+  if (loopArgs.size()) {
+    std::size_t loopVarTypeSize = 0;
+    for (const Fortran::semantics::Symbol *arg : loopArgs)
+      loopVarTypeSize = std::max(loopVarTypeSize, arg->GetUltimate().size());
+    mlir::Type loopVarType = getLoopVarType(converter, loopVarTypeSize);
+    std::fill_n(std::back_inserter(blockArgTypes), loopArgs.size(),
+                loopVarType);
+    std::fill_n(std::back_inserter(blockArgLocs), loopArgs.size(), loc);
+  }
+  if (reductionArgs.size()) {
+    llvm::copy(reductionTypes, std::back_inserter(blockArgTypes));
+    std::fill_n(std::back_inserter(blockArgLocs), reductionArgs.size(), loc);
+  }
+  entryBlock = firOpBuilder.createBlock(&op->getRegion(0), {}, blockArgTypes,
+                                        blockArgLocs);
+  // The argument is not currently in memory, so make a temporary for the
+  // argument, and store it there, then bind that location to the argument.
+  if (loopArgs.size()) {
+    mlir::Operation *storeOp = nullptr;
+    for (auto [argIndex, argSymbol] : llvm::enumerate(loopArgs)) {
+      mlir::Value indexVal =
+          fir::getBase(op->getRegion(0).front().getArgument(argIndex));
+      storeOp =
+          createAndSetPrivatizedLoopVar(converter, loc, indexVal, argSymbol);
+    }
+    firOpBuilder.setInsertionPointAfter(storeOp);
+  }
+  // Bind the reduction arguments to their block arguments
+  for (auto [arg, prv] : llvm::zip_equal(
+           reductionArgs,
+           llvm::drop_begin(entryBlock->getArguments(), loopArgs.size()))) {
+    converter.bindSymbol(*arg, prv);
+  }
+
+  return llvm::SmallVector<const Fortran::semantics::Symbol *>(loopArgs);
+}
+
+static void
+markDeclareTarget(mlir::Operation *op,
+                  Fortran::lower::AbstractConverter &converter,
+                  mlir::omp::DeclareTargetCaptureClause captureClause,
+                  mlir::omp::DeclareTargetDeviceType deviceType) {
+  // TODO: Add support for program local variables with declare target applied
+  auto declareTargetOp = llvm::dyn_cast<mlir::omp::DeclareTargetInterface>(op);
+  if (!declareTargetOp)
+    fir::emitFatalError(
+        converter.getCurrentLocation(),
+        "Attempt to apply declare target on unsupported operation");
+
+  // The function or global already has a declare target applied to it, very
+  // likely through implicit capture (usage in another declare target
+  // function/subroutine). It should be marked as any if it has been assigned
+  // both host and nohost, else we skip, as there is no change
+  if (declareTargetOp.isDeclareTarget()) {
+    if (declareTargetOp.getDeclareTargetDeviceType() != deviceType)
+      declareTargetOp.setDeclareTarget(mlir::omp::DeclareTargetDeviceType::any,
+                                       captureClause);
+    return;
+  }
+
+  declareTargetOp.setDeclareTarget(deviceType, captureClause);
+}
+
+//===----------------------------------------------------------------------===//
+// Op body generation helper structures and functions
+//===----------------------------------------------------------------------===//
+
 struct OpWithBodyGenInfo {
   /// A type for a code-gen callback function. This takes as argument the op for
   /// which the code is being generated and returns the arguments of the op's
@@ -493,548 +763,737 @@ static void genBodyOfTargetDataOp(
     genNestedEvaluations(converter, eval);
 }
 
-template <typename OpTy, typename... Args>
-static OpTy genOpWithBody(OpWithBodyGenInfo &info, Args &&...args) {
-  auto op = info.converter.getFirOpBuilder().create<OpTy>(
-      info.loc, std::forward<Args>(args)...);
-  createBodyOfOp<OpTy>(op, info);
-  return op;
-}
-
-static mlir::omp::MasterOp
-genMasterOp(Fortran::lower::AbstractConverter &converter,
-            Fortran::semantics::SemanticsContext &semaCtx,
-            Fortran::lower::pft::Evaluation &eval, bool genNested,
-            mlir::Location currentLocation) {
-  return genOpWithBody<mlir::omp::MasterOp>(
-      OpWithBodyGenInfo(converter, semaCtx, currentLocation, eval)
-          .setGenNested(genNested));
-}
-
-static mlir::omp::OrderedRegionOp
-genOrderedRegionOp(Fortran::lower::AbstractConverter &converter,
-                   Fortran::semantics::SemanticsContext &semaCtx,
-                   Fortran::lower::pft::Evaluation &eval, bool genNested,
-                   mlir::Location currentLocation,
-                   const Fortran::parser::OmpClauseList &clauseList) {
-  mlir::omp::OrderedRegionClauseOps clauseOps;
-
-  ClauseProcessor cp(converter, semaCtx, clauseList);
-  cp.processTODO<clause::Simd>(currentLocation,
-                               llvm::omp::Directive::OMPD_ordered);
-
-  return genOpWithBody<mlir::omp::OrderedRegionOp>(
-      OpWithBodyGenInfo(converter, semaCtx, currentLocation, eval)
-          .setGenNested(genNested),
-      clauseOps);
-}
+// This functions creates a block for the body of the targetOp's region. It adds
+// all the symbols present in mapSymbols as block arguments to this block.
+static void
+genBodyOfTargetOp(Fortran::lower::AbstractConverter &converter,
+                  Fortran::semantics::SemanticsContext &semaCtx,
+                  Fortran::lower::pft::Evaluation &eval, bool genNested,
+                  mlir::omp::TargetOp &targetOp,
+                  llvm::ArrayRef<const Fortran::semantics::Symbol *> mapSyms,
+                  llvm::ArrayRef<mlir::Location> mapSymLocs,
+                  llvm::ArrayRef<mlir::Type> mapSymTypes,
+                  const mlir::Location &currentLocation) {
+  assert(mapSymTypes.size() == mapSymLocs.size());
 
-static mlir::omp::ParallelOp
-genParallelOp(Fortran::lower::AbstractConverter &converter,
-              Fortran::lower::SymMap &symTable,
-              Fortran::semantics::SemanticsContext &semaCtx,
-              Fortran::lower::pft::Evaluation &eval, bool genNested,
-              mlir::Location currentLocation,
-              const Fortran::parser::OmpClauseList &clauseList,
-              bool outerCombined = false) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-  Fortran::lower::StatementContext stmtCtx;
-  mlir::omp::ParallelClauseOps clauseOps;
-  llvm::SmallVector<const Fortran::semantics::Symbol *> privateSyms;
-  llvm::SmallVector<mlir::Type> reductionTypes;
-  llvm::SmallVector<const Fortran::semantics::Symbol *> reductionSyms;
-
-  ClauseProcessor cp(converter, semaCtx, clauseList);
-  cp.processIf(llvm::omp::Directive::OMPD_parallel, clauseOps);
-  cp.processNumThreads(stmtCtx, clauseOps);
-  cp.processProcBind(clauseOps);
-  cp.processDefault();
-  cp.processAllocate(clauseOps);
-
-  if (!outerCombined)
-    cp.processReduction(currentLocation, clauseOps, &reductionTypes,
-                        &reductionSyms);
+  mlir::Region &region = targetOp.getRegion();
 
-  if (ReductionProcessor::doReductionByRef(clauseOps.reductionVars))
-    clauseOps.reductionByRefAttr = firOpBuilder.getUnitAttr();
+  auto *regionBlock =
+      firOpBuilder.createBlock(&region, {}, mapSymTypes, mapSymLocs);
 
-  auto reductionCallback = [&](mlir::Operation *op) {
-    llvm::SmallVector<mlir::Location> locs(clauseOps.reductionVars.size(),
-                                           currentLocation);
-    auto *block =
-        firOpBuilder.createBlock(&op->getRegion(0), {}, reductionTypes, locs);
-    for (auto [arg, prv] :
-         llvm::zip_equal(reductionSyms, block->getArguments())) {
-      converter.bindSymbol(*arg, prv);
+  // Clones the `bounds` placing them inside the target region and returns them.
+  auto cloneBound = [&](mlir::Value bound) {
+    if (mlir::isMemoryEffectFree(bound.getDefiningOp())) {
+      mlir::Operation *clonedOp = bound.getDefiningOp()->clone();
+      regionBlock->push_back(clonedOp);
+      return clonedOp->getResult(0);
     }
-    return reductionSyms;
+    TODO(converter.getCurrentLocation(),
+         "target map clause operand unsupported bound type");
   };
 
-  OpWithBodyGenInfo genInfo =
-      OpWithBodyGenInfo(converter, semaCtx, currentLocation, eval)
-          .setGenNested(genNested)
-          .setOuterCombined(outerCombined)
-          .setClauses(&clauseList)
-          .setReductions(&reductionSyms, &reductionTypes)
-          .setGenRegionEntryCb(reductionCallback);
+  auto cloneBounds = [cloneBound](llvm::ArrayRef<mlir::Value> bounds) {
+    llvm::SmallVector<mlir::Value> clonedBounds;
+    for (mlir::Value bound : bounds)
+      clonedBounds.emplace_back(cloneBound(bound));
+    return clonedBounds;
+  };
 
-  if (!enableDelayedPrivatization)
-    return genOpWithBody<mlir::omp::ParallelOp>(genInfo, clauseOps);
+  // Bind the symbols to their corresponding block arguments.
+  for (auto [argIndex, argSymbol] : llvm::enumerate(mapSyms)) {
+    const mlir::BlockArgument &arg = region.getArgument(argIndex);
+    // Avoid capture of a reference to a structured binding.
+    const Fortran::semantics::Symbol *sym = argSymbol;
+    // Structure component symbols don't have bindings.
+    if (sym->owner().IsDerivedType())
+      continue;
+    fir::ExtendedValue extVal = converter.getSymbolExtendedValue(*sym);
+    extVal.match(
+        [&](const fir::BoxValue &v) {
+          converter.bindSymbol(*sym,
+                               fir::BoxValue(arg, cloneBounds(v.getLBounds()),
+                                             v.getExplicitParameters(),
+                                             v.getExplicitExtents()));
+        },
+        [&](const fir::MutableBoxValue &v) {
+          converter.bindSymbol(
+              *sym, fir::MutableBoxValue(arg, cloneBounds(v.getLBounds()),
+                                         v.getMutableProperties()));
+        },
+        [&](const fir::ArrayBoxValue &v) {
+          converter.bindSymbol(
+              *sym, fir::ArrayBoxValue(arg, cloneBounds(v.getExtents()),
+                                       cloneBounds(v.getLBounds()),
+                                       v.getSourceBox()));
+        },
+        [&](const fir::CharArrayBoxValue &v) {
+          converter.bindSymbol(
+              *sym, fir::CharArrayBoxValue(arg, cloneBound(v.getLen()),
+                                           cloneBounds(v.getExtents()),
+                                           cloneBounds(v.getLBounds())));
+        },
+        [&](const fir::CharBoxValue &v) {
+          converter.bindSymbol(*sym,
+                               fir::CharBoxValue(arg, cloneBound(v.getLen())));
+        },
+        [&](const fir::UnboxedValue &v) { converter.bindSymbol(*sym, arg); },
+        [&](const auto &) {
+          TODO(converter.getCurrentLocation(),
+               "target map clause operand unsupported type");
+        });
+  }
 
-  bool privatize = !outerCombined;
-  DataSharingProcessor dsp(converter, semaCtx, clauseList, eval,
-                           /*useDelayedPrivatization=*/true, &symTable);
+  // Check if cloning the bounds introduced any dependency on the outer region.
+  // If so, then either clone them as well if they are MemoryEffectFree, or else
+  // copy them to a new temporary and add them to the map and block_argument
+  // lists and replace their uses with the new temporary.
+  llvm::SetVector<mlir::Value> valuesDefinedAbove;
+  mlir::getUsedValuesDefinedAbove(region, valuesDefinedAbove);
+  while (!valuesDefinedAbove.empty()) {
+    for (mlir::Value val : valuesDefinedAbove) {
+      mlir::Operation *valOp = val.getDefiningOp();
+      if (mlir::isMemoryEffectFree(valOp)) {
+        mlir::Operation *clonedOp = valOp->clone();
+        regionBlock->push_front(clonedOp);
+        val.replaceUsesWithIf(
+            clonedOp->getResult(0), [regionBlock](mlir::OpOperand &use) {
+              return use.getOwner()->getBlock() == regionBlock;
+            });
+      } else {
+        auto savedIP = firOpBuilder.getInsertionPoint();
+        firOpBuilder.setInsertionPointAfter(valOp);
+        auto copyVal =
+            firOpBuilder.createTemporary(val.getLoc(), val.getType());
+        firOpBuilder.createStoreWithConvert(copyVal.getLoc(), val, copyVal);
 
-  if (privatize)
-    dsp.processStep1(&clauseOps, &privateSyms);
+        llvm::SmallVector<mlir::Value> bounds;
+        std::stringstream name;
+        firOpBuilder.setInsertionPoint(targetOp);
+        mlir::Value mapOp = createMapInfoOp(
+            firOpBuilder, copyVal.getLoc(), copyVal, mlir::Value{}, name.str(),
+            bounds, llvm::SmallVector<mlir::Value>{},
+            static_cast<
+                std::underlying_type_t<llvm::omp::OpenMPOffloadMappingFlags>>(
+                llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT),
+            mlir::omp::VariableCaptureKind::ByCopy, copyVal.getType());
+        targetOp.getMapOperandsMutable().append(mapOp);
+        mlir::Value clonedValArg =
+            region.addArgument(copyVal.getType(), copyVal.getLoc());
+        firOpBuilder.setInsertionPointToStart(regionBlock);
+        auto loadOp = firOpBuilder.create<fir::LoadOp>(clonedValArg.getLoc(),
+                                                       clonedValArg);
+        val.replaceUsesWithIf(
+            loadOp->getResult(0), [regionBlock](mlir::OpOperand &use) {
+              return use.getOwner()->getBlock() == regionBlock;
+            });
+        firOpBuilder.setInsertionPoint(regionBlock, savedIP);
+      }
+    }
+    valuesDefinedAbove.clear();
+    mlir::getUsedValuesDefinedAbove(region, valuesDefinedAbove);
+  }
 
-  auto genRegionEntryCB = [&](mlir::Operation *op) {
-    auto parallelOp = llvm::cast<mlir::omp::ParallelOp>(op);
+  // Insert dummy instruction to remember the insertion position. The
+  // marker will be deleted since there are not uses.
+  // In the HLFIR flow there are hlfir.declares inserted above while
+  // setting block arguments.
+  mlir::Value undefMarker = firOpBuilder.create<fir::UndefOp>(
+      targetOp.getOperation()->getLoc(), firOpBuilder.getIndexType());
 
-    llvm::SmallVector<mlir::Location> reductionLocs(
-        clauseOps.reductionVars.size(), currentLocation);
+  // Create blocks for unstructured regions. This has to be done since
+  // blocks are initially allocated with the function as the parent region.
+  if (eval.lowerAsUnstructured()) {
+    Fortran::lower::createEmptyRegionBlocks<mlir::omp::TerminatorOp,
+                                            mlir::omp::YieldOp>(
+        firOpBuilder, eval.getNestedEvaluations());
+  }
 
-    mlir::OperandRange privateVars = parallelOp.getPrivateVars();
-    mlir::Region &region = parallelOp.getRegion();
+  firOpBuilder.create<mlir::omp::TerminatorOp>(currentLocation);
 
-    llvm::SmallVector<mlir::Type> privateVarTypes = reductionTypes;
-    privateVarTypes.reserve(privateVarTypes.size() + privateVars.size());
-    llvm::transform(privateVars, std::back_inserter(privateVarTypes),
-                    [](mlir::Value v) { return v.getType(); });
+  // Create the insertion point after the marker.
+  firOpBuilder.setInsertionPointAfter(undefMarker.getDefiningOp());
+  if (genNested)
+    genNestedEvaluations(converter, eval);
+}
 
-    llvm::SmallVector<mlir::Location> privateVarLocs = reductionLocs;
-    privateVarLocs.reserve(privateVarLocs.size() + privateVars.size());
-    llvm::transform(privateVars, std::back_inserter(privateVarLocs),
-                    [](mlir::Value v) { return v.getLoc(); });
+template <typename OpTy, typename... Args>
+static OpTy genOpWithBody(OpWithBodyGenInfo &info, Args &&...args) {
+  auto op = info.converter.getFirOpBuilder().create<OpTy>(
+      info.loc, std::forward<Args>(args)...);
+  createBodyOfOp<OpTy>(op, info);
+  return op;
+}
 
-    firOpBuilder.createBlock(&region, /*insertPt=*/{}, privateVarTypes,
-                             privateVarLocs);
+//===----------------------------------------------------------------------===//
+// Code generation functions for clauses
+//===----------------------------------------------------------------------===//
 
-    llvm::SmallVector<const Fortran::semantics::Symbol *> allSymbols =
-        reductionSyms;
-    allSymbols.append(privateSyms);
-    for (auto [arg, prv] : llvm::zip_equal(allSymbols, region.getArguments())) {
-      converter.bindSymbol(*arg, prv);
-    }
+static void genCriticalDeclareClauses(
+    Fortran::lower::AbstractConverter &converter,
+    Fortran::semantics::SemanticsContext &semaCtx,
+    const Fortran::parser::OmpClauseList &clauses, mlir::Location loc,
+    mlir::omp::CriticalClauseOps &clauseOps, llvm::StringRef name) {
+  ClauseProcessor cp(converter, semaCtx, clauses);
+  cp.processHint(clauseOps);
+  clauseOps.nameAttr =
+      mlir::StringAttr::get(converter.getFirOpBuilder().getContext(), name);
+}
 
-    return allSymbols;
-  };
+static void genFlushClauses(
+    Fortran::lower::AbstractConverter &converter,
+    Fortran::semantics::SemanticsContext &semaCtx,
+    const std::optional<Fortran::parser::OmpObjectList> &objects,
+    const std::optional<std::list<Fortran::parser::OmpMemoryOrderClause>>
+        &clauses,
+    mlir::Location loc, llvm::SmallVectorImpl<mlir::Value> &operandRange) {
+  if (objects)
+    genObjectList2(*objects, converter, operandRange);
+
+  if (clauses && clauses->size() > 0)
+    TODO(converter.getCurrentLocation(), "Handle OmpMemoryOrderClause");
+}
 
-  // TODO Merge with the reduction CB.
-  genInfo.setGenRegionEntryCb(genRegionEntryCB).setDataSharingProcessor(&dsp);
-  return genOpWithBody<mlir::omp::ParallelOp>(genInfo, clauseOps);
+static void
+genOrderedRegionClauses(Fortran::lower::AbstractConverter &converter,
+                        Fortran::semantics::SemanticsContext &semaCtx,
+                        const Fortran::parser::OmpClauseList &clauses,
+                        mlir::Location loc,
+                        mlir::omp::OrderedRegionClauseOps &clauseOps) {
+  ClauseProcessor cp(converter, semaCtx, clauses);
+  cp.processTODO<clause::Simd>(loc, llvm::omp::Directive::OMPD_ordered);
+}
+
+static void genParallelClauses(
+    Fortran::lower::AbstractConverter &converter,
+    Fortran::semantics::SemanticsContext &semaCtx,
+    Fortran::lower::StatementContext &stmtCtx,
+    const Fortran::parser::OmpClauseList &clauses, mlir::Location loc,
+    bool processReduction, mlir::omp::ParallelClauseOps &clauseOps,
+    llvm::SmallVectorImpl<mlir::Type> &reductionTypes,
+    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &reductionSyms) {
+  ClauseProcessor cp(converter, semaCtx, clauses);
+  cp.processAllocate(clauseOps);
+  cp.processDefault();
+  cp.processIf(llvm::omp::Directive::OMPD_parallel, clauseOps);
+  cp.processNumThreads(stmtCtx, clauseOps);
+  cp.processProcBind(clauseOps);
+
+  if (processReduction) {
+    cp.processReduction(loc, clauseOps, &reductionTypes, &reductionSyms);
+    if (ReductionProcessor::doReductionByRef(clauseOps.reductionVars))
+      clauseOps.reductionByRefAttr = converter.getFirOpBuilder().getUnitAttr();
+  }
 }
 
-static mlir::omp::SectionOp
-genSectionOp(Fortran::lower::AbstractConverter &converter,
-             Fortran::semantics::SemanticsContext &semaCtx,
-             Fortran::lower::pft::Evaluation &eval, bool genNested,
-             mlir::Location currentLocation,
-             const Fortran::parser::OmpClauseList &sectionsClauseList) {
-  // Currently only private/firstprivate clause is handled, and
-  // all privatization is done within `omp.section` operations.
-  return genOpWithBody<mlir::omp::SectionOp>(
-      OpWithBodyGenInfo(converter, semaCtx, currentLocation, eval)
-          .setGenNested(genNested)
-          .setClauses(&sectionsClauseList));
+static void genSectionsClauses(Fortran::lower::AbstractConverter &converter,
+                               Fortran::semantics::SemanticsContext &semaCtx,
+                               const Fortran::parser::OmpClauseList &clauses,
+                               mlir::Location loc,
+                               bool clausesFromBeginSections,
+                               mlir::omp::SectionsClauseOps &clauseOps) {
+  ClauseProcessor cp(converter, semaCtx, clauses);
+  if (clausesFromBeginSections) {
+    cp.processAllocate(clauseOps);
+    cp.processSectionsReduction(loc, clauseOps);
+    // TODO Support delayed privatization.
+  } else {
+    cp.processNowait(clauseOps);
+  }
 }
 
-static mlir::omp::SingleOp
-genSingleOp(Fortran::lower::AbstractConverter &converter,
-            Fortran::semantics::SemanticsContext &semaCtx,
-            Fortran::lower::pft::Evaluation &eval, bool genNested,
-            mlir::Location currentLocation,
-            const Fortran::parser::OmpClauseList &beginClauseList,
-            const Fortran::parser::OmpClauseList &endClauseList) {
-  mlir::omp::SingleClauseOps clauseOps;
+static void genSimdLoopClauses(
+    Fortran::lower::AbstractConverter &converter,
+    Fortran::semantics::SemanticsContext &semaCtx,
+    Fortran::lower::StatementContext &stmtCtx,
+    Fortran::lower::pft::Evaluation &eval,
+    const Fortran::parser::OmpClauseList &clauses, mlir::Location loc,
+    mlir::omp::SimdLoopClauseOps &clauseOps,
+    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &iv) {
+  ClauseProcessor cp(converter, semaCtx, clauses);
+  cp.processCollapse(loc, eval, clauseOps, iv);
+  cp.processIf(llvm::omp::Directive::OMPD_simd, clauseOps);
+  cp.processReduction(loc, clauseOps);
+  cp.processSafelen(clauseOps);
+  cp.processSimdlen(clauseOps);
+  clauseOps.loopInclusiveAttr = converter.getFirOpBuilder().getUnitAttr();
+  // TODO Support delayed privatization.
 
-  ClauseProcessor cp(converter, semaCtx, beginClauseList);
-  cp.processAllocate(clauseOps);
+  cp.processTODO<clause::Aligned, clause::Allocate, clause::Linear,
+                 clause::Nontemporal, clause::Order>(
+      loc, llvm::omp::Directive::OMPD_simd);
+}
+
+static void genSingleClauses(Fortran::lower::AbstractConverter &converter,
+                             Fortran::semantics::SemanticsContext &semaCtx,
+                             const Fortran::parser::OmpClauseList &beginClauses,
+                             const Fortran::parser::OmpClauseList &endClauses,
+                             mlir::Location loc,
+                             mlir::omp::SingleClauseOps &clauseOps) {
+  ClauseProcessor bcp(converter, semaCtx, beginClauses);
+  bcp.processAllocate(clauseOps);
   // TODO Support delayed privatization.
 
-  ClauseProcessor ecp(converter, semaCtx, endClauseList);
+  ClauseProcessor ecp(converter, semaCtx, endClauses);
+  ecp.processCopyprivate(loc, clauseOps);
   ecp.processNowait(clauseOps);
-  ecp.processCopyprivate(currentLocation, clauseOps);
+}
 
-  return genOpWithBody<mlir::omp::SingleOp>(
-      OpWithBodyGenInfo(converter, semaCtx, currentLocation, eval)
-          .setGenNested(genNested)
-          .setClauses(&beginClauseList),
-      clauseOps);
+static void genTargetClauses(
+    Fortran::lower::AbstractConverter &converter,
+    Fortran::semantics::SemanticsContext &semaCtx,
+    Fortran::lower::StatementContext &stmtCtx,
+    const Fortran::parser::OmpClauseList &clauses, mlir::Location loc,
+    bool processHostOnlyClauses, bool processReduction,
+    mlir::omp::TargetClauseOps &clauseOps,
+    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &mapSyms,
+    llvm::SmallVectorImpl<mlir::Location> &mapLocs,
+    llvm::SmallVectorImpl<mlir::Type> &mapTypes,
+    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &deviceAddrSyms,
+    llvm::SmallVectorImpl<mlir::Location> &deviceAddrLocs,
+    llvm::SmallVectorImpl<mlir::Type> &deviceAddrTypes,
+    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &devicePtrSyms,
+    llvm::SmallVectorImpl<mlir::Location> &devicePtrLocs,
+    llvm::SmallVectorImpl<mlir::Type> &devicePtrTypes) {
+  ClauseProcessor cp(converter, semaCtx, clauses);
+  cp.processDepend(clauseOps);
+  cp.processDevice(stmtCtx, clauseOps);
+  cp.processHasDeviceAddr(clauseOps, deviceAddrTypes, deviceAddrLocs,
+                          deviceAddrSyms);
+  cp.processIf(llvm::omp::Directive::OMPD_target, clauseOps);
+  cp.processIsDevicePtr(clauseOps, devicePtrTypes, devicePtrLocs,
+                        devicePtrSyms);
+  cp.processMap(loc, stmtCtx, clauseOps, &mapSyms, &mapLocs, &mapTypes);
+  cp.processThreadLimit(stmtCtx, clauseOps);
+  // TODO Support delayed privatization.
+
+  if (processHostOnlyClauses)
+    cp.processNowait(clauseOps);
+
+  cp.processTODO<clause::Allocate, clause::Defaultmap, clause::Firstprivate,
+                 clause::InReduction, clause::Private, clause::Reduction,
+                 clause::UsesAllocators>(loc,
+                                         llvm::omp::Directive::OMPD_target);
 }
 
-static mlir::omp::TaskOp
-genTaskOp(Fortran::lower::AbstractConverter &converter,
-          Fortran::semantics::SemanticsContext &semaCtx,
-          Fortran::lower::pft::Evaluation &eval, bool genNested,
-          mlir::Location currentLocation,
-          const Fortran::parser::OmpClauseList &clauseList) {
-  Fortran::lower::StatementContext stmtCtx;
-  mlir::omp::TaskClauseOps clauseOps;
+static void genTargetDataClauses(
+    Fortran::lower::AbstractConverter &converter,
+    Fortran::semantics::SemanticsContext &semaCtx,
+    Fortran::lower::StatementContext &stmtCtx,
+    const Fortran::parser::OmpClauseList &clauses, mlir::Location loc,
+    mlir::omp::TargetDataClauseOps &clauseOps,
+    llvm::SmallVectorImpl<mlir::Type> &useDeviceTypes,
+    llvm::SmallVectorImpl<mlir::Location> &useDeviceLocs,
+    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &useDeviceSyms) {
+  ClauseProcessor cp(converter, semaCtx, clauses);
+  cp.processDevice(stmtCtx, clauseOps);
+  cp.processIf(llvm::omp::Directive::OMPD_target_data, clauseOps);
+  cp.processMap(loc, stmtCtx, clauseOps);
+  cp.processUseDeviceAddr(clauseOps, useDeviceTypes, useDeviceLocs,
+                          useDeviceSyms);
+  cp.processUseDevicePtr(clauseOps, useDeviceTypes, useDeviceLocs,
+                         useDeviceSyms);
 
-  ClauseProcessor cp(converter, semaCtx, clauseList);
-  cp.processIf(llvm::omp::Directive::OMPD_task, clauseOps);
+  // This function implements the deprecated functionality of use_device_ptr
+  // that allows users to provide non-CPTR arguments to it with the caveat
+  // that the compiler will treat them as use_device_addr. A lot of legacy
+  // code may still depend on this functionality, so we should support it
+  // in some manner. We do so currently by simply shifting non-cptr operands
+  // from the use_device_ptr list into the front of the use_device_addr list
+  // whilst maintaining the ordering of useDeviceLocs, useDeviceSyms and
+  // useDeviceTypes to use_device_ptr/use_device_addr input for BlockArg
+  // ordering.
+  // TODO: Perhaps create a user provideable compiler option that will
+  // re-introduce a hard-error rather than a warning in these cases.
+  promoteNonCPtrUseDevicePtrArgsToUseDeviceAddr(clauseOps, useDeviceTypes,
+                                                useDeviceLocs, useDeviceSyms);
+}
+
+static void genTargetEnterExitUpdateDataClauses(
+    Fortran::lower::AbstractConverter &converter,
+    Fortran::semantics::SemanticsContext &semaCtx,
+    Fortran::lower::StatementContext &stmtCtx,
+    const Fortran::parser::OmpClauseList &clauses, mlir::Location loc,
+    llvm::omp::Directive directive,
+    mlir::omp::TargetEnterExitUpdateDataClauseOps &clauseOps) {
+  ClauseProcessor cp(converter, semaCtx, clauses);
+  cp.processDepend(clauseOps);
+  cp.processDevice(stmtCtx, clauseOps);
+  cp.processIf(directive, clauseOps);
+  cp.processNowait(clauseOps);
+
+  if (directive == llvm::omp::Directive::OMPD_target_update) {
+    cp.processMotionClauses<clause::To>(stmtCtx, clauseOps);
+    cp.processMotionClauses<clause::From>(stmtCtx, clauseOps);
+  } else {
+    cp.processMap(loc, stmtCtx, clauseOps);
+  }
+}
+
+static void genTaskClauses(Fortran::lower::AbstractConverter &converter,
+                           Fortran::semantics::SemanticsContext &semaCtx,
+                           Fortran::lower::StatementContext &stmtCtx,
+                           const Fortran::parser::OmpClauseList &clauses,
+                           mlir::Location loc,
+                           mlir::omp::TaskClauseOps &clauseOps) {
+  ClauseProcessor cp(converter, semaCtx, clauses);
   cp.processAllocate(clauseOps);
   cp.processDefault();
+  cp.processDepend(clauseOps);
   cp.processFinal(stmtCtx, clauseOps);
-  cp.processUntied(clauseOps);
+  cp.processIf(llvm::omp::Directive::OMPD_task, clauseOps);
   cp.processMergeable(clauseOps);
   cp.processPriority(stmtCtx, clauseOps);
-  cp.processDepend(clauseOps);
+  cp.processUntied(clauseOps);
   // TODO Support delayed privatization.
 
-  cp.processTODO<clause::InReduction, clause::Detach, clause::Affinity>(
-      currentLocation, llvm::omp::Directive::OMPD_task);
+  cp.processTODO<clause::Affinity, clause::Detach, clause::InReduction>(
+      loc, llvm::omp::Directive::OMPD_task);
+}
 
-  return genOpWithBody<mlir::omp::TaskOp>(
-      OpWithBodyGenInfo(converter, semaCtx, currentLocation, eval)
-          .setGenNested(genNested)
-          .setClauses(&clauseList),
-      clauseOps);
+static void genTaskgroupClauses(Fortran::lower::AbstractConverter &converter,
+                                Fortran::semantics::SemanticsContext &semaCtx,
+                                const Fortran::parser::OmpClauseList &clauses,
+                                mlir::Location loc,
+                                mlir::omp::TaskgroupClauseOps &clauseOps) {
+  ClauseProcessor cp(converter, semaCtx, clauses);
+  cp.processAllocate(clauseOps);
+
+  cp.processTODO<clause::TaskReduction>(loc,
+                                        llvm::omp::Directive::OMPD_taskgroup);
 }
 
-static mlir::omp::TaskgroupOp
-genTaskgroupOp(Fortran::lower::AbstractConverter &converter,
-               Fortran::semantics::SemanticsContext &semaCtx,
-               Fortran::lower::pft::Evaluation &eval, bool genNested,
-               mlir::Location currentLocation,
-               const Fortran::parser::OmpClauseList &clauseList) {
-  mlir::omp::TaskgroupClauseOps clauseOps;
+static void genTaskwaitClauses(Fortran::lower::AbstractConverter &converter,
+                               Fortran::semantics::SemanticsContext &semaCtx,
+                               const Fortran::parser::OmpClauseList &clauses,
+                               mlir::Location loc,
+                               mlir::omp::TaskwaitClauseOps &clauseOps) {
+  ClauseProcessor cp(converter, semaCtx, clauses);
+  cp.processTODO<clause::Depend, clause::Nowait>(
+      loc, llvm::omp::Directive::OMPD_taskwait);
+}
 
-  ClauseProcessor cp(converter, semaCtx, clauseList);
+static void genTeamsClauses(Fortran::lower::AbstractConverter &converter,
+                            Fortran::semantics::SemanticsContext &semaCtx,
+                            Fortran::lower::StatementContext &stmtCtx,
+                            const Fortran::parser::OmpClauseList &clauses,
+                            mlir::Location loc,
+                            mlir::omp::TeamsClauseOps &clauseOps) {
+  ClauseProcessor cp(converter, semaCtx, clauses);
   cp.processAllocate(clauseOps);
-  cp.processTODO<clause::TaskReduction>(currentLocation,
-                                        llvm::omp::Directive::OMPD_taskgroup);
+  cp.processDefault();
+  cp.processIf(llvm::omp::Directive::OMPD_teams, clauseOps);
+  cp.processNumTeams(stmtCtx, clauseOps);
+  cp.processThreadLimit(stmtCtx, clauseOps);
+  // TODO Support delayed privatization.
 
-  return genOpWithBody<mlir::omp::TaskgroupOp>(
-      OpWithBodyGenInfo(converter, semaCtx, currentLocation, eval)
-          .setGenNested(genNested)
-          .setClauses(&clauseList),
-      clauseOps);
+  cp.processTODO<clause::Reduction>(loc, llvm::omp::Directive::OMPD_teams);
 }
 
-// This helper function implements the functionality of "promoting"
-// non-CPTR arguments of use_device_ptr to use_device_addr
-// arguments (automagic conversion of use_device_ptr ->
-// use_device_addr in these cases). The way we do so currently is
-// through the shuffling of operands from the devicePtrOperands to
-// deviceAddrOperands where neccesary and re-organizing the types,
-// locations and symbols to maintain the correct ordering of ptr/addr
-// input -> BlockArg.
-//
-// This effectively implements some deprecated OpenMP functionality
-// that some legacy applications unfortunately depend on
-// (deprecated in specification version 5.2):
-//
-// "If a list item in a use_device_ptr clause is not of type C_PTR,
-//  the behavior is as if the list item appeared in a use_device_addr
-//  clause. Support for such list items in a use_device_ptr clause
-//  is deprecated."
-static void promoteNonCPtrUseDevicePtrArgsToUseDeviceAddr(
-    mlir::omp::UseDeviceClauseOps &clauseOps,
-    llvm::SmallVectorImpl<mlir::Type> &useDeviceTypes,
-    llvm::SmallVectorImpl<mlir::Location> &useDeviceLocs,
-    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *>
-        &useDeviceSymbols) {
-  auto moveElementToBack = [](size_t idx, auto &vector) {
-    auto *iter = std::next(vector.begin(), idx);
-    vector.push_back(*iter);
-    vector.erase(iter);
-  };
+static void genWsloopClauses(
+    Fortran::lower::AbstractConverter &converter,
+    Fortran::semantics::SemanticsContext &semaCtx,
+    Fortran::lower::StatementContext &stmtCtx,
+    Fortran::lower::pft::Evaluation &eval,
+    const Fortran::parser::OmpClauseList &beginClauses,
+    const Fortran::parser::OmpClauseList *endClauses, mlir::Location loc,
+    mlir::omp::WsloopClauseOps &clauseOps,
+    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &iv,
+    llvm::SmallVectorImpl<mlir::Type> &reductionTypes,
+    llvm::SmallVectorImpl<const Fortran::semantics::Symbol *> &reductionSyms) {
+  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+  ClauseProcessor bcp(converter, semaCtx, beginClauses);
+  bcp.processCollapse(loc, eval, clauseOps, iv);
+  bcp.processOrdered(clauseOps);
+  bcp.processReduction(loc, clauseOps, &reductionTypes, &reductionSyms);
+  bcp.processSchedule(stmtCtx, clauseOps);
+  clauseOps.loopInclusiveAttr = firOpBuilder.getUnitAttr();
+  // TODO Support delayed privatization.
 
-  // Iterate over our use_device_ptr list and shift all non-cptr arguments into
-  // use_device_addr.
-  for (auto *it = clauseOps.useDevicePtrVars.begin();
-       it != clauseOps.useDevicePtrVars.end();) {
-    if (!fir::isa_builtin_cptr_type(fir::unwrapRefType(it->getType()))) {
-      clauseOps.useDeviceAddrVars.push_back(*it);
-      // We have to shuffle the symbols around as well, to maintain
-      // the correct Input -> BlockArg for use_device_ptr/use_device_addr.
-      // NOTE: However, as map's do not seem to be included currently
-      // this isn't as pertinent, but we must try to maintain for
-      // future alterations. I believe the reason they are not currently
-      // is that the BlockArg assign/lowering needs to be extended
-      // to a greater set of types.
-      auto idx = std::distance(clauseOps.useDevicePtrVars.begin(), it);
-      moveElementToBack(idx, useDeviceTypes);
-      moveElementToBack(idx, useDeviceLocs);
-      moveElementToBack(idx, useDeviceSymbols);
-      it = clauseOps.useDevicePtrVars.erase(it);
-      continue;
+  if (ReductionProcessor::doReductionByRef(clauseOps.reductionVars))
+    clauseOps.reductionByRefAttr = firOpBuilder.getUnitAttr();
+
+  if (endClauses) {
+    ClauseProcessor ecp(converter, semaCtx, *endClauses);
+    ecp.processNowait(clauseOps);
+  }
+
+  bcp.processTODO<clause::Allocate, clause::Linear, clause::Order>(
+      loc, llvm::omp::Directive::OMPD_do);
+}
+
+//===----------------------------------------------------------------------===//
+// Code generation functions for leaf constructs
+//===----------------------------------------------------------------------===//
+
+static mlir::omp::BarrierOp
+genBarrierOp(Fortran::lower::AbstractConverter &converter,
+             Fortran::semantics::SemanticsContext &semaCtx,
+             Fortran::lower::pft::Evaluation &eval, mlir::Location loc) {
+  return converter.getFirOpBuilder().create<mlir::omp::BarrierOp>(loc);
+}
+
+static mlir::omp::CriticalOp
+genCriticalOp(Fortran::lower::AbstractConverter &converter,
+              Fortran::semantics::SemanticsContext &semaCtx,
+              Fortran::lower::pft::Evaluation &eval, bool genNested,
+              mlir::Location loc,
+              const Fortran::parser::OmpClauseList &clauseList,
+              const std::optional<Fortran::parser::Name> &name) {
+  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+  mlir::FlatSymbolRefAttr nameAttr;
+
+  if (name) {
+    std::string nameStr = name->ToString();
+    mlir::ModuleOp mod = firOpBuilder.getModule();
+    auto global = mod.lookupSymbol<mlir::omp::CriticalDeclareOp>(nameStr);
+    if (!global) {
+      mlir::omp::CriticalClauseOps clauseOps;
+      genCriticalDeclareClauses(converter, semaCtx, clauseList, loc, clauseOps,
+                                nameStr);
+
+      mlir::OpBuilder modBuilder(mod.getBodyRegion());
+      global = modBuilder.create<mlir::omp::CriticalDeclareOp>(loc, clauseOps);
     }
-    ++it;
+    nameAttr = mlir::FlatSymbolRefAttr::get(firOpBuilder.getContext(),
+                                            global.getSymName());
   }
+
+  return genOpWithBody<mlir::omp::CriticalOp>(
+      OpWithBodyGenInfo(converter, semaCtx, loc, eval).setGenNested(genNested),
+      nameAttr);
 }
 
-static mlir::omp::TargetDataOp
-genTargetDataOp(Fortran::lower::AbstractConverter &converter,
+static mlir::omp::DistributeOp
+genDistributeOp(Fortran::lower::AbstractConverter &converter,
                 Fortran::semantics::SemanticsContext &semaCtx,
                 Fortran::lower::pft::Evaluation &eval, bool genNested,
-                mlir::Location currentLocation,
+                mlir::Location loc,
                 const Fortran::parser::OmpClauseList &clauseList) {
-  Fortran::lower::StatementContext stmtCtx;
-  mlir::omp::TargetDataClauseOps clauseOps;
-  llvm::SmallVector<mlir::Type> useDeviceTypes;
-  llvm::SmallVector<mlir::Location> useDeviceLocs;
-  llvm::SmallVector<const Fortran::semantics::Symbol *> useDeviceSyms;
+  TODO(loc, "Distribute construct");
+  return nullptr;
+}
 
-  ClauseProcessor cp(converter, semaCtx, clauseList);
-  cp.processIf(llvm::omp::Directive::OMPD_target_data, clauseOps);
-  cp.processDevice(stmtCtx, clauseOps);
-  cp.processUseDevicePtr(clauseOps, useDeviceTypes, useDeviceLocs,
-                         useDeviceSyms);
-  cp.processUseDeviceAddr(clauseOps, useDeviceTypes, useDeviceLocs,
-                          useDeviceSyms);
+static mlir::omp::FlushOp
+genFlushOp(Fortran::lower::AbstractConverter &converter,
+           Fortran::semantics::SemanticsContext &semaCtx,
+           Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+           const std::optional<Fortran::parser::OmpObjectList> &objectList,
+           const std::optional<std::list<Fortran::parser::OmpMemoryOrderClause>>
+               &clauseList) {
+  llvm::SmallVector<mlir::Value> operandRange;
+  genFlushClauses(converter, semaCtx, objectList, clauseList, loc,
+                  operandRange);
+
+  return converter.getFirOpBuilder().create<mlir::omp::FlushOp>(
+      converter.getCurrentLocation(), operandRange);
+}
 
-  // This function implements the deprecated functionality of use_device_ptr
-  // that allows users to provide non-CPTR arguments to it with the caveat
-  // that the compiler will treat them as use_device_addr. A lot of legacy
-  // code may still depend on this functionality, so we should support it
-  // in some manner. We do so currently by simply shifting non-cptr operands
-  // from the use_device_ptr list into the front of the use_device_addr list
-  // whilst maintaining the ordering of useDeviceLocs, useDeviceSymbols and
-  // useDeviceTypes to use_device_ptr/use_device_addr input for BlockArg
-  // ordering.
-  // TODO: Perhaps create a user provideable compiler option that will
-  // re-introduce a hard-error rather than a warning in these cases.
-  promoteNonCPtrUseDevicePtrArgsToUseDeviceAddr(clauseOps, useDeviceTypes,
-                                                useDeviceLocs, useDeviceSyms);
-  cp.processMap(currentLocation, llvm::omp::Directive::OMPD_target_data,
-                stmtCtx, clauseOps);
+static mlir::omp::MasterOp
+genMasterOp(Fortran::lower::AbstractConverter &converter,
+            Fortran::semantics::SemanticsContext &semaCtx,
+            Fortran::lower::pft::Evaluation &eval, bool genNested,
+            mlir::Location loc) {
+  return genOpWithBody<mlir::omp::MasterOp>(
+      OpWithBodyGenInfo(converter, semaCtx, loc, eval).setGenNested(genNested));
+}
+
+static mlir::omp::OrderedOp
+genOrderedOp(Fortran::lower::AbstractConverter &converter,
+             Fortran::semantics::SemanticsContext &semaCtx,
+             Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+             const Fortran::parser::OmpClauseList &clauseList) {
+  TODO(loc, "OMPD_ordered");
+  return nullptr;
+}
+
+static mlir::omp::OrderedRegionOp
+genOrderedRegionOp(Fortran::lower::AbstractConverter &converter,
+                   Fortran::semantics::SemanticsContext &semaCtx,
+                   Fortran::lower::pft::Evaluation &eval, bool genNested,
+                   mlir::Location loc,
+                   const Fortran::parser::OmpClauseList &clauseList) {
+  mlir::omp::OrderedRegionClauseOps clauseOps;
+  genOrderedRegionClauses(converter, semaCtx, clauseList, loc, clauseOps);
+
+  return genOpWithBody<mlir::omp::OrderedRegionOp>(
+      OpWithBodyGenInfo(converter, semaCtx, loc, eval).setGenNested(genNested),
+      clauseOps);
+}
+
+static mlir::omp::ParallelOp
+genParallelOp(Fortran::lower::AbstractConverter &converter,
+              Fortran::lower::SymMap &symTable,
+              Fortran::semantics::SemanticsContext &semaCtx,
+              Fortran::lower::pft::Evaluation &eval, bool genNested,
+              mlir::Location loc,
+              const Fortran::parser::OmpClauseList &clauseList,
+              bool outerCombined = false) {
+  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+  Fortran::lower::StatementContext stmtCtx;
+  mlir::omp::ParallelClauseOps clauseOps;
+  llvm::SmallVector<const Fortran::semantics::Symbol *> privateSyms;
+  llvm::SmallVector<mlir::Type> reductionTypes;
+  llvm::SmallVector<const Fortran::semantics::Symbol *> reductionSyms;
+  genParallelClauses(converter, semaCtx, stmtCtx, clauseList, loc,
+                     /*processReduction=*/!outerCombined, clauseOps,
+                     reductionTypes, reductionSyms);
 
-  auto dataOp = converter.getFirOpBuilder().create<mlir::omp::TargetDataOp>(
-      currentLocation, clauseOps);
+  auto reductionCallback = [&](mlir::Operation *op) {
+    genReductionVars(op, converter, loc, reductionSyms, reductionTypes);
+    return reductionSyms;
+  };
 
-  genBodyOfTargetDataOp(converter, semaCtx, eval, genNested, dataOp,
-                        useDeviceTypes, useDeviceLocs, useDeviceSyms,
-                        currentLocation);
-  return dataOp;
-}
+  OpWithBodyGenInfo genInfo =
+      OpWithBodyGenInfo(converter, semaCtx, loc, eval)
+          .setGenNested(genNested)
+          .setOuterCombined(outerCombined)
+          .setClauses(&clauseList)
+          .setReductions(&reductionSyms, &reductionTypes)
+          .setGenRegionEntryCb(reductionCallback);
 
-template <typename OpTy>
-static OpTy genTargetEnterExitDataUpdateOp(
-    Fortran::lower::AbstractConverter &converter,
-    Fortran::semantics::SemanticsContext &semaCtx,
-    mlir::Location currentLocation,
-    const Fortran::parser::OmpClauseList &clauseList) {
-  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-  Fortran::lower::StatementContext stmtCtx;
-  mlir::omp::TargetEnterExitUpdateDataClauseOps clauseOps;
+  if (!enableDelayedPrivatization)
+    return genOpWithBody<mlir::omp::ParallelOp>(genInfo, clauseOps);
 
-  // GCC 9.3.0 emits a (probably) bogus warning about an unused variable.
-  [[maybe_unused]] llvm::omp::Directive directive;
-  if constexpr (std::is_same_v<OpTy, mlir::omp::TargetEnterDataOp>) {
-    directive = llvm::omp::Directive::OMPD_target_enter_data;
-  } else if constexpr (std::is_same_v<OpTy, mlir::omp::TargetExitDataOp>) {
-    directive = llvm::omp::Directive::OMPD_target_exit_data;
-  } else if constexpr (std::is_same_v<OpTy, mlir::omp::TargetUpdateOp>) {
-    directive = llvm::omp::Directive::OMPD_target_update;
-  } else {
-    return nullptr;
-  }
+  bool privatize = !outerCombined;
+  DataSharingProcessor dsp(converter, semaCtx, clauseList, eval,
+                           /*useDelayedPrivatization=*/true, &symTable);
 
-  ClauseProcessor cp(converter, semaCtx, clauseList);
-  cp.processIf(directive, clauseOps);
-  cp.processDevice(stmtCtx, clauseOps);
-  cp.processDepend(clauseOps);
-  cp.processNowait(clauseOps);
+  if (privatize)
+    dsp.processStep1(&clauseOps, &privateSyms);
 
-  if constexpr (std::is_same_v<OpTy, mlir::omp::TargetUpdateOp>) {
-    cp.processMotionClauses<clause::To>(stmtCtx, clauseOps);
-    cp.processMotionClauses<clause::From>(stmtCtx, clauseOps);
-  } else {
-    cp.processMap(currentLocation, directive, stmtCtx, clauseOps);
-  }
+  auto genRegionEntryCB = [&](mlir::Operation *op) {
+    auto parallelOp = llvm::cast<mlir::omp::ParallelOp>(op);
 
-  return firOpBuilder.create<OpTy>(currentLocation, clauseOps);
-}
+    llvm::SmallVector<mlir::Location> reductionLocs(
+        clauseOps.reductionVars.size(), loc);
 
-// This functions creates a block for the body of the targetOp's region. It adds
-// all the symbols present in mapSymbols as block arguments to this block.
-static void
-genBodyOfTargetOp(Fortran::lower::AbstractConverter &converter,
-                  Fortran::semantics::SemanticsContext &semaCtx,
-                  Fortran::lower::pft::Evaluation &eval, bool genNested,
-                  mlir::omp::TargetOp &targetOp,
-                  llvm::ArrayRef<const Fortran::semantics::Symbol *> mapSyms,
-                  llvm::ArrayRef<mlir::Location> mapSymLocs,
-                  llvm::ArrayRef<mlir::Type> mapSymTypes,
-                  const mlir::Location &currentLocation) {
-  assert(mapSymTypes.size() == mapSymLocs.size());
+    mlir::OperandRange privateVars = parallelOp.getPrivateVars();
+    mlir::Region &region = parallelOp.getRegion();
 
-  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-  mlir::Region &region = targetOp.getRegion();
+    llvm::SmallVector<mlir::Type> privateVarTypes = reductionTypes;
+    privateVarTypes.reserve(privateVarTypes.size() + privateVars.size());
+    llvm::transform(privateVars, std::back_inserter(privateVarTypes),
+                    [](mlir::Value v) { return v.getType(); });
 
-  auto *regionBlock =
-      firOpBuilder.createBlock(&region, {}, mapSymTypes, mapSymLocs);
+    llvm::SmallVector<mlir::Location> privateVarLocs = reductionLocs;
+    privateVarLocs.reserve(privateVarLocs.size() + privateVars.size());
+    llvm::transform(privateVars, std::back_inserter(privateVarLocs),
+                    [](mlir::Value v) { return v.getLoc(); });
 
-  // Clones the `bounds` placing them inside the target region and returns them.
-  auto cloneBound = [&](mlir::Value bound) {
-    if (mlir::isMemoryEffectFree(bound.getDefiningOp())) {
-      mlir::Operation *clonedOp = bound.getDefiningOp()->clone();
-      regionBlock->push_back(clonedOp);
-      return clonedOp->getResult(0);
+    firOpBuilder.createBlock(&region, /*insertPt=*/{}, privateVarTypes,
+                             privateVarLocs);
+
+    llvm::SmallVector<const Fortran::semantics::Symbol *> allSymbols =
+        reductionSyms;
+    allSymbols.append(privateSyms);
+    for (auto [arg, prv] : llvm::zip_equal(allSymbols, region.getArguments())) {
+      converter.bindSymbol(*arg, prv);
     }
-    TODO(converter.getCurrentLocation(),
-         "target map clause operand unsupported bound type");
-  };
 
-  auto cloneBounds = [cloneBound](llvm::ArrayRef<mlir::Value> bounds) {
-    llvm::SmallVector<mlir::Value> clonedBounds;
-    for (mlir::Value bound : bounds)
-      clonedBounds.emplace_back(cloneBound(bound));
-    return clonedBounds;
+    return allSymbols;
   };
 
-  // Bind the symbols to their corresponding block arguments.
-  for (auto [argIndex, argSymbol] : llvm::enumerate(mapSyms)) {
-    const mlir::BlockArgument &arg = region.getArgument(argIndex);
-    // Avoid capture of a reference to a structured binding.
-    const Fortran::semantics::Symbol *sym = argSymbol;
-    // Structure component symbols don't have bindings.
-    if (sym->owner().IsDerivedType())
-      continue;
-    fir::ExtendedValue extVal = converter.getSymbolExtendedValue(*sym);
-    extVal.match(
-        [&](const fir::BoxValue &v) {
-          converter.bindSymbol(*sym,
-                               fir::BoxValue(arg, cloneBounds(v.getLBounds()),
-                                             v.getExplicitParameters(),
-                                             v.getExplicitExtents()));
-        },
-        [&](const fir::MutableBoxValue &v) {
-          converter.bindSymbol(
-              *sym, fir::MutableBoxValue(arg, cloneBounds(v.getLBounds()),
-                                         v.getMutableProperties()));
-        },
-        [&](const fir::ArrayBoxValue &v) {
-          converter.bindSymbol(
-              *sym, fir::ArrayBoxValue(arg, cloneBounds(v.getExtents()),
-                                       cloneBounds(v.getLBounds()),
-                                       v.getSourceBox()));
-        },
-        [&](const fir::CharArrayBoxValue &v) {
-          converter.bindSymbol(
-              *sym, fir::CharArrayBoxValue(arg, cloneBound(v.getLen()),
-                                           cloneBounds(v.getExtents()),
-                                           cloneBounds(v.getLBounds())));
-        },
-        [&](const fir::CharBoxValue &v) {
-          converter.bindSymbol(*sym,
-                               fir::CharBoxValue(arg, cloneBound(v.getLen())));
-        },
-        [&](const fir::UnboxedValue &v) { converter.bindSymbol(*sym, arg); },
-        [&](const auto &) {
-          TODO(converter.getCurrentLocation(),
-               "target map clause operand unsupported type");
-        });
-  }
+  // TODO Merge with the reduction CB.
+  genInfo.setGenRegionEntryCb(genRegionEntryCB).setDataSharingProcessor(&dsp);
+  return genOpWithBody<mlir::omp::ParallelOp>(genInfo, clauseOps);
+}
 
-  // Check if cloning the bounds introduced any dependency on the outer region.
-  // If so, then either clone them as well if they are MemoryEffectFree, or else
-  // copy them to a new temporary and add them to the map and block_argument
-  // lists and replace their uses with the new temporary.
-  llvm::SetVector<mlir::Value> valuesDefinedAbove;
-  mlir::getUsedValuesDefinedAbove(region, valuesDefinedAbove);
-  while (!valuesDefinedAbove.empty()) {
-    for (mlir::Value val : valuesDefinedAbove) {
-      mlir::Operation *valOp = val.getDefiningOp();
-      if (mlir::isMemoryEffectFree(valOp)) {
-        mlir::Operation *clonedOp = valOp->clone();
-        regionBlock->push_front(clonedOp);
-        val.replaceUsesWithIf(
-            clonedOp->getResult(0), [regionBlock](mlir::OpOperand &use) {
-              return use.getOwner()->getBlock() == regionBlock;
-            });
-      } else {
-        auto savedIP = firOpBuilder.getInsertionPoint();
-        firOpBuilder.setInsertionPointAfter(valOp);
-        auto copyVal =
-            firOpBuilder.createTemporary(val.getLoc(), val.getType());
-        firOpBuilder.createStoreWithConvert(copyVal.getLoc(), val, copyVal);
+static mlir::omp::SectionOp
+genSectionOp(Fortran::lower::AbstractConverter &converter,
+             Fortran::semantics::SemanticsContext &semaCtx,
+             Fortran::lower::pft::Evaluation &eval, bool genNested,
+             mlir::Location loc,
+             const Fortran::parser::OmpClauseList &clauseList) {
+  // Currently only private/firstprivate clause is handled, and
+  // all privatization is done within `omp.section` operations.
+  return genOpWithBody<mlir::omp::SectionOp>(
+      OpWithBodyGenInfo(converter, semaCtx, loc, eval)
+          .setGenNested(genNested)
+          .setClauses(&clauseList));
+}
 
-        llvm::SmallVector<mlir::Value> bounds;
-        std::stringstream name;
-        firOpBuilder.setInsertionPoint(targetOp);
-        mlir::Value mapOp = createMapInfoOp(
-            firOpBuilder, copyVal.getLoc(), copyVal, mlir::Value{}, name.str(),
-            bounds, llvm::SmallVector<mlir::Value>{},
-            static_cast<
-                std::underlying_type_t<llvm::omp::OpenMPOffloadMappingFlags>>(
-                llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT),
-            mlir::omp::VariableCaptureKind::ByCopy, copyVal.getType());
-        targetOp.getMapOperandsMutable().append(mapOp);
-        mlir::Value clonedValArg =
-            region.addArgument(copyVal.getType(), copyVal.getLoc());
-        firOpBuilder.setInsertionPointToStart(regionBlock);
-        auto loadOp = firOpBuilder.create<fir::LoadOp>(clonedValArg.getLoc(),
-                                                       clonedValArg);
-        val.replaceUsesWithIf(
-            loadOp->getResult(0), [regionBlock](mlir::OpOperand &use) {
-              return use.getOwner()->getBlock() == regionBlock;
-            });
-        firOpBuilder.setInsertionPoint(regionBlock, savedIP);
-      }
-    }
-    valuesDefinedAbove.clear();
-    mlir::getUsedValuesDefinedAbove(region, valuesDefinedAbove);
-  }
+static mlir::omp::SectionsOp
+genSectionsOp(Fortran::lower::AbstractConverter &converter,
+              Fortran::semantics::SemanticsContext &semaCtx,
+              Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+              const mlir::omp::SectionsClauseOps &clauseOps) {
+  return genOpWithBody<mlir::omp::SectionsOp>(
+      OpWithBodyGenInfo(converter, semaCtx, loc, eval).setGenNested(false),
+      clauseOps);
+}
 
-  // Insert dummy instruction to remember the insertion position. The
-  // marker will be deleted since there are not uses.
-  // In the HLFIR flow there are hlfir.declares inserted above while
-  // setting block arguments.
-  mlir::Value undefMarker = firOpBuilder.create<fir::UndefOp>(
-      targetOp.getOperation()->getLoc(), firOpBuilder.getIndexType());
+static mlir::omp::SimdLoopOp
+genSimdLoopOp(Fortran::lower::AbstractConverter &converter,
+              Fortran::semantics::SemanticsContext &semaCtx,
+              Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+              const Fortran::parser::OmpClauseList &clauseList) {
+  DataSharingProcessor dsp(converter, semaCtx, clauseList, eval);
+  dsp.processStep1();
 
-  // Create blocks for unstructured regions. This has to be done since
-  // blocks are initially allocated with the function as the parent region.
-  if (eval.lowerAsUnstructured()) {
-    Fortran::lower::createEmptyRegionBlocks<mlir::omp::TerminatorOp,
-                                            mlir::omp::YieldOp>(
-        firOpBuilder, eval.getNestedEvaluations());
-  }
+  Fortran::lower::StatementContext stmtCtx;
+  mlir::omp::SimdLoopClauseOps clauseOps;
+  llvm::SmallVector<const Fortran::semantics::Symbol *> iv;
+  genSimdLoopClauses(converter, semaCtx, stmtCtx, eval, clauseList, loc,
+                     clauseOps, iv);
 
-  firOpBuilder.create<mlir::omp::TerminatorOp>(currentLocation);
+  auto *nestedEval =
+      getCollapsedLoopEval(eval, Fortran::lower::getCollapseValue(clauseList));
 
-  // Create the insertion point after the marker.
-  firOpBuilder.setInsertionPointAfter(undefMarker.getDefiningOp());
-  if (genNested)
-    genNestedEvaluations(converter, eval);
+  auto ivCallback = [&](mlir::Operation *op) {
+    return genLoopVars(op, converter, loc, iv);
+  };
+
+  return genOpWithBody<mlir::omp::SimdLoopOp>(
+      OpWithBodyGenInfo(converter, semaCtx, loc, *nestedEval)
+          .setClauses(&clauseList)
+          .setDataSharingProcessor(&dsp)
+          .setGenRegionEntryCb(ivCallback),
+      clauseOps);
+}
+
+static mlir::omp::SingleOp
+genSingleOp(Fortran::lower::AbstractConverter &converter,
+            Fortran::semantics::SemanticsContext &semaCtx,
+            Fortran::lower::pft::Evaluation &eval, bool genNested,
+            mlir::Location loc,
+            const Fortran::parser::OmpClauseList &beginClauseList,
+            const Fortran::parser::OmpClauseList &endClauseList) {
+  mlir::omp::SingleClauseOps clauseOps;
+  genSingleClauses(converter, semaCtx, beginClauseList, endClauseList, loc,
+                   clauseOps);
+
+  return genOpWithBody<mlir::omp::SingleOp>(
+      OpWithBodyGenInfo(converter, semaCtx, loc, eval)
+          .setGenNested(genNested)
+          .setClauses(&beginClauseList),
+      clauseOps);
 }
 
 static mlir::omp::TargetOp
 genTargetOp(Fortran::lower::AbstractConverter &converter,
             Fortran::semantics::SemanticsContext &semaCtx,
             Fortran::lower::pft::Evaluation &eval, bool genNested,
-            mlir::Location currentLocation,
+            mlir::Location loc,
             const Fortran::parser::OmpClauseList &clauseList,
-            llvm::omp::Directive directive, bool outerCombined = false) {
+            bool outerCombined = false) {
+  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
   Fortran::lower::StatementContext stmtCtx;
+
+  bool processHostOnlyClauses =
+      !llvm::cast<mlir::omp::OffloadModuleInterface>(*converter.getModuleOp())
+           .getIsTargetDevice();
+
   mlir::omp::TargetClauseOps clauseOps;
-  llvm::SmallVector<mlir::Type> mapTypes, devicePtrTypes, deviceAddrTypes;
-  llvm::SmallVector<mlir::Location> mapLocs, devicePtrLocs, deviceAddrLocs;
   llvm::SmallVector<const Fortran::semantics::Symbol *> mapSyms, devicePtrSyms,
       deviceAddrSyms;
-
-  ClauseProcessor cp(converter, semaCtx, clauseList);
-  cp.processIf(llvm::omp::Directive::OMPD_target, clauseOps);
-  cp.processDevice(stmtCtx, clauseOps);
-  cp.processThreadLimit(stmtCtx, clauseOps);
-  cp.processDepend(clauseOps);
-  cp.processNowait(clauseOps);
-  cp.processMap(currentLocation, directive, stmtCtx, clauseOps, &mapSyms,
-                &mapLocs, &mapTypes);
-  cp.processIsDevicePtr(clauseOps, devicePtrTypes, devicePtrLocs,
-                        devicePtrSyms);
-  cp.processHasDeviceAddr(clauseOps, deviceAddrTypes, deviceAddrLocs,
-                          deviceAddrSyms);
-  // TODO Support delayed privatization.
-
-  cp.processTODO<clause::Private, clause::Firstprivate, clause::Reduction,
-                 clause::InReduction, clause::Allocate, clause::UsesAllocators,
-                 clause::Defaultmap>(currentLocation,
-                                     llvm::omp::Directive::OMPD_target);
+  llvm::SmallVector<mlir::Location> mapLocs, devicePtrLocs, deviceAddrLocs;
+  llvm::SmallVector<mlir::Type> mapTypes, devicePtrTypes, deviceAddrTypes;
+  genTargetClauses(converter, semaCtx, stmtCtx, clauseList, loc,
+                   processHostOnlyClauses, /*processReduction=*/outerCombined,
+                   clauseOps, mapSyms, mapLocs, mapTypes, deviceAddrSyms,
+                   deviceAddrLocs, deviceAddrTypes, devicePtrSyms,
+                   devicePtrLocs, devicePtrTypes);
 
   // 5.8.1 Implicit Data-Mapping Attribute Rules
   // The following code follows the implicit data-mapping rules to map all the
@@ -1056,22 +1515,21 @@ genTargetOp(Fortran::lower::AbstractConverter &converter,
         fir::ExtendedValue dataExv = converter.getSymbolExtendedValue(sym);
         name << sym.name().ToString();
 
-        Fortran::lower::AddrAndBoundsInfo info =
-            getDataOperandBaseAddr(converter, converter.getFirOpBuilder(), sym,
-                                   converter.getCurrentLocation());
+        Fortran::lower::AddrAndBoundsInfo info = getDataOperandBaseAddr(
+            converter, firOpBuilder, sym, converter.getCurrentLocation());
         if (fir::unwrapRefType(info.addr.getType()).isa<fir::BaseBoxType>())
           bounds =
               Fortran::lower::genBoundsOpsFromBox<mlir::omp::MapBoundsOp,
                                                   mlir::omp::MapBoundsType>(
-                  converter.getFirOpBuilder(), converter.getCurrentLocation(),
-                  converter, dataExv, info);
+                  firOpBuilder, converter.getCurrentLocation(), converter,
+                  dataExv, info);
         if (fir::unwrapRefType(info.addr.getType()).isa<fir::SequenceType>()) {
           bool dataExvIsAssumedSize =
               Fortran::semantics::IsAssumedSizeArray(sym.GetUltimate());
           bounds = Fortran::lower::genBaseBoundsOps<mlir::omp::MapBoundsOp,
                                                     mlir::omp::MapBoundsType>(
-              converter.getFirOpBuilder(), converter.getCurrentLocation(),
-              converter, dataExv, dataExvIsAssumedSize);
+              firOpBuilder, converter.getCurrentLocation(), converter, dataExv,
+              dataExvIsAssumedSize);
         }
 
         llvm::omp::OpenMPOffloadMappingFlags mapFlag =
@@ -1085,7 +1543,7 @@ genTargetOp(Fortran::lower::AbstractConverter &converter,
 
         // If a variable is specified in declare target link and if device
         // type is not specified as `nohost`, it needs to be mapped tofrom
-        mlir::ModuleOp mod = converter.getFirOpBuilder().getModule();
+        mlir::ModuleOp mod = firOpBuilder.getModule();
         mlir::Operation *op = mod.lookupSymbol(converter.mangleName(sym));
         auto declareTargetOp =
             llvm::dyn_cast_if_present<mlir::omp::DeclareTargetInterface>(op);
@@ -1105,8 +1563,8 @@ genTargetOp(Fortran::lower::AbstractConverter &converter,
         }
 
         mlir::Value mapOp = createMapInfoOp(
-            converter.getFirOpBuilder(), baseOp.getLoc(), baseOp, mlir::Value{},
-            name.str(), bounds, {},
+            firOpBuilder, baseOp.getLoc(), baseOp, mlir::Value{}, name.str(),
+            bounds, {},
             static_cast<
                 std::underlying_type_t<llvm::omp::OpenMPOffloadMappingFlags>>(
                 mapFlag),
@@ -1118,341 +1576,147 @@ genTargetOp(Fortran::lower::AbstractConverter &converter,
         mapTypes.push_back(baseOp.getType());
       }
     }
-  };
-  Fortran::lower::pft::visitAllSymbols(eval, captureImplicitMap);
-
-  auto targetOp = converter.getFirOpBuilder().create<mlir::omp::TargetOp>(
-      currentLocation, clauseOps);
-
-  genBodyOfTargetOp(converter, semaCtx, eval, genNested, targetOp, mapSyms,
-                    mapLocs, mapTypes, currentLocation);
-
-  return targetOp;
-}
-
-static mlir::omp::TeamsOp
-genTeamsOp(Fortran::lower::AbstractConverter &converter,
-           Fortran::semantics::SemanticsContext &semaCtx,
-           Fortran::lower::pft::Evaluation &eval, bool genNested,
-           mlir::Location currentLocation,
-           const Fortran::parser::OmpClauseList &clauseList,
-           bool outerCombined = false) {
-  Fortran::lower::StatementContext stmtCtx;
-  mlir::omp::TeamsClauseOps clauseOps;
-
-  ClauseProcessor cp(converter, semaCtx, clauseList);
-  cp.processIf(llvm::omp::Directive::OMPD_teams, clauseOps);
-  cp.processAllocate(clauseOps);
-  cp.processDefault();
-  cp.processNumTeams(stmtCtx, clauseOps);
-  cp.processThreadLimit(stmtCtx, clauseOps);
-  // TODO Support delayed privatization.
-
-  cp.processTODO<clause::Reduction>(currentLocation,
-                                    llvm::omp::Directive::OMPD_teams);
-
-  return genOpWithBody<mlir::omp::TeamsOp>(
-      OpWithBodyGenInfo(converter, semaCtx, currentLocation, eval)
-          .setGenNested(genNested)
-          .setOuterCombined(outerCombined)
-          .setClauses(&clauseList),
-      clauseOps);
-}
-
-/// Extract the list of function and variable symbols affected by the given
-/// 'declare target' directive and return the intended device type for them.
-static void getDeclareTargetInfo(
-    Fortran::lower::AbstractConverter &converter,
-    Fortran::semantics::SemanticsContext &semaCtx,
-    Fortran::lower::pft::Evaluation &eval,
-    const Fortran::parser::OpenMPDeclareTargetConstruct &declareTargetConstruct,
-    mlir::omp::DeclareTargetClauseOps &clauseOps,
-    llvm::SmallVectorImpl<DeclareTargetCapturePair> &symbolAndClause) {
-  const auto &spec = std::get<Fortran::parser::OmpDeclareTargetSpecifier>(
-      declareTargetConstruct.t);
-  if (const auto *objectList{
-          Fortran::parser::Unwrap<Fortran::parser::OmpObjectList>(spec.u)}) {
-    ObjectList objects{makeObjects(*objectList, semaCtx)};
-    // Case: declare target(func, var1, var2)
-    gatherFuncAndVarSyms(objects, mlir::omp::DeclareTargetCaptureClause::to,
-                         symbolAndClause);
-  } else if (const auto *clauseList{
-                 Fortran::parser::Unwrap<Fortran::parser::OmpClauseList>(
-                     spec.u)}) {
-    if (clauseList->v.empty()) {
-      // Case: declare target, implicit capture of function
-      symbolAndClause.emplace_back(
-          mlir::omp::DeclareTargetCaptureClause::to,
-          eval.getOwningProcedure()->getSubprogramSymbol());
-    }
-
-    ClauseProcessor cp(converter, semaCtx, *clauseList);
-    cp.processTo(symbolAndClause);
-    cp.processEnter(symbolAndClause);
-    cp.processLink(symbolAndClause);
-    cp.processDeviceType(clauseOps);
-    cp.processTODO<clause::Indirect>(converter.getCurrentLocation(),
-                                     llvm::omp::Directive::OMPD_declare_target);
-  }
-}
-
-static void collectDeferredDeclareTargets(
-    Fortran::lower::AbstractConverter &converter,
-    Fortran::semantics::SemanticsContext &semaCtx,
-    Fortran::lower::pft::Evaluation &eval,
-    const Fortran::parser::OpenMPDeclareTargetConstruct &declareTargetConstruct,
-    llvm::SmallVectorImpl<Fortran::lower::OMPDeferredDeclareTargetInfo>
-        &deferredDeclareTarget) {
-  mlir::omp::DeclareTargetClauseOps clauseOps;
-  llvm::SmallVector<DeclareTargetCapturePair> symbolAndClause;
-  getDeclareTargetInfo(converter, semaCtx, eval, declareTargetConstruct,
-                       clauseOps, symbolAndClause);
-  // Return the device type only if at least one of the targets for the
-  // directive is a function or subroutine
-  mlir::ModuleOp mod = converter.getFirOpBuilder().getModule();
-
-  for (const DeclareTargetCapturePair &symClause : symbolAndClause) {
-    mlir::Operation *op = mod.lookupSymbol(converter.mangleName(
-        std::get<const Fortran::semantics::Symbol &>(symClause)));
-
-    if (!op) {
-      deferredDeclareTarget.push_back({std::get<0>(symClause),
-                                       clauseOps.deviceType,
-                                       std::get<1>(symClause)});
-    }
-  }
-}
-
-static std::optional<mlir::omp::DeclareTargetDeviceType>
-getDeclareTargetFunctionDevice(
-    Fortran::lower::AbstractConverter &converter,
-    Fortran::semantics::SemanticsContext &semaCtx,
-    Fortran::lower::pft::Evaluation &eval,
-    const Fortran::parser::OpenMPDeclareTargetConstruct
-        &declareTargetConstruct) {
-  mlir::omp::DeclareTargetClauseOps clauseOps;
-  llvm::SmallVector<DeclareTargetCapturePair> symbolAndClause;
-  getDeclareTargetInfo(converter, semaCtx, eval, declareTargetConstruct,
-                       clauseOps, symbolAndClause);
-
-  // Return the device type only if at least one of the targets for the
-  // directive is a function or subroutine
-  mlir::ModuleOp mod = converter.getFirOpBuilder().getModule();
-  for (const DeclareTargetCapturePair &symClause : symbolAndClause) {
-    mlir::Operation *op = mod.lookupSymbol(converter.mangleName(
-        std::get<const Fortran::semantics::Symbol &>(symClause)));
-
-    if (mlir::isa_and_nonnull<mlir::func::FuncOp>(op))
-      return clauseOps.deviceType;
-  }
-
-  return std::nullopt;
-}
-
-//===----------------------------------------------------------------------===//
-// genOMP() Code generation helper functions
-//===----------------------------------------------------------------------===//
-
-static void
-genOmpSimpleStandalone(Fortran::lower::AbstractConverter &converter,
-                       Fortran::semantics::SemanticsContext &semaCtx,
-                       Fortran::lower::pft::Evaluation &eval, bool genNested,
-                       const Fortran::parser::OpenMPSimpleStandaloneConstruct
-                           &simpleStandaloneConstruct) {
-  const auto &directive =
-      std::get<Fortran::parser::OmpSimpleStandaloneDirective>(
-          simpleStandaloneConstruct.t);
-  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-  const auto &opClauseList =
-      std::get<Fortran::parser::OmpClauseList>(simpleStandaloneConstruct.t);
-  mlir::Location currentLocation = converter.genLocation(directive.source);
-
-  switch (directive.v) {
-  default:
-    break;
-  case llvm::omp::Directive::OMPD_barrier:
-    firOpBuilder.create<mlir::omp::BarrierOp>(currentLocation);
-    break;
-  case llvm::omp::Directive::OMPD_taskwait: {
-    mlir::omp::TaskwaitClauseOps clauseOps;
-    ClauseProcessor cp(converter, semaCtx, opClauseList);
-    cp.processTODO<clause::Depend, clause::Nowait>(
-        currentLocation, llvm::omp::Directive::OMPD_taskwait);
-    firOpBuilder.create<mlir::omp::TaskwaitOp>(currentLocation, clauseOps);
-    break;
-  }
-  case llvm::omp::Directive::OMPD_taskyield:
-    firOpBuilder.create<mlir::omp::TaskyieldOp>(currentLocation);
-    break;
-  case llvm::omp::Directive::OMPD_target_data:
-    genTargetDataOp(converter, semaCtx, eval, genNested, currentLocation,
-                    opClauseList);
-    break;
-  case llvm::omp::Directive::OMPD_target_enter_data:
-    genTargetEnterExitDataUpdateOp<mlir::omp::TargetEnterDataOp>(
-        converter, semaCtx, currentLocation, opClauseList);
-    break;
-  case llvm::omp::Directive::OMPD_target_exit_data:
-    genTargetEnterExitDataUpdateOp<mlir::omp::TargetExitDataOp>(
-        converter, semaCtx, currentLocation, opClauseList);
-    break;
-  case llvm::omp::Directive::OMPD_target_update:
-    genTargetEnterExitDataUpdateOp<mlir::omp::TargetUpdateOp>(
-        converter, semaCtx, currentLocation, opClauseList);
-    break;
-  case llvm::omp::Directive::OMPD_ordered:
-    TODO(currentLocation, "OMPD_ordered");
-  }
-}
-
-static void
-genOmpFlush(Fortran::lower::AbstractConverter &converter,
-            Fortran::semantics::SemanticsContext &semaCtx,
-            Fortran::lower::pft::Evaluation &eval,
-            const Fortran::parser::OpenMPFlushConstruct &flushConstruct) {
-  llvm::SmallVector<mlir::Value, 4> operandRange;
-  if (const auto &ompObjectList =
-          std::get<std::optional<Fortran::parser::OmpObjectList>>(
-              flushConstruct.t))
-    genObjectList2(*ompObjectList, converter, operandRange);
-  const auto &memOrderClause =
-      std::get<std::optional<std::list<Fortran::parser::OmpMemoryOrderClause>>>(
-          flushConstruct.t);
-  if (memOrderClause && memOrderClause->size() > 0)
-    TODO(converter.getCurrentLocation(), "Handle OmpMemoryOrderClause");
-  converter.getFirOpBuilder().create<mlir::omp::FlushOp>(
-      converter.getCurrentLocation(), operandRange);
-}
-
-static llvm::SmallVector<const Fortran::semantics::Symbol *>
-genLoopVars(mlir::Operation *op, Fortran::lower::AbstractConverter &converter,
-            mlir::Location &loc,
-            llvm::ArrayRef<const Fortran::semantics::Symbol *> args) {
-  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-  auto &region = op->getRegion(0);
-
-  std::size_t loopVarTypeSize = 0;
-  for (const Fortran::semantics::Symbol *arg : args)
-    loopVarTypeSize = std::max(loopVarTypeSize, arg->GetUltimate().size());
-  mlir::Type loopVarType = getLoopVarType(converter, loopVarTypeSize);
-  llvm::SmallVector<mlir::Type> tiv(args.size(), loopVarType);
-  llvm::SmallVector<mlir::Location> locs(args.size(), loc);
-  firOpBuilder.createBlock(&region, {}, tiv, locs);
-  // The argument is not currently in memory, so make a temporary for the
-  // argument, and store it there, then bind that location to the argument.
-  mlir::Operation *storeOp = nullptr;
-  for (auto [argIndex, argSymbol] : llvm::enumerate(args)) {
-    mlir::Value indexVal = fir::getBase(region.front().getArgument(argIndex));
-    storeOp =
-        createAndSetPrivatizedLoopVar(converter, loc, indexVal, argSymbol);
-  }
-  firOpBuilder.setInsertionPointAfter(storeOp);
-
-  return llvm::SmallVector<const Fortran::semantics::Symbol *>(args);
-}
-
-static llvm::SmallVector<const Fortran::semantics::Symbol *>
-genLoopAndReductionVars(
-    mlir::Operation *op, Fortran::lower::AbstractConverter &converter,
-    mlir::Location &loc,
-    llvm::ArrayRef<const Fortran::semantics::Symbol *> loopArgs,
-    llvm::ArrayRef<const Fortran::semantics::Symbol *> reductionArgs,
-    llvm::ArrayRef<mlir::Type> reductionTypes) {
-  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-
-  llvm::SmallVector<mlir::Type> blockArgTypes;
-  llvm::SmallVector<mlir::Location> blockArgLocs;
-  blockArgTypes.reserve(loopArgs.size() + reductionArgs.size());
-  blockArgLocs.reserve(blockArgTypes.size());
-  mlir::Block *entryBlock;
-
-  if (loopArgs.size()) {
-    std::size_t loopVarTypeSize = 0;
-    for (const Fortran::semantics::Symbol *arg : loopArgs)
-      loopVarTypeSize = std::max(loopVarTypeSize, arg->GetUltimate().size());
-    mlir::Type loopVarType = getLoopVarType(converter, loopVarTypeSize);
-    std::fill_n(std::back_inserter(blockArgTypes), loopArgs.size(),
-                loopVarType);
-    std::fill_n(std::back_inserter(blockArgLocs), loopArgs.size(), loc);
-  }
-  if (reductionArgs.size()) {
-    llvm::copy(reductionTypes, std::back_inserter(blockArgTypes));
-    std::fill_n(std::back_inserter(blockArgLocs), reductionArgs.size(), loc);
-  }
-  entryBlock = firOpBuilder.createBlock(&op->getRegion(0), {}, blockArgTypes,
-                                        blockArgLocs);
-  // The argument is not currently in memory, so make a temporary for the
-  // argument, and store it there, then bind that location to the argument.
-  if (loopArgs.size()) {
-    mlir::Operation *storeOp = nullptr;
-    for (auto [argIndex, argSymbol] : llvm::enumerate(loopArgs)) {
-      mlir::Value indexVal =
-          fir::getBase(op->getRegion(0).front().getArgument(argIndex));
-      storeOp =
-          createAndSetPrivatizedLoopVar(converter, loc, indexVal, argSymbol);
-    }
-    firOpBuilder.setInsertionPointAfter(storeOp);
-  }
-  // Bind the reduction arguments to their block arguments
-  for (auto [arg, prv] : llvm::zip_equal(
-           reductionArgs,
-           llvm::drop_begin(entryBlock->getArguments(), loopArgs.size()))) {
-    converter.bindSymbol(*arg, prv);
-  }
+  };
+  Fortran::lower::pft::visitAllSymbols(eval, captureImplicitMap);
 
-  return llvm::SmallVector<const Fortran::semantics::Symbol *>(loopArgs);
+  auto targetOp = firOpBuilder.create<mlir::omp::TargetOp>(loc, clauseOps);
+  genBodyOfTargetOp(converter, semaCtx, eval, genNested, targetOp, mapSyms,
+                    mapLocs, mapTypes, loc);
+  return targetOp;
 }
 
-static void
-createSimdLoop(Fortran::lower::AbstractConverter &converter,
-               Fortran::semantics::SemanticsContext &semaCtx,
-               Fortran::lower::pft::Evaluation &eval,
-               llvm::omp::Directive ompDirective,
-               const Fortran::parser::OmpClauseList &loopOpClauseList,
-               mlir::Location loc) {
+static mlir::omp::TargetDataOp
+genTargetDataOp(Fortran::lower::AbstractConverter &converter,
+                Fortran::semantics::SemanticsContext &semaCtx,
+                Fortran::lower::pft::Evaluation &eval, bool genNested,
+                mlir::Location loc,
+                const Fortran::parser::OmpClauseList &clauseList) {
+  Fortran::lower::StatementContext stmtCtx;
+  mlir::omp::TargetDataClauseOps clauseOps;
+  llvm::SmallVector<mlir::Type> useDeviceTypes;
+  llvm::SmallVector<mlir::Location> useDeviceLocs;
+  llvm::SmallVector<const Fortran::semantics::Symbol *> useDeviceSyms;
+  genTargetDataClauses(converter, semaCtx, stmtCtx, clauseList, loc, clauseOps,
+                       useDeviceTypes, useDeviceLocs, useDeviceSyms);
+
+  auto targetDataOp =
+      converter.getFirOpBuilder().create<mlir::omp::TargetDataOp>(loc,
+                                                                  clauseOps);
+  genBodyOfTargetDataOp(converter, semaCtx, eval, genNested, targetDataOp,
+                        useDeviceTypes, useDeviceLocs, useDeviceSyms, loc);
+  return targetDataOp;
+}
+
+template <typename OpTy>
+static OpTy genTargetEnterExitUpdateDataOp(
+    Fortran::lower::AbstractConverter &converter,
+    Fortran::semantics::SemanticsContext &semaCtx, mlir::Location loc,
+    const Fortran::parser::OmpClauseList &clauseList) {
   fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-  DataSharingProcessor dsp(converter, semaCtx, loopOpClauseList, eval);
-  dsp.processStep1();
+  Fortran::lower::StatementContext stmtCtx;
+
+  // GCC 9.3.0 emits a (probably) bogus warning about an unused variable.
+  [[maybe_unused]] llvm::omp::Directive directive;
+  if constexpr (std::is_same_v<OpTy, mlir::omp::TargetEnterDataOp>) {
+    directive = llvm::omp::Directive::OMPD_target_enter_data;
+  } else if constexpr (std::is_same_v<OpTy, mlir::omp::TargetExitDataOp>) {
+    directive = llvm::omp::Directive::OMPD_target_exit_data;
+  } else if constexpr (std::is_same_v<OpTy, mlir::omp::TargetUpdateOp>) {
+    directive = llvm::omp::Directive::OMPD_target_update;
+  } else {
+    llvm_unreachable("Unexpected TARGET DATA construct");
+  }
+
+  mlir::omp::TargetEnterExitUpdateDataClauseOps clauseOps;
+  genTargetEnterExitUpdateDataClauses(converter, semaCtx, stmtCtx, clauseList,
+                                      loc, directive, clauseOps);
+
+  return firOpBuilder.create<OpTy>(loc, clauseOps);
+}
 
+static mlir::omp::TaskOp
+genTaskOp(Fortran::lower::AbstractConverter &converter,
+          Fortran::semantics::SemanticsContext &semaCtx,
+          Fortran::lower::pft::Evaluation &eval, bool genNested,
+          mlir::Location loc,
+          const Fortran::parser::OmpClauseList &clauseList) {
   Fortran::lower::StatementContext stmtCtx;
-  mlir::omp::SimdLoopClauseOps clauseOps;
-  llvm::SmallVector<const Fortran::semantics::Symbol *> iv;
+  mlir::omp::TaskClauseOps clauseOps;
+  genTaskClauses(converter, semaCtx, stmtCtx, clauseList, loc, clauseOps);
 
-  ClauseProcessor cp(converter, semaCtx, loopOpClauseList);
-  cp.processCollapse(loc, eval, clauseOps, iv);
-  cp.processReduction(loc, clauseOps);
-  cp.processIf(llvm::omp::Directive::OMPD_simd, clauseOps);
-  cp.processSimdlen(clauseOps);
-  cp.processSafelen(clauseOps);
-  clauseOps.loopInclusiveAttr = firOpBuilder.getUnitAttr();
-  // TODO Support delayed privatization.
+  return genOpWithBody<mlir::omp::TaskOp>(
+      OpWithBodyGenInfo(converter, semaCtx, loc, eval)
+          .setGenNested(genNested)
+          .setClauses(&clauseList),
+      clauseOps);
+}
 
-  cp.processTODO<clause::Aligned, clause::Allocate, clause::Linear,
-                 clause::Nontemporal, clause::Order>(loc, ompDirective);
+static mlir::omp::TaskgroupOp
+genTaskgroupOp(Fortran::lower::AbstractConverter &converter,
+               Fortran::semantics::SemanticsContext &semaCtx,
+               Fortran::lower::pft::Evaluation &eval, bool genNested,
+               mlir::Location loc,
+               const Fortran::parser::OmpClauseList &clauseList) {
+  mlir::omp::TaskgroupClauseOps clauseOps;
+  genTaskgroupClauses(converter, semaCtx, clauseList, loc, clauseOps);
 
-  auto *nestedEval = getCollapsedLoopEval(
-      eval, Fortran::lower::getCollapseValue(loopOpClauseList));
+  return genOpWithBody<mlir::omp::TaskgroupOp>(
+      OpWithBodyGenInfo(converter, semaCtx, loc, eval)
+          .setGenNested(genNested)
+          .setClauses(&clauseList),
+      clauseOps);
+}
 
-  auto ivCallback = [&](mlir::Operation *op) {
-    return genLoopVars(op, converter, loc, iv);
-  };
+static mlir::omp::TaskloopOp
+genTaskloopOp(Fortran::lower::AbstractConverter &converter,
+              Fortran::semantics::SemanticsContext &semaCtx,
+              Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+              const Fortran::parser::OmpClauseList &clauseList) {
+  TODO(loc, "Taskloop construct");
+}
 
-  genOpWithBody<mlir::omp::SimdLoopOp>(
-      OpWithBodyGenInfo(converter, semaCtx, loc, *nestedEval)
-          .setClauses(&loopOpClauseList)
-          .setDataSharingProcessor(&dsp)
-          .setGenRegionEntryCb(ivCallback),
+static mlir::omp::TaskwaitOp
+genTaskwaitOp(Fortran::lower::AbstractConverter &converter,
+              Fortran::semantics::SemanticsContext &semaCtx,
+              Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+              const Fortran::parser::OmpClauseList &clauseList) {
+  mlir::omp::TaskwaitClauseOps clauseOps;
+  genTaskwaitClauses(converter, semaCtx, clauseList, loc, clauseOps);
+  return converter.getFirOpBuilder().create<mlir::omp::TaskwaitOp>(loc,
+                                                                   clauseOps);
+}
+
+static mlir::omp::TaskyieldOp
+genTaskyieldOp(Fortran::lower::AbstractConverter &converter,
+               Fortran::semantics::SemanticsContext &semaCtx,
+               Fortran::lower::pft::Evaluation &eval, mlir::Location loc) {
+  return converter.getFirOpBuilder().create<mlir::omp::TaskyieldOp>(loc);
+}
+
+static mlir::omp::TeamsOp
+genTeamsOp(Fortran::lower::AbstractConverter &converter,
+           Fortran::semantics::SemanticsContext &semaCtx,
+           Fortran::lower::pft::Evaluation &eval, bool genNested,
+           mlir::Location loc, const Fortran::parser::OmpClauseList &clauseList,
+           bool outerCombined = false) {
+  Fortran::lower::StatementContext stmtCtx;
+  mlir::omp::TeamsClauseOps clauseOps;
+  genTeamsClauses(converter, semaCtx, stmtCtx, clauseList, loc, clauseOps);
+
+  return genOpWithBody<mlir::omp::TeamsOp>(
+      OpWithBodyGenInfo(converter, semaCtx, loc, eval)
+          .setGenNested(genNested)
+          .setOuterCombined(outerCombined)
+          .setClauses(&clauseList),
       clauseOps);
 }
 
-static void createWsloop(Fortran::lower::AbstractConverter &converter,
-                         Fortran::semantics::SemanticsContext &semaCtx,
-                         Fortran::lower::pft::Evaluation &eval,
-                         llvm::omp::Directive ompDirective,
-                         const Fortran::parser::OmpClauseList &beginClauseList,
-                         const Fortran::parser::OmpClauseList *endClauseList,
-                         mlir::Location loc) {
-  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+static mlir::omp::WsloopOp
+genWsloopOp(Fortran::lower::AbstractConverter &converter,
+            Fortran::semantics::SemanticsContext &semaCtx,
+            Fortran::lower::pft::Evaluation &eval, mlir::Location loc,
+            const Fortran::parser::OmpClauseList &beginClauseList,
+            const Fortran::parser::OmpClauseList *endClauseList) {
   DataSharingProcessor dsp(converter, semaCtx, beginClauseList, eval);
   dsp.processStep1();
 
@@ -1461,30 +1725,9 @@ static void createWsloop(Fortran::lower::AbstractConverter &converter,
   llvm::SmallVector<const Fortran::semantics::Symbol *> iv;
   llvm::SmallVector<mlir::Type> reductionTypes;
   llvm::SmallVector<const Fortran::semantics::Symbol *> reductionSyms;
-
-  ClauseProcessor cp(converter, semaCtx, beginClauseList);
-  cp.processCollapse(loc, eval, clauseOps, iv);
-  cp.processSchedule(stmtCtx, clauseOps);
-  cp.processReduction(loc, clauseOps, &reductionTypes, &reductionSyms);
-  cp.processOrdered(clauseOps);
-  clauseOps.loopInclusiveAttr = firOpBuilder.getUnitAttr();
-  // TODO Support delayed privatization.
-
-  if (ReductionProcessor::doReductionByRef(clauseOps.reductionVars))
-    clauseOps.reductionByRefAttr = firOpBuilder.getUnitAttr();
-
-  cp.processTODO<clause::Allocate, clause::Linear, clause::Order>(loc,
-                                                                  ompDirective);
-
-  // In FORTRAN `nowait` clause occur at the end of `omp do` directive.
-  // i.e
-  // !$omp do
-  // <...>
-  // !$omp end do nowait
-  if (endClauseList) {
-    ClauseProcessor ecp(converter, semaCtx, *endClauseList);
-    ecp.processNowait(clauseOps);
-  }
+  genWsloopClauses(converter, semaCtx, stmtCtx, eval, beginClauseList,
+                   endClauseList, loc, clauseOps, iv, reductionTypes,
+                   reductionSyms);
 
   auto *nestedEval = getCollapsedLoopEval(
       eval, Fortran::lower::getCollapseValue(beginClauseList));
@@ -1494,7 +1737,7 @@ static void createWsloop(Fortran::lower::AbstractConverter &converter,
                                    reductionTypes);
   };
 
-  genOpWithBody<mlir::omp::WsloopOp>(
+  return genOpWithBody<mlir::omp::WsloopOp>(
       OpWithBodyGenInfo(converter, semaCtx, loc, *nestedEval)
           .setClauses(&beginClauseList)
           .setDataSharingProcessor(&dsp)
@@ -1503,7 +1746,11 @@ static void createWsloop(Fortran::lower::AbstractConverter &converter,
       clauseOps);
 }
 
-static void createSimdWsloop(
+//===----------------------------------------------------------------------===//
+// Code generation functions for composite constructs
+//===----------------------------------------------------------------------===//
+
+static void genCompositeDoSimd(
     Fortran::lower::AbstractConverter &converter,
     Fortran::semantics::SemanticsContext &semaCtx,
     Fortran::lower::pft::Evaluation &eval, llvm::omp::Directive ompDirective,
@@ -1511,7 +1758,7 @@ static void createSimdWsloop(
     const Fortran::parser::OmpClauseList *endClauseList, mlir::Location loc) {
   ClauseProcessor cp(converter, semaCtx, beginClauseList);
   cp.processTODO<clause::Aligned, clause::Allocate, clause::Linear,
-                 clause::Safelen, clause::Simdlen, clause::Order>(loc,
+                 clause::Order, clause::Safelen, clause::Simdlen>(loc,
                                                                   ompDirective);
   // TODO: Add support for vectorization - add vectorization hints inside loop
   // body.
@@ -1521,34 +1768,7 @@ static void createSimdWsloop(
   // When support for vectorization is enabled, then we need to add handling of
   // if clause. Currently if clause can be skipped because we always assume
   // SIMD length = 1.
-  createWsloop(converter, semaCtx, eval, ompDirective, beginClauseList,
-               endClauseList, loc);
-}
-
-static void
-markDeclareTarget(mlir::Operation *op,
-                  Fortran::lower::AbstractConverter &converter,
-                  mlir::omp::DeclareTargetCaptureClause captureClause,
-                  mlir::omp::DeclareTargetDeviceType deviceType) {
-  // TODO: Add support for program local variables with declare target applied
-  auto declareTargetOp = llvm::dyn_cast<mlir::omp::DeclareTargetInterface>(op);
-  if (!declareTargetOp)
-    fir::emitFatalError(
-        converter.getCurrentLocation(),
-        "Attempt to apply declare target on unsupported operation");
-
-  // The function or global already has a declare target applied to it, very
-  // likely through implicit capture (usage in another declare target
-  // function/subroutine). It should be marked as any if it has been assigned
-  // both host and nohost, else we skip, as there is no change
-  if (declareTargetOp.isDeclareTarget()) {
-    if (declareTargetOp.getDeclareTargetDeviceType() != deviceType)
-      declareTargetOp.setDeclareTarget(mlir::omp::DeclareTargetDeviceType::any,
-                                       captureClause);
-    return;
-  }
-
-  declareTargetOp.setDeclareTarget(deviceType, captureClause);
+  genWsloopOp(converter, semaCtx, eval, loc, beginClauseList, endClauseList);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1643,6 +1863,102 @@ genOMP(Fortran::lower::AbstractConverter &converter,
       ompDeclConstruct.u);
 }
 
+//===----------------------------------------------------------------------===//
+// OpenMPStandaloneConstruct visitors
+//===----------------------------------------------------------------------===//
+
+static void genOMP(Fortran::lower::AbstractConverter &converter,
+                   Fortran::lower::SymMap &symTable,
+                   Fortran::semantics::SemanticsContext &semaCtx,
+                   Fortran::lower::pft::Evaluation &eval,
+                   const Fortran::parser::OpenMPSimpleStandaloneConstruct
+                       &simpleStandaloneConstruct) {
+  const auto &directive =
+      std::get<Fortran::parser::OmpSimpleStandaloneDirective>(
+          simpleStandaloneConstruct.t);
+  const auto &clauseList =
+      std::get<Fortran::parser::OmpClauseList>(simpleStandaloneConstruct.t);
+  mlir::Location currentLocation = converter.genLocation(directive.source);
+
+  switch (directive.v) {
+  default:
+    break;
+  case llvm::omp::Directive::OMPD_barrier:
+    genBarrierOp(converter, semaCtx, eval, currentLocation);
+    break;
+  case llvm::omp::Directive::OMPD_taskwait:
+    genTaskwaitOp(converter, semaCtx, eval, currentLocation, clauseList);
+    break;
+  case llvm::omp::Directive::OMPD_taskyield:
+    genTaskyieldOp(converter, semaCtx, eval, currentLocation);
+    break;
+  case llvm::omp::Directive::OMPD_target_data:
+    genTargetDataOp(converter, semaCtx, eval, /*genNested=*/true,
+                    currentLocation, clauseList);
+    break;
+  case llvm::omp::Directive::OMPD_target_enter_data:
+    genTargetEnterExitUpdateDataOp<mlir::omp::TargetEnterDataOp>(
+        converter, semaCtx, currentLocation, clauseList);
+    break;
+  case llvm::omp::Directive::OMPD_target_exit_data:
+    genTargetEnterExitUpdateDataOp<mlir::omp::TargetExitDataOp>(
+        converter, semaCtx, currentLocation, clauseList);
+    break;
+  case llvm::omp::Directive::OMPD_target_update:
+    genTargetEnterExitUpdateDataOp<mlir::omp::TargetUpdateOp>(
+        converter, semaCtx, currentLocation, clauseList);
+    break;
+  case llvm::omp::Directive::OMPD_ordered:
+    genOrderedOp(converter, semaCtx, eval, currentLocation, clauseList);
+    break;
+  }
+}
+
+static void
+genOMP(Fortran::lower::AbstractConverter &converter,
+       Fortran::lower::SymMap &symTable,
+       Fortran::semantics::SemanticsContext &semaCtx,
+       Fortran::lower::pft::Evaluation &eval,
+       const Fortran::parser::OpenMPFlushConstruct &flushConstruct) {
+  const auto &verbatim = std::get<Fortran::parser::Verbatim>(flushConstruct.t);
+  const auto &objectList =
+      std::get<std::optional<Fortran::parser::OmpObjectList>>(flushConstruct.t);
+  const auto &clauseList =
+      std::get<std::optional<std::list<Fortran::parser::OmpMemoryOrderClause>>>(
+          flushConstruct.t);
+  mlir::Location currentLocation = converter.genLocation(verbatim.source);
+  genFlushOp(converter, semaCtx, eval, currentLocation, objectList, clauseList);
+}
+
+static void
+genOMP(Fortran::lower::AbstractConverter &converter,
+       Fortran::lower::SymMap &symTable,
+       Fortran::semantics::SemanticsContext &semaCtx,
+       Fortran::lower::pft::Evaluation &eval,
+       const Fortran::parser::OpenMPCancelConstruct &cancelConstruct) {
+  TODO(converter.getCurrentLocation(), "OpenMPCancelConstruct");
+}
+
+static void genOMP(Fortran::lower::AbstractConverter &converter,
+                   Fortran::lower::SymMap &symTable,
+                   Fortran::semantics::SemanticsContext &semaCtx,
+                   Fortran::lower::pft::Evaluation &eval,
+                   const Fortran::parser::OpenMPCancellationPointConstruct
+                       &cancellationPointConstruct) {
+  TODO(converter.getCurrentLocation(), "OpenMPCancelConstruct");
+}
+
+static void
+genOMP(Fortran::lower::AbstractConverter &converter,
+       Fortran::lower::SymMap &symTable,
+       Fortran::semantics::SemanticsContext &semaCtx,
+       Fortran::lower::pft::Evaluation &eval,
+       const Fortran::parser::OpenMPStandaloneConstruct &standaloneConstruct) {
+  std::visit(
+      [&](auto &&s) { return genOMP(converter, symTable, semaCtx, eval, s); },
+      standaloneConstruct.u);
+}
+
 //===----------------------------------------------------------------------===//
 // OpenMPConstruct visitors
 //===----------------------------------------------------------------------===//
@@ -1774,7 +2090,7 @@ genOMP(Fortran::lower::AbstractConverter &converter,
     break;
   case llvm::omp::Directive::OMPD_target:
     genTargetOp(converter, semaCtx, eval, /*genNested=*/true, currentLocation,
-                beginClauseList, directive.v);
+                beginClauseList);
     break;
   case llvm::omp::Directive::OMPD_target_data:
     genTargetDataOp(converter, semaCtx, eval, /*genNested=*/true,
@@ -1790,8 +2106,7 @@ genOMP(Fortran::lower::AbstractConverter &converter,
     break;
   case llvm::omp::Directive::OMPD_teams:
     genTeamsOp(converter, semaCtx, eval, /*genNested=*/true, currentLocation,
-               beginClauseList,
-               /*outerCombined=*/false);
+               beginClauseList);
     break;
   case llvm::omp::Directive::OMPD_workshare:
     // FIXME: Workshare is not a commonly used OpenMP construct, an
@@ -1813,8 +2128,7 @@ genOMP(Fortran::lower::AbstractConverter &converter,
   if ((llvm::omp::allTargetSet & llvm::omp::blockConstructSet)
           .test(directive.v)) {
     genTargetOp(converter, semaCtx, eval, /*genNested=*/false, currentLocation,
-                beginClauseList, directive.v,
-                /*outerCombined=*/true);
+                beginClauseList, /*outerCombined=*/true);
     combinedDirective = true;
   }
   if ((llvm::omp::allTeamsSet & llvm::omp::blockConstructSet)
@@ -1851,44 +2165,13 @@ genOMP(Fortran::lower::AbstractConverter &converter,
        Fortran::semantics::SemanticsContext &semaCtx,
        Fortran::lower::pft::Evaluation &eval,
        const Fortran::parser::OpenMPCriticalConstruct &criticalConstruct) {
-  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
-  mlir::Location currentLocation = converter.getCurrentLocation();
-  std::string name;
-  const Fortran::parser::OmpCriticalDirective &cd =
+  const auto &cd =
       std::get<Fortran::parser::OmpCriticalDirective>(criticalConstruct.t);
-  if (std::get<std::optional<Fortran::parser::Name>>(cd.t).has_value()) {
-    name =
-        std::get<std::optional<Fortran::parser::Name>>(cd.t).value().ToString();
-  }
-
-  mlir::omp::CriticalOp criticalOp = [&]() {
-    if (name.empty()) {
-      return firOpBuilder.create<mlir::omp::CriticalOp>(
-          currentLocation, mlir::FlatSymbolRefAttr());
-    }
-
-    mlir::ModuleOp module = firOpBuilder.getModule();
-    mlir::OpBuilder modBuilder(module.getBodyRegion());
-    auto global = module.lookupSymbol<mlir::omp::CriticalDeclareOp>(name);
-    if (!global) {
-      mlir::omp::CriticalClauseOps clauseOps;
-      const auto &clauseList = std::get<Fortran::parser::OmpClauseList>(cd.t);
-
-      ClauseProcessor cp(converter, semaCtx, clauseList);
-      cp.processHint(clauseOps);
-      clauseOps.nameAttr =
-          mlir::StringAttr::get(firOpBuilder.getContext(), name);
-
-      global = modBuilder.create<mlir::omp::CriticalDeclareOp>(currentLocation,
-                                                               clauseOps);
-    }
-
-    return firOpBuilder.create<mlir::omp::CriticalOp>(
-        currentLocation, mlir::FlatSymbolRefAttr::get(firOpBuilder.getContext(),
-                                                      global.getSymName()));
-  }();
-  auto genInfo = OpWithBodyGenInfo(converter, semaCtx, currentLocation, eval);
-  createBodyOfOp<mlir::omp::CriticalOp>(criticalOp, genInfo);
+  const auto &clauseList = std::get<Fortran::parser::OmpClauseList>(cd.t);
+  const auto &name = std::get<std::optional<Fortran::parser::Name>>(cd.t);
+  mlir::Location currentLocation = converter.getCurrentLocation();
+  genCriticalOp(converter, semaCtx, eval, /*genNested=*/true, currentLocation,
+                clauseList, name);
 }
 
 static void
@@ -1907,7 +2190,7 @@ static void genOMP(Fortran::lower::AbstractConverter &converter,
                    const Fortran::parser::OpenMPLoopConstruct &loopConstruct) {
   const auto &beginLoopDirective =
       std::get<Fortran::parser::OmpBeginLoopDirective>(loopConstruct.t);
-  const auto &loopOpClauseList =
+  const auto &beginClauseList =
       std::get<Fortran::parser::OmpClauseList>(beginLoopDirective.t);
   mlir::Location currentLocation =
       converter.genLocation(beginLoopDirective.source);
@@ -1928,33 +2211,31 @@ static void genOMP(Fortran::lower::AbstractConverter &converter,
   bool validDirective = false;
   if (llvm::omp::topTaskloopSet.test(ompDirective)) {
     validDirective = true;
-    TODO(currentLocation, "Taskloop construct");
+    genTaskloopOp(converter, semaCtx, eval, currentLocation, beginClauseList);
   } else {
     // Create omp.{target, teams, distribute, parallel} nested operations
     if ((llvm::omp::allTargetSet & llvm::omp::loopConstructSet)
             .test(ompDirective)) {
       validDirective = true;
       genTargetOp(converter, semaCtx, eval, /*genNested=*/false,
-                  currentLocation, loopOpClauseList, ompDirective,
-                  /*outerCombined=*/true);
+                  currentLocation, beginClauseList, /*outerCombined=*/true);
     }
     if ((llvm::omp::allTeamsSet & llvm::omp::loopConstructSet)
             .test(ompDirective)) {
       validDirective = true;
       genTeamsOp(converter, semaCtx, eval, /*genNested=*/false, currentLocation,
-                 loopOpClauseList,
-                 /*outerCombined=*/true);
+                 beginClauseList, /*outerCombined=*/true);
     }
     if (llvm::omp::allDistributeSet.test(ompDirective)) {
       validDirective = true;
-      TODO(currentLocation, "Distribute construct");
+      genDistributeOp(converter, semaCtx, eval, /*genNested=*/false,
+                      currentLocation, beginClauseList);
     }
     if ((llvm::omp::allParallelSet & llvm::omp::loopConstructSet)
             .test(ompDirective)) {
       validDirective = true;
       genParallelOp(converter, symTable, semaCtx, eval, /*genNested=*/false,
-                    currentLocation, loopOpClauseList,
-                    /*outerCombined=*/true);
+                    currentLocation, beginClauseList, /*outerCombined=*/true);
     }
   }
   if ((llvm::omp::allDoSet | llvm::omp::allSimdSet).test(ompDirective))
@@ -1968,16 +2249,14 @@ static void genOMP(Fortran::lower::AbstractConverter &converter,
 
   if (llvm::omp::allDoSimdSet.test(ompDirective)) {
     // 2.9.3.2 Workshare SIMD construct
-    createSimdWsloop(converter, semaCtx, eval, ompDirective, loopOpClauseList,
-                     endClauseList, currentLocation);
-
+    genCompositeDoSimd(converter, semaCtx, eval, ompDirective, beginClauseList,
+                       endClauseList, currentLocation);
   } else if (llvm::omp::allSimdSet.test(ompDirective)) {
     // 2.9.3.1 SIMD construct
-    createSimdLoop(converter, semaCtx, eval, ompDirective, loopOpClauseList,
-                   currentLocation);
+    genSimdLoopOp(converter, semaCtx, eval, currentLocation, beginClauseList);
   } else {
-    createWsloop(converter, semaCtx, eval, ompDirective, loopOpClauseList,
-                 endClauseList, currentLocation);
+    genWsloopOp(converter, semaCtx, eval, currentLocation, beginClauseList,
+                endClauseList);
   }
 }
 
@@ -1997,44 +2276,39 @@ genOMP(Fortran::lower::AbstractConverter &converter,
        Fortran::semantics::SemanticsContext &semaCtx,
        Fortran::lower::pft::Evaluation &eval,
        const Fortran::parser::OpenMPSectionsConstruct &sectionsConstruct) {
-  mlir::Location currentLocation = converter.getCurrentLocation();
-  mlir::omp::SectionsClauseOps clauseOps;
   const auto &beginSectionsDirective =
       std::get<Fortran::parser::OmpBeginSectionsDirective>(sectionsConstruct.t);
-  const auto &sectionsClauseList =
+  const auto &beginClauseList =
       std::get<Fortran::parser::OmpClauseList>(beginSectionsDirective.t);
 
   // Process clauses before optional omp.parallel, so that new variables are
   // allocated outside of the parallel region
-  ClauseProcessor cp(converter, semaCtx, sectionsClauseList);
-  cp.processSectionsReduction(currentLocation, clauseOps);
-  cp.processAllocate(clauseOps);
-  // TODO Support delayed privatization.
+  mlir::Location currentLocation = converter.getCurrentLocation();
+  mlir::omp::SectionsClauseOps clauseOps;
+  genSectionsClauses(converter, semaCtx, beginClauseList, currentLocation,
+                     /*clausesFromBeginSections=*/true, clauseOps);
 
+  // Parallel wrapper of PARALLEL SECTIONS construct
   llvm::omp::Directive dir =
       std::get<Fortran::parser::OmpSectionsDirective>(beginSectionsDirective.t)
           .v;
-
-  // Parallel wrapper of PARALLEL SECTIONS construct
   if (dir == llvm::omp::Directive::OMPD_parallel_sections) {
     genParallelOp(converter, symTable, semaCtx, eval,
-                  /*genNested=*/false, currentLocation, sectionsClauseList,
+                  /*genNested=*/false, currentLocation, beginClauseList,
                   /*outerCombined=*/true);
   } else {
     const auto &endSectionsDirective =
         std::get<Fortran::parser::OmpEndSectionsDirective>(sectionsConstruct.t);
-    const auto &endSectionsClauseList =
+    const auto &endClauseList =
         std::get<Fortran::parser::OmpClauseList>(endSectionsDirective.t);
-    ClauseProcessor(converter, semaCtx, endSectionsClauseList)
-        .processNowait(clauseOps);
+    genSectionsClauses(converter, semaCtx, endClauseList, currentLocation,
+                       /*clausesFromBeginSections=*/false, clauseOps);
   }
 
-  // SECTIONS construct
-  genOpWithBody<mlir::omp::SectionsOp>(
-      OpWithBodyGenInfo(converter, semaCtx, currentLocation, eval)
-          .setGenNested(false),
-      clauseOps);
+  // SECTIONS construct.
+  genSectionsOp(converter, semaCtx, eval, currentLocation, clauseOps);
 
+  // Generate nested SECTION operations recursively.
   const auto &sectionBlocks =
       std::get<Fortran::parser::OmpSectionBlocks>(sectionsConstruct.t);
   auto &firOpBuilder = converter.getFirOpBuilder();
@@ -2043,40 +2317,12 @@ genOMP(Fortran::lower::AbstractConverter &converter,
        llvm::zip(sectionBlocks.v, eval.getNestedEvaluations())) {
     symTable.pushScope();
     genSectionOp(converter, semaCtx, neval, /*genNested=*/true, currentLocation,
-                 sectionsClauseList);
+                 beginClauseList);
     symTable.popScope();
     firOpBuilder.restoreInsertionPoint(ip);
   }
 }
 
-static void
-genOMP(Fortran::lower::AbstractConverter &converter,
-       Fortran::lower::SymMap &symTable,
-       Fortran::semantics::SemanticsContext &semaCtx,
-       Fortran::lower::pft::Evaluation &eval,
-       const Fortran::parser::OpenMPStandaloneConstruct &standaloneConstruct) {
-  std::visit(
-      Fortran::common::visitors{
-          [&](const Fortran::parser::OpenMPSimpleStandaloneConstruct
-                  &simpleStandaloneConstruct) {
-            genOmpSimpleStandalone(converter, semaCtx, eval,
-                                   /*genNested=*/true,
-                                   simpleStandaloneConstruct);
-          },
-          [&](const Fortran::parser::OpenMPFlushConstruct &flushConstruct) {
-            genOmpFlush(converter, semaCtx, eval, flushConstruct);
-          },
-          [&](const Fortran::parser::OpenMPCancelConstruct &cancelConstruct) {
-            TODO(converter.getCurrentLocation(), "OpenMPCancelConstruct");
-          },
-          [&](const Fortran::parser::OpenMPCancellationPointConstruct
-                  &cancellationPointConstruct) {
-            TODO(converter.getCurrentLocation(), "OpenMPCancelConstruct");
-          },
-      },
-      standaloneConstruct.u);
-}
-
 static void genOMP(Fortran::lower::AbstractConverter &converter,
                    Fortran::lower::SymMap &symTable,
                    Fortran::semantics::SemanticsContext &semaCtx,
diff --git a/flang/test/Lower/OpenMP/FIR/target.f90 b/flang/test/Lower/OpenMP/FIR/target.f90
index 022327f9c25d..ca3162340d78 100644
--- a/flang/test/Lower/OpenMP/FIR/target.f90
+++ b/flang/test/Lower/OpenMP/FIR/target.f90
@@ -411,8 +411,8 @@ end subroutine omp_target_implicit_bounds
 !CHECK-LABEL: func.func @_QPomp_target_thread_limit() {
 subroutine omp_target_thread_limit
    integer :: a
-   !CHECK: %[[VAL_1:.*]] = arith.constant 64 : i32
    !CHECK: %[[MAP:.*]] = omp.map.info var_ptr({{.*}})   map_clauses(tofrom) capture(ByRef) -> !fir.ref<i32> {name = "a"}
+   !CHECK: %[[VAL_1:.*]] = arith.constant 64 : i32
    !CHECK: omp.target   thread_limit(%[[VAL_1]] : i32) map_entries(%[[MAP]] -> %[[ARG_0:.*]] : !fir.ref<i32>) {
    !CHECK: ^bb0(%[[ARG_0]]: !fir.ref<i32>):
    !$omp target map(tofrom: a) thread_limit(64)
diff --git a/flang/test/Lower/OpenMP/target.f90 b/flang/test/Lower/OpenMP/target.f90
index 6f72b5a34d06..51b66327dfb2 100644
--- a/flang/test/Lower/OpenMP/target.f90
+++ b/flang/test/Lower/OpenMP/target.f90
@@ -490,8 +490,8 @@ end subroutine omp_target_implicit_bounds
 !CHECK-LABEL: func.func @_QPomp_target_thread_limit() {
 subroutine omp_target_thread_limit
    integer :: a
-   !CHECK: %[[VAL_1:.*]] = arith.constant 64 : i32
    !CHECK: %[[MAP:.*]] = omp.map.info var_ptr({{.*}})   map_clauses(tofrom) capture(ByRef) -> !fir.ref<i32> {name = "a"}
+   !CHECK: %[[VAL_1:.*]] = arith.constant 64 : i32
    !CHECK: omp.target   thread_limit(%[[VAL_1]] : i32) map_entries(%[[MAP]] -> %{{.*}} : !fir.ref<i32>) {
    !CHECK: ^bb0(%{{.*}}: !fir.ref<i32>):
    !$omp target map(tofrom: a) thread_limit(64)
diff --git a/flang/test/Lower/OpenMP/use-device-ptr-to-use-device-addr.f90 b/flang/test/Lower/OpenMP/use-device-ptr-to-use-device-addr.f90
index 33b597165601..d849dd206b94 100644
--- a/flang/test/Lower/OpenMP/use-device-ptr-to-use-device-addr.f90
+++ b/flang/test/Lower/OpenMP/use-device-ptr-to-use-device-addr.f90
@@ -21,7 +21,7 @@ end subroutine
 
 !CHECK: func.func @{{.*}}mix_use_device_ptr_and_addr()
 !CHECK: omp.target_data use_device_ptr({{.*}} : !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>) use_device_addr(%{{.*}}, %{{.*}} : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>) {
-!CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, %{{.*}}: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, %{{.*}}: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>):
+!CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, %{{.*}}: !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, %{{.*}}: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>):
 subroutine mix_use_device_ptr_and_addr 
     use iso_c_binding
     integer, pointer, dimension(:) :: array
@@ -47,7 +47,7 @@ end subroutine
 
 !CHECK: func.func @{{.*}}mix_use_device_ptr_and_addr_and_map()
 !CHECK: omp.target_data map_entries(%{{.*}}, %{{.*}} : !fir.ref<i32>, !fir.ref<i32>) use_device_ptr(%{{.*}} : !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>) use_device_addr(%{{.*}}, %{{.*}} : !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>) {
-!CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, %{{.*}}: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>, %{{.*}}: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>):
+!CHECK: ^bb0(%{{.*}}: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xf32>>>>, %{{.*}}: !fir.ref<!fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}>>, %{{.*}}: !fir.ref<!fir.box<!fir.ptr<!fir.array<?xi32>>>>):
 subroutine mix_use_device_ptr_and_addr_and_map
     use iso_c_binding
     integer :: i, j
-- 
GitLab


From 31424be3aef4290dd84065b9371fcd0c5014e097 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Mon, 15 Apr 2024 13:14:41 +0200
Subject: [PATCH 370/464] [clang][Interp][NFC] Compare std::optionals directly

---
 clang/lib/AST/Interp/ByteCodeExprGen.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
index 93059edc4622..4a7b40440770 100644
--- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp
+++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
@@ -1624,7 +1624,7 @@ bool ByteCodeExprGen<Emitter>::VisitCompoundAssignOperator(
     return false;
   if (!this->emitLoad(*LT, E))
     return false;
-  if (*LT != *LHSComputationT) {
+  if (LT != LHSComputationT) {
     if (!this->emitCast(*LT, *LHSComputationT, E))
       return false;
   }
@@ -1680,7 +1680,7 @@ bool ByteCodeExprGen<Emitter>::VisitCompoundAssignOperator(
   }
 
   // And now cast from LHSComputationT to ResultT.
-  if (*ResultT != *LHSComputationT) {
+  if (ResultT != LHSComputationT) {
     if (!this->emitCast(*LHSComputationT, *ResultT, E))
       return false;
   }
-- 
GitLab


From a831c54357c2bb7b8b457ccea22836c23e8b8625 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Mon, 15 Apr 2024 14:17:31 +0200
Subject: [PATCH 371/464] [clang][Interp] Avoid calling invalid functions

Check if the non-null function pointer is even valid before calling
the function.
---
 clang/lib/AST/Interp/FunctionPointer.h |  1 +
 clang/lib/AST/Interp/Interp.h          |  4 ++++
 clang/test/AST/Interp/functions.cpp    | 15 +++++++++++++--
 3 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/clang/lib/AST/Interp/FunctionPointer.h b/clang/lib/AST/Interp/FunctionPointer.h
index c2ea295b82bd..fc3d7a4214a7 100644
--- a/clang/lib/AST/Interp/FunctionPointer.h
+++ b/clang/lib/AST/Interp/FunctionPointer.h
@@ -32,6 +32,7 @@ public:
 
   const Function *getFunction() const { return Func; }
   bool isZero() const { return !Func; }
+  bool isValid() const { return Valid; }
   bool isWeak() const {
     if (!Func || !Valid)
       return false;
diff --git a/clang/lib/AST/Interp/Interp.h b/clang/lib/AST/Interp/Interp.h
index 4182254357eb..dd0bacd73acb 100644
--- a/clang/lib/AST/Interp/Interp.h
+++ b/clang/lib/AST/Interp/Interp.h
@@ -2236,6 +2236,10 @@ inline bool CallPtr(InterpState &S, CodePtr OpPC, uint32_t ArgSize,
         << const_cast<Expr *>(E) << E->getSourceRange();
     return false;
   }
+
+  if (!FuncPtr.isValid())
+    return false;
+
   assert(F);
 
   // Check argument nullability state.
diff --git a/clang/test/AST/Interp/functions.cpp b/clang/test/AST/Interp/functions.cpp
index 4fb3c816000a..f9bb5d53634e 100644
--- a/clang/test/AST/Interp/functions.cpp
+++ b/clang/test/AST/Interp/functions.cpp
@@ -584,9 +584,20 @@ namespace VariadicOperator {
 namespace WeakCompare {
   [[gnu::weak]]void weak_method();
   static_assert(weak_method != nullptr, ""); // both-error {{not an integral constant expression}} \
-                                         // both-note {{comparison against address of weak declaration '&weak_method' can only be performed at runtim}}
+                                             // both-note {{comparison against address of weak declaration '&weak_method' can only be performed at runtim}}
 
   constexpr auto A = &weak_method;
   static_assert(A != nullptr, ""); // both-error {{not an integral constant expression}} \
-                               // both-note {{comparison against address of weak declaration '&weak_method' can only be performed at runtim}}
+                                   // both-note {{comparison against address of weak declaration '&weak_method' can only be performed at runtim}}
+}
+
+namespace FromIntegral {
+#if __cplusplus >= 202002L
+  typedef double (*DoubleFn)();
+  int a[(int)DoubleFn((void*)-1)()]; // both-error {{not allowed at file scope}} \
+                                    // both-warning {{variable length arrays}}
+  int b[(int)DoubleFn((void*)(-1 + 1))()]; // both-error {{not allowed at file scope}} \
+                                           // expected-note {{evaluates to a null function pointer}} \
+                                           // both-warning {{variable length arrays}}
+#endif
 }
-- 
GitLab


From b0194d2894db49d7cf4d36aed87952c3e0c6a390 Mon Sep 17 00:00:00 2001
From: Phoebe Wang <phoebe.wang@intel.com>
Date: Tue, 16 Apr 2024 18:19:50 +0800
Subject: [PATCH 372/464] [SEH] Ignore async exception flag when the
 environment is not MSVC (#88101)

Fixes #62449
---
 clang/lib/Driver/ToolChains/Clang.cpp         | 28 +++++++++++++------
 .../test/Driver/windows-seh-async-verify.cpp  | 24 ++++++++++++++++
 2 files changed, 44 insertions(+), 8 deletions(-)
 create mode 100644 clang/test/Driver/windows-seh-async-verify.cpp

diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 6d52eced1042..096ed14f9570 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -346,11 +346,14 @@ static bool addExceptionArgs(const ArgList &Args, types::ID InputType,
   bool EH = Args.hasFlag(options::OPT_fexceptions, options::OPT_fno_exceptions,
                          false);
 
-  bool EHa = Args.hasFlag(options::OPT_fasync_exceptions,
-                          options::OPT_fno_async_exceptions, false);
-  if (EHa) {
-    CmdArgs.push_back("-fasync-exceptions");
-    EH = true;
+  // Async exceptions are Windows MSVC only.
+  if (Triple.isWindowsMSVCEnvironment()) {
+    bool EHa = Args.hasFlag(options::OPT_fasync_exceptions,
+                            options::OPT_fno_async_exceptions, false);
+    if (EHa) {
+      CmdArgs.push_back("-fasync-exceptions");
+      EH = true;
+    }
   }
 
   // Obj-C exceptions are enabled by default, regardless of -fexceptions. This
@@ -8102,7 +8105,8 @@ struct EHFlags {
 ///      The 'a' modifier is unimplemented and fundamentally hard in LLVM IR.
 /// - c: Assume that extern "C" functions are implicitly nounwind.
 /// The default is /EHs-c-, meaning cleanups are disabled.
-static EHFlags parseClangCLEHFlags(const Driver &D, const ArgList &Args) {
+static EHFlags parseClangCLEHFlags(const Driver &D, const ArgList &Args,
+                                   bool isWindowsMSVC) {
   EHFlags EH;
 
   std::vector<std::string> EHArgs =
@@ -8112,8 +8116,15 @@ static EHFlags parseClangCLEHFlags(const Driver &D, const ArgList &Args) {
       switch (EHVal[I]) {
       case 'a':
         EH.Asynch = maybeConsumeDash(EHVal, I);
-        if (EH.Asynch)
+        if (EH.Asynch) {
+          // Async exceptions are Windows MSVC only.
+          if (!isWindowsMSVC) {
+            EH.Asynch = false;
+            D.Diag(clang::diag::warn_drv_unused_argument) << "/EHa" << EHVal;
+            continue;
+          }
           EH.Synch = false;
+        }
         continue;
       case 'c':
         EH.NoUnwindC = maybeConsumeDash(EHVal, I);
@@ -8177,7 +8188,8 @@ void Clang::AddClangCLArgs(const ArgList &Args, types::ID InputType,
 
   const Driver &D = getToolChain().getDriver();
 
-  EHFlags EH = parseClangCLEHFlags(D, Args);
+  bool IsWindowsMSVC = getToolChain().getTriple().isWindowsMSVCEnvironment();
+  EHFlags EH = parseClangCLEHFlags(D, Args, IsWindowsMSVC);
   if (!isNVPTX && (EH.Synch || EH.Asynch)) {
     if (types::isCXX(InputType))
       CmdArgs.push_back("-fcxx-exceptions");
diff --git a/clang/test/Driver/windows-seh-async-verify.cpp b/clang/test/Driver/windows-seh-async-verify.cpp
new file mode 100644
index 000000000000..5fda6a77dba0
--- /dev/null
+++ b/clang/test/Driver/windows-seh-async-verify.cpp
@@ -0,0 +1,24 @@
+// RUN: %clang --target=x86_64-pc-windows -fasync-exceptions -fsyntax-only %s -### 2>&1 | FileCheck %s
+// RUN: %clang_cl --target=x86_64-pc-windows /EHa -fsyntax-only %s -### 2>&1 | FileCheck %s
+// RUN: %clang --target=x86_64-pc-windows-gnu -fasync-exceptions -fsyntax-only %s -### 2>&1 | FileCheck %s --check-prefixes=GNU-ALL,GNU
+// RUN: %clang_cl --target=x86_64-pc-windows-gnu /EHa -fsyntax-only %s -### 2>&1 | FileCheck %s --check-prefixes=GNU-ALL,CL-GNU
+
+// CHECK-NOT: warning
+// GNU: warning: argument unused during compilation: '-fasync-exceptions' [-Wunused-command-line-argument]
+// CL-GNU: warning: argument unused during compilation: '/EHa' [-Wunused-command-line-argument]
+
+// CHECK: -fasync-exceptions
+// GNU-ALL-NOT: -fasync-exceptions
+struct S {
+    union _Un {
+        ~_Un() {}
+        char _Buf[12];
+    };
+    _Un _un;
+};
+
+struct Embed {
+    S v2;
+};
+
+void PR62449() { Embed v{}; }
-- 
GitLab


From f4960da6023b8034ae68925c3223d51624621b37 Mon Sep 17 00:00:00 2001
From: Jon Chesterfield <jonathanchesterfield@gmail.com>
Date: Tue, 16 Apr 2024 11:24:45 +0100
Subject: [PATCH 373/464] Revert "[Verifier] Reject va_start in non-variadic
 function (#88809)"

This reverts commit 61717c1aa1f08eb57839a21fb2d9004739022e0d. Failed a MLIR test
---
 llvm/lib/Analysis/Lint.cpp                      | 5 ++++-
 llvm/lib/IR/Verifier.cpp                        | 5 -----
 llvm/test/CodeGen/AArch64/GlobalISel/vastart.ll | 2 +-
 llvm/test/Other/lint.ll                         | 7 +++++++
 llvm/test/Verifier/variadic.ll                  | 8 --------
 5 files changed, 12 insertions(+), 15 deletions(-)
 delete mode 100644 llvm/test/Verifier/variadic.ll

diff --git a/llvm/lib/Analysis/Lint.cpp b/llvm/lib/Analysis/Lint.cpp
index 1ab856ac8830..0694c2995dfc 100644
--- a/llvm/lib/Analysis/Lint.cpp
+++ b/llvm/lib/Analysis/Lint.cpp
@@ -350,7 +350,10 @@ void Lint::visitCallBase(CallBase &I) {
     }
 
     case Intrinsic::vastart:
-      // vastart in non-varargs function is rejected by the verifier
+      Check(I.getParent()->getParent()->isVarArg(),
+            "Undefined behavior: va_start called in a non-varargs function",
+            &I);
+
       visitMemoryReference(I, MemoryLocation::getForArgument(&I, 0, TLI),
                            std::nullopt, nullptr, MemRef::Read | MemRef::Write);
       break;
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 4cd61e6e531b..516d4a051556 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -5798,11 +5798,6 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
 
     break;
   }
-  case Intrinsic::vastart: {
-    Check(Call.getFunction()->isVarArg(),
-          "va_start called in a non-varargs function");
-    break;
-  }
   case Intrinsic::vector_reduce_and:
   case Intrinsic::vector_reduce_or:
   case Intrinsic::vector_reduce_xor:
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/vastart.ll b/llvm/test/CodeGen/AArch64/GlobalISel/vastart.ll
index 8c6e01d934c2..bd576d0f70e9 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/vastart.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/vastart.ll
@@ -3,7 +3,7 @@
 
 
 declare void @llvm.va_start(ptr)
-define void @test_va_start(ptr %list, ...) {
+define void @test_va_start(ptr %list) {
 ; CHECK-LABEL: name: test_va_start
 ; CHECK: [[LIST:%[0-9]+]]:_(p0) = COPY $x0
 ; CHECK-IOS: G_VASTART [[LIST]](p0) :: (store (s64) into %ir.list, align 1)
diff --git a/llvm/test/Other/lint.ll b/llvm/test/Other/lint.ll
index 6fd2d40cd2f2..6b31b31a78c9 100644
--- a/llvm/test/Other/lint.ll
+++ b/llvm/test/Other/lint.ll
@@ -124,6 +124,13 @@ define void @0() nounwind {
   ret void
 }
 
+; CHECK: va_start called in a non-varargs function
+declare void @llvm.va_start(ptr)
+define void @not_vararg(ptr %p) nounwind {
+  call void @llvm.va_start(ptr %p)
+  ret void
+}
+
 ; CHECK: Undefined behavior: Branch to non-blockaddress
 define void @use_indbr() {
   indirectbr ptr @foo, [label %block]
diff --git a/llvm/test/Verifier/variadic.ll b/llvm/test/Verifier/variadic.ll
deleted file mode 100644
index 55e4a4da0a92..000000000000
--- a/llvm/test/Verifier/variadic.ll
+++ /dev/null
@@ -1,8 +0,0 @@
-; RUN: not opt -S -passes=verify 2>&1 < %s | FileCheck %s
-
-; CHECK: va_start called in a non-varargs function
-declare void @llvm.va_start(ptr)
-define void @not_vararg(ptr %p) nounwind {
-  call void @llvm.va_start(ptr %p)
-  ret void
-}
-- 
GitLab


From 01f79899ba349a0200586c8d05f5e22cca2ced31 Mon Sep 17 00:00:00 2001
From: Jinyang He <hejinyang@loongson.cn>
Date: Tue, 16 Apr 2024 18:31:03 +0800
Subject: [PATCH 374/464] [LoongArch] Use R_LARCH_ALIGN with section symbol
 (#84741)

In LoongArch psABI v2.30, the R_LARCH_ALIGN requires symbol index to
support the third parameter of alignment directive. Create symbol for
each section is redundant because they have section symbol which can
also be used as symbol index. So use section symbol directly for
R_LARCH_ALIGN.
---
 lld/ELF/InputSection.cpp                      |  6 +++-
 lld/test/ELF/loongarch-relax-align-ldr.s      | 28 +++++++++++++++++++
 lld/test/ELF/loongarch-relax-emit-relocs.s    |  5 ++--
 .../MCTargetDesc/LoongArchAsmBackend.cpp      |  7 ++---
 .../MC/LoongArch/Relocations/relax-addsub.s   |  2 +-
 .../MC/LoongArch/Relocations/relax-align.s    | 14 ++++++----
 6 files changed, 47 insertions(+), 15 deletions(-)
 create mode 100644 lld/test/ELF/loongarch-relax-align-ldr.s

diff --git a/lld/ELF/InputSection.cpp b/lld/ELF/InputSection.cpp
index c8350652e65a..fa48552b8f7a 100644
--- a/lld/ELF/InputSection.cpp
+++ b/lld/ELF/InputSection.cpp
@@ -464,7 +464,11 @@ void InputSection::copyRelocations(uint8_t *buf,
         addend += sec->getFile<ELFT>()->mipsGp0;
       }
 
-      if (RelTy::IsRela)
+      if (config->emachine == EM_LOONGARCH && type == R_LARCH_ALIGN)
+        // LoongArch psABI v2.30, the R_LARCH_ALIGN requires symbol index.
+        // If it use the section symbol, the addend should not be changed.
+        p->r_addend = addend;
+      else if (RelTy::IsRela)
         p->r_addend = sym.getVA(addend) - section->getOutputSection()->addr;
       // For SHF_ALLOC sections relocated by REL, append a relocation to
       // sec->relocations so that relocateAlloc transitively called by
diff --git a/lld/test/ELF/loongarch-relax-align-ldr.s b/lld/test/ELF/loongarch-relax-align-ldr.s
new file mode 100644
index 000000000000..6534dc906cfd
--- /dev/null
+++ b/lld/test/ELF/loongarch-relax-align-ldr.s
@@ -0,0 +1,28 @@
+# REQUIRES: loongarch
+## Test `ld -r` not changes the addend of R_LARCH_ALIGN.
+
+# RUN: llvm-mc --filetype=obj --triple=loongarch64 --mattr=+relax %s -o %t.64.o
+# RUN: ld.lld -r %t.64.o %t.64.o -o %t.64.r
+# RUN: llvm-objdump -dr --no-show-raw-insn %t.64.r | FileCheck %s
+
+# CHECK:      <.text>:
+# CHECK-NEXT:   break 1
+# CHECK-NEXT:   nop
+# CHECK-NEXT:   {{0*}}04:  R_LARCH_ALIGN        .text+0x804
+# CHECK-NEXT:   nop
+# CHECK-NEXT:   nop
+# CHECK-NEXT:   break 2
+# CHECK-NEXT:   break 0
+# CHECK-NEXT:   break 0
+# CHECK-NEXT:   break 0
+# CHECK-NEXT:   break 1
+# CHECK-NEXT:   nop
+# CHECK-NEXT:   {{0*}}24:  R_LARCH_ALIGN        .text+0x804
+# CHECK-NEXT:   nop
+# CHECK-NEXT:   nop
+# CHECK-NEXT:   break 2
+
+.text
+break 1
+.p2align 4, , 8
+break 2
diff --git a/lld/test/ELF/loongarch-relax-emit-relocs.s b/lld/test/ELF/loongarch-relax-emit-relocs.s
index 581fce8c95ca..9007f8fcc114 100644
--- a/lld/test/ELF/loongarch-relax-emit-relocs.s
+++ b/lld/test/ELF/loongarch-relax-emit-relocs.s
@@ -25,7 +25,7 @@
 # CHECK-NEXT:     R_LARCH_PCALA_LO12 _start
 # CHECK-NEXT:     R_LARCH_RELAX *ABS*
 # CHECK-NEXT:   nop
-# CHECK-NEXT:     R_LARCH_ALIGN .Lla-relax-align0+0x4
+# CHECK-NEXT:     R_LARCH_ALIGN .text+0x4
 # CHECK-NEXT:   nop
 # CHECK-NEXT:   ret
 
@@ -37,11 +37,12 @@
 # CHECKR-NEXT:     R_LARCH_PCALA_LO12 _start
 # CHECKR-NEXT:     R_LARCH_RELAX *ABS*
 # CHECKR-NEXT:   nop
-# CHECKR-NEXT:     R_LARCH_ALIGN .Lla-relax-align0+0x4
+# CHECKR-NEXT:     R_LARCH_ALIGN .text+0x4
 # CHECKR-NEXT:   nop
 # CHECKR-NEXT:   nop
 # CHECKR-NEXT:   ret
 
+.text
 .global _start
 _start:
   la.pcrel $a0, _start
diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
index de492f2b1f0a..98f5014a34b1 100644
--- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
+++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp
@@ -226,11 +226,8 @@ bool LoongArchAsmBackend::shouldInsertFixupForCodeAlign(
       MCFixup::create(0, Dummy, MCFixupKind(LoongArch::fixup_loongarch_align));
   const MCSymbolRefExpr *MCSym = getSecToAlignSym()[Sec];
   if (MCSym == nullptr) {
-    // Create a symbol and make the value of symbol is zero.
-    MCSymbol *Sym = Ctx.createNamedTempSymbol("la-relax-align");
-    Sym->setFragment(&*Sec->getBeginSymbol()->getFragment());
-    Asm.registerSymbol(*Sym);
-    MCSym = MCSymbolRefExpr::create(Sym, Ctx);
+    // Use section symbol directly.
+    MCSym = MCSymbolRefExpr::create(Sec->getBeginSymbol(), Ctx);
     getSecToAlignSym()[Sec] = MCSym;
   }
 
diff --git a/llvm/test/MC/LoongArch/Relocations/relax-addsub.s b/llvm/test/MC/LoongArch/Relocations/relax-addsub.s
index 18e0ede5e293..0e27d6301bb3 100644
--- a/llvm/test/MC/LoongArch/Relocations/relax-addsub.s
+++ b/llvm/test/MC/LoongArch/Relocations/relax-addsub.s
@@ -28,7 +28,7 @@
 
 # RELAX:       Relocations [
 # RELAX-NEXT:    Section ({{.*}}) .rela.text {
-# RELAX-NEXT:      0x4 R_LARCH_ALIGN {{.*}} 0x4
+# RELAX-NEXT:      0x4 R_LARCH_ALIGN .text 0x4
 # RELAX-NEXT:      0x10 R_LARCH_PCALA_HI20 .L1 0x0
 # RELAX-NEXT:      0x10 R_LARCH_RELAX - 0x0
 # RELAX-NEXT:      0x14 R_LARCH_PCALA_LO12 .L1 0x0
diff --git a/llvm/test/MC/LoongArch/Relocations/relax-align.s b/llvm/test/MC/LoongArch/Relocations/relax-align.s
index 294fd9fb916c..0246d5b46431 100644
--- a/llvm/test/MC/LoongArch/Relocations/relax-align.s
+++ b/llvm/test/MC/LoongArch/Relocations/relax-align.s
@@ -63,17 +63,19 @@ ret
 ## Test the symbol index is different from .text.
 .section .text2, "ax"
 .p2align 4
+.p2align 4, , 4
 break 7
 
 # RELOC:            Relocations [
 # RELAX-RELOC-NEXT:   Section ({{.*}}) .rela.text {
-# RELAX-RELOC-NEXT:     0x24 R_LARCH_ALIGN .Lla-relax-align0 0x4
-# RELAX-RELOC-NEXT:     0x34 R_LARCH_ALIGN .Lla-relax-align0 0x5
-# RELAX-RELOC-NEXT:     0x50 R_LARCH_ALIGN .Lla-relax-align0 0x4
-# RELAX-RELOC-NEXT:     0x60 R_LARCH_ALIGN .Lla-relax-align0 0xB04
-# RELAX-RELOC-NEXT:     0x70 R_LARCH_ALIGN .Lla-relax-align0 0x4
+# RELAX-RELOC-NEXT:     0x24 R_LARCH_ALIGN .text 0x4
+# RELAX-RELOC-NEXT:     0x34 R_LARCH_ALIGN .text 0x5
+# RELAX-RELOC-NEXT:     0x50 R_LARCH_ALIGN .text 0x4
+# RELAX-RELOC-NEXT:     0x60 R_LARCH_ALIGN .text 0xB04
+# RELAX-RELOC-NEXT:     0x70 R_LARCH_ALIGN .text 0x4
 # RELAX-RELOC-NEXT:   }
 # RELAX-RELOC-NEXT:   Section ({{.*}}) .rela.text2 {
-# RELAX-RELOC-NEXT:     0x0 R_LARCH_ALIGN .Lla-relax-align1 0x4
+# RELAX-RELOC-NEXT:     0x0 R_LARCH_ALIGN .text2 0x4
+# RELAX-RELOC-NEXT:     0xC R_LARCH_ALIGN .text2 0x404
 # RELAX-RELOC-NEXT:   }
 # RELOC-NEXT:       ]
-- 
GitLab


From c09384e2b419c7b4e4167e0d0295d9018cc6169c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Mon, 15 Apr 2024 15:11:23 +0200
Subject: [PATCH 375/464] [clang][Interp] Support MemberExprs pointing to
 VarDecls

---
 clang/lib/AST/Interp/ByteCodeExprGen.cpp | 11 +++++++++--
 clang/test/AST/Interp/records.cpp        |  8 ++++++++
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
index 4a7b40440770..a069f3ec27e7 100644
--- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp
+++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
@@ -1267,10 +1267,19 @@ template <class Emitter>
 bool ByteCodeExprGen<Emitter>::VisitMemberExpr(const MemberExpr *E) {
   // 'Base.Member'
   const Expr *Base = E->getBase();
+  const ValueDecl *Member = E->getMemberDecl();
 
   if (DiscardResult)
     return this->discard(Base);
 
+  if (const auto *VD = dyn_cast<VarDecl>(Member)) {
+    // I am almost confident in saying that a var decl must be static
+    // and therefore registered as a global variable. But this will probably
+    // turn out to be wrong some time in the future, as always.
+    if (auto GlobalIndex = P.getGlobal(VD))
+      return this->emitGetPtrGlobal(*GlobalIndex, E);
+  }
+
   if (Initializing) {
     if (!this->delegate(Base))
       return false;
@@ -1280,8 +1289,6 @@ bool ByteCodeExprGen<Emitter>::VisitMemberExpr(const MemberExpr *E) {
   }
 
   // Base above gives us a pointer on the stack.
-  // TODO: Implement non-FieldDecl members.
-  const ValueDecl *Member = E->getMemberDecl();
   if (const auto *FD = dyn_cast<FieldDecl>(Member)) {
     const RecordDecl *RD = FD->getParent();
     const Record *R = getRecord(RD);
diff --git a/clang/test/AST/Interp/records.cpp b/clang/test/AST/Interp/records.cpp
index f251497ed701..2c33fa1bf884 100644
--- a/clang/test/AST/Interp/records.cpp
+++ b/clang/test/AST/Interp/records.cpp
@@ -1309,3 +1309,11 @@ namespace pr18633 {
     func2<int>();
   }
 }
+
+namespace {
+  struct F {
+    static constexpr int Z = 12;
+  };
+  F f;
+  static_assert(f.Z == 12, "");
+}
-- 
GitLab


From 1120d8e6f799121b611aa23bdc128e40cf9c6c58 Mon Sep 17 00:00:00 2001
From: Alex Voicu <alexandru.voicu@amd.com>
Date: Tue, 16 Apr 2024 13:37:29 +0300
Subject: [PATCH 376/464] [clang][CodeGen] Add AS for Globals to SPIR & SPIRV
 datalayouts (#88455)

Currently neither the SPIR nor the SPIRV targets specify the AS for
globals in their datalayout strings. This is problematic because
CodeGen/LLVM will default to AS0 in this case, which produces Globals
that end up in the private address space for e.g. OCL, HIPSPV or SYCL.
This patch addresses it by completing the datalayout string.
---
 clang/lib/Basic/Targets/SPIR.h                |  8 +++---
 clang/test/CodeGen/target-data.c              |  4 +--
 llvm/lib/IR/AutoUpgrade.cpp                   |  9 ++++---
 llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp  |  4 +--
 .../Bitcode/DataLayoutUpgradeTest.cpp         | 27 +++++++++++++++++++
 5 files changed, 40 insertions(+), 12 deletions(-)

diff --git a/clang/lib/Basic/Targets/SPIR.h b/clang/lib/Basic/Targets/SPIR.h
index e25991e3dfe8..9a4a8b501460 100644
--- a/clang/lib/Basic/Targets/SPIR.h
+++ b/clang/lib/Basic/Targets/SPIR.h
@@ -259,7 +259,7 @@ public:
     SizeType = TargetInfo::UnsignedInt;
     PtrDiffType = IntPtrType = TargetInfo::SignedInt;
     resetDataLayout("e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-"
-                    "v96:128-v192:256-v256:256-v512:512-v1024:1024");
+                    "v96:128-v192:256-v256:256-v512:512-v1024:1024-G1");
   }
 
   void getTargetDefines(const LangOptions &Opts,
@@ -276,7 +276,7 @@ public:
     SizeType = TargetInfo::UnsignedLong;
     PtrDiffType = IntPtrType = TargetInfo::SignedLong;
     resetDataLayout("e-i64:64-v16:16-v24:32-v32:32-v48:64-"
-                    "v96:128-v192:256-v256:256-v512:512-v1024:1024");
+                    "v96:128-v192:256-v256:256-v512:512-v1024:1024-G1");
   }
 
   void getTargetDefines(const LangOptions &Opts,
@@ -336,7 +336,7 @@ public:
     SizeType = TargetInfo::UnsignedInt;
     PtrDiffType = IntPtrType = TargetInfo::SignedInt;
     resetDataLayout("e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-"
-                    "v96:128-v192:256-v256:256-v512:512-v1024:1024");
+                    "v96:128-v192:256-v256:256-v512:512-v1024:1024-G1");
   }
 
   void getTargetDefines(const LangOptions &Opts,
@@ -357,7 +357,7 @@ public:
     SizeType = TargetInfo::UnsignedLong;
     PtrDiffType = IntPtrType = TargetInfo::SignedLong;
     resetDataLayout("e-i64:64-v16:16-v24:32-v32:32-v48:64-"
-                    "v96:128-v192:256-v256:256-v512:512-v1024:1024");
+                    "v96:128-v192:256-v256:256-v512:512-v1024:1024-G1");
   }
 
   void getTargetDefines(const LangOptions &Opts,
diff --git a/clang/test/CodeGen/target-data.c b/clang/test/CodeGen/target-data.c
index acff367d50eb..c184f314f68f 100644
--- a/clang/test/CodeGen/target-data.c
+++ b/clang/test/CodeGen/target-data.c
@@ -251,11 +251,11 @@
 
 // RUN: %clang_cc1 -triple spir-unknown -o - -emit-llvm %s | \
 // RUN: FileCheck %s -check-prefix=SPIR
-// SPIR: target datalayout = "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024"
+// SPIR: target datalayout = "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
 
 // RUN: %clang_cc1 -triple spir64-unknown -o - -emit-llvm %s | \
 // RUN: FileCheck %s -check-prefix=SPIR64
-// SPIR64: target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024"
+// SPIR64: target datalayout = "e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-G1"
 
 // RUN: %clang_cc1 -triple bpfel -o - -emit-llvm %s | \
 // RUN: FileCheck %s -check-prefix=BPFEL
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index 2c480fb76ee4..634b2dd5119e 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -5341,10 +5341,11 @@ MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) {
 
 std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
   Triple T(TT);
-  // The only data layout upgrades needed for pre-GCN are setting the address
-  // space of globals to 1.
-  if (T.isAMDGPU() && !T.isAMDGCN() && !DL.contains("-G") &&
-      !DL.starts_with("G")) {
+  // The only data layout upgrades needed for pre-GCN, SPIR or SPIRV are setting
+  // the address space of globals to 1. This does not apply to SPIRV Logical.
+  if (((T.isAMDGPU() && !T.isAMDGCN()) ||
+       (T.isSPIR() || (T.isSPIRV() && !T.isSPIRVLogical()))) &&
+      !DL.contains("-G") && !DL.starts_with("G")) {
     return DL.empty() ? std::string("G1") : (DL + "-G1").str();
   }
 
diff --git a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
index fbf64f2b1dfb..ae8baa3f1191 100644
--- a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
+++ b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
@@ -55,9 +55,9 @@ static std::string computeDataLayout(const Triple &TT) {
   // mean anything.
   if (Arch == Triple::spirv32)
     return "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-"
-           "v96:128-v192:256-v256:256-v512:512-v1024:1024";
+           "v96:128-v192:256-v256:256-v512:512-v1024:1024-G1";
   return "e-i64:64-v16:16-v24:32-v32:32-v48:64-"
-         "v96:128-v192:256-v256:256-v512:512-v1024:1024";
+         "v96:128-v192:256-v256:256-v512:512-v1024:1024-G1";
 }
 
 static Reloc::Model getEffectiveRelocModel(std::optional<Reloc::Model> RM) {
diff --git a/llvm/unittests/Bitcode/DataLayoutUpgradeTest.cpp b/llvm/unittests/Bitcode/DataLayoutUpgradeTest.cpp
index 4865616e3e2b..d7e4dba4ac17 100644
--- a/llvm/unittests/Bitcode/DataLayoutUpgradeTest.cpp
+++ b/llvm/unittests/Bitcode/DataLayoutUpgradeTest.cpp
@@ -59,6 +59,14 @@ TEST(DataLayoutUpgradeTest, ValidDataLayoutUpgrade) {
   EXPECT_EQ(UpgradeDataLayoutString("e-m:e-p:64:64-i64:64-i128:128-n64-S128",
                                     "riscv64"),
             "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128");
+
+  // Check that SPIR && SPIRV targets add -G1 if it's not present.
+  EXPECT_EQ(UpgradeDataLayoutString("e-p:32:32", "spir"), "e-p:32:32-G1");
+  EXPECT_EQ(UpgradeDataLayoutString("e-p:32:32", "spir64"), "e-p:32:32-G1");
+  EXPECT_EQ(UpgradeDataLayoutString("e-p:32:32", "spirv32"), "e-p:32:32-G1");
+  EXPECT_EQ(UpgradeDataLayoutString("e-p:32:32", "spirv64"), "e-p:32:32-G1");
+  // but that SPIRV Logical does not.
+  EXPECT_EQ(UpgradeDataLayoutString("e-p:32:32", "spirv"), "e-p:32:32");
 }
 
 TEST(DataLayoutUpgradeTest, NoDataLayoutUpgrade) {
@@ -100,6 +108,17 @@ TEST(DataLayoutUpgradeTest, NoDataLayoutUpgrade) {
             "p7:64:64-G2-e-p:64:64-ni:7:8:9-p8:128:128-p9:192:256:256:32");
   EXPECT_EQ(UpgradeDataLayoutString("e-p:64:64-p7:64:64-G1", "amdgcn"),
             "e-p:64:64-p7:64:64-G1-ni:7:8:9-p8:128:128-p9:192:256:256:32");
+
+  // Check that SPIR & SPIRV targets don't add -G1 if there is already a -G
+  // flag.
+  EXPECT_EQ(UpgradeDataLayoutString("e-p:32:32-G2", "spir"), "e-p:32:32-G2");
+  EXPECT_EQ(UpgradeDataLayoutString("e-p:32:32-G2", "spir64"), "e-p:32:32-G2");
+  EXPECT_EQ(UpgradeDataLayoutString("e-p:32:32-G2", "spirv32"), "e-p:32:32-G2");
+  EXPECT_EQ(UpgradeDataLayoutString("e-p:32:32-G2", "spirv64"), "e-p:32:32-G2");
+  EXPECT_EQ(UpgradeDataLayoutString("G2", "spir"), "G2");
+  EXPECT_EQ(UpgradeDataLayoutString("G2", "spir64"), "G2");
+  EXPECT_EQ(UpgradeDataLayoutString("G2", "spirv32"), "G2");
+  EXPECT_EQ(UpgradeDataLayoutString("G2", "spirv64"), "G2");
 }
 
 TEST(DataLayoutUpgradeTest, EmptyDataLayout) {
@@ -113,6 +132,14 @@ TEST(DataLayoutUpgradeTest, EmptyDataLayout) {
   EXPECT_EQ(UpgradeDataLayoutString("", "r600"), "G1");
   EXPECT_EQ(UpgradeDataLayoutString("", "amdgcn"),
             "G1-ni:7:8:9-p7:160:256:256:32-p8:128:128-p9:192:256:256:32");
+
+  // Check that SPIR & SPIRV targets add G1 if it's not present.
+  EXPECT_EQ(UpgradeDataLayoutString("", "spir"), "G1");
+  EXPECT_EQ(UpgradeDataLayoutString("", "spir64"), "G1");
+  EXPECT_EQ(UpgradeDataLayoutString("", "spirv32"), "G1");
+  EXPECT_EQ(UpgradeDataLayoutString("", "spirv64"), "G1");
+  // but SPIRV Logical does not.
+  EXPECT_EQ(UpgradeDataLayoutString("", "spirv"), "");
 }
 
 } // end namespace
-- 
GitLab


From 36b3c26451bf9a42f0b6b415993d3942bb73abdd Mon Sep 17 00:00:00 2001
From: XChy <xxs_chy@outlook.com>
Date: Tue, 16 Apr 2024 18:51:03 +0800
Subject: [PATCH 377/464] [JumpThreading] Thread over BB with only an
 unconditional branch (#86312)

Fixes #76609
This patch does:
- relax the phis constraint in `CanRedirectPredsOfEmptyBBToSucc`
- guarantee the BB has multiple different predecessors to redirect, so
that we can handle the case without phis in BB. Without this change and
phi constraint, we may redirect the CommonPred.

The motivation is consistent with JumpThreading. We always want the
branch to jump more direct to the destination, without passing the
middle block. In this way, we can expose more other optimization
opportunities.

An obivous example proposed by @dtcxzyw is like:
```llvm
define i32 @test(...) {
entry:
   br i1 %c, label %do.end, label %if.then

if.then:                                          ; preds = %entry
   %call2 = call i32 @dummy()
   %tobool3.not = icmp eq i32 %call2, 0
   br i1 %tobool3.not, label %do.end, label %return

do.end:                                           ; preds = %entry, %if.then
   br label %return

return:                                           ; preds = %if.then, %do.end
   %retval.0 = phi i32 [ 0, %do.end ], [ %call2, %if.then ]
   ret i32 %retval.0
}
```
`entry` can directly jump to return, without passing `do.end`, and then
the if-else pattern can be simplified further:
```llvm
define i32 @test(...) {
entry:
   br i1 %c, label %return, label %if.then

if.then:                                          ; preds = %entry
   %call2 = call i32 @dummy()
   br label %return

return:                                           ; preds = %if.then
   %retval.0 = phi i32 [ 0, %entry ], [ %call2, %if.then ]
   ret i32 %retval.0
}
```
---
 llvm/lib/Transforms/Utils/Local.cpp           |  10 +-
 llvm/test/CodeGen/AArch64/and-sink.ll         |   9 +-
 .../AArch64/combine-comparisons-by-cse.ll     | 122 +++++++----------
 llvm/test/CodeGen/Hexagon/vect/zext-v4i1.ll   |  18 +--
 llvm/test/Transforms/JumpThreading/pr79175.ll |   8 +-
 llvm/test/Transforms/JumpThreading/select.ll  |  50 +++----
 .../Transforms/JumpThreading/thread-prob-7.ll |   8 +-
 .../Transforms/JumpThreading/uncond-no-phi.ll | 123 ++++++++++++++++++
 .../PhaseOrdering/thread-uncond-bb.ll         |  62 +++++++++
 9 files changed, 284 insertions(+), 126 deletions(-)
 create mode 100644 llvm/test/Transforms/JumpThreading/uncond-no-phi.ll
 create mode 100644 llvm/test/Transforms/PhaseOrdering/thread-uncond-bb.ll

diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp
index a42ef0c4e6ae..baec51a07fcb 100644
--- a/llvm/lib/Transforms/Utils/Local.cpp
+++ b/llvm/lib/Transforms/Utils/Local.cpp
@@ -1019,12 +1019,14 @@ CanRedirectPredsOfEmptyBBToSucc(BasicBlock *BB, BasicBlock *Succ,
                                 const SmallPtrSetImpl<BasicBlock *> &SuccPreds,
                                 BasicBlock *&CommonPred) {
 
-  // There must be phis in BB, otherwise BB will be merged into Succ directly
-  if (BB->phis().empty() || Succ->phis().empty())
+  // When Succ has no phis, BB may be merged into Succ directly. We don't need
+  // to redirect the predecessors of BB in this case.
+  if (Succ->phis().empty())
     return false;
 
-  // BB must have predecessors not shared that can be redirected to Succ
-  if (!BB->hasNPredecessorsOrMore(2))
+  // BB must have multiple different predecessors, so that at least one of
+  // predecessors can be redirected to Succ, except the common predecessor.
+  if (BB->getUniquePredecessor() || pred_empty(BB))
     return false;
 
   // Get single common predecessors of both BB and Succ
diff --git a/llvm/test/CodeGen/AArch64/and-sink.ll b/llvm/test/CodeGen/AArch64/and-sink.ll
index f298a55dab72..a57e9d54f307 100644
--- a/llvm/test/CodeGen/AArch64/and-sink.ll
+++ b/llvm/test/CodeGen/AArch64/and-sink.ll
@@ -11,15 +11,14 @@
 define dso_local i32 @and_sink1(i32 %a, i1 %c) {
 ; CHECK-LABEL: and_sink1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    tbz w1, #0, .LBB0_3
+; CHECK-NEXT:    tbz w1, #0, .LBB0_2
 ; CHECK-NEXT:  // %bb.1: // %bb0
+; CHECK-NEXT:    tst w0, #0x4
 ; CHECK-NEXT:    adrp x8, A
+; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    str wzr, [x8, :lo12:A]
-; CHECK-NEXT:    tbnz w0, #2, .LBB0_3
-; CHECK-NEXT:  // %bb.2:
-; CHECK-NEXT:    mov w0, #1 // =0x1
 ; CHECK-NEXT:    ret
-; CHECK-NEXT:  .LBB0_3: // %bb2
+; CHECK-NEXT:  .LBB0_2:
 ; CHECK-NEXT:    mov w0, wzr
 ; CHECK-NEXT:    ret
 
diff --git a/llvm/test/CodeGen/AArch64/combine-comparisons-by-cse.ll b/llvm/test/CodeGen/AArch64/combine-comparisons-by-cse.ll
index 6449c3e11d66..dde3e81833a6 100644
--- a/llvm/test/CodeGen/AArch64/combine-comparisons-by-cse.ll
+++ b/llvm/test/CodeGen/AArch64/combine-comparisons-by-cse.ll
@@ -13,10 +13,10 @@ define i32 @combine_gt_ge_10() #0 {
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    adrp x8, :got:a
 ; CHECK-NEXT:    ldr x8, [x8, :got_lo12:a]
-; CHECK-NEXT:    ldr w8, [x8]
-; CHECK-NEXT:    cmp w8, #10
+; CHECK-NEXT:    ldr w9, [x8]
 ; CHECK-NEXT:    adrp x8, :got:b
 ; CHECK-NEXT:    ldr x8, [x8, :got_lo12:b]
+; CHECK-NEXT:    cmp w9, #10
 ; CHECK-NEXT:    b.le .LBB0_3
 ; CHECK-NEXT:  // %bb.1: // %land.lhs.true
 ; CHECK-NEXT:    adrp x9, :got:c
@@ -29,18 +29,17 @@ define i32 @combine_gt_ge_10() #0 {
 ; CHECK-NEXT:    mov w0, #1 // =0x1
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB0_3: // %lor.lhs.false
-; CHECK-NEXT:    b.lt .LBB0_6
+; CHECK-NEXT:    cmp w9, #10
+; CHECK-NEXT:    b.lt .LBB0_5
 ; CHECK-NEXT:  .LBB0_4: // %land.lhs.true3
 ; CHECK-NEXT:    adrp x9, :got:d
 ; CHECK-NEXT:    ldr x9, [x9, :got_lo12:d]
 ; CHECK-NEXT:    ldr w8, [x8]
 ; CHECK-NEXT:    ldr w9, [x9]
 ; CHECK-NEXT:    cmp w8, w9
-; CHECK-NEXT:    b.ne .LBB0_6
-; CHECK-NEXT:  // %bb.5:
-; CHECK-NEXT:    mov w0, #1 // =0x1
+; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
-; CHECK-NEXT:  .LBB0_6: // %if.end
+; CHECK-NEXT:  .LBB0_5:
 ; CHECK-NEXT:    mov w0, wzr
 ; CHECK-NEXT:    ret
 entry:
@@ -145,10 +144,10 @@ define i32 @combine_lt_ge_5() #0 {
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    adrp x8, :got:a
 ; CHECK-NEXT:    ldr x8, [x8, :got_lo12:a]
-; CHECK-NEXT:    ldr w8, [x8]
-; CHECK-NEXT:    cmp w8, #5
+; CHECK-NEXT:    ldr w9, [x8]
 ; CHECK-NEXT:    adrp x8, :got:b
 ; CHECK-NEXT:    ldr x8, [x8, :got_lo12:b]
+; CHECK-NEXT:    cmp w9, #5
 ; CHECK-NEXT:    b.ge .LBB2_3
 ; CHECK-NEXT:  // %bb.1: // %land.lhs.true
 ; CHECK-NEXT:    adrp x9, :got:c
@@ -161,18 +160,17 @@ define i32 @combine_lt_ge_5() #0 {
 ; CHECK-NEXT:    mov w0, #1 // =0x1
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB2_3: // %lor.lhs.false
-; CHECK-NEXT:    b.gt .LBB2_6
+; CHECK-NEXT:    cmp w9, #5
+; CHECK-NEXT:    b.gt .LBB2_5
 ; CHECK-NEXT:  .LBB2_4: // %land.lhs.true3
 ; CHECK-NEXT:    adrp x9, :got:d
 ; CHECK-NEXT:    ldr x9, [x9, :got_lo12:d]
 ; CHECK-NEXT:    ldr w8, [x8]
 ; CHECK-NEXT:    ldr w9, [x9]
 ; CHECK-NEXT:    cmp w8, w9
-; CHECK-NEXT:    b.ne .LBB2_6
-; CHECK-NEXT:  // %bb.5:
-; CHECK-NEXT:    mov w0, #1 // =0x1
+; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
-; CHECK-NEXT:  .LBB2_6: // %if.end
+; CHECK-NEXT:  .LBB2_5:
 ; CHECK-NEXT:    mov w0, wzr
 ; CHECK-NEXT:    ret
 entry:
@@ -499,24 +497,17 @@ define i32 @do_nothing_if_resultant_opcodes_would_differ() #0 {
 ; CHECK-NEXT:  // %bb.3: // %while.cond.while.end_crit_edge
 ; CHECK-NEXT:    ldr w8, [x19]
 ; CHECK-NEXT:  .LBB7_4: // %while.end
-; CHECK-NEXT:    cmp w8, #1
-; CHECK-NEXT:    b.gt .LBB7_7
-; CHECK-NEXT:  // %bb.5: // %land.lhs.true
-; CHECK-NEXT:    adrp x8, :got:b
-; CHECK-NEXT:    adrp x9, :got:d
-; CHECK-NEXT:    ldr x8, [x8, :got_lo12:b]
-; CHECK-NEXT:    ldr x9, [x9, :got_lo12:d]
-; CHECK-NEXT:    ldr w8, [x8]
-; CHECK-NEXT:    ldr w9, [x9]
-; CHECK-NEXT:    cmp w8, w9
-; CHECK-NEXT:    b.ne .LBB7_7
-; CHECK-NEXT:  // %bb.6:
-; CHECK-NEXT:    mov w0, #123 // =0x7b
-; CHECK-NEXT:    b .LBB7_8
-; CHECK-NEXT:  .LBB7_7: // %if.end
-; CHECK-NEXT:    mov w0, wzr
-; CHECK-NEXT:  .LBB7_8: // %return
+; CHECK-NEXT:    adrp x9, :got:b
+; CHECK-NEXT:    adrp x10, :got:d
+; CHECK-NEXT:    ldr x9, [x9, :got_lo12:b]
+; CHECK-NEXT:    ldr x10, [x10, :got_lo12:d]
 ; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr w9, [x9]
+; CHECK-NEXT:    ldr w10, [x10]
+; CHECK-NEXT:    cmp w9, w10
+; CHECK-NEXT:    ccmp w8, #2, #0, eq
+; CHECK-NEXT:    mov w8, #123 // =0x7b
+; CHECK-NEXT:    csel w0, w8, wzr, lt
 ; CHECK-NEXT:    ldr x30, [sp], #32 // 8-byte Folded Reload
 ; CHECK-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-NEXT:    .cfi_restore w19
@@ -564,52 +555,42 @@ return:                                           ; preds = %if.end, %land.lhs.t
 define i32 @do_nothing_if_compares_can_not_be_adjusted_to_each_other() #0 {
 ; CHECK-LABEL: do_nothing_if_compares_can_not_be_adjusted_to_each_other:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    .cfi_offset w19, -8
-; CHECK-NEXT:    .cfi_offset w30, -16
-; CHECK-NEXT:    .cfi_remember_state
 ; CHECK-NEXT:    adrp x8, :got:a
 ; CHECK-NEXT:    ldr x8, [x8, :got_lo12:a]
 ; CHECK-NEXT:    ldr w8, [x8]
 ; CHECK-NEXT:    cmp w8, #0
-; CHECK-NEXT:    b.gt .LBB8_3
+; CHECK-NEXT:    b.gt .LBB8_4
 ; CHECK-NEXT:  // %bb.1: // %while.body.preheader
+; CHECK-NEXT:    stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    sub w19, w8, #1
 ; CHECK-NEXT:  .LBB8_2: // %while.body
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    bl do_something
 ; CHECK-NEXT:    adds w19, w19, #1
 ; CHECK-NEXT:    b.mi .LBB8_2
-; CHECK-NEXT:  .LBB8_3: // %while.end
-; CHECK-NEXT:    adrp x8, :got:c
-; CHECK-NEXT:    ldr x8, [x8, :got_lo12:c]
-; CHECK-NEXT:    ldr w8, [x8]
-; CHECK-NEXT:    cmn w8, #2
-; CHECK-NEXT:    b.lt .LBB8_6
-; CHECK-NEXT:  // %bb.4: // %land.lhs.true
+; CHECK-NEXT:  // %bb.3:
+; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:  .LBB8_4: // %while.end
 ; CHECK-NEXT:    adrp x8, :got:b
 ; CHECK-NEXT:    adrp x9, :got:d
+; CHECK-NEXT:    adrp x10, :got:c
 ; CHECK-NEXT:    ldr x8, [x8, :got_lo12:b]
 ; CHECK-NEXT:    ldr x9, [x9, :got_lo12:d]
+; CHECK-NEXT:    ldr x10, [x10, :got_lo12:c]
 ; CHECK-NEXT:    ldr w8, [x8]
 ; CHECK-NEXT:    ldr w9, [x9]
+; CHECK-NEXT:    ldr w10, [x10]
 ; CHECK-NEXT:    cmp w8, w9
-; CHECK-NEXT:    b.ne .LBB8_6
-; CHECK-NEXT:  // %bb.5:
-; CHECK-NEXT:    mov w0, #123 // =0x7b
-; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
-; CHECK-NEXT:    .cfi_def_cfa_offset 0
-; CHECK-NEXT:    .cfi_restore w19
-; CHECK-NEXT:    .cfi_restore w30
-; CHECK-NEXT:    ret
-; CHECK-NEXT:  .LBB8_6: // %if.end
-; CHECK-NEXT:    .cfi_restore_state
-; CHECK-NEXT:    mov w0, wzr
-; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
-; CHECK-NEXT:    .cfi_def_cfa_offset 0
-; CHECK-NEXT:    .cfi_restore w19
-; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    mov w8, #-3 // =0xfffffffd
+; CHECK-NEXT:    ccmp w10, w8, #4, eq
+; CHECK-NEXT:    mov w8, #123 // =0x7b
+; CHECK-NEXT:    csel w0, w8, wzr, gt
 ; CHECK-NEXT:    ret
 entry:
   %0 = load i32, ptr @a, align 4
@@ -782,12 +763,14 @@ define i32 @combine_gt_ge_sel(i64 %v, ptr %p) #0 {
 ; CHECK-NEXT:    cmp w8, #0
 ; CHECK-NEXT:    csel x9, x0, xzr, gt
 ; CHECK-NEXT:    str x9, [x1]
-; CHECK-NEXT:    b.le .LBB11_2
+; CHECK-NEXT:    b.le .LBB11_3
 ; CHECK-NEXT:  // %bb.1: // %lor.lhs.false
 ; CHECK-NEXT:    cmp w8, #2
-; CHECK-NEXT:    b.ge .LBB11_4
-; CHECK-NEXT:    b .LBB11_6
-; CHECK-NEXT:  .LBB11_2: // %land.lhs.true
+; CHECK-NEXT:    b.ge .LBB11_5
+; CHECK-NEXT:  // %bb.2:
+; CHECK-NEXT:    mov w0, wzr
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB11_3: // %land.lhs.true
 ; CHECK-NEXT:    adrp x8, :got:b
 ; CHECK-NEXT:    adrp x9, :got:c
 ; CHECK-NEXT:    ldr x8, [x8, :got_lo12:b]
@@ -795,11 +778,11 @@ define i32 @combine_gt_ge_sel(i64 %v, ptr %p) #0 {
 ; CHECK-NEXT:    ldr w8, [x8]
 ; CHECK-NEXT:    ldr w9, [x9]
 ; CHECK-NEXT:    cmp w8, w9
-; CHECK-NEXT:    b.ne .LBB11_4
-; CHECK-NEXT:  // %bb.3:
+; CHECK-NEXT:    b.ne .LBB11_5
+; CHECK-NEXT:  // %bb.4:
 ; CHECK-NEXT:    mov w0, #1 // =0x1
 ; CHECK-NEXT:    ret
-; CHECK-NEXT:  .LBB11_4: // %land.lhs.true3
+; CHECK-NEXT:  .LBB11_5: // %land.lhs.true3
 ; CHECK-NEXT:    adrp x8, :got:b
 ; CHECK-NEXT:    adrp x9, :got:d
 ; CHECK-NEXT:    ldr x8, [x8, :got_lo12:b]
@@ -807,12 +790,7 @@ define i32 @combine_gt_ge_sel(i64 %v, ptr %p) #0 {
 ; CHECK-NEXT:    ldr w8, [x8]
 ; CHECK-NEXT:    ldr w9, [x9]
 ; CHECK-NEXT:    cmp w8, w9
-; CHECK-NEXT:    b.ne .LBB11_6
-; CHECK-NEXT:  // %bb.5:
-; CHECK-NEXT:    mov w0, #1 // =0x1
-; CHECK-NEXT:    ret
-; CHECK-NEXT:  .LBB11_6: // %if.end
-; CHECK-NEXT:    mov w0, wzr
+; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    ret
 entry:
   %0 = load i32, ptr @a, align 4
diff --git a/llvm/test/CodeGen/Hexagon/vect/zext-v4i1.ll b/llvm/test/CodeGen/Hexagon/vect/zext-v4i1.ll
index dddc4bd953d7..c33c81841be6 100644
--- a/llvm/test/CodeGen/Hexagon/vect/zext-v4i1.ll
+++ b/llvm/test/CodeGen/Hexagon/vect/zext-v4i1.ll
@@ -10,12 +10,13 @@ define i32 @fred(ptr %a0) #0 {
 ; CHECK-LABEL: fred:
 ; CHECK:       // %bb.0: // %b0
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     if (p0) jump:nt .LBB0_2
+; CHECK-NEXT:     r1:0 = combine(r0,#0)
+; CHECK-NEXT:     if (p0) jumpr r31
 ; CHECK-NEXT:    }
-; CHECK-NEXT:  // %bb.1: // %b2
+; CHECK-NEXT:  .LBB0_1: // %b2
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     r3:2 = combine(#0,#0)
-; CHECK-NEXT:     r1:0 = memd(r0+#0)
+; CHECK-NEXT:     r1:0 = memd(r1+#0)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     p0 = vcmph.eq(r1:0,r3:2)
@@ -27,16 +28,7 @@ define i32 @fred(ptr %a0) #0 {
 ; CHECK-NEXT:     r0 = and(r0,#1)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     p0 = cmp.eq(r0,#11)
-; CHECK-NEXT:     r0 = #1
-; CHECK-NEXT:    }
-; CHECK-NEXT:    {
-; CHECK-NEXT:     if (p0) r0 = #0
-; CHECK-NEXT:     jumpr r31
-; CHECK-NEXT:    }
-; CHECK-NEXT:  .LBB0_2: // %b14
-; CHECK-NEXT:    {
-; CHECK-NEXT:     r0 = #0
+; CHECK-NEXT:     r0 = !cmp.eq(r0,#11)
 ; CHECK-NEXT:     jumpr r31
 ; CHECK-NEXT:    }
 b0:
diff --git a/llvm/test/Transforms/JumpThreading/pr79175.ll b/llvm/test/Transforms/JumpThreading/pr79175.ll
index 2c7ee0770cdc..cce30ce07999 100644
--- a/llvm/test/Transforms/JumpThreading/pr79175.ll
+++ b/llvm/test/Transforms/JumpThreading/pr79175.ll
@@ -17,11 +17,11 @@ define i32 @test(i64 %idx, i32 %val) {
 ; CHECK:       cond.end:
 ; CHECK-NEXT:    [[CMP_I:%.*]] = icmp sgt i32 [[VAL]], 0
 ; CHECK-NEXT:    [[COND_FR:%.*]] = freeze i1 [[CMP_I]]
-; CHECK-NEXT:    br i1 [[COND_FR]], label [[COND_END_THREAD]], label [[TMP0:%.*]]
-; CHECK:       cond.end.thread:
-; CHECK-NEXT:    br label [[TMP0]]
+; CHECK-NEXT:    br i1 [[COND_FR]], label [[TMP0:%.*]], label [[COND_END_THREAD]]
 ; CHECK:       0:
-; CHECK-NEXT:    [[TMP1:%.*]] = phi i32 [ 0, [[COND_END_THREAD]] ], [ [[VAL]], [[COND_END]] ]
+; CHECK-NEXT:    br label [[COND_END_THREAD]]
+; CHECK:       cond.end.thread:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i32 [ [[VAL]], [[COND_END]] ], [ 0, [[TMP0]] ], [ 0, [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[F_IDX:%.*]] = getelementptr inbounds i32, ptr @f, i64 [[IDX]]
 ; CHECK-NEXT:    store i32 [[TMP1]], ptr [[F_IDX]], align 4
 ; CHECK-NEXT:    [[F_RELOAD:%.*]] = load i32, ptr @f, align 4
diff --git a/llvm/test/Transforms/JumpThreading/select.ll b/llvm/test/Transforms/JumpThreading/select.ll
index 4ec55a66bb8a..27ebf4c25da5 100644
--- a/llvm/test/Transforms/JumpThreading/select.ll
+++ b/llvm/test/Transforms/JumpThreading/select.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals
 ; RUN: opt -S -passes="jump-threading" -debug-only=branch-prob < %s 2>&1 | FileCheck %s
-; RUN: opt -S -passes="require<branch-prob>,jump-threading" -debug-only=branch-prob < %s 2>&1 | FileCheck -check-prefixes=CHECK,CHECK-BPI %s
+; RUN: opt -S -passes="require<branch-prob>,jump-threading" -debug-only=branch-prob  -disable-output < %s 2>&1 | FileCheck -check-prefix=CHECK-BPI %s
 ; REQUIRES: asserts
 
 ; CHECK-BPI-LABEL:  ---- Branch Probability Info : unfold1 ----
@@ -21,7 +21,7 @@ declare void @quux()
 ; booleans where at least one operand is true/false/undef.
 
 ;.
-; CHECK: @[[ANCHOR:[a-zA-Z0-9_$"\\.-]+]] = constant [3 x ptr] [ptr blockaddress(@test_indirectbr, [[L1:%.*]]), ptr inttoptr (i32 1 to ptr), ptr blockaddress(@test_indirectbr, [[L3:%.*]])]
+; CHECK: @anchor = constant [3 x ptr] [ptr blockaddress(@test_indirectbr, %L1), ptr inttoptr (i32 1 to ptr), ptr blockaddress(@test_indirectbr, %L3)]
 ;.
 define void @test_br(i1 %cond, i1 %value) nounwind {
 ; CHECK-LABEL: @test_br(
@@ -66,8 +66,8 @@ define void @test_switch(i1 %cond, i8 %value) nounwind {
 ; CHECK-NEXT:    call void @quux()
 ; CHECK-NEXT:    [[EXPR:%.*]] = select i1 [[COND]], i8 1, i8 [[VALUE:%.*]]
 ; CHECK-NEXT:    switch i8 [[EXPR]], label [[L3:%.*]] [
-; CHECK-NEXT:    i8 1, label [[L1]]
-; CHECK-NEXT:    i8 2, label [[L2:%.*]]
+; CHECK-NEXT:      i8 1, label [[L1]]
+; CHECK-NEXT:      i8 2, label [[L2:%.*]]
 ; CHECK-NEXT:    ]
 ; CHECK:       L1:
 ; CHECK-NEXT:    call void @foo()
@@ -192,8 +192,8 @@ define void @test_switch_cmp(i1 %cond, i32 %val, i8 %value) nounwind {
 ; CHECK:       0:
 ; CHECK-NEXT:    [[TMP1:%.*]] = phi i8 [ [[VALUE:%.*]], [[L0]] ]
 ; CHECK-NEXT:    switch i8 [[TMP1]], label [[L3:%.*]] [
-; CHECK-NEXT:    i8 1, label [[L1]]
-; CHECK-NEXT:    i8 2, label [[L2:%.*]]
+; CHECK-NEXT:      i8 1, label [[L1]]
+; CHECK-NEXT:      i8 2, label [[L2:%.*]]
 ; CHECK-NEXT:    ]
 ; CHECK:       L1:
 ; CHECK-NEXT:    call void @foo()
@@ -237,8 +237,8 @@ define void @test_switch_default(ptr nocapture %status) nounwind {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[STATUS:%.*]], align 4
 ; CHECK-NEXT:    switch i32 [[TMP0]], label [[L2:%.*]] [
-; CHECK-NEXT:    i32 5061, label [[L2_THREAD:%.*]]
-; CHECK-NEXT:    i32 0, label [[L2]]
+; CHECK-NEXT:      i32 5061, label [[L2_THREAD:%.*]]
+; CHECK-NEXT:      i32 0, label [[L2]]
 ; CHECK-NEXT:    ]
 ; CHECK:       L2.thread:
 ; CHECK-NEXT:    store i32 10025, ptr [[STATUS]], align 4
@@ -377,21 +377,21 @@ define i32 @unfold3(i32 %u, i32 %v, i32 %w, i32 %x, i32 %y, i32 %z, i32 %j) noun
 ; CHECK-NEXT:    br i1 [[CMP_I]], label [[DOTEXIT_THREAD4:%.*]], label [[COND_FALSE_I:%.*]]
 ; CHECK:       cond.false.i:
 ; CHECK-NEXT:    [[CMP4_I:%.*]] = icmp sgt i32 [[U]], [[V]]
-; CHECK-NEXT:    br i1 [[CMP4_I]], label [[DOTEXIT_THREAD:%.*]], label [[COND_FALSE_6_I:%.*]]
+; CHECK-NEXT:    br i1 [[CMP4_I]], label [[DOTEXIT_THREAD4]], label [[COND_FALSE_6_I:%.*]]
 ; CHECK:       cond.false.6.i:
 ; CHECK-NEXT:    [[CMP8_I:%.*]] = icmp slt i32 [[W:%.*]], [[X:%.*]]
 ; CHECK-NEXT:    br i1 [[CMP8_I]], label [[DOTEXIT_THREAD4]], label [[COND_FALSE_10_I:%.*]]
 ; CHECK:       cond.false.10.i:
 ; CHECK-NEXT:    [[CMP13_I:%.*]] = icmp sgt i32 [[W]], [[X]]
-; CHECK-NEXT:    br i1 [[CMP13_I]], label [[DOTEXIT_THREAD]], label [[DOTEXIT:%.*]]
+; CHECK-NEXT:    br i1 [[CMP13_I]], label [[DOTEXIT_THREAD4]], label [[DOTEXIT:%.*]]
 ; CHECK:       .exit:
 ; CHECK-NEXT:    [[PHITMP:%.*]] = icmp sge i32 [[Y:%.*]], [[Z:%.*]]
 ; CHECK-NEXT:    [[COND_FR:%.*]] = freeze i1 [[PHITMP]]
-; CHECK-NEXT:    br i1 [[COND_FR]], label [[DOTEXIT_THREAD]], label [[DOTEXIT_THREAD4]]
-; CHECK:       .exit.thread:
+; CHECK-NEXT:    br i1 [[COND_FR]], label [[DOTEXIT_THREAD:%.*]], label [[DOTEXIT_THREAD4]]
+; CHECK:       0:
 ; CHECK-NEXT:    br label [[DOTEXIT_THREAD4]]
-; CHECK:       .exit.thread4:
-; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ [[J]], [[DOTEXIT_THREAD]] ], [ [[ADD3]], [[DOTEXIT]] ], [ [[ADD3]], [[ENTRY:%.*]] ], [ [[ADD3]], [[COND_FALSE_6_I]] ]
+; CHECK:       .exit.thread:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ [[ADD3]], [[DOTEXIT]] ], [ [[J]], [[DOTEXIT_THREAD]] ], [ [[J]], [[COND_FALSE_I]] ], [ [[J]], [[COND_FALSE_10_I]] ], [ [[ADD3]], [[ENTRY:%.*]] ], [ [[ADD3]], [[COND_FALSE_6_I]] ]
 ; CHECK-NEXT:    ret i32 [[TMP0]]
 ;
 entry:
@@ -430,23 +430,23 @@ define i32 @unfold4(i32 %u, i32 %v, i32 %w, i32 %x, i32 %y, i32 %z, i32 %j) noun
 ; CHECK-NEXT:    br i1 [[CMP_I]], label [[DOTEXIT_THREAD:%.*]], label [[COND_FALSE_I:%.*]]
 ; CHECK:       cond.false.i:
 ; CHECK-NEXT:    [[CMP4_I:%.*]] = icmp sgt i32 [[U]], [[V]]
-; CHECK-NEXT:    br i1 [[CMP4_I]], label [[DOTEXIT_THREAD5:%.*]], label [[COND_FALSE_6_I:%.*]]
+; CHECK-NEXT:    br i1 [[CMP4_I]], label [[DOTEXIT_THREAD]], label [[COND_FALSE_6_I:%.*]]
 ; CHECK:       cond.false.6.i:
 ; CHECK-NEXT:    [[CMP8_I:%.*]] = icmp slt i32 [[W:%.*]], [[X:%.*]]
 ; CHECK-NEXT:    br i1 [[CMP8_I]], label [[DOTEXIT_THREAD]], label [[COND_FALSE_10_I:%.*]]
 ; CHECK:       cond.false.10.i:
 ; CHECK-NEXT:    [[CMP13_I:%.*]] = icmp sgt i32 [[W]], [[X]]
-; CHECK-NEXT:    br i1 [[CMP13_I]], label [[DOTEXIT_THREAD5]], label [[DOTEXIT:%.*]]
+; CHECK-NEXT:    br i1 [[CMP13_I]], label [[DOTEXIT_THREAD]], label [[DOTEXIT:%.*]]
 ; CHECK:       .exit:
 ; CHECK-NEXT:    [[CMP19_I:%.*]] = icmp sge i32 [[Y:%.*]], [[Z:%.*]]
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP19_I]] to i32
 ; CHECK-NEXT:    [[LNOT_I18:%.*]] = icmp eq i32 [[CONV]], 1
 ; CHECK-NEXT:    [[COND_FR:%.*]] = freeze i1 [[LNOT_I18]]
-; CHECK-NEXT:    br i1 [[COND_FR]], label [[DOTEXIT_THREAD]], label [[DOTEXIT_THREAD5]]
+; CHECK-NEXT:    br i1 [[COND_FR]], label [[TMP1:%.*]], label [[DOTEXIT_THREAD]]
+; CHECK:       0:
+; CHECK-NEXT:    br label [[DOTEXIT_THREAD]]
 ; CHECK:       .exit.thread:
-; CHECK-NEXT:    br label [[DOTEXIT_THREAD5]]
-; CHECK:       .exit.thread5:
-; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ [[J]], [[DOTEXIT_THREAD]] ], [ [[ADD3]], [[DOTEXIT]] ], [ [[ADD3]], [[COND_FALSE_I]] ], [ [[ADD3]], [[COND_FALSE_10_I]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ [[ADD3]], [[DOTEXIT]] ], [ [[J]], [[TMP1]] ], [ [[J]], [[ENTRY:%.*]] ], [ [[J]], [[COND_FALSE_6_I]] ], [ [[ADD3]], [[COND_FALSE_I]] ], [ [[ADD3]], [[COND_FALSE_10_I]] ]
 ; CHECK-NEXT:    ret i32 [[TMP0]]
 ;
 entry:
@@ -560,10 +560,10 @@ define void @test_func(ptr nocapture readonly %a, ptr nocapture readonly %b, ptr
 ; CHECK:       if.end:
 ; CHECK-NEXT:    [[LOCAL_VAR_0:%.*]] = phi i32 [ [[TMP1]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    switch i32 [[LOCAL_VAR_0]], label [[SW_DEFAULT]] [
-; CHECK-NEXT:    i32 2, label [[SW_BB]]
-; CHECK-NEXT:    i32 4, label [[SW_BB7]]
-; CHECK-NEXT:    i32 5, label [[SW_BB8:%.*]]
-; CHECK-NEXT:    i32 7, label [[SW_BB9:%.*]]
+; CHECK-NEXT:      i32 2, label [[SW_BB]]
+; CHECK-NEXT:      i32 4, label [[SW_BB7]]
+; CHECK-NEXT:      i32 5, label [[SW_BB8:%.*]]
+; CHECK-NEXT:      i32 7, label [[SW_BB9:%.*]]
 ; CHECK-NEXT:    ]
 ; CHECK:       sw.bb:
 ; CHECK-NEXT:    call void @foo()
@@ -674,3 +674,5 @@ if.end:
 ; CHECK: [[META0:![0-9]+]] = !{!"function_entry_count", i64 1984}
 ; CHECK: [[PROF1]] = !{!"branch_weights", i64 1073741824, i64 3221225472}
 ;.
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK-BPI: {{.*}}
diff --git a/llvm/test/Transforms/JumpThreading/thread-prob-7.ll b/llvm/test/Transforms/JumpThreading/thread-prob-7.ll
index 8c9d89871d00..4623a579be48 100644
--- a/llvm/test/Transforms/JumpThreading/thread-prob-7.ll
+++ b/llvm/test/Transforms/JumpThreading/thread-prob-7.ll
@@ -14,15 +14,15 @@ define i32 @func0(i32 %a0, i32 %a1) !prof !0 {
 ; CHECK-NEXT:    br i1 [[CMP1]], label [[BB_JOIN_THREAD:%.*]], label [[TEST2_FALSE:%.*]], !prof [[PROF2:![0-9]+]]
 ; CHECK:       test2_false:
 ; CHECK-NEXT:    call void @foobar()
-; CHECK-NEXT:    br label [[TMP0:%.*]]
+; CHECK-NEXT:    br label [[BB_JOIN_THREAD]]
 ; CHECK:       bb_join:
 ; CHECK-NEXT:    [[C:%.*]] = phi i1 [ [[CX]], [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[COND_FR:%.*]] = freeze i1 [[C]]
-; CHECK-NEXT:    br i1 [[COND_FR]], label [[BB_JOIN_THREAD]], label [[TMP0]], !prof [[PROF3:![0-9]+]]
+; CHECK-NEXT:    br i1 [[COND_FR]], label [[BB_JOIN_THREAD1:%.*]], label [[BB_JOIN_THREAD]], !prof [[PROF3:![0-9]+]]
 ; CHECK:       bb_join.thread:
-; CHECK-NEXT:    br label [[TMP0]]
+; CHECK-NEXT:    br label [[BB_JOIN_THREAD]]
 ; CHECK:       0:
-; CHECK-NEXT:    [[TMP1:%.*]] = phi i32 [ 42, [[BB_JOIN_THREAD]] ], [ 7, [[BB_JOIN]] ], [ 7, [[TEST2_FALSE]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i32 [ 7, [[BB_JOIN]] ], [ 7, [[TEST2_FALSE]] ], [ 42, [[TEST2]] ], [ 42, [[BB_JOIN_THREAD1]] ]
 ; CHECK-NEXT:    ret i32 [[TMP1]]
 ;
 entry:
diff --git a/llvm/test/Transforms/JumpThreading/uncond-no-phi.ll b/llvm/test/Transforms/JumpThreading/uncond-no-phi.ll
new file mode 100644
index 000000000000..6104e8f8778b
--- /dev/null
+++ b/llvm/test/Transforms/JumpThreading/uncond-no-phi.ll
@@ -0,0 +1,123 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -passes=jump-threading -S < %s | FileCheck %s
+
+define i1 @if_else(i1 %c, i1 %c1) {
+; CHECK-LABEL: define i1 @if_else(
+; CHECK-SAME: i1 [[C:%.*]], i1 [[C1:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C]], label [[THEN:%.*]], label [[RETURN:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    call void @dummy()
+; CHECK-NEXT:    br i1 [[C1]], label [[ELSE:%.*]], label [[RETURN]]
+; CHECK:       else:
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i1 [ false, [[THEN]] ], [ true, [[ENTRY:%.*]] ], [ true, [[ELSE]] ]
+; CHECK-NEXT:    ret i1 [[RETVAL_0]]
+;
+entry:
+  br i1 %c, label %then, label %else
+
+then:
+  call void @dummy()
+  br i1 %c1, label %else, label %return
+
+else:
+  br label %return
+
+return:
+  %retval.0 = phi i1 [ true, %else ], [ false, %then ]
+  ret i1 %retval.0
+}
+
+define i8 @switch_uncond(i8 %arg) {
+; CHECK-LABEL: define i8 @switch_uncond(
+; CHECK-SAME: i8 [[ARG:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i8 [[ARG]], label [[DEFAULT:%.*]] [
+; CHECK-NEXT:      i8 0, label [[BB1:%.*]]
+; CHECK-NEXT:      i8 1, label [[BB3:%.*]]
+; CHECK-NEXT:      i8 2, label [[BB2:%.*]]
+; CHECK-NEXT:      i8 3, label [[END:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       default:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb:
+; CHECK-NEXT:    call void @dummy()
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       bb1:
+; CHECK-NEXT:    call void @dummy()
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       bb2:
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i8 [ 1, [[ENTRY:%.*]] ], [ 0, [[BB3]] ], [ 0, [[BB1]] ], [ 0, [[BB2]] ]
+; CHECK-NEXT:    ret i8 [[PHI]]
+;
+entry:
+  switch i8 %arg, label %default [
+  i8 0, label %bb
+  i8 1, label %bb1
+  i8 2, label %bb2
+  i8 3, label %end
+  ]
+
+default:
+  unreachable
+
+bb:
+  call void @dummy()
+  br label %bb2
+
+bb1:
+  call void @dummy()
+  br label %bb2
+
+; Predecessors of %bb2 are %bb and %bb1, they are not identical.
+; So we can thread %bb2.
+bb2:
+  br label %end
+
+end:
+  %phi = phi i8 [ 0, %bb2 ], [ 1, %entry ]
+  ret i8 %phi
+}
+
+define i8 @switch_uncond_fail(i8 %arg) {
+; CHECK-LABEL: define i8 @switch_uncond_fail(
+; CHECK-SAME: i8 [[ARG:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i8 [[ARG]], label [[DEFAULT:%.*]] [
+; CHECK-NEXT:      i8 0, label [[BB:%.*]]
+; CHECK-NEXT:      i8 1, label [[BB]]
+; CHECK-NEXT:      i8 2, label [[END:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       default:
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       bb:
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i8 [ 0, [[BB]] ], [ 1, [[ENTRY:%.*]] ], [ 2, [[DEFAULT]] ]
+; CHECK-NEXT:    ret i8 [[PHI]]
+;
+entry:
+  switch i8 %arg, label %default [
+  i8 0, label %bb
+  i8 1, label %bb
+  i8 2, label %end
+  ]
+
+default:
+  br label %end
+
+; Predecessor of %bb is only %entry (though there are two in predecessor list),
+; thus it's unthreadable.
+bb:
+  br label %end
+
+end:
+  %phi = phi i8 [ 0, %bb ], [ 1, %entry ], [ 2, %default ]
+  ret i8 %phi
+}
+
+declare void @dummy()
diff --git a/llvm/test/Transforms/PhaseOrdering/thread-uncond-bb.ll b/llvm/test/Transforms/PhaseOrdering/thread-uncond-bb.ll
new file mode 100644
index 000000000000..17146d7d5987
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/thread-uncond-bb.ll
@@ -0,0 +1,62 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt < %s -O3 -S | FileCheck %s
+
+define i32 @thread_uncond_bb_cmp(i1 %c, i32 %v) {
+; CHECK-LABEL: define i32 @thread_uncond_bb_cmp(
+; CHECK-SAME: i1 [[C:%.*]], i32 [[V:%.*]]) local_unnamed_addr {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C]], label [[DO_END:%.*]], label [[IF_THEN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    tail call void @dummy()
+; CHECK-NEXT:    br label [[DO_END]]
+; CHECK:       return:
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[V]], [[IF_THEN]] ]
+; CHECK-NEXT:    ret i32 [[RETVAL]]
+;
+entry:
+  br i1 %c, label %do.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  call void @dummy()
+  %tobool = icmp eq i32 %v, 0
+  br i1 %tobool, label %do.end, label %return
+
+do.end:                                           ; preds = %entry, %if.then
+  br label %return
+
+return:                                           ; preds = %if.then, %do.end
+  %retval = phi i32 [ 0, %do.end ], [ %v, %if.then ]
+  ret i32 %retval
+}
+
+define i32 @thread_uncond_bb_cmp_zext(i1 %c, i32 %v) {
+; CHECK-LABEL: define i32 @thread_uncond_bb_cmp_zext(
+; CHECK-SAME: i1 [[C:%.*]], i32 [[V:%.*]]) local_unnamed_addr {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C]], label [[DO_END:%.*]], label [[IF_THEN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    tail call void @dummy()
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[V]], 0
+; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = zext i1 [[TOBOOL]] to i32
+; CHECK-NEXT:    br label [[DO_END]]
+; CHECK:       return:
+; CHECK-NEXT:    [[RETVAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[SPEC_SELECT]], [[IF_THEN]] ]
+; CHECK-NEXT:    ret i32 [[RETVAL]]
+;
+entry:
+  br i1 %c, label %do.end, label %if.then
+
+if.then:                                          ; preds = %entry
+  call void @dummy()
+  %tobool = icmp eq i32 %v, 0
+  br i1 %tobool, label %do.end, label %return
+
+do.end:                                           ; preds = %entry, %if.then
+  br label %return
+
+return:                                           ; preds = %if.then, %do.end
+  %retval = phi i32 [ 0, %do.end ], [ 1, %if.then ]
+  ret i32 %retval
+}
+
+declare void @dummy()
-- 
GitLab


From 32b74ca6e41768c91eee8b8ca26235b110a65deb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Mon, 15 Apr 2024 15:41:33 +0200
Subject: [PATCH 378/464] [clang][Interp] Load value from MemberExpr if
 required

---
 clang/lib/AST/Interp/ByteCodeExprGen.cpp | 17 ++++++++++++++---
 clang/test/AST/Interp/cxx03.cpp          | 14 ++++++++++++++
 2 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
index a069f3ec27e7..00c4a9f16130 100644
--- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp
+++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
@@ -1272,12 +1272,23 @@ bool ByteCodeExprGen<Emitter>::VisitMemberExpr(const MemberExpr *E) {
   if (DiscardResult)
     return this->discard(Base);
 
+  // MemberExprs are almost always lvalues, in which case we don't need to
+  // do the load. But sometimes they aren't.
+  const auto maybeLoadValue = [&]() -> bool {
+    if (E->isGLValue())
+      return true;
+    if (std::optional<PrimType> T = classify(E))
+      return this->emitLoadPop(*T, E);
+    return false;
+  };
+
   if (const auto *VD = dyn_cast<VarDecl>(Member)) {
     // I am almost confident in saying that a var decl must be static
     // and therefore registered as a global variable. But this will probably
     // turn out to be wrong some time in the future, as always.
     if (auto GlobalIndex = P.getGlobal(VD))
-      return this->emitGetPtrGlobal(*GlobalIndex, E);
+      return this->emitGetPtrGlobal(*GlobalIndex, E) && maybeLoadValue();
+    return false;
   }
 
   if (Initializing) {
@@ -1295,8 +1306,8 @@ bool ByteCodeExprGen<Emitter>::VisitMemberExpr(const MemberExpr *E) {
     const Record::Field *F = R->getField(FD);
     // Leave a pointer to the field on the stack.
     if (F->Decl->getType()->isReferenceType())
-      return this->emitGetFieldPop(PT_Ptr, F->Offset, E);
-    return this->emitGetPtrField(F->Offset, E);
+      return this->emitGetFieldPop(PT_Ptr, F->Offset, E) && maybeLoadValue();
+    return this->emitGetPtrField(F->Offset, E) && maybeLoadValue();
   }
 
   return false;
diff --git a/clang/test/AST/Interp/cxx03.cpp b/clang/test/AST/Interp/cxx03.cpp
index d30cbb2fd7a2..b6aaf0840cfb 100644
--- a/clang/test/AST/Interp/cxx03.cpp
+++ b/clang/test/AST/Interp/cxx03.cpp
@@ -10,3 +10,17 @@ namespace NonInitializingMemberExpr {
                                                                                               // both-note {{required by}} \
                                                                                               // both-note {{subexpression not valid}}
 }
+
+
+namespace NonLValueMemberExpr {
+  struct PODType {
+    int value;
+  };
+
+#define ATTR __attribute__((require_constant_initialization))
+  struct TT1 {
+    ATTR static const int &subobj_init;
+  };
+
+  const int &TT1::subobj_init = PODType().value;
+}
-- 
GitLab


From 92e96c7bbacbb477265c7e5ff6c49a6de5d4ee69 Mon Sep 17 00:00:00 2001
From: Malay Sanghi <malay.sanghi@intel.com>
Date: Tue, 16 Apr 2024 16:36:17 +0530
Subject: [PATCH 379/464] [X86][GISel] Add DU chain lookups for LOAD & STORE
 (#87453)

For G_LOAD and G_STORE we want this information during regbankselect.
Today we treat load dest as integer and insert converts.

---------

Co-authored-by: Evgenii Kudriashov <evgenii.kudriashov@intel.com>
---
 llvm/include/llvm/CodeGen/GlobalISel/Utils.h  |   4 +
 llvm/lib/CodeGen/GlobalISel/Utils.cpp         |  44 +++++
 .../AArch64/GISel/AArch64RegisterBankInfo.cpp |  37 -----
 llvm/lib/Target/Mips/MipsRegisterBankInfo.cpp |  24 +--
 .../PowerPC/GISel/PPCRegisterBankInfo.cpp     |  39 +----
 .../RISCV/GISel/RISCVRegisterBankInfo.cpp     |  40 -----
 .../Target/X86/GISel/X86RegisterBankInfo.cpp  | 139 +++++++++++++++-
 .../Target/X86/GISel/X86RegisterBankInfo.h    |  16 ++
 llvm/test/CodeGen/X86/GlobalISel/fconstant.ll |  13 +-
 .../regbankselect-sse-intrinsics.ll           | 153 ++++++++++++++++++
 .../X86/GlobalISel/regbankselect-x87.ll       |  29 ++--
 11 files changed, 367 insertions(+), 171 deletions(-)
 create mode 100644 llvm/test/CodeGen/X86/GlobalISel/regbankselect-sse-intrinsics.ll

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
index 807cec3c177d..c4174cee5e10 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/Utils.h
@@ -555,5 +555,9 @@ void eraseInstr(MachineInstr &MI, MachineRegisterInfo &MRI,
 /// debug users of \p MI by writing the effect of \p MI in a DIExpression.
 void salvageDebugInfo(const MachineRegisterInfo &MRI, MachineInstr &MI);
 
+/// Returns whether opcode \p Opc is a pre-isel generic floating-point opcode,
+/// having only floating-point operands.
+bool isPreISelGenericFloatingPointOpcode(unsigned Opc);
+
 } // End namespace llvm.
 #endif
diff --git a/llvm/lib/CodeGen/GlobalISel/Utils.cpp b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
index c3bc3203b636..ae43e9ccf611 100644
--- a/llvm/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Utils.cpp
@@ -1665,3 +1665,47 @@ void llvm::salvageDebugInfo(const MachineRegisterInfo &MRI, MachineInstr &MI) {
     }
   }
 }
+
+bool llvm::isPreISelGenericFloatingPointOpcode(unsigned Opc) {
+  switch (Opc) {
+  case TargetOpcode::G_FABS:
+  case TargetOpcode::G_FADD:
+  case TargetOpcode::G_FCANONICALIZE:
+  case TargetOpcode::G_FCEIL:
+  case TargetOpcode::G_FCONSTANT:
+  case TargetOpcode::G_FCOPYSIGN:
+  case TargetOpcode::G_FCOS:
+  case TargetOpcode::G_FDIV:
+  case TargetOpcode::G_FEXP2:
+  case TargetOpcode::G_FEXP:
+  case TargetOpcode::G_FFLOOR:
+  case TargetOpcode::G_FLOG10:
+  case TargetOpcode::G_FLOG2:
+  case TargetOpcode::G_FLOG:
+  case TargetOpcode::G_FMA:
+  case TargetOpcode::G_FMAD:
+  case TargetOpcode::G_FMAXIMUM:
+  case TargetOpcode::G_FMAXNUM:
+  case TargetOpcode::G_FMAXNUM_IEEE:
+  case TargetOpcode::G_FMINIMUM:
+  case TargetOpcode::G_FMINNUM:
+  case TargetOpcode::G_FMINNUM_IEEE:
+  case TargetOpcode::G_FMUL:
+  case TargetOpcode::G_FNEARBYINT:
+  case TargetOpcode::G_FNEG:
+  case TargetOpcode::G_FPEXT:
+  case TargetOpcode::G_FPOW:
+  case TargetOpcode::G_FPTRUNC:
+  case TargetOpcode::G_FREM:
+  case TargetOpcode::G_FRINT:
+  case TargetOpcode::G_FSIN:
+  case TargetOpcode::G_FSQRT:
+  case TargetOpcode::G_FSUB:
+  case TargetOpcode::G_INTRINSIC_ROUND:
+  case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
+  case TargetOpcode::G_INTRINSIC_TRUNC:
+    return true;
+  default:
+    return false;
+  }
+}
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp
index d39de770eaf1..d5c4ce1888e7 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp
@@ -424,43 +424,6 @@ void AArch64RegisterBankInfo::applyMappingImpl(
   }
 }
 
-/// Returns whether opcode \p Opc is a pre-isel generic floating-point opcode,
-/// having only floating-point operands.
-static bool isPreISelGenericFloatingPointOpcode(unsigned Opc) {
-  switch (Opc) {
-  case TargetOpcode::G_FADD:
-  case TargetOpcode::G_FSUB:
-  case TargetOpcode::G_FMUL:
-  case TargetOpcode::G_FMA:
-  case TargetOpcode::G_FDIV:
-  case TargetOpcode::G_FCONSTANT:
-  case TargetOpcode::G_FPEXT:
-  case TargetOpcode::G_FPTRUNC:
-  case TargetOpcode::G_FCEIL:
-  case TargetOpcode::G_FFLOOR:
-  case TargetOpcode::G_FNEARBYINT:
-  case TargetOpcode::G_FNEG:
-  case TargetOpcode::G_FCOS:
-  case TargetOpcode::G_FSIN:
-  case TargetOpcode::G_FLOG10:
-  case TargetOpcode::G_FLOG:
-  case TargetOpcode::G_FLOG2:
-  case TargetOpcode::G_FSQRT:
-  case TargetOpcode::G_FABS:
-  case TargetOpcode::G_FEXP:
-  case TargetOpcode::G_FRINT:
-  case TargetOpcode::G_INTRINSIC_TRUNC:
-  case TargetOpcode::G_INTRINSIC_ROUND:
-  case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
-  case TargetOpcode::G_FMAXNUM:
-  case TargetOpcode::G_FMINNUM:
-  case TargetOpcode::G_FMAXIMUM:
-  case TargetOpcode::G_FMINIMUM:
-    return true;
-  }
-  return false;
-}
-
 const RegisterBankInfo::InstructionMapping &
 AArch64RegisterBankInfo::getSameKindOfOperandsMapping(
     const MachineInstr &MI) const {
diff --git a/llvm/lib/Target/Mips/MipsRegisterBankInfo.cpp b/llvm/lib/Target/Mips/MipsRegisterBankInfo.cpp
index 6af1fd8c88e5..62b58cba9f24 100644
--- a/llvm/lib/Target/Mips/MipsRegisterBankInfo.cpp
+++ b/llvm/lib/Target/Mips/MipsRegisterBankInfo.cpp
@@ -104,26 +104,6 @@ MipsRegisterBankInfo::getRegBankFromRegClass(const TargetRegisterClass &RC,
   }
 }
 
-// Instructions where all register operands are floating point.
-static bool isFloatingPointOpcode(unsigned Opc) {
-  switch (Opc) {
-  case TargetOpcode::G_FCONSTANT:
-  case TargetOpcode::G_FADD:
-  case TargetOpcode::G_FSUB:
-  case TargetOpcode::G_FMUL:
-  case TargetOpcode::G_FDIV:
-  case TargetOpcode::G_FABS:
-  case TargetOpcode::G_FSQRT:
-  case TargetOpcode::G_FCEIL:
-  case TargetOpcode::G_FFLOOR:
-  case TargetOpcode::G_FPEXT:
-  case TargetOpcode::G_FPTRUNC:
-    return true;
-  default:
-    return false;
-  }
-}
-
 // Instructions where use operands are floating point registers.
 // Def operands are general purpose.
 static bool isFloatingPointOpcodeUse(unsigned Opc) {
@@ -133,7 +113,7 @@ static bool isFloatingPointOpcodeUse(unsigned Opc) {
   case TargetOpcode::G_FCMP:
     return true;
   default:
-    return isFloatingPointOpcode(Opc);
+    return isPreISelGenericFloatingPointOpcode(Opc);
   }
 }
 
@@ -145,7 +125,7 @@ static bool isFloatingPointOpcodeDef(unsigned Opc) {
   case TargetOpcode::G_UITOFP:
     return true;
   default:
-    return isFloatingPointOpcode(Opc);
+    return isPreISelGenericFloatingPointOpcode(Opc);
   }
 }
 
diff --git a/llvm/lib/Target/PowerPC/GISel/PPCRegisterBankInfo.cpp b/llvm/lib/Target/PowerPC/GISel/PPCRegisterBankInfo.cpp
index 6aeef145e307..125a49de7b27 100644
--- a/llvm/lib/Target/PowerPC/GISel/PPCRegisterBankInfo.cpp
+++ b/llvm/lib/Target/PowerPC/GISel/PPCRegisterBankInfo.cpp
@@ -13,6 +13,7 @@
 #include "PPCRegisterBankInfo.h"
 #include "PPCRegisterInfo.h"
 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
+#include "llvm/CodeGen/GlobalISel/Utils.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/Support/Debug.h"
@@ -239,44 +240,6 @@ PPCRegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
   return getInstructionMapping(MappingID, Cost, OperandsMapping, NumOperands);
 }
 
-/// Returns whether opcode \p Opc is a pre-isel generic floating-point opcode,
-/// having only floating-point operands.
-/// FIXME: this is copied from target AArch64. Needs some code refactor here to
-/// put this function in GlobalISel/Utils.cpp.
-static bool isPreISelGenericFloatingPointOpcode(unsigned Opc) {
-  switch (Opc) {
-  case TargetOpcode::G_FADD:
-  case TargetOpcode::G_FSUB:
-  case TargetOpcode::G_FMUL:
-  case TargetOpcode::G_FMA:
-  case TargetOpcode::G_FDIV:
-  case TargetOpcode::G_FCONSTANT:
-  case TargetOpcode::G_FPEXT:
-  case TargetOpcode::G_FPTRUNC:
-  case TargetOpcode::G_FCEIL:
-  case TargetOpcode::G_FFLOOR:
-  case TargetOpcode::G_FNEARBYINT:
-  case TargetOpcode::G_FNEG:
-  case TargetOpcode::G_FCOS:
-  case TargetOpcode::G_FSIN:
-  case TargetOpcode::G_FLOG10:
-  case TargetOpcode::G_FLOG:
-  case TargetOpcode::G_FLOG2:
-  case TargetOpcode::G_FSQRT:
-  case TargetOpcode::G_FABS:
-  case TargetOpcode::G_FEXP:
-  case TargetOpcode::G_FRINT:
-  case TargetOpcode::G_INTRINSIC_TRUNC:
-  case TargetOpcode::G_INTRINSIC_ROUND:
-  case TargetOpcode::G_FMAXNUM:
-  case TargetOpcode::G_FMINNUM:
-  case TargetOpcode::G_FMAXIMUM:
-  case TargetOpcode::G_FMINIMUM:
-    return true;
-  }
-  return false;
-}
-
 /// \returns true if a given intrinsic \p ID only uses and defines FPRs.
 static bool isFPIntrinsic(unsigned ID) {
   // TODO: Add more intrinsics.
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp
index 86e44343b508..cc534f29685f 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVRegisterBankInfo.cpp
@@ -154,46 +154,6 @@ static const RegisterBankInfo::ValueMapping *getFPValueMapping(unsigned Size) {
   return &RISCV::ValueMappings[Idx];
 }
 
-/// Returns whether opcode \p Opc is a pre-isel generic floating-point opcode,
-/// having only floating-point operands.
-/// FIXME: this is copied from target AArch64. Needs some code refactor here to
-/// put this function in GlobalISel/Utils.cpp.
-static bool isPreISelGenericFloatingPointOpcode(unsigned Opc) {
-  switch (Opc) {
-  case TargetOpcode::G_FADD:
-  case TargetOpcode::G_FSUB:
-  case TargetOpcode::G_FMUL:
-  case TargetOpcode::G_FMA:
-  case TargetOpcode::G_FDIV:
-  case TargetOpcode::G_FCONSTANT:
-  case TargetOpcode::G_FPEXT:
-  case TargetOpcode::G_FPTRUNC:
-  case TargetOpcode::G_FCEIL:
-  case TargetOpcode::G_FFLOOR:
-  case TargetOpcode::G_FNEARBYINT:
-  case TargetOpcode::G_FNEG:
-  case TargetOpcode::G_FCOPYSIGN:
-  case TargetOpcode::G_FCOS:
-  case TargetOpcode::G_FSIN:
-  case TargetOpcode::G_FLOG10:
-  case TargetOpcode::G_FLOG:
-  case TargetOpcode::G_FLOG2:
-  case TargetOpcode::G_FSQRT:
-  case TargetOpcode::G_FABS:
-  case TargetOpcode::G_FEXP:
-  case TargetOpcode::G_FRINT:
-  case TargetOpcode::G_INTRINSIC_TRUNC:
-  case TargetOpcode::G_INTRINSIC_ROUND:
-  case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
-  case TargetOpcode::G_FMAXNUM:
-  case TargetOpcode::G_FMINNUM:
-  case TargetOpcode::G_FMAXIMUM:
-  case TargetOpcode::G_FMINIMUM:
-    return true;
-  }
-  return false;
-}
-
 // TODO: Make this more like AArch64?
 bool RISCVRegisterBankInfo::hasFPConstraints(
     const MachineInstr &MI, const MachineRegisterInfo &MRI,
diff --git a/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.cpp b/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.cpp
index e7c9e60ba95f..9e85424e76e6 100644
--- a/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.cpp
+++ b/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.cpp
@@ -13,10 +13,13 @@
 #include "X86RegisterBankInfo.h"
 #include "X86InstrInfo.h"
 #include "X86Subtarget.h"
+#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
+#include "llvm/CodeGen/GlobalISel/Utils.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/RegisterBank.h"
 #include "llvm/CodeGen/RegisterBankInfo.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
+#include "llvm/IR/IntrinsicsX86.h"
 
 #define GET_TARGET_REGBANK_IMPL
 #include "X86GenRegisterBank.inc"
@@ -68,6 +71,98 @@ X86RegisterBankInfo::getRegBankFromRegClass(const TargetRegisterClass &RC,
   llvm_unreachable("Unsupported register kind yet.");
 }
 
+// \returns true if a given intrinsic only uses and defines FPRs.
+static bool isFPIntrinsic(const MachineRegisterInfo &MRI,
+                          const MachineInstr &MI) {
+  // TODO: Add more intrinsics.
+  switch (cast<GIntrinsic>(MI).getIntrinsicID()) {
+  default:
+    return false;
+  // SSE1
+  case Intrinsic::x86_sse_rcp_ss:
+  case Intrinsic::x86_sse_rcp_ps:
+  case Intrinsic::x86_sse_rsqrt_ss:
+  case Intrinsic::x86_sse_rsqrt_ps:
+  case Intrinsic::x86_sse_min_ss:
+  case Intrinsic::x86_sse_min_ps:
+  case Intrinsic::x86_sse_max_ss:
+  case Intrinsic::x86_sse_max_ps:
+    return true;
+  }
+  return false;
+}
+
+bool X86RegisterBankInfo::hasFPConstraints(const MachineInstr &MI,
+                                           const MachineRegisterInfo &MRI,
+                                           const TargetRegisterInfo &TRI,
+                                           unsigned Depth) const {
+  unsigned Op = MI.getOpcode();
+  if (Op == TargetOpcode::G_INTRINSIC && isFPIntrinsic(MRI, MI))
+    return true;
+
+  // Do we have an explicit floating point instruction?
+  if (isPreISelGenericFloatingPointOpcode(Op))
+    return true;
+
+  // No. Check if we have a copy-like instruction. If we do, then we could
+  // still be fed by floating point instructions.
+  if (Op != TargetOpcode::COPY && !MI.isPHI() &&
+      !isPreISelGenericOptimizationHint(Op))
+    return false;
+
+  // Check if we already know the register bank.
+  auto *RB = getRegBank(MI.getOperand(0).getReg(), MRI, TRI);
+  if (RB == &getRegBank(X86::PSRRegBankID))
+    return true;
+  if (RB == &getRegBank(X86::GPRRegBankID))
+    return false;
+
+  // We don't know anything.
+  //
+  // If we have a phi, we may be able to infer that it will be assigned a fp
+  // type based off of its inputs.
+  if (!MI.isPHI() || Depth > MaxFPRSearchDepth)
+    return false;
+
+  return any_of(MI.explicit_uses(), [&](const MachineOperand &Op) {
+    return Op.isReg() &&
+           onlyDefinesFP(*MRI.getVRegDef(Op.getReg()), MRI, TRI, Depth + 1);
+  });
+}
+
+bool X86RegisterBankInfo::onlyUsesFP(const MachineInstr &MI,
+                                     const MachineRegisterInfo &MRI,
+                                     const TargetRegisterInfo &TRI,
+                                     unsigned Depth) const {
+  switch (MI.getOpcode()) {
+  case TargetOpcode::G_FPTOSI:
+  case TargetOpcode::G_FPTOUI:
+  case TargetOpcode::G_FCMP:
+  case TargetOpcode::G_LROUND:
+  case TargetOpcode::G_LLROUND:
+  case TargetOpcode::G_INTRINSIC_TRUNC:
+  case TargetOpcode::G_INTRINSIC_ROUND:
+    return true;
+  default:
+    break;
+  }
+  return hasFPConstraints(MI, MRI, TRI, Depth);
+}
+
+bool X86RegisterBankInfo::onlyDefinesFP(const MachineInstr &MI,
+                                        const MachineRegisterInfo &MRI,
+                                        const TargetRegisterInfo &TRI,
+                                        unsigned Depth) const {
+  switch (MI.getOpcode()) {
+  case TargetOpcode::G_SITOFP:
+  case TargetOpcode::G_UITOFP:
+    return true;
+  default:
+    break;
+  }
+  return hasFPConstraints(MI, MRI, TRI, Depth);
+}
+
 X86GenRegisterBankInfo::PartialMappingIdx
 X86GenRegisterBankInfo::getPartialMappingIdx(const MachineInstr &MI,
                                              const LLT &Ty, bool isFP) {
@@ -180,11 +275,13 @@ X86RegisterBankInfo::getSameOperandsMapping(const MachineInstr &MI,
 const RegisterBankInfo::InstructionMapping &
 X86RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
   const MachineFunction &MF = *MI.getParent()->getParent();
+  const TargetSubtargetInfo &STI = MF.getSubtarget();
+  const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
   const MachineRegisterInfo &MRI = MF.getRegInfo();
   unsigned Opc = MI.getOpcode();
 
-  // Try the default logic for non-generic instructions that are either copies
-  // or already have some operands assigned to banks.
+  // Try the default logic for non-generic instructions that are either
+  // copies or already have some operands assigned to banks.
   if (!isPreISelGenericOpcode(Opc) || Opc == TargetOpcode::G_PHI) {
     const InstructionMapping &Mapping = getInstrMappingImpl(MI);
     if (Mapping.isValid())
@@ -221,13 +318,14 @@ X86RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
   case TargetOpcode::G_FPEXT:
   case TargetOpcode::G_FPTRUNC:
   case TargetOpcode::G_FCONSTANT:
-    // Instruction having only floating-point operands (all scalars in VECRReg)
+    // Instruction having only floating-point operands (all scalars in
+    // VECRReg)
     getInstrPartialMappingIdxs(MI, MRI, /* isFP= */ true, OpRegBankIdx);
     break;
   case TargetOpcode::G_SITOFP:
   case TargetOpcode::G_FPTOSI: {
-    // Some of the floating-point instructions have mixed GPR and FP operands:
-    // fine-tune the computed mapping.
+    // Some of the floating-point instructions have mixed GPR and FP
+    // operands: fine-tune the computed mapping.
     auto &Op0 = MI.getOperand(0);
     auto &Op1 = MI.getOperand(1);
     const LLT Ty0 = MRI.getType(Op0.getReg());
@@ -271,9 +369,36 @@ X86RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
 
     getInstrPartialMappingIdxs(MI, MRI, /* isFP= */ isFPTrunc || isFPAnyExt,
                                OpRegBankIdx);
-  } break;
+    break;
+  }
+  case TargetOpcode::G_LOAD: {
+    // Check if that load feeds fp instructions.
+    // In that case, we want the default mapping to be on FPR
+    // instead of blind map every scalar to GPR.
+    bool IsFP = any_of(MRI.use_nodbg_instructions(cast<GLoad>(MI).getDstReg()),
+                       [&](const MachineInstr &UseMI) {
+                         // If we have at least one direct use in a FP
+                         // instruction, assume this was a floating point load
+                         // in the IR. If it was not, we would have had a
+                         // bitcast before reaching that instruction.
+                         return onlyUsesFP(UseMI, MRI, TRI);
+                       });
+    getInstrPartialMappingIdxs(MI, MRI, IsFP, OpRegBankIdx);
+    break;
+  }
+  case TargetOpcode::G_STORE: {
+    // Check if that store is fed by fp instructions.
+    Register VReg = cast<GStore>(MI).getValueReg();
+    if (!VReg)
+      break;
+    MachineInstr *DefMI = MRI.getVRegDef(VReg);
+    bool IsFP = onlyDefinesFP(*DefMI, MRI, TRI);
+    getInstrPartialMappingIdxs(MI, MRI, IsFP, OpRegBankIdx);
+    break;
+  }
   default:
-    // Track the bank of each register, use NotFP mapping (all scalars in GPRs)
+    // Track the bank of each register, use NotFP mapping (all scalars in
+    // GPRs)
     getInstrPartialMappingIdxs(MI, MRI, /* isFP= */ false, OpRegBankIdx);
     break;
   }
diff --git a/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.h b/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.h
index 989c5956ad59..8f38e717e36b 100644
--- a/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.h
+++ b/llvm/lib/Target/X86/GISel/X86RegisterBankInfo.h
@@ -62,6 +62,22 @@ private:
                        const SmallVectorImpl<PartialMappingIdx> &OpRegBankIdx,
                        SmallVectorImpl<const ValueMapping *> &OpdsMapping);
 
+  // Maximum recursion depth for hasFPConstraints.
+  const unsigned MaxFPRSearchDepth = 2;
+
+  /// \returns true if \p MI only uses and defines FPRs.
+  bool hasFPConstraints(const MachineInstr &MI, const MachineRegisterInfo &MRI,
+                        const TargetRegisterInfo &TRI,
+                        unsigned Depth = 0) const;
+
+  /// \returns true if \p MI only uses FPRs.
+  bool onlyUsesFP(const MachineInstr &MI, const MachineRegisterInfo &MRI,
+                  const TargetRegisterInfo &TRI, unsigned Depth = 0) const;
+
+  /// \returns true if \p MI only defines FPRs.
+  bool onlyDefinesFP(const MachineInstr &MI, const MachineRegisterInfo &MRI,
+                     const TargetRegisterInfo &TRI, unsigned Depth = 0) const;
+
 public:
   X86RegisterBankInfo(const TargetRegisterInfo &TRI);
 
diff --git a/llvm/test/CodeGen/X86/GlobalISel/fconstant.ll b/llvm/test/CodeGen/X86/GlobalISel/fconstant.ll
index a9b2037e9947..8d2ee3c50f21 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/fconstant.ll
+++ b/llvm/test/CodeGen/X86/GlobalISel/fconstant.ll
@@ -10,27 +10,22 @@ define void @test_float(ptr %a , float %b) {
 ; CHECK64_SMALL:       # %bb.0: # %entry
 ; CHECK64_SMALL-NEXT:    movss {{.*#+}} xmm1 = [5.5E+0,0.0E+0,0.0E+0,0.0E+0]
 ; CHECK64_SMALL-NEXT:    addss %xmm0, %xmm1
-; CHECK64_SMALL-NEXT:    movd %xmm1, %eax
-; CHECK64_SMALL-NEXT:    movl %eax, (%rdi)
+; CHECK64_SMALL-NEXT:    movss %xmm1, (%rdi)
 ; CHECK64_SMALL-NEXT:    retq
 ;
 ; CHECK64_LARGE-LABEL: test_float:
 ; CHECK64_LARGE:       # %bb.0: # %entry
 ; CHECK64_LARGE-NEXT:    movabsq ${{\.?LCPI[0-9]+_[0-9]+}}, %rax
 ; CHECK64_LARGE-NEXT:    addss (%rax), %xmm0
-; CHECK64_LARGE-NEXT:    movd %xmm0, %eax
-; CHECK64_LARGE-NEXT:    movl %eax, (%rdi)
+; CHECK64_LARGE-NEXT:    movss %xmm0, (%rdi)
 ; CHECK64_LARGE-NEXT:    retq
 ;
 ; CHECK32-LABEL: test_float:
 ; CHECK32:       # %bb.0: # %entry
 ; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; CHECK32-NEXT:    movss {{.*#+}} xmm0 = [5.5E+0,0.0E+0,0.0E+0,0.0E+0]
-; CHECK32-NEXT:    movd %ecx, %xmm1
-; CHECK32-NEXT:    addss %xmm0, %xmm1
-; CHECK32-NEXT:    movd %xmm1, %ecx
-; CHECK32-NEXT:    movl %ecx, (%eax)
+; CHECK32-NEXT:    addss {{[0-9]+}}(%esp), %xmm0
+; CHECK32-NEXT:    movss %xmm0, (%eax)
 ; CHECK32-NEXT:    retl
 entry:
   %aa = fadd float 5.500000e+00, %b
diff --git a/llvm/test/CodeGen/X86/GlobalISel/regbankselect-sse-intrinsics.ll b/llvm/test/CodeGen/X86/GlobalISel/regbankselect-sse-intrinsics.ll
new file mode 100644
index 000000000000..3388af605d96
--- /dev/null
+++ b/llvm/test/CodeGen/X86/GlobalISel/regbankselect-sse-intrinsics.ll
@@ -0,0 +1,153 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
+; RUN: llc < %s -mtriple=i686-- -mattr=+sse -global-isel -stop-after=regbankselect | FileCheck %s
+
+define void @test_x86_sse_max_ps(ptr %p1, ptr %p2) {
+  ; CHECK-LABEL: name: test_x86_sse_max_ps
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK-NEXT:   [[FRAME_INDEX:%[0-9]+]]:gpr(p0) = G_FRAME_INDEX %fixed-stack.1
+  ; CHECK-NEXT:   [[LOAD:%[0-9]+]]:gpr(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load (p0) from %fixed-stack.1)
+  ; CHECK-NEXT:   [[FRAME_INDEX1:%[0-9]+]]:gpr(p0) = G_FRAME_INDEX %fixed-stack.0
+  ; CHECK-NEXT:   [[LOAD1:%[0-9]+]]:gpr(p0) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load (p0) from %fixed-stack.0)
+  ; CHECK-NEXT:   [[LOAD2:%[0-9]+]]:vecr(<4 x s32>) = G_LOAD [[LOAD]](p0) :: (load (<4 x s32>) from %ir.p1)
+  ; CHECK-NEXT:   [[LOAD3:%[0-9]+]]:vecr(<4 x s32>) = G_LOAD [[LOAD1]](p0) :: (load (<4 x s32>) from %ir.p2)
+  ; CHECK-NEXT:   [[INT:%[0-9]+]]:vecr(<4 x s32>) = G_INTRINSIC intrinsic(@llvm.x86.sse.max.ps), [[LOAD2]](<4 x s32>), [[LOAD3]](<4 x s32>)
+  ; CHECK-NEXT:   G_STORE [[INT]](<4 x s32>), [[LOAD]](p0) :: (store (<4 x s32>) into %ir.p1)
+  ; CHECK-NEXT:   RET 0
+  %a0 = load <4 x float>, ptr %p1, align 16
+  %a1 = load <4 x float>, ptr %p2, align 16
+  %res = call <4 x float> @llvm.x86.sse.max.ps(<4 x float> %a0, <4 x float> %a1) ; <<4 x float>> [#uses=1]
+  store <4 x float> %res, ptr %p1
+  ret void
+}
+declare <4 x float> @llvm.x86.sse.max.ps(<4 x float>, <4 x float>) nounwind readnone
+
+
+define void @test_x86_sse_max_ss(ptr %p1, ptr %p2) {
+  ; CHECK-LABEL: name: test_x86_sse_max_ss
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK-NEXT:   [[FRAME_INDEX:%[0-9]+]]:gpr(p0) = G_FRAME_INDEX %fixed-stack.1
+  ; CHECK-NEXT:   [[LOAD:%[0-9]+]]:gpr(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load (p0) from %fixed-stack.1)
+  ; CHECK-NEXT:   [[FRAME_INDEX1:%[0-9]+]]:gpr(p0) = G_FRAME_INDEX %fixed-stack.0
+  ; CHECK-NEXT:   [[LOAD1:%[0-9]+]]:gpr(p0) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load (p0) from %fixed-stack.0)
+  ; CHECK-NEXT:   [[LOAD2:%[0-9]+]]:vecr(<4 x s32>) = G_LOAD [[LOAD]](p0) :: (load (<4 x s32>) from %ir.p1)
+  ; CHECK-NEXT:   [[LOAD3:%[0-9]+]]:vecr(<4 x s32>) = G_LOAD [[LOAD1]](p0) :: (load (<4 x s32>) from %ir.p2)
+  ; CHECK-NEXT:   [[INT:%[0-9]+]]:vecr(<4 x s32>) = G_INTRINSIC intrinsic(@llvm.x86.sse.max.ss), [[LOAD2]](<4 x s32>), [[LOAD3]](<4 x s32>)
+  ; CHECK-NEXT:   G_STORE [[INT]](<4 x s32>), [[LOAD]](p0) :: (store (<4 x s32>) into %ir.p1)
+  ; CHECK-NEXT:   RET 0
+  %a0 = load <4 x float>, ptr %p1, align 16
+  %a1 = load <4 x float>, ptr %p2, align 16
+  %res = call <4 x float> @llvm.x86.sse.max.ss(<4 x float> %a0, <4 x float> %a1) ; <<4 x float>> [#uses=1]
+  store <4 x float> %res, ptr %p1
+  ret void
+}
+declare <4 x float> @llvm.x86.sse.max.ss(<4 x float>, <4 x float>) nounwind readnone
+
+
+define void @test_x86_sse_min_ps(ptr %p1, ptr %p2) {
+  ; CHECK-LABEL: name: test_x86_sse_min_ps
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK-NEXT:   [[FRAME_INDEX:%[0-9]+]]:gpr(p0) = G_FRAME_INDEX %fixed-stack.1
+  ; CHECK-NEXT:   [[LOAD:%[0-9]+]]:gpr(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load (p0) from %fixed-stack.1)
+  ; CHECK-NEXT:   [[FRAME_INDEX1:%[0-9]+]]:gpr(p0) = G_FRAME_INDEX %fixed-stack.0
+  ; CHECK-NEXT:   [[LOAD1:%[0-9]+]]:gpr(p0) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load (p0) from %fixed-stack.0)
+  ; CHECK-NEXT:   [[LOAD2:%[0-9]+]]:vecr(<4 x s32>) = G_LOAD [[LOAD]](p0) :: (load (<4 x s32>) from %ir.p1)
+  ; CHECK-NEXT:   [[LOAD3:%[0-9]+]]:vecr(<4 x s32>) = G_LOAD [[LOAD1]](p0) :: (load (<4 x s32>) from %ir.p2)
+  ; CHECK-NEXT:   [[INT:%[0-9]+]]:vecr(<4 x s32>) = G_INTRINSIC intrinsic(@llvm.x86.sse.min.ps), [[LOAD2]](<4 x s32>), [[LOAD3]](<4 x s32>)
+  ; CHECK-NEXT:   G_STORE [[INT]](<4 x s32>), [[LOAD]](p0) :: (store (<4 x s32>) into %ir.p1)
+  ; CHECK-NEXT:   RET 0
+  %a0 = load <4 x float>, ptr %p1, align 16
+  %a1 = load <4 x float>, ptr %p2, align 16
+  %res = call <4 x float> @llvm.x86.sse.min.ps(<4 x float> %a0, <4 x float> %a1) ; <<4 x float>> [#uses=1]
+  store <4 x float> %res, ptr %p1
+  ret void
+}
+declare <4 x float> @llvm.x86.sse.min.ps(<4 x float>, <4 x float>) nounwind readnone
+
+
+define void @test_x86_sse_min_ss(ptr %p1, ptr %p2) {
+  ; CHECK-LABEL: name: test_x86_sse_min_ss
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK-NEXT:   [[FRAME_INDEX:%[0-9]+]]:gpr(p0) = G_FRAME_INDEX %fixed-stack.1
+  ; CHECK-NEXT:   [[LOAD:%[0-9]+]]:gpr(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load (p0) from %fixed-stack.1)
+  ; CHECK-NEXT:   [[FRAME_INDEX1:%[0-9]+]]:gpr(p0) = G_FRAME_INDEX %fixed-stack.0
+  ; CHECK-NEXT:   [[LOAD1:%[0-9]+]]:gpr(p0) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load (p0) from %fixed-stack.0)
+  ; CHECK-NEXT:   [[LOAD2:%[0-9]+]]:vecr(<4 x s32>) = G_LOAD [[LOAD]](p0) :: (load (<4 x s32>) from %ir.p1)
+  ; CHECK-NEXT:   [[LOAD3:%[0-9]+]]:vecr(<4 x s32>) = G_LOAD [[LOAD1]](p0) :: (load (<4 x s32>) from %ir.p2)
+  ; CHECK-NEXT:   [[INT:%[0-9]+]]:vecr(<4 x s32>) = G_INTRINSIC intrinsic(@llvm.x86.sse.min.ss), [[LOAD2]](<4 x s32>), [[LOAD3]](<4 x s32>)
+  ; CHECK-NEXT:   G_STORE [[INT]](<4 x s32>), [[LOAD]](p0) :: (store (<4 x s32>) into %ir.p1)
+  ; CHECK-NEXT:   RET 0
+  %a0 = load <4 x float>, ptr %p1, align 16
+  %a1 = load <4 x float>, ptr %p2, align 16
+  %res = call <4 x float> @llvm.x86.sse.min.ss(<4 x float> %a0, <4 x float> %a1) ; <<4 x float>> [#uses=1]
+  store <4 x float> %res, ptr %p1
+  ret void
+}
+declare <4 x float> @llvm.x86.sse.min.ss(<4 x float>, <4 x float>) nounwind readnone
+
+
+define void @test_x86_sse_rcp_ps(ptr %p1, ptr %p2) {
+  ; CHECK-LABEL: name: test_x86_sse_rcp_ps
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK-NEXT:   [[FRAME_INDEX:%[0-9]+]]:gpr(p0) = G_FRAME_INDEX %fixed-stack.1
+  ; CHECK-NEXT:   [[LOAD:%[0-9]+]]:gpr(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load (p0) from %fixed-stack.1)
+  ; CHECK-NEXT:   [[LOAD1:%[0-9]+]]:vecr(<4 x s32>) = G_LOAD [[LOAD]](p0) :: (load (<4 x s32>) from %ir.p1)
+  ; CHECK-NEXT:   [[INT:%[0-9]+]]:vecr(<4 x s32>) = G_INTRINSIC intrinsic(@llvm.x86.sse.rcp.ps), [[LOAD1]](<4 x s32>)
+  ; CHECK-NEXT:   G_STORE [[INT]](<4 x s32>), [[LOAD]](p0) :: (store (<4 x s32>) into %ir.p1)
+  ; CHECK-NEXT:   RET 0
+  %a0 = load <4 x float>, ptr %p1, align 16
+  %res = call <4 x float> @llvm.x86.sse.rcp.ps(<4 x float> %a0) ; <<4 x float>> [#uses=1]
+  store <4 x float> %res, ptr %p1
+  ret void
+}
+declare <4 x float> @llvm.x86.sse.rcp.ps(<4 x float>) nounwind readnone
+
+
+define void @test_x86_sse_rcp_ss(ptr %p1, ptr %p2) {
+  ; CHECK-LABEL: name: test_x86_sse_rcp_ss
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK-NEXT:   [[FRAME_INDEX:%[0-9]+]]:gpr(p0) = G_FRAME_INDEX %fixed-stack.1
+  ; CHECK-NEXT:   [[LOAD:%[0-9]+]]:gpr(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load (p0) from %fixed-stack.1)
+  ; CHECK-NEXT:   [[LOAD1:%[0-9]+]]:vecr(<4 x s32>) = G_LOAD [[LOAD]](p0) :: (load (<4 x s32>) from %ir.p1)
+  ; CHECK-NEXT:   [[INT:%[0-9]+]]:vecr(<4 x s32>) = G_INTRINSIC intrinsic(@llvm.x86.sse.rcp.ss), [[LOAD1]](<4 x s32>)
+  ; CHECK-NEXT:   G_STORE [[INT]](<4 x s32>), [[LOAD]](p0) :: (store (<4 x s32>) into %ir.p1)
+  ; CHECK-NEXT:   RET 0
+  %a0 = load <4 x float>, ptr %p1, align 16
+  %res = call <4 x float> @llvm.x86.sse.rcp.ss(<4 x float> %a0) ; <<4 x float>> [#uses=1]
+  store <4 x float> %res, ptr %p1
+  ret void
+}
+declare <4 x float> @llvm.x86.sse.rcp.ss(<4 x float>) nounwind readnone
+
+
+define void @test_x86_sse_rsqrt_ps(ptr %p1, ptr %p2) {
+  ; CHECK-LABEL: name: test_x86_sse_rsqrt_ps
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK-NEXT:   [[FRAME_INDEX:%[0-9]+]]:gpr(p0) = G_FRAME_INDEX %fixed-stack.1
+  ; CHECK-NEXT:   [[LOAD:%[0-9]+]]:gpr(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load (p0) from %fixed-stack.1)
+  ; CHECK-NEXT:   [[LOAD1:%[0-9]+]]:vecr(<4 x s32>) = G_LOAD [[LOAD]](p0) :: (load (<4 x s32>) from %ir.p1)
+  ; CHECK-NEXT:   [[INT:%[0-9]+]]:vecr(<4 x s32>) = G_INTRINSIC intrinsic(@llvm.x86.sse.rsqrt.ps), [[LOAD1]](<4 x s32>)
+  ; CHECK-NEXT:   G_STORE [[INT]](<4 x s32>), [[LOAD]](p0) :: (store (<4 x s32>) into %ir.p1)
+  ; CHECK-NEXT:   RET 0
+  %a0 = load <4 x float>, ptr %p1, align 16
+  %res = call <4 x float> @llvm.x86.sse.rsqrt.ps(<4 x float> %a0) ; <<4 x float>> [#uses=1]
+  store <4 x float> %res, ptr %p1
+  ret void
+}
+declare <4 x float> @llvm.x86.sse.rsqrt.ps(<4 x float>) nounwind readnone
+
+
+define void @test_x86_sse_rsqrt_ss(ptr %p1, ptr %p2) {
+  ; CHECK-LABEL: name: test_x86_sse_rsqrt_ss
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK-NEXT:   [[FRAME_INDEX:%[0-9]+]]:gpr(p0) = G_FRAME_INDEX %fixed-stack.1
+  ; CHECK-NEXT:   [[LOAD:%[0-9]+]]:gpr(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load (p0) from %fixed-stack.1)
+  ; CHECK-NEXT:   [[LOAD1:%[0-9]+]]:vecr(<4 x s32>) = G_LOAD [[LOAD]](p0) :: (load (<4 x s32>) from %ir.p1)
+  ; CHECK-NEXT:   [[INT:%[0-9]+]]:vecr(<4 x s32>) = G_INTRINSIC intrinsic(@llvm.x86.sse.rsqrt.ss), [[LOAD1]](<4 x s32>)
+  ; CHECK-NEXT:   G_STORE [[INT]](<4 x s32>), [[LOAD]](p0) :: (store (<4 x s32>) into %ir.p1)
+  ; CHECK-NEXT:   RET 0
+  %a0 = load <4 x float>, ptr %p1, align 16
+  %res = call <4 x float> @llvm.x86.sse.rsqrt.ss(<4 x float> %a0) ; <<4 x float>> [#uses=1]
+  store <4 x float> %res, ptr %p1
+  ret void
+}
+declare <4 x float> @llvm.x86.sse.rsqrt.ss(<4 x float>) nounwind readnone
diff --git a/llvm/test/CodeGen/X86/GlobalISel/regbankselect-x87.ll b/llvm/test/CodeGen/X86/GlobalISel/regbankselect-x87.ll
index d09db0f2474c..99d458a183a9 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/regbankselect-x87.ll
+++ b/llvm/test/CodeGen/X86/GlobalISel/regbankselect-x87.ll
@@ -142,7 +142,7 @@ define float @f4(float %val) {
   ; X86-LABEL: name: f4
   ; X86: bb.1 (%ir-block.0):
   ; X86-NEXT:   [[FRAME_INDEX:%[0-9]+]]:gpr(p0) = G_FRAME_INDEX %fixed-stack.0
-  ; X86-NEXT:   [[LOAD:%[0-9]+]]:gpr(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load (s32) from %fixed-stack.0)
+  ; X86-NEXT:   [[LOAD:%[0-9]+]]:psr(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load (s32) from %fixed-stack.0)
   ; X86-NEXT:   $fp0 = COPY [[LOAD]](s32)
   ; X86-NEXT:   RET 0, implicit $fp0
   ;
@@ -187,13 +187,10 @@ define void @f5(ptr %a, ptr %b) {
   ; X64-NEXT: {{  $}}
   ; X64-NEXT:   [[COPY:%[0-9]+]]:gpr(p0) = COPY $rdi
   ; X64-NEXT:   [[COPY1:%[0-9]+]]:gpr(p0) = COPY $rsi
-  ; X64-NEXT:   [[LOAD:%[0-9]+]]:gpr(s64) = G_LOAD [[COPY]](p0) :: (load (s64) from %ir.a)
-  ; X64-NEXT:   [[LOAD1:%[0-9]+]]:gpr(s64) = G_LOAD [[COPY1]](p0) :: (load (s64) from %ir.b)
-  ; X64-NEXT:   [[COPY2:%[0-9]+]]:psr(s64) = COPY [[LOAD]](s64)
-  ; X64-NEXT:   [[COPY3:%[0-9]+]]:psr(s64) = COPY [[LOAD1]](s64)
-  ; X64-NEXT:   [[FADD:%[0-9]+]]:psr(s64) = G_FADD [[COPY2]], [[COPY3]]
-  ; X64-NEXT:   [[COPY4:%[0-9]+]]:gpr(s64) = COPY [[FADD]](s64)
-  ; X64-NEXT:   G_STORE [[COPY4]](s64), [[COPY]](p0) :: (store (s64) into %ir.a)
+  ; X64-NEXT:   [[LOAD:%[0-9]+]]:psr(s64) = G_LOAD [[COPY]](p0) :: (load (s64) from %ir.a)
+  ; X64-NEXT:   [[LOAD1:%[0-9]+]]:psr(s64) = G_LOAD [[COPY1]](p0) :: (load (s64) from %ir.b)
+  ; X64-NEXT:   [[FADD:%[0-9]+]]:psr(s64) = G_FADD [[LOAD]], [[LOAD1]]
+  ; X64-NEXT:   G_STORE [[FADD]](s64), [[COPY]](p0) :: (store (s64) into %ir.a)
   ; X64-NEXT:   RET 0
   %load1 = load double, ptr %a, align 8
   %load2 = load double, ptr %b, align 8
@@ -210,11 +207,9 @@ define void @f6(ptr %0, ptr %1) {
   ; X86-NEXT:   [[FRAME_INDEX1:%[0-9]+]]:gpr(p0) = G_FRAME_INDEX %fixed-stack.0
   ; X86-NEXT:   [[LOAD1:%[0-9]+]]:gpr(p0) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load (p0) from %fixed-stack.0)
   ; X86-NEXT:   [[C:%[0-9]+]]:psr(s32) = G_FCONSTANT float 2.000000e+01
-  ; X86-NEXT:   [[LOAD2:%[0-9]+]]:gpr(s32) = G_LOAD [[LOAD]](p0) :: (load (s32) from %ir.0)
-  ; X86-NEXT:   [[COPY:%[0-9]+]]:psr(s32) = COPY [[LOAD2]](s32)
-  ; X86-NEXT:   [[FADD:%[0-9]+]]:psr(s32) = G_FADD [[COPY]], [[C]]
-  ; X86-NEXT:   [[COPY1:%[0-9]+]]:gpr(s32) = COPY [[FADD]](s32)
-  ; X86-NEXT:   G_STORE [[COPY1]](s32), [[LOAD1]](p0) :: (store (s32) into %ir.1)
+  ; X86-NEXT:   [[LOAD2:%[0-9]+]]:psr(s32) = G_LOAD [[LOAD]](p0) :: (load (s32) from %ir.0)
+  ; X86-NEXT:   [[FADD:%[0-9]+]]:psr(s32) = G_FADD [[LOAD2]], [[C]]
+  ; X86-NEXT:   G_STORE [[FADD]](s32), [[LOAD1]](p0) :: (store (s32) into %ir.1)
   ; X86-NEXT:   RET 0
   ;
   ; X64-LABEL: name: f6
@@ -224,11 +219,9 @@ define void @f6(ptr %0, ptr %1) {
   ; X64-NEXT:   [[COPY:%[0-9]+]]:gpr(p0) = COPY $rdi
   ; X64-NEXT:   [[COPY1:%[0-9]+]]:gpr(p0) = COPY $rsi
   ; X64-NEXT:   [[C:%[0-9]+]]:psr(s32) = G_FCONSTANT float 2.000000e+01
-  ; X64-NEXT:   [[LOAD:%[0-9]+]]:gpr(s32) = G_LOAD [[COPY]](p0) :: (load (s32) from %ir.0)
-  ; X64-NEXT:   [[COPY2:%[0-9]+]]:psr(s32) = COPY [[LOAD]](s32)
-  ; X64-NEXT:   [[FADD:%[0-9]+]]:psr(s32) = G_FADD [[COPY2]], [[C]]
-  ; X64-NEXT:   [[COPY3:%[0-9]+]]:gpr(s32) = COPY [[FADD]](s32)
-  ; X64-NEXT:   G_STORE [[COPY3]](s32), [[COPY1]](p0) :: (store (s32) into %ir.1)
+  ; X64-NEXT:   [[LOAD:%[0-9]+]]:psr(s32) = G_LOAD [[COPY]](p0) :: (load (s32) from %ir.0)
+  ; X64-NEXT:   [[FADD:%[0-9]+]]:psr(s32) = G_FADD [[LOAD]], [[C]]
+  ; X64-NEXT:   G_STORE [[FADD]](s32), [[COPY1]](p0) :: (store (s32) into %ir.1)
   ; X64-NEXT:   RET 0
   %load1 = load float, ptr %0
   %add = fadd float %load1, 20.0
-- 
GitLab


From ac6b4c618a52c62cef9b143a767991dbba7453e1 Mon Sep 17 00:00:00 2001
From: Jon Chesterfield <jonathanchesterfield@gmail.com>
Date: Tue, 16 Apr 2024 12:25:14 +0100
Subject: [PATCH 380/464] Reapply "[Verifier] Reject va_start in non-variadic
 function (#88809)"

This reverts commit f4960da6023b8034ae68925c3223d51624621b37.
Includes a fix for the MLIR test case.
---
 llvm/lib/Analysis/Lint.cpp                      | 5 +----
 llvm/lib/IR/Verifier.cpp                        | 5 +++++
 llvm/test/CodeGen/AArch64/GlobalISel/vastart.ll | 2 +-
 llvm/test/Other/lint.ll                         | 7 -------
 llvm/test/Verifier/variadic.ll                  | 8 ++++++++
 mlir/test/Target/LLVMIR/Import/intrinsic.ll     | 2 +-
 6 files changed, 16 insertions(+), 13 deletions(-)
 create mode 100644 llvm/test/Verifier/variadic.ll

diff --git a/llvm/lib/Analysis/Lint.cpp b/llvm/lib/Analysis/Lint.cpp
index 0694c2995dfc..1ab856ac8830 100644
--- a/llvm/lib/Analysis/Lint.cpp
+++ b/llvm/lib/Analysis/Lint.cpp
@@ -350,10 +350,7 @@ void Lint::visitCallBase(CallBase &I) {
     }
 
     case Intrinsic::vastart:
-      Check(I.getParent()->getParent()->isVarArg(),
-            "Undefined behavior: va_start called in a non-varargs function",
-            &I);
-
+      // vastart in non-varargs function is rejected by the verifier
       visitMemoryReference(I, MemoryLocation::getForArgument(&I, 0, TLI),
                            std::nullopt, nullptr, MemRef::Read | MemRef::Write);
       break;
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 516d4a051556..4cd61e6e531b 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -5798,6 +5798,11 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
 
     break;
   }
+  case Intrinsic::vastart: {
+    Check(Call.getFunction()->isVarArg(),
+          "va_start called in a non-varargs function");
+    break;
+  }
   case Intrinsic::vector_reduce_and:
   case Intrinsic::vector_reduce_or:
   case Intrinsic::vector_reduce_xor:
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/vastart.ll b/llvm/test/CodeGen/AArch64/GlobalISel/vastart.ll
index bd576d0f70e9..8c6e01d934c2 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/vastart.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/vastart.ll
@@ -3,7 +3,7 @@
 
 
 declare void @llvm.va_start(ptr)
-define void @test_va_start(ptr %list) {
+define void @test_va_start(ptr %list, ...) {
 ; CHECK-LABEL: name: test_va_start
 ; CHECK: [[LIST:%[0-9]+]]:_(p0) = COPY $x0
 ; CHECK-IOS: G_VASTART [[LIST]](p0) :: (store (s64) into %ir.list, align 1)
diff --git a/llvm/test/Other/lint.ll b/llvm/test/Other/lint.ll
index 6b31b31a78c9..6fd2d40cd2f2 100644
--- a/llvm/test/Other/lint.ll
+++ b/llvm/test/Other/lint.ll
@@ -124,13 +124,6 @@ define void @0() nounwind {
   ret void
 }
 
-; CHECK: va_start called in a non-varargs function
-declare void @llvm.va_start(ptr)
-define void @not_vararg(ptr %p) nounwind {
-  call void @llvm.va_start(ptr %p)
-  ret void
-}
-
 ; CHECK: Undefined behavior: Branch to non-blockaddress
 define void @use_indbr() {
   indirectbr ptr @foo, [label %block]
diff --git a/llvm/test/Verifier/variadic.ll b/llvm/test/Verifier/variadic.ll
new file mode 100644
index 000000000000..55e4a4da0a92
--- /dev/null
+++ b/llvm/test/Verifier/variadic.ll
@@ -0,0 +1,8 @@
+; RUN: not opt -S -passes=verify 2>&1 < %s | FileCheck %s
+
+; CHECK: va_start called in a non-varargs function
+declare void @llvm.va_start(ptr)
+define void @not_vararg(ptr %p) nounwind {
+  call void @llvm.va_start(ptr %p)
+  ret void
+}
diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
index 81a6eadbadd3..bf6847a32ff4 100644
--- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll
+++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
@@ -597,7 +597,7 @@ define void @ushl_sat_test(i32 %0, i32 %1, <8 x i32> %2, <8 x i32> %3) {
 }
 
 ; CHECK-LABEL: llvm.func @va_intrinsics_test
-define void @va_intrinsics_test(ptr %0, ptr %1) {
+define void @va_intrinsics_test(ptr %0, ptr %1, ...) {
 ; CHECK: llvm.intr.vastart %{{.*}}
   call void @llvm.va_start.p0(ptr %0)
 ; CHECK: llvm.intr.vacopy %{{.*}} to %{{.*}}
-- 
GitLab


From 09e7d7585cf881fb598eb56738579b84d027318c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Tue, 16 Apr 2024 07:52:39 +0200
Subject: [PATCH 381/464] [clang][Interp] Don't add 'in call to' diagnostics
 for builtin frames

---
 clang/lib/AST/Interp/InterpFrame.cpp        |  7 ++++++
 clang/lib/AST/Interp/State.cpp              |  5 +++--
 clang/test/AST/Interp/builtin-functions.cpp | 24 +++++++--------------
 3 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/clang/lib/AST/Interp/InterpFrame.cpp b/clang/lib/AST/Interp/InterpFrame.cpp
index 12e2e6ff9155..ba957546473e 100644
--- a/clang/lib/AST/Interp/InterpFrame.cpp
+++ b/clang/lib/AST/Interp/InterpFrame.cpp
@@ -152,6 +152,13 @@ void print(llvm::raw_ostream &OS, const Pointer &P, ASTContext &Ctx,
 }
 
 void InterpFrame::describe(llvm::raw_ostream &OS) const {
+  // We create frames for builtin functions as well, but we can't reliably
+  // diagnose them. The 'in call to' diagnostics for them add no value to the
+  // user _and_ it doesn't generally work since the argument types don't always
+  // match the function prototype. Just ignore them.
+  if (const auto *F = getFunction(); F && F->isBuiltin())
+    return;
+
   const FunctionDecl *F = getCallee();
   if (const auto *M = dyn_cast<CXXMethodDecl>(F);
       M && M->isInstance() && !isa<CXXConstructorDecl>(F)) {
diff --git a/clang/lib/AST/Interp/State.cpp b/clang/lib/AST/Interp/State.cpp
index 47fbf5145cd4..0d9dadec4b95 100644
--- a/clang/lib/AST/Interp/State.cpp
+++ b/clang/lib/AST/Interp/State.cpp
@@ -155,7 +155,8 @@ void State::addCallStack(unsigned Limit) {
     SmallString<128> Buffer;
     llvm::raw_svector_ostream Out(Buffer);
     F->describe(Out);
-    addDiag(CallRange.getBegin(), diag::note_constexpr_call_here)
-        << Out.str() << CallRange;
+    if (!Buffer.empty())
+      addDiag(CallRange.getBegin(), diag::note_constexpr_call_here)
+          << Out.str() << CallRange;
   }
 }
diff --git a/clang/test/AST/Interp/builtin-functions.cpp b/clang/test/AST/Interp/builtin-functions.cpp
index a7adc92d3714..1a29a664d7ce 100644
--- a/clang/test/AST/Interp/builtin-functions.cpp
+++ b/clang/test/AST/Interp/builtin-functions.cpp
@@ -24,16 +24,13 @@ namespace strcmp {
   static_assert(__builtin_strcmp("abab", "abab\0banana") == 0, "");
   static_assert(__builtin_strcmp("abab\0banana", "abab\0canada") == 0, "");
   static_assert(__builtin_strcmp(0, "abab") == 0, ""); // both-error {{not an integral constant}} \
-                                                       // both-note {{dereferenced null}} \
-                                                       // expected-note {{in call to}}
+                                                       // both-note {{dereferenced null}}
   static_assert(__builtin_strcmp("abab", 0) == 0, ""); // both-error {{not an integral constant}} \
-                                                       // both-note {{dereferenced null}} \
-                                                       // expected-note {{in call to}}
+                                                       // both-note {{dereferenced null}}
 
   static_assert(__builtin_strcmp(kFoobar, kFoobazfoobar) == -1, "");
   static_assert(__builtin_strcmp(kFoobar, kFoobazfoobar + 6) == 0, ""); // both-error {{not an integral constant}} \
-                                                                        // both-note {{dereferenced one-past-the-end}} \
-                                                                        // expected-note {{in call to}}
+                                                                        // both-note {{dereferenced one-past-the-end}}
 
   /// Used to assert because we're passing a dummy pointer to
   /// __builtin_strcmp() when evaluating the return statement.
@@ -72,14 +69,11 @@ constexpr const char *a = "foo\0quux";
   static_assert(check(c), "");
 
   constexpr int over1 = __builtin_strlen(a + 9); // both-error {{constant expression}} \
-                                                 // both-note {{one-past-the-end}} \
-                                                 // expected-note {{in call to}}
+                                                 // both-note {{one-past-the-end}}
   constexpr int over2 = __builtin_strlen(b + 9); // both-error {{constant expression}} \
-                                                 // both-note {{one-past-the-end}} \
-                                                 // expected-note {{in call to}}
+                                                 // both-note {{one-past-the-end}}
   constexpr int over3 = __builtin_strlen(c + 9); // both-error {{constant expression}} \
-                                                 // both-note {{one-past-the-end}} \
-                                                 // expected-note {{in call to}}
+                                                 // both-note {{one-past-the-end}}
 
   constexpr int under1 = __builtin_strlen(a - 1); // both-error {{constant expression}} \
                                                   // both-note {{cannot refer to element -1}}
@@ -90,8 +84,7 @@ constexpr const char *a = "foo\0quux";
 
   constexpr char d[] = { 'f', 'o', 'o' }; // no nul terminator.
   constexpr int bad = __builtin_strlen(d); // both-error {{constant expression}} \
-                                           // both-note {{one-past-the-end}} \
-                                           // expected-note {{in call to}}
+                                           // both-note {{one-past-the-end}}
 }
 
 namespace nan {
@@ -114,8 +107,7 @@ namespace nan {
   /// FIXME: Current interpreter misses diagnostics.
   constexpr char f2[] = {'0', 'x', 'A', 'E'}; /// No trailing 0 byte.
   constexpr double NaN7 = __builtin_nan(f2); // both-error {{must be initialized by a constant expression}} \
-                                             // expected-note {{read of dereferenced one-past-the-end pointer}} \
-                                             // expected-note {{in call to}}
+                                             // expected-note {{read of dereferenced one-past-the-end pointer}}
   static_assert(!__builtin_issignaling(__builtin_nan("")), "");
   static_assert(__builtin_issignaling(__builtin_nans("")), "");
 }
-- 
GitLab


From a8de3ee8994023ea7669397587f8118ae5bba9c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Tue, 16 Apr 2024 13:29:10 +0200
Subject: [PATCH 382/464] [clang][Interp][NFC] Fix some build warnings

Fixes:
/buildbot/worker/arc-folder/llvm-project/clang/lib/AST/Interp/Disasm.cpp:143:25: warning: cast from type 'const clang::interp::Block*' to type 'void*' casts away qualifiers [-Wcast-qual]
/buildbot/worker/arc-folder/llvm-project/clang/lib/AST/Interp/Disasm.cpp:271:23: warning: cast from type 'const clang::interp::Block*' to type 'void*' casts away qualifiers [-Wcast-qual]
---
 clang/lib/AST/Interp/Disasm.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/clang/lib/AST/Interp/Disasm.cpp b/clang/lib/AST/Interp/Disasm.cpp
index ebc4e4f195ba..d127f33223e8 100644
--- a/clang/lib/AST/Interp/Disasm.cpp
+++ b/clang/lib/AST/Interp/Disasm.cpp
@@ -140,7 +140,7 @@ LLVM_DUMP_METHOD void Program::dump(llvm::raw_ostream &OS) const {
     const Descriptor *Desc = G->block()->getDescriptor();
     Pointer GP = getPtrGlobal(GI);
 
-    OS << GI << ": " << (void *)G->block() << " ";
+    OS << GI << ": " << (const void *)G->block() << " ";
     {
       ColorScope SC(OS, true,
                     GP.isInitialized()
@@ -268,7 +268,7 @@ LLVM_DUMP_METHOD void Record::dump(llvm::raw_ostream &OS, unsigned Indentation,
 LLVM_DUMP_METHOD void Block::dump(llvm::raw_ostream &OS) const {
   {
     ColorScope SC(OS, true, {llvm::raw_ostream::BRIGHT_BLUE, true});
-    OS << "Block " << (void *)this << "\n";
+    OS << "Block " << (const void *)this << "\n";
   }
   unsigned NPointers = 0;
   for (const Pointer *P = Pointers; P; P = P->Next) {
-- 
GitLab


From c18a3b6bd30456305cf1b3d78ad5a805577388c1 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 16 Apr 2024 12:30:13 +0100
Subject: [PATCH 383/464] [DAG] Fold
 extract_subvector(insert_subvector(x,y,c1),c2) --> extract_subvector(y,c2-c1)
 (#87925) (REAPPLIED)

If the extract_subvector is cheap, attempt to extract directly from an inserted subvector

Reapplied with a check to ensure we only attempt this for fixed vectors
---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp |   16 +
 .../any_extend_vector_inreg_of_broadcast.ll   |   42 +-
 llvm/test/CodeGen/X86/dpbusd.ll               |    2 +-
 llvm/test/CodeGen/X86/dpbusd_i4.ll            |    2 +-
 .../vector-interleaved-load-i16-stride-3.ll   | 1300 ++++++++---------
 .../vector-interleaved-store-i8-stride-7.ll   |  602 ++++----
 .../zero_extend_vector_inreg_of_broadcast.ll  |   28 +-
 7 files changed, 986 insertions(+), 1006 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 0fa0bf2609bb..c36b1cc9039c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -24467,6 +24467,22 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode *N) {
       if (!LegalOperations || TLI.isOperationLegal(ISD::SPLAT_VECTOR, NVT))
         return DAG.getSplatVector(NVT, DL, V.getOperand(0));
 
+  // extract_subvector(insert_subvector(x,y,c1),c2)
+  //  --> extract_subvector(y,c2-c1)
+  // iff we're just extracting from the inserted subvector.
+  if (V.getOpcode() == ISD::INSERT_SUBVECTOR) {
+    SDValue InsSub = V.getOperand(1);
+    EVT InsSubVT = InsSub.getValueType();
+    unsigned NumInsElts = InsSubVT.getVectorMinNumElements();
+    unsigned InsIdx = V.getConstantOperandVal(2);
+    unsigned NumSubElts = NVT.getVectorMinNumElements();
+    if (InsIdx <= ExtIdx && (ExtIdx + NumSubElts) <= (InsIdx + NumInsElts) &&
+        TLI.isExtractSubvectorCheap(NVT, InsSubVT, ExtIdx - InsIdx) &&
+        InsSubVT.isFixedLengthVector() && NVT.isFixedLengthVector())
+      return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NVT, InsSub,
+                         DAG.getVectorIdxConstant(ExtIdx - InsIdx, DL));
+  }
+
   // Try to move vector bitcast after extract_subv by scaling extraction index:
   // extract_subv (bitcast X), Index --> bitcast (extract_subv X, Index')
   if (V.getOpcode() == ISD::BITCAST &&
diff --git a/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll b/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll
index 4242d8483e72..39c7ce1413d1 100644
--- a/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll
+++ b/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll
@@ -314,8 +314,8 @@ define void @vec64_i16_widen_to_i32_factor2_broadcast_to_v2i32_factor2(ptr %in.v
 ;
 ; AVX512F-LABEL: vec64_i16_widen_to_i32_factor2_broadcast_to_v2i32_factor2:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,10,11,0,1,14,15,u,u,u,u,u,u,u,u]
 ; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)
@@ -324,8 +324,8 @@ define void @vec64_i16_widen_to_i32_factor2_broadcast_to_v2i32_factor2(ptr %in.v
 ;
 ; AVX512DQ-LABEL: vec64_i16_widen_to_i32_factor2_broadcast_to_v2i32_factor2:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512DQ-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,10,11,0,1,14,15,u,u,u,u,u,u,u,u]
 ; AVX512DQ-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, (%rcx)
@@ -981,7 +981,7 @@ define void @vec128_i32_widen_to_i64_factor2_broadcast_to_v2i64_factor2(ptr %in.
 ; AVX512F-NEXT:    vpmovsxbd {{.*#+}} xmm0 = [0,5,0,7]
 ; AVX512F-NEXT:    vmovdqa (%rdi), %ymm1
 ; AVX512F-NEXT:    vpaddb (%rsi), %ymm1, %ymm1
-; AVX512F-NEXT:    vpermd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpermd %ymm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)
 ; AVX512F-NEXT:    vzeroupper
@@ -992,7 +992,7 @@ define void @vec128_i32_widen_to_i64_factor2_broadcast_to_v2i64_factor2(ptr %in.
 ; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} xmm0 = [0,5,0,7]
 ; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm1
 ; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpermd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpermd %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, (%rcx)
 ; AVX512DQ-NEXT:    vzeroupper
@@ -3507,13 +3507,12 @@ define void @vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12(ptr %i
 ;
 ; AVX512F-LABEL: vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX512F-NEXT:    vmovdqa 48(%rdi), %xmm1
 ; AVX512F-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX512F-NEXT:    vpbroadcastw %xmm0, %ymm2
-; AVX512F-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2],ymm1[3],ymm2[4],ymm1[5],ymm2[6],ymm1[7],ymm2[8],ymm1[9],ymm2[10],ymm1[11],ymm2[12],ymm1[13],ymm2[14],ymm1[15]
+; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512F-NEXT:    vpbroadcastw %xmm0, %ymm0
+; AVX512F-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7],ymm0[8],ymm1[9],ymm0[10],ymm1[11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
 ; AVX512F-NEXT:    vpaddb (%rdx), %ymm1, %ymm1
 ; AVX512F-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm0, 32(%rcx)
@@ -3523,13 +3522,12 @@ define void @vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12(ptr %i
 ;
 ; AVX512DQ-LABEL: vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX512DQ-NEXT:    vmovdqa 48(%rdi), %xmm1
 ; AVX512DQ-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX512DQ-NEXT:    vpbroadcastw %xmm0, %ymm2
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2],ymm1[3],ymm2[4],ymm1[5],ymm2[6],ymm1[7],ymm2[8],ymm1[9],ymm2[10],ymm1[11],ymm2[12],ymm1[13],ymm2[14],ymm1[15]
+; AVX512DQ-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpbroadcastw %xmm0, %ymm0
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7],ymm0[8],ymm1[9],ymm0[10],ymm1[11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
 ; AVX512DQ-NEXT:    vpaddb (%rdx), %ymm1, %ymm1
 ; AVX512DQ-NEXT:    vpaddb 32(%rdx), %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, 32(%rcx)
@@ -3768,10 +3766,10 @@ define void @vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6(ptr %in.
 ;
 ; AVX512F-LABEL: vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX512F-NEXT:    vmovdqa 48(%rdi), %xmm1
 ; AVX512F-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
+; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512F-NEXT:    vpbroadcastq %xmm0, %ymm2
 ; AVX512F-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1,2,3],ymm2[4],ymm1[5,6,7],ymm2[8],ymm1[9,10,11],ymm2[12],ymm1[13,14,15]
 ; AVX512F-NEXT:    vpbroadcastw %xmm0, %ymm0
@@ -3784,10 +3782,10 @@ define void @vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6(ptr %in.
 ;
 ; AVX512DQ-LABEL: vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX512DQ-NEXT:    vmovdqa 48(%rdi), %xmm1
 ; AVX512DQ-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpbroadcastq %xmm0, %ymm2
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1,2,3],ymm2[4],ymm1[5,6,7],ymm2[8],ymm1[9,10,11],ymm2[12],ymm1[13,14,15]
 ; AVX512DQ-NEXT:    vpbroadcastw %xmm0, %ymm0
@@ -4147,9 +4145,9 @@ define void @vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2(ptr %
 ;
 ; AVX512F-LABEL: vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX512F-NEXT:    vmovdqa 48(%rdi), %xmm1
+; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512F-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
 ; AVX512F-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]
 ; AVX512F-NEXT:    vpbroadcastw %xmm0, %xmm0
@@ -4161,9 +4159,9 @@ define void @vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2(ptr %
 ;
 ; AVX512DQ-LABEL: vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX512DQ-NEXT:    vmovdqa 48(%rdi), %xmm1
+; AVX512DQ-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]
 ; AVX512DQ-NEXT:    vpbroadcastw %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/dpbusd.ll b/llvm/test/CodeGen/X86/dpbusd.ll
index fbea08eb1e55..04d7a9691b64 100644
--- a/llvm/test/CodeGen/X86/dpbusd.ll
+++ b/llvm/test/CodeGen/X86/dpbusd.ll
@@ -26,7 +26,7 @@ define i32 @no_dpbusd(ptr%a, ptr%b, i32 %c, i32 %n) {
 ; AVX512-NEXT:    vpmovzxbw {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
 ; AVX512-NEXT:    vpmaddwd %ymm0, %ymm1, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
diff --git a/llvm/test/CodeGen/X86/dpbusd_i4.ll b/llvm/test/CodeGen/X86/dpbusd_i4.ll
index 906fead7f8db..a212f99680ef 100644
--- a/llvm/test/CodeGen/X86/dpbusd_i4.ll
+++ b/llvm/test/CodeGen/X86/dpbusd_i4.ll
@@ -86,7 +86,7 @@ define i32 @mul_sext_i4i4(<16 x i4> %a, <16 x i4> %b, i32 %c) {
 ; CHECK-NEXT:    vpsraw $12, %ymm0, %ymm0
 ; CHECK-NEXT:    vpmaddwd %ymm1, %ymm0, %ymm0
 ; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; CHECK-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-3.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-3.ll
index 1436922f9dd1..6d5fc9ed0ab5 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-3.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-3.ll
@@ -1828,22 +1828,22 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX512-LABEL: load_i16_stride3_vf32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm1 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
 ; AVX512-NEXT:    vmovdqa 128(%rdi), %ymm5
 ; AVX512-NEXT:    vmovdqa 160(%rdi), %ymm6
-; AVX512-NEXT:    vmovdqa %ymm1, %ymm0
-; AVX512-NEXT:    vpternlogq $202, %ymm5, %ymm6, %ymm0
-; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2,3],ymm2[4],ymm0[5,6],ymm2[7],ymm0[8],ymm2[9],ymm0[10,11],ymm2[12],ymm0[13,14],ymm2[15]
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm3 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
-; AVX512-NEXT:    vmovdqa 112(%rdi), %xmm0
+; AVX512-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512-NEXT:    vpternlogq $202, %ymm5, %ymm6, %ymm1
+; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
+; AVX512-NEXT:    vmovdqa 112(%rdi), %xmm1
 ; AVX512-NEXT:    vmovdqa 96(%rdi), %xmm2
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm2[0],xmm0[1],xmm2[2,3],xmm0[4],xmm2[5,6],xmm0[7]
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
 ; AVX512-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[0,1,6,7,12,13,2,3,8,9,14,15,u,u,u,u]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm7 = ymm4[0,1,2],ymm3[3,4,5,6,7]
 ; AVX512-NEXT:    vmovdqa (%rdi), %ymm8
 ; AVX512-NEXT:    vmovdqa 32(%rdi), %ymm9
-; AVX512-NEXT:    vmovdqa %ymm1, %ymm3
+; AVX512-NEXT:    vmovdqa %ymm0, %ymm3
 ; AVX512-NEXT:    vpternlogq $202, %ymm9, %ymm8, %ymm3
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm3[2,3,0,1]
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7],ymm3[8],ymm4[9],ymm3[10,11],ymm4[12],ymm3[13,14],ymm4[15]
@@ -1857,14 +1857,14 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,6,5,4,7]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm11[4,5,6,7]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm10, %zmm7
-; AVX512-NEXT:    vmovdqa %ymm1, %ymm10
+; AVX512-NEXT:    vmovdqa %ymm0, %ymm10
 ; AVX512-NEXT:    vpternlogq $202, %ymm6, %ymm5, %ymm10
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm11 = ymm10[2,3,0,1]
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0,1],ymm11[2],ymm10[3,4],ymm11[5],ymm10[6,7,8,9],ymm11[10],ymm10[11,12],ymm11[13],ymm10[14,15]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm11 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
 ; AVX512-NEXT:    vpshufb %ymm11, %ymm10, %ymm10
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]
-; AVX512-NEXT:    vpshufb %xmm11, %xmm12, %xmm12
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm12 = xmm12[2,3,8,9,14,15,4,5,10,11,10,11,10,11,10,11]
 ; AVX512-NEXT:    vpblendw {{.*#+}} xmm12 = xmm12[0,1,2,3,4],xmm10[5,6,7]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm10 = ymm12[0,1,2,3],ymm10[4,5,6,7]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm12 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
@@ -1885,21 +1885,19 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1,2],ymm5[3],ymm12[4,5],ymm5[6],ymm12[7],ymm5[8],ymm12[9,10],ymm5[11],ymm12[12,13],ymm5[14],ymm12[15]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm6 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
 ; AVX512-NEXT:    vpshufb %ymm6, %ymm5, %ymm5
-; AVX512-NEXT:    vpternlogq $202, %ymm8, %ymm9, %ymm1
-; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm1[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm8[0],ymm1[1,2],ymm8[3],ymm1[4,5],ymm8[6],ymm1[7],ymm8[8],ymm1[9,10],ymm8[11],ymm1[12,13],ymm8[14],ymm1[15]
-; AVX512-NEXT:    vpshufb %ymm6, %ymm1, %ymm1
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]
-; AVX512-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
-; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5,6,7]
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2],xmm0[3,4],xmm2[5],xmm0[6,7]
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; AVX512-NEXT:    vextracti32x4 $2, %zmm0, %xmm0
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm5[5,6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3,4],xmm2[5],xmm1[6,7]
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm5[5,6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm5[4,5,6,7]
+; AVX512-NEXT:    vpternlogq $202, %ymm8, %ymm9, %ymm0
+; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1,2],ymm2[3],ymm0[4,5],ymm2[6],ymm0[7],ymm2[8],ymm0[9,10],ymm2[11],ymm0[12,13],ymm2[14],ymm0[15]
+; AVX512-NEXT:    vpshufb %ymm6, %ymm0, %ymm0
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]
+; AVX512-NEXT:    vpshufb %xmm6, %xmm2, %xmm2
+; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5,6,7]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vmovdqa64 %zmm7, (%rsi)
 ; AVX512-NEXT:    vmovdqa64 %zmm10, (%rdx)
 ; AVX512-NEXT:    vmovdqa64 %zmm0, (%rcx)
@@ -1908,22 +1906,22 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX512-FCP-LABEL: load_i16_stride3_vf32:
 ; AVX512-FCP:       # %bb.0:
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm1 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
 ; AVX512-FCP-NEXT:    vmovdqa 128(%rdi), %ymm5
 ; AVX512-FCP-NEXT:    vmovdqa 160(%rdi), %ymm6
-; AVX512-FCP-NEXT:    vmovdqa %ymm1, %ymm0
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm5, %ymm6, %ymm0
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2,3],ymm2[4],ymm0[5,6],ymm2[7],ymm0[8],ymm2[9],ymm0[10,11],ymm2[12],ymm0[13,14],ymm2[15]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
-; AVX512-FCP-NEXT:    vmovdqa 112(%rdi), %xmm0
+; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm5, %ymm6, %ymm1
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
+; AVX512-FCP-NEXT:    vmovdqa 112(%rdi), %xmm1
 ; AVX512-FCP-NEXT:    vmovdqa 96(%rdi), %xmm2
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm2[0],xmm0[1],xmm2[2,3],xmm0[4],xmm2[5,6],xmm0[7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
 ; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[0,1,6,7,12,13,2,3,8,9,14,15,u,u,u,u]
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm7 = ymm4[0,1,2],ymm3[3,4,5,6,7]
 ; AVX512-FCP-NEXT:    vmovdqa (%rdi), %ymm8
 ; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %ymm9
-; AVX512-FCP-NEXT:    vmovdqa %ymm1, %ymm3
+; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm3
 ; AVX512-FCP-NEXT:    vpternlogq $202, %ymm9, %ymm8, %ymm3
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm3[2,3,0,1]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7],ymm3[8],ymm4[9],ymm3[10,11],ymm4[12],ymm3[13,14],ymm4[15]
@@ -1937,14 +1935,14 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,6,5,4,7]
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm11[4,5,6,7]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm10, %zmm7
-; AVX512-FCP-NEXT:    vmovdqa %ymm1, %ymm10
+; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm10
 ; AVX512-FCP-NEXT:    vpternlogq $202, %ymm6, %ymm5, %ymm10
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm11 = ymm10[2,3,0,1]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0,1],ymm11[2],ymm10[3,4],ymm11[5],ymm10[6,7,8,9],ymm11[10],ymm10[11,12],ymm11[13],ymm10[14,15]
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm11 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
 ; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm10, %ymm10
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]
-; AVX512-FCP-NEXT:    vpshufb %xmm11, %xmm12, %xmm12
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm12 = xmm12[2,3,8,9,14,15,4,5,10,11,10,11,10,11,10,11]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm12 = xmm12[0,1,2,3,4],xmm10[5,6,7]
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm10 = ymm12[0,1,2,3],ymm10[4,5,6,7]
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm12 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
@@ -1965,21 +1963,19 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1,2],ymm5[3],ymm12[4,5],ymm5[6],ymm12[7],ymm5[8],ymm12[9,10],ymm5[11],ymm12[12,13],ymm5[14],ymm12[15]
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm6 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
 ; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm5, %ymm5
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm8, %ymm9, %ymm1
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm1[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm8[0],ymm1[1,2],ymm8[3],ymm1[4,5],ymm8[6],ymm1[7],ymm8[8],ymm1[9,10],ymm8[11],ymm1[12,13],ymm8[14],ymm1[15]
-; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm1, %ymm1
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]
-; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
-; AVX512-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5,6,7]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2],xmm0[3,4],xmm2[5],xmm0[6,7]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; AVX512-FCP-NEXT:    vextracti32x4 $2, %zmm0, %xmm0
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm5[5,6,7]
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3,4],xmm2[5],xmm1[6,7]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm5[5,6,7]
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm5[4,5,6,7]
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm8, %ymm9, %ymm0
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1,2],ymm2[3],ymm0[4,5],ymm2[6],ymm0[7],ymm2[8],ymm0[9,10],ymm2[11],ymm0[12,13],ymm2[14],ymm0[15]
+; AVX512-FCP-NEXT:    vpshufb %ymm6, %ymm0, %ymm0
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]
+; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm2, %xmm2
+; AVX512-FCP-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5,6,7]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm7, (%rsi)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm10, (%rdx)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, (%rcx)
@@ -1988,22 +1984,22 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX512DQ-LABEL: load_i16_stride3_vf32:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm1 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
 ; AVX512DQ-NEXT:    vmovdqa 128(%rdi), %ymm5
 ; AVX512DQ-NEXT:    vmovdqa 160(%rdi), %ymm6
-; AVX512DQ-NEXT:    vmovdqa %ymm1, %ymm0
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm5, %ymm6, %ymm0
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2,3],ymm2[4],ymm0[5,6],ymm2[7],ymm0[8],ymm2[9],ymm0[10,11],ymm2[12],ymm0[13,14],ymm2[15]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm3 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
-; AVX512DQ-NEXT:    vmovdqa 112(%rdi), %xmm0
+; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm5, %ymm6, %ymm1
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
+; AVX512DQ-NEXT:    vmovdqa 112(%rdi), %xmm1
 ; AVX512DQ-NEXT:    vmovdqa 96(%rdi), %xmm2
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm2[0],xmm0[1],xmm2[2,3],xmm0[4],xmm2[5,6],xmm0[7]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[0,1,6,7,12,13,2,3,8,9,14,15,u,u,u,u]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm7 = ymm4[0,1,2],ymm3[3,4,5,6,7]
 ; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm8
 ; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm9
-; AVX512DQ-NEXT:    vmovdqa %ymm1, %ymm3
+; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm3
 ; AVX512DQ-NEXT:    vpternlogq $202, %ymm9, %ymm8, %ymm3
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm3[2,3,0,1]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7],ymm3[8],ymm4[9],ymm3[10,11],ymm4[12],ymm3[13,14],ymm4[15]
@@ -2017,14 +2013,14 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,6,5,4,7]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm11[4,5,6,7]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm10, %zmm7
-; AVX512DQ-NEXT:    vmovdqa %ymm1, %ymm10
+; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm10
 ; AVX512DQ-NEXT:    vpternlogq $202, %ymm6, %ymm5, %ymm10
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm11 = ymm10[2,3,0,1]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0,1],ymm11[2],ymm10[3,4],ymm11[5],ymm10[6,7,8,9],ymm11[10],ymm10[11,12],ymm11[13],ymm10[14,15]
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm11 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
 ; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm10, %ymm10
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]
-; AVX512DQ-NEXT:    vpshufb %xmm11, %xmm12, %xmm12
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm12 = xmm12[2,3,8,9,14,15,4,5,10,11,10,11,10,11,10,11]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm12 = xmm12[0,1,2,3,4],xmm10[5,6,7]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm10 = ymm12[0,1,2,3],ymm10[4,5,6,7]
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm12 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
@@ -2045,21 +2041,19 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1,2],ymm5[3],ymm12[4,5],ymm5[6],ymm12[7],ymm5[8],ymm12[9,10],ymm5[11],ymm12[12,13],ymm5[14],ymm12[15]
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm6 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
 ; AVX512DQ-NEXT:    vpshufb %ymm6, %ymm5, %ymm5
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm8, %ymm9, %ymm1
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm1[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm8[0],ymm1[1,2],ymm8[3],ymm1[4,5],ymm8[6],ymm1[7],ymm8[8],ymm1[9,10],ymm8[11],ymm1[12,13],ymm8[14],ymm1[15]
-; AVX512DQ-NEXT:    vpshufb %ymm6, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]
-; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5,6,7]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2],xmm0[3,4],xmm2[5],xmm0[6,7]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; AVX512DQ-NEXT:    vextracti32x4 $2, %zmm0, %xmm0
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm5[5,6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3,4],xmm2[5],xmm1[6,7]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm5[5,6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm5[4,5,6,7]
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm8, %ymm9, %ymm0
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1,2],ymm2[3],ymm0[4,5],ymm2[6],ymm0[7],ymm2[8],ymm0[9,10],ymm2[11],ymm0[12,13],ymm2[14],ymm0[15]
+; AVX512DQ-NEXT:    vpshufb %ymm6, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]
+; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm2, %xmm2
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5,6,7]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm7, (%rsi)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm10, (%rdx)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm0, (%rcx)
@@ -2068,22 +2062,22 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX512DQ-FCP-LABEL: load_i16_stride3_vf32:
 ; AVX512DQ-FCP:       # %bb.0:
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm1 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
 ; AVX512DQ-FCP-NEXT:    vmovdqa 128(%rdi), %ymm5
 ; AVX512DQ-FCP-NEXT:    vmovdqa 160(%rdi), %ymm6
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm1, %ymm0
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm5, %ymm6, %ymm0
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2,3],ymm2[4],ymm0[5,6],ymm2[7],ymm0[8],ymm2[9],ymm0[10,11],ymm2[12],ymm0[13,14],ymm2[15]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
-; AVX512DQ-FCP-NEXT:    vmovdqa 112(%rdi), %xmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm5, %ymm6, %ymm1
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
+; AVX512DQ-FCP-NEXT:    vmovdqa 112(%rdi), %xmm1
 ; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdi), %xmm2
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm2[0],xmm0[1],xmm2[2,3],xmm0[4],xmm2[5,6],xmm0[7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
 ; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[0,1,6,7,12,13,2,3,8,9,14,15,u,u,u,u]
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm7 = ymm4[0,1,2],ymm3[3,4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %ymm8
 ; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %ymm9
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm1, %ymm3
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm3
 ; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm9, %ymm8, %ymm3
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm3[2,3,0,1]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7],ymm3[8],ymm4[9],ymm3[10,11],ymm4[12],ymm3[13,14],ymm4[15]
@@ -2097,14 +2091,14 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,6,5,4,7]
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm11[4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm10, %zmm7
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm1, %ymm10
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm10
 ; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm6, %ymm5, %ymm10
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm11 = ymm10[2,3,0,1]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0,1],ymm11[2],ymm10[3,4],ymm11[5],ymm10[6,7,8,9],ymm11[10],ymm10[11,12],ymm11[13],ymm10[14,15]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm11 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm10, %ymm10
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm11, %xmm12, %xmm12
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm12 = xmm12[2,3,8,9,14,15,4,5,10,11,10,11,10,11,10,11]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm12 = xmm12[0,1,2,3,4],xmm10[5,6,7]
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm10 = ymm12[0,1,2,3],ymm10[4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm12 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
@@ -2125,21 +2119,19 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1,2],ymm5[3],ymm12[4,5],ymm5[6],ymm12[7],ymm5[8],ymm12[9,10],ymm5[11],ymm12[12,13],ymm5[14],ymm12[15]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm6 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm5, %ymm5
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm8, %ymm9, %ymm1
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm1[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm8[0],ymm1[1,2],ymm8[3],ymm1[4,5],ymm8[6],ymm1[7],ymm8[8],ymm1[9,10],ymm8[11],ymm1[12,13],ymm8[14],ymm1[15]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm1, %ymm1
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
-; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5,6,7]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2],xmm0[3,4],xmm2[5],xmm0[6,7]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; AVX512DQ-FCP-NEXT:    vextracti32x4 $2, %zmm0, %xmm0
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm5[5,6,7]
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3,4],xmm2[5],xmm1[6,7]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm5[5,6,7]
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm5[4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm8, %ymm9, %ymm0
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1,2],ymm2[3],ymm0[4,5],ymm2[6],ymm0[7],ymm2[8],ymm0[9,10],ymm2[11],ymm0[12,13],ymm2[14],ymm0[15]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm6, %ymm0, %ymm0
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm2, %xmm2
+; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5,6,7]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm7, (%rsi)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm10, (%rdx)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, (%rcx)
@@ -3500,688 +3492,668 @@ define void @load_i16_stride3_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512-LABEL: load_i16_stride3_vf64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
-; AVX512-NEXT:    vmovdqa64 224(%rdi), %ymm20
-; AVX512-NEXT:    vmovdqa64 192(%rdi), %ymm21
+; AVX512-NEXT:    vmovdqa64 224(%rdi), %ymm18
+; AVX512-NEXT:    vmovdqa64 192(%rdi), %ymm20
 ; AVX512-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512-NEXT:    vpternlogq $202, %ymm20, %ymm21, %ymm1
-; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm1[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm3[1],ymm1[2,3],ymm3[4],ymm1[5,6],ymm3[7],ymm1[8],ymm3[9],ymm1[10,11],ymm3[12],ymm1[13,14],ymm3[15]
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]
-; AVX512-NEXT:    vpshufb %ymm3, %ymm1, %ymm5
-; AVX512-NEXT:    vmovdqa 272(%rdi), %xmm8
+; AVX512-NEXT:    vpternlogq $202, %ymm18, %ymm20, %ymm1
+; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm7 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]
+; AVX512-NEXT:    vpshufb %ymm7, %ymm2, %ymm5
+; AVX512-NEXT:    vmovdqa 272(%rdi), %xmm1
 ; AVX512-NEXT:    vmovdqa 256(%rdi), %xmm2
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm6 = xmm2[0,1],xmm8[2],xmm2[3,4],xmm8[5],xmm2[6,7]
-; AVX512-NEXT:    vmovdqa %xmm2, %xmm14
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm9 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]
-; AVX512-NEXT:    vpshufb %xmm9, %xmm6, %xmm6
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm6 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]
+; AVX512-NEXT:    vmovdqa %xmm2, %xmm3
+; AVX512-NEXT:    vmovdqa64 %xmm1, %xmm19
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm13 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]
+; AVX512-NEXT:    vpshufb %xmm13, %xmm6, %xmm6
 ; AVX512-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm6 = ymm5[0,1,2],ymm6[3,4,5,6,7],ymm5[8,9,10],ymm6[11,12,13,14,15]
 ; AVX512-NEXT:    vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,6,5,4,7]
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4,5,6,7]
-; AVX512-NEXT:    vmovdqa64 320(%rdi), %ymm22
-; AVX512-NEXT:    vmovdqa64 352(%rdi), %ymm23
-; AVX512-NEXT:    vmovdqa %ymm0, %ymm6
-; AVX512-NEXT:    vpternlogq $202, %ymm22, %ymm23, %ymm6
-; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0],ymm7[1],ymm6[2,3],ymm7[4],ymm6[5,6],ymm7[7],ymm6[8],ymm7[9],ymm6[10,11],ymm7[12],ymm6[13,14],ymm7[15]
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm11 = [0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
-; AVX512-NEXT:    vpshufb %ymm11, %ymm6, %ymm12
+; AVX512-NEXT:    vmovdqa64 320(%rdi), %ymm21
+; AVX512-NEXT:    vmovdqa64 352(%rdi), %ymm22
+; AVX512-NEXT:    vmovdqa %ymm0, %ymm8
+; AVX512-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm8
+; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm8[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm8 = ymm8[0],ymm9[1],ymm8[2,3],ymm9[4],ymm8[5,6],ymm9[7],ymm8[8],ymm9[9],ymm8[10,11],ymm9[12],ymm8[13,14],ymm9[15]
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm10 = [0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
+; AVX512-NEXT:    vpshufb %ymm10, %ymm8, %ymm11
 ; AVX512-NEXT:    vmovdqa 304(%rdi), %xmm1
 ; AVX512-NEXT:    vmovdqa 288(%rdi), %xmm2
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm13 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
 ; AVX512-NEXT:    vmovdqa %xmm2, %xmm4
-; AVX512-NEXT:    vmovdqa %xmm1, %xmm6
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm15 = [0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]
-; AVX512-NEXT:    vpshufb %xmm15, %xmm13, %xmm13
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm12 = ymm13[0,1,2],ymm12[3,4,5,6,7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm12, %zmm5, %zmm16
-; AVX512-NEXT:    vmovdqa64 128(%rdi), %ymm24
-; AVX512-NEXT:    vmovdqa 160(%rdi), %ymm13
+; AVX512-NEXT:    vmovdqa %xmm1, %xmm8
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm14 = [0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]
+; AVX512-NEXT:    vpshufb %xmm14, %xmm12, %xmm12
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm11 = ymm12[0,1,2],ymm11[3,4,5,6,7]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm11, %zmm5, %zmm16
+; AVX512-NEXT:    vmovdqa64 128(%rdi), %ymm23
+; AVX512-NEXT:    vmovdqa 160(%rdi), %ymm11
 ; AVX512-NEXT:    vmovdqa %ymm0, %ymm5
-; AVX512-NEXT:    vpternlogq $202, %ymm24, %ymm13, %ymm5
+; AVX512-NEXT:    vpternlogq $202, %ymm23, %ymm11, %ymm5
 ; AVX512-NEXT:    vpermq {{.*#+}} ymm12 = ymm5[2,3,0,1]
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1],ymm5[2,3],ymm12[4],ymm5[5,6],ymm12[7],ymm5[8],ymm12[9],ymm5[10,11],ymm12[12],ymm5[13,14],ymm12[15]
-; AVX512-NEXT:    vpshufb %ymm11, %ymm5, %ymm5
-; AVX512-NEXT:    vmovdqa 112(%rdi), %xmm11
-; AVX512-NEXT:    vmovdqa 96(%rdi), %xmm12
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm10 = xmm12[0],xmm11[1],xmm12[2,3],xmm11[4],xmm12[5,6],xmm11[7]
-; AVX512-NEXT:    vpshufb %xmm15, %xmm10, %xmm10
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm10[0,1,2],ymm5[3,4,5,6,7]
-; AVX512-NEXT:    vmovdqa64 (%rdi), %ymm17
-; AVX512-NEXT:    vmovdqa 32(%rdi), %ymm5
+; AVX512-NEXT:    vpshufb %ymm10, %ymm5, %ymm10
+; AVX512-NEXT:    vmovdqa 112(%rdi), %xmm15
+; AVX512-NEXT:    vmovdqa 96(%rdi), %xmm5
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm12 = xmm5[0],xmm15[1],xmm5[2,3],xmm15[4],xmm5[5,6],xmm15[7]
+; AVX512-NEXT:    vpshufb %xmm14, %xmm12, %xmm12
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm6 = ymm12[0,1,2],ymm10[3,4,5,6,7]
+; AVX512-NEXT:    vmovdqa64 (%rdi), %ymm24
+; AVX512-NEXT:    vmovdqa 32(%rdi), %ymm12
 ; AVX512-NEXT:    vmovdqa %ymm0, %ymm10
-; AVX512-NEXT:    vpternlogq $202, %ymm5, %ymm17, %ymm10
-; AVX512-NEXT:    vpermq {{.*#+}} ymm15 = ymm10[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0],ymm15[1],ymm10[2,3],ymm15[4],ymm10[5,6],ymm15[7],ymm10[8],ymm15[9],ymm10[10,11],ymm15[12],ymm10[13,14],ymm15[15]
-; AVX512-NEXT:    vpshufb %ymm3, %ymm10, %ymm2
+; AVX512-NEXT:    vpternlogq $202, %ymm12, %ymm24, %ymm10
+; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm10[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm10[0],ymm1[1],ymm10[2,3],ymm1[4],ymm10[5,6],ymm1[7],ymm10[8],ymm1[9],ymm10[10,11],ymm1[12],ymm10[13,14],ymm1[15]
+; AVX512-NEXT:    vpshufb %ymm7, %ymm1, %ymm7
 ; AVX512-NEXT:    vmovdqa 80(%rdi), %xmm10
-; AVX512-NEXT:    vmovdqa 64(%rdi), %xmm15
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm3 = xmm15[0,1],xmm10[2],xmm15[3,4],xmm10[5],xmm15[6,7]
-; AVX512-NEXT:    vpshufb %xmm9, %xmm3, %xmm3
-; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm2[0,1,2],ymm3[3,4,5,6,7],ymm2[8,9,10],ymm3[11,12,13,14,15]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,4,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm18
-; AVX512-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512-NEXT:    vpternlogq $202, %ymm23, %ymm22, %ymm1
-; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7,8,9],ymm2[10],ymm1[11,12],ymm2[13],ymm1[14,15]
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm2 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
-; AVX512-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
-; AVX512-NEXT:    vmovdqa64 %ymm2, %ymm28
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1],xmm6[2],xmm4[3,4],xmm6[5],xmm4[6,7]
-; AVX512-NEXT:    vmovdqa64 %xmm6, %xmm25
+; AVX512-NEXT:    vmovdqa 64(%rdi), %xmm1
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm2 = xmm1[0,1],xmm10[2],xmm1[3,4],xmm10[5],xmm1[6,7]
+; AVX512-NEXT:    vpshufb %xmm13, %xmm2, %xmm2
+; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm7[0,1,2],ymm2[3,4,5,6,7],ymm7[8,9,10],ymm2[11,12,13,14,15]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,6,5,4,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm6, %zmm2, %zmm17
+; AVX512-NEXT:    vmovdqa %ymm0, %ymm2
+; AVX512-NEXT:    vpternlogq $202, %ymm22, %ymm21, %ymm2
+; AVX512-NEXT:    vpermq {{.*#+}} ymm6 = ymm2[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm6[2],ymm2[3,4],ymm6[5],ymm2[6,7,8,9],ymm6[10],ymm2[11,12],ymm6[13],ymm2[14,15]
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm9 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
+; AVX512-NEXT:    vpshufb %ymm9, %ymm2, %ymm2
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm7 = xmm4[0,1],xmm8[2],xmm4[3,4],xmm8[5],xmm4[6,7]
+; AVX512-NEXT:    vmovdqa64 %xmm8, %xmm25
 ; AVX512-NEXT:    vmovdqa64 %xmm4, %xmm26
 ; AVX512-NEXT:    vmovdqa {{.*#+}} xmm6 = [2,3,8,9,14,15,4,5,10,11,10,11,10,11,10,11]
-; AVX512-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm1[5,6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm1[4,5,6,7]
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm9 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
-; AVX512-NEXT:    vmovdqa %ymm9, %ymm1
-; AVX512-NEXT:    vpternlogq $202, %ymm21, %ymm20, %ymm1
-; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm1[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm4[2],ymm1[3,4],ymm4[5],ymm1[6,7,8,9],ymm4[10],ymm1[11,12],ymm4[13],ymm1[14,15]
+; AVX512-NEXT:    vpshufb %xmm6, %xmm7, %xmm7
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm7 = xmm7[0,1,2,3,4],xmm2[5,6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm13 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
+; AVX512-NEXT:    vmovdqa %ymm13, %ymm2
+; AVX512-NEXT:    vpternlogq $202, %ymm20, %ymm18, %ymm2
+; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm2[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm4[2],ymm2[3,4],ymm4[5],ymm2[6,7,8,9],ymm4[10],ymm2[11,12],ymm4[13],ymm2[14,15]
 ; AVX512-NEXT:    vmovdqa {{.*#+}} ymm4 = [2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]
-; AVX512-NEXT:    vpshufb %ymm4, %ymm1, %ymm1
-; AVX512-NEXT:    vmovdqa %xmm14, %xmm7
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm14 = xmm8[0,1],xmm14[2],xmm8[3,4],xmm14[5],xmm8[6,7]
-; AVX512-NEXT:    vmovdqa64 %xmm8, %xmm27
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]
-; AVX512-NEXT:    vpshufb %xmm2, %xmm14, %xmm14
-; AVX512-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm14 = ymm1[0,1,2],ymm14[3,4,5,6,7],ymm1[8,9,10],ymm14[11,12,13,14,15]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,6,7,4]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm14[4,5,6,7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm19
-; AVX512-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512-NEXT:    vpternlogq $202, %ymm13, %ymm24, %ymm1
-; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm1[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm3[2],ymm1[3,4],ymm3[5],ymm1[6,7,8,9],ymm3[10],ymm1[11,12],ymm3[13],ymm1[14,15]
-; AVX512-NEXT:    vmovdqa64 %ymm28, %ymm3
-; AVX512-NEXT:    vpshufb %ymm3, %ymm1, %ymm1
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm3 = xmm12[0,1],xmm11[2],xmm12[3,4],xmm11[5],xmm12[6,7]
-; AVX512-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm1[5,6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
-; AVX512-NEXT:    vmovdqa %ymm9, %ymm3
-; AVX512-NEXT:    vpternlogq $202, %ymm17, %ymm5, %ymm3
-; AVX512-NEXT:    vpermq {{.*#+}} ymm6 = ymm3[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1],ymm6[2],ymm3[3,4],ymm6[5],ymm3[6,7,8,9],ymm6[10],ymm3[11,12],ymm6[13],ymm3[14,15]
-; AVX512-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm10[0,1],xmm15[2],xmm10[3,4],xmm15[5],xmm10[6,7]
-; AVX512-NEXT:    vpshufb %xmm2, %xmm4, %xmm2
-; AVX512-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3,4,5,6,7],ymm3[8,9,10],ymm2[11,12,13,14,15]
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,6,7,4]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX512-NEXT:    vpternlogq $226, %ymm24, %ymm9, %ymm13
-; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm13[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm13[1,2],ymm2[3],ymm13[4,5],ymm2[6],ymm13[7],ymm2[8],ymm13[9,10],ymm2[11],ymm13[12,13],ymm2[14],ymm13[15]
-; AVX512-NEXT:    vpternlogq $226, %ymm17, %ymm0, %ymm5
-; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm5[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm5[1,2],ymm3[3],ymm5[4,5],ymm3[6],ymm5[7],ymm3[8],ymm5[9,10],ymm3[11],ymm5[12,13],ymm3[14],ymm5[15]
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm4 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
-; AVX512-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm5 = xmm15[0],xmm10[1],xmm15[2,3],xmm10[4],xmm15[5,6],xmm10[7]
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm6 = [0,1,2,3,0,1,6,7,12,13,2,3,8,9,14,15]
-; AVX512-NEXT:    vpshufb %xmm6, %xmm5, %xmm5
-; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm5[5,6,7]
 ; AVX512-NEXT:    vpshufb %ymm4, %ymm2, %ymm2
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm5 = xmm11[0,1],xmm12[2],xmm11[3,4],xmm12[5],xmm11[6,7]
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm8 = [4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
-; AVX512-NEXT:    vpshufb %xmm8, %xmm5, %xmm5
-; AVX512-NEXT:    vinserti64x4 $1, %ymm5, %zmm3, %zmm5
-; AVX512-NEXT:    vextracti32x4 $2, %zmm5, %xmm5
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4],xmm2[5,6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-NEXT:    vmovdqa64 %xmm19, %xmm8
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm14 = xmm8[0,1],xmm3[2],xmm8[3,4],xmm3[5],xmm8[6,7]
+; AVX512-NEXT:    vmovdqa64 %xmm3, %xmm27
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]
+; AVX512-NEXT:    vpshufb %xmm3, %xmm14, %xmm14
+; AVX512-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm14 = ymm2[0,1,2],ymm14[3,4,5,6,7],ymm2[8,9,10],ymm14[11,12,13,14,15]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,6,7,4]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm14[4,5,6,7]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm2, %zmm19
+; AVX512-NEXT:    vmovdqa %ymm0, %ymm2
+; AVX512-NEXT:    vpternlogq $202, %ymm11, %ymm23, %ymm2
+; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm2[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm7[2],ymm2[3,4],ymm7[5],ymm2[6,7,8,9],ymm7[10],ymm2[11,12],ymm7[13],ymm2[14,15]
+; AVX512-NEXT:    vpshufb %ymm9, %ymm2, %ymm2
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm7 = xmm5[0,1],xmm15[2],xmm5[3,4],xmm15[5],xmm5[6,7]
+; AVX512-NEXT:    vpshufb %xmm6, %xmm7, %xmm6
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4],xmm2[5,6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-NEXT:    vmovdqa %ymm13, %ymm6
+; AVX512-NEXT:    vpternlogq $202, %ymm24, %ymm12, %ymm6
+; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0,1],ymm7[2],ymm6[3,4],ymm7[5],ymm6[6,7,8,9],ymm7[10],ymm6[11,12],ymm7[13],ymm6[14,15]
+; AVX512-NEXT:    vpshufb %ymm4, %ymm6, %ymm4
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm6 = xmm10[0,1],xmm1[2],xmm10[3,4],xmm1[5],xmm10[6,7]
+; AVX512-NEXT:    vpshufb %xmm3, %xmm6, %xmm3
+; AVX512-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3,4,5,6,7],ymm4[8,9,10],ymm3[11,12,13,14,15]
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,6,7,4]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
-; AVX512-NEXT:    vpternlogq $202, %ymm22, %ymm23, %ymm9
-; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm9[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm9[1,2],ymm3[3],ymm9[4,5],ymm3[6],ymm9[7],ymm3[8],ymm9[9,10],ymm3[11],ymm9[12,13],ymm3[14],ymm9[15]
-; AVX512-NEXT:    vpternlogq $202, %ymm21, %ymm20, %ymm0
-; AVX512-NEXT:    vpermq {{.*#+}} ymm5 = ymm0[2,3,0,1]
-; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm5[0],ymm0[1,2],ymm5[3],ymm0[4,5],ymm5[6],ymm0[7],ymm5[8],ymm0[9,10],ymm5[11],ymm0[12,13],ymm5[14],ymm0[15]
-; AVX512-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
-; AVX512-NEXT:    vpshufb %ymm4, %ymm0, %ymm0
+; AVX512-NEXT:    vpternlogq $226, %ymm23, %ymm13, %ymm11
+; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm11[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm11[1,2],ymm3[3],ymm11[4,5],ymm3[6],ymm11[7],ymm3[8],ymm11[9,10],ymm3[11],ymm11[12,13],ymm3[14],ymm11[15]
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm11 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
+; AVX512-NEXT:    vpshufb %ymm11, %ymm3, %ymm3
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm15[0,1],xmm5[2],xmm15[3,4],xmm5[5],xmm15[6,7]
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm5 = [4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
+; AVX512-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512-NEXT:    vpternlogq $226, %ymm24, %ymm0, %ymm12
+; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm12[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0],ymm12[1,2],ymm4[3],ymm12[4,5],ymm4[6],ymm12[7],ymm4[8],ymm12[9,10],ymm4[11],ymm12[12,13],ymm4[14],ymm12[15]
+; AVX512-NEXT:    vpshufb %ymm11, %ymm4, %ymm4
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm10[1],xmm1[2,3],xmm10[4],xmm1[5,6],xmm10[7]
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm6 = [0,1,2,3,0,1,6,7,12,13,2,3,8,9,14,15]
+; AVX512-NEXT:    vpshufb %xmm6, %xmm1, %xmm1
+; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4],ymm1[5,6,7]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
+; AVX512-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm13
+; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm13[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm13[1,2],ymm3[3],ymm13[4,5],ymm3[6],ymm13[7],ymm3[8],ymm13[9,10],ymm3[11],ymm13[12,13],ymm3[14],ymm13[15]
+; AVX512-NEXT:    vmovdqa64 %xmm25, %xmm4
+; AVX512-NEXT:    vmovdqa64 %xmm26, %xmm7
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm7[2],xmm4[3,4],xmm7[5],xmm4[6,7]
+; AVX512-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
+; AVX512-NEXT:    vpshufb %ymm11, %ymm3, %ymm3
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
+; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512-NEXT:    vpternlogq $202, %ymm20, %ymm18, %ymm0
+; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm0[2,3,0,1]
+; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm4[0],ymm0[1,2],ymm4[3],ymm0[4,5],ymm4[6],ymm0[7],ymm4[8],ymm0[9,10],ymm4[11],ymm0[12,13],ymm4[14],ymm0[15]
+; AVX512-NEXT:    vpshufb %ymm11, %ymm0, %ymm0
 ; AVX512-NEXT:    vmovdqa64 %xmm27, %xmm4
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm7[0],xmm4[1],xmm7[2,3],xmm4[4],xmm7[5,6],xmm4[7]
+; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0],xmm8[1],xmm4[2,3],xmm8[4],xmm4[5,6],xmm8[7]
 ; AVX512-NEXT:    vpshufb %xmm6, %xmm4, %xmm4
 ; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm4[5,6,7]
-; AVX512-NEXT:    vmovdqa64 %xmm25, %xmm4
-; AVX512-NEXT:    vmovdqa64 %xmm26, %xmm5
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2],xmm4[3,4],xmm5[5],xmm4[6,7]
-; AVX512-NEXT:    vpshufb %xmm8, %xmm4, %xmm4
-; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm4
-; AVX512-NEXT:    vextracti32x4 $2, %zmm4, %xmm4
-; AVX512-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
-; AVX512-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
-; AVX512-NEXT:    vmovdqa64 %zmm18, (%rsi)
+; AVX512-NEXT:    vmovdqa64 %zmm17, (%rsi)
 ; AVX512-NEXT:    vmovdqa64 %zmm16, 64(%rsi)
 ; AVX512-NEXT:    vmovdqa64 %zmm19, 64(%rdx)
-; AVX512-NEXT:    vmovdqa64 %zmm1, (%rdx)
+; AVX512-NEXT:    vmovdqa64 %zmm2, (%rdx)
 ; AVX512-NEXT:    vmovdqa64 %zmm0, 64(%rcx)
-; AVX512-NEXT:    vmovdqa64 %zmm2, (%rcx)
+; AVX512-NEXT:    vmovdqa64 %zmm1, (%rcx)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
 ; AVX512-FCP-LABEL: load_i16_stride3_vf64:
 ; AVX512-FCP:       # %bb.0:
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
-; AVX512-FCP-NEXT:    vmovdqa64 224(%rdi), %ymm20
-; AVX512-FCP-NEXT:    vmovdqa64 192(%rdi), %ymm21
+; AVX512-FCP-NEXT:    vmovdqa64 224(%rdi), %ymm18
+; AVX512-FCP-NEXT:    vmovdqa64 192(%rdi), %ymm20
 ; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm20, %ymm21, %ymm1
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm1[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm3[1],ymm1[2,3],ymm3[4],ymm1[5,6],ymm3[7],ymm1[8],ymm3[9],ymm1[10,11],ymm3[12],ymm1[13,14],ymm3[15]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]
-; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm1, %ymm5
-; AVX512-FCP-NEXT:    vmovdqa 272(%rdi), %xmm8
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm18, %ymm20, %ymm1
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm7 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]
+; AVX512-FCP-NEXT:    vpshufb %ymm7, %ymm2, %ymm5
+; AVX512-FCP-NEXT:    vmovdqa 272(%rdi), %xmm1
 ; AVX512-FCP-NEXT:    vmovdqa 256(%rdi), %xmm2
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm6 = xmm2[0,1],xmm8[2],xmm2[3,4],xmm8[5],xmm2[6,7]
-; AVX512-FCP-NEXT:    vmovdqa %xmm2, %xmm14
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm9 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]
-; AVX512-FCP-NEXT:    vpshufb %xmm9, %xmm6, %xmm6
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm6 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]
+; AVX512-FCP-NEXT:    vmovdqa %xmm2, %xmm3
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm1, %xmm19
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm13 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]
+; AVX512-FCP-NEXT:    vpshufb %xmm13, %xmm6, %xmm6
 ; AVX512-FCP-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm5[0,1,2],ymm6[3,4,5,6,7],ymm5[8,9,10],ymm6[11,12,13,14,15]
 ; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,6,5,4,7]
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4,5,6,7]
-; AVX512-FCP-NEXT:    vmovdqa64 320(%rdi), %ymm22
-; AVX512-FCP-NEXT:    vmovdqa64 352(%rdi), %ymm23
-; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm6
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm22, %ymm23, %ymm6
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0],ymm7[1],ymm6[2,3],ymm7[4],ymm6[5,6],ymm7[7],ymm6[8],ymm7[9],ymm6[10,11],ymm7[12],ymm6[13,14],ymm7[15]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm11 = [0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
-; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm6, %ymm12
+; AVX512-FCP-NEXT:    vmovdqa64 320(%rdi), %ymm21
+; AVX512-FCP-NEXT:    vmovdqa64 352(%rdi), %ymm22
+; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm8
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm8
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm8[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm8 = ymm8[0],ymm9[1],ymm8[2,3],ymm9[4],ymm8[5,6],ymm9[7],ymm8[8],ymm9[9],ymm8[10,11],ymm9[12],ymm8[13,14],ymm9[15]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm10 = [0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
+; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm8, %ymm11
 ; AVX512-FCP-NEXT:    vmovdqa 304(%rdi), %xmm1
 ; AVX512-FCP-NEXT:    vmovdqa 288(%rdi), %xmm2
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm13 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
 ; AVX512-FCP-NEXT:    vmovdqa %xmm2, %xmm4
-; AVX512-FCP-NEXT:    vmovdqa %xmm1, %xmm6
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm15 = [0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]
-; AVX512-FCP-NEXT:    vpshufb %xmm15, %xmm13, %xmm13
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm12 = ymm13[0,1,2],ymm12[3,4,5,6,7]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm5, %zmm16
-; AVX512-FCP-NEXT:    vmovdqa64 128(%rdi), %ymm24
-; AVX512-FCP-NEXT:    vmovdqa 160(%rdi), %ymm13
+; AVX512-FCP-NEXT:    vmovdqa %xmm1, %xmm8
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm14 = [0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]
+; AVX512-FCP-NEXT:    vpshufb %xmm14, %xmm12, %xmm12
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm11 = ymm12[0,1,2],ymm11[3,4,5,6,7]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm5, %zmm16
+; AVX512-FCP-NEXT:    vmovdqa64 128(%rdi), %ymm23
+; AVX512-FCP-NEXT:    vmovdqa 160(%rdi), %ymm11
 ; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm5
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm24, %ymm13, %ymm5
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm23, %ymm11, %ymm5
 ; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm12 = ymm5[2,3,0,1]
 ; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1],ymm5[2,3],ymm12[4],ymm5[5,6],ymm12[7],ymm5[8],ymm12[9],ymm5[10,11],ymm12[12],ymm5[13,14],ymm12[15]
-; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm5, %ymm5
-; AVX512-FCP-NEXT:    vmovdqa 112(%rdi), %xmm11
-; AVX512-FCP-NEXT:    vmovdqa 96(%rdi), %xmm12
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm10 = xmm12[0],xmm11[1],xmm12[2,3],xmm11[4],xmm12[5,6],xmm11[7]
-; AVX512-FCP-NEXT:    vpshufb %xmm15, %xmm10, %xmm10
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm10[0,1,2],ymm5[3,4,5,6,7]
-; AVX512-FCP-NEXT:    vmovdqa64 (%rdi), %ymm17
-; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %ymm5
+; AVX512-FCP-NEXT:    vpshufb %ymm10, %ymm5, %ymm10
+; AVX512-FCP-NEXT:    vmovdqa 112(%rdi), %xmm15
+; AVX512-FCP-NEXT:    vmovdqa 96(%rdi), %xmm5
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm12 = xmm5[0],xmm15[1],xmm5[2,3],xmm15[4],xmm5[5,6],xmm15[7]
+; AVX512-FCP-NEXT:    vpshufb %xmm14, %xmm12, %xmm12
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm6 = ymm12[0,1,2],ymm10[3,4,5,6,7]
+; AVX512-FCP-NEXT:    vmovdqa64 (%rdi), %ymm24
+; AVX512-FCP-NEXT:    vmovdqa 32(%rdi), %ymm12
 ; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm10
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm5, %ymm17, %ymm10
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm15 = ymm10[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0],ymm15[1],ymm10[2,3],ymm15[4],ymm10[5,6],ymm15[7],ymm10[8],ymm15[9],ymm10[10,11],ymm15[12],ymm10[13,14],ymm15[15]
-; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm10, %ymm2
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm12, %ymm24, %ymm10
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm10[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm10[0],ymm1[1],ymm10[2,3],ymm1[4],ymm10[5,6],ymm1[7],ymm10[8],ymm1[9],ymm10[10,11],ymm1[12],ymm10[13,14],ymm1[15]
+; AVX512-FCP-NEXT:    vpshufb %ymm7, %ymm1, %ymm7
 ; AVX512-FCP-NEXT:    vmovdqa 80(%rdi), %xmm10
-; AVX512-FCP-NEXT:    vmovdqa 64(%rdi), %xmm15
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm15[0,1],xmm10[2],xmm15[3,4],xmm10[5],xmm15[6,7]
-; AVX512-FCP-NEXT:    vpshufb %xmm9, %xmm3, %xmm3
-; AVX512-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm2[0,1,2],ymm3[3,4,5,6,7],ymm2[8,9,10],ymm3[11,12,13,14,15]
-; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,4,7]
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm18
-; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm23, %ymm22, %ymm1
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7,8,9],ymm2[10],ymm1[11,12],ymm2[13],ymm1[14,15]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm2 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
-; AVX512-FCP-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm2, %ymm28
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1],xmm6[2],xmm4[3,4],xmm6[5],xmm4[6,7]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm6, %xmm25
+; AVX512-FCP-NEXT:    vmovdqa 64(%rdi), %xmm1
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm2 = xmm1[0,1],xmm10[2],xmm1[3,4],xmm10[5],xmm1[6,7]
+; AVX512-FCP-NEXT:    vpshufb %xmm13, %xmm2, %xmm2
+; AVX512-FCP-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm7[0,1,2],ymm2[3,4,5,6,7],ymm7[8,9,10],ymm2[11,12,13,14,15]
+; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,6,5,4,7]
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm2, %zmm17
+; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm2
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm22, %ymm21, %ymm2
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm2[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm6[2],ymm2[3,4],ymm6[5],ymm2[6,7,8,9],ymm6[10],ymm2[11,12],ymm6[13],ymm2[14,15]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm9 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
+; AVX512-FCP-NEXT:    vpshufb %ymm9, %ymm2, %ymm2
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm7 = xmm4[0,1],xmm8[2],xmm4[3,4],xmm8[5],xmm4[6,7]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm8, %xmm25
 ; AVX512-FCP-NEXT:    vmovdqa64 %xmm4, %xmm26
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm6 = [2,3,8,9,14,15,4,5,10,11,10,11,10,11,10,11]
-; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm1[5,6,7]
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm1[4,5,6,7]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm9 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
-; AVX512-FCP-NEXT:    vmovdqa %ymm9, %ymm1
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm21, %ymm20, %ymm1
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm1[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm4[2],ymm1[3,4],ymm4[5],ymm1[6,7,8,9],ymm4[10],ymm1[11,12],ymm4[13],ymm1[14,15]
+; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm7, %xmm7
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm7 = xmm7[0,1,2,3,4],xmm2[5,6,7]
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm13 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
+; AVX512-FCP-NEXT:    vmovdqa %ymm13, %ymm2
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm20, %ymm18, %ymm2
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm2[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm4[2],ymm2[3,4],ymm4[5],ymm2[6,7,8,9],ymm4[10],ymm2[11,12],ymm4[13],ymm2[14,15]
 ; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm4 = [2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]
-; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm1, %ymm1
-; AVX512-FCP-NEXT:    vmovdqa %xmm14, %xmm7
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm14 = xmm8[0,1],xmm14[2],xmm8[3,4],xmm14[5],xmm8[6,7]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm8, %xmm27
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]
-; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm14, %xmm14
-; AVX512-FCP-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm14 = ymm1[0,1,2],ymm14[3,4,5,6,7],ymm1[8,9,10],ymm14[11,12,13,14,15]
-; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,6,7,4]
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm14[4,5,6,7]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm19
-; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm13, %ymm24, %ymm1
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm1[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm3[2],ymm1[3,4],ymm3[5],ymm1[6,7,8,9],ymm3[10],ymm1[11,12],ymm3[13],ymm1[14,15]
-; AVX512-FCP-NEXT:    vmovdqa64 %ymm28, %ymm3
-; AVX512-FCP-NEXT:    vpshufb %ymm3, %ymm1, %ymm1
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm12[0,1],xmm11[2],xmm12[3,4],xmm11[5],xmm12[6,7]
-; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm1[5,6,7]
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
-; AVX512-FCP-NEXT:    vmovdqa %ymm9, %ymm3
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm17, %ymm5, %ymm3
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm3[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1],ymm6[2],ymm3[3,4],ymm6[5],ymm3[6,7,8,9],ymm6[10],ymm3[11,12],ymm6[13],ymm3[14,15]
-; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm10[0,1],xmm15[2],xmm10[3,4],xmm15[5],xmm10[6,7]
-; AVX512-FCP-NEXT:    vpshufb %xmm2, %xmm4, %xmm2
-; AVX512-FCP-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3,4,5,6,7],ymm3[8,9,10],ymm2[11,12,13,14,15]
-; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,6,7,4]
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX512-FCP-NEXT:    vpternlogq $226, %ymm24, %ymm9, %ymm13
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm13[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm13[1,2],ymm2[3],ymm13[4,5],ymm2[6],ymm13[7],ymm2[8],ymm13[9,10],ymm2[11],ymm13[12,13],ymm2[14],ymm13[15]
-; AVX512-FCP-NEXT:    vpternlogq $226, %ymm17, %ymm0, %ymm5
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm5[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm5[1,2],ymm3[3],ymm5[4,5],ymm3[6],ymm5[7],ymm3[8],ymm5[9,10],ymm3[11],ymm5[12,13],ymm3[14],ymm5[15]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm4 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
-; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm5 = xmm15[0],xmm10[1],xmm15[2,3],xmm10[4],xmm15[5,6],xmm10[7]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm6 = [0,1,2,3,0,1,6,7,12,13,2,3,8,9,14,15]
-; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm5, %xmm5
-; AVX512-FCP-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm5[5,6,7]
 ; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm2, %ymm2
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm5 = xmm11[0,1],xmm12[2],xmm11[3,4],xmm12[5],xmm11[6,7]
-; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm8 = [4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
-; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm5, %xmm5
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm3, %zmm5
-; AVX512-FCP-NEXT:    vextracti32x4 $2, %zmm5, %xmm5
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4],xmm2[5,6,7]
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm19, %xmm8
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm14 = xmm8[0,1],xmm3[2],xmm8[3,4],xmm3[5],xmm8[6,7]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm3, %xmm27
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]
+; AVX512-FCP-NEXT:    vpshufb %xmm3, %xmm14, %xmm14
+; AVX512-FCP-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm14 = ymm2[0,1,2],ymm14[3,4,5,6,7],ymm2[8,9,10],ymm14[11,12,13,14,15]
+; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,6,7,4]
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm14[4,5,6,7]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm2, %zmm19
+; AVX512-FCP-NEXT:    vmovdqa %ymm0, %ymm2
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm11, %ymm23, %ymm2
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm2[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm7[2],ymm2[3,4],ymm7[5],ymm2[6,7,8,9],ymm7[10],ymm2[11,12],ymm7[13],ymm2[14,15]
+; AVX512-FCP-NEXT:    vpshufb %ymm9, %ymm2, %ymm2
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm7 = xmm5[0,1],xmm15[2],xmm5[3,4],xmm15[5],xmm5[6,7]
+; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm7, %xmm6
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4],xmm2[5,6,7]
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm2[4,5,6,7]
+; AVX512-FCP-NEXT:    vmovdqa %ymm13, %ymm6
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm24, %ymm12, %ymm6
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0,1],ymm7[2],ymm6[3,4],ymm7[5],ymm6[6,7,8,9],ymm7[10],ymm6[11,12],ymm7[13],ymm6[14,15]
+; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm6, %ymm4
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm6 = xmm10[0,1],xmm1[2],xmm10[3,4],xmm1[5],xmm10[6,7]
+; AVX512-FCP-NEXT:    vpshufb %xmm3, %xmm6, %xmm3
+; AVX512-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3,4,5,6,7],ymm4[8,9,10],ymm3[11,12,13,14,15]
+; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,6,7,4]
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm22, %ymm23, %ymm9
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm9[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm9[1,2],ymm3[3],ymm9[4,5],ymm3[6],ymm9[7],ymm3[8],ymm9[9,10],ymm3[11],ymm9[12,13],ymm3[14],ymm9[15]
-; AVX512-FCP-NEXT:    vpternlogq $202, %ymm21, %ymm20, %ymm0
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm5 = ymm0[2,3,0,1]
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm5[0],ymm0[1,2],ymm5[3],ymm0[4,5],ymm5[6],ymm0[7],ymm5[8],ymm0[9,10],ymm5[11],ymm0[12,13],ymm5[14],ymm0[15]
-; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
-; AVX512-FCP-NEXT:    vpshufb %ymm4, %ymm0, %ymm0
+; AVX512-FCP-NEXT:    vpternlogq $226, %ymm23, %ymm13, %ymm11
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm11[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm11[1,2],ymm3[3],ymm11[4,5],ymm3[6],ymm11[7],ymm3[8],ymm11[9,10],ymm3[11],ymm11[12,13],ymm3[14],ymm11[15]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} ymm11 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
+; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm3, %ymm3
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm15[0,1],xmm5[2],xmm15[3,4],xmm5[5],xmm15[6,7]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm5 = [4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
+; AVX512-FCP-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512-FCP-NEXT:    vpternlogq $226, %ymm24, %ymm0, %ymm12
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm12[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0],ymm12[1,2],ymm4[3],ymm12[4,5],ymm4[6],ymm12[7],ymm4[8],ymm12[9,10],ymm4[11],ymm12[12,13],ymm4[14],ymm12[15]
+; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm4, %ymm4
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm10[1],xmm1[2,3],xmm10[4],xmm1[5,6],xmm10[7]
+; AVX512-FCP-NEXT:    vmovdqa {{.*#+}} xmm6 = [0,1,2,3,0,1,6,7,12,13,2,3,8,9,14,15]
+; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm1, %xmm1
+; AVX512-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4],ymm1[5,6,7]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm13
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm13[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm13[1,2],ymm3[3],ymm13[4,5],ymm3[6],ymm13[7],ymm3[8],ymm13[9,10],ymm3[11],ymm13[12,13],ymm3[14],ymm13[15]
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm25, %xmm4
+; AVX512-FCP-NEXT:    vmovdqa64 %xmm26, %xmm7
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm7[2],xmm4[3,4],xmm7[5],xmm4[6,7]
+; AVX512-FCP-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
+; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm3, %ymm3
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
+; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512-FCP-NEXT:    vpternlogq $202, %ymm20, %ymm18, %ymm0
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm0[2,3,0,1]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm4[0],ymm0[1,2],ymm4[3],ymm0[4,5],ymm4[6],ymm0[7],ymm4[8],ymm0[9,10],ymm4[11],ymm0[12,13],ymm4[14],ymm0[15]
+; AVX512-FCP-NEXT:    vpshufb %ymm11, %ymm0, %ymm0
 ; AVX512-FCP-NEXT:    vmovdqa64 %xmm27, %xmm4
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm7[0],xmm4[1],xmm7[2,3],xmm4[4],xmm7[5,6],xmm4[7]
+; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0],xmm8[1],xmm4[2,3],xmm8[4],xmm4[5,6],xmm8[7]
 ; AVX512-FCP-NEXT:    vpshufb %xmm6, %xmm4, %xmm4
 ; AVX512-FCP-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm4[5,6,7]
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm25, %xmm4
-; AVX512-FCP-NEXT:    vmovdqa64 %xmm26, %xmm5
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2],xmm4[3,4],xmm5[5],xmm4[6,7]
-; AVX512-FCP-NEXT:    vpshufb %xmm8, %xmm4, %xmm4
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm4
-; AVX512-FCP-NEXT:    vextracti32x4 $2, %zmm4, %xmm4
-; AVX512-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
-; AVX512-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm18, (%rsi)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm17, (%rsi)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm16, 64(%rsi)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm19, 64(%rdx)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, (%rdx)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, (%rdx)
 ; AVX512-FCP-NEXT:    vmovdqa64 %zmm0, 64(%rcx)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm2, (%rcx)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, (%rcx)
 ; AVX512-FCP-NEXT:    vzeroupper
 ; AVX512-FCP-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: load_i16_stride3_vf64:
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
-; AVX512DQ-NEXT:    vmovdqa64 224(%rdi), %ymm20
-; AVX512DQ-NEXT:    vmovdqa64 192(%rdi), %ymm21
+; AVX512DQ-NEXT:    vmovdqa64 224(%rdi), %ymm18
+; AVX512DQ-NEXT:    vmovdqa64 192(%rdi), %ymm20
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm20, %ymm21, %ymm1
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm1[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm3[1],ymm1[2,3],ymm3[4],ymm1[5,6],ymm3[7],ymm1[8],ymm3[9],ymm1[10,11],ymm3[12],ymm1[13,14],ymm3[15]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]
-; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm1, %ymm5
-; AVX512DQ-NEXT:    vmovdqa 272(%rdi), %xmm8
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm18, %ymm20, %ymm1
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm7 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]
+; AVX512DQ-NEXT:    vpshufb %ymm7, %ymm2, %ymm5
+; AVX512DQ-NEXT:    vmovdqa 272(%rdi), %xmm1
 ; AVX512DQ-NEXT:    vmovdqa 256(%rdi), %xmm2
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm6 = xmm2[0,1],xmm8[2],xmm2[3,4],xmm8[5],xmm2[6,7]
-; AVX512DQ-NEXT:    vmovdqa %xmm2, %xmm14
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm9 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]
-; AVX512DQ-NEXT:    vpshufb %xmm9, %xmm6, %xmm6
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm6 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]
+; AVX512DQ-NEXT:    vmovdqa %xmm2, %xmm3
+; AVX512DQ-NEXT:    vmovdqa64 %xmm1, %xmm19
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm13 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]
+; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm6, %xmm6
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm6 = ymm5[0,1,2],ymm6[3,4,5,6,7],ymm5[8,9,10],ymm6[11,12,13,14,15]
 ; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,6,5,4,7]
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa64 320(%rdi), %ymm22
-; AVX512DQ-NEXT:    vmovdqa64 352(%rdi), %ymm23
-; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm6
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm22, %ymm23, %ymm6
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0],ymm7[1],ymm6[2,3],ymm7[4],ymm6[5,6],ymm7[7],ymm6[8],ymm7[9],ymm6[10,11],ymm7[12],ymm6[13,14],ymm7[15]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm11 = [0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
-; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm6, %ymm12
+; AVX512DQ-NEXT:    vmovdqa64 320(%rdi), %ymm21
+; AVX512DQ-NEXT:    vmovdqa64 352(%rdi), %ymm22
+; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm8
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm8
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm8[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm8 = ymm8[0],ymm9[1],ymm8[2,3],ymm9[4],ymm8[5,6],ymm9[7],ymm8[8],ymm9[9],ymm8[10,11],ymm9[12],ymm8[13,14],ymm9[15]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm10 = [0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
+; AVX512DQ-NEXT:    vpshufb %ymm10, %ymm8, %ymm11
 ; AVX512DQ-NEXT:    vmovdqa 304(%rdi), %xmm1
 ; AVX512DQ-NEXT:    vmovdqa 288(%rdi), %xmm2
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm13 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
 ; AVX512DQ-NEXT:    vmovdqa %xmm2, %xmm4
-; AVX512DQ-NEXT:    vmovdqa %xmm1, %xmm6
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm15 = [0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]
-; AVX512DQ-NEXT:    vpshufb %xmm15, %xmm13, %xmm13
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm12 = ymm13[0,1,2],ymm12[3,4,5,6,7]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm12, %zmm5, %zmm16
-; AVX512DQ-NEXT:    vmovdqa64 128(%rdi), %ymm24
-; AVX512DQ-NEXT:    vmovdqa 160(%rdi), %ymm13
+; AVX512DQ-NEXT:    vmovdqa %xmm1, %xmm8
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm14 = [0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm12, %xmm12
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm11 = ymm12[0,1,2],ymm11[3,4,5,6,7]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm11, %zmm5, %zmm16
+; AVX512DQ-NEXT:    vmovdqa64 128(%rdi), %ymm23
+; AVX512DQ-NEXT:    vmovdqa 160(%rdi), %ymm11
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm5
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm24, %ymm13, %ymm5
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm23, %ymm11, %ymm5
 ; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm12 = ymm5[2,3,0,1]
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1],ymm5[2,3],ymm12[4],ymm5[5,6],ymm12[7],ymm5[8],ymm12[9],ymm5[10,11],ymm12[12],ymm5[13,14],ymm12[15]
-; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm5, %ymm5
-; AVX512DQ-NEXT:    vmovdqa 112(%rdi), %xmm11
-; AVX512DQ-NEXT:    vmovdqa 96(%rdi), %xmm12
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm10 = xmm12[0],xmm11[1],xmm12[2,3],xmm11[4],xmm12[5,6],xmm11[7]
-; AVX512DQ-NEXT:    vpshufb %xmm15, %xmm10, %xmm10
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm10[0,1,2],ymm5[3,4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa64 (%rdi), %ymm17
-; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm5
+; AVX512DQ-NEXT:    vpshufb %ymm10, %ymm5, %ymm10
+; AVX512DQ-NEXT:    vmovdqa 112(%rdi), %xmm15
+; AVX512DQ-NEXT:    vmovdqa 96(%rdi), %xmm5
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm12 = xmm5[0],xmm15[1],xmm5[2,3],xmm15[4],xmm5[5,6],xmm15[7]
+; AVX512DQ-NEXT:    vpshufb %xmm14, %xmm12, %xmm12
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm6 = ymm12[0,1,2],ymm10[3,4,5,6,7]
+; AVX512DQ-NEXT:    vmovdqa64 (%rdi), %ymm24
+; AVX512DQ-NEXT:    vmovdqa 32(%rdi), %ymm12
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm10
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm5, %ymm17, %ymm10
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm15 = ymm10[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0],ymm15[1],ymm10[2,3],ymm15[4],ymm10[5,6],ymm15[7],ymm10[8],ymm15[9],ymm10[10,11],ymm15[12],ymm10[13,14],ymm15[15]
-; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm10, %ymm2
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm12, %ymm24, %ymm10
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm10[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm10[0],ymm1[1],ymm10[2,3],ymm1[4],ymm10[5,6],ymm1[7],ymm10[8],ymm1[9],ymm10[10,11],ymm1[12],ymm10[13,14],ymm1[15]
+; AVX512DQ-NEXT:    vpshufb %ymm7, %ymm1, %ymm7
 ; AVX512DQ-NEXT:    vmovdqa 80(%rdi), %xmm10
-; AVX512DQ-NEXT:    vmovdqa 64(%rdi), %xmm15
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm3 = xmm15[0,1],xmm10[2],xmm15[3,4],xmm10[5],xmm15[6,7]
-; AVX512DQ-NEXT:    vpshufb %xmm9, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm2[0,1,2],ymm3[3,4,5,6,7],ymm2[8,9,10],ymm3[11,12,13,14,15]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,4,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm18
-; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm23, %ymm22, %ymm1
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7,8,9],ymm2[10],ymm1[11,12],ymm2[13],ymm1[14,15]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm2 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
-; AVX512DQ-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vmovdqa64 %ymm2, %ymm28
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1],xmm6[2],xmm4[3,4],xmm6[5],xmm4[6,7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm6, %xmm25
+; AVX512DQ-NEXT:    vmovdqa 64(%rdi), %xmm1
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm2 = xmm1[0,1],xmm10[2],xmm1[3,4],xmm10[5],xmm1[6,7]
+; AVX512DQ-NEXT:    vpshufb %xmm13, %xmm2, %xmm2
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm7[0,1,2],ymm2[3,4,5,6,7],ymm7[8,9,10],ymm2[11,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,6,5,4,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm6, %zmm2, %zmm17
+; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm2
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm22, %ymm21, %ymm2
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm6 = ymm2[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm6[2],ymm2[3,4],ymm6[5],ymm2[6,7,8,9],ymm6[10],ymm2[11,12],ymm6[13],ymm2[14,15]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm9 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
+; AVX512DQ-NEXT:    vpshufb %ymm9, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm7 = xmm4[0,1],xmm8[2],xmm4[3,4],xmm8[5],xmm4[6,7]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm8, %xmm25
 ; AVX512DQ-NEXT:    vmovdqa64 %xmm4, %xmm26
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm6 = [2,3,8,9,14,15,4,5,10,11,10,11,10,11,10,11]
-; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm1[5,6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm9 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
-; AVX512DQ-NEXT:    vmovdqa %ymm9, %ymm1
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm21, %ymm20, %ymm1
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm1[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm4[2],ymm1[3,4],ymm4[5],ymm1[6,7,8,9],ymm4[10],ymm1[11,12],ymm4[13],ymm1[14,15]
+; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm7, %xmm7
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm7 = xmm7[0,1,2,3,4],xmm2[5,6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm13 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
+; AVX512DQ-NEXT:    vmovdqa %ymm13, %ymm2
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm20, %ymm18, %ymm2
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm2[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm4[2],ymm2[3,4],ymm4[5],ymm2[6,7,8,9],ymm4[10],ymm2[11,12],ymm4[13],ymm2[14,15]
 ; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm4 = [2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]
-; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vmovdqa %xmm14, %xmm7
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm14 = xmm8[0,1],xmm14[2],xmm8[3,4],xmm14[5],xmm8[6,7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm8, %xmm27
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm2 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm14, %xmm14
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm14 = ymm1[0,1,2],ymm14[3,4,5,6,7],ymm1[8,9,10],ymm14[11,12,13,14,15]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,6,7,4]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm14[4,5,6,7]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm19
-; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm13, %ymm24, %ymm1
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm1[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm3[2],ymm1[3,4],ymm3[5],ymm1[6,7,8,9],ymm3[10],ymm1[11,12],ymm3[13],ymm1[14,15]
-; AVX512DQ-NEXT:    vmovdqa64 %ymm28, %ymm3
-; AVX512DQ-NEXT:    vpshufb %ymm3, %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm3 = xmm12[0,1],xmm11[2],xmm12[3,4],xmm11[5],xmm12[6,7]
-; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm1[5,6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-NEXT:    vmovdqa %ymm9, %ymm3
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm17, %ymm5, %ymm3
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm6 = ymm3[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1],ymm6[2],ymm3[3,4],ymm6[5],ymm3[6,7,8,9],ymm6[10],ymm3[11,12],ymm6[13],ymm3[14,15]
-; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm10[0,1],xmm15[2],xmm10[3,4],xmm15[5],xmm10[6,7]
-; AVX512DQ-NEXT:    vpshufb %xmm2, %xmm4, %xmm2
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3,4,5,6,7],ymm3[8,9,10],ymm2[11,12,13,14,15]
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,6,7,4]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX512DQ-NEXT:    vpternlogq $226, %ymm24, %ymm9, %ymm13
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm13[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm13[1,2],ymm2[3],ymm13[4,5],ymm2[6],ymm13[7],ymm2[8],ymm13[9,10],ymm2[11],ymm13[12,13],ymm2[14],ymm13[15]
-; AVX512DQ-NEXT:    vpternlogq $226, %ymm17, %ymm0, %ymm5
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm5[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm5[1,2],ymm3[3],ymm5[4,5],ymm3[6],ymm5[7],ymm3[8],ymm5[9,10],ymm3[11],ymm5[12,13],ymm3[14],ymm5[15]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm4 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
-; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm5 = xmm15[0],xmm10[1],xmm15[2,3],xmm10[4],xmm15[5,6],xmm10[7]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm6 = [0,1,2,3,0,1,6,7,12,13,2,3,8,9,14,15]
-; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm5, %xmm5
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm5[5,6,7]
 ; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm5 = xmm11[0,1],xmm12[2],xmm11[3,4],xmm12[5],xmm11[6,7]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm8 = [4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
-; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm5, %xmm5
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm5, %zmm3, %zmm5
-; AVX512DQ-NEXT:    vextracti32x4 $2, %zmm5, %xmm5
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4],xmm2[5,6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm19, %xmm8
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm14 = xmm8[0,1],xmm3[2],xmm8[3,4],xmm3[5],xmm8[6,7]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm3, %xmm27
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm3 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]
+; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm14, %xmm14
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm14 = ymm2[0,1,2],ymm14[3,4,5,6,7],ymm2[8,9,10],ymm14[11,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,6,7,4]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm14[4,5,6,7]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm2, %zmm19
+; AVX512DQ-NEXT:    vmovdqa %ymm0, %ymm2
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm11, %ymm23, %ymm2
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm2[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm7[2],ymm2[3,4],ymm7[5],ymm2[6,7,8,9],ymm7[10],ymm2[11,12],ymm7[13],ymm2[14,15]
+; AVX512DQ-NEXT:    vpshufb %ymm9, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm7 = xmm5[0,1],xmm15[2],xmm5[3,4],xmm15[5],xmm5[6,7]
+; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm7, %xmm6
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4],xmm2[5,6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-NEXT:    vmovdqa %ymm13, %ymm6
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm24, %ymm12, %ymm6
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0,1],ymm7[2],ymm6[3,4],ymm7[5],ymm6[6,7,8,9],ymm7[10],ymm6[11,12],ymm7[13],ymm6[14,15]
+; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm6, %ymm4
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm6 = xmm10[0,1],xmm1[2],xmm10[3,4],xmm1[5],xmm10[6,7]
+; AVX512DQ-NEXT:    vpshufb %xmm3, %xmm6, %xmm3
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3,4,5,6,7],ymm4[8,9,10],ymm3[11,12,13,14,15]
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,6,7,4]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm22, %ymm23, %ymm9
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm9[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm9[1,2],ymm3[3],ymm9[4,5],ymm3[6],ymm9[7],ymm3[8],ymm9[9,10],ymm3[11],ymm9[12,13],ymm3[14],ymm9[15]
-; AVX512DQ-NEXT:    vpternlogq $202, %ymm21, %ymm20, %ymm0
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm5 = ymm0[2,3,0,1]
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm5[0],ymm0[1,2],ymm5[3],ymm0[4,5],ymm5[6],ymm0[7],ymm5[8],ymm0[9,10],ymm5[11],ymm0[12,13],ymm5[14],ymm0[15]
-; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
-; AVX512DQ-NEXT:    vpshufb %ymm4, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpternlogq $226, %ymm23, %ymm13, %ymm11
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm11[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm11[1,2],ymm3[3],ymm11[4,5],ymm3[6],ymm11[7],ymm3[8],ymm11[9,10],ymm3[11],ymm11[12,13],ymm3[14],ymm11[15]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm11 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
+; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm15[0,1],xmm5[2],xmm15[3,4],xmm5[5],xmm15[6,7]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm5 = [4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
+; AVX512DQ-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512DQ-NEXT:    vpternlogq $226, %ymm24, %ymm0, %ymm12
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm12[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0],ymm12[1,2],ymm4[3],ymm12[4,5],ymm4[6],ymm12[7],ymm4[8],ymm12[9,10],ymm4[11],ymm12[12,13],ymm4[14],ymm12[15]
+; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm4, %ymm4
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm10[1],xmm1[2,3],xmm10[4],xmm1[5,6],xmm10[7]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} xmm6 = [0,1,2,3,0,1,6,7,12,13,2,3,8,9,14,15]
+; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm1, %xmm1
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4],ymm1[5,6,7]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm13
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm13[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm13[1,2],ymm3[3],ymm13[4,5],ymm3[6],ymm13[7],ymm3[8],ymm13[9,10],ymm3[11],ymm13[12,13],ymm3[14],ymm13[15]
+; AVX512DQ-NEXT:    vmovdqa64 %xmm25, %xmm4
+; AVX512DQ-NEXT:    vmovdqa64 %xmm26, %xmm7
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm7[2],xmm4[3,4],xmm7[5],xmm4[6,7]
+; AVX512DQ-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
+; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm3, %ymm3
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
+; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512DQ-NEXT:    vpternlogq $202, %ymm20, %ymm18, %ymm0
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm0[2,3,0,1]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} ymm0 = ymm4[0],ymm0[1,2],ymm4[3],ymm0[4,5],ymm4[6],ymm0[7],ymm4[8],ymm0[9,10],ymm4[11],ymm0[12,13],ymm4[14],ymm0[15]
+; AVX512DQ-NEXT:    vpshufb %ymm11, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa64 %xmm27, %xmm4
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm7[0],xmm4[1],xmm7[2,3],xmm4[4],xmm7[5,6],xmm4[7]
+; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0],xmm8[1],xmm4[2,3],xmm8[4],xmm4[5,6],xmm8[7]
 ; AVX512DQ-NEXT:    vpshufb %xmm6, %xmm4, %xmm4
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm4[5,6,7]
-; AVX512DQ-NEXT:    vmovdqa64 %xmm25, %xmm4
-; AVX512DQ-NEXT:    vmovdqa64 %xmm26, %xmm5
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2],xmm4[3,4],xmm5[5],xmm4[6,7]
-; AVX512DQ-NEXT:    vpshufb %xmm8, %xmm4, %xmm4
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm4
-; AVX512DQ-NEXT:    vextracti32x4 $2, %zmm4, %xmm4
-; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
-; AVX512DQ-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vmovdqa64 %zmm18, (%rsi)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm17, (%rsi)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm16, 64(%rsi)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm19, 64(%rdx)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm1, (%rdx)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm2, (%rdx)
 ; AVX512DQ-NEXT:    vmovdqa64 %zmm0, 64(%rcx)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm2, (%rcx)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm1, (%rcx)
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; AVX512DQ-FCP-LABEL: load_i16_stride3_vf64:
 ; AVX512DQ-FCP:       # %bb.0:
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 224(%rdi), %ymm20
-; AVX512DQ-FCP-NEXT:    vmovdqa64 192(%rdi), %ymm21
+; AVX512DQ-FCP-NEXT:    vmovdqa64 224(%rdi), %ymm18
+; AVX512DQ-FCP-NEXT:    vmovdqa64 192(%rdi), %ymm20
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm20, %ymm21, %ymm1
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm1[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm3[1],ymm1[2,3],ymm3[4],ymm1[5,6],ymm3[7],ymm1[8],ymm3[9],ymm1[10,11],ymm3[12],ymm1[13,14],ymm3[15]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm1, %ymm5
-; AVX512DQ-FCP-NEXT:    vmovdqa 272(%rdi), %xmm8
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm18, %ymm20, %ymm1
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm7 = [0,1,6,7,12,13,2,3,4,5,14,15,8,9,10,11,16,17,22,23,28,29,18,19,20,21,30,31,24,25,26,27]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm7, %ymm2, %ymm5
+; AVX512DQ-FCP-NEXT:    vmovdqa 272(%rdi), %xmm1
 ; AVX512DQ-FCP-NEXT:    vmovdqa 256(%rdi), %xmm2
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm6 = xmm2[0,1],xmm8[2],xmm2[3,4],xmm8[5],xmm2[6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm2, %xmm14
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm9 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm9, %xmm6, %xmm6
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm6 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm2, %xmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm1, %xmm19
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm13 = [4,5,14,15,0,1,2,3,8,9,14,15,4,5,10,11]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm13, %xmm6, %xmm6
 ; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm5[0,1,2],ymm6[3,4,5,6,7],ymm5[8,9,10],ymm6[11,12,13,14,15]
 ; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,6,5,4,7]
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 320(%rdi), %ymm22
-; AVX512DQ-FCP-NEXT:    vmovdqa64 352(%rdi), %ymm23
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm6
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm22, %ymm23, %ymm6
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0],ymm7[1],ymm6[2,3],ymm7[4],ymm6[5,6],ymm7[7],ymm6[8],ymm7[9],ymm6[10,11],ymm7[12],ymm6[13,14],ymm7[15]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm11 = [0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm6, %ymm12
+; AVX512DQ-FCP-NEXT:    vmovdqa64 320(%rdi), %ymm21
+; AVX512DQ-FCP-NEXT:    vmovdqa64 352(%rdi), %ymm22
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm8
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm8
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm8[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm8 = ymm8[0],ymm9[1],ymm8[2,3],ymm9[4],ymm8[5,6],ymm9[7],ymm8[8],ymm9[9],ymm8[10,11],ymm9[12],ymm8[13,14],ymm9[15]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm10 = [0,1,6,7,12,13,2,3,8,9,14,15,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm8, %ymm11
 ; AVX512DQ-FCP-NEXT:    vmovdqa 304(%rdi), %xmm1
 ; AVX512DQ-FCP-NEXT:    vmovdqa 288(%rdi), %xmm2
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm13 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
 ; AVX512DQ-FCP-NEXT:    vmovdqa %xmm2, %xmm4
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, %xmm6
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm15 = [0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm15, %xmm13, %xmm13
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm12 = ymm13[0,1,2],ymm12[3,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm5, %zmm16
-; AVX512DQ-FCP-NEXT:    vmovdqa64 128(%rdi), %ymm24
-; AVX512DQ-FCP-NEXT:    vmovdqa 160(%rdi), %ymm13
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, %xmm8
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm14 = [0,1,6,7,12,13,2,3,8,9,14,15,12,13,14,15]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm14, %xmm12, %xmm12
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm11 = ymm12[0,1,2],ymm11[3,4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm5, %zmm16
+; AVX512DQ-FCP-NEXT:    vmovdqa64 128(%rdi), %ymm23
+; AVX512DQ-FCP-NEXT:    vmovdqa 160(%rdi), %ymm11
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm5
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm24, %ymm13, %ymm5
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm23, %ymm11, %ymm5
 ; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm12 = ymm5[2,3,0,1]
 ; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1],ymm5[2,3],ymm12[4],ymm5[5,6],ymm12[7],ymm5[8],ymm12[9],ymm5[10,11],ymm12[12],ymm5[13,14],ymm12[15]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm5, %ymm5
-; AVX512DQ-FCP-NEXT:    vmovdqa 112(%rdi), %xmm11
-; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdi), %xmm12
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm10 = xmm12[0],xmm11[1],xmm12[2,3],xmm11[4],xmm12[5,6],xmm11[7]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm15, %xmm10, %xmm10
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm10[0,1,2],ymm5[3,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 (%rdi), %ymm17
-; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %ymm5
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm10, %ymm5, %ymm10
+; AVX512DQ-FCP-NEXT:    vmovdqa 112(%rdi), %xmm15
+; AVX512DQ-FCP-NEXT:    vmovdqa 96(%rdi), %xmm5
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm12 = xmm5[0],xmm15[1],xmm5[2,3],xmm15[4],xmm5[5,6],xmm15[7]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm14, %xmm12, %xmm12
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm6 = ymm12[0,1,2],ymm10[3,4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 (%rdi), %ymm24
+; AVX512DQ-FCP-NEXT:    vmovdqa 32(%rdi), %ymm12
 ; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm10
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm5, %ymm17, %ymm10
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm15 = ymm10[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0],ymm15[1],ymm10[2,3],ymm15[4],ymm10[5,6],ymm15[7],ymm10[8],ymm15[9],ymm10[10,11],ymm15[12],ymm10[13,14],ymm15[15]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm10, %ymm2
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm12, %ymm24, %ymm10
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm10[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm10[0],ymm1[1],ymm10[2,3],ymm1[4],ymm10[5,6],ymm1[7],ymm10[8],ymm1[9],ymm10[10,11],ymm1[12],ymm10[13,14],ymm1[15]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm7, %ymm1, %ymm7
 ; AVX512DQ-FCP-NEXT:    vmovdqa 80(%rdi), %xmm10
-; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdi), %xmm15
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm15[0,1],xmm10[2],xmm15[3,4],xmm10[5],xmm15[6,7]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm9, %xmm3, %xmm3
-; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm2[0,1,2],ymm3[3,4,5,6,7],ymm2[8,9,10],ymm3[11,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,4,7]
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm18
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm23, %ymm22, %ymm1
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7,8,9],ymm2[10],ymm1[11,12],ymm2[13],ymm1[14,15]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm2 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm2, %ymm28
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1],xmm6[2],xmm4[3,4],xmm6[5],xmm4[6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm6, %xmm25
+; AVX512DQ-FCP-NEXT:    vmovdqa 64(%rdi), %xmm1
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm2 = xmm1[0,1],xmm10[2],xmm1[3,4],xmm10[5],xmm1[6,7]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm13, %xmm2, %xmm2
+; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm7[0,1,2],ymm2[3,4,5,6,7],ymm7[8,9,10],ymm2[11,12,13,14,15]
+; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,6,5,4,7]
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm2, %zmm17
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm2
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm22, %ymm21, %ymm2
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm2[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm6[2],ymm2[3,4],ymm6[5],ymm2[6,7,8,9],ymm6[10],ymm2[11,12],ymm6[13],ymm2[14,15]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm9 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm9, %ymm2, %ymm2
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm7 = xmm4[0,1],xmm8[2],xmm4[3,4],xmm8[5],xmm4[6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm8, %xmm25
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm4, %xmm26
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm6 = [2,3,8,9,14,15,4,5,10,11,10,11,10,11,10,11]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm1[5,6,7]
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm9 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm9, %ymm1
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm21, %ymm20, %ymm1
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm1[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm4[2],ymm1[3,4],ymm4[5],ymm1[6,7,8,9],ymm4[10],ymm1[11,12],ymm4[13],ymm1[14,15]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm7, %xmm7
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm7 = xmm7[0,1,2,3,4],xmm2[5,6,7]
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm13 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm13, %ymm2
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm20, %ymm18, %ymm2
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm2[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm4[2],ymm2[3,4],ymm4[5],ymm2[6,7,8,9],ymm4[10],ymm2[11,12],ymm4[13],ymm2[14,15]
 ; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm4 = [2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm1, %ymm1
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm14, %xmm7
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm14 = xmm8[0,1],xmm14[2],xmm8[3,4],xmm14[5],xmm8[6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm8, %xmm27
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm2, %xmm14, %xmm14
-; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm14 = ymm1[0,1,2],ymm14[3,4,5,6,7],ymm1[8,9,10],ymm14[11,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,6,7,4]
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm14[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm19
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm13, %ymm24, %ymm1
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm1[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm3[2],ymm1[3,4],ymm3[5],ymm1[6,7,8,9],ymm3[10],ymm1[11,12],ymm3[13],ymm1[14,15]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %ymm28, %ymm3
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm3, %ymm1, %ymm1
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm12[0,1],xmm11[2],xmm12[3,4],xmm11[5],xmm12[6,7]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3,4],xmm1[5,6,7]
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm9, %ymm3
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm17, %ymm5, %ymm3
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm3[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1],ymm6[2],ymm3[3,4],ymm6[5],ymm3[6,7,8,9],ymm6[10],ymm3[11,12],ymm6[13],ymm3[14,15]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm10[0,1],xmm15[2],xmm10[3,4],xmm15[5],xmm10[6,7]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm2, %xmm4, %xmm2
-; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3,4,5,6,7],ymm3[8,9,10],ymm2[11,12,13,14,15]
-; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,6,7,4]
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %ymm24, %ymm9, %ymm13
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm13[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm13[1,2],ymm2[3],ymm13[4,5],ymm2[6],ymm13[7],ymm2[8],ymm13[9,10],ymm2[11],ymm13[12,13],ymm2[14],ymm13[15]
-; AVX512DQ-FCP-NEXT:    vpternlogq $226, %ymm17, %ymm0, %ymm5
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm5[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm5[1,2],ymm3[3],ymm5[4,5],ymm3[6],ymm5[7],ymm3[8],ymm5[9,10],ymm3[11],ymm5[12,13],ymm3[14],ymm5[15]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm4 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm5 = xmm15[0],xmm10[1],xmm15[2,3],xmm10[4],xmm15[5,6],xmm10[7]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm6 = [0,1,2,3,0,1,6,7,12,13,2,3,8,9,14,15]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm5, %xmm5
-; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm5[5,6,7]
 ; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm2, %ymm2
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm5 = xmm11[0,1],xmm12[2],xmm11[3,4],xmm12[5],xmm11[6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm8 = [4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm5, %xmm5
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm5, %zmm3, %zmm5
-; AVX512DQ-FCP-NEXT:    vextracti32x4 $2, %zmm5, %xmm5
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3,4],xmm2[5,6,7]
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm19, %xmm8
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm14 = xmm8[0,1],xmm3[2],xmm8[3,4],xmm3[5],xmm8[6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm3, %xmm27
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm3 = [4,5,4,5,4,5,4,5,10,11,0,1,6,7,12,13]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm3, %xmm14, %xmm14
+; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm14 = ymm2[0,1,2],ymm14[3,4,5,6,7],ymm2[8,9,10],ymm14[11,12,13,14,15]
+; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,6,7,4]
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm14[4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm2, %zmm19
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, %ymm2
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm11, %ymm23, %ymm2
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm2[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm7[2],ymm2[3,4],ymm7[5],ymm2[6,7,8,9],ymm7[10],ymm2[11,12],ymm7[13],ymm2[14,15]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm9, %ymm2, %ymm2
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm7 = xmm5[0,1],xmm15[2],xmm5[3,4],xmm15[5],xmm5[6,7]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm7, %xmm6
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3,4],xmm2[5,6,7]
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm13, %ymm6
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm24, %ymm12, %ymm6
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0,1],ymm7[2],ymm6[3,4],ymm7[5],ymm6[6,7,8,9],ymm7[10],ymm6[11,12],ymm7[13],ymm6[14,15]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm6, %ymm4
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm6 = xmm10[0,1],xmm1[2],xmm10[3,4],xmm1[5],xmm10[6,7]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm3, %xmm6, %xmm3
+; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3,4,5,6,7],ymm4[8,9,10],ymm3[11,12,13,14,15]
+; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,6,7,4]
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm22, %ymm23, %ymm9
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm9[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm9[1,2],ymm3[3],ymm9[4,5],ymm3[6],ymm9[7],ymm3[8],ymm9[9,10],ymm3[11],ymm9[12,13],ymm3[14],ymm9[15]
-; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm21, %ymm20, %ymm0
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm5 = ymm0[2,3,0,1]
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm5[0],ymm0[1,2],ymm5[3],ymm0[4,5],ymm5[6],ymm0[7],ymm5[8],ymm0[9,10],ymm5[11],ymm0[12,13],ymm5[14],ymm0[15]
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
-; AVX512DQ-FCP-NEXT:    vpshufb %ymm4, %ymm0, %ymm0
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %ymm23, %ymm13, %ymm11
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm11[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm11[1,2],ymm3[3],ymm11[4,5],ymm3[6],ymm11[7],ymm3[8],ymm11[9,10],ymm3[11],ymm11[12,13],ymm3[14],ymm11[15]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} ymm11 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm3, %ymm3
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm15[0,1],xmm5[2],xmm15[3,4],xmm5[5],xmm15[6,7]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm5 = [4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vpternlogq $226, %ymm24, %ymm0, %ymm12
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm12[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0],ymm12[1,2],ymm4[3],ymm12[4,5],ymm4[6],ymm12[7],ymm4[8],ymm12[9,10],ymm4[11],ymm12[12,13],ymm4[14],ymm12[15]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm4, %ymm4
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm10[1],xmm1[2,3],xmm10[4],xmm1[5,6],xmm10[7]
+; AVX512DQ-FCP-NEXT:    vmovdqa {{.*#+}} xmm6 = [0,1,2,3,0,1,6,7,12,13,2,3,8,9,14,15]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm1, %xmm1
+; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3,4],ymm1[5,6,7]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm13
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm13[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm13[1,2],ymm3[3],ymm13[4,5],ymm3[6],ymm13[7],ymm3[8],ymm13[9,10],ymm3[11],ymm13[12,13],ymm3[14],ymm13[15]
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm25, %xmm4
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm26, %xmm7
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm7[2],xmm4[3,4],xmm7[5],xmm4[6,7]
+; AVX512DQ-FCP-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm3, %ymm3
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
+; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vpternlogq $202, %ymm20, %ymm18, %ymm0
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm0[2,3,0,1]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} ymm0 = ymm4[0],ymm0[1,2],ymm4[3],ymm0[4,5],ymm4[6],ymm0[7],ymm4[8],ymm0[9,10],ymm4[11],ymm0[12,13],ymm4[14],ymm0[15]
+; AVX512DQ-FCP-NEXT:    vpshufb %ymm11, %ymm0, %ymm0
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm27, %xmm4
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm7[0],xmm4[1],xmm7[2,3],xmm4[4],xmm7[5,6],xmm4[7]
+; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0],xmm8[1],xmm4[2,3],xmm8[4],xmm4[5,6],xmm8[7]
 ; AVX512DQ-FCP-NEXT:    vpshufb %xmm6, %xmm4, %xmm4
 ; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm4[5,6,7]
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm25, %xmm4
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %xmm26, %xmm5
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2],xmm4[3,4],xmm5[5],xmm4[6,7]
-; AVX512DQ-FCP-NEXT:    vpshufb %xmm8, %xmm4, %xmm4
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm4
-; AVX512DQ-FCP-NEXT:    vextracti32x4 $2, %zmm4, %xmm4
-; AVX512DQ-FCP-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3,4],xmm3[5,6,7]
-; AVX512DQ-FCP-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm18, (%rsi)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm17, (%rsi)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm16, 64(%rsi)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm19, 64(%rdx)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, (%rdx)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm2, (%rdx)
 ; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm0, 64(%rcx)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm2, (%rcx)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, (%rcx)
 ; AVX512DQ-FCP-NEXT:    vzeroupper
 ; AVX512DQ-FCP-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
index 8b6ba51506ab..8091afbbfd70 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
@@ -1246,29 +1246,28 @@ define void @store_i8_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecp
 ; AVX512BW-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
 ; AVX512BW-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm1 = [0,2,4,0]
 ; AVX512BW-FCP-NEXT:    vpermi2q %ymm3, %ymm0, %ymm1
-; AVX512BW-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [1,3,5,7,1,3,5,7]
-; AVX512BW-FCP-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX512BW-FCP-NEXT:    vpermd %ymm2, %ymm0, %ymm0
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm0[1,5,9,13],zero,zero,zero,ymm0[2,6,10,14],zero,zero,zero,ymm0[19,23,27,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm3 = [1,3,5,0,5,1,3,0]
-; AVX512BW-FCP-NEXT:    vpermd %ymm1, %ymm3, %ymm3
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[0,4,8],zero,zero,zero,zero,ymm3[1,5,9],zero,zero,zero,zero,ymm3[2,6,18],zero,zero,zero,zero,ymm3[23,27,19],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm2[0,8],zero,zero,zero,zero,zero,ymm2[1,9],zero,zero,zero,zero,zero,ymm2[2,10,18,26],zero,zero,zero,zero,zero,ymm2[19,27],zero,zero,zero,zero,zero,ymm2[20,28]
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm2[2,3,0,1]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,ymm3[0,8],zero,zero,zero,zero,zero,ymm3[1,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[19,27],zero,zero,zero,zero,zero,ymm3[20,28],zero,zero
 ; AVX512BW-FCP-NEXT:    vpor %ymm0, %ymm3, %ymm0
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm2[0,8],zero,zero,zero,zero,zero,ymm2[1,9],zero,zero,zero,zero,zero,ymm2[2,10,18,26],zero,zero,zero,zero,zero,ymm2[19,27],zero,zero,zero,zero,zero,ymm2[20,28]
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,3,0,1]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,ymm2[0,8],zero,zero,zero,zero,zero,ymm2[1,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[19,27],zero,zero,zero,zero,zero,ymm2[20,28],zero,zero
-; AVX512BW-FCP-NEXT:    vpor %ymm3, %ymm2, %ymm2
 ; AVX512BW-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [0,2,4,6,0,2,4,6]
 ; AVX512BW-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
-; AVX512BW-FCP-NEXT:    vpermd %ymm1, %ymm3, %ymm1
+; AVX512BW-FCP-NEXT:    vpermd %ymm1, %ymm3, %ymm3
 ; AVX512BW-FCP-NEXT:    movl $236730480, %ecx # imm = 0xE1C3870
 ; AVX512BW-FCP-NEXT:    kmovd %ecx, %k1
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm2 {%k1} = ymm1[u,u,u,u,0,4,8,u,u,u,u,1,5,9,u,u,u,u,18,22,26,u,u,u,u,19,23,27,u,u,u,u]
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm0
-; AVX512BW-FCP-NEXT:    vmovdqa %ymm2, (%rax)
-; AVX512BW-FCP-NEXT:    vextracti32x4 $2, %zmm0, 32(%rax)
-; AVX512BW-FCP-NEXT:    vextracti32x4 $3, %zmm0, %xmm0
-; AVX512BW-FCP-NEXT:    vmovq %xmm0, 48(%rax)
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm0 {%k1} = ymm3[u,u,u,u,0,4,8,u,u,u,u,1,5,9,u,u,u,u,18,22,26,u,u,u,u,19,23,27,u,u,u,u]
+; AVX512BW-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [1,3,5,7,1,3,5,7]
+; AVX512BW-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
+; AVX512BW-FCP-NEXT:    vpermd %ymm2, %ymm3, %ymm2
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,zero,ymm2[1,5,9,13],zero,zero,zero,ymm2[2,6,10,14],zero,zero,zero,ymm2[19,23,27,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm3 = [1,3,5,0,5,1,3,0]
+; AVX512BW-FCP-NEXT:    vpermd %ymm1, %ymm3, %ymm1
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[0,4,8],zero,zero,zero,zero,ymm1[1,5,9],zero,zero,zero,zero,ymm1[2,6,18],zero,zero,zero,zero,ymm1[23,27,19],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-FCP-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512BW-FCP-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX512BW-FCP-NEXT:    vmovq %xmm2, 48(%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa %xmm1, 32(%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa %ymm0, (%rax)
 ; AVX512BW-FCP-NEXT:    vzeroupper
 ; AVX512BW-FCP-NEXT:    retq
 ;
@@ -1326,29 +1325,28 @@ define void @store_i8_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecp
 ; AVX512DQ-BW-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
 ; AVX512DQ-BW-FCP-NEXT:    vpmovsxbq {{.*#+}} ymm1 = [0,2,4,0]
 ; AVX512DQ-BW-FCP-NEXT:    vpermi2q %ymm3, %ymm0, %ymm1
-; AVX512DQ-BW-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [1,3,5,7,1,3,5,7]
-; AVX512DQ-BW-FCP-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm2, %ymm0, %ymm0
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm0[1,5,9,13],zero,zero,zero,ymm0[2,6,10,14],zero,zero,zero,ymm0[19,23,27,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm3 = [1,3,5,0,5,1,3,0]
-; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm1, %ymm3, %ymm3
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[0,4,8],zero,zero,zero,zero,ymm3[1,5,9],zero,zero,zero,zero,ymm3[2,6,18],zero,zero,zero,zero,ymm3[23,27,19],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm0 = ymm2[0,8],zero,zero,zero,zero,zero,ymm2[1,9],zero,zero,zero,zero,zero,ymm2[2,10,18,26],zero,zero,zero,zero,zero,ymm2[19,27],zero,zero,zero,zero,zero,ymm2[20,28]
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm2[2,3,0,1]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,ymm3[0,8],zero,zero,zero,zero,zero,ymm3[1,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[19,27],zero,zero,zero,zero,zero,ymm3[20,28],zero,zero
 ; AVX512DQ-BW-FCP-NEXT:    vpor %ymm0, %ymm3, %ymm0
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm3 = ymm2[0,8],zero,zero,zero,zero,zero,ymm2[1,9],zero,zero,zero,zero,zero,ymm2[2,10,18,26],zero,zero,zero,zero,zero,ymm2[19,27],zero,zero,zero,zero,zero,ymm2[20,28]
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,3,0,1]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,ymm2[0,8],zero,zero,zero,zero,zero,ymm2[1,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[19,27],zero,zero,zero,zero,zero,ymm2[20,28],zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vpor %ymm3, %ymm2, %ymm2
 ; AVX512DQ-BW-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [0,2,4,6,0,2,4,6]
 ; AVX512DQ-BW-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
-; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm1, %ymm3, %ymm1
+; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm1, %ymm3, %ymm3
 ; AVX512DQ-BW-FCP-NEXT:    movl $236730480, %ecx # imm = 0xE1C3870
 ; AVX512DQ-BW-FCP-NEXT:    kmovd %ecx, %k1
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm2 {%k1} = ymm1[u,u,u,u,0,4,8,u,u,u,u,1,5,9,u,u,u,u,18,22,26,u,u,u,u,19,23,27,u,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm0
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa %ymm2, (%rax)
-; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $2, %zmm0, 32(%rax)
-; AVX512DQ-BW-FCP-NEXT:    vextracti32x4 $3, %zmm0, %xmm0
-; AVX512DQ-BW-FCP-NEXT:    vmovq %xmm0, 48(%rax)
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm0 {%k1} = ymm3[u,u,u,u,0,4,8,u,u,u,u,1,5,9,u,u,u,u,18,22,26,u,u,u,u,19,23,27,u,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [1,3,5,7,1,3,5,7]
+; AVX512DQ-BW-FCP-NEXT:    # ymm3 = mem[0,1,0,1]
+; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm2, %ymm3, %ymm2
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,zero,ymm2[1,5,9,13],zero,zero,zero,ymm2[2,6,10,14],zero,zero,zero,ymm2[19,23,27,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm3 = [1,3,5,0,5,1,3,0]
+; AVX512DQ-BW-FCP-NEXT:    vpermd %ymm1, %ymm3, %ymm1
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[0,4,8],zero,zero,zero,zero,ymm1[1,5,9],zero,zero,zero,zero,ymm1[2,6,18],zero,zero,zero,zero,ymm1[23,27,19],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512DQ-BW-FCP-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX512DQ-BW-FCP-NEXT:    vmovq %xmm2, 48(%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa %xmm1, 32(%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa %ymm0, (%rax)
 ; AVX512DQ-BW-FCP-NEXT:    vzeroupper
 ; AVX512DQ-BW-FCP-NEXT:    retq
   %in.vec0 = load <8 x i8>, ptr %in.vecptr0, align 64
@@ -2053,77 +2051,76 @@ define void @store_i8_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512-NEXT:    vmovdqa (%rdi), %xmm4
-; AVX512-NEXT:    vmovdqa (%rsi), %xmm5
-; AVX512-NEXT:    vmovdqa (%rdx), %xmm6
-; AVX512-NEXT:    vmovdqa (%rcx), %xmm7
-; AVX512-NEXT:    vmovdqa (%r8), %xmm0
-; AVX512-NEXT:    vmovdqa (%r10), %xmm1
-; AVX512-NEXT:    vinserti128 $1, %xmm7, %ymm6, %ymm3
-; AVX512-NEXT:    vinserti128 $1, %xmm5, %ymm4, %ymm2
-; AVX512-NEXT:    vinserti128 $1, (%r9), %ymm0, %ymm0
-; AVX512-NEXT:    vinserti32x4 $2, %xmm1, %zmm0, %zmm0
-; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm6[8],xmm7[8],xmm6[9],xmm7[9],xmm6[10],xmm7[10],xmm6[11],xmm7[11],xmm6[12],xmm7[12],xmm6[13],xmm7[13],xmm6[14],xmm7[14],xmm6[15],xmm7[15]
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[u,u],zero,zero,xmm6[12,13,u,u,u],zero,zero,xmm6[14,15,u,u,u]
-; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm4[8],xmm5[8],xmm4[9],xmm5[9],xmm4[10],xmm5[10],xmm4[11],xmm5[11],xmm4[12],xmm5[12],xmm4[13],xmm5[13],xmm4[14],xmm5[14],xmm4[15],xmm5[15]
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,12,13],zero,zero,xmm4[u,u,u,14,15],zero,zero,xmm4[u,u,u]
-; AVX512-NEXT:    vpor %xmm6, %xmm4, %xmm4
-; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm5
-; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[10],zero,xmm5[u,u,u,u,13,12],zero,xmm5[u,u,u,u,15,14],zero
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm6 = zero,xmm1[13,u,u,u,u],zero,zero,xmm1[14,u,u,u,u],zero,zero,xmm1[15]
-; AVX512-NEXT:    vpor %xmm6, %xmm5, %xmm5
-; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4, %xmm5
-; AVX512-NEXT:    vpermq {{.*#+}} ymm4 = ymm2[3,1,1,3]
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm4 = ymm4[1],zero,zero,ymm4[u,u,u,10,2],zero,zero,ymm4[u,u,u,11,3],zero,zero,ymm4[u,u,u,20,28],zero,zero,ymm4[u,u,u,21,29],zero,zero,ymm4[u]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm6 = ymm3[1,3,3,1]
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm6 = zero,ymm6[1,9,u,u,u],zero,zero,ymm6[2,10,u,u,u],zero,zero,ymm6[3,19,u,u,u],zero,zero,ymm6[28,20,u,u,u],zero,zero,ymm6[29,21,u]
-; AVX512-NEXT:    vpor %ymm4, %ymm6, %ymm4
-; AVX512-NEXT:    vpshufhw {{.*#+}} xmm6 = xmm1[0,1,2,3,4,5,5,6]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,2,3,3]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,1,0,1]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm0[1,3,1,3]
+; AVX512-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-NEXT:    vmovdqa (%rsi), %xmm1
+; AVX512-NEXT:    vmovdqa (%rdx), %xmm5
+; AVX512-NEXT:    vmovdqa (%rcx), %xmm6
+; AVX512-NEXT:    vmovdqa (%r8), %xmm3
+; AVX512-NEXT:    vmovdqa (%r9), %xmm4
+; AVX512-NEXT:    vmovdqa (%r10), %xmm2
+; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm8
+; AVX512-NEXT:    vinserti128 $1, %xmm6, %ymm5, %ymm9
+; AVX512-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm7
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm10 = ymm9[u,u,u,u,u,5],zero,ymm9[u,u,u,u,u,6],zero,ymm9[u,u,u,u,u],zero,ymm9[23,u,u,u,u,u],zero,ymm9[24,u,u,u,u]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm11 = ymm9[2,3,0,1]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[u,u,u,u,u],zero,ymm11[5,u,u,u,u,u],zero,ymm11[6,u,u,u,u,u,23],zero,ymm11[u,u,u,u,u,24],zero,ymm11[u,u,u,u]
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm12 = [255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255]
+; AVX512-NEXT:    vpternlogq $50, %ymm10, %ymm12, %ymm11
+; AVX512-NEXT:    vpermq {{.*#+}} ymm10 = ymm9[0,2,0,2]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm10 = zero,zero,ymm10[0,8,u,u,u],zero,zero,ymm10[1,9,u,u,u],zero,zero,ymm10[18,26,u,u,u],zero,zero,ymm10[19,27,u,u,u],zero,zero,ymm10[20,28]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm11, %zmm10, %zmm10
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm11 = ymm8[u,u,u,5],zero,ymm8[u,u,u,u,u,6],zero,ymm8[u,u,u,u,u],zero,ymm8[23,u,u,u,u,u],zero,ymm8[24,u,u,u,u,u],zero
+; AVX512-NEXT:    vpermq {{.*#+}} ymm13 = ymm8[2,3,0,1]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm13 = ymm13[u,u,u],zero,ymm13[5,u,u,u,u,u],zero,ymm13[6,u,u,u,u,u,23],zero,ymm13[u,u,u,u,u,24],zero,ymm13[u,u,u,u,u,25]
+; AVX512-NEXT:    vpternlogq $200, %ymm11, %ymm12, %ymm13
+; AVX512-NEXT:    vpermq {{.*#+}} ymm11 = ymm8[0,2,0,2]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[0,8],zero,zero,ymm11[u,u,u,1,9],zero,zero,ymm11[u,u,u,2,10],zero,zero,ymm11[u,u,u,19,27],zero,zero,ymm11[u,u,u,20,28],zero,zero
+; AVX512-NEXT:    vinserti64x4 $1, %ymm13, %zmm11, %zmm11
+; AVX512-NEXT:    vporq %zmm10, %zmm11, %zmm10
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm11 = ymm7[4],zero,ymm7[u,u,u,u,u,5],zero,ymm7[u,u,u,u,u,6],zero,ymm7[u,u,u,u,u],zero,ymm7[23,u,u,u,u,u],zero,ymm7[24,u,u]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm12 = ymm7[2,3,0,1]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm12 = zero,ymm12[4,u,u,u,u,u],zero,ymm12[5,u,u,u,u,u],zero,ymm12[6,u,u,u,u,u,23],zero,ymm12[u,u,u,u,u,24],zero,ymm12[u,u]
+; AVX512-NEXT:    vmovdqa {{.*#+}} ymm13 = [255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255]
+; AVX512-NEXT:    vpternlogq $200, %ymm11, %ymm13, %ymm12
+; AVX512-NEXT:    vpermq {{.*#+}} ymm11 = ymm7[0,2,0,2]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[u,u,u,u,0,8],zero,ymm11[u,u,u,u,1,9],zero,ymm11[u,u,u,u,18,26],zero,ymm11[u,u,u,u,19,27],zero,ymm11[u,u,u,u]
+; AVX512-NEXT:    vinserti64x4 $1, %ymm12, %zmm11, %zmm11
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm12 = xmm2[4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,0,1,0]
+; AVX512-NEXT:    vpandn %ymm12, %ymm13, %ymm12
+; AVX512-NEXT:    vpshuflw {{.*#+}} xmm13 = xmm2[1,1,0,0,4,5,6,7]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm13 = xmm13[0,1,2,0]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,0,1,0]
+; AVX512-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm13, %ymm13
+; AVX512-NEXT:    vinserti64x4 $1, %ymm12, %zmm13, %zmm12
+; AVX512-NEXT:    vporq %zmm12, %zmm11, %zmm11
+; AVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm11
+; AVX512-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[3,1,1,3]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm8 = ymm8[1],zero,zero,ymm8[u,u,u,10,2],zero,zero,ymm8[u,u,u,11,3],zero,zero,ymm8[u,u,u,20,28],zero,zero,ymm8[u,u,u,21,29],zero,zero,ymm8[u]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[1,3,3,1]
+; AVX512-NEXT:    vpshufb {{.*#+}} ymm9 = zero,ymm9[1,9,u,u,u],zero,zero,ymm9[2,10,u,u,u],zero,zero,ymm9[3,19,u,u,u],zero,zero,ymm9[28,20,u,u,u],zero,zero,ymm9[29,21,u]
+; AVX512-NEXT:    vpor %ymm8, %ymm9, %ymm8
+; AVX512-NEXT:    vpshufhw {{.*#+}} xmm9 = xmm2[0,1,2,3,4,5,5,6]
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm9 = xmm9[2,2,3,3]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,1,0,1]
+; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[1,3,1,3]
 ; AVX512-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,1,9],zero,ymm7[u,u,u,u,2,10],zero,ymm7[u,u,u,u,19,27],zero,ymm7[u,u,u,u,20,28],zero,ymm7[u,u,u,u,21]
-; AVX512-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm6, %ymm7
-; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm7
-; AVX512-NEXT:    vinserti32x4 $2, %xmm5, %zmm7, %zmm4
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm6 = ymm3[u,u,u,u,u,5],zero,ymm3[u,u,u,u,u,6],zero,ymm3[u,u,u,u,u],zero,ymm3[23,u,u,u,u,u],zero,ymm3[24,u,u,u,u]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm3[2,3,0,1]
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,u,u],zero,ymm7[5,u,u,u,u,u],zero,ymm7[6,u,u,u,u,u,23],zero,ymm7[u,u,u,u,u,24],zero,ymm7[u,u,u,u]
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm8 = [255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255]
-; AVX512-NEXT:    vpternlogq $50, %ymm6, %ymm8, %ymm7
-; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,2,0,2]
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,ymm3[0,8,u,u,u],zero,zero,ymm3[1,9,u,u,u],zero,zero,ymm3[18,26,u,u,u],zero,zero,ymm3[19,27,u,u,u],zero,zero,ymm3[20,28]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm3, %zmm3
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm6 = ymm2[u,u,u,5],zero,ymm2[u,u,u,u,u,6],zero,ymm2[u,u,u,u,u],zero,ymm2[23,u,u,u,u,u],zero,ymm2[24,u,u,u,u,u],zero
-; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm2[2,3,0,1]
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u],zero,ymm7[5,u,u,u,u,u],zero,ymm7[6,u,u,u,u,u,23],zero,ymm7[u,u,u,u,u,24],zero,ymm7[u,u,u,u,u,25]
-; AVX512-NEXT:    vpternlogq $200, %ymm6, %ymm8, %ymm7
-; AVX512-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,0,2]
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[0,8],zero,zero,ymm2[u,u,u,1,9],zero,zero,ymm2[u,u,u,2,10],zero,zero,ymm2[u,u,u,19,27],zero,zero,ymm2[u,u,u,20,28],zero,zero
-; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm2, %zmm2
-; AVX512-NEXT:    vporq %zmm3, %zmm2, %zmm2
-; AVX512-NEXT:    vpshufb {{.*#+}} xmm3 = xmm1[4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,1,0]
-; AVX512-NEXT:    vmovdqa {{.*#+}} ymm6 = [255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255]
-; AVX512-NEXT:    vpandn %ymm3, %ymm6, %ymm3
-; AVX512-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[1,1,0,0,4,5,6,7]
-; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,1,0]
-; AVX512-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX512-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm3 = ymm0[4],zero,ymm0[u,u,u,u,u,5],zero,ymm0[u,u,u,u,u,6],zero,ymm0[u,u,u,u,u],zero,ymm0[23,u,u,u,u,u],zero,ymm0[24,u,u]
-; AVX512-NEXT:    vpermq {{.*#+}} ymm7 = ymm0[2,3,0,1]
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm7 = zero,ymm7[4,u,u,u,u,u],zero,ymm7[5,u,u,u,u,u],zero,ymm7[6,u,u,u,u,u,23],zero,ymm7[u,u,u,u,u,24],zero,ymm7[u,u]
-; AVX512-NEXT:    vpternlogq $200, %ymm3, %ymm6, %ymm7
-; AVX512-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,0,2]
-; AVX512-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,0,8],zero,ymm0[u,u,u,u,1,9],zero,ymm0[u,u,u,u,18,26],zero,ymm0[u,u,u,u,19,27],zero,ymm0[u,u,u,u]
-; AVX512-NEXT:    vinserti64x4 $1, %ymm7, %zmm0, %zmm0
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0
-; AVX512-NEXT:    vmovdqa %xmm5, 96(%rax)
-; AVX512-NEXT:    vmovdqa64 %zmm0, (%rax)
-; AVX512-NEXT:    vmovdqa %ymm4, 64(%rax)
+; AVX512-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm9, %ymm7
+; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm8, %ymm7
+; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm5[8],xmm6[8],xmm5[9],xmm6[9],xmm5[10],xmm6[10],xmm5[11],xmm6[11],xmm5[12],xmm6[12],xmm5[13],xmm6[13],xmm5[14],xmm6[14],xmm5[15],xmm6[15]
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u],zero,zero,xmm5[12,13,u,u,u],zero,zero,xmm5[14,15,u,u,u]
+; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,12,13],zero,zero,xmm0[u,u,u,14,15],zero,zero,xmm0[u,u,u]
+; AVX512-NEXT:    vpor %xmm5, %xmm0, %xmm0
+; AVX512-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm4[8],xmm3[8],xmm4[9],xmm3[9],xmm4[10],xmm3[10],xmm4[11],xmm3[11],xmm4[12],xmm3[12],xmm4[13],xmm3[13],xmm4[14],xmm3[14],xmm4[15],xmm3[15]
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[10],zero,xmm1[u,u,u,u,13,12],zero,xmm1[u,u,u,u,15,14],zero
+; AVX512-NEXT:    vpshufb {{.*#+}} xmm2 = zero,xmm2[13,u,u,u,u],zero,zero,xmm2[14,u,u,u,u],zero,zero,xmm2[15]
+; AVX512-NEXT:    vpor %xmm2, %xmm1, %xmm1
+; AVX512-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
+; AVX512-NEXT:    vinserti32x4 $2, %xmm1, %zmm7, %zmm0
+; AVX512-NEXT:    vmovdqa %xmm1, 96(%rax)
+; AVX512-NEXT:    vmovdqa %ymm0, 64(%rax)
+; AVX512-NEXT:    vmovdqa64 %zmm11, (%rax)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 ;
@@ -2131,70 +2128,69 @@ define void @store_i8_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512-FCP:       # %bb.0:
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm2
-; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm3
-; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm4
-; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm5
-; AVX512-FCP-NEXT:    vmovdqa (%r8), %xmm1
-; AVX512-FCP-NEXT:    vmovdqa (%r10), %xmm0
-; AVX512-FCP-NEXT:    vinserti128 $1, %xmm5, %ymm4, %ymm6
-; AVX512-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm7
-; AVX512-FCP-NEXT:    vinserti128 $1, (%r9), %ymm1, %ymm1
-; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm1
-; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm4[8],xmm5[8],xmm4[9],xmm5[9],xmm4[10],xmm5[10],xmm4[11],xmm5[11],xmm4[12],xmm5[12],xmm4[13],xmm5[13],xmm4[14],xmm5[14],xmm4[15],xmm5[15]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u],zero,zero,xmm4[12,13,u,u,u],zero,zero,xmm4[14,15,u,u,u]
-; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,12,13],zero,zero,xmm2[u,u,u,14,15],zero,zero,xmm2[u,u,u]
-; AVX512-FCP-NEXT:    vpor %xmm4, %xmm2, %xmm2
-; AVX512-FCP-NEXT:    vextracti128 $1, %ymm1, %xmm3
-; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[10],zero,xmm3[u,u,u,u,13,12],zero,xmm3[u,u,u,u,15,14],zero
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = zero,xmm0[13,u,u,u,u],zero,zero,xmm0[14,u,u,u,u],zero,zero,xmm0[15]
-; AVX512-FCP-NEXT:    vpor %xmm4, %xmm3, %xmm3
-; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm3
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm7[3,1,1,3]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[1],zero,zero,ymm2[u,u,u,10,2],zero,zero,ymm2[u,u,u,11,3],zero,zero,ymm2[u,u,u,20,28],zero,zero,ymm2[u,u,u,21,29],zero,zero,ymm2[u]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm6[1,3,3,1]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = zero,ymm4[1,9,u,u,u],zero,zero,ymm4[2,10,u,u,u],zero,zero,ymm4[3,19,u,u,u],zero,zero,ymm4[28,20,u,u,u],zero,zero,ymm4[29,21,u]
-; AVX512-FCP-NEXT:    vpor %ymm2, %ymm4, %ymm2
-; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm0[0,1,2,3,4,5,5,6]
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [2,2,3,3,2,2,3,3]
-; AVX512-FCP-NEXT:    # ymm5 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpermd %ymm4, %ymm5, %ymm4
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm5 = ymm1[1,3,1,3]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm5 = ymm5[u,u,u,1,9],zero,ymm5[u,u,u,u,2,10],zero,ymm5[u,u,u,u,19,27],zero,ymm5[u,u,u,u,20,28],zero,ymm5[u,u,u,u,21]
-; AVX512-FCP-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm5
-; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm5
-; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm3, %zmm5, %zmm2
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm6[0,2,0,2]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = zero,zero,ymm4[0,8,u,u,u],zero,zero,ymm4[1,9,u,u,u],zero,zero,ymm4[18,26,u,u,u],zero,zero,ymm4[19,27,u,u,u],zero,zero,ymm4[20,28]
-; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [1,5,2,6,1,5,2,6]
-; AVX512-FCP-NEXT:    # ymm5 = mem[0,1,0,1]
-; AVX512-FCP-NEXT:    vpermd %ymm6, %ymm5, %ymm6
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[u,u,u],zero,zero,ymm6[1,5,u,u,u],zero,zero,ymm6[2,6,u,u,u],zero,zero,ymm6[19,23,u,u,u],zero,zero,ymm6[24,28,u,u,u],zero
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm4, %zmm4
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm7[0,2,0,2]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[0,8],zero,zero,ymm6[u,u,u,1,9],zero,zero,ymm6[u,u,u,2,10],zero,zero,ymm6[u,u,u,19,27],zero,zero,ymm6[u,u,u,20,28],zero,zero
-; AVX512-FCP-NEXT:    vpermd %ymm7, %ymm5, %ymm7
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,1,5],zero,zero,ymm7[u,u,u,2,6],zero,zero,ymm7[u,u,u,19,23],zero,zero,ymm7[u,u,u,24,28],zero,zero,ymm7[u,u,u,25]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm6, %zmm6
-; AVX512-FCP-NEXT:    vporq %zmm4, %zmm6, %zmm4
-; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm0[1,1,0,0,4,5,6,7]
-; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm7 = [0,1,0,1,0,0,0,0]
-; AVX512-FCP-NEXT:    vpermd %ymm6, %ymm7, %ymm6
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm6, %zmm0
-; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm1[0,2,0,2]
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[u,u,u,u,0,8],zero,ymm6[u,u,u,u,1,9],zero,ymm6[u,u,u,u,18,26],zero,ymm6[u,u,u,u,19,27],zero,ymm6[u,u,u,u]
-; AVX512-FCP-NEXT:    vpermd %ymm1, %ymm5, %ymm1
-; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[0,4],zero,ymm1[u,u,u,u,1,5],zero,ymm1[u,u,u,u,2,6],zero,ymm1[u,u,u,u,19,23],zero,ymm1[u,u,u,u,24,28],zero,ymm1[u]
-; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm6, %zmm1
-; AVX512-FCP-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1
-; AVX512-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm1
-; AVX512-FCP-NEXT:    vmovdqa %xmm3, 96(%rax)
-; AVX512-FCP-NEXT:    vmovdqa64 %zmm1, (%rax)
-; AVX512-FCP-NEXT:    vmovdqa %ymm2, 64(%rax)
+; AVX512-FCP-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512-FCP-NEXT:    vmovdqa (%rsi), %xmm1
+; AVX512-FCP-NEXT:    vmovdqa (%rdx), %xmm5
+; AVX512-FCP-NEXT:    vmovdqa (%rcx), %xmm6
+; AVX512-FCP-NEXT:    vmovdqa (%r8), %xmm3
+; AVX512-FCP-NEXT:    vmovdqa (%r9), %xmm4
+; AVX512-FCP-NEXT:    vmovdqa (%r10), %xmm2
+; AVX512-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm7
+; AVX512-FCP-NEXT:    vinserti128 $1, %xmm6, %ymm5, %ymm8
+; AVX512-FCP-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm9
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm8[0,2,0,2]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm10 = zero,zero,ymm10[0,8,u,u,u],zero,zero,ymm10[1,9,u,u,u],zero,zero,ymm10[18,26,u,u,u],zero,zero,ymm10[19,27,u,u,u],zero,zero,ymm10[20,28]
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm11 = [1,5,2,6,1,5,2,6]
+; AVX512-FCP-NEXT:    # ymm11 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vpermd %ymm8, %ymm11, %ymm12
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm12 = ymm12[u,u,u],zero,zero,ymm12[1,5,u,u,u],zero,zero,ymm12[2,6,u,u,u],zero,zero,ymm12[19,23,u,u,u],zero,zero,ymm12[24,28,u,u,u],zero
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm10, %zmm10
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm12 = ymm7[0,2,0,2]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm12 = ymm12[0,8],zero,zero,ymm12[u,u,u,1,9],zero,zero,ymm12[u,u,u,2,10],zero,zero,ymm12[u,u,u,19,27],zero,zero,ymm12[u,u,u,20,28],zero,zero
+; AVX512-FCP-NEXT:    vpermd %ymm7, %ymm11, %ymm13
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm13 = ymm13[u,u,u,1,5],zero,zero,ymm13[u,u,u,2,6],zero,zero,ymm13[u,u,u,19,23],zero,zero,ymm13[u,u,u,24,28],zero,zero,ymm13[u,u,u,25]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm12, %zmm12
+; AVX512-FCP-NEXT:    vporq %zmm10, %zmm12, %zmm10
+; AVX512-FCP-NEXT:    vpshuflw {{.*#+}} xmm12 = xmm2[1,1,0,0,4,5,6,7]
+; AVX512-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm13 = [0,1,0,1,0,0,0,0]
+; AVX512-FCP-NEXT:    vpermd %ymm12, %ymm13, %ymm12
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm13 = xmm2[4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,0,1,0]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm12, %zmm12
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm13 = ymm9[0,2,0,2]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm13 = ymm13[u,u,u,u,0,8],zero,ymm13[u,u,u,u,1,9],zero,ymm13[u,u,u,u,18,26],zero,ymm13[u,u,u,u,19,27],zero,ymm13[u,u,u,u]
+; AVX512-FCP-NEXT:    vpermd %ymm9, %ymm11, %ymm11
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[0,4],zero,ymm11[u,u,u,u,1,5],zero,ymm11[u,u,u,u,2,6],zero,ymm11[u,u,u,u,19,23],zero,ymm11[u,u,u,u,24,28],zero,ymm11[u]
+; AVX512-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm13, %zmm11
+; AVX512-FCP-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm12, %zmm11
+; AVX512-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm11
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[3,1,1,3]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[1],zero,zero,ymm7[u,u,u,10,2],zero,zero,ymm7[u,u,u,11,3],zero,zero,ymm7[u,u,u,20,28],zero,zero,ymm7[u,u,u,21,29],zero,zero,ymm7[u]
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[1,3,3,1]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm8 = zero,ymm8[1,9,u,u,u],zero,zero,ymm8[2,10,u,u,u],zero,zero,ymm8[3,19,u,u,u],zero,zero,ymm8[28,20,u,u,u],zero,zero,ymm8[29,21,u]
+; AVX512-FCP-NEXT:    vpor %ymm7, %ymm8, %ymm7
+; AVX512-FCP-NEXT:    vpshufhw {{.*#+}} xmm8 = xmm2[0,1,2,3,4,5,5,6]
+; AVX512-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [2,2,3,3,2,2,3,3]
+; AVX512-FCP-NEXT:    # ymm10 = mem[0,1,0,1]
+; AVX512-FCP-NEXT:    vpermd %ymm8, %ymm10, %ymm8
+; AVX512-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[1,3,1,3]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} ymm9 = ymm9[u,u,u,1,9],zero,ymm9[u,u,u,u,2,10],zero,ymm9[u,u,u,u,19,27],zero,ymm9[u,u,u,u,20,28],zero,ymm9[u,u,u,u,21]
+; AVX512-FCP-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm8, %ymm9
+; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm7, %ymm9
+; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm5[8],xmm6[8],xmm5[9],xmm6[9],xmm5[10],xmm6[10],xmm5[11],xmm6[11],xmm5[12],xmm6[12],xmm5[13],xmm6[13],xmm5[14],xmm6[14],xmm5[15],xmm6[15]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u],zero,zero,xmm5[12,13,u,u,u],zero,zero,xmm5[14,15,u,u,u]
+; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,12,13],zero,zero,xmm0[u,u,u,14,15],zero,zero,xmm0[u,u,u]
+; AVX512-FCP-NEXT:    vpor %xmm5, %xmm0, %xmm0
+; AVX512-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm4[8],xmm3[8],xmm4[9],xmm3[9],xmm4[10],xmm3[10],xmm4[11],xmm3[11],xmm4[12],xmm3[12],xmm4[13],xmm3[13],xmm4[14],xmm3[14],xmm4[15],xmm3[15]
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[10],zero,xmm1[u,u,u,u,13,12],zero,xmm1[u,u,u,u,15,14],zero
+; AVX512-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = zero,xmm2[13,u,u,u,u],zero,zero,xmm2[14,u,u,u,u],zero,zero,xmm2[15]
+; AVX512-FCP-NEXT:    vpor %xmm2, %xmm1, %xmm1
+; AVX512-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
+; AVX512-FCP-NEXT:    vinserti32x4 $2, %xmm1, %zmm9, %zmm0
+; AVX512-FCP-NEXT:    vmovdqa %xmm1, 96(%rax)
+; AVX512-FCP-NEXT:    vmovdqa64 %zmm11, (%rax)
+; AVX512-FCP-NEXT:    vmovdqa %ymm0, 64(%rax)
 ; AVX512-FCP-NEXT:    vzeroupper
 ; AVX512-FCP-NEXT:    retq
 ;
@@ -2202,77 +2198,76 @@ define void @store_i8_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm4
-; AVX512DQ-NEXT:    vmovdqa (%rsi), %xmm5
-; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm6
-; AVX512DQ-NEXT:    vmovdqa (%rcx), %xmm7
-; AVX512DQ-NEXT:    vmovdqa (%r8), %xmm0
-; AVX512DQ-NEXT:    vmovdqa (%r10), %xmm1
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm7, %ymm6, %ymm3
-; AVX512DQ-NEXT:    vinserti128 $1, %xmm5, %ymm4, %ymm2
-; AVX512DQ-NEXT:    vinserti128 $1, (%r9), %ymm0, %ymm0
-; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm1, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm6[8],xmm7[8],xmm6[9],xmm7[9],xmm6[10],xmm7[10],xmm6[11],xmm7[11],xmm6[12],xmm7[12],xmm6[13],xmm7[13],xmm6[14],xmm7[14],xmm6[15],xmm7[15]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[u,u],zero,zero,xmm6[12,13,u,u,u],zero,zero,xmm6[14,15,u,u,u]
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm4[8],xmm5[8],xmm4[9],xmm5[9],xmm4[10],xmm5[10],xmm4[11],xmm5[11],xmm4[12],xmm5[12],xmm4[13],xmm5[13],xmm4[14],xmm5[14],xmm4[15],xmm5[15]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,12,13],zero,zero,xmm4[u,u,u,14,15],zero,zero,xmm4[u,u,u]
-; AVX512DQ-NEXT:    vpor %xmm6, %xmm4, %xmm4
-; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm5
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[10],zero,xmm5[u,u,u,u,13,12],zero,xmm5[u,u,u,u,15,14],zero
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm6 = zero,xmm1[13,u,u,u,u],zero,zero,xmm1[14,u,u,u,u],zero,zero,xmm1[15]
-; AVX512DQ-NEXT:    vpor %xmm6, %xmm5, %xmm5
-; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4, %xmm5
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm4 = ymm2[3,1,1,3]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm4 = ymm4[1],zero,zero,ymm4[u,u,u,10,2],zero,zero,ymm4[u,u,u,11,3],zero,zero,ymm4[u,u,u,20,28],zero,zero,ymm4[u,u,u,21,29],zero,zero,ymm4[u]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm6 = ymm3[1,3,3,1]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm6 = zero,ymm6[1,9,u,u,u],zero,zero,ymm6[2,10,u,u,u],zero,zero,ymm6[3,19,u,u,u],zero,zero,ymm6[28,20,u,u,u],zero,zero,ymm6[29,21,u]
-; AVX512DQ-NEXT:    vpor %ymm4, %ymm6, %ymm4
-; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm6 = xmm1[0,1,2,3,4,5,5,6]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,2,3,3]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,1,0,1]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm0[1,3,1,3]
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512DQ-NEXT:    vmovdqa (%rsi), %xmm1
+; AVX512DQ-NEXT:    vmovdqa (%rdx), %xmm5
+; AVX512DQ-NEXT:    vmovdqa (%rcx), %xmm6
+; AVX512DQ-NEXT:    vmovdqa (%r8), %xmm3
+; AVX512DQ-NEXT:    vmovdqa (%r9), %xmm4
+; AVX512DQ-NEXT:    vmovdqa (%r10), %xmm2
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm8
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm6, %ymm5, %ymm9
+; AVX512DQ-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm7
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm10 = ymm9[u,u,u,u,u,5],zero,ymm9[u,u,u,u,u,6],zero,ymm9[u,u,u,u,u],zero,ymm9[23,u,u,u,u,u],zero,ymm9[24,u,u,u,u]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm11 = ymm9[2,3,0,1]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[u,u,u,u,u],zero,ymm11[5,u,u,u,u,u],zero,ymm11[6,u,u,u,u,u,23],zero,ymm11[u,u,u,u,u,24],zero,ymm11[u,u,u,u]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm12 = [255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255]
+; AVX512DQ-NEXT:    vpternlogq $50, %ymm10, %ymm12, %ymm11
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm10 = ymm9[0,2,0,2]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm10 = zero,zero,ymm10[0,8,u,u,u],zero,zero,ymm10[1,9,u,u,u],zero,zero,ymm10[18,26,u,u,u],zero,zero,ymm10[19,27,u,u,u],zero,zero,ymm10[20,28]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm11, %zmm10, %zmm10
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm11 = ymm8[u,u,u,5],zero,ymm8[u,u,u,u,u,6],zero,ymm8[u,u,u,u,u],zero,ymm8[23,u,u,u,u,u],zero,ymm8[24,u,u,u,u,u],zero
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm13 = ymm8[2,3,0,1]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm13 = ymm13[u,u,u],zero,ymm13[5,u,u,u,u,u],zero,ymm13[6,u,u,u,u,u,23],zero,ymm13[u,u,u,u,u,24],zero,ymm13[u,u,u,u,u,25]
+; AVX512DQ-NEXT:    vpternlogq $200, %ymm11, %ymm12, %ymm13
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm11 = ymm8[0,2,0,2]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[0,8],zero,zero,ymm11[u,u,u,1,9],zero,zero,ymm11[u,u,u,2,10],zero,zero,ymm11[u,u,u,19,27],zero,zero,ymm11[u,u,u,20,28],zero,zero
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm13, %zmm11, %zmm11
+; AVX512DQ-NEXT:    vporq %zmm10, %zmm11, %zmm10
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm11 = ymm7[4],zero,ymm7[u,u,u,u,u,5],zero,ymm7[u,u,u,u,u,6],zero,ymm7[u,u,u,u,u],zero,ymm7[23,u,u,u,u,u],zero,ymm7[24,u,u]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm12 = ymm7[2,3,0,1]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm12 = zero,ymm12[4,u,u,u,u,u],zero,ymm12[5,u,u,u,u,u],zero,ymm12[6,u,u,u,u,u,23],zero,ymm12[u,u,u,u,u,24],zero,ymm12[u,u]
+; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm13 = [255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255]
+; AVX512DQ-NEXT:    vpternlogq $200, %ymm11, %ymm13, %ymm12
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm11 = ymm7[0,2,0,2]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[u,u,u,u,0,8],zero,ymm11[u,u,u,u,1,9],zero,ymm11[u,u,u,u,18,26],zero,ymm11[u,u,u,u,19,27],zero,ymm11[u,u,u,u]
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm12, %zmm11, %zmm11
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm12 = xmm2[4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,0,1,0]
+; AVX512DQ-NEXT:    vpandn %ymm12, %ymm13, %ymm12
+; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm13 = xmm2[1,1,0,0,4,5,6,7]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm13 = xmm13[0,1,2,0]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,0,1,0]
+; AVX512DQ-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm13, %ymm13
+; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm12, %zmm13, %zmm12
+; AVX512DQ-NEXT:    vporq %zmm12, %zmm11, %zmm11
+; AVX512DQ-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm11
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[3,1,1,3]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm8 = ymm8[1],zero,zero,ymm8[u,u,u,10,2],zero,zero,ymm8[u,u,u,11,3],zero,zero,ymm8[u,u,u,20,28],zero,zero,ymm8[u,u,u,21,29],zero,zero,ymm8[u]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[1,3,3,1]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm9 = zero,ymm9[1,9,u,u,u],zero,zero,ymm9[2,10,u,u,u],zero,zero,ymm9[3,19,u,u,u],zero,zero,ymm9[28,20,u,u,u],zero,zero,ymm9[29,21,u]
+; AVX512DQ-NEXT:    vpor %ymm8, %ymm9, %ymm8
+; AVX512DQ-NEXT:    vpshufhw {{.*#+}} xmm9 = xmm2[0,1,2,3,4,5,5,6]
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm9 = xmm9[2,2,3,3]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,1,0,1]
+; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[1,3,1,3]
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,1,9],zero,ymm7[u,u,u,u,2,10],zero,ymm7[u,u,u,u,19,27],zero,ymm7[u,u,u,u,20,28],zero,ymm7[u,u,u,u,21]
-; AVX512DQ-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm6, %ymm7
-; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm7
-; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm5, %zmm7, %zmm4
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm6 = ymm3[u,u,u,u,u,5],zero,ymm3[u,u,u,u,u,6],zero,ymm3[u,u,u,u,u],zero,ymm3[23,u,u,u,u,u],zero,ymm3[24,u,u,u,u]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm3[2,3,0,1]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,u,u],zero,ymm7[5,u,u,u,u,u],zero,ymm7[6,u,u,u,u,u,23],zero,ymm7[u,u,u,u,u,24],zero,ymm7[u,u,u,u]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm8 = [255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255,255,0,0,255,255,255,255]
-; AVX512DQ-NEXT:    vpternlogq $50, %ymm6, %ymm8, %ymm7
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,2,0,2]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,ymm3[0,8,u,u,u],zero,zero,ymm3[1,9,u,u,u],zero,zero,ymm3[18,26,u,u,u],zero,zero,ymm3[19,27,u,u,u],zero,zero,ymm3[20,28]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm3, %zmm3
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm6 = ymm2[u,u,u,5],zero,ymm2[u,u,u,u,u,6],zero,ymm2[u,u,u,u,u],zero,ymm2[23,u,u,u,u,u],zero,ymm2[24,u,u,u,u,u],zero
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm2[2,3,0,1]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u],zero,ymm7[5,u,u,u,u,u],zero,ymm7[6,u,u,u,u,u,23],zero,ymm7[u,u,u,u,u,24],zero,ymm7[u,u,u,u,u,25]
-; AVX512DQ-NEXT:    vpternlogq $200, %ymm6, %ymm8, %ymm7
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,0,2]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[0,8],zero,zero,ymm2[u,u,u,1,9],zero,zero,ymm2[u,u,u,2,10],zero,zero,ymm2[u,u,u,19,27],zero,zero,ymm2[u,u,u,20,28],zero,zero
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm2, %zmm2
-; AVX512DQ-NEXT:    vporq %zmm3, %zmm2, %zmm2
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm3 = xmm1[4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,0,1,0]
-; AVX512DQ-NEXT:    vmovdqa {{.*#+}} ymm6 = [255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255]
-; AVX512DQ-NEXT:    vpandn %ymm3, %ymm6, %ymm3
-; AVX512DQ-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[1,1,0,0,4,5,6,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,0]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,0,1,0]
-; AVX512DQ-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm3 = ymm0[4],zero,ymm0[u,u,u,u,u,5],zero,ymm0[u,u,u,u,u,6],zero,ymm0[u,u,u,u,u],zero,ymm0[23,u,u,u,u,u],zero,ymm0[24,u,u]
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm7 = ymm0[2,3,0,1]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm7 = zero,ymm7[4,u,u,u,u,u],zero,ymm7[5,u,u,u,u,u],zero,ymm7[6,u,u,u,u,u,23],zero,ymm7[u,u,u,u,u,24],zero,ymm7[u,u]
-; AVX512DQ-NEXT:    vpternlogq $200, %ymm3, %ymm6, %ymm7
-; AVX512DQ-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,0,2]
-; AVX512DQ-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,0,8],zero,ymm0[u,u,u,u,1,9],zero,ymm0[u,u,u,u,18,26],zero,ymm0[u,u,u,u,19,27],zero,ymm0[u,u,u,u]
-; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm7, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vporq %zmm1, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0
-; AVX512DQ-NEXT:    vmovdqa %xmm5, 96(%rax)
-; AVX512DQ-NEXT:    vmovdqa64 %zmm0, (%rax)
-; AVX512DQ-NEXT:    vmovdqa %ymm4, 64(%rax)
+; AVX512DQ-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm9, %ymm7
+; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm8, %ymm7
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm5[8],xmm6[8],xmm5[9],xmm6[9],xmm5[10],xmm6[10],xmm5[11],xmm6[11],xmm5[12],xmm6[12],xmm5[13],xmm6[13],xmm5[14],xmm6[14],xmm5[15],xmm6[15]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u],zero,zero,xmm5[12,13,u,u,u],zero,zero,xmm5[14,15,u,u,u]
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,12,13],zero,zero,xmm0[u,u,u,14,15],zero,zero,xmm0[u,u,u]
+; AVX512DQ-NEXT:    vpor %xmm5, %xmm0, %xmm0
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm4[8],xmm3[8],xmm4[9],xmm3[9],xmm4[10],xmm3[10],xmm4[11],xmm3[11],xmm4[12],xmm3[12],xmm4[13],xmm3[13],xmm4[14],xmm3[14],xmm4[15],xmm3[15]
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[10],zero,xmm1[u,u,u,u,13,12],zero,xmm1[u,u,u,u,15,14],zero
+; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm2 = zero,xmm2[13,u,u,u,u],zero,zero,xmm2[14,u,u,u,u],zero,zero,xmm2[15]
+; AVX512DQ-NEXT:    vpor %xmm2, %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
+; AVX512DQ-NEXT:    vinserti32x4 $2, %xmm1, %zmm7, %zmm0
+; AVX512DQ-NEXT:    vmovdqa %xmm1, 96(%rax)
+; AVX512DQ-NEXT:    vmovdqa %ymm0, 64(%rax)
+; AVX512DQ-NEXT:    vmovdqa64 %zmm11, (%rax)
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
@@ -2280,70 +2275,69 @@ define void @store_i8_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FCP:       # %bb.0:
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm2
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm3
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm4
-; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm5
-; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %xmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa (%r10), %xmm0
-; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm5, %ymm4, %ymm6
-; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm7
-; AVX512DQ-FCP-NEXT:    vinserti128 $1, (%r9), %ymm1, %ymm1
-; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm0, %zmm1, %zmm1
-; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm4[8],xmm5[8],xmm4[9],xmm5[9],xmm4[10],xmm5[10],xmm4[11],xmm5[11],xmm4[12],xmm5[12],xmm4[13],xmm5[13],xmm4[14],xmm5[14],xmm4[15],xmm5[15]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u],zero,zero,xmm4[12,13,u,u,u],zero,zero,xmm4[14,15,u,u,u]
-; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,12,13],zero,zero,xmm2[u,u,u,14,15],zero,zero,xmm2[u,u,u]
-; AVX512DQ-FCP-NEXT:    vpor %xmm4, %xmm2, %xmm2
-; AVX512DQ-FCP-NEXT:    vextracti128 $1, %ymm1, %xmm3
-; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[10],zero,xmm3[u,u,u,u,13,12],zero,xmm3[u,u,u,u,15,14],zero
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm4 = zero,xmm0[13,u,u,u,u],zero,zero,xmm0[14,u,u,u,u],zero,zero,xmm0[15]
-; AVX512DQ-FCP-NEXT:    vpor %xmm4, %xmm3, %xmm3
-; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm3
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm2 = ymm7[3,1,1,3]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[1],zero,zero,ymm2[u,u,u,10,2],zero,zero,ymm2[u,u,u,11,3],zero,zero,ymm2[u,u,u,20,28],zero,zero,ymm2[u,u,u,21,29],zero,zero,ymm2[u]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm6[1,3,3,1]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = zero,ymm4[1,9,u,u,u],zero,zero,ymm4[2,10,u,u,u],zero,zero,ymm4[3,19,u,u,u],zero,zero,ymm4[28,20,u,u,u],zero,zero,ymm4[29,21,u]
-; AVX512DQ-FCP-NEXT:    vpor %ymm2, %ymm4, %ymm2
-; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm0[0,1,2,3,4,5,5,6]
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [2,2,3,3,2,2,3,3]
-; AVX512DQ-FCP-NEXT:    # ymm5 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm4, %ymm5, %ymm4
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm5 = ymm1[1,3,1,3]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm5 = ymm5[u,u,u,1,9],zero,ymm5[u,u,u,u,2,10],zero,ymm5[u,u,u,u,19,27],zero,ymm5[u,u,u,u,20,28],zero,ymm5[u,u,u,u,21]
-; AVX512DQ-FCP-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm5
-; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm5
-; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm3, %zmm5, %zmm2
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm4 = ymm6[0,2,0,2]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm4 = zero,zero,ymm4[0,8,u,u,u],zero,zero,ymm4[1,9,u,u,u],zero,zero,ymm4[18,26,u,u,u],zero,zero,ymm4[19,27,u,u,u],zero,zero,ymm4[20,28]
-; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [1,5,2,6,1,5,2,6]
-; AVX512DQ-FCP-NEXT:    # ymm5 = mem[0,1,0,1]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm6, %ymm5, %ymm6
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[u,u,u],zero,zero,ymm6[1,5,u,u,u],zero,zero,ymm6[2,6,u,u,u],zero,zero,ymm6[19,23,u,u,u],zero,zero,ymm6[24,28,u,u,u],zero
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm6, %zmm4, %zmm4
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm7[0,2,0,2]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[0,8],zero,zero,ymm6[u,u,u,1,9],zero,zero,ymm6[u,u,u,2,10],zero,zero,ymm6[u,u,u,19,27],zero,zero,ymm6[u,u,u,20,28],zero,zero
-; AVX512DQ-FCP-NEXT:    vpermd %ymm7, %ymm5, %ymm7
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,1,5],zero,zero,ymm7[u,u,u,2,6],zero,zero,ymm7[u,u,u,19,23],zero,zero,ymm7[u,u,u,24,28],zero,zero,ymm7[u,u,u,25]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm7, %zmm6, %zmm6
-; AVX512DQ-FCP-NEXT:    vporq %zmm4, %zmm6, %zmm4
-; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm0[1,1,0,0,4,5,6,7]
-; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm7 = [0,1,0,1,0,0,0,0]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm6, %ymm7, %ymm6
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm6, %zmm0
-; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm6 = ymm1[0,2,0,2]
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[u,u,u,u,0,8],zero,ymm6[u,u,u,u,1,9],zero,ymm6[u,u,u,u,18,26],zero,ymm6[u,u,u,u,19,27],zero,ymm6[u,u,u,u]
-; AVX512DQ-FCP-NEXT:    vpermd %ymm1, %ymm5, %ymm1
-; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[0,4],zero,ymm1[u,u,u,u,1,5],zero,ymm1[u,u,u,u,2,6],zero,ymm1[u,u,u,u,19,23],zero,ymm1[u,u,u,u,24,28],zero,ymm1[u]
-; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm6, %zmm1
-; AVX512DQ-FCP-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1
-; AVX512DQ-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm1
-; AVX512DQ-FCP-NEXT:    vmovdqa %xmm3, 96(%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm1, (%rax)
-; AVX512DQ-FCP-NEXT:    vmovdqa %ymm2, 64(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rsi), %xmm1
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rdx), %xmm5
+; AVX512DQ-FCP-NEXT:    vmovdqa (%rcx), %xmm6
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r8), %xmm3
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r9), %xmm4
+; AVX512DQ-FCP-NEXT:    vmovdqa (%r10), %xmm2
+; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm7
+; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm6, %ymm5, %ymm8
+; AVX512DQ-FCP-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm9
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm10 = ymm8[0,2,0,2]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm10 = zero,zero,ymm10[0,8,u,u,u],zero,zero,ymm10[1,9,u,u,u],zero,zero,ymm10[18,26,u,u,u],zero,zero,ymm10[19,27,u,u,u],zero,zero,ymm10[20,28]
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm11 = [1,5,2,6,1,5,2,6]
+; AVX512DQ-FCP-NEXT:    # ymm11 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm8, %ymm11, %ymm12
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm12 = ymm12[u,u,u],zero,zero,ymm12[1,5,u,u,u],zero,zero,ymm12[2,6,u,u,u],zero,zero,ymm12[19,23,u,u,u],zero,zero,ymm12[24,28,u,u,u],zero
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm12, %zmm10, %zmm10
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm12 = ymm7[0,2,0,2]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm12 = ymm12[0,8],zero,zero,ymm12[u,u,u,1,9],zero,zero,ymm12[u,u,u,2,10],zero,zero,ymm12[u,u,u,19,27],zero,zero,ymm12[u,u,u,20,28],zero,zero
+; AVX512DQ-FCP-NEXT:    vpermd %ymm7, %ymm11, %ymm13
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm13 = ymm13[u,u,u,1,5],zero,zero,ymm13[u,u,u,2,6],zero,zero,ymm13[u,u,u,19,23],zero,zero,ymm13[u,u,u,24,28],zero,zero,ymm13[u,u,u,25]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm12, %zmm12
+; AVX512DQ-FCP-NEXT:    vporq %zmm10, %zmm12, %zmm10
+; AVX512DQ-FCP-NEXT:    vpshuflw {{.*#+}} xmm12 = xmm2[1,1,0,0,4,5,6,7]
+; AVX512DQ-FCP-NEXT:    vpmovsxbd {{.*#+}} ymm13 = [0,1,0,1,0,0,0,0]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm12, %ymm13, %ymm12
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm13 = xmm2[4,5,4,5,4,5,8,9,6,7,6,7,6,7,6,7]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,0,1,0]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm13, %zmm12, %zmm12
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm13 = ymm9[0,2,0,2]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm13 = ymm13[u,u,u,u,0,8],zero,ymm13[u,u,u,u,1,9],zero,ymm13[u,u,u,u,18,26],zero,ymm13[u,u,u,u,19,27],zero,ymm13[u,u,u,u]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm9, %ymm11, %ymm11
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[0,4],zero,ymm11[u,u,u,u,1,5],zero,ymm11[u,u,u,u,2,6],zero,ymm11[u,u,u,u,19,23],zero,ymm11[u,u,u,u,24,28],zero,ymm11[u]
+; AVX512DQ-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm13, %zmm11
+; AVX512DQ-FCP-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm12, %zmm11
+; AVX512DQ-FCP-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm11
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[3,1,1,3]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[1],zero,zero,ymm7[u,u,u,10,2],zero,zero,ymm7[u,u,u,11,3],zero,zero,ymm7[u,u,u,20,28],zero,zero,ymm7[u,u,u,21,29],zero,zero,ymm7[u]
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[1,3,3,1]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm8 = zero,ymm8[1,9,u,u,u],zero,zero,ymm8[2,10,u,u,u],zero,zero,ymm8[3,19,u,u,u],zero,zero,ymm8[28,20,u,u,u],zero,zero,ymm8[29,21,u]
+; AVX512DQ-FCP-NEXT:    vpor %ymm7, %ymm8, %ymm7
+; AVX512DQ-FCP-NEXT:    vpshufhw {{.*#+}} xmm8 = xmm2[0,1,2,3,4,5,5,6]
+; AVX512DQ-FCP-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [2,2,3,3,2,2,3,3]
+; AVX512DQ-FCP-NEXT:    # ymm10 = mem[0,1,0,1]
+; AVX512DQ-FCP-NEXT:    vpermd %ymm8, %ymm10, %ymm8
+; AVX512DQ-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[1,3,1,3]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} ymm9 = ymm9[u,u,u,1,9],zero,ymm9[u,u,u,u,2,10],zero,ymm9[u,u,u,u,19,27],zero,ymm9[u,u,u,u,20,28],zero,ymm9[u,u,u,u,21]
+; AVX512DQ-FCP-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm8, %ymm9
+; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm7, %ymm9
+; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm5[8],xmm6[8],xmm5[9],xmm6[9],xmm5[10],xmm6[10],xmm5[11],xmm6[11],xmm5[12],xmm6[12],xmm5[13],xmm6[13],xmm5[14],xmm6[14],xmm5[15],xmm6[15]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u],zero,zero,xmm5[12,13,u,u,u],zero,zero,xmm5[14,15,u,u,u]
+; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,12,13],zero,zero,xmm0[u,u,u,14,15],zero,zero,xmm0[u,u,u]
+; AVX512DQ-FCP-NEXT:    vpor %xmm5, %xmm0, %xmm0
+; AVX512DQ-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm4[8],xmm3[8],xmm4[9],xmm3[9],xmm4[10],xmm3[10],xmm4[11],xmm3[11],xmm4[12],xmm3[12],xmm4[13],xmm3[13],xmm4[14],xmm3[14],xmm4[15],xmm3[15]
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[10],zero,xmm1[u,u,u,u,13,12],zero,xmm1[u,u,u,u,15,14],zero
+; AVX512DQ-FCP-NEXT:    vpshufb {{.*#+}} xmm2 = zero,xmm2[13,u,u,u,u],zero,zero,xmm2[14,u,u,u,u],zero,zero,xmm2[15]
+; AVX512DQ-FCP-NEXT:    vpor %xmm2, %xmm1, %xmm1
+; AVX512DQ-FCP-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
+; AVX512DQ-FCP-NEXT:    vinserti32x4 $2, %xmm1, %zmm9, %zmm0
+; AVX512DQ-FCP-NEXT:    vmovdqa %xmm1, 96(%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa64 %zmm11, (%rax)
+; AVX512DQ-FCP-NEXT:    vmovdqa %ymm0, 64(%rax)
 ; AVX512DQ-FCP-NEXT:    vzeroupper
 ; AVX512DQ-FCP-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll b/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
index 11f422d67154..99e8cdb179c8 100644
--- a/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
+++ b/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
@@ -314,8 +314,8 @@ define void @vec64_i16_widen_to_i32_factor2_broadcast_to_v2i32_factor2(ptr %in.v
 ;
 ; AVX512F-LABEL: vec64_i16_widen_to_i32_factor2_broadcast_to_v2i32_factor2:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,10,11,0,1,14,15,u,u,u,u,u,u,u,u]
 ; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)
@@ -324,8 +324,8 @@ define void @vec64_i16_widen_to_i32_factor2_broadcast_to_v2i32_factor2(ptr %in.v
 ;
 ; AVX512DQ-LABEL: vec64_i16_widen_to_i32_factor2_broadcast_to_v2i32_factor2:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512DQ-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,10,11,0,1,14,15,u,u,u,u,u,u,u,u]
 ; AVX512DQ-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, (%rcx)
@@ -981,7 +981,7 @@ define void @vec128_i32_widen_to_i64_factor2_broadcast_to_v2i64_factor2(ptr %in.
 ; AVX512F-NEXT:    vpmovsxbd {{.*#+}} xmm0 = [0,5,0,7]
 ; AVX512F-NEXT:    vmovdqa (%rdi), %ymm1
 ; AVX512F-NEXT:    vpaddb (%rsi), %ymm1, %ymm1
-; AVX512F-NEXT:    vpermd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpermd %ymm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm0, (%rcx)
 ; AVX512F-NEXT:    vzeroupper
@@ -992,7 +992,7 @@ define void @vec128_i32_widen_to_i64_factor2_broadcast_to_v2i64_factor2(ptr %in.
 ; AVX512DQ-NEXT:    vpmovsxbd {{.*#+}} xmm0 = [0,5,0,7]
 ; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm1
 ; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm1, %ymm1
-; AVX512DQ-NEXT:    vpermd %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT:    vpermd %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpaddb (%rdx), %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vmovdqa %ymm0, (%rcx)
 ; AVX512DQ-NEXT:    vzeroupper
@@ -4026,10 +4026,10 @@ define void @vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6(ptr %in.
 ;
 ; AVX512F-FAST-LABEL: vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6:
 ; AVX512F-FAST:       # %bb.0:
-; AVX512F-FAST-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512F-FAST-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX512F-FAST-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX512F-FAST-NEXT:    vmovdqa 48(%rdi), %xmm1
 ; AVX512F-FAST-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
+; AVX512F-FAST-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512F-FAST-NEXT:    vpbroadcastq %xmm0, %ymm2
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1,2,3],ymm2[4],ymm1[5,6,7],ymm2[8],ymm1[9,10,11],ymm2[12],ymm1[13,14,15]
 ; AVX512F-FAST-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
@@ -4062,10 +4062,10 @@ define void @vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6(ptr %in.
 ;
 ; AVX512DQ-FAST-LABEL: vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6:
 ; AVX512DQ-FAST:       # %bb.0:
-; AVX512DQ-FAST-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512DQ-FAST-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX512DQ-FAST-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX512DQ-FAST-NEXT:    vmovdqa 48(%rdi), %xmm1
 ; AVX512DQ-FAST-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
+; AVX512DQ-FAST-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512DQ-FAST-NEXT:    vpbroadcastq %xmm0, %ymm2
 ; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1,2,3],ymm2[4],ymm1[5,6,7],ymm2[8],ymm1[9,10,11],ymm2[12],ymm1[13,14,15]
 ; AVX512DQ-FAST-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
@@ -4541,9 +4541,9 @@ define void @vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2(ptr %
 ;
 ; AVX512F-LABEL: vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512F-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX512F-NEXT:    vmovdqa 48(%rdi), %xmm1
+; AVX512F-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512F-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
 ; AVX512F-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]
 ; AVX512F-NEXT:    vpbroadcastw %xmm0, %ymm0
@@ -4559,9 +4559,9 @@ define void @vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2(ptr %
 ;
 ; AVX512DQ-LABEL: vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512DQ-NEXT:    vpaddb (%rsi), %ymm0, %ymm0
+; AVX512DQ-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX512DQ-NEXT:    vmovdqa 48(%rdi), %xmm1
+; AVX512DQ-NEXT:    vpaddb (%rsi), %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpaddb 48(%rsi), %xmm1, %xmm1
 ; AVX512DQ-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]
 ; AVX512DQ-NEXT:    vpbroadcastw %xmm0, %ymm0
-- 
GitLab


From 71b9f6648222771470473431bc8ef2a2c25e872c Mon Sep 17 00:00:00 2001
From: Krystian Stasiowski <sdkrystian@gmail.com>
Date: Tue, 16 Apr 2024 07:34:27 -0400
Subject: [PATCH 384/464] [clang][Index] Use canonical function parameter types
 in USRs (#68222)

This is necessary to ensure that functions declared in different
translation units whose parameter types only differ in top-level
cv-qualification generate the same USR.

For example:
```
// A.cpp
void f(const int x); // c:@F@f#1I#

// B.cpp
void f(int x);       // c:@F@f#I#
```
With this patch, the USR for both functions will be
`c:@F@f#I#`.
---
 clang/lib/Index/USRGeneration.cpp  |  9 ++++++---
 clang/test/Index/USR/func-type.cpp | 12 ++++++++++++
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/clang/lib/Index/USRGeneration.cpp b/clang/lib/Index/USRGeneration.cpp
index 5acc86191f8f..31c4a3345c09 100644
--- a/clang/lib/Index/USRGeneration.cpp
+++ b/clang/lib/Index/USRGeneration.cpp
@@ -267,10 +267,13 @@ void USRGenerator::VisitFunctionDecl(const FunctionDecl *D) {
     Out << '>';
   }
 
+  QualType CanonicalType = D->getType().getCanonicalType();
   // Mangle in type information for the arguments.
-  for (auto *PD : D->parameters()) {
-    Out << '#';
-    VisitType(PD->getType());
+  if (const auto *FPT = CanonicalType->getAs<FunctionProtoType>()) {
+    for (QualType PT : FPT->param_types()) {
+      Out << '#';
+      VisitType(PT);
+    }
   }
   if (D->isVariadic())
     Out << '.';
diff --git a/clang/test/Index/USR/func-type.cpp b/clang/test/Index/USR/func-type.cpp
index ff1cd37a7fc4..459a8cd6da55 100644
--- a/clang/test/Index/USR/func-type.cpp
+++ b/clang/test/Index/USR/func-type.cpp
@@ -16,3 +16,15 @@ void Func( void (* (*)(int, int))(int, int) );
 // CHECK: {{[0-9]+}}:6 | function/C | Func | c:@F@Func#*F*Fv(#I#I)(#I#I)# |
 void Func( void (* (*)(int, int, int))(int) );
 // CHECK: {{[0-9]+}}:6 | function/C | Func | c:@F@Func#*F*Fv(#I)(#I#I#I)# |
+
+// Functions with parameter types that only differ in top-level cv-qualification should generate the same USR.
+
+void f( const int );
+// CHECK: {{[0-9]+}}:6 | function/C | f | c:@F@f#I# |
+void f( int );
+// CHECK: {{[0-9]+}}:6 | function/C | f | c:@F@f#I# |
+
+void g( int );
+// CHECK: {{[0-9]+}}:6 | function/C | g | c:@F@g#I# |
+void g( const int );
+// CHECK: {{[0-9]+}}:6 | function/C | g | c:@F@g#I# |
-- 
GitLab


From dadcaf82274805456b7d85131cf94f921b5398b7 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell@arm.com>
Date: Tue, 16 Apr 2024 12:54:01 +0100
Subject: [PATCH 385/464] [mlir][ArmSME] Support decomposing constant splats
 into ArmSME tiles (#88762)

This adds a simple rewrite/legalization to decompose constant splats
larger than a single ArmSME tile into multiple SME virtual tile sized
splats. E.g. a constant splat to `vector<[8]x[8]xi32>` would decompose
into four `vector<[4]x[4]xi32>` splats.
---
 .../ArmSME/Transforms/VectorLegalization.cpp  | 32 ++++++++++++++++++-
 .../Dialect/ArmSME/vector-legalization.mlir   | 11 +++++++
 2 files changed, 42 insertions(+), 1 deletion(-)

diff --git a/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp b/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
index 31500c62c0d6..b595c6dd8a68 100644
--- a/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
+++ b/mlir/lib/Dialect/ArmSME/Transforms/VectorLegalization.cpp
@@ -165,6 +165,35 @@ int getNumberOfSMETilesForVectorType(VectorType type) {
   return (vectorRows * vectorCols) / (minNumElts * minNumElts);
 }
 
+/// Legalize `arith.constant dense<value>` splat operations to fit within SME
+/// tiles by decomposing them into tile-sized operations.
+struct LegalizeArithConstantOpsByDecomposition
+    : public OneToNOpConversionPattern<arith::ConstantOp> {
+  using OneToNOpConversionPattern::OneToNOpConversionPattern;
+
+  LogicalResult
+  matchAndRewrite(arith::ConstantOp constantOp, OpAdaptor adaptor,
+                  OneToNPatternRewriter &rewriter) const override {
+    auto vectorType = dyn_cast<VectorType>(constantOp.getType());
+    auto denseAttr = dyn_cast<DenseElementsAttr>(constantOp.getValueAttr());
+    if (!vectorType || !denseAttr || !denseAttr.isSplat())
+      return failure();
+
+    if (!isMultipleOfSMETileVectorType(vectorType))
+      return rewriter.notifyMatchFailure(constantOp,
+                                         kMatchFailureNotSMETileTypeMultiple);
+
+    auto smeTileType = getSMETileTypeForElement(vectorType.getElementType());
+    auto tileCount = getNumberOfSMETilesForVectorType(vectorType);
+    auto tileSplat = rewriter.create<arith::ConstantOp>(
+        constantOp.getLoc(), denseAttr.resizeSplat(smeTileType));
+    rewriter.replaceOp(constantOp, SmallVector<Value>(tileCount, tileSplat),
+                       adaptor.getResultMapping());
+
+    return success();
+  }
+};
+
 /// Legalize `vector.outerproduct` operations to fit within SME tiles by
 /// decomposing them into tile-sized operations.
 struct LegalizeVectorOuterProductOpsByDecomposition
@@ -637,7 +666,8 @@ struct VectorLegalizationPass
     // Note: High benefit to ensure masked outer products are lowered first.
     patterns.add<LegalizeMaskedVectorOuterProductOpsByDecomposition>(
         converter, context, 1024);
-    patterns.add<LegalizeVectorOuterProductOpsByDecomposition,
+    patterns.add<LegalizeArithConstantOpsByDecomposition,
+                 LegalizeVectorOuterProductOpsByDecomposition,
                  LegalizeTransferReadOpsByDecomposition,
                  LegalizeTransferWriteOpsByDecomposition>(converter, context);
     populateFuncTypeConversionPatterns(converter, patterns);
diff --git a/mlir/test/Dialect/ArmSME/vector-legalization.mlir b/mlir/test/Dialect/ArmSME/vector-legalization.mlir
index f8be697548c1..f43ef1cce787 100644
--- a/mlir/test/Dialect/ArmSME/vector-legalization.mlir
+++ b/mlir/test/Dialect/ArmSME/vector-legalization.mlir
@@ -433,3 +433,14 @@ func.func @lift_illegal_1d_shape_cast_to_memory(%a: index, %b: index, %memref: m
   %cast = vector.shape_cast %illegalRead : vector<[4]x1xf32> to vector<[4]xf32>
   return %cast : vector<[4]xf32>
 }
+
+// -----
+
+// CHECK-LABEL: @multi_tile_splat
+func.func @multi_tile_splat() -> vector<[8]x[8]xi32>
+{
+  // CHECK: %[[SPLAT:.*]] = arith.constant dense<42> : vector<[4]x[4]xi32>
+  // CHECK-NEXT: return %[[SPLAT]], %[[SPLAT]], %[[SPLAT]], %[[SPLAT]] : vector<[4]x[4]xi32>, vector<[4]x[4]xi32>, vector<[4]x[4]xi32>, vector<[4]x[4]xi32>
+  %0 = arith.constant dense<42> : vector<[8]x[8]xi32>
+  return %0 : vector<[8]x[8]xi32>
+}
-- 
GitLab


From 711df7b0ae4a9ea45e431d5c0ff4a0c8b2e732c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andrzej=20Warzy=C5=84ski?= <andrzej.warzynski@arm.com>
Date: Tue, 16 Apr 2024 12:57:59 +0100
Subject: [PATCH 386/464] [vector][mlir] Restrict vector.shuffle to fixed-width
 vectors (#88733)

At the moment there is no support for vector.shuffle for scalable
vectors - various hooks/helpers related to `vector.shuffle` simply
ignore the scalable flags (e.g. ` ShuffleOp::inferReturnTypes`).

This is unlikely to change any time soon (vector shuffles are known to
be tricky for scalable vectors), hence this patch restricts
`vector.shuffle` to fixed width vectors.
---
 mlir/include/mlir/Dialect/Vector/IR/VectorOps.td | 4 +++-
 mlir/test/Dialect/Vector/canonicalize.mlir       | 8 --------
 mlir/test/Dialect/Vector/invalid.mlir            | 7 +++++++
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
index 147bc2354977..332b5ad08ced 100644
--- a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
+++ b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
@@ -420,7 +420,7 @@ def Vector_ShuffleOp :
      PredOpTrait<"second operand v2 and result have same element type",
                  TCresVTEtIsSameAsOpBase<0, 1>>,
      InferTypeOpAdaptor]>,
-     Arguments<(ins AnyVectorOfAnyRank:$v1, AnyVectorOfAnyRank:$v2,
+     Arguments<(ins AnyFixedVector:$v1, AnyFixedVector:$v2,
                     I64ArrayAttr:$mask)>,
      Results<(outs AnyVector:$vector)> {
   let summary = "shuffle operation";
@@ -444,6 +444,8 @@ def Vector_ShuffleOp :
       mask values must be within range, viz. given two k-D operands v1 and v2
       above, all mask values are in the range [0,s_1+t_1)
 
+    Note, scalable vectors are not supported.
+
     Example:
 
     ```mlir
diff --git a/mlir/test/Dialect/Vector/canonicalize.mlir b/mlir/test/Dialect/Vector/canonicalize.mlir
index 627ac54cf145..61a5f2a96e1c 100644
--- a/mlir/test/Dialect/Vector/canonicalize.mlir
+++ b/mlir/test/Dialect/Vector/canonicalize.mlir
@@ -1943,14 +1943,6 @@ func.func @shuffle_nofold1(%v0 : vector<4xi32>, %v1 : vector<2xi32>) -> vector<5
   return %shuffle : vector<5xi32>
 }
 
-// CHECK-LABEL: func @shuffle_nofold2
-//       CHECK:   %[[V:.+]] = vector.shuffle %arg0, %arg1 [0, 1, 2, 3] : vector<[4]xi32>, vector<[2]xi32>
-//       CHECK:   return %[[V]]
-func.func @shuffle_nofold2(%v0 : vector<[4]xi32>, %v1 : vector<[2]xi32>) -> vector<4xi32> {
-  %shuffle = vector.shuffle %v0, %v1 [0, 1, 2, 3] : vector<[4]xi32>, vector<[2]xi32>
-  return %shuffle : vector<4xi32>
-}
-
 // -----
 
 // CHECK-LABEL: func @transpose_scalar_broadcast1
diff --git a/mlir/test/Dialect/Vector/invalid.mlir b/mlir/test/Dialect/Vector/invalid.mlir
index c16f1cb2876d..c9f7e9c6e2fb 100644
--- a/mlir/test/Dialect/Vector/invalid.mlir
+++ b/mlir/test/Dialect/Vector/invalid.mlir
@@ -84,6 +84,13 @@ func.func @shuffle_index_out_of_range(%arg0: vector<2xf32>, %arg1: vector<2xf32>
 
 // -----
 
+func.func @shuffle_scalable_vec(%arg0: vector<[2]xf32>, %arg1: vector<[2]xf32>) {
+  // expected-error@+1 {{'vector.shuffle' op operand #0 must be fixed-length vector of any type values}}
+  %1 = vector.shuffle %arg0, %arg1 [0, 1, 2, 3] : vector<[2]xf32>, vector<[2]xf32>
+}
+
+// -----
+
 func.func @shuffle_empty_mask(%arg0: vector<2xf32>, %arg1: vector<2xf32>) {
   // expected-error@+1 {{'vector.shuffle' op invalid mask length}}
   %1 = vector.shuffle %arg0, %arg1 [] : vector<2xf32>, vector<2xf32>
-- 
GitLab


From 75244a1043d2be5003dea6914d5edc940c437cd5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Tue, 16 Apr 2024 09:44:54 +0200
Subject: [PATCH 387/464] [clang][Interp] Implement align builtins

__builtin_is_aligned
__builtin_is_align_up
__builtin_is_align_down
---
 clang/lib/AST/Interp/InterpBuiltin.cpp      | 118 +++++++++
 clang/test/AST/Interp/builtin-align-cxx.cpp | 258 ++++++++++++++++++++
 2 files changed, 376 insertions(+)
 create mode 100644 clang/test/AST/Interp/builtin-align-cxx.cpp

diff --git a/clang/lib/AST/Interp/InterpBuiltin.cpp b/clang/lib/AST/Interp/InterpBuiltin.cpp
index 984ba4f7f268..f562f9e1cb19 100644
--- a/clang/lib/AST/Interp/InterpBuiltin.cpp
+++ b/clang/lib/AST/Interp/InterpBuiltin.cpp
@@ -977,6 +977,117 @@ static bool interp__builtin_complex(InterpState &S, CodePtr OpPC,
   return true;
 }
 
+/// __builtin_is_aligned()
+/// __builtin_align_up()
+/// __builtin_align_down()
+/// The first parameter is either an integer or a pointer.
+/// The second parameter is the requested alignment as an integer.
+static bool interp__builtin_is_aligned_up_down(InterpState &S, CodePtr OpPC,
+                                               const InterpFrame *Frame,
+                                               const Function *Func,
+                                               const CallExpr *Call) {
+  unsigned BuiltinOp = Func->getBuiltinID();
+  unsigned CallSize = callArgSize(S, Call);
+
+  PrimType AlignmentT = *S.Ctx.classify(Call->getArg(1));
+  const APSInt &Alignment = peekToAPSInt(S.Stk, AlignmentT);
+
+  if (Alignment < 0 || !Alignment.isPowerOf2()) {
+    S.FFDiag(Call, diag::note_constexpr_invalid_alignment) << Alignment;
+    return false;
+  }
+  unsigned SrcWidth = S.getCtx().getIntWidth(Call->getArg(0)->getType());
+  APSInt MaxValue(APInt::getOneBitSet(SrcWidth, SrcWidth - 1));
+  if (APSInt::compareValues(Alignment, MaxValue) > 0) {
+    S.FFDiag(Call, diag::note_constexpr_alignment_too_big)
+        << MaxValue << Call->getArg(0)->getType() << Alignment;
+    return false;
+  }
+
+  // The first parameter is either an integer or a pointer (but not a function
+  // pointer).
+  PrimType FirstArgT = *S.Ctx.classify(Call->getArg(0));
+
+  if (isIntegralType(FirstArgT)) {
+    const APSInt &Src = peekToAPSInt(S.Stk, FirstArgT, CallSize);
+    APSInt Align = Alignment.extOrTrunc(Src.getBitWidth());
+    if (BuiltinOp == Builtin::BI__builtin_align_up) {
+      APSInt AlignedVal =
+          APSInt((Src + (Align - 1)) & ~(Align - 1), Src.isUnsigned());
+      pushInteger(S, AlignedVal, Call->getType());
+    } else if (BuiltinOp == Builtin::BI__builtin_align_down) {
+      APSInt AlignedVal = APSInt(Src & ~(Align - 1), Src.isUnsigned());
+      pushInteger(S, AlignedVal, Call->getType());
+    } else {
+      assert(*S.Ctx.classify(Call->getType()) == PT_Bool);
+      S.Stk.push<Boolean>((Src & (Align - 1)) == 0);
+    }
+    return true;
+  }
+
+  assert(FirstArgT == PT_Ptr);
+  const Pointer &Ptr = S.Stk.peek<Pointer>(CallSize);
+
+  unsigned PtrOffset = Ptr.getByteOffset();
+  PtrOffset = Ptr.getIndex();
+  CharUnits BaseAlignment =
+      S.getCtx().getDeclAlign(Ptr.getDeclDesc()->asValueDecl());
+  CharUnits PtrAlign =
+      BaseAlignment.alignmentAtOffset(CharUnits::fromQuantity(PtrOffset));
+
+  if (BuiltinOp == Builtin::BI__builtin_is_aligned) {
+    if (PtrAlign.getQuantity() >= Alignment) {
+      S.Stk.push<Boolean>(true);
+      return true;
+    }
+    // If the alignment is not known to be sufficient, some cases could still
+    // be aligned at run time. However, if the requested alignment is less or
+    // equal to the base alignment and the offset is not aligned, we know that
+    // the run-time value can never be aligned.
+    if (BaseAlignment.getQuantity() >= Alignment &&
+        PtrAlign.getQuantity() < Alignment) {
+      S.Stk.push<Boolean>(false);
+      return true;
+    }
+
+    S.FFDiag(Call->getArg(0), diag::note_constexpr_alignment_compute)
+        << Alignment;
+    return false;
+  }
+
+  assert(BuiltinOp == Builtin::BI__builtin_align_down ||
+         BuiltinOp == Builtin::BI__builtin_align_up);
+
+  // For align_up/align_down, we can return the same value if the alignment
+  // is known to be greater or equal to the requested value.
+  if (PtrAlign.getQuantity() >= Alignment) {
+    S.Stk.push<Pointer>(Ptr);
+    return true;
+  }
+
+  // The alignment could be greater than the minimum at run-time, so we cannot
+  // infer much about the resulting pointer value. One case is possible:
+  // For `_Alignas(32) char buf[N]; __builtin_align_down(&buf[idx], 32)` we
+  // can infer the correct index if the requested alignment is smaller than
+  // the base alignment so we can perform the computation on the offset.
+  if (BaseAlignment.getQuantity() >= Alignment) {
+    assert(Alignment.getBitWidth() <= 64 &&
+           "Cannot handle > 64-bit address-space");
+    uint64_t Alignment64 = Alignment.getZExtValue();
+    CharUnits NewOffset =
+        CharUnits::fromQuantity(BuiltinOp == Builtin::BI__builtin_align_down
+                                    ? llvm::alignDown(PtrOffset, Alignment64)
+                                    : llvm::alignTo(PtrOffset, Alignment64));
+
+    S.Stk.push<Pointer>(Ptr.atIndex(NewOffset.getQuantity()));
+    return true;
+  }
+
+  // Otherwise, we cannot constant-evaluate the result.
+  S.FFDiag(Call->getArg(0), diag::note_constexpr_alignment_adjust) << Alignment;
+  return false;
+}
+
 bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const Function *F,
                       const CallExpr *Call) {
   const InterpFrame *Frame = S.Current;
@@ -1291,6 +1402,13 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const Function *F,
       return false;
     break;
 
+  case Builtin::BI__builtin_is_aligned:
+  case Builtin::BI__builtin_align_up:
+  case Builtin::BI__builtin_align_down:
+    if (!interp__builtin_is_aligned_up_down(S, OpPC, Frame, F, Call))
+      return false;
+    break;
+
   default:
     S.FFDiag(S.Current->getLocation(OpPC),
              diag::note_invalid_subexpr_in_const_expr)
diff --git a/clang/test/AST/Interp/builtin-align-cxx.cpp b/clang/test/AST/Interp/builtin-align-cxx.cpp
new file mode 100644
index 000000000000..62d73dba929b
--- /dev/null
+++ b/clang/test/AST/Interp/builtin-align-cxx.cpp
@@ -0,0 +1,258 @@
+// C++-specific checks for the alignment builtins
+// RUN: %clang_cc1 -triple=x86_64-unknown-unknown -std=c++11 %s -fsyntax-only -verify=expected,both -fexperimental-new-constant-interpreter
+// RUN: %clang_cc1 -triple=x86_64-unknown-unknown -std=c++11 %s -fsyntax-only -verify=ref,both
+
+
+/// This is just a copy of the one from test/SemaCXX/ with some of the
+/// diagnostic output adapted.
+/// Also, align32array has an initializer now, which means it's not just
+/// a dummy pointer for us and we do actually have type information for it.
+/// In the future, we need to retain type information for dummy pointers as
+/// well, so here is a test that will break once we do that:
+namespace {
+  _Alignas(32) char heh[4];
+  static_assert(!__builtin_is_aligned(&heh[1], 4), ""); // expected-error {{failed}}
+}
+
+
+// Check that we don't crash when using dependent types in __builtin_align:
+template <typename a, a b>
+void *c(void *d) { // both-note{{candidate template ignored}}
+  return __builtin_align_down(d, b);
+}
+
+struct x {};
+x foo;
+void test(void *value) {
+  c<int, 16>(value);
+  c<struct x, foo>(value); // both-error{{no matching function for call to 'c'}}
+}
+
+template <typename T, long Alignment, long ArraySize = 16>
+void test_templated_arguments() {
+  T array[ArraySize];                                                           // both-error{{variable has incomplete type 'fwddecl'}}
+  static_assert(__is_same(decltype(__builtin_align_up(array, Alignment)), T *), // both-error{{requested alignment is not a power of 2}}
+                "return type should be the decayed array type");
+  static_assert(__is_same(decltype(__builtin_align_down(array, Alignment)), T *),
+                "return type should be the decayed array type");
+  static_assert(__is_same(decltype(__builtin_is_aligned(array, Alignment)), bool),
+                "return type should be bool");
+  T *x1 = __builtin_align_up(array, Alignment);
+  T *x2 = __builtin_align_down(array, Alignment);
+  bool x3 = __builtin_align_up(array, Alignment);
+}
+
+void test() {
+  test_templated_arguments<int, 32>(); // fine
+  test_templated_arguments<struct fwddecl, 16>();
+  // both-note@-1{{in instantiation of function template specialization 'test_templated_arguments<fwddecl, 16L, 16L>'}}
+  // both-note@-2{{forward declaration of 'fwddecl'}}
+  test_templated_arguments<int, 7>(); // invalid alignment value
+  // both-note@-1{{in instantiation of function template specialization 'test_templated_arguments<int, 7L, 16L>'}}
+}
+
+template <typename T, long ArraySize>
+void test_incorrect_alignment_without_instatiation(T value) {
+  int array[32];
+  static_assert(__is_same(decltype(__builtin_align_up(array, 31)), int *), // both-error{{requested alignment is not a power of 2}}
+                "return type should be the decayed array type");
+  static_assert(__is_same(decltype(__builtin_align_down(array, 7)), int *), // both-error{{requested alignment is not a power of 2}}
+                "return type should be the decayed array type");
+  static_assert(__is_same(decltype(__builtin_is_aligned(array, -1)), bool), // both-error{{requested alignment must be 1 or greater}}
+                "return type should be bool");
+  __builtin_align_up(array);       // both-error{{too few arguments to function call, expected 2, have 1}}
+  __builtin_align_up(array, 31);   // both-error{{requested alignment is not a power of 2}}
+  __builtin_align_down(array, 31); // both-error{{requested alignment is not a power of 2}}
+  __builtin_align_up(array, 31);   // both-error{{requested alignment is not a power of 2}}
+  __builtin_align_up(value, 31);   // This shouldn't want since the type is dependent
+  __builtin_align_up(value);       // Same here
+
+  __builtin_align_up(array, sizeof(sizeof(value)) - 1); // both-error{{requested alignment is not a power of 2}}
+  __builtin_align_up(array, value); // no diagnostic as the alignment is value dependent.
+  (void)__builtin_align_up(array, ArraySize); // The same above here
+}
+
+// The original fix for the issue above broke some legitimate code.
+// Here is a regression test:
+typedef __SIZE_TYPE__ size_t;
+void *allocate_impl(size_t size);
+template <typename T>
+T *allocate() {
+  constexpr size_t allocation_size =
+      __builtin_align_up(sizeof(T), sizeof(void *));
+  return static_cast<T *>(
+      __builtin_assume_aligned(allocate_impl(allocation_size), sizeof(void *)));
+}
+struct Foo {
+  int value;
+};
+void *test2() {
+  return allocate<struct Foo>();
+}
+
+// Check that pointers-to-members cannot be used:
+class MemPtr {
+public:
+  int data;
+  void func();
+  virtual void vfunc();
+};
+void test_member_ptr() {
+  __builtin_align_up(&MemPtr::data, 64);    // both-error{{operand of type 'int MemPtr::*' where arithmetic or pointer type is required}}
+  __builtin_align_down(&MemPtr::func, 64);  // both-error{{operand of type 'void (MemPtr::*)()' where arithmetic or pointer type is required}}
+  __builtin_is_aligned(&MemPtr::vfunc, 64); // both-error{{operand of type 'void (MemPtr::*)()' where arithmetic or pointer type is required}}
+}
+
+void test_references(Foo &i) {
+  // Check that the builtins look at the referenced type rather than the reference itself.
+  (void)__builtin_align_up(i, 64);                            // both-error{{operand of type 'Foo' where arithmetic or pointer type is required}}
+  (void)__builtin_align_up(static_cast<Foo &>(i), 64);        // both-error{{operand of type 'Foo' where arithmetic or pointer type is required}}
+  (void)__builtin_align_up(static_cast<const Foo &>(i), 64);  // both-error{{operand of type 'const Foo' where arithmetic or pointer type is required}}
+  (void)__builtin_align_up(static_cast<Foo &&>(i), 64);       // both-error{{operand of type 'Foo' where arithmetic or pointer type is required}}
+  (void)__builtin_align_up(static_cast<const Foo &&>(i), 64); // both-error{{operand of type 'const Foo' where arithmetic or pointer type is required}}
+  (void)__builtin_align_up(&i, 64);
+}
+
+// Check that constexpr wrapper functions can be constant-evaluated.
+template <typename T>
+constexpr bool wrap_is_aligned(T ptr, long align) {
+  return __builtin_is_aligned(ptr, align);
+  // both-note@-1{{requested alignment -3 is not a positive power of two}}
+  // both-note@-2{{requested alignment 19 is not a positive power of two}}
+  // both-note@-3{{requested alignment must be 128 or less for type 'char'; 4194304 is invalid}}
+}
+template <typename T>
+constexpr T wrap_align_up(T ptr, long align) {
+  return __builtin_align_up(ptr, align);
+  // both-note@-1{{requested alignment -2 is not a positive power of two}}
+  // both-note@-2{{requested alignment 18 is not a positive power of two}}
+  // both-note@-3{{requested alignment must be 2147483648 or less for type 'int'; 8589934592 is invalid}}
+  // both-error@-4{{operand of type 'bool' where arithmetic or pointer type is required}}
+}
+
+template <typename T>
+constexpr T wrap_align_down(T ptr, long align) {
+  return __builtin_align_down(ptr, align);
+  // both-note@-1{{requested alignment -1 is not a positive power of two}}
+  // both-note@-2{{requested alignment 17 is not a positive power of two}}
+  // both-note@-3{{requested alignment must be 32768 or less for type 'short'; 1048576 is invalid}}
+}
+
+constexpr int a1 = wrap_align_up(22, 32);
+static_assert(a1 == 32, "");
+constexpr int a2 = wrap_align_down(22, 16);
+static_assert(a2 == 16, "");
+constexpr bool a3 = wrap_is_aligned(22, 32);
+static_assert(!a3, "");
+static_assert(wrap_align_down(wrap_align_up(22, 16), 32) == 32, "");
+static_assert(wrap_is_aligned(wrap_align_down(wrap_align_up(22, 16), 32), 32), "");
+static_assert(!wrap_is_aligned(wrap_align_down(wrap_align_up(22, 16), 32), 64), "");
+
+constexpr long const_value(long l) { return l; }
+// Check some invalid values during constant-evaluation
+static_assert(wrap_align_down(1, const_value(-1)), ""); // both-error{{not an integral constant expression}}
+// both-note@-1{{in call to}}
+static_assert(wrap_align_up(1, const_value(-2)), ""); // both-error{{not an integral constant expression}}
+// both-note@-1{{in call to}}
+static_assert(wrap_is_aligned(1, const_value(-3)), ""); // both-error{{not an integral constant expression}}
+// both-note@-1{{in call to}}
+static_assert(wrap_align_down(1, const_value(17)), ""); // both-error{{not an integral constant expression}}
+// both-note@-1{{in call to}}
+static_assert(wrap_align_up(1, const_value(18)), ""); // both-error{{not an integral constant expression}}
+// both-note@-1{{in call to}}
+static_assert(wrap_is_aligned(1, const_value(19)), ""); // both-error{{not an integral constant expression}}
+// both-note@-1{{in call to}}
+
+// Check invalid values for smaller types:
+static_assert(wrap_align_down(static_cast<short>(1), const_value(1 << 20)), ""); // both-error{{not an integral constant expression}}
+// both-note@-1{{in call to }}
+// Check invalid boolean type
+static_assert(wrap_align_up(static_cast<int>(1), const_value(1ull << 33)), ""); // both-error{{not an integral constant expression}}
+// both-note@-1{{in call to}}
+static_assert(wrap_is_aligned(static_cast<char>(1), const_value(1 << 22)), ""); // both-error{{not an integral constant expression}}
+// both-note@-1{{in call to}}
+
+// Check invalid boolean type
+static_assert(wrap_align_up(static_cast<bool>(1), const_value(1 << 21)), ""); // both-error{{not an integral constant expression}}
+// both-note@-1{{in instantiation of function template specialization 'wrap_align_up<bool>' requested here}}
+
+// Check constant evaluation for pointers:
+_Alignas(32) char align32array[128] = {};
+static_assert(&align32array[0] == &align32array[0], "");
+// __builtin_align_up/down can be constant evaluated as a no-op for values
+// that are known to have greater alignment:
+static_assert(__builtin_align_up(&align32array[0], 32) == &align32array[0], "");
+static_assert(__builtin_align_up(&align32array[0], 4) == &align32array[0], "");
+static_assert(__builtin_align_down(&align32array[0], 4) == __builtin_align_up(&align32array[0], 8), "");
+// But it can not be evaluated if the alignment is greater than the minimum
+// known alignment, since in that case the value might be the same if it happens
+// to actually be aligned to 64 bytes at run time.
+static_assert(&align32array[0] == __builtin_align_up(&align32array[0], 64), ""); // both-error{{not an integral constant expression}}
+// both-note@-1{{cannot constant evaluate the result of adjusting alignment to 64}}
+static_assert(__builtin_align_up(&align32array[0], 64) == __builtin_align_up(&align32array[0], 64), ""); // both-error{{not an integral constant expression}}
+// both-note@-1{{cannot constant evaluate the result of adjusting alignment to 64}}
+
+// However, we can compute in case the requested alignment is less than the
+// base alignment:
+static_assert(__builtin_align_up(&align32array[0], 4) == &align32array[0], "");
+static_assert(__builtin_align_up(&align32array[1], 4) == &align32array[4], "");
+static_assert(__builtin_align_up(&align32array[2], 4) == &align32array[4], "");
+static_assert(__builtin_align_up(&align32array[3], 4) == &align32array[4], "");
+static_assert(__builtin_align_up(&align32array[4], 4) == &align32array[4], "");
+static_assert(__builtin_align_up(&align32array[5], 4) == &align32array[8], "");
+static_assert(__builtin_align_up(&align32array[6], 4) == &align32array[8], "");
+static_assert(__builtin_align_up(&align32array[7], 4) == &align32array[8], "");
+static_assert(__builtin_align_up(&align32array[8], 4) == &align32array[8], "");
+
+static_assert(__builtin_align_down(&align32array[0], 4) == &align32array[0], "");
+static_assert(__builtin_align_down(&align32array[1], 4) == &align32array[0], "");
+static_assert(__builtin_align_down(&align32array[2], 4) == &align32array[0], "");
+static_assert(__builtin_align_down(&align32array[3], 4) == &align32array[0], "");
+static_assert(__builtin_align_down(&align32array[4], 4) == &align32array[4], "");
+static_assert(__builtin_align_down(&align32array[5], 4) == &align32array[4], "");
+static_assert(__builtin_align_down(&align32array[6], 4) == &align32array[4], "");
+static_assert(__builtin_align_down(&align32array[7], 4) == &align32array[4], "");
+static_assert(__builtin_align_down(&align32array[8], 4) == &align32array[8], "");
+
+// Achieving the same thing using casts to uintptr_t is not allowed:
+static_assert((char *)((__UINTPTR_TYPE__)&align32array[7] & ~3) == &align32array[4], ""); // both-error{{not an integral constant expression}} \
+                                                                                          // expected-note {{cast that performs the conversions of a reinterpret_cast is not allowed in a constant expression}}
+
+static_assert(__builtin_align_down(&align32array[1], 4) == &align32array[0], "");
+static_assert(__builtin_align_down(&align32array[1], 64) == &align32array[0], ""); // both-error{{not an integral constant expression}}
+// both-note@-1{{cannot constant evaluate the result of adjusting alignment to 64}}
+
+// Add some checks for __builtin_is_aligned:
+static_assert(__builtin_is_aligned(&align32array[0], 32), "");
+static_assert(__builtin_is_aligned(&align32array[4], 4), "");
+// We cannot constant evaluate whether the array is aligned to > 32 since this
+// may well be true at run time.
+static_assert(!__builtin_is_aligned(&align32array[0], 64), ""); // both-error{{not an integral constant expression}}
+// both-note@-1{{cannot constant evaluate whether run-time alignment is at least 64}}
+
+// However, if the alignment being checked is less than the minimum alignment of
+// the base object we can check the low bits of the alignment:
+static_assert(__builtin_is_aligned(&align32array[0], 4), "");
+static_assert(!__builtin_is_aligned(&align32array[1], 4), "");
+static_assert(!__builtin_is_aligned(&align32array[2], 4), "");
+static_assert(!__builtin_is_aligned(&align32array[3], 4), "");
+static_assert(__builtin_is_aligned(&align32array[4], 4), "");
+
+// TODO: this should evaluate to true even though we can't evaluate the result
+//  of __builtin_align_up() to a concrete value
+static_assert(__builtin_is_aligned(__builtin_align_up(&align32array[0], 64), 64), ""); // both-error{{not an integral constant expression}}
+// both-note@-1{{cannot constant evaluate the result of adjusting alignment to 64}}
+
+// Check different source and alignment type widths are handled correctly.
+static_assert(!__builtin_is_aligned(static_cast<signed long>(7), static_cast<signed short>(4)), "");
+static_assert(!__builtin_is_aligned(static_cast<signed short>(7), static_cast<signed long>(4)), "");
+// Also check signed -- unsigned mismatch.
+static_assert(!__builtin_is_aligned(static_cast<signed long>(7), static_cast<signed long>(4)), "");
+static_assert(!__builtin_is_aligned(static_cast<unsigned long>(7), static_cast<unsigned long>(4)), "");
+static_assert(!__builtin_is_aligned(static_cast<signed long>(7), static_cast<unsigned long>(4)), "");
+static_assert(!__builtin_is_aligned(static_cast<unsigned long>(7), static_cast<signed long>(4)), "");
+static_assert(!__builtin_is_aligned(static_cast<signed long>(7), static_cast<unsigned short>(4)), "");
+static_assert(!__builtin_is_aligned(static_cast<unsigned short>(7), static_cast<signed long>(4)), "");
+
+// Check the diagnostic message
+_Alignas(void) char align_void_array[1]; // both-error {{invalid application of '_Alignas' to an incomplete type 'void'}}
-- 
GitLab


From 91dd844aa499d69c7ff75bf3156e2e3593a88057 Mon Sep 17 00:00:00 2001
From: Brandon Wu <brandon.wu@sifive.com>
Date: Tue, 16 Apr 2024 19:59:36 +0800
Subject: [PATCH 388/464] Recommit [RISCV] RISCV vector calling convention
 (2/2) (#79096) (#87736)

Bug fix: Handle RVV return type in calling convention correctly.
Return values are handled in a same way as function arguments.
One thing to mention is that if a type can be broken down into
homogeneous
vector types, e.g. {<vscale x 4 x i32>, {<vscale x 4 x i32>, <vscale x 4
x i32>}},
it is considered as a vector tuple type and need to be handled by tuple
type rule.
---
 llvm/lib/CodeGen/TargetLoweringBase.cpp       |  12 +-
 .../Target/RISCV/GISel/RISCVCallLowering.cpp  |  57 ++--
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp   | 244 +++++++++++++++---
 llvm/lib/Target/RISCV/RISCVISelLowering.h     |  59 ++++-
 llvm/test/CodeGen/RISCV/rvv/calling-conv.ll   | 203 +++++++++++++++
 .../RISCV/rvv/vector-deinterleave-load.ll     |   6 +-
 .../CodeGen/RISCV/rvv/vector-deinterleave.ll  |  19 +-
 7 files changed, 515 insertions(+), 85 deletions(-)

diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index f64ded4f2cf9..6e7b67ded23c 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -1809,8 +1809,16 @@ void llvm::GetReturnInfo(CallingConv::ID CC, Type *ReturnType,
     else if (attr.hasRetAttr(Attribute::ZExt))
       Flags.setZExt();
 
-    for (unsigned i = 0; i < NumParts; ++i)
-      Outs.push_back(ISD::OutputArg(Flags, PartVT, VT, /*isfixed=*/true, 0, 0));
+    for (unsigned i = 0; i < NumParts; ++i) {
+      ISD::ArgFlagsTy OutFlags = Flags;
+      if (NumParts > 1 && i == 0)
+        OutFlags.setSplit();
+      else if (i == NumParts - 1 && i != 0)
+        OutFlags.setSplitEnd();
+
+      Outs.push_back(
+          ISD::OutputArg(OutFlags, PartVT, VT, /*isfixed=*/true, 0, 0));
+    }
   }
 }
 
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVCallLowering.cpp b/llvm/lib/Target/RISCV/GISel/RISCVCallLowering.cpp
index 45e19cdea300..c18892ac62f2 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVCallLowering.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVCallLowering.cpp
@@ -34,14 +34,15 @@ private:
   // Whether this is assigning args for a return.
   bool IsRet;
 
-  // true if assignArg has been called for a mask argument, false otherwise.
-  bool AssignedFirstMaskArg = false;
+  RVVArgDispatcher &RVVDispatcher;
 
 public:
   RISCVOutgoingValueAssigner(
-      RISCVTargetLowering::RISCVCCAssignFn *RISCVAssignFn_, bool IsRet)
+      RISCVTargetLowering::RISCVCCAssignFn *RISCVAssignFn_, bool IsRet,
+      RVVArgDispatcher &RVVDispatcher)
       : CallLowering::OutgoingValueAssigner(nullptr),
-        RISCVAssignFn(RISCVAssignFn_), IsRet(IsRet) {}
+        RISCVAssignFn(RISCVAssignFn_), IsRet(IsRet),
+        RVVDispatcher(RVVDispatcher) {}
 
   bool assignArg(unsigned ValNo, EVT OrigVT, MVT ValVT, MVT LocVT,
                  CCValAssign::LocInfo LocInfo,
@@ -51,16 +52,9 @@ public:
     const DataLayout &DL = MF.getDataLayout();
     const RISCVSubtarget &Subtarget = MF.getSubtarget<RISCVSubtarget>();
 
-    std::optional<unsigned> FirstMaskArgument;
-    if (Subtarget.hasVInstructions() && !AssignedFirstMaskArg &&
-        ValVT.isVector() && ValVT.getVectorElementType() == MVT::i1) {
-      FirstMaskArgument = ValNo;
-      AssignedFirstMaskArg = true;
-    }
-
     if (RISCVAssignFn(DL, Subtarget.getTargetABI(), ValNo, ValVT, LocVT,
                       LocInfo, Flags, State, Info.IsFixed, IsRet, Info.Ty,
-                      *Subtarget.getTargetLowering(), FirstMaskArgument))
+                      *Subtarget.getTargetLowering(), RVVDispatcher))
       return true;
 
     StackSize = State.getStackSize();
@@ -181,14 +175,15 @@ private:
   // Whether this is assigning args from a return.
   bool IsRet;
 
-  // true if assignArg has been called for a mask argument, false otherwise.
-  bool AssignedFirstMaskArg = false;
+  RVVArgDispatcher &RVVDispatcher;
 
 public:
   RISCVIncomingValueAssigner(
-      RISCVTargetLowering::RISCVCCAssignFn *RISCVAssignFn_, bool IsRet)
+      RISCVTargetLowering::RISCVCCAssignFn *RISCVAssignFn_, bool IsRet,
+      RVVArgDispatcher &RVVDispatcher)
       : CallLowering::IncomingValueAssigner(nullptr),
-        RISCVAssignFn(RISCVAssignFn_), IsRet(IsRet) {}
+        RISCVAssignFn(RISCVAssignFn_), IsRet(IsRet),
+        RVVDispatcher(RVVDispatcher) {}
 
   bool assignArg(unsigned ValNo, EVT OrigVT, MVT ValVT, MVT LocVT,
                  CCValAssign::LocInfo LocInfo,
@@ -201,16 +196,9 @@ public:
     if (LocVT.isScalableVector())
       MF.getInfo<RISCVMachineFunctionInfo>()->setIsVectorCall();
 
-    std::optional<unsigned> FirstMaskArgument;
-    if (Subtarget.hasVInstructions() && !AssignedFirstMaskArg &&
-        ValVT.isVector() && ValVT.getVectorElementType() == MVT::i1) {
-      FirstMaskArgument = ValNo;
-      AssignedFirstMaskArg = true;
-    }
-
     if (RISCVAssignFn(DL, Subtarget.getTargetABI(), ValNo, ValVT, LocVT,
                       LocInfo, Flags, State, /*IsFixed=*/true, IsRet, Info.Ty,
-                      *Subtarget.getTargetLowering(), FirstMaskArgument))
+                      *Subtarget.getTargetLowering(), RVVDispatcher))
       return true;
 
     StackSize = State.getStackSize();
@@ -420,9 +408,11 @@ bool RISCVCallLowering::lowerReturnVal(MachineIRBuilder &MIRBuilder,
   SmallVector<ArgInfo, 4> SplitRetInfos;
   splitToValueTypes(OrigRetInfo, SplitRetInfos, DL, CC);
 
+  RVVArgDispatcher Dispatcher{&MF, getTLI<RISCVTargetLowering>(),
+                              ArrayRef(F.getReturnType())};
   RISCVOutgoingValueAssigner Assigner(
       CC == CallingConv::Fast ? RISCV::CC_RISCV_FastCC : RISCV::CC_RISCV,
-      /*IsRet=*/true);
+      /*IsRet=*/true, Dispatcher);
   RISCVOutgoingValueHandler Handler(MIRBuilder, MF.getRegInfo(), Ret);
   return determineAndHandleAssignments(Handler, Assigner, SplitRetInfos,
                                        MIRBuilder, CC, F.isVarArg());
@@ -531,6 +521,7 @@ bool RISCVCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
   CallingConv::ID CC = F.getCallingConv();
 
   SmallVector<ArgInfo, 32> SplitArgInfos;
+  SmallVector<Type *, 4> TypeList;
   unsigned Index = 0;
   for (auto &Arg : F.args()) {
     // Construct the ArgInfo object from destination register and argument type.
@@ -542,12 +533,16 @@ bool RISCVCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
     // correspondingly and appended to SplitArgInfos.
     splitToValueTypes(AInfo, SplitArgInfos, DL, CC);
 
+    TypeList.push_back(Arg.getType());
+
     ++Index;
   }
 
+  RVVArgDispatcher Dispatcher{&MF, getTLI<RISCVTargetLowering>(),
+                              ArrayRef(TypeList)};
   RISCVIncomingValueAssigner Assigner(
       CC == CallingConv::Fast ? RISCV::CC_RISCV_FastCC : RISCV::CC_RISCV,
-      /*IsRet=*/false);
+      /*IsRet=*/false, Dispatcher);
   RISCVFormalArgHandler Handler(MIRBuilder, MF.getRegInfo());
 
   SmallVector<CCValAssign, 16> ArgLocs;
@@ -585,11 +580,13 @@ bool RISCVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
 
   SmallVector<ArgInfo, 32> SplitArgInfos;
   SmallVector<ISD::OutputArg, 8> Outs;
+  SmallVector<Type *, 4> TypeList;
   for (auto &AInfo : Info.OrigArgs) {
     // Handle any required unmerging of split value types from a given VReg into
     // physical registers. ArgInfo objects are constructed correspondingly and
     // appended to SplitArgInfos.
     splitToValueTypes(AInfo, SplitArgInfos, DL, CC);
+    TypeList.push_back(AInfo.Ty);
   }
 
   // TODO: Support tail calls.
@@ -607,9 +604,11 @@ bool RISCVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
   const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
   Call.addRegMask(TRI->getCallPreservedMask(MF, Info.CallConv));
 
+  RVVArgDispatcher ArgDispatcher{&MF, getTLI<RISCVTargetLowering>(),
+                                 ArrayRef(TypeList)};
   RISCVOutgoingValueAssigner ArgAssigner(
       CC == CallingConv::Fast ? RISCV::CC_RISCV_FastCC : RISCV::CC_RISCV,
-      /*IsRet=*/false);
+      /*IsRet=*/false, ArgDispatcher);
   RISCVOutgoingValueHandler ArgHandler(MIRBuilder, MF.getRegInfo(), Call);
   if (!determineAndHandleAssignments(ArgHandler, ArgAssigner, SplitArgInfos,
                                      MIRBuilder, CC, Info.IsVarArg))
@@ -637,9 +636,11 @@ bool RISCVCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
   SmallVector<ArgInfo, 4> SplitRetInfos;
   splitToValueTypes(Info.OrigRet, SplitRetInfos, DL, CC);
 
+  RVVArgDispatcher RetDispatcher{&MF, getTLI<RISCVTargetLowering>(),
+                                 ArrayRef(F.getReturnType())};
   RISCVIncomingValueAssigner RetAssigner(
       CC == CallingConv::Fast ? RISCV::CC_RISCV_FastCC : RISCV::CC_RISCV,
-      /*IsRet=*/true);
+      /*IsRet=*/true, RetDispatcher);
   RISCVCallReturnHandler RetHandler(MIRBuilder, MF.getRegInfo(), Call);
   if (!determineAndHandleAssignments(RetHandler, RetAssigner, SplitRetInfos,
                                      MIRBuilder, CC, Info.IsVarArg))
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 1d1ea6bae6c1..765838aafb58 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -22,6 +22,7 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/MemoryLocation.h"
 #include "llvm/Analysis/VectorUtils.h"
+#include "llvm/CodeGen/Analysis.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
@@ -18223,33 +18224,12 @@ static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1,
   return false;
 }
 
-static unsigned allocateRVVReg(MVT ValVT, unsigned ValNo,
-                               std::optional<unsigned> FirstMaskArgument,
-                               CCState &State, const RISCVTargetLowering &TLI) {
-  const TargetRegisterClass *RC = TLI.getRegClassFor(ValVT);
-  if (RC == &RISCV::VRRegClass) {
-    // Assign the first mask argument to V0.
-    // This is an interim calling convention and it may be changed in the
-    // future.
-    if (FirstMaskArgument && ValNo == *FirstMaskArgument)
-      return State.AllocateReg(RISCV::V0);
-    return State.AllocateReg(ArgVRs);
-  }
-  if (RC == &RISCV::VRM2RegClass)
-    return State.AllocateReg(ArgVRM2s);
-  if (RC == &RISCV::VRM4RegClass)
-    return State.AllocateReg(ArgVRM4s);
-  if (RC == &RISCV::VRM8RegClass)
-    return State.AllocateReg(ArgVRM8s);
-  llvm_unreachable("Unhandled register class for ValueType");
-}
-
 // Implements the RISC-V calling convention. Returns true upon failure.
 bool RISCV::CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo,
                      MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo,
                      ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed,
                      bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI,
-                     std::optional<unsigned> FirstMaskArgument) {
+                     RVVArgDispatcher &RVVDispatcher) {
   unsigned XLen = DL.getLargestLegalIntTypeSizeInBits();
   assert(XLen == 32 || XLen == 64);
   MVT XLenVT = XLen == 32 ? MVT::i32 : MVT::i64;
@@ -18418,7 +18398,7 @@ bool RISCV::CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo,
   else if (ValVT == MVT::f64 && !UseGPRForF64)
     Reg = State.AllocateReg(ArgFPR64s);
   else if (ValVT.isVector()) {
-    Reg = allocateRVVReg(ValVT, ValNo, FirstMaskArgument, State, TLI);
+    Reg = RVVDispatcher.getNextPhysReg();
     if (!Reg) {
       // For return values, the vector must be passed fully via registers or
       // via the stack.
@@ -18504,9 +18484,15 @@ void RISCVTargetLowering::analyzeInputArgs(
   unsigned NumArgs = Ins.size();
   FunctionType *FType = MF.getFunction().getFunctionType();
 
-  std::optional<unsigned> FirstMaskArgument;
-  if (Subtarget.hasVInstructions())
-    FirstMaskArgument = preAssignMask(Ins);
+  RVVArgDispatcher Dispatcher;
+  if (IsRet) {
+    Dispatcher = RVVArgDispatcher{&MF, this, ArrayRef(Ins)};
+  } else {
+    SmallVector<Type *, 4> TypeList;
+    for (const Argument &Arg : MF.getFunction().args())
+      TypeList.push_back(Arg.getType());
+    Dispatcher = RVVArgDispatcher{&MF, this, ArrayRef(TypeList)};
+  }
 
   for (unsigned i = 0; i != NumArgs; ++i) {
     MVT ArgVT = Ins[i].VT;
@@ -18521,7 +18507,7 @@ void RISCVTargetLowering::analyzeInputArgs(
     RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
     if (Fn(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full,
            ArgFlags, CCInfo, /*IsFixed=*/true, IsRet, ArgTy, *this,
-           FirstMaskArgument)) {
+           Dispatcher)) {
       LLVM_DEBUG(dbgs() << "InputArg #" << i << " has unhandled type "
                         << ArgVT << '\n');
       llvm_unreachable(nullptr);
@@ -18535,9 +18521,13 @@ void RISCVTargetLowering::analyzeOutputArgs(
     CallLoweringInfo *CLI, RISCVCCAssignFn Fn) const {
   unsigned NumArgs = Outs.size();
 
-  std::optional<unsigned> FirstMaskArgument;
-  if (Subtarget.hasVInstructions())
-    FirstMaskArgument = preAssignMask(Outs);
+  SmallVector<Type *, 4> TypeList;
+  if (IsRet)
+    TypeList.push_back(MF.getFunction().getReturnType());
+  else if (CLI)
+    for (const TargetLowering::ArgListEntry &Arg : CLI->getArgs())
+      TypeList.push_back(Arg.Ty);
+  RVVArgDispatcher Dispatcher{&MF, this, ArrayRef(TypeList)};
 
   for (unsigned i = 0; i != NumArgs; i++) {
     MVT ArgVT = Outs[i].VT;
@@ -18547,7 +18537,7 @@ void RISCVTargetLowering::analyzeOutputArgs(
     RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
     if (Fn(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full,
            ArgFlags, CCInfo, Outs[i].IsFixed, IsRet, OrigTy, *this,
-           FirstMaskArgument)) {
+           Dispatcher)) {
       LLVM_DEBUG(dbgs() << "OutputArg #" << i << " has unhandled type "
                         << ArgVT << "\n");
       llvm_unreachable(nullptr);
@@ -18728,7 +18718,7 @@ bool RISCV::CC_RISCV_FastCC(const DataLayout &DL, RISCVABI::ABI ABI,
                             ISD::ArgFlagsTy ArgFlags, CCState &State,
                             bool IsFixed, bool IsRet, Type *OrigTy,
                             const RISCVTargetLowering &TLI,
-                            std::optional<unsigned> FirstMaskArgument) {
+                            RVVArgDispatcher &RVVDispatcher) {
   if (LocVT == MVT::i32 || LocVT == MVT::i64) {
     if (unsigned Reg = State.AllocateReg(getFastCCArgGPRs(ABI))) {
       State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
@@ -18806,13 +18796,14 @@ bool RISCV::CC_RISCV_FastCC(const DataLayout &DL, RISCVABI::ABI ABI,
   }
 
   if (LocVT.isVector()) {
-    if (unsigned Reg =
-            allocateRVVReg(ValVT, ValNo, FirstMaskArgument, State, TLI)) {
+    MCPhysReg AllocatedVReg = RVVDispatcher.getNextPhysReg();
+    if (AllocatedVReg) {
       // Fixed-length vectors are located in the corresponding scalable-vector
       // container types.
       if (ValVT.isFixedLengthVector())
         LocVT = TLI.getContainerForFixedLengthVector(LocVT);
-      State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
+      State.addLoc(
+          CCValAssign::getReg(ValNo, ValVT, AllocatedVReg, LocVT, LocInfo));
     } else {
       // Try and pass the address via a "fast" GPR.
       if (unsigned GPRReg = State.AllocateReg(getFastCCArgGPRs(ABI))) {
@@ -19440,17 +19431,15 @@ bool RISCVTargetLowering::CanLowerReturn(
   SmallVector<CCValAssign, 16> RVLocs;
   CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
 
-  std::optional<unsigned> FirstMaskArgument;
-  if (Subtarget.hasVInstructions())
-    FirstMaskArgument = preAssignMask(Outs);
+  RVVArgDispatcher Dispatcher{&MF, this, ArrayRef(Outs)};
 
   for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
     MVT VT = Outs[i].VT;
     ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
     RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
     if (RISCV::CC_RISCV(MF.getDataLayout(), ABI, i, VT, VT, CCValAssign::Full,
-                 ArgFlags, CCInfo, /*IsFixed=*/true, /*IsRet=*/true, nullptr,
-                 *this, FirstMaskArgument))
+                        ArgFlags, CCInfo, /*IsFixed=*/true, /*IsRet=*/true,
+                        nullptr, *this, Dispatcher))
       return false;
   }
   return true;
@@ -21247,6 +21236,181 @@ unsigned RISCVTargetLowering::getMinimumJumpTableEntries() const {
   return Subtarget.getMinimumJumpTableEntries();
 }
 
+// Handle single arg such as return value.
+template <typename Arg>
+void RVVArgDispatcher::constructArgInfos(ArrayRef<Arg> ArgList) {
+  // This lambda determines whether an array of types are constructed by
+  // homogeneous vector types.
+  auto isHomogeneousScalableVectorType = [](ArrayRef<Arg> ArgList) {
+    // First, extract the first element in the argument type.
+    auto It = ArgList.begin();
+    MVT FirstArgRegType = It->VT;
+
+    // Return if there is no return or the type needs split.
+    if (It == ArgList.end() || It->Flags.isSplit())
+      return false;
+
+    ++It;
+
+    // Return if this argument type contains only 1 element, or it's not a
+    // vector type.
+    if (It == ArgList.end() || !FirstArgRegType.isScalableVector())
+      return false;
+
+    // Second, check if the following elements in this argument type are all the
+    // same.
+    for (; It != ArgList.end(); ++It)
+      if (It->Flags.isSplit() || It->VT != FirstArgRegType)
+        return false;
+
+    return true;
+  };
+
+  if (isHomogeneousScalableVectorType(ArgList)) {
+    // Handle as tuple type
+    RVVArgInfos.push_back({(unsigned)ArgList.size(), ArgList[0].VT, false});
+  } else {
+    // Handle as normal vector type
+    bool FirstVMaskAssigned = false;
+    for (const auto &OutArg : ArgList) {
+      MVT RegisterVT = OutArg.VT;
+
+      // Skip non-RVV register type
+      if (!RegisterVT.isVector())
+        continue;
+
+      if (RegisterVT.isFixedLengthVector())
+        RegisterVT = TLI->getContainerForFixedLengthVector(RegisterVT);
+
+      if (!FirstVMaskAssigned && RegisterVT.getVectorElementType() == MVT::i1) {
+        RVVArgInfos.push_back({1, RegisterVT, true});
+        FirstVMaskAssigned = true;
+        continue;
+      }
+
+      RVVArgInfos.push_back({1, RegisterVT, false});
+    }
+  }
+}
+
+// Handle multiple args.
+template <>
+void RVVArgDispatcher::constructArgInfos<Type *>(ArrayRef<Type *> TypeList) {
+  const DataLayout &DL = MF->getDataLayout();
+  const Function &F = MF->getFunction();
+  LLVMContext &Context = F.getContext();
+
+  bool FirstVMaskAssigned = false;
+  for (Type *Ty : TypeList) {
+    StructType *STy = dyn_cast<StructType>(Ty);
+    if (STy && STy->containsHomogeneousScalableVectorTypes()) {
+      Type *ElemTy = STy->getTypeAtIndex(0U);
+      EVT VT = TLI->getValueType(DL, ElemTy);
+      MVT RegisterVT =
+          TLI->getRegisterTypeForCallingConv(Context, F.getCallingConv(), VT);
+      unsigned NumRegs =
+          TLI->getNumRegistersForCallingConv(Context, F.getCallingConv(), VT);
+
+      RVVArgInfos.push_back(
+          {NumRegs * STy->getNumElements(), RegisterVT, false});
+    } else {
+      SmallVector<EVT, 4> ValueVTs;
+      ComputeValueVTs(*TLI, DL, Ty, ValueVTs);
+
+      for (unsigned Value = 0, NumValues = ValueVTs.size(); Value != NumValues;
+           ++Value) {
+        EVT VT = ValueVTs[Value];
+        MVT RegisterVT =
+            TLI->getRegisterTypeForCallingConv(Context, F.getCallingConv(), VT);
+        unsigned NumRegs =
+            TLI->getNumRegistersForCallingConv(Context, F.getCallingConv(), VT);
+
+        // Skip non-RVV register type
+        if (!RegisterVT.isVector())
+          continue;
+
+        if (RegisterVT.isFixedLengthVector())
+          RegisterVT = TLI->getContainerForFixedLengthVector(RegisterVT);
+
+        if (!FirstVMaskAssigned &&
+            RegisterVT.getVectorElementType() == MVT::i1) {
+          RVVArgInfos.push_back({1, RegisterVT, true});
+          FirstVMaskAssigned = true;
+          --NumRegs;
+        }
+
+        RVVArgInfos.insert(RVVArgInfos.end(), NumRegs, {1, RegisterVT, false});
+      }
+    }
+  }
+}
+
+void RVVArgDispatcher::allocatePhysReg(unsigned NF, unsigned LMul,
+                                       unsigned StartReg) {
+  assert((StartReg % LMul) == 0 &&
+         "Start register number should be multiple of lmul");
+  const MCPhysReg *VRArrays;
+  switch (LMul) {
+  default:
+    report_fatal_error("Invalid lmul");
+  case 1:
+    VRArrays = ArgVRs;
+    break;
+  case 2:
+    VRArrays = ArgVRM2s;
+    break;
+  case 4:
+    VRArrays = ArgVRM4s;
+    break;
+  case 8:
+    VRArrays = ArgVRM8s;
+    break;
+  }
+
+  for (unsigned i = 0; i < NF; ++i)
+    if (StartReg)
+      AllocatedPhysRegs.push_back(VRArrays[(StartReg - 8) / LMul + i]);
+    else
+      AllocatedPhysRegs.push_back(MCPhysReg());
+}
+
+/// This function determines if each RVV argument is passed by register, if the
+/// argument can be assigned to a VR, then give it a specific register.
+/// Otherwise, assign the argument to 0 which is a invalid MCPhysReg.
+void RVVArgDispatcher::compute() {
+  uint32_t AssignedMap = 0;
+  auto allocate = [&](const RVVArgInfo &ArgInfo) {
+    // Allocate first vector mask argument to V0.
+    if (ArgInfo.FirstVMask) {
+      AllocatedPhysRegs.push_back(RISCV::V0);
+      return;
+    }
+
+    unsigned RegsNeeded = divideCeil(
+        ArgInfo.VT.getSizeInBits().getKnownMinValue(), RISCV::RVVBitsPerBlock);
+    unsigned TotalRegsNeeded = ArgInfo.NF * RegsNeeded;
+    for (unsigned StartReg = 0; StartReg + TotalRegsNeeded <= NumArgVRs;
+         StartReg += RegsNeeded) {
+      uint32_t Map = ((1 << TotalRegsNeeded) - 1) << StartReg;
+      if ((AssignedMap & Map) == 0) {
+        allocatePhysReg(ArgInfo.NF, RegsNeeded, StartReg + 8);
+        AssignedMap |= Map;
+        return;
+      }
+    }
+
+    allocatePhysReg(ArgInfo.NF, RegsNeeded, 0);
+  };
+
+  for (unsigned i = 0; i < RVVArgInfos.size(); ++i)
+    allocate(RVVArgInfos[i]);
+}
+
+MCPhysReg RVVArgDispatcher::getNextPhysReg() {
+  assert(CurIdx < AllocatedPhysRegs.size() && "Index out of range");
+  return AllocatedPhysRegs[CurIdx++];
+}
+
 namespace llvm::RISCVVIntrinsicsTable {
 
 #define GET_RISCVVIntrinsicsTable_IMPL
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index e2633733c31b..b10da3d40bef 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -24,6 +24,7 @@ namespace llvm {
 class InstructionCost;
 class RISCVSubtarget;
 struct RISCVRegisterInfo;
+class RVVArgDispatcher;
 
 namespace RISCVISD {
 // clang-format off
@@ -875,7 +876,7 @@ public:
                                ISD::ArgFlagsTy ArgFlags, CCState &State,
                                bool IsFixed, bool IsRet, Type *OrigTy,
                                const RISCVTargetLowering &TLI,
-                               std::optional<unsigned> FirstMaskArgument);
+                               RVVArgDispatcher &RVVDispatcher);
 
 private:
   void analyzeInputArgs(MachineFunction &MF, CCState &CCInfo,
@@ -1017,19 +1018,71 @@ private:
   unsigned getMinimumJumpTableEntries() const override;
 };
 
+/// As per the spec, the rules for passing vector arguments are as follows:
+///
+/// 1. For the first vector mask argument, use v0 to pass it.
+/// 2. For vector data arguments or rest vector mask arguments, starting from
+/// the v8 register, if a vector register group between v8-v23 that has not been
+/// allocated can be found and the first register number is a multiple of LMUL,
+/// then allocate this vector register group to the argument and mark these
+/// registers as allocated. Otherwise, pass it by reference and are replaced in
+/// the argument list with the address.
+/// 3. For tuple vector data arguments, starting from the v8 register, if
+/// NFIELDS consecutive vector register groups between v8-v23 that have not been
+/// allocated can be found and the first register number is a multiple of LMUL,
+/// then allocate these vector register groups to the argument and mark these
+/// registers as allocated. Otherwise, pass it by reference and are replaced in
+/// the argument list with the address.
+class RVVArgDispatcher {
+public:
+  static constexpr unsigned NumArgVRs = 16;
+
+  struct RVVArgInfo {
+    unsigned NF;
+    MVT VT;
+    bool FirstVMask = false;
+  };
+
+  template <typename Arg>
+  RVVArgDispatcher(const MachineFunction *MF, const RISCVTargetLowering *TLI,
+                   ArrayRef<Arg> ArgList)
+      : MF(MF), TLI(TLI) {
+    constructArgInfos(ArgList);
+    compute();
+  }
+
+  RVVArgDispatcher() = default;
+
+  MCPhysReg getNextPhysReg();
+
+private:
+  SmallVector<RVVArgInfo, 4> RVVArgInfos;
+  SmallVector<MCPhysReg, 4> AllocatedPhysRegs;
+
+  const MachineFunction *MF = nullptr;
+  const RISCVTargetLowering *TLI = nullptr;
+
+  unsigned CurIdx = 0;
+
+  template <typename Arg> void constructArgInfos(ArrayRef<Arg> Ret);
+  void compute();
+  void allocatePhysReg(unsigned NF = 1, unsigned LMul = 1,
+                       unsigned StartReg = 0);
+};
+
 namespace RISCV {
 
 bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo,
               MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo,
               ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed,
               bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI,
-              std::optional<unsigned> FirstMaskArgument);
+              RVVArgDispatcher &RVVDispatcher);
 
 bool CC_RISCV_FastCC(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo,
                      MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo,
                      ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed,
                      bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI,
-                     std::optional<unsigned> FirstMaskArgument);
+                     RVVArgDispatcher &RVVDispatcher);
 
 bool CC_RISCV_GHC(unsigned ValNo, MVT ValVT, MVT LocVT,
                   CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
diff --git a/llvm/test/CodeGen/RISCV/rvv/calling-conv.ll b/llvm/test/CodeGen/RISCV/rvv/calling-conv.ll
index 78e8700a9fef..647d3158b616 100644
--- a/llvm/test/CodeGen/RISCV/rvv/calling-conv.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/calling-conv.ll
@@ -162,3 +162,206 @@ define void @caller_tuple_argument({<vscale x 4 x i32>, <vscale x 4 x i32>} %x)
 }
 
 declare void @callee_tuple_argument({<vscale x 4 x i32>, <vscale x 4 x i32>})
+
+; %0 -> v8
+; %1 -> v9
+define <vscale x 1 x i64> @case1(<vscale x 1 x i64> %0, <vscale x 1 x i64> %1) {
+; CHECK-LABEL: case1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v8, v9
+; CHECK-NEXT:    ret
+  %a = add <vscale x 1 x i64> %0, %1
+  ret <vscale x 1 x i64> %a
+}
+
+; %0 -> v8
+; %1 -> v10-v11
+; %2 -> v9
+define <vscale x 1 x i64> @case2_1(<vscale x 1 x i64> %0, <vscale x 2 x i64> %1, <vscale x 1 x i64> %2) {
+; CHECK-LABEL: case2_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v8, v9
+; CHECK-NEXT:    ret
+  %a = add <vscale x 1 x i64> %0, %2
+  ret <vscale x 1 x i64> %a
+}
+define <vscale x 2 x i64> @case2_2(<vscale x 1 x i64> %0, <vscale x 2 x i64> %1, <vscale x 1 x i64> %2) {
+; CHECK-LABEL: case2_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v10, v10
+; CHECK-NEXT:    ret
+  %a = add <vscale x 2 x i64> %1, %1
+  ret <vscale x 2 x i64> %a
+}
+
+; %0 -> v8
+; %1 -> {v10-v11, v12-v13}
+; %2 -> v9
+define <vscale x 1 x i64> @case3_1(<vscale x 1 x i64> %0, {<vscale x 2 x i64>, <vscale x 2 x i64>} %1, <vscale x 1 x i64> %2) {
+; CHECK-LABEL: case3_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v8, v9
+; CHECK-NEXT:    ret
+  %add = add <vscale x 1 x i64> %0, %2
+  ret <vscale x 1 x i64> %add
+}
+define <vscale x 2 x i64> @case3_2(<vscale x 1 x i64> %0, {<vscale x 2 x i64>, <vscale x 2 x i64>} %1, <vscale x 1 x i64> %2) {
+; CHECK-LABEL: case3_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v10, v12
+; CHECK-NEXT:    ret
+  %a = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } %1, 0
+  %b = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } %1, 1
+  %add = add <vscale x 2 x i64> %a, %b
+  ret <vscale x 2 x i64> %add
+}
+
+; %0 -> v8
+; %1 -> {by-ref, by-ref}
+; %2 -> v9
+define <vscale x 8 x i64> @case4_1(<vscale x 1 x i64> %0, {<vscale x 8 x i64>, <vscale x 8 x i64>} %1, <vscale x 1 x i64> %2) {
+; CHECK-LABEL: case4_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    csrr a1, vlenb
+; CHECK-NEXT:    slli a1, a1, 3
+; CHECK-NEXT:    add a1, a0, a1
+; CHECK-NEXT:    vl8re64.v v8, (a1)
+; CHECK-NEXT:    vl8re64.v v16, (a0)
+; CHECK-NEXT:    vsetvli a0, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v16, v8
+; CHECK-NEXT:    ret
+  %a = extractvalue { <vscale x 8 x i64>, <vscale x 8 x i64> } %1, 0
+  %b = extractvalue { <vscale x 8 x i64>, <vscale x 8 x i64> } %1, 1
+  %add = add <vscale x 8 x i64> %a, %b
+  ret <vscale x 8 x i64> %add
+}
+define <vscale x 1 x i64> @case4_2(<vscale x 1 x i64> %0, {<vscale x 8 x i64>, <vscale x 8 x i64>} %1, <vscale x 1 x i64> %2) {
+; CHECK-LABEL: case4_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v8, v9
+; CHECK-NEXT:    ret
+  %add = add <vscale x 1 x i64> %0, %2
+  ret <vscale x 1 x i64> %add
+}
+
+declare <vscale x 1 x i64> @callee1()
+declare void @callee2(<vscale x 1 x i64>)
+declare void @callee3(<vscale x 4 x i32>)
+define void @caller() {
+; RV32-LABEL: caller:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    .cfi_def_cfa_offset 16
+; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT:    .cfi_offset ra, -4
+; RV32-NEXT:    call callee1
+; RV32-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; RV32-NEXT:    vadd.vv v8, v8, v8
+; RV32-NEXT:    call callee2
+; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: caller:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi sp, sp, -16
+; RV64-NEXT:    .cfi_def_cfa_offset 16
+; RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT:    .cfi_offset ra, -8
+; RV64-NEXT:    call callee1
+; RV64-NEXT:    vsetvli a0, zero, e64, m1, ta, ma
+; RV64-NEXT:    vadd.vv v8, v8, v8
+; RV64-NEXT:    call callee2
+; RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64-NEXT:    addi sp, sp, 16
+; RV64-NEXT:    ret
+  %a = call <vscale x 1 x i64> @callee1()
+  %add = add <vscale x 1 x i64> %a, %a
+  call void @callee2(<vscale x 1 x i64> %add)
+  ret void
+}
+
+declare {<vscale x 4 x i32>, <vscale x 4 x i32>} @callee_tuple()
+define void @caller_tuple() {
+; RV32-LABEL: caller_tuple:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    .cfi_def_cfa_offset 16
+; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT:    .cfi_offset ra, -4
+; RV32-NEXT:    call callee_tuple
+; RV32-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; RV32-NEXT:    vadd.vv v8, v8, v10
+; RV32-NEXT:    call callee3
+; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: caller_tuple:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi sp, sp, -16
+; RV64-NEXT:    .cfi_def_cfa_offset 16
+; RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT:    .cfi_offset ra, -8
+; RV64-NEXT:    call callee_tuple
+; RV64-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; RV64-NEXT:    vadd.vv v8, v8, v10
+; RV64-NEXT:    call callee3
+; RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64-NEXT:    addi sp, sp, 16
+; RV64-NEXT:    ret
+  %a = call {<vscale x 4 x i32>, <vscale x 4 x i32>} @callee_tuple()
+  %b = extractvalue {<vscale x 4 x i32>, <vscale x 4 x i32>} %a, 0
+  %c = extractvalue {<vscale x 4 x i32>, <vscale x 4 x i32>} %a, 1
+  %add = add <vscale x 4 x i32> %b, %c
+  call void @callee3(<vscale x 4 x i32> %add)
+  ret void
+}
+
+declare {<vscale x 4 x i32>, {<vscale x 4 x i32>, <vscale x 4 x i32>}} @callee_nested()
+define void @caller_nested() {
+; RV32-LABEL: caller_nested:
+; RV32:       # %bb.0:
+; RV32-NEXT:    addi sp, sp, -16
+; RV32-NEXT:    .cfi_def_cfa_offset 16
+; RV32-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT:    .cfi_offset ra, -4
+; RV32-NEXT:    call callee_nested
+; RV32-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; RV32-NEXT:    vadd.vv v8, v8, v10
+; RV32-NEXT:    vadd.vv v8, v8, v12
+; RV32-NEXT:    call callee3
+; RV32-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT:    addi sp, sp, 16
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: caller_nested:
+; RV64:       # %bb.0:
+; RV64-NEXT:    addi sp, sp, -16
+; RV64-NEXT:    .cfi_def_cfa_offset 16
+; RV64-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT:    .cfi_offset ra, -8
+; RV64-NEXT:    call callee_nested
+; RV64-NEXT:    vsetvli a0, zero, e32, m2, ta, ma
+; RV64-NEXT:    vadd.vv v8, v8, v10
+; RV64-NEXT:    vadd.vv v8, v8, v12
+; RV64-NEXT:    call callee3
+; RV64-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64-NEXT:    addi sp, sp, 16
+; RV64-NEXT:    ret
+  %a = call {<vscale x 4 x i32>, {<vscale x 4 x i32>, <vscale x 4 x i32>}} @callee_nested()
+  %b = extractvalue {<vscale x 4 x i32>, {<vscale x 4 x i32>, <vscale x 4 x i32>}} %a, 0
+  %c = extractvalue {<vscale x 4 x i32>, {<vscale x 4 x i32>, <vscale x 4 x i32>}} %a, 1
+  %c0 = extractvalue {<vscale x 4 x i32>, <vscale x 4 x i32>} %c, 0
+  %c1 = extractvalue {<vscale x 4 x i32>, <vscale x 4 x i32>} %c, 1
+  %add0 = add <vscale x 4 x i32> %b, %c0
+  %add1 = add <vscale x 4 x i32> %add0, %c1
+  call void @callee3(<vscale x 4 x i32> %add1)
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll
index a320aecc6fce..6a712080fda7 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-load.ll
@@ -18,10 +18,10 @@ define {<vscale x 16 x i1>, <vscale x 16 x i1>} @vector_deinterleave_load_nxv16i
 ; CHECK-NEXT:    vmerge.vim v14, v10, 1, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    vmerge.vim v12, v10, 1, v0
-; CHECK-NEXT:    vnsrl.wi v8, v12, 0
-; CHECK-NEXT:    vmsne.vi v0, v8, 0
-; CHECK-NEXT:    vnsrl.wi v10, v12, 8
+; CHECK-NEXT:    vnsrl.wi v10, v12, 0
 ; CHECK-NEXT:    vmsne.vi v8, v10, 0
+; CHECK-NEXT:    vnsrl.wi v10, v12, 8
+; CHECK-NEXT:    vmsne.vi v9, v10, 0
 ; CHECK-NEXT:    ret
   %vec = load <vscale x 32 x i1>, ptr %p
   %retval = call {<vscale x 16 x i1>, <vscale x 16 x i1>} @llvm.experimental.vector.deinterleave2.nxv32i1(<vscale x 32 x i1> %vec)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll
index ef4baf34d23f..d98597fabcd9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll
@@ -8,18 +8,18 @@ define {<vscale x 16 x i1>, <vscale x 16 x i1>} @vector_deinterleave_nxv16i1_nxv
 ; CHECK-LABEL: vector_deinterleave_nxv16i1_nxv32i1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
-; CHECK-NEXT:    vmv.v.i v10, 0
-; CHECK-NEXT:    vmerge.vim v8, v10, 1, v0
+; CHECK-NEXT:    vmv.v.i v8, 0
+; CHECK-NEXT:    vmerge.vim v12, v8, 1, v0
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 2
 ; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v0, v0, a0
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
-; CHECK-NEXT:    vmerge.vim v10, v10, 1, v0
-; CHECK-NEXT:    vnsrl.wi v12, v8, 0
-; CHECK-NEXT:    vmsne.vi v0, v12, 0
-; CHECK-NEXT:    vnsrl.wi v12, v8, 8
-; CHECK-NEXT:    vmsne.vi v8, v12, 0
+; CHECK-NEXT:    vmerge.vim v14, v8, 1, v0
+; CHECK-NEXT:    vnsrl.wi v10, v12, 0
+; CHECK-NEXT:    vmsne.vi v8, v10, 0
+; CHECK-NEXT:    vnsrl.wi v10, v12, 8
+; CHECK-NEXT:    vmsne.vi v9, v10, 0
 ; CHECK-NEXT:    ret
 %retval = call {<vscale x 16 x i1>, <vscale x 16 x i1>} @llvm.experimental.vector.deinterleave2.nxv32i1(<vscale x 32 x i1> %vec)
 ret {<vscale x 16 x i1>, <vscale x 16 x i1>} %retval
@@ -102,12 +102,13 @@ define {<vscale x 64 x i1>, <vscale x 64 x i1>} @vector_deinterleave_nxv64i1_nxv
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m4, ta, ma
 ; CHECK-NEXT:    vnsrl.wi v28, v8, 0
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
-; CHECK-NEXT:    vmsne.vi v0, v24, 0
+; CHECK-NEXT:    vmsne.vi v7, v24, 0
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m4, ta, ma
 ; CHECK-NEXT:    vnsrl.wi v24, v16, 8
 ; CHECK-NEXT:    vnsrl.wi v28, v8, 8
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m8, ta, ma
-; CHECK-NEXT:    vmsne.vi v8, v24, 0
+; CHECK-NEXT:    vmsne.vi v9, v24, 0
+; CHECK-NEXT:    vmv1r.v v8, v7
 ; CHECK-NEXT:    ret
 %retval = call {<vscale x 64 x i1>, <vscale x 64 x i1>} @llvm.experimental.vector.deinterleave2.nxv128i1(<vscale x 128 x i1> %vec)
 ret {<vscale x 64 x i1>, <vscale x 64 x i1>} %retval
-- 
GitLab


From ac1f2de7b581c26a768c4d2a2aad36505cc63c31 Mon Sep 17 00:00:00 2001
From: Frederik Harwath <frederik.harwath@amd.com>
Date: Tue, 16 Apr 2024 14:09:25 +0200
Subject: [PATCH 389/464] [MLIR][docs] Mention declarePromisedInterface in
 Interfaces doc (#88689)

Co-authored-by: Frederik Harwath <fharwath@amd.com>
Co-authored-by: Mehdi Amini <joker.eph@gmail.com>
---
 mlir/docs/Interfaces.md | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/mlir/docs/Interfaces.md b/mlir/docs/Interfaces.md
index 536e7613e509..51747db546bb 100644
--- a/mlir/docs/Interfaces.md
+++ b/mlir/docs/Interfaces.md
@@ -299,6 +299,30 @@ owner of the dialect containing the object nor the owner of the interface are
 aware of an interface implementation, which can lead to duplicate or
 diverging implementations.
 
+Forgetting to register an external model can lead to bugs which are hard to
+track down. The `declarePromisedInterface` function can be used to declare that
+an external model implementation for an operation must eventually be provided.
+
+```
+  void MyDialect::initialize() {
+    declarePromisedInterface<SomeInterface, SomeOp>();
+     ...
+  }
+```
+
+Now attempting to use the interface, e.g in a cast, without a prior registration
+of the external model will lead to a runtime error that will look similar to
+this:
+
+```
+LLVM ERROR: checking for an interface (`SomeInterface`) that was promised by dialect 'mydialect' but never implemented. This is generally an indication that the dialect extension implementing the interface was never registered.
+```
+
+If you encounter this error for a dialect and an interface provided by MLIR, you
+may look for a method that will be named like
+`register<Dialect><Interface>ExternalModels(DialectRegistry &registry);` ; try
+to find it with `git grep 'register.*SomeInterface.*Model' mlir`.
+
 #### Dialect Fallback for OpInterface
 
 Some dialects have an open ecosystem and don't register all of the possible
-- 
GitLab


From 1c076b43c294c7d29d99dd50f6853b33a5b99789 Mon Sep 17 00:00:00 2001
From: Spenser Bauman <sbauman@mathworks.com>
Date: Tue, 16 Apr 2024 08:14:22 -0400
Subject: [PATCH 390/464] [mlir][tosa] Implement dynamic shape support for
 tosa.max_pool2d lowering (#87538)

The existing lowering for tosa.max_pool2d only supports dynamic
dimensions when the dynamic dimension is the batch dimension. This
change updates the lowering to support arbitrary dynamic dimensions on
the inputs and outputs of the tosa.max_pool2d operation.

This change also fixes a bug in the implementation of implicit
broadcasting in the tosa-to-linalg pass, which was introducing uses of
constant ops that violated dominance requirements.
---
 .../mlir/Dialect/Tosa/IR/TosaTypesBase.td     |  10 +-
 .../Conversion/TosaToLinalg/TosaToLinalg.cpp  |  12 +-
 .../TosaToLinalg/TosaToLinalgNamed.cpp        |  88 ++++++++++----
 .../TosaToLinalg/tosa-to-linalg-named.mlir    |  54 +++++++++
 .../TosaToLinalg/tosa-to-linalg.mlir          |  12 +-
 .../Tosa/CPU/test-maxpool-dynamic.mlir        | 112 ++++++++++++++++++
 6 files changed, 251 insertions(+), 37 deletions(-)
 create mode 100644 mlir/test/Integration/Dialect/Tosa/CPU/test-maxpool-dynamic.mlir

diff --git a/mlir/include/mlir/Dialect/Tosa/IR/TosaTypesBase.td b/mlir/include/mlir/Dialect/Tosa/IR/TosaTypesBase.td
index cff3de0a69af..3687891fe4b7 100644
--- a/mlir/include/mlir/Dialect/Tosa/IR/TosaTypesBase.td
+++ b/mlir/include/mlir/Dialect/Tosa/IR/TosaTypesBase.td
@@ -130,11 +130,11 @@ def Tosa_ScalarTensor : TensorRankOf<[Tosa_AnyNumber], [0]>;
 // to not include any remaining unranked tensors.
 def Tosa_UnrankedTensor : UnrankedTensorOf<[Tosa_AnyNumber]>;
 
-def Tosa_Tensor1D : AnyTypeOf<[Tosa_UnrankedTensor, 1DTensorOf<[Tosa_AnyNumber]>]>;
-def Tosa_Tensor2D : AnyTypeOf<[Tosa_UnrankedTensor, 2DTensorOf<[Tosa_AnyNumber]>]>;
-def Tosa_Tensor3D : AnyTypeOf<[Tosa_UnrankedTensor, 3DTensorOf<[Tosa_AnyNumber]>]>;
-def Tosa_Tensor4D : AnyTypeOf<[Tosa_UnrankedTensor, 4DTensorOf<[Tosa_AnyNumber]>]>;
-def Tosa_Tensor5D : AnyTypeOf<[Tosa_UnrankedTensor, TensorRankOf<[Tosa_AnyNumber], [5]>]>;
+def Tosa_Tensor1D : AnyTypeOf<[Tosa_UnrankedTensor, 1DTensorOf<[Tosa_AnyNumber]>], "1-d tensor", "::mlir::TensorType">;
+def Tosa_Tensor2D : AnyTypeOf<[Tosa_UnrankedTensor, 2DTensorOf<[Tosa_AnyNumber]>], "2-d tensor", "::mlir::TensorType">;
+def Tosa_Tensor3D : AnyTypeOf<[Tosa_UnrankedTensor, 3DTensorOf<[Tosa_AnyNumber]>], "3-d tensor", "::mlir::TensorType">;
+def Tosa_Tensor4D : AnyTypeOf<[Tosa_UnrankedTensor, 4DTensorOf<[Tosa_AnyNumber]>], "4-d tensor", "::mlir::TensorType">;
+def Tosa_Tensor5D : AnyTypeOf<[Tosa_UnrankedTensor, TensorRankOf<[Tosa_AnyNumber], [5]>], "5-d tensor", "::mlir::TensorType">;
 
 // Ranked tensors up to given rank.
 def Tosa_Tensor1Dto4D : AnyTypeOf<[
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
index 7c477f2e1412..d8dd1c93722b 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalg.cpp
@@ -766,11 +766,15 @@ static Value broadcastDynamicDimension(PatternRewriter &rewriter, Location loc,
 
   // Emit 'then' region of 'scf.if'
   auto emitThenRegion = [&](OpBuilder &opBuilder, Location loc) {
+    // It is not safe to cache constants across regions.
+    // New constants could potentially violate dominance requirements.
+    IndexPool localPool;
+
     // Emit 'tensor.empty' op
     SmallVector<OpFoldResult> outputTensorShape;
     for (auto index : llvm::seq<int64_t>(0, rank)) {
       auto size = index == dim ? targetSize
-                               : getOrFoldTensorDim(rewriter, loc, indexPool,
+                               : getOrFoldTensorDim(rewriter, loc, localPool,
                                                     operand, index);
       outputTensorShape.push_back(size);
     }
@@ -812,9 +816,9 @@ static Value broadcastDynamicDimensions(PatternRewriter &rewriter, Location loc,
                                         IndexPool &indexPool, Value operand,
                                         ArrayRef<OpFoldResult> targetShape,
                                         ArrayRef<Value> masterOperands) {
-  size_t rank = operand.getType().cast<RankedTensorType>().getRank();
-  assert(targetShape.size() == rank);
-  assert(masterOperands.size() == rank);
+  int64_t rank = operand.getType().cast<RankedTensorType>().getRank();
+  assert((int64_t)targetShape.size() == rank);
+  assert((int64_t)masterOperands.size() == rank);
   for (auto index : llvm::seq<int64_t>(0, rank))
     operand =
         broadcastDynamicDimension(rewriter, loc, indexPool, operand, index,
diff --git a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
index 3f39cbf03a9a..8fb8d1648656 100644
--- a/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
+++ b/mlir/lib/Conversion/TosaToLinalg/TosaToLinalgNamed.cpp
@@ -26,6 +26,8 @@
 #include "mlir/Transforms/DialectConversion.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
+#include "mlir/Interfaces/InferTypeOpInterface.h"
+
 #include <numeric>
 #include <type_traits>
 
@@ -34,7 +36,7 @@ using namespace mlir::tosa;
 
 static mlir::Value applyPad(Location loc, Value input, ArrayRef<int64_t> pad,
                             TypedAttr padAttr, OpBuilder &rewriter) {
-  // Input should be padded if necessary.
+  // Input should be padded only if necessary.
   if (llvm::all_of(pad, [](int64_t p) { return p == 0; }))
     return input;
 
@@ -47,7 +49,7 @@ static mlir::Value applyPad(Location loc, Value input, ArrayRef<int64_t> pad,
   SmallVector<int64_t, 4> paddedShape;
   SmallVector<OpFoldResult, 8> lowIndices;
   SmallVector<OpFoldResult, 8> highIndices;
-  for (int i = 0, s = inputShape.size(); i < s; i++) {
+  for (size_t i : llvm::seq(inputShape.size())) {
     auto lowPad = pad[i * 2];
     auto highPad = pad[i * 2 + 1];
     if (ShapedType::isDynamic(inputShape[i]))
@@ -131,20 +133,19 @@ static mlir::Value linalgBroadcastAndMaybeExtSI(PatternRewriter &rewriter,
 
 static mlir::Value reifyConstantDim(int64_t attr,
                                     ImplicitLocOpBuilder &builder) {
-  return builder.createOrFold<arith::IndexCastOp>(
-      builder.getIndexType(),
-      builder.create<arith::ConstantOp>(builder.getI64IntegerAttr(attr)));
+  return builder.create<arith::ConstantIndexOp>(attr);
 }
 
 // Calculating the output width/height using the formula:
 // H = ((IH+pad_top+pad_bottom-(dilation_y*(KH-1)+1))/stride_y)+1
 // W = ((IW+pad_left+pad_right-(dilation_x*(KW-1)+1))/stride_x)+1
 
-static mlir::Value getConvOutputDim(Location loc, Value inputDim,
-                                    int64_t padBeforeAttr, int64_t padAfterAttr,
-                                    Value kernelDim, int64_t strideAttr,
-                                    int64_t dilationAttr, Type inputETy,
-                                    OpBuilder &rewriter) {
+static mlir::Value getConvOrPoolOutputDim(Location loc, Value inputDim,
+                                          int64_t padBeforeAttr,
+                                          int64_t padAfterAttr, Value kernelDim,
+                                          int64_t strideAttr,
+                                          int64_t dilationAttr,
+                                          OpBuilder &rewriter) {
   ImplicitLocOpBuilder builder(loc, rewriter);
   auto one = rewriter.create<arith::ConstantOp>(
       loc, IntegerAttr::get(inputDim.getType(), 1));
@@ -171,7 +172,6 @@ static SmallVector<Value> inferDynamicDimsForConv(
     ArrayRef<int64_t> dilationAttr, ArrayRef<int64_t> inputSizeDims,
     ArrayRef<int64_t> kernelSizeDims, OpBuilder &rewriter) {
   ShapedType inputTy = cast<ShapedType>(input.getType());
-  Type inputETy = inputTy.getElementType();
   int64_t inputRank = inputTy.getRank();
 
   SmallVector<Value> dynDims;
@@ -190,8 +190,8 @@ static SmallVector<Value> inferDynamicDimsForConv(
           rewriter.create<tensor::DimOp>(loc, weight, kernelDim);
       // H = F(IH, pad_top, pad_bottom, dilation_y, KH, stride_y)
       dynDims[inputDim] =
-          getConvOutputDim(loc, initDynDim, padTop, padBottom, kernelDynDim,
-                           stride, dilation, inputETy, rewriter);
+          getConvOrPoolOutputDim(loc, initDynDim, padTop, padBottom,
+                                 kernelDynDim, stride, dilation, rewriter);
     }
   }
 
@@ -685,20 +685,61 @@ class MaxPool2dConverter : public OpRewritePattern<tosa::MaxPool2dOp> {
 public:
   using OpRewritePattern<tosa::MaxPool2dOp>::OpRewritePattern;
 
+  // Compute the dynamic output sizes of the maxpool operation.
+  static SmallVector<Value>
+  computeDynamicOutputSizes(tosa::MaxPool2dOp op, PatternRewriter &rewriter) {
+    TensorType resultTy = op.getType();
+    Location loc = op.getLoc();
+
+    TypedValue<TensorType> input = op.getInput();
+    ArrayRef<int64_t> kernel = op.getKernel();
+    ArrayRef<int64_t> pad = op.getPad();
+    ArrayRef<int64_t> stride = op.getStride();
+
+    SmallVector<Value> dynamicDims;
+
+    // Batch dimension
+    if (resultTy.isDynamicDim(0))
+      dynamicDims.push_back(rewriter.create<tensor::DimOp>(loc, input, 0));
+
+    // Height/width dimensions
+    for (int64_t dim : {1, 2}) {
+      if (!resultTy.isDynamicDim(dim))
+        continue;
+
+      // Index into the attribute arrays
+      int64_t index = dim - 1;
+
+      // Input height/width
+      Value ihw = rewriter.create<tensor::DimOp>(loc, input, dim);
+
+      // Kernel height/width
+      Value khw = rewriter.create<arith::ConstantIndexOp>(loc, kernel[index]);
+
+      // Output height/width
+      Value ohw = getConvOrPoolOutputDim(loc, ihw, pad[index * 2],
+                                         pad[index * 2 + 1], khw, stride[index],
+                                         /*dilationAttr=*/1, rewriter);
+      dynamicDims.push_back(ohw);
+    }
+
+    // Channel dimension
+    if (resultTy.isDynamicDim(3))
+      dynamicDims.push_back(rewriter.create<tensor::DimOp>(loc, input, 3));
+
+    return dynamicDims;
+  }
+
   LogicalResult matchAndRewrite(tosa::MaxPool2dOp op,
                                 PatternRewriter &rewriter) const final {
     Location loc = op.getLoc();
-    Value input = op.getInput();
-    ShapedType inputTy = cast<ShapedType>(input.getType());
+    TypedValue<TensorType> input = op.getInput();
+    ShapedType inputTy = input.getType();
 
-    ShapedType resultTy = cast<ShapedType>(op.getType());
+    ShapedType resultTy = op.getType();
     Type resultETy = inputTy.getElementType();
 
-    auto dynamicDimsOr =
-        checkHasDynamicBatchDims(rewriter, op, {input, op.getOutput()});
-    if (!dynamicDimsOr.has_value())
-      return failure();
-    SmallVector<Value> dynamicDims = *dynamicDimsOr;
+    SmallVector<Value> dynamicDims = computeDynamicOutputSizes(op, rewriter);
 
     // Determine what the initial value needs to be for the max pool op.
     TypedAttr initialAttr;
@@ -721,6 +762,7 @@ public:
     pad.resize(2, 0);
     llvm::append_range(pad, op.getPad());
     pad.resize(pad.size() + 2, 0);
+
     Value paddedInput = applyPad(loc, input, pad, initialAttr, rewriter);
 
     Value initialValue = rewriter.create<arith::ConstantOp>(loc, initialAttr);
@@ -736,9 +778,7 @@ public:
         loc, resultTy.getShape(), resultTy.getElementType(), dynamicDims);
 
     Value filledEmptyTensor =
-        rewriter
-            .create<linalg::FillOp>(loc, ValueRange{initialValue},
-                                    ValueRange{emptyTensor})
+        rewriter.create<linalg::FillOp>(loc, initialValue, emptyTensor)
             .result();
 
     Value fakeWindowDims =
diff --git a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-named.mlir b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-named.mlir
index e64903671e59..b4049000c50d 100644
--- a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-named.mlir
+++ b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-named.mlir
@@ -1,5 +1,6 @@
 // RUN: mlir-opt --split-input-file -pass-pipeline="builtin.module(func.func(tosa-to-linalg-named))" %s -verify-diagnostics -o -| FileCheck %s
 // RUN: mlir-opt --split-input-file -pass-pipeline="builtin.module(func.func(tosa-to-linalg-named{prefer-conv2d-kernel-layout-hwcf=true}))" %s -verify-diagnostics -o -| FileCheck --check-prefix="HWCF" %s
+// RUN: mlir-opt --split-input-file -pass-pipeline="builtin.module(func.func(tosa-to-linalg-named,cse))" %s -verify-diagnostics -o -| FileCheck --check-prefix="CHECK-CSE" %s
 
 // CHECK-LABEL: @matmul
 func.func @matmul(%arg0: tensor<1x5x3xf32>, %arg1: tensor<1x3x6xf32>) -> (tensor<1x5x6xf32>) {
@@ -215,6 +216,59 @@ func.func @max_pool_i32(%arg0: tensor<1x6x34x62xi32>) -> () {
   return
 }
 
+// CHECK-CSE-LABEL: @max_pool_all_dynamic
+func.func @max_pool_all_dynamic(%arg0: tensor<?x?x?x?xf32>) -> tensor<?x?x?x?xf32> {
+  // Batch size
+  // CHECK-CSE: %[[C0:.+]] = arith.constant 0 : index
+  // CHECK-CSE: %[[BATCH:.+]] = tensor.dim %arg0, %[[C0]] : tensor<?x?x?x?xf32>
+
+  // Compute output height
+  // CHECK-CSE: %[[C1:.+]] = arith.constant 1 : index
+  // CHECK-CSE: %[[IH:.+]] = tensor.dim %arg0, %[[C1]] : tensor<?x?x?x?xf32>
+  // CHECK-CSE: %[[C2:.+]] = arith.constant 2 : index
+  // CHECK-CSE: %[[PADDED_BEFORE:.+]] = arith.addi %[[IH]], %[[C0]] : index
+  // CHECK-CSE: %[[PADDED_AFTER:.+]] = arith.addi %[[PADDED_BEFORE]], %[[C0]] : index
+  // CHECK-CSE: %[[SUB_ONE:.+]] = arith.subi %[[C2]], %[[C1]] : index
+  // CHECK-CSE: %[[DILATED:.+]] = arith.muli %[[C1]], %[[SUB_ONE]] : index
+  // CHECK-CSE: %[[ADD_ONE:.+]] = arith.addi %[[DILATED]], %[[C1]] : index
+  // CHECK-CSE: %[[SUBTRACT:.+]] = arith.subi %[[PADDED_AFTER]], %[[ADD_ONE]] : index
+  // CHECK-CSE: %[[DIVIDE:.+]] = arith.divui %[[SUBTRACT]], %[[C1]] : index
+  // CHECK-CSE: %[[HEIGHT:.+]] = arith.addi %[[DIVIDE]], %[[C1]] : index
+
+  // Compute output width
+  // CHECK-CSE: %[[IW:.+]] = tensor.dim %arg0, %[[C2]] : tensor<?x?x?x?xf32>
+  // CHECK-CSE: %[[C5:.+]] = arith.constant 5 : index
+  // CHECK-CSE: %[[PADDED_BEFORE:.+]] = arith.addi %[[IW]], %[[C2]] : index
+  // CHECK-CSE: %[[PADDED_AFTER:.+]] = arith.addi %[[PADDED_BEFORE]], %[[C2]] : index
+  // CHECK-CSE: %[[SUB_ONE:.+]] = arith.subi %[[C5]], %[[C1]] : index
+  // CHECK-CSE: %[[DILATED:.+]] = arith.muli %[[C1]], %[[SUB_ONE]] : index
+  // CHECK-CSE: %[[ADD_ONE:.+]] = arith.addi %[[DILATED]], %[[C1]] : index
+  // CHECK-CSE: %[[SUBTRACT:.+]] = arith.subi %[[PADDED_AFTER]], %[[ADD_ONE]] : index
+  // CHECK-CSE: %[[DIVIDE:.+]] = arith.divui %[[SUBTRACT]], %[[C1]] : index
+  // CHECK-CSE: %[[WIDTH:.+]] = arith.addi %14, %[[C1]] : index
+
+  // Channel size
+  // CHECK-CSE: %[[C3:.+]] = arith.constant 3 : index
+  // CHECK-CSE: %[[CHANNEL:.+]] = tensor.dim %arg0, %[[C3]] : tensor<?x?x?x?xf32>
+
+  // Pad the input
+  // CHECK-CSE: %[[FLOAT_MIN:.+]] = arith.constant -3.40282347E+38 : f32
+  // CHECK-CSE: %[[PADDED:.+]] = tensor.pad %arg0 low[0, 0, 2, 0] high[0, 0, 2, 0] {
+  // CHECK-CSE:   tensor.yield %[[FLOAT_MIN]] : f32
+
+  // Allocate the output and fill with minimum value
+  // CHECK-CSE: %[[INIT:.+]] = tensor.empty(%[[BATCH]], %[[HEIGHT]], %[[WIDTH]], %[[CHANNEL]]) : tensor<?x?x?x?xf32>
+  // CHECK-CSE: %[[FILL:.+]] = linalg.fill ins(%[[FLOAT_MIN]] : f32) outs(%[[INIT]] : tensor<?x?x?x?xf32>) -> tensor<?x?x?x?xf32>
+  // CHECK-CSE: %[[FAKE_WINDOW:.+]] = tensor.empty() : tensor<2x5xf32>
+
+  // Compute max pool
+  // CHECK-CSE: %[[OUT:.+]] = linalg.pooling_nhwc_max {dilations = dense<1> : vector<2xi64>, strides = dense<1> : vector<2xi64>} ins(%[[PADDED]], %[[FAKE_WINDOW]] : tensor<?x?x?x?xf32>, tensor<2x5xf32>) outs(%[[FILL]] : tensor<?x?x?x?xf32>) -> tensor<?x?x?x?xf32>
+  // CHECK-CSE: return %[[OUT]]
+
+  %0 = tosa.max_pool2d %arg0 {kernel = array<i64: 2, 5>, pad = array<i64: 0, 0, 2, 2>, stride = array<i64: 1, 1>} : (tensor<?x?x?x?xf32>) -> tensor<?x?x?x?xf32>
+  return %0 : tensor<?x?x?x?xf32>
+}
+
 // -----
 
 // CHECK-LABEL: @avg_pool_f32
diff --git a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir
index 1fa783f05f04..445e8be47678 100644
--- a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir
+++ b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg.mlir
@@ -270,7 +270,8 @@ func.func @test_add_2d_all_dynamic(%arg0: tensor<?x?xf32>, %arg1: tensor<?x?xf32
   // CHECK: %[[VAL_0:.*]] = tensor.dim %[[ARG0]], %[[CONST0]] : tensor<?x?xf32>
   // CHECK: %[[VAL_1:.*]] = arith.cmpi eq, %[[VAL_0]], %[[CONST1]] : index
   // CHECK: %[[ARG0_DIM0_BROADCAST:.*]] = scf.if %[[VAL_1]] -> (tensor<?x?xf32>) {
-  // CHECK:   %[[VAL_2:.*]] = tensor.dim %[[ARG0]], %[[CONST1]] : tensor<?x?xf32>
+  // CHECK:   %[[LOCAL_CONST1:.*]] = arith.constant 1 : index
+  // CHECK:   %[[VAL_2:.*]] = tensor.dim %[[ARG0]], %[[LOCAL_CONST1]] : tensor<?x?xf32>
   // CHECK:   %[[VAL_3:.*]] = tensor.empty(%[[MAX_DIM0]], %[[VAL_2]]) : tensor<?x?xf32>
   // CHECK:   %[[VAL_4:.*]] = linalg.generic {indexing_maps = [#[[$MAP0]], #[[$MAP1]]], iterator_types = ["parallel", "parallel"]} ins(%[[ARG0]] : tensor<?x?xf32>) outs(%[[VAL_3]] : tensor<?x?xf32>) {
   // CHECK:   ^bb0(%[[VAL_5:.*]]: f32, %[[VAL_6:.*]]: f32):
@@ -284,7 +285,8 @@ func.func @test_add_2d_all_dynamic(%arg0: tensor<?x?xf32>, %arg1: tensor<?x?xf32
   // CHECK: %[[VAL_7:.*]] = tensor.dim %[[ARG0_DIM0_BROADCAST]], %[[CONST1]] : tensor<?x?xf32>
   // CHECK: %[[VAL_8:.*]] = arith.cmpi eq, %[[VAL_7]], %[[CONST1]] : index
   // CHECK: %[[ARG0_DIM1_BROADCAST:.*]] = scf.if %[[VAL_8]] -> (tensor<?x?xf32>) {
-  // CHECK:   %[[VAL_9:.*]] = tensor.dim %[[ARG0_DIM0_BROADCAST]], %[[CONST0]] : tensor<?x?xf32>
+  // CHECK:   %[[LOCAL_CONST0:.*]] = arith.constant 0 : index
+  // CHECK:   %[[VAL_9:.*]] = tensor.dim %[[ARG0_DIM0_BROADCAST]], %[[LOCAL_CONST0]] : tensor<?x?xf32>
   // CHECK:   %[[VAL_10:.*]] = tensor.empty(%[[VAL_9]], %[[MAX_DIM1]]) : tensor<?x?xf32>
   // CHECK:   %[[VAL_11:.*]] = linalg.generic {indexing_maps = [#[[$MAP2]], #[[$MAP1]]], iterator_types = ["parallel", "parallel"]} ins(%[[ARG0_DIM0_BROADCAST]] : tensor<?x?xf32>) outs(%[[VAL_10]] : tensor<?x?xf32>) {
   // CHECK:   ^bb0(%[[VAL_12:.*]]: f32, %[[VAL_13:.*]]: f32):
@@ -298,7 +300,8 @@ func.func @test_add_2d_all_dynamic(%arg0: tensor<?x?xf32>, %arg1: tensor<?x?xf32
   // CHECK: %[[VAL_14:.*]] = tensor.dim %[[ARG1]], %[[CONST0]] : tensor<?x?xf32>
   // CHECK: %[[VAL_15:.*]] = arith.cmpi eq, %[[VAL_14]], %[[CONST1]] : index
   // CHECK: %[[ARG1_DIM0_BROADCAST:.*]] = scf.if %[[VAL_15]] -> (tensor<?x?xf32>) {
-  // CHECK:   %[[VAL_16:.*]] = tensor.dim %[[ARG1]], %[[CONST1]] : tensor<?x?xf32>
+  // CHECK:   %[[LOCAL_CONST1:.*]] = arith.constant 1 : index
+  // CHECK:   %[[VAL_16:.*]] = tensor.dim %[[ARG1]], %[[LOCAL_CONST1]] : tensor<?x?xf32>
   // CHECK:   %[[VAL_17:.*]] = tensor.empty(%[[MAX_DIM0]], %[[VAL_16]]) : tensor<?x?xf32>
   // CHECK:   %[[VAL_18:.*]] = linalg.generic {indexing_maps = [#[[$MAP0]], #[[$MAP1]]], iterator_types = ["parallel", "parallel"]} ins(%[[ARG1]] : tensor<?x?xf32>) outs(%[[VAL_17]] : tensor<?x?xf32>) {
   // CHECK:   ^bb0(%[[VAL_19:.*]]: f32, %[[VAL_20:.*]]: f32):
@@ -312,7 +315,8 @@ func.func @test_add_2d_all_dynamic(%arg0: tensor<?x?xf32>, %arg1: tensor<?x?xf32
   // CHECK: %[[VAL_21:.*]] = tensor.dim %[[ARG1_DIM0_BROADCAST]], %[[CONST1]] : tensor<?x?xf32>
   // CHECK: %[[VAL_22:.*]] = arith.cmpi eq, %[[VAL_21]], %[[CONST1]] : index
   // CHECK: %[[ARG1_DIM1_BROADCAST:.*]] = scf.if %[[VAL_22]] -> (tensor<?x?xf32>) {
-  // CHECK:   %[[VAL_23:.*]] = tensor.dim %[[ARG1_DIM0_BROADCAST]], %[[CONST0]] : tensor<?x?xf32>
+  // CHECK:   %[[LOCAL_CONST0:.*]] = arith.constant 0 : index
+  // CHECK:   %[[VAL_23:.*]] = tensor.dim %[[ARG1_DIM0_BROADCAST]], %[[LOCAL_CONST0]] : tensor<?x?xf32>
   // CHECK:   %[[VAL_24:.*]] = tensor.empty(%[[VAL_23]], %[[MAX_DIM1]]) : tensor<?x?xf32>
   // CHECK:   %[[VAL_25:.*]] = linalg.generic {indexing_maps = [#[[$MAP2]], #[[$MAP1]]], iterator_types = ["parallel", "parallel"]} ins(%[[ARG1_DIM0_BROADCAST]] : tensor<?x?xf32>) outs(%[[VAL_24]] : tensor<?x?xf32>) {
   // CHECK:   ^bb0(%[[VAL_26:.*]]: f32, %[[VAL_27:.*]]: f32):
diff --git a/mlir/test/Integration/Dialect/Tosa/CPU/test-maxpool-dynamic.mlir b/mlir/test/Integration/Dialect/Tosa/CPU/test-maxpool-dynamic.mlir
new file mode 100644
index 000000000000..05a78e32b9e1
--- /dev/null
+++ b/mlir/test/Integration/Dialect/Tosa/CPU/test-maxpool-dynamic.mlir
@@ -0,0 +1,112 @@
+// DEFINE: %{tosa-to-linalg-pipeline} = -pass-pipeline="builtin.module(func.func(tosa-infer-shapes,tosa-to-linalg-named,tosa-to-linalg,tosa-to-arith))"
+
+// RUN:   mlir-opt %s \
+// RUN:     %{tosa-to-linalg-pipeline} \
+// RUN: | mlir-opt \
+// RUN:     -one-shot-bufferize="bufferize-function-boundaries" \
+// RUN:     -buffer-deallocation-pipeline \
+// RUN:     -test-lower-to-llvm \
+// RUN: | mlir-cpu-runner \
+// RUN:     -entry-point-result=void \
+// RUN:     -shared-libs=%mlir_runner_utils,%mlir_c_runner_utils \
+// RUN: | FileCheck %s
+
+// Validate that the TOSA lowering for tosa.max_pool2d produces the same results when
+// for fully static and fully dynamic inputs.
+
+!tensor_type = tensor<1x4x4x1xf32>
+!memref_type = memref<1x4x4x1xf32>
+
+// Utility functions
+func.func private @printMemrefF32(memref<*xf32>) attributes { llvm.emit_c_interface }
+
+func.func @max_pool_static(%arg0: !tensor_type) -> (!tensor_type) {
+  %0 = tosa.max_pool2d %arg0 {
+    pad = array<i64: 1, 1, 1, 1>,
+    kernel = array<i64: 3, 3>,
+    stride = array<i64: 1, 1>
+  } : (tensor<1x4x4x1xf32>) -> tensor<1x4x4x1xf32>
+  return %0 : tensor<1x4x4x1xf32>
+}
+
+func.func @max_pool_dynamic(%arg0: tensor<?x?x?x?xf32>) -> (tensor<?x?x?x?xf32>) {
+  %0 = tosa.max_pool2d %arg0 {
+    pad = array<i64: 1, 1, 1, 1>,
+    kernel = array<i64: 3, 3>,
+    stride = array<i64: 1, 1>
+  } : (tensor<?x?x?x?xf32>) -> tensor<?x?x?x?xf32>
+  return %0 : tensor<?x?x?x?xf32>
+}
+
+// Test harness to compare the results of a fully statically shaped max_pool2d with
+// a fully dynamically shaped max_pool2d on the same inputs.
+func.func @main() {
+  %A = arith.constant dense<[[
+    [[0.0], [0.1], [0.2], [0.3]], // H = 0
+    [[1.0], [1.1], [1.2], [1.3]], // H = 1
+    [[2.0], [2.1], [2.2], [2.3]], // H = 2
+    [[3.0], [3.1], [3.2], [3.3]]  // H = 3
+  ]]> : tensor<1x4x4x1xf32>
+
+  %A_dynamic = tensor.cast %A : !tensor_type to tensor<?x?x?x?xf32>
+
+  // Call both static and dynamically sized variants
+  %result_static  = func.call @max_pool_static(%A) : (!tensor_type) -> !tensor_type
+  %result_dynamic = func.call @max_pool_dynamic(%A_dynamic) : (tensor<?x?x?x?xf32>) -> tensor<?x?x?x?xf32>
+
+  %static_buffer = bufferization.to_memref %result_static : !memref_type
+  %unranked_static_buffer = memref.cast %static_buffer : !memref_type to memref<*xf32>
+
+  // CHECK: Unranked Memref base@ = {{.*}} rank = 4 offset = 0 sizes = [1, 4, 4, 1] strides = [16, 4, 1, 1] data =
+
+  // CHECK-NEXT: 1.1
+  // CHECK-NEXT: 1.2
+  // CHECK-NEXT: 1.3
+  // CHECK-NEXT: 1.3
+
+  // CHECK-NEXT: 2.1
+  // CHECK-NEXT: 2.2
+  // CHECK-NEXT: 2.3
+  // CHECK-NEXT: 2.3
+
+  // CHECK-NEXT: 3.1
+  // CHECK-NEXT: 3.2
+  // CHECK-NEXT: 3.3
+  // CHECK-NEXT: 3.3
+
+  // CHECK-NEXT: 3.1
+  // CHECK-NEXT: 3.2
+  // CHECK-NEXT: 3.3
+  // CHECK-NEXT: 3.3
+
+  func.call @printMemrefF32(%unranked_static_buffer) : (memref<*xf32>) -> ()
+
+  %dynamic_buffer = bufferization.to_memref %result_dynamic : memref<?x?x?x?xf32>
+  %unranked_dynamic_buffer = memref.cast %dynamic_buffer : memref<?x?x?x?xf32> to memref<*xf32>
+
+  // CHECK: Unranked Memref base@ = {{.*}} rank = 4 offset = 0 sizes = [1, 4, 4, 1] strides = [16, 4, 1, 1] data =
+  // CHECK-NEXT: 1.1
+  // CHECK-NEXT: 1.2
+  // CHECK-NEXT: 1.3
+  // CHECK-NEXT: 1.3
+
+  // CHECK-NEXT: 2.1
+  // CHECK-NEXT: 2.2
+  // CHECK-NEXT: 2.3
+  // CHECK-NEXT: 2.3
+
+  // CHECK-NEXT: 3.1
+  // CHECK-NEXT: 3.2
+  // CHECK-NEXT: 3.3
+  // CHECK-NEXT: 3.3
+
+  // CHECK-NEXT: 3.1
+  // CHECK-NEXT: 3.2
+  // CHECK-NEXT: 3.3
+  // CHECK-NEXT: 3.3
+
+  func.call @printMemrefF32(%unranked_dynamic_buffer) : (memref<*xf32>) -> ()
+
+  return
+}
+
-- 
GitLab


From d4602a96b48b00a50c4d891673fc622ca2e37f0a Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Tue, 16 Apr 2024 08:14:41 -0400
Subject: [PATCH 391/464] [gn] port fe48bf672e1ab2

---
 llvm/utils/gn/secondary/lldb/test/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/lldb/test/BUILD.gn b/llvm/utils/gn/secondary/lldb/test/BUILD.gn
index 414ea4933c51..c8245739842d 100644
--- a/llvm/utils/gn/secondary/lldb/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/test/BUILD.gn
@@ -118,6 +118,7 @@ write_lit_cfg("lit_shell_site_cfg") {
     "LLDB_TOOLS_DIR=" + rebase_path("$root_out_dir/bin"),
     "LLDB_USE_SYSTEM_DEBUGSERVER=1",  # XXX port //lldb/tools/debugserver (?)
     "LLVM_HOST_TRIPLE=$llvm_current_triple",
+    "LLVM_USE_SANITIZER=",
   ]
 
   if (llvm_enable_zlib) {
-- 
GitLab


From c309dc6d0759b23b570c563f611530ff1a49e1bd Mon Sep 17 00:00:00 2001
From: mahtohappy <Happy.Kumar@Windriver.com>
Date: Tue, 16 Apr 2024 17:48:45 +0530
Subject: [PATCH 392/464] [Clang][Sema] placement new initializes typedef array
 with correct size (#83124)

When in-place new-ing a local variable of an array of trivial type, the
generated code calls 'memset' with the correct size of the array,
earlier it was generating size (squared of the typedef array + size).

The cause: `typedef TYPE TArray[8]; TArray x;` The type of declarator is
Tarray[8] and in `SemaExprCXX.cpp::BuildCXXNew` we check if it's of
typedef and of constant size then we get the original type and it works
fine for non-dependent cases.
But in case of template we do `TreeTransform.h:TransformCXXNEWExpr` and
there we again check the allocated type which is TArray[8] and it stays
that way, so ArraySize=(Tarray[8] type, alloc Tarray[8*type]) so the
squared size allocation.

ArraySize gets calculated earlier in `TreeTransform.h` so that
`if(!ArraySize)` condition was failing.
fix: I changed that condition to `if(ArraySize)`.


Fixes #41441
---
 clang/docs/ReleaseNotes.rst                   |  2 ++
 clang/lib/Sema/TreeTransform.h                | 14 ++++++++++++-
 .../instantiate-new-placement-size.cpp        | 20 +++++++++++++++++++
 3 files changed, 35 insertions(+), 1 deletion(-)
 create mode 100644 clang/test/SemaCXX/instantiate-new-placement-size.cpp

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 76701dc723b6..255d2cc04404 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -539,6 +539,8 @@ Bug Fixes to C++ Support
   Fixes (#GH70604), (#GH79754), (#GH84163), (#GH84425), (#GH86054), (#GH86398), and (#GH86399).
 - Fix a crash when deducing ``auto`` from an invalid dereference (#GH88329).
 - Fix a crash in requires expression with templated base class member function. Fixes (#GH84020).
+- placement new initializes typedef array with correct size
+  (`#GH41441 <https://github.com/llvm/llvm-project/issues/41441>`_)
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 8c96134af7c8..9d15f3eacbb0 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -12802,6 +12802,19 @@ TreeTransform<Derived>::TransformCXXNewExpr(CXXNewExpr *E) {
     ArraySize = NewArraySize.get();
   }
 
+  // Per C++0x [expr.new]p5, the type being constructed may be a
+  // typedef of an array type.
+  QualType AllocType = AllocTypeInfo->getType();
+  if (ArraySize) {
+    if (const ConstantArrayType *Array =
+            SemaRef.Context.getAsConstantArrayType(AllocType)) {
+      ArraySize = IntegerLiteral::Create(SemaRef.Context, Array->getSize(),
+                                         SemaRef.Context.getSizeType(),
+                                         E->getBeginLoc());
+      AllocType = Array->getElementType();
+    }
+  }
+
   // Transform the placement arguments (if any).
   bool ArgumentChanged = false;
   SmallVector<Expr*, 8> PlacementArgs;
@@ -12863,7 +12876,6 @@ TreeTransform<Derived>::TransformCXXNewExpr(CXXNewExpr *E) {
     return E;
   }
 
-  QualType AllocType = AllocTypeInfo->getType();
   if (!ArraySize) {
     // If no array size was specified, but the new expression was
     // instantiated with an array type (e.g., "new T" where T is
diff --git a/clang/test/SemaCXX/instantiate-new-placement-size.cpp b/clang/test/SemaCXX/instantiate-new-placement-size.cpp
new file mode 100644
index 000000000000..7a29d3dee849
--- /dev/null
+++ b/clang/test/SemaCXX/instantiate-new-placement-size.cpp
@@ -0,0 +1,20 @@
+// RUN: %clang -S -fno-discard-value-names -emit-llvm -o - %s | FileCheck %s
+// Issue no: 41441
+#include <new>
+
+// CHECK: call void @llvm.memset.p0.i64(ptr align 1 %x, i8 0, i64 8, i1 false)
+// CHECK: call void @llvm.memset.p0.i64(ptr align 16 %x, i8 0, i64 32, i1 false)
+template <typename TYPE>
+void f()
+{
+    typedef TYPE TArray[8];
+
+    TArray x;
+    new(&x) TArray();
+}
+
+int main()
+{
+    f<char>();
+    f<int>();
+}
-- 
GitLab


From 34013e7ce25868aa8ddea116f79184e8603af56c Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 16 Apr 2024 13:03:09 +0100
Subject: [PATCH 393/464] [X86] Add shuffle tests for
 BLEND(PERMUTE(X),PERMUTE(Y)) patterns

Some very basic tests for a case where we could fold BLEND(PERMUTE(X),PERMUTE(Y)) -> PERMUTE(BLEND(X,Y))

These assume the permute masks are the same, and "complete" (no undefs/duplicate elements) but we could relax that depending on the blend mask
---
 .../X86/vector-shuffle-combining-avx.ll       | 31 +++++++++++++
 .../X86/vector-shuffle-combining-avx512f.ll   | 44 +++++++++++++++++++
 .../X86/vector-shuffle-combining-sse41.ll     | 15 +++++++
 3 files changed, 90 insertions(+)

diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx.ll
index 0c76c14afb0a..4859a8e0eaaa 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx.ll
@@ -305,6 +305,37 @@ define <4 x float> @combine_vpermilvar_4f32_as_insertps(<4 x float> %a0) {
   ret <4 x float> %2
 }
 
+define <8 x i32> @combine_blend_of_permutes_v8i32(<4 x i64> %a0, <4 x i64> %a1) {
+; AVX1-LABEL: combine_blend_of_permutes_v8i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,0,1]
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,0,1]
+; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3,4],ymm0[5,6],ymm1[7]
+; AVX1-NEXT:    ret{{[l|q]}}
+;
+; AVX2-LABEL: combine_blend_of_permutes_v8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,3,0,1]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[2,3,0,1]
+; AVX2-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3,4],ymm0[5,6],ymm1[7]
+; AVX2-NEXT:    ret{{[l|q]}}
+;
+; AVX512-LABEL: combine_blend_of_permutes_v8i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512-NEXT:    vpmovsxbd {{.*#+}} ymm2 = [4,21,6,23,16,1,2,19]
+; AVX512-NEXT:    vpermt2d %zmm1, %zmm2, %zmm0
+; AVX512-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512-NEXT:    ret{{[l|q]}}
+  %s0 = shufflevector <4 x i64> %a0, <4 x i64> undef, <4 x i32> <i32 2, i32 3, i32 0, i32 1>
+  %s1 = shufflevector <4 x i64> %a1, <4 x i64> undef, <4 x i32> <i32 2, i32 3, i32 0, i32 1>
+  %x0 = bitcast <4 x i64> %s0 to <8 x i32>
+  %x1 = bitcast <4 x i64> %s1 to <8 x i32>
+  %r = shufflevector <8 x i32> %x0, <8 x i32> %x1, <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 12, i32 5, i32 6, i32 15>
+  ret <8 x i32> %r
+}
+
 define <2 x double> @constant_fold_vpermilvar_pd() {
 ; CHECK-LABEL: constant_fold_vpermilvar_pd:
 ; CHECK:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512f.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512f.ll
index f53b1eeaf8f5..e87e810971e1 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512f.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx512f.ll
@@ -973,3 +973,47 @@ define <8 x i64> @combine_vpermvar_insertion_as_broadcast_v8i64(i64 %a0) {
   %2 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %1, <8 x i64> zeroinitializer)
   ret <8 x i64> %2
 }
+
+define <16 x i32> @blend_of_permutes_v16i32(<8 x i64> %a0, <8x i64> %a1) {
+; X86-AVX512F-LABEL: blend_of_permutes_v16i32:
+; X86-AVX512F:       # %bb.0:
+; X86-AVX512F-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[2,3,0,1,6,7,4,5]
+; X86-AVX512F-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,3,0,1,6,7,4,5]
+; X86-AVX512F-NEXT:    movw $-25958, %ax # imm = 0x9A9A
+; X86-AVX512F-NEXT:    kmovw %eax, %k1
+; X86-AVX512F-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
+; X86-AVX512F-NEXT:    retl
+;
+; X86-AVX512BW-LABEL: blend_of_permutes_v16i32:
+; X86-AVX512BW:       # %bb.0:
+; X86-AVX512BW-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[2,3,0,1,6,7,4,5]
+; X86-AVX512BW-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,3,0,1,6,7,4,5]
+; X86-AVX512BW-NEXT:    movw $-25958, %ax # imm = 0x9A9A
+; X86-AVX512BW-NEXT:    kmovd %eax, %k1
+; X86-AVX512BW-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
+; X86-AVX512BW-NEXT:    retl
+;
+; X64-AVX512F-LABEL: blend_of_permutes_v16i32:
+; X64-AVX512F:       # %bb.0:
+; X64-AVX512F-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[2,3,0,1,6,7,4,5]
+; X64-AVX512F-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,3,0,1,6,7,4,5]
+; X64-AVX512F-NEXT:    movw $-25958, %ax # imm = 0x9A9A
+; X64-AVX512F-NEXT:    kmovw %eax, %k1
+; X64-AVX512F-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
+; X64-AVX512F-NEXT:    retq
+;
+; X64-AVX512BW-LABEL: blend_of_permutes_v16i32:
+; X64-AVX512BW:       # %bb.0:
+; X64-AVX512BW-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[2,3,0,1,6,7,4,5]
+; X64-AVX512BW-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,3,0,1,6,7,4,5]
+; X64-AVX512BW-NEXT:    movw $-25958, %ax # imm = 0x9A9A
+; X64-AVX512BW-NEXT:    kmovd %eax, %k1
+; X64-AVX512BW-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
+; X64-AVX512BW-NEXT:    retq
+  %s0 = shufflevector <8 x i64> %a0, <8 x i64> undef, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 6, i32 7, i32 4, i32 5>
+  %s1 = shufflevector <8 x i64> %a1, <8 x i64> undef, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 6, i32 7, i32 4, i32 5>
+  %x0 = bitcast <8 x i64> %s0 to <16 x i32>
+  %x1 = bitcast <8 x i64> %s1 to <16 x i32>
+  %r = shufflevector <16 x i32> %x0, <16 x i32> %x1, <16 x i32> <i32 0, i32 17, i32 2, i32 19, i32 20, i32 5, i32 6, i32 23, i32 8, i32 25, i32 10, i32 27, i32 28, i32 13, i32 14, i32 31>
+  ret <16 x i32> %r
+}
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-sse41.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-sse41.ll
index 5eb017bc80ca..33851f56fe8d 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-combining-sse41.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-sse41.ll
@@ -22,6 +22,21 @@ define <16 x i8> @combine_vpshufb_as_movzx(<16 x i8> %a0) {
   ret <16 x i8> %res0
 }
 
+define <4 x i32> @combine_blend_of_permutes_v4i32(<2 x i64> %a0, <2 x i64> %a1) {
+; SSE-LABEL: combine_blend_of_permutes_v4i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE-NEXT:    pblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3],xmm2[4,5],xmm0[6,7]
+; SSE-NEXT:    retq
+  %s0 = shufflevector <2 x i64> %a0, <2 x i64> undef, <2 x i32> <i32 1, i32 0>
+  %s1 = shufflevector <2 x i64> %a1, <2 x i64> undef, <2 x i32> <i32 1, i32 0>
+  %x0 = bitcast <2 x i64> %s0 to <4 x i32>
+  %x1 = bitcast <2 x i64> %s1 to <4 x i32>
+  %r = shufflevector <4 x i32> %x0, <4 x i32> %x1, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+  ret <4 x i32> %r
+}
+
 define <16 x i8> @PR50049(ptr %p1, ptr %p2) {
 ; SSE-LABEL: PR50049:
 ; SSE:       # %bb.0:
-- 
GitLab


From b73476c7843f21966acb2fb5cab8515d9ec02905 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Tue, 16 Apr 2024 13:29:34 +0100
Subject: [PATCH 394/464] [SLP] Make sure MinVF is a power-of-2 by using
 PowerOf2Ceil.

This should ensure we explore the same VFs as before 6d66db3890a18e39.

Fixes https://github.com/llvm/llvm-project/issues/88640.
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    |  4 +--
 .../trunc-store-value-ty-not-power-of-2.ll    | 33 +++++++++++++++++++
 2 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index c63b500f546f..d0bcdceae392 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -15155,8 +15155,8 @@ bool SLPVectorizerPass::vectorizeStores(ArrayRef<StoreInst *> Stores,
       Type *ValueTy = StoreTy;
       if (auto *Trunc = dyn_cast<TruncInst>(Store->getValueOperand()))
         ValueTy = Trunc->getSrcTy();
-      unsigned MinVF = TTI->getStoreMinimumVF(
-          R.getMinVF(DL->getTypeSizeInBits(StoreTy)), StoreTy, ValueTy);
+      unsigned MinVF = PowerOf2Ceil(TTI->getStoreMinimumVF(
+          R.getMinVF(DL->getTypeStoreSizeInBits(StoreTy)), StoreTy, ValueTy));
 
       if (MaxVF < MinVF) {
         LLVM_DEBUG(dbgs() << "SLP: Vectorization infeasible as MaxVF (" << MaxVF
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/trunc-store-value-ty-not-power-of-2.ll b/llvm/test/Transforms/SLPVectorizer/X86/trunc-store-value-ty-not-power-of-2.ll
index 81b4ee40e7fd..2f0fad70b593 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/trunc-store-value-ty-not-power-of-2.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/trunc-store-value-ty-not-power-of-2.ll
@@ -107,3 +107,36 @@ define void @test_4_trunc_i24_to_i16(i24 %x, ptr %A) {
   store i16 %t, ptr %gep.3, align 1
   ret void
 }
+
+%struct.d = type { [3 x i8], [3 x i8], [2 x i8] }
+
+; Test case for https://github.com/llvm/llvm-project/issues/88640.
+define void @test_access_i24_directly(ptr %src, ptr noalias %dst) "target-cpu"="btver2" {
+; CHECK-LABEL: define void @test_access_i24_directly(
+; CHECK-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64, ptr [[SRC]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[TMP0]] to i24
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds [[STRUCT_D:%.*]], ptr [[SRC]], i64 0, i32 1
+; CHECK-NEXT:    [[BF_LOAD:%.*]] = load i24, ptr [[GEP_SRC]], align 1
+; CHECK-NEXT:    [[BF_VALUE:%.*]] = and i24 [[TMP1]], 8388607
+; CHECK-NEXT:    [[BF_CLEAR:%.*]] = and i24 [[BF_LOAD]], -8388608
+; CHECK-NEXT:    [[BF_SET:%.*]] = or disjoint i24 [[BF_CLEAR]], [[BF_VALUE]]
+; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds [[STRUCT_D]], ptr [[DST]], i64 0, i32 1
+; CHECK-NEXT:    store i24 [[BF_SET]], ptr [[GEP_DST]], align 1
+; CHECK-NEXT:    store i24 0, ptr [[DST]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %0 = load i64, ptr %src, align 8
+  %1 = trunc i64 %0 to i24
+  %gep.src = getelementptr inbounds %struct.d, ptr %src, i64 0, i32 1
+  %bf.load = load i24, ptr %gep.src, align 1
+  %bf.value = and i24 %1, 8388607
+  %bf.clear = and i24 %bf.load, -8388608
+  %bf.set = or disjoint i24 %bf.clear, %bf.value
+  %gep.dst = getelementptr inbounds %struct.d, ptr %dst, i64 0, i32 1
+  store i24 %bf.set, ptr %gep.dst, align 1
+  store i24 0, ptr %dst, align 8
+  ret void
+}
-- 
GitLab


From e272c37934a06cd80b9b072afc09afae5fd8c218 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Tue, 16 Apr 2024 08:31:35 -0400
Subject: [PATCH 395/464] clang; Try to get windows-seh-async-verify.cpp to
 pass on mac

On macOS, file paths start with /Users/..., which clang-cl interptrets
as the /U switch followed by a preprocessor macro name to undefine.

Put the filename after `--` to prevent this. For consistency, move %s
to the end of the regular `clang` lines (where this isn't needed) as
well.
---
 clang/test/Driver/windows-seh-async-verify.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/clang/test/Driver/windows-seh-async-verify.cpp b/clang/test/Driver/windows-seh-async-verify.cpp
index 5fda6a77dba0..ace93cf44a31 100644
--- a/clang/test/Driver/windows-seh-async-verify.cpp
+++ b/clang/test/Driver/windows-seh-async-verify.cpp
@@ -1,7 +1,7 @@
-// RUN: %clang --target=x86_64-pc-windows -fasync-exceptions -fsyntax-only %s -### 2>&1 | FileCheck %s
-// RUN: %clang_cl --target=x86_64-pc-windows /EHa -fsyntax-only %s -### 2>&1 | FileCheck %s
-// RUN: %clang --target=x86_64-pc-windows-gnu -fasync-exceptions -fsyntax-only %s -### 2>&1 | FileCheck %s --check-prefixes=GNU-ALL,GNU
-// RUN: %clang_cl --target=x86_64-pc-windows-gnu /EHa -fsyntax-only %s -### 2>&1 | FileCheck %s --check-prefixes=GNU-ALL,CL-GNU
+// RUN: %clang --target=x86_64-pc-windows -fasync-exceptions -fsyntax-only -### %s 2>&1 | FileCheck %s
+// RUN: %clang_cl --target=x86_64-pc-windows /EHa -fsyntax-only -### -- %s 2>&1 | FileCheck %s
+// RUN: %clang --target=x86_64-pc-windows-gnu -fasync-exceptions -fsyntax-only -### %s 2>&1 | FileCheck %s --check-prefixes=GNU-ALL,GNU
+// RUN: %clang_cl --target=x86_64-pc-windows-gnu /EHa -fsyntax-only -### -- %s 2>&1 | FileCheck %s --check-prefixes=GNU-ALL,CL-GNU
 
 // CHECK-NOT: warning
 // GNU: warning: argument unused during compilation: '-fasync-exceptions' [-Wunused-command-line-argument]
-- 
GitLab


From f69ded0d9965a6b2b76ce12db876c70f249d96d1 Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Tue, 16 Apr 2024 16:36:53 +0400
Subject: [PATCH 396/464] [clang] Introduce `SemaOpenMP` (#88642)

This patch moves OpenMP-related entities out of `Sema` to a newly
created `SemaOpenMP` class. This is a part of the effort to split `Sema`
up, and follows the recent example of CUDA, OpenACC, SYCL, HLSL.
Additional context can be found in
https://github.com/llvm/llvm-project/pull/82217,
https://github.com/llvm/llvm-project/pull/84184,
https://github.com/llvm/llvm-project/pull/87634.
---
 clang/include/clang/Parse/Parser.h            |   15 +-
 clang/include/clang/Sema/Sema.h               | 1419 +-----
 clang/include/clang/Sema/SemaOpenMP.h         | 1447 ++++++
 clang/lib/Parse/ParseDecl.cpp                 |    3 +-
 clang/lib/Parse/ParseExpr.cpp                 |    5 +-
 clang/lib/Parse/ParseOpenMP.cpp               |  240 +-
 clang/lib/Parse/ParseStmt.cpp                 |    3 +-
 clang/lib/Sema/Sema.cpp                       |   25 +-
 clang/lib/Sema/SemaDecl.cpp                   |   32 +-
 clang/lib/Sema/SemaDeclCXX.cpp                |    9 +-
 clang/lib/Sema/SemaExpr.cpp                   |  610 +--
 clang/lib/Sema/SemaExprMember.cpp             |    9 +-
 clang/lib/Sema/SemaLambda.cpp                 |    3 +-
 clang/lib/Sema/SemaOpenMP.cpp                 | 4333 ++++++++++-------
 clang/lib/Sema/SemaStmt.cpp                   |    6 +-
 .../lib/Sema/SemaTemplateInstantiateDecl.cpp  |   60 +-
 clang/lib/Sema/SemaType.cpp                   |    3 +-
 clang/lib/Sema/TreeTransform.h                |  684 +--
 18 files changed, 4565 insertions(+), 4341 deletions(-)
 create mode 100644 clang/include/clang/Sema/SemaOpenMP.h

diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index 5950dd74cfe8..23b268126de4 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -18,6 +18,7 @@
 #include "clang/Lex/CodeCompletionHandler.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Sema/Sema.h"
+#include "clang/Sema/SemaOpenMP.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Frontend/OpenMP/OMPContext.h"
 #include "llvm/Support/SaveAndRestore.h"
@@ -2537,7 +2538,7 @@ private:
   /// Returns true for declaration, false for expression.
   bool isForInitDeclaration() {
     if (getLangOpts().OpenMP)
-      Actions.startOpenMPLoop();
+      Actions.OpenMP().startOpenMPLoop();
     if (getLangOpts().CPlusPlus)
       return Tok.is(tok::kw_using) ||
              isCXXSimpleDeclaration(/*AllowForRangeDecl=*/true);
@@ -3396,7 +3397,7 @@ private:
                           SourceLocation Loc);
 
   /// Parse clauses for '#pragma omp [begin] declare target'.
-  void ParseOMPDeclareTargetClauses(Sema::DeclareTargetContextInfo &DTCI);
+  void ParseOMPDeclareTargetClauses(SemaOpenMP::DeclareTargetContextInfo &DTCI);
 
   /// Parse '#pragma omp end declare target'.
   void ParseOMPEndDeclareTargetDirective(OpenMPDirectiveKind BeginDKind,
@@ -3486,7 +3487,7 @@ private:
   /// Parses indirect clause
   /// \param ParseOnly true to skip the clause's semantic actions and return
   // false;
-  bool ParseOpenMPIndirectClause(Sema::DeclareTargetContextInfo &DTCI,
+  bool ParseOpenMPIndirectClause(SemaOpenMP::DeclareTargetContextInfo &DTCI,
                                  bool ParseOnly);
   /// Parses clause with a single expression and an additional argument
   /// of a kind \a Kind.
@@ -3556,12 +3557,12 @@ public:
 
   /// Parses a reserved locator like 'omp_all_memory'.
   bool ParseOpenMPReservedLocator(OpenMPClauseKind Kind,
-                                  Sema::OpenMPVarListDataTy &Data,
+                                  SemaOpenMP::OpenMPVarListDataTy &Data,
                                   const LangOptions &LangOpts);
   /// Parses clauses with list.
   bool ParseOpenMPVarList(OpenMPDirectiveKind DKind, OpenMPClauseKind Kind,
                           SmallVectorImpl<Expr *> &Vars,
-                          Sema::OpenMPVarListDataTy &Data);
+                          SemaOpenMP::OpenMPVarListDataTy &Data);
   bool ParseUnqualifiedId(CXXScopeSpec &SS, ParsedType ObjectType,
                           bool ObjectHadErrors, bool EnteringContext,
                           bool AllowDestructorName, bool AllowConstructorName,
@@ -3569,11 +3570,11 @@ public:
                           SourceLocation *TemplateKWLoc, UnqualifiedId &Result);
 
   /// Parses the mapper modifier in map, to, and from clauses.
-  bool parseMapperModifier(Sema::OpenMPVarListDataTy &Data);
+  bool parseMapperModifier(SemaOpenMP::OpenMPVarListDataTy &Data);
   /// Parses map-type-modifiers in map clause.
   /// map([ [map-type-modifier[,] [map-type-modifier[,] ...] map-type : ] list)
   /// where, map-type-modifier ::= always | close | mapper(mapper-identifier)
-  bool parseMapTypeModifiers(Sema::OpenMPVarListDataTy &Data);
+  bool parseMapTypeModifiers(SemaOpenMP::OpenMPVarListDataTy &Data);
 
   //===--------------------------------------------------------------------===//
   // OpenACC Parsing.
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index d93ac7863b72..a5fe83a539aa 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -26,14 +26,12 @@
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/ExprConcepts.h"
 #include "clang/AST/ExprObjC.h"
-#include "clang/AST/ExprOpenMP.h"
 #include "clang/AST/ExternalASTSource.h"
 #include "clang/AST/LocInfoType.h"
 #include "clang/AST/MangleNumberingContext.h"
 #include "clang/AST/NSAPI.h"
 #include "clang/AST/PrettyPrinter.h"
 #include "clang/AST/StmtCXX.h"
-#include "clang/AST/StmtOpenMP.h"
 #include "clang/AST/TypeLoc.h"
 #include "clang/AST/TypeOrdering.h"
 #include "clang/Basic/BitmaskEnum.h"
@@ -43,7 +41,6 @@
 #include "clang/Basic/ExpressionTraits.h"
 #include "clang/Basic/Module.h"
 #include "clang/Basic/OpenCLOptions.h"
-#include "clang/Basic/OpenMPKinds.h"
 #include "clang/Basic/PragmaKinds.h"
 #include "clang/Basic/Specifiers.h"
 #include "clang/Basic/TemplateKinds.h"
@@ -68,7 +65,6 @@
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/TinyPtrVector.h"
-#include "llvm/Frontend/OpenMP/OMPConstants.h"
 #include <deque>
 #include <memory>
 #include <optional>
@@ -167,12 +163,6 @@ class ObjCMessageExpr;
 class ObjCMethodDecl;
 class ObjCPropertyDecl;
 class ObjCProtocolDecl;
-class OMPThreadPrivateDecl;
-class OMPRequiresDecl;
-class OMPDeclareReductionDecl;
-class OMPDeclareSimdDecl;
-class OMPClause;
-struct OMPVarListLocTy;
 struct OverloadCandidate;
 enum class OverloadCandidateParamOrder : char;
 enum OverloadCandidateRewriteKind : unsigned;
@@ -187,6 +177,7 @@ class QualType;
 class SemaCUDA;
 class SemaHLSL;
 class SemaOpenACC;
+class SemaOpenMP;
 class SemaSYCL;
 class StandardConversionSequence;
 class Stmt;
@@ -480,7 +471,6 @@ class Sema final : public SemaBase {
   // 35. Code Completion (SemaCodeComplete.cpp)
   // 36. FixIt Helpers (SemaFixItUtils.cpp)
   // 37. Name Lookup for RISC-V Vector Intrinsic (SemaRISCVVectorLookup.cpp)
-  // 38. OpenMP Directives and Clauses (SemaOpenMP.cpp)
 
   /// \name Semantic Analysis
   /// Implementations are in Sema.cpp
@@ -997,6 +987,11 @@ public:
     return *OpenACCPtr;
   }
 
+  SemaOpenMP &OpenMP() {
+    assert(OpenMPPtr && "SemaOpenMP is dead");
+    return *OpenMPPtr;
+  }
+
   SemaSYCL &SYCL() {
     assert(SYCLPtr);
     return *SYCLPtr;
@@ -1035,6 +1030,7 @@ private:
   std::unique_ptr<SemaCUDA> CUDAPtr;
   std::unique_ptr<SemaHLSL> HLSLPtr;
   std::unique_ptr<SemaOpenACC> OpenACCPtr;
+  std::unique_ptr<SemaOpenMP> OpenMPPtr;
   std::unique_ptr<SemaSYCL> SYCLPtr;
 
   ///@}
@@ -3443,14 +3439,6 @@ public:
 
   sema::LambdaScopeInfo *RebuildLambdaScopeInfo(CXXMethodDecl *CallOperator);
 
-  /// The declarator \p D defines a function in the scope \p S which is nested
-  /// in an `omp begin/end declare variant` scope. In this method we create a
-  /// declaration for \p D and rename \p D according to the OpenMP context
-  /// selector of the surrounding scope. Return all base functions in \p Bases.
-  void ActOnStartOfFunctionDefinitionInOpenMPDeclareVariantScope(
-      Scope *S, Declarator &D, MultiTemplateParamsArg TemplateParameterLists,
-      SmallVectorImpl<FunctionDecl *> &Bases);
-
   // Heuristically tells if the function is `get_return_object` member of a
   // coroutine promise_type by matching the function name.
   static bool CanBeGetReturnObject(const FunctionDecl *FD);
@@ -5533,32 +5521,6 @@ public:
                                               Expr *ColumnIdx,
                                               SourceLocation RBLoc);
 
-  ExprResult ActOnOMPArraySectionExpr(Expr *Base, SourceLocation LBLoc,
-                                      Expr *LowerBound,
-                                      SourceLocation ColonLocFirst,
-                                      SourceLocation ColonLocSecond,
-                                      Expr *Length, Expr *Stride,
-                                      SourceLocation RBLoc);
-  ExprResult ActOnOMPArrayShapingExpr(Expr *Base, SourceLocation LParenLoc,
-                                      SourceLocation RParenLoc,
-                                      ArrayRef<Expr *> Dims,
-                                      ArrayRef<SourceRange> Brackets);
-
-  /// Data structure for iterator expression.
-  struct OMPIteratorData {
-    IdentifierInfo *DeclIdent = nullptr;
-    SourceLocation DeclIdentLoc;
-    ParsedType Type;
-    OMPIteratorExpr::IteratorRange Range;
-    SourceLocation AssignLoc;
-    SourceLocation ColonLoc;
-    SourceLocation SecColonLoc;
-  };
-
-  ExprResult ActOnOMPIteratorExpr(Scope *S, SourceLocation IteratorKwLoc,
-                                  SourceLocation LLoc, SourceLocation RLoc,
-                                  ArrayRef<OMPIteratorData> Data);
-
   bool ConvertArgumentsForCall(CallExpr *Call, Expr *Fn, FunctionDecl *FDecl,
                                const FunctionProtoType *Proto,
                                ArrayRef<Expr *> Args, SourceLocation RParenLoc,
@@ -12863,1373 +12825,6 @@ private:
   std::unique_ptr<sema::RISCVIntrinsicManager> RVIntrinsicManager;
 
   ///@}
-
-  //
-  //
-  // -------------------------------------------------------------------------
-  //
-  //
-
-  /// \name OpenMP Directives and Clauses
-  /// Implementations are in SemaOpenMP.cpp
-  ///@{
-
-public:
-  /// Creates a SemaDiagnosticBuilder that emits the diagnostic if the current
-  /// context is "used as device code".
-  ///
-  /// - If CurContext is a `declare target` function or it is known that the
-  /// function is emitted for the device, emits the diagnostics immediately.
-  /// - If CurContext is a non-`declare target` function and we are compiling
-  ///   for the device, creates a diagnostic which is emitted if and when we
-  ///   realize that the function will be codegen'ed.
-  ///
-  /// Example usage:
-  ///
-  ///  // Variable-length arrays are not allowed in NVPTX device code.
-  ///  if (diagIfOpenMPDeviceCode(Loc, diag::err_vla_unsupported))
-  ///    return ExprError();
-  ///  // Otherwise, continue parsing as normal.
-  SemaDiagnosticBuilder diagIfOpenMPDeviceCode(SourceLocation Loc,
-                                               unsigned DiagID,
-                                               const FunctionDecl *FD);
-
-  /// Creates a SemaDiagnosticBuilder that emits the diagnostic if the current
-  /// context is "used as host code".
-  ///
-  /// - If CurContext is a `declare target` function or it is known that the
-  /// function is emitted for the host, emits the diagnostics immediately.
-  /// - If CurContext is a non-host function, just ignore it.
-  ///
-  /// Example usage:
-  ///
-  ///  // Variable-length arrays are not allowed in NVPTX device code.
-  ///  if (diagIfOpenMPHostode(Loc, diag::err_vla_unsupported))
-  ///    return ExprError();
-  ///  // Otherwise, continue parsing as normal.
-  SemaDiagnosticBuilder diagIfOpenMPHostCode(SourceLocation Loc,
-                                             unsigned DiagID,
-                                             const FunctionDecl *FD);
-
-  /// Register \p D as specialization of all base functions in \p Bases in the
-  /// current `omp begin/end declare variant` scope.
-  void ActOnFinishedFunctionDefinitionInOpenMPDeclareVariantScope(
-      Decl *D, SmallVectorImpl<FunctionDecl *> &Bases);
-
-  /// Act on \p D, a function definition inside of an `omp [begin/end] assumes`.
-  void ActOnFinishedFunctionDefinitionInOpenMPAssumeScope(Decl *D);
-
-  /// Can we exit an OpenMP declare variant scope at the moment.
-  bool isInOpenMPDeclareVariantScope() const {
-    return !OMPDeclareVariantScopes.empty();
-  }
-
-  ExprResult
-  VerifyPositiveIntegerConstantInClause(Expr *Op, OpenMPClauseKind CKind,
-                                        bool StrictlyPositive = true,
-                                        bool SuppressExprDiags = false);
-
-  /// Given the potential call expression \p Call, determine if there is a
-  /// specialization via the OpenMP declare variant mechanism available. If
-  /// there is, return the specialized call expression, otherwise return the
-  /// original \p Call.
-  ExprResult ActOnOpenMPCall(ExprResult Call, Scope *Scope,
-                             SourceLocation LParenLoc, MultiExprArg ArgExprs,
-                             SourceLocation RParenLoc, Expr *ExecConfig);
-
-  /// Handle a `omp begin declare variant`.
-  void ActOnOpenMPBeginDeclareVariant(SourceLocation Loc, OMPTraitInfo &TI);
-
-  /// Handle a `omp end declare variant`.
-  void ActOnOpenMPEndDeclareVariant();
-
-  /// Function tries to capture lambda's captured variables in the OpenMP region
-  /// before the original lambda is captured.
-  void tryCaptureOpenMPLambdas(ValueDecl *V);
-
-  /// Return true if the provided declaration \a VD should be captured by
-  /// reference.
-  /// \param Level Relative level of nested OpenMP construct for that the check
-  /// is performed.
-  /// \param OpenMPCaptureLevel Capture level within an OpenMP construct.
-  bool isOpenMPCapturedByRef(const ValueDecl *D, unsigned Level,
-                             unsigned OpenMPCaptureLevel) const;
-
-  /// Check if the specified variable is used in one of the private
-  /// clauses (private, firstprivate, lastprivate, reduction etc.) in OpenMP
-  /// constructs.
-  VarDecl *isOpenMPCapturedDecl(ValueDecl *D, bool CheckScopeInfo = false,
-                                unsigned StopAt = 0);
-
-  /// The member expression(this->fd) needs to be rebuilt in the template
-  /// instantiation to generate private copy for OpenMP when default
-  /// clause is used. The function will return true if default
-  /// cluse is used.
-  bool isOpenMPRebuildMemberExpr(ValueDecl *D);
-
-  ExprResult getOpenMPCapturedExpr(VarDecl *Capture, ExprValueKind VK,
-                                   ExprObjectKind OK, SourceLocation Loc);
-
-  /// If the current region is a loop-based region, mark the start of the loop
-  /// construct.
-  void startOpenMPLoop();
-
-  /// If the current region is a range loop-based region, mark the start of the
-  /// loop construct.
-  void startOpenMPCXXRangeFor();
-
-  /// Check if the specified variable is used in 'private' clause.
-  /// \param Level Relative level of nested OpenMP construct for that the check
-  /// is performed.
-  OpenMPClauseKind isOpenMPPrivateDecl(ValueDecl *D, unsigned Level,
-                                       unsigned CapLevel) const;
-
-  /// Sets OpenMP capture kind (OMPC_private, OMPC_firstprivate, OMPC_map etc.)
-  /// for \p FD based on DSA for the provided corresponding captured declaration
-  /// \p D.
-  void setOpenMPCaptureKind(FieldDecl *FD, const ValueDecl *D, unsigned Level);
-
-  /// Check if the specified variable is captured  by 'target' directive.
-  /// \param Level Relative level of nested OpenMP construct for that the check
-  /// is performed.
-  bool isOpenMPTargetCapturedDecl(const ValueDecl *D, unsigned Level,
-                                  unsigned CaptureLevel) const;
-
-  /// Check if the specified global variable must be captured  by outer capture
-  /// regions.
-  /// \param Level Relative level of nested OpenMP construct for that
-  /// the check is performed.
-  bool isOpenMPGlobalCapturedDecl(ValueDecl *D, unsigned Level,
-                                  unsigned CaptureLevel) const;
-
-  ExprResult PerformOpenMPImplicitIntegerConversion(SourceLocation OpLoc,
-                                                    Expr *Op);
-  /// Called on start of new data sharing attribute block.
-  void StartOpenMPDSABlock(OpenMPDirectiveKind K,
-                           const DeclarationNameInfo &DirName, Scope *CurScope,
-                           SourceLocation Loc);
-  /// Start analysis of clauses.
-  void StartOpenMPClause(OpenMPClauseKind K);
-  /// End analysis of clauses.
-  void EndOpenMPClause();
-  /// Called on end of data sharing attribute block.
-  void EndOpenMPDSABlock(Stmt *CurDirective);
-
-  /// Check if the current region is an OpenMP loop region and if it is,
-  /// mark loop control variable, used in \p Init for loop initialization, as
-  /// private by default.
-  /// \param Init First part of the for loop.
-  void ActOnOpenMPLoopInitialization(SourceLocation ForLoc, Stmt *Init);
-
-  /// Called on well-formed '\#pragma omp metadirective' after parsing
-  /// of the  associated statement.
-  StmtResult ActOnOpenMPMetaDirective(ArrayRef<OMPClause *> Clauses,
-                                      Stmt *AStmt, SourceLocation StartLoc,
-                                      SourceLocation EndLoc);
-
-  // OpenMP directives and clauses.
-  /// Called on correct id-expression from the '#pragma omp
-  /// threadprivate'.
-  ExprResult ActOnOpenMPIdExpression(Scope *CurScope, CXXScopeSpec &ScopeSpec,
-                                     const DeclarationNameInfo &Id,
-                                     OpenMPDirectiveKind Kind);
-  /// Called on well-formed '#pragma omp threadprivate'.
-  DeclGroupPtrTy ActOnOpenMPThreadprivateDirective(SourceLocation Loc,
-                                                   ArrayRef<Expr *> VarList);
-  /// Builds a new OpenMPThreadPrivateDecl and checks its correctness.
-  OMPThreadPrivateDecl *CheckOMPThreadPrivateDecl(SourceLocation Loc,
-                                                  ArrayRef<Expr *> VarList);
-  /// Called on well-formed '#pragma omp allocate'.
-  DeclGroupPtrTy ActOnOpenMPAllocateDirective(SourceLocation Loc,
-                                              ArrayRef<Expr *> VarList,
-                                              ArrayRef<OMPClause *> Clauses,
-                                              DeclContext *Owner = nullptr);
-
-  /// Called on well-formed '#pragma omp [begin] assume[s]'.
-  void ActOnOpenMPAssumesDirective(SourceLocation Loc,
-                                   OpenMPDirectiveKind DKind,
-                                   ArrayRef<std::string> Assumptions,
-                                   bool SkippedClauses);
-
-  /// Check if there is an active global `omp begin assumes` directive.
-  bool isInOpenMPAssumeScope() const { return !OMPAssumeScoped.empty(); }
-
-  /// Check if there is an active global `omp assumes` directive.
-  bool hasGlobalOpenMPAssumes() const { return !OMPAssumeGlobal.empty(); }
-
-  /// Called on well-formed '#pragma omp end assumes'.
-  void ActOnOpenMPEndAssumesDirective();
-
-  /// Called on well-formed '#pragma omp requires'.
-  DeclGroupPtrTy ActOnOpenMPRequiresDirective(SourceLocation Loc,
-                                              ArrayRef<OMPClause *> ClauseList);
-  /// Check restrictions on Requires directive
-  OMPRequiresDecl *CheckOMPRequiresDecl(SourceLocation Loc,
-                                        ArrayRef<OMPClause *> Clauses);
-  /// Check if the specified type is allowed to be used in 'omp declare
-  /// reduction' construct.
-  QualType ActOnOpenMPDeclareReductionType(SourceLocation TyLoc,
-                                           TypeResult ParsedType);
-  /// Called on start of '#pragma omp declare reduction'.
-  DeclGroupPtrTy ActOnOpenMPDeclareReductionDirectiveStart(
-      Scope *S, DeclContext *DC, DeclarationName Name,
-      ArrayRef<std::pair<QualType, SourceLocation>> ReductionTypes,
-      AccessSpecifier AS, Decl *PrevDeclInScope = nullptr);
-  /// Initialize declare reduction construct initializer.
-  void ActOnOpenMPDeclareReductionCombinerStart(Scope *S, Decl *D);
-  /// Finish current declare reduction construct initializer.
-  void ActOnOpenMPDeclareReductionCombinerEnd(Decl *D, Expr *Combiner);
-  /// Initialize declare reduction construct initializer.
-  /// \return omp_priv variable.
-  VarDecl *ActOnOpenMPDeclareReductionInitializerStart(Scope *S, Decl *D);
-  /// Finish current declare reduction construct initializer.
-  void ActOnOpenMPDeclareReductionInitializerEnd(Decl *D, Expr *Initializer,
-                                                 VarDecl *OmpPrivParm);
-  /// Called at the end of '#pragma omp declare reduction'.
-  DeclGroupPtrTy ActOnOpenMPDeclareReductionDirectiveEnd(
-      Scope *S, DeclGroupPtrTy DeclReductions, bool IsValid);
-
-  /// Check variable declaration in 'omp declare mapper' construct.
-  TypeResult ActOnOpenMPDeclareMapperVarDecl(Scope *S, Declarator &D);
-  /// Check if the specified type is allowed to be used in 'omp declare
-  /// mapper' construct.
-  QualType ActOnOpenMPDeclareMapperType(SourceLocation TyLoc,
-                                        TypeResult ParsedType);
-  /// Called on start of '#pragma omp declare mapper'.
-  DeclGroupPtrTy ActOnOpenMPDeclareMapperDirective(
-      Scope *S, DeclContext *DC, DeclarationName Name, QualType MapperType,
-      SourceLocation StartLoc, DeclarationName VN, AccessSpecifier AS,
-      Expr *MapperVarRef, ArrayRef<OMPClause *> Clauses,
-      Decl *PrevDeclInScope = nullptr);
-  /// Build the mapper variable of '#pragma omp declare mapper'.
-  ExprResult ActOnOpenMPDeclareMapperDirectiveVarDecl(Scope *S,
-                                                      QualType MapperType,
-                                                      SourceLocation StartLoc,
-                                                      DeclarationName VN);
-  void ActOnOpenMPIteratorVarDecl(VarDecl *VD);
-  bool isOpenMPDeclareMapperVarDeclAllowed(const VarDecl *VD) const;
-  const ValueDecl *getOpenMPDeclareMapperVarName() const;
-
-  struct DeclareTargetContextInfo {
-    struct MapInfo {
-      OMPDeclareTargetDeclAttr::MapTypeTy MT;
-      SourceLocation Loc;
-    };
-    /// Explicitly listed variables and functions in a 'to' or 'link' clause.
-    llvm::DenseMap<NamedDecl *, MapInfo> ExplicitlyMapped;
-
-    /// The 'device_type' as parsed from the clause.
-    OMPDeclareTargetDeclAttr::DevTypeTy DT = OMPDeclareTargetDeclAttr::DT_Any;
-
-    /// The directive kind, `begin declare target` or `declare target`.
-    OpenMPDirectiveKind Kind;
-
-    /// The directive with indirect clause.
-    std::optional<Expr *> Indirect;
-
-    /// The directive location.
-    SourceLocation Loc;
-
-    DeclareTargetContextInfo(OpenMPDirectiveKind Kind, SourceLocation Loc)
-        : Kind(Kind), Loc(Loc) {}
-  };
-
-  /// Called on the start of target region i.e. '#pragma omp declare target'.
-  bool ActOnStartOpenMPDeclareTargetContext(DeclareTargetContextInfo &DTCI);
-
-  /// Called at the end of target region i.e. '#pragma omp end declare target'.
-  const DeclareTargetContextInfo ActOnOpenMPEndDeclareTargetDirective();
-
-  /// Called once a target context is completed, that can be when a
-  /// '#pragma omp end declare target' was encountered or when a
-  /// '#pragma omp declare target' without declaration-definition-seq was
-  /// encountered.
-  void ActOnFinishedOpenMPDeclareTargetContext(DeclareTargetContextInfo &DTCI);
-
-  /// Report unterminated 'omp declare target' or 'omp begin declare target' at
-  /// the end of a compilation unit.
-  void DiagnoseUnterminatedOpenMPDeclareTarget();
-
-  /// Searches for the provided declaration name for OpenMP declare target
-  /// directive.
-  NamedDecl *lookupOpenMPDeclareTargetName(Scope *CurScope,
-                                           CXXScopeSpec &ScopeSpec,
-                                           const DeclarationNameInfo &Id);
-
-  /// Called on correct id-expression from the '#pragma omp declare target'.
-  void ActOnOpenMPDeclareTargetName(NamedDecl *ND, SourceLocation Loc,
-                                    OMPDeclareTargetDeclAttr::MapTypeTy MT,
-                                    DeclareTargetContextInfo &DTCI);
-
-  /// Check declaration inside target region.
-  void
-  checkDeclIsAllowedInOpenMPTarget(Expr *E, Decl *D,
-                                   SourceLocation IdLoc = SourceLocation());
-
-  /// Adds OMPDeclareTargetDeclAttr to referenced variables in declare target
-  /// directive.
-  void ActOnOpenMPDeclareTargetInitializer(Decl *D);
-
-  /// Finishes analysis of the deferred functions calls that may be declared as
-  /// host/nohost during device/host compilation.
-  void finalizeOpenMPDelayedAnalysis(const FunctionDecl *Caller,
-                                     const FunctionDecl *Callee,
-                                     SourceLocation Loc);
-
-  /// Return true if currently in OpenMP task with untied clause context.
-  bool isInOpenMPTaskUntiedContext() const;
-
-  /// Return true inside OpenMP declare target region.
-  bool isInOpenMPDeclareTargetContext() const {
-    return !DeclareTargetNesting.empty();
-  }
-  /// Return true inside OpenMP target region.
-  bool isInOpenMPTargetExecutionDirective() const;
-
-  /// Return the number of captured regions created for an OpenMP directive.
-  static int getOpenMPCaptureLevels(OpenMPDirectiveKind Kind);
-
-  /// Initialization of captured region for OpenMP region.
-  void ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind, Scope *CurScope);
-
-  /// Called for syntactical loops (ForStmt or CXXForRangeStmt) associated to
-  /// an OpenMP loop directive.
-  StmtResult ActOnOpenMPCanonicalLoop(Stmt *AStmt);
-
-  /// Process a canonical OpenMP loop nest that can either be a canonical
-  /// literal loop (ForStmt or CXXForRangeStmt), or the generated loop of an
-  /// OpenMP loop transformation construct.
-  StmtResult ActOnOpenMPLoopnest(Stmt *AStmt);
-
-  /// End of OpenMP region.
-  ///
-  /// \param S Statement associated with the current OpenMP region.
-  /// \param Clauses List of clauses for the current OpenMP region.
-  ///
-  /// \returns Statement for finished OpenMP region.
-  StmtResult ActOnOpenMPRegionEnd(StmtResult S, ArrayRef<OMPClause *> Clauses);
-  StmtResult ActOnOpenMPExecutableDirective(
-      OpenMPDirectiveKind Kind, const DeclarationNameInfo &DirName,
-      OpenMPDirectiveKind CancelRegion, ArrayRef<OMPClause *> Clauses,
-      Stmt *AStmt, SourceLocation StartLoc, SourceLocation EndLoc,
-      OpenMPDirectiveKind PrevMappedDirective = llvm::omp::OMPD_unknown);
-  /// Called on well-formed '\#pragma omp parallel' after parsing
-  /// of the  associated statement.
-  StmtResult ActOnOpenMPParallelDirective(ArrayRef<OMPClause *> Clauses,
-                                          Stmt *AStmt, SourceLocation StartLoc,
-                                          SourceLocation EndLoc);
-  using VarsWithInheritedDSAType =
-      llvm::SmallDenseMap<const ValueDecl *, const Expr *, 4>;
-  /// Called on well-formed '\#pragma omp simd' after parsing
-  /// of the associated statement.
-  StmtResult
-  ActOnOpenMPSimdDirective(ArrayRef<OMPClause *> Clauses, Stmt *AStmt,
-                           SourceLocation StartLoc, SourceLocation EndLoc,
-                           VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '#pragma omp tile' after parsing of its clauses and
-  /// the associated statement.
-  StmtResult ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
-                                      Stmt *AStmt, SourceLocation StartLoc,
-                                      SourceLocation EndLoc);
-  /// Called on well-formed '#pragma omp unroll' after parsing of its clauses
-  /// and the associated statement.
-  StmtResult ActOnOpenMPUnrollDirective(ArrayRef<OMPClause *> Clauses,
-                                        Stmt *AStmt, SourceLocation StartLoc,
-                                        SourceLocation EndLoc);
-  /// Called on well-formed '\#pragma omp for' after parsing
-  /// of the associated statement.
-  StmtResult
-  ActOnOpenMPForDirective(ArrayRef<OMPClause *> Clauses, Stmt *AStmt,
-                          SourceLocation StartLoc, SourceLocation EndLoc,
-                          VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp for simd' after parsing
-  /// of the associated statement.
-  StmtResult
-  ActOnOpenMPForSimdDirective(ArrayRef<OMPClause *> Clauses, Stmt *AStmt,
-                              SourceLocation StartLoc, SourceLocation EndLoc,
-                              VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp sections' after parsing
-  /// of the associated statement.
-  StmtResult ActOnOpenMPSectionsDirective(ArrayRef<OMPClause *> Clauses,
-                                          Stmt *AStmt, SourceLocation StartLoc,
-                                          SourceLocation EndLoc);
-  /// Called on well-formed '\#pragma omp section' after parsing of the
-  /// associated statement.
-  StmtResult ActOnOpenMPSectionDirective(Stmt *AStmt, SourceLocation StartLoc,
-                                         SourceLocation EndLoc);
-  /// Called on well-formed '\#pragma omp scope' after parsing of the
-  /// associated statement.
-  StmtResult ActOnOpenMPScopeDirective(ArrayRef<OMPClause *> Clauses,
-                                       Stmt *AStmt, SourceLocation StartLoc,
-                                       SourceLocation EndLoc);
-  /// Called on well-formed '\#pragma omp single' after parsing of the
-  /// associated statement.
-  StmtResult ActOnOpenMPSingleDirective(ArrayRef<OMPClause *> Clauses,
-                                        Stmt *AStmt, SourceLocation StartLoc,
-                                        SourceLocation EndLoc);
-  /// Called on well-formed '\#pragma omp master' after parsing of the
-  /// associated statement.
-  StmtResult ActOnOpenMPMasterDirective(Stmt *AStmt, SourceLocation StartLoc,
-                                        SourceLocation EndLoc);
-  /// Called on well-formed '\#pragma omp critical' after parsing of the
-  /// associated statement.
-  StmtResult ActOnOpenMPCriticalDirective(const DeclarationNameInfo &DirName,
-                                          ArrayRef<OMPClause *> Clauses,
-                                          Stmt *AStmt, SourceLocation StartLoc,
-                                          SourceLocation EndLoc);
-  /// Called on well-formed '\#pragma omp parallel for' after parsing
-  /// of the  associated statement.
-  StmtResult ActOnOpenMPParallelForDirective(
-      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
-      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp parallel for simd' after
-  /// parsing of the  associated statement.
-  StmtResult ActOnOpenMPParallelForSimdDirective(
-      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
-      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp parallel master' after
-  /// parsing of the  associated statement.
-  StmtResult ActOnOpenMPParallelMasterDirective(ArrayRef<OMPClause *> Clauses,
-                                                Stmt *AStmt,
-                                                SourceLocation StartLoc,
-                                                SourceLocation EndLoc);
-  /// Called on well-formed '\#pragma omp parallel masked' after
-  /// parsing of the associated statement.
-  StmtResult ActOnOpenMPParallelMaskedDirective(ArrayRef<OMPClause *> Clauses,
-                                                Stmt *AStmt,
-                                                SourceLocation StartLoc,
-                                                SourceLocation EndLoc);
-  /// Called on well-formed '\#pragma omp parallel sections' after
-  /// parsing of the  associated statement.
-  StmtResult ActOnOpenMPParallelSectionsDirective(ArrayRef<OMPClause *> Clauses,
-                                                  Stmt *AStmt,
-                                                  SourceLocation StartLoc,
-                                                  SourceLocation EndLoc);
-  /// Called on well-formed '\#pragma omp task' after parsing of the
-  /// associated statement.
-  StmtResult ActOnOpenMPTaskDirective(ArrayRef<OMPClause *> Clauses,
-                                      Stmt *AStmt, SourceLocation StartLoc,
-                                      SourceLocation EndLoc);
-  /// Called on well-formed '\#pragma omp taskyield'.
-  StmtResult ActOnOpenMPTaskyieldDirective(SourceLocation StartLoc,
-                                           SourceLocation EndLoc);
-  /// Called on well-formed '\#pragma omp error'.
-  /// Error direcitive is allowed in both declared and excutable contexts.
-  /// Adding InExContext to identify which context is called from.
-  StmtResult ActOnOpenMPErrorDirective(ArrayRef<OMPClause *> Clauses,
-                                       SourceLocation StartLoc,
-                                       SourceLocation EndLoc,
-                                       bool InExContext = true);
-  /// Called on well-formed '\#pragma omp barrier'.
-  StmtResult ActOnOpenMPBarrierDirective(SourceLocation StartLoc,
-                                         SourceLocation EndLoc);
-  /// Called on well-formed '\#pragma omp taskwait'.
-  StmtResult ActOnOpenMPTaskwaitDirective(ArrayRef<OMPClause *> Clauses,
-                                          SourceLocation StartLoc,
-                                          SourceLocation EndLoc);
-  /// Called on well-formed '\#pragma omp taskgroup'.
-  StmtResult ActOnOpenMPTaskgroupDirective(ArrayRef<OMPClause *> Clauses,
-                                           Stmt *AStmt, SourceLocation StartLoc,
-                                           SourceLocation EndLoc);
-  /// Called on well-formed '\#pragma omp flush'.
-  StmtResult ActOnOpenMPFlushDirective(ArrayRef<OMPClause *> Clauses,
-                                       SourceLocation StartLoc,
-                                       SourceLocation EndLoc);
-  /// Called on well-formed '\#pragma omp depobj'.
-  StmtResult ActOnOpenMPDepobjDirective(ArrayRef<OMPClause *> Clauses,
-                                        SourceLocation StartLoc,
-                                        SourceLocation EndLoc);
-  /// Called on well-formed '\#pragma omp scan'.
-  StmtResult ActOnOpenMPScanDirective(ArrayRef<OMPClause *> Clauses,
-                                      SourceLocation StartLoc,
-                                      SourceLocation EndLoc);
-  /// Called on well-formed '\#pragma omp ordered' after parsing of the
-  /// associated statement.
-  StmtResult ActOnOpenMPOrderedDirective(ArrayRef<OMPClause *> Clauses,
-                                         Stmt *AStmt, SourceLocation StartLoc,
-                                         SourceLocation EndLoc);
-  /// Called on well-formed '\#pragma omp atomic' after parsing of the
-  /// associated statement.
-  StmtResult ActOnOpenMPAtomicDirective(ArrayRef<OMPClause *> Clauses,
-                                        Stmt *AStmt, SourceLocation StartLoc,
-                                        SourceLocation EndLoc);
-  /// Called on well-formed '\#pragma omp target' after parsing of the
-  /// associated statement.
-  StmtResult ActOnOpenMPTargetDirective(ArrayRef<OMPClause *> Clauses,
-                                        Stmt *AStmt, SourceLocation StartLoc,
-                                        SourceLocation EndLoc);
-  /// Called on well-formed '\#pragma omp target data' after parsing of
-  /// the associated statement.
-  StmtResult ActOnOpenMPTargetDataDirective(ArrayRef<OMPClause *> Clauses,
-                                            Stmt *AStmt,
-                                            SourceLocation StartLoc,
-                                            SourceLocation EndLoc);
-  /// Called on well-formed '\#pragma omp target enter data' after
-  /// parsing of the associated statement.
-  StmtResult ActOnOpenMPTargetEnterDataDirective(ArrayRef<OMPClause *> Clauses,
-                                                 SourceLocation StartLoc,
-                                                 SourceLocation EndLoc,
-                                                 Stmt *AStmt);
-  /// Called on well-formed '\#pragma omp target exit data' after
-  /// parsing of the associated statement.
-  StmtResult ActOnOpenMPTargetExitDataDirective(ArrayRef<OMPClause *> Clauses,
-                                                SourceLocation StartLoc,
-                                                SourceLocation EndLoc,
-                                                Stmt *AStmt);
-  /// Called on well-formed '\#pragma omp target parallel' after
-  /// parsing of the associated statement.
-  StmtResult ActOnOpenMPTargetParallelDirective(ArrayRef<OMPClause *> Clauses,
-                                                Stmt *AStmt,
-                                                SourceLocation StartLoc,
-                                                SourceLocation EndLoc);
-  /// Called on well-formed '\#pragma omp target parallel for' after
-  /// parsing of the  associated statement.
-  StmtResult ActOnOpenMPTargetParallelForDirective(
-      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
-      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp teams' after parsing of the
-  /// associated statement.
-  StmtResult ActOnOpenMPTeamsDirective(ArrayRef<OMPClause *> Clauses,
-                                       Stmt *AStmt, SourceLocation StartLoc,
-                                       SourceLocation EndLoc);
-  /// Called on well-formed '\#pragma omp teams loop' after parsing of the
-  /// associated statement.
-  StmtResult ActOnOpenMPTeamsGenericLoopDirective(
-      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
-      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp target teams loop' after parsing of
-  /// the associated statement.
-  StmtResult ActOnOpenMPTargetTeamsGenericLoopDirective(
-      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
-      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp parallel loop' after parsing of the
-  /// associated statement.
-  StmtResult ActOnOpenMPParallelGenericLoopDirective(
-      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
-      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp target parallel loop' after parsing
-  /// of the associated statement.
-  StmtResult ActOnOpenMPTargetParallelGenericLoopDirective(
-      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
-      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp cancellation point'.
-  StmtResult
-  ActOnOpenMPCancellationPointDirective(SourceLocation StartLoc,
-                                        SourceLocation EndLoc,
-                                        OpenMPDirectiveKind CancelRegion);
-  /// Called on well-formed '\#pragma omp cancel'.
-  StmtResult ActOnOpenMPCancelDirective(ArrayRef<OMPClause *> Clauses,
-                                        SourceLocation StartLoc,
-                                        SourceLocation EndLoc,
-                                        OpenMPDirectiveKind CancelRegion);
-  /// Called on well-formed '\#pragma omp taskloop' after parsing of the
-  /// associated statement.
-  StmtResult
-  ActOnOpenMPTaskLoopDirective(ArrayRef<OMPClause *> Clauses, Stmt *AStmt,
-                               SourceLocation StartLoc, SourceLocation EndLoc,
-                               VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp taskloop simd' after parsing of
-  /// the associated statement.
-  StmtResult ActOnOpenMPTaskLoopSimdDirective(
-      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
-      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp master taskloop' after parsing of the
-  /// associated statement.
-  StmtResult ActOnOpenMPMasterTaskLoopDirective(
-      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
-      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp master taskloop simd' after parsing of
-  /// the associated statement.
-  StmtResult ActOnOpenMPMasterTaskLoopSimdDirective(
-      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
-      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp parallel master taskloop' after
-  /// parsing of the associated statement.
-  StmtResult ActOnOpenMPParallelMasterTaskLoopDirective(
-      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
-      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp parallel master taskloop simd' after
-  /// parsing of the associated statement.
-  StmtResult ActOnOpenMPParallelMasterTaskLoopSimdDirective(
-      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
-      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp masked taskloop' after parsing of the
-  /// associated statement.
-  StmtResult ActOnOpenMPMaskedTaskLoopDirective(
-      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
-      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp masked taskloop simd' after parsing of
-  /// the associated statement.
-  StmtResult ActOnOpenMPMaskedTaskLoopSimdDirective(
-      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
-      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp parallel masked taskloop' after
-  /// parsing of the associated statement.
-  StmtResult ActOnOpenMPParallelMaskedTaskLoopDirective(
-      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
-      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp parallel masked taskloop simd' after
-  /// parsing of the associated statement.
-  StmtResult ActOnOpenMPParallelMaskedTaskLoopSimdDirective(
-      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
-      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp distribute' after parsing
-  /// of the associated statement.
-  StmtResult
-  ActOnOpenMPDistributeDirective(ArrayRef<OMPClause *> Clauses, Stmt *AStmt,
-                                 SourceLocation StartLoc, SourceLocation EndLoc,
-                                 VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp target update'.
-  StmtResult ActOnOpenMPTargetUpdateDirective(ArrayRef<OMPClause *> Clauses,
-                                              SourceLocation StartLoc,
-                                              SourceLocation EndLoc,
-                                              Stmt *AStmt);
-  /// Called on well-formed '\#pragma omp distribute parallel for' after
-  /// parsing of the associated statement.
-  StmtResult ActOnOpenMPDistributeParallelForDirective(
-      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
-      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp distribute parallel for simd'
-  /// after parsing of the associated statement.
-  StmtResult ActOnOpenMPDistributeParallelForSimdDirective(
-      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
-      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp distribute simd' after
-  /// parsing of the associated statement.
-  StmtResult ActOnOpenMPDistributeSimdDirective(
-      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
-      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp target parallel for simd' after
-  /// parsing of the associated statement.
-  StmtResult ActOnOpenMPTargetParallelForSimdDirective(
-      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
-      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp target simd' after parsing of
-  /// the associated statement.
-  StmtResult
-  ActOnOpenMPTargetSimdDirective(ArrayRef<OMPClause *> Clauses, Stmt *AStmt,
-                                 SourceLocation StartLoc, SourceLocation EndLoc,
-                                 VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp teams distribute' after parsing of
-  /// the associated statement.
-  StmtResult ActOnOpenMPTeamsDistributeDirective(
-      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
-      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp teams distribute simd' after parsing
-  /// of the associated statement.
-  StmtResult ActOnOpenMPTeamsDistributeSimdDirective(
-      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
-      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp teams distribute parallel for simd'
-  /// after parsing of the associated statement.
-  StmtResult ActOnOpenMPTeamsDistributeParallelForSimdDirective(
-      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
-      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp teams distribute parallel for'
-  /// after parsing of the associated statement.
-  StmtResult ActOnOpenMPTeamsDistributeParallelForDirective(
-      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
-      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp target teams' after parsing of the
-  /// associated statement.
-  StmtResult ActOnOpenMPTargetTeamsDirective(ArrayRef<OMPClause *> Clauses,
-                                             Stmt *AStmt,
-                                             SourceLocation StartLoc,
-                                             SourceLocation EndLoc);
-  /// Called on well-formed '\#pragma omp target teams distribute' after parsing
-  /// of the associated statement.
-  StmtResult ActOnOpenMPTargetTeamsDistributeDirective(
-      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
-      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp target teams distribute parallel for'
-  /// after parsing of the associated statement.
-  StmtResult ActOnOpenMPTargetTeamsDistributeParallelForDirective(
-      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
-      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp target teams distribute parallel for
-  /// simd' after parsing of the associated statement.
-  StmtResult ActOnOpenMPTargetTeamsDistributeParallelForSimdDirective(
-      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
-      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp target teams distribute simd' after
-  /// parsing of the associated statement.
-  StmtResult ActOnOpenMPTargetTeamsDistributeSimdDirective(
-      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
-      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
-  /// Called on well-formed '\#pragma omp interop'.
-  StmtResult ActOnOpenMPInteropDirective(ArrayRef<OMPClause *> Clauses,
-                                         SourceLocation StartLoc,
-                                         SourceLocation EndLoc);
-  /// Called on well-formed '\#pragma omp dispatch' after parsing of the
-  // /associated statement.
-  StmtResult ActOnOpenMPDispatchDirective(ArrayRef<OMPClause *> Clauses,
-                                          Stmt *AStmt, SourceLocation StartLoc,
-                                          SourceLocation EndLoc);
-  /// Called on well-formed '\#pragma omp masked' after parsing of the
-  // /associated statement.
-  StmtResult ActOnOpenMPMaskedDirective(ArrayRef<OMPClause *> Clauses,
-                                        Stmt *AStmt, SourceLocation StartLoc,
-                                        SourceLocation EndLoc);
-
-  /// Called on well-formed '\#pragma omp loop' after parsing of the
-  /// associated statement.
-  StmtResult ActOnOpenMPGenericLoopDirective(
-      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
-      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
-
-  /// Checks correctness of linear modifiers.
-  bool CheckOpenMPLinearModifier(OpenMPLinearClauseKind LinKind,
-                                 SourceLocation LinLoc);
-  /// Checks that the specified declaration matches requirements for the linear
-  /// decls.
-  bool CheckOpenMPLinearDecl(const ValueDecl *D, SourceLocation ELoc,
-                             OpenMPLinearClauseKind LinKind, QualType Type,
-                             bool IsDeclareSimd = false);
-
-  /// Called on well-formed '\#pragma omp declare simd' after parsing of
-  /// the associated method/function.
-  DeclGroupPtrTy ActOnOpenMPDeclareSimdDirective(
-      DeclGroupPtrTy DG, OMPDeclareSimdDeclAttr::BranchStateTy BS,
-      Expr *Simdlen, ArrayRef<Expr *> Uniforms, ArrayRef<Expr *> Aligneds,
-      ArrayRef<Expr *> Alignments, ArrayRef<Expr *> Linears,
-      ArrayRef<unsigned> LinModifiers, ArrayRef<Expr *> Steps, SourceRange SR);
-
-  /// Checks '\#pragma omp declare variant' variant function and original
-  /// functions after parsing of the associated method/function.
-  /// \param DG Function declaration to which declare variant directive is
-  /// applied to.
-  /// \param VariantRef Expression that references the variant function, which
-  /// must be used instead of the original one, specified in \p DG.
-  /// \param TI The trait info object representing the match clause.
-  /// \param NumAppendArgs The number of omp_interop_t arguments to account for
-  /// in checking.
-  /// \returns std::nullopt, if the function/variant function are not compatible
-  /// with the pragma, pair of original function/variant ref expression
-  /// otherwise.
-  std::optional<std::pair<FunctionDecl *, Expr *>>
-  checkOpenMPDeclareVariantFunction(DeclGroupPtrTy DG, Expr *VariantRef,
-                                    OMPTraitInfo &TI, unsigned NumAppendArgs,
-                                    SourceRange SR);
-
-  /// Called on well-formed '\#pragma omp declare variant' after parsing of
-  /// the associated method/function.
-  /// \param FD Function declaration to which declare variant directive is
-  /// applied to.
-  /// \param VariantRef Expression that references the variant function, which
-  /// must be used instead of the original one, specified in \p DG.
-  /// \param TI The context traits associated with the function variant.
-  /// \param AdjustArgsNothing The list of 'nothing' arguments.
-  /// \param AdjustArgsNeedDevicePtr The list of 'need_device_ptr' arguments.
-  /// \param AppendArgs The list of 'append_args' arguments.
-  /// \param AdjustArgsLoc The Location of an 'adjust_args' clause.
-  /// \param AppendArgsLoc The Location of an 'append_args' clause.
-  /// \param SR The SourceRange of the 'declare variant' directive.
-  void ActOnOpenMPDeclareVariantDirective(
-      FunctionDecl *FD, Expr *VariantRef, OMPTraitInfo &TI,
-      ArrayRef<Expr *> AdjustArgsNothing,
-      ArrayRef<Expr *> AdjustArgsNeedDevicePtr,
-      ArrayRef<OMPInteropInfo> AppendArgs, SourceLocation AdjustArgsLoc,
-      SourceLocation AppendArgsLoc, SourceRange SR);
-
-  OMPClause *ActOnOpenMPSingleExprClause(OpenMPClauseKind Kind, Expr *Expr,
-                                         SourceLocation StartLoc,
-                                         SourceLocation LParenLoc,
-                                         SourceLocation EndLoc);
-  /// Called on well-formed 'allocator' clause.
-  OMPClause *ActOnOpenMPAllocatorClause(Expr *Allocator,
-                                        SourceLocation StartLoc,
-                                        SourceLocation LParenLoc,
-                                        SourceLocation EndLoc);
-  /// Called on well-formed 'if' clause.
-  OMPClause *ActOnOpenMPIfClause(OpenMPDirectiveKind NameModifier,
-                                 Expr *Condition, SourceLocation StartLoc,
-                                 SourceLocation LParenLoc,
-                                 SourceLocation NameModifierLoc,
-                                 SourceLocation ColonLoc,
-                                 SourceLocation EndLoc);
-  /// Called on well-formed 'final' clause.
-  OMPClause *ActOnOpenMPFinalClause(Expr *Condition, SourceLocation StartLoc,
-                                    SourceLocation LParenLoc,
-                                    SourceLocation EndLoc);
-  /// Called on well-formed 'num_threads' clause.
-  OMPClause *ActOnOpenMPNumThreadsClause(Expr *NumThreads,
-                                         SourceLocation StartLoc,
-                                         SourceLocation LParenLoc,
-                                         SourceLocation EndLoc);
-  /// Called on well-formed 'align' clause.
-  OMPClause *ActOnOpenMPAlignClause(Expr *Alignment, SourceLocation StartLoc,
-                                    SourceLocation LParenLoc,
-                                    SourceLocation EndLoc);
-  /// Called on well-formed 'safelen' clause.
-  OMPClause *ActOnOpenMPSafelenClause(Expr *Length, SourceLocation StartLoc,
-                                      SourceLocation LParenLoc,
-                                      SourceLocation EndLoc);
-  /// Called on well-formed 'simdlen' clause.
-  OMPClause *ActOnOpenMPSimdlenClause(Expr *Length, SourceLocation StartLoc,
-                                      SourceLocation LParenLoc,
-                                      SourceLocation EndLoc);
-  /// Called on well-form 'sizes' clause.
-  OMPClause *ActOnOpenMPSizesClause(ArrayRef<Expr *> SizeExprs,
-                                    SourceLocation StartLoc,
-                                    SourceLocation LParenLoc,
-                                    SourceLocation EndLoc);
-  /// Called on well-form 'full' clauses.
-  OMPClause *ActOnOpenMPFullClause(SourceLocation StartLoc,
-                                   SourceLocation EndLoc);
-  /// Called on well-form 'partial' clauses.
-  OMPClause *ActOnOpenMPPartialClause(Expr *FactorExpr, SourceLocation StartLoc,
-                                      SourceLocation LParenLoc,
-                                      SourceLocation EndLoc);
-  /// Called on well-formed 'collapse' clause.
-  OMPClause *ActOnOpenMPCollapseClause(Expr *NumForLoops,
-                                       SourceLocation StartLoc,
-                                       SourceLocation LParenLoc,
-                                       SourceLocation EndLoc);
-  /// Called on well-formed 'ordered' clause.
-  OMPClause *
-  ActOnOpenMPOrderedClause(SourceLocation StartLoc, SourceLocation EndLoc,
-                           SourceLocation LParenLoc = SourceLocation(),
-                           Expr *NumForLoops = nullptr);
-  /// Called on well-formed 'grainsize' clause.
-  OMPClause *ActOnOpenMPGrainsizeClause(OpenMPGrainsizeClauseModifier Modifier,
-                                        Expr *Size, SourceLocation StartLoc,
-                                        SourceLocation LParenLoc,
-                                        SourceLocation ModifierLoc,
-                                        SourceLocation EndLoc);
-  /// Called on well-formed 'num_tasks' clause.
-  OMPClause *ActOnOpenMPNumTasksClause(OpenMPNumTasksClauseModifier Modifier,
-                                       Expr *NumTasks, SourceLocation StartLoc,
-                                       SourceLocation LParenLoc,
-                                       SourceLocation ModifierLoc,
-                                       SourceLocation EndLoc);
-  /// Called on well-formed 'hint' clause.
-  OMPClause *ActOnOpenMPHintClause(Expr *Hint, SourceLocation StartLoc,
-                                   SourceLocation LParenLoc,
-                                   SourceLocation EndLoc);
-  /// Called on well-formed 'detach' clause.
-  OMPClause *ActOnOpenMPDetachClause(Expr *Evt, SourceLocation StartLoc,
-                                     SourceLocation LParenLoc,
-                                     SourceLocation EndLoc);
-
-  OMPClause *ActOnOpenMPSimpleClause(OpenMPClauseKind Kind, unsigned Argument,
-                                     SourceLocation ArgumentLoc,
-                                     SourceLocation StartLoc,
-                                     SourceLocation LParenLoc,
-                                     SourceLocation EndLoc);
-  /// Called on well-formed 'when' clause.
-  OMPClause *ActOnOpenMPWhenClause(OMPTraitInfo &TI, SourceLocation StartLoc,
-                                   SourceLocation LParenLoc,
-                                   SourceLocation EndLoc);
-  /// Called on well-formed 'default' clause.
-  OMPClause *ActOnOpenMPDefaultClause(llvm::omp::DefaultKind Kind,
-                                      SourceLocation KindLoc,
-                                      SourceLocation StartLoc,
-                                      SourceLocation LParenLoc,
-                                      SourceLocation EndLoc);
-  /// Called on well-formed 'proc_bind' clause.
-  OMPClause *ActOnOpenMPProcBindClause(llvm::omp::ProcBindKind Kind,
-                                       SourceLocation KindLoc,
-                                       SourceLocation StartLoc,
-                                       SourceLocation LParenLoc,
-                                       SourceLocation EndLoc);
-  /// Called on well-formed 'order' clause.
-  OMPClause *ActOnOpenMPOrderClause(OpenMPOrderClauseModifier Modifier,
-                                    OpenMPOrderClauseKind Kind,
-                                    SourceLocation StartLoc,
-                                    SourceLocation LParenLoc,
-                                    SourceLocation MLoc, SourceLocation KindLoc,
-                                    SourceLocation EndLoc);
-  /// Called on well-formed 'update' clause.
-  OMPClause *ActOnOpenMPUpdateClause(OpenMPDependClauseKind Kind,
-                                     SourceLocation KindLoc,
-                                     SourceLocation StartLoc,
-                                     SourceLocation LParenLoc,
-                                     SourceLocation EndLoc);
-
-  OMPClause *ActOnOpenMPSingleExprWithArgClause(
-      OpenMPClauseKind Kind, ArrayRef<unsigned> Arguments, Expr *Expr,
-      SourceLocation StartLoc, SourceLocation LParenLoc,
-      ArrayRef<SourceLocation> ArgumentsLoc, SourceLocation DelimLoc,
-      SourceLocation EndLoc);
-  /// Called on well-formed 'schedule' clause.
-  OMPClause *ActOnOpenMPScheduleClause(
-      OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2,
-      OpenMPScheduleClauseKind Kind, Expr *ChunkSize, SourceLocation StartLoc,
-      SourceLocation LParenLoc, SourceLocation M1Loc, SourceLocation M2Loc,
-      SourceLocation KindLoc, SourceLocation CommaLoc, SourceLocation EndLoc);
-
-  OMPClause *ActOnOpenMPClause(OpenMPClauseKind Kind, SourceLocation StartLoc,
-                               SourceLocation EndLoc);
-  /// Called on well-formed 'nowait' clause.
-  OMPClause *ActOnOpenMPNowaitClause(SourceLocation StartLoc,
-                                     SourceLocation EndLoc);
-  /// Called on well-formed 'untied' clause.
-  OMPClause *ActOnOpenMPUntiedClause(SourceLocation StartLoc,
-                                     SourceLocation EndLoc);
-  /// Called on well-formed 'mergeable' clause.
-  OMPClause *ActOnOpenMPMergeableClause(SourceLocation StartLoc,
-                                        SourceLocation EndLoc);
-  /// Called on well-formed 'read' clause.
-  OMPClause *ActOnOpenMPReadClause(SourceLocation StartLoc,
-                                   SourceLocation EndLoc);
-  /// Called on well-formed 'write' clause.
-  OMPClause *ActOnOpenMPWriteClause(SourceLocation StartLoc,
-                                    SourceLocation EndLoc);
-  /// Called on well-formed 'update' clause.
-  OMPClause *ActOnOpenMPUpdateClause(SourceLocation StartLoc,
-                                     SourceLocation EndLoc);
-  /// Called on well-formed 'capture' clause.
-  OMPClause *ActOnOpenMPCaptureClause(SourceLocation StartLoc,
-                                      SourceLocation EndLoc);
-  /// Called on well-formed 'compare' clause.
-  OMPClause *ActOnOpenMPCompareClause(SourceLocation StartLoc,
-                                      SourceLocation EndLoc);
-  /// Called on well-formed 'fail' clause.
-  OMPClause *ActOnOpenMPFailClause(SourceLocation StartLoc,
-                                   SourceLocation EndLoc);
-  OMPClause *ActOnOpenMPFailClause(OpenMPClauseKind Kind,
-                                   SourceLocation KindLoc,
-                                   SourceLocation StartLoc,
-                                   SourceLocation LParenLoc,
-                                   SourceLocation EndLoc);
-
-  /// Called on well-formed 'seq_cst' clause.
-  OMPClause *ActOnOpenMPSeqCstClause(SourceLocation StartLoc,
-                                     SourceLocation EndLoc);
-  /// Called on well-formed 'acq_rel' clause.
-  OMPClause *ActOnOpenMPAcqRelClause(SourceLocation StartLoc,
-                                     SourceLocation EndLoc);
-  /// Called on well-formed 'acquire' clause.
-  OMPClause *ActOnOpenMPAcquireClause(SourceLocation StartLoc,
-                                      SourceLocation EndLoc);
-  /// Called on well-formed 'release' clause.
-  OMPClause *ActOnOpenMPReleaseClause(SourceLocation StartLoc,
-                                      SourceLocation EndLoc);
-  /// Called on well-formed 'relaxed' clause.
-  OMPClause *ActOnOpenMPRelaxedClause(SourceLocation StartLoc,
-                                      SourceLocation EndLoc);
-  /// Called on well-formed 'weak' clause.
-  OMPClause *ActOnOpenMPWeakClause(SourceLocation StartLoc,
-                                   SourceLocation EndLoc);
-
-  /// Called on well-formed 'init' clause.
-  OMPClause *
-  ActOnOpenMPInitClause(Expr *InteropVar, OMPInteropInfo &InteropInfo,
-                        SourceLocation StartLoc, SourceLocation LParenLoc,
-                        SourceLocation VarLoc, SourceLocation EndLoc);
-
-  /// Called on well-formed 'use' clause.
-  OMPClause *ActOnOpenMPUseClause(Expr *InteropVar, SourceLocation StartLoc,
-                                  SourceLocation LParenLoc,
-                                  SourceLocation VarLoc, SourceLocation EndLoc);
-
-  /// Called on well-formed 'destroy' clause.
-  OMPClause *ActOnOpenMPDestroyClause(Expr *InteropVar, SourceLocation StartLoc,
-                                      SourceLocation LParenLoc,
-                                      SourceLocation VarLoc,
-                                      SourceLocation EndLoc);
-  /// Called on well-formed 'novariants' clause.
-  OMPClause *ActOnOpenMPNovariantsClause(Expr *Condition,
-                                         SourceLocation StartLoc,
-                                         SourceLocation LParenLoc,
-                                         SourceLocation EndLoc);
-  /// Called on well-formed 'nocontext' clause.
-  OMPClause *ActOnOpenMPNocontextClause(Expr *Condition,
-                                        SourceLocation StartLoc,
-                                        SourceLocation LParenLoc,
-                                        SourceLocation EndLoc);
-  /// Called on well-formed 'filter' clause.
-  OMPClause *ActOnOpenMPFilterClause(Expr *ThreadID, SourceLocation StartLoc,
-                                     SourceLocation LParenLoc,
-                                     SourceLocation EndLoc);
-  /// Called on well-formed 'threads' clause.
-  OMPClause *ActOnOpenMPThreadsClause(SourceLocation StartLoc,
-                                      SourceLocation EndLoc);
-  /// Called on well-formed 'simd' clause.
-  OMPClause *ActOnOpenMPSIMDClause(SourceLocation StartLoc,
-                                   SourceLocation EndLoc);
-  /// Called on well-formed 'nogroup' clause.
-  OMPClause *ActOnOpenMPNogroupClause(SourceLocation StartLoc,
-                                      SourceLocation EndLoc);
-  /// Called on well-formed 'unified_address' clause.
-  OMPClause *ActOnOpenMPUnifiedAddressClause(SourceLocation StartLoc,
-                                             SourceLocation EndLoc);
-
-  /// Called on well-formed 'unified_address' clause.
-  OMPClause *ActOnOpenMPUnifiedSharedMemoryClause(SourceLocation StartLoc,
-                                                  SourceLocation EndLoc);
-
-  /// Called on well-formed 'reverse_offload' clause.
-  OMPClause *ActOnOpenMPReverseOffloadClause(SourceLocation StartLoc,
-                                             SourceLocation EndLoc);
-
-  /// Called on well-formed 'dynamic_allocators' clause.
-  OMPClause *ActOnOpenMPDynamicAllocatorsClause(SourceLocation StartLoc,
-                                                SourceLocation EndLoc);
-
-  /// Called on well-formed 'atomic_default_mem_order' clause.
-  OMPClause *ActOnOpenMPAtomicDefaultMemOrderClause(
-      OpenMPAtomicDefaultMemOrderClauseKind Kind, SourceLocation KindLoc,
-      SourceLocation StartLoc, SourceLocation LParenLoc, SourceLocation EndLoc);
-
-  /// Called on well-formed 'at' clause.
-  OMPClause *ActOnOpenMPAtClause(OpenMPAtClauseKind Kind,
-                                 SourceLocation KindLoc,
-                                 SourceLocation StartLoc,
-                                 SourceLocation LParenLoc,
-                                 SourceLocation EndLoc);
-
-  /// Called on well-formed 'severity' clause.
-  OMPClause *ActOnOpenMPSeverityClause(OpenMPSeverityClauseKind Kind,
-                                       SourceLocation KindLoc,
-                                       SourceLocation StartLoc,
-                                       SourceLocation LParenLoc,
-                                       SourceLocation EndLoc);
-
-  /// Called on well-formed 'message' clause.
-  /// passing string for message.
-  OMPClause *ActOnOpenMPMessageClause(Expr *MS, SourceLocation StartLoc,
-                                      SourceLocation LParenLoc,
-                                      SourceLocation EndLoc);
-
-  /// Data used for processing a list of variables in OpenMP clauses.
-  struct OpenMPVarListDataTy final {
-    Expr *DepModOrTailExpr = nullptr;
-    Expr *IteratorExpr = nullptr;
-    SourceLocation ColonLoc;
-    SourceLocation RLoc;
-    CXXScopeSpec ReductionOrMapperIdScopeSpec;
-    DeclarationNameInfo ReductionOrMapperId;
-    int ExtraModifier = -1; ///< Additional modifier for linear, map, depend or
-                            ///< lastprivate clause.
-    SmallVector<OpenMPMapModifierKind, NumberOfOMPMapClauseModifiers>
-        MapTypeModifiers;
-    SmallVector<SourceLocation, NumberOfOMPMapClauseModifiers>
-        MapTypeModifiersLoc;
-    SmallVector<OpenMPMotionModifierKind, NumberOfOMPMotionModifiers>
-        MotionModifiers;
-    SmallVector<SourceLocation, NumberOfOMPMotionModifiers> MotionModifiersLoc;
-    bool IsMapTypeImplicit = false;
-    SourceLocation ExtraModifierLoc;
-    SourceLocation OmpAllMemoryLoc;
-    SourceLocation
-        StepModifierLoc; /// 'step' modifier location for linear clause
-  };
-
-  OMPClause *ActOnOpenMPVarListClause(OpenMPClauseKind Kind,
-                                      ArrayRef<Expr *> Vars,
-                                      const OMPVarListLocTy &Locs,
-                                      OpenMPVarListDataTy &Data);
-  /// Called on well-formed 'inclusive' clause.
-  OMPClause *ActOnOpenMPInclusiveClause(ArrayRef<Expr *> VarList,
-                                        SourceLocation StartLoc,
-                                        SourceLocation LParenLoc,
-                                        SourceLocation EndLoc);
-  /// Called on well-formed 'exclusive' clause.
-  OMPClause *ActOnOpenMPExclusiveClause(ArrayRef<Expr *> VarList,
-                                        SourceLocation StartLoc,
-                                        SourceLocation LParenLoc,
-                                        SourceLocation EndLoc);
-  /// Called on well-formed 'allocate' clause.
-  OMPClause *
-  ActOnOpenMPAllocateClause(Expr *Allocator, ArrayRef<Expr *> VarList,
-                            SourceLocation StartLoc, SourceLocation ColonLoc,
-                            SourceLocation LParenLoc, SourceLocation EndLoc);
-  /// Called on well-formed 'private' clause.
-  OMPClause *ActOnOpenMPPrivateClause(ArrayRef<Expr *> VarList,
-                                      SourceLocation StartLoc,
-                                      SourceLocation LParenLoc,
-                                      SourceLocation EndLoc);
-  /// Called on well-formed 'firstprivate' clause.
-  OMPClause *ActOnOpenMPFirstprivateClause(ArrayRef<Expr *> VarList,
-                                           SourceLocation StartLoc,
-                                           SourceLocation LParenLoc,
-                                           SourceLocation EndLoc);
-  /// Called on well-formed 'lastprivate' clause.
-  OMPClause *ActOnOpenMPLastprivateClause(
-      ArrayRef<Expr *> VarList, OpenMPLastprivateModifier LPKind,
-      SourceLocation LPKindLoc, SourceLocation ColonLoc,
-      SourceLocation StartLoc, SourceLocation LParenLoc, SourceLocation EndLoc);
-  /// Called on well-formed 'shared' clause.
-  OMPClause *ActOnOpenMPSharedClause(ArrayRef<Expr *> VarList,
-                                     SourceLocation StartLoc,
-                                     SourceLocation LParenLoc,
-                                     SourceLocation EndLoc);
-  /// Called on well-formed 'reduction' clause.
-  OMPClause *ActOnOpenMPReductionClause(
-      ArrayRef<Expr *> VarList, OpenMPReductionClauseModifier Modifier,
-      SourceLocation StartLoc, SourceLocation LParenLoc,
-      SourceLocation ModifierLoc, SourceLocation ColonLoc,
-      SourceLocation EndLoc, CXXScopeSpec &ReductionIdScopeSpec,
-      const DeclarationNameInfo &ReductionId,
-      ArrayRef<Expr *> UnresolvedReductions = std::nullopt);
-  /// Called on well-formed 'task_reduction' clause.
-  OMPClause *ActOnOpenMPTaskReductionClause(
-      ArrayRef<Expr *> VarList, SourceLocation StartLoc,
-      SourceLocation LParenLoc, SourceLocation ColonLoc, SourceLocation EndLoc,
-      CXXScopeSpec &ReductionIdScopeSpec,
-      const DeclarationNameInfo &ReductionId,
-      ArrayRef<Expr *> UnresolvedReductions = std::nullopt);
-  /// Called on well-formed 'in_reduction' clause.
-  OMPClause *ActOnOpenMPInReductionClause(
-      ArrayRef<Expr *> VarList, SourceLocation StartLoc,
-      SourceLocation LParenLoc, SourceLocation ColonLoc, SourceLocation EndLoc,
-      CXXScopeSpec &ReductionIdScopeSpec,
-      const DeclarationNameInfo &ReductionId,
-      ArrayRef<Expr *> UnresolvedReductions = std::nullopt);
-  /// Called on well-formed 'linear' clause.
-  OMPClause *ActOnOpenMPLinearClause(
-      ArrayRef<Expr *> VarList, Expr *Step, SourceLocation StartLoc,
-      SourceLocation LParenLoc, OpenMPLinearClauseKind LinKind,
-      SourceLocation LinLoc, SourceLocation ColonLoc,
-      SourceLocation StepModifierLoc, SourceLocation EndLoc);
-  /// Called on well-formed 'aligned' clause.
-  OMPClause *ActOnOpenMPAlignedClause(ArrayRef<Expr *> VarList, Expr *Alignment,
-                                      SourceLocation StartLoc,
-                                      SourceLocation LParenLoc,
-                                      SourceLocation ColonLoc,
-                                      SourceLocation EndLoc);
-  /// Called on well-formed 'copyin' clause.
-  OMPClause *ActOnOpenMPCopyinClause(ArrayRef<Expr *> VarList,
-                                     SourceLocation StartLoc,
-                                     SourceLocation LParenLoc,
-                                     SourceLocation EndLoc);
-  /// Called on well-formed 'copyprivate' clause.
-  OMPClause *ActOnOpenMPCopyprivateClause(ArrayRef<Expr *> VarList,
-                                          SourceLocation StartLoc,
-                                          SourceLocation LParenLoc,
-                                          SourceLocation EndLoc);
-  /// Called on well-formed 'flush' pseudo clause.
-  OMPClause *ActOnOpenMPFlushClause(ArrayRef<Expr *> VarList,
-                                    SourceLocation StartLoc,
-                                    SourceLocation LParenLoc,
-                                    SourceLocation EndLoc);
-  /// Called on well-formed 'depobj' pseudo clause.
-  OMPClause *ActOnOpenMPDepobjClause(Expr *Depobj, SourceLocation StartLoc,
-                                     SourceLocation LParenLoc,
-                                     SourceLocation EndLoc);
-  /// Called on well-formed 'depend' clause.
-  OMPClause *ActOnOpenMPDependClause(const OMPDependClause::DependDataTy &Data,
-                                     Expr *DepModifier,
-                                     ArrayRef<Expr *> VarList,
-                                     SourceLocation StartLoc,
-                                     SourceLocation LParenLoc,
-                                     SourceLocation EndLoc);
-  /// Called on well-formed 'device' clause.
-  OMPClause *ActOnOpenMPDeviceClause(OpenMPDeviceClauseModifier Modifier,
-                                     Expr *Device, SourceLocation StartLoc,
-                                     SourceLocation LParenLoc,
-                                     SourceLocation ModifierLoc,
-                                     SourceLocation EndLoc);
-  /// Called on well-formed 'map' clause.
-  OMPClause *ActOnOpenMPMapClause(
-      Expr *IteratorModifier, ArrayRef<OpenMPMapModifierKind> MapTypeModifiers,
-      ArrayRef<SourceLocation> MapTypeModifiersLoc,
-      CXXScopeSpec &MapperIdScopeSpec, DeclarationNameInfo &MapperId,
-      OpenMPMapClauseKind MapType, bool IsMapTypeImplicit,
-      SourceLocation MapLoc, SourceLocation ColonLoc, ArrayRef<Expr *> VarList,
-      const OMPVarListLocTy &Locs, bool NoDiagnose = false,
-      ArrayRef<Expr *> UnresolvedMappers = std::nullopt);
-  /// Called on well-formed 'num_teams' clause.
-  OMPClause *ActOnOpenMPNumTeamsClause(Expr *NumTeams, SourceLocation StartLoc,
-                                       SourceLocation LParenLoc,
-                                       SourceLocation EndLoc);
-  /// Called on well-formed 'thread_limit' clause.
-  OMPClause *ActOnOpenMPThreadLimitClause(Expr *ThreadLimit,
-                                          SourceLocation StartLoc,
-                                          SourceLocation LParenLoc,
-                                          SourceLocation EndLoc);
-  /// Called on well-formed 'priority' clause.
-  OMPClause *ActOnOpenMPPriorityClause(Expr *Priority, SourceLocation StartLoc,
-                                       SourceLocation LParenLoc,
-                                       SourceLocation EndLoc);
-  /// Called on well-formed 'dist_schedule' clause.
-  OMPClause *ActOnOpenMPDistScheduleClause(
-      OpenMPDistScheduleClauseKind Kind, Expr *ChunkSize,
-      SourceLocation StartLoc, SourceLocation LParenLoc, SourceLocation KindLoc,
-      SourceLocation CommaLoc, SourceLocation EndLoc);
-  /// Called on well-formed 'defaultmap' clause.
-  OMPClause *ActOnOpenMPDefaultmapClause(
-      OpenMPDefaultmapClauseModifier M, OpenMPDefaultmapClauseKind Kind,
-      SourceLocation StartLoc, SourceLocation LParenLoc, SourceLocation MLoc,
-      SourceLocation KindLoc, SourceLocation EndLoc);
-  /// Called on well-formed 'to' clause.
-  OMPClause *
-  ActOnOpenMPToClause(ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
-                      ArrayRef<SourceLocation> MotionModifiersLoc,
-                      CXXScopeSpec &MapperIdScopeSpec,
-                      DeclarationNameInfo &MapperId, SourceLocation ColonLoc,
-                      ArrayRef<Expr *> VarList, const OMPVarListLocTy &Locs,
-                      ArrayRef<Expr *> UnresolvedMappers = std::nullopt);
-  /// Called on well-formed 'from' clause.
-  OMPClause *
-  ActOnOpenMPFromClause(ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
-                        ArrayRef<SourceLocation> MotionModifiersLoc,
-                        CXXScopeSpec &MapperIdScopeSpec,
-                        DeclarationNameInfo &MapperId, SourceLocation ColonLoc,
-                        ArrayRef<Expr *> VarList, const OMPVarListLocTy &Locs,
-                        ArrayRef<Expr *> UnresolvedMappers = std::nullopt);
-  /// Called on well-formed 'use_device_ptr' clause.
-  OMPClause *ActOnOpenMPUseDevicePtrClause(ArrayRef<Expr *> VarList,
-                                           const OMPVarListLocTy &Locs);
-  /// Called on well-formed 'use_device_addr' clause.
-  OMPClause *ActOnOpenMPUseDeviceAddrClause(ArrayRef<Expr *> VarList,
-                                            const OMPVarListLocTy &Locs);
-  /// Called on well-formed 'is_device_ptr' clause.
-  OMPClause *ActOnOpenMPIsDevicePtrClause(ArrayRef<Expr *> VarList,
-                                          const OMPVarListLocTy &Locs);
-  /// Called on well-formed 'has_device_addr' clause.
-  OMPClause *ActOnOpenMPHasDeviceAddrClause(ArrayRef<Expr *> VarList,
-                                            const OMPVarListLocTy &Locs);
-  /// Called on well-formed 'nontemporal' clause.
-  OMPClause *ActOnOpenMPNontemporalClause(ArrayRef<Expr *> VarList,
-                                          SourceLocation StartLoc,
-                                          SourceLocation LParenLoc,
-                                          SourceLocation EndLoc);
-
-  /// Data for list of allocators.
-  struct UsesAllocatorsData {
-    /// Allocator.
-    Expr *Allocator = nullptr;
-    /// Allocator traits.
-    Expr *AllocatorTraits = nullptr;
-    /// Locations of '(' and ')' symbols.
-    SourceLocation LParenLoc, RParenLoc;
-  };
-  /// Called on well-formed 'uses_allocators' clause.
-  OMPClause *ActOnOpenMPUsesAllocatorClause(SourceLocation StartLoc,
-                                            SourceLocation LParenLoc,
-                                            SourceLocation EndLoc,
-                                            ArrayRef<UsesAllocatorsData> Data);
-  /// Called on well-formed 'affinity' clause.
-  OMPClause *ActOnOpenMPAffinityClause(SourceLocation StartLoc,
-                                       SourceLocation LParenLoc,
-                                       SourceLocation ColonLoc,
-                                       SourceLocation EndLoc, Expr *Modifier,
-                                       ArrayRef<Expr *> Locators);
-  /// Called on a well-formed 'bind' clause.
-  OMPClause *ActOnOpenMPBindClause(OpenMPBindClauseKind Kind,
-                                   SourceLocation KindLoc,
-                                   SourceLocation StartLoc,
-                                   SourceLocation LParenLoc,
-                                   SourceLocation EndLoc);
-
-  /// Called on a well-formed 'ompx_dyn_cgroup_mem' clause.
-  OMPClause *ActOnOpenMPXDynCGroupMemClause(Expr *Size, SourceLocation StartLoc,
-                                            SourceLocation LParenLoc,
-                                            SourceLocation EndLoc);
-
-  /// Called on well-formed 'doacross' clause.
-  OMPClause *
-  ActOnOpenMPDoacrossClause(OpenMPDoacrossClauseModifier DepType,
-                            SourceLocation DepLoc, SourceLocation ColonLoc,
-                            ArrayRef<Expr *> VarList, SourceLocation StartLoc,
-                            SourceLocation LParenLoc, SourceLocation EndLoc);
-
-  /// Called on a well-formed 'ompx_attribute' clause.
-  OMPClause *ActOnOpenMPXAttributeClause(ArrayRef<const Attr *> Attrs,
-                                         SourceLocation StartLoc,
-                                         SourceLocation LParenLoc,
-                                         SourceLocation EndLoc);
-
-  /// Called on a well-formed 'ompx_bare' clause.
-  OMPClause *ActOnOpenMPXBareClause(SourceLocation StartLoc,
-                                    SourceLocation EndLoc);
-
-private:
-  void *VarDataSharingAttributesStack;
-
-  /// Number of nested '#pragma omp declare target' directives.
-  SmallVector<DeclareTargetContextInfo, 4> DeclareTargetNesting;
-
-  /// Initialization of data-sharing attributes stack.
-  void InitDataSharingAttributesStack();
-  void DestroyDataSharingAttributesStack();
-
-  /// Returns OpenMP nesting level for current directive.
-  unsigned getOpenMPNestingLevel() const;
-
-  /// Adjusts the function scopes index for the target-based regions.
-  void adjustOpenMPTargetScopeIndex(unsigned &FunctionScopesIndex,
-                                    unsigned Level) const;
-
-  /// Returns the number of scopes associated with the construct on the given
-  /// OpenMP level.
-  int getNumberOfConstructScopes(unsigned Level) const;
-
-  /// Push new OpenMP function region for non-capturing function.
-  void pushOpenMPFunctionRegion();
-
-  /// Pop OpenMP function region for non-capturing function.
-  void popOpenMPFunctionRegion(const sema::FunctionScopeInfo *OldFSI);
-
-  /// Analyzes and checks a loop nest for use by a loop transformation.
-  ///
-  /// \param Kind          The loop transformation directive kind.
-  /// \param NumLoops      How many nested loops the directive is expecting.
-  /// \param AStmt         Associated statement of the transformation directive.
-  /// \param LoopHelpers   [out] The loop analysis result.
-  /// \param Body          [out] The body code nested in \p NumLoops loop.
-  /// \param OriginalInits [out] Collection of statements and declarations that
-  ///                      must have been executed/declared before entering the
-  ///                      loop.
-  ///
-  /// \return Whether there was any error.
-  bool checkTransformableLoopNest(
-      OpenMPDirectiveKind Kind, Stmt *AStmt, int NumLoops,
-      SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
-      Stmt *&Body,
-      SmallVectorImpl<SmallVector<llvm::PointerUnion<Stmt *, Decl *>, 0>>
-          &OriginalInits);
-
-  /// Helper to keep information about the current `omp begin/end declare
-  /// variant` nesting.
-  struct OMPDeclareVariantScope {
-    /// The associated OpenMP context selector.
-    OMPTraitInfo *TI;
-
-    /// The associated OpenMP context selector mangling.
-    std::string NameSuffix;
-
-    OMPDeclareVariantScope(OMPTraitInfo &TI);
-  };
-
-  /// Return the OMPTraitInfo for the surrounding scope, if any.
-  OMPTraitInfo *getOMPTraitInfoForSurroundingScope() {
-    return OMPDeclareVariantScopes.empty() ? nullptr
-                                           : OMPDeclareVariantScopes.back().TI;
-  }
-
-  /// The current `omp begin/end declare variant` scopes.
-  SmallVector<OMPDeclareVariantScope, 4> OMPDeclareVariantScopes;
-
-  /// The current `omp begin/end assumes` scopes.
-  SmallVector<OMPAssumeAttr *, 4> OMPAssumeScoped;
-
-  /// All `omp assumes` we encountered so far.
-  SmallVector<OMPAssumeAttr *, 4> OMPAssumeGlobal;
-
-  /// OMPD_loop is mapped to OMPD_for, OMPD_distribute or OMPD_simd depending
-  /// on the parameter of the bind clause. In the methods for the
-  /// mapped directives, check the parameters of the lastprivate clause.
-  bool checkLastPrivateForMappedDirectives(ArrayRef<OMPClause *> Clauses);
-  /// Depending on the bind clause of OMPD_loop map the directive to new
-  /// directives.
-  ///    1) loop bind(parallel) --> OMPD_for
-  ///    2) loop bind(teams) --> OMPD_distribute
-  ///    3) loop bind(thread) --> OMPD_simd
-  /// This is being handled in Sema instead of Codegen because of the need for
-  /// rigorous semantic checking in the new mapped directives.
-  bool mapLoopConstruct(llvm::SmallVector<OMPClause *> &ClausesWithoutBind,
-                        ArrayRef<OMPClause *> Clauses,
-                        OpenMPBindClauseKind &BindKind,
-                        OpenMPDirectiveKind &Kind,
-                        OpenMPDirectiveKind &PrevMappedDirective,
-                        SourceLocation StartLoc, SourceLocation EndLoc,
-                        const DeclarationNameInfo &DirName,
-                        OpenMPDirectiveKind CancelRegion);
-
-  ///@}
 };
 
 DeductionFailureInfo
diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
new file mode 100644
index 000000000000..9927459bbc59
--- /dev/null
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -0,0 +1,1447 @@
+//===----- SemaOpenMP.h -- Semantic Analysis for OpenMP constructs -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+/// This file declares semantic analysis for OpenMP constructs and
+/// clauses.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_SEMA_SEMAOPENMP_H
+#define LLVM_CLANG_SEMA_SEMAOPENMP_H
+
+#include "clang/AST/Attr.h"
+#include "clang/AST/Decl.h"
+#include "clang/AST/DeclBase.h"
+#include "clang/AST/DeclOpenMP.h"
+#include "clang/AST/DeclarationName.h"
+#include "clang/AST/Expr.h"
+#include "clang/AST/ExprOpenMP.h"
+#include "clang/AST/OpenMPClause.h"
+#include "clang/AST/Stmt.h"
+#include "clang/AST/StmtOpenMP.h"
+#include "clang/AST/Type.h"
+#include "clang/Basic/IdentifierTable.h"
+#include "clang/Basic/LLVM.h"
+#include "clang/Basic/OpenMPKinds.h"
+#include "clang/Basic/SourceLocation.h"
+#include "clang/Basic/Specifiers.h"
+#include "clang/Sema/DeclSpec.h"
+#include "clang/Sema/Ownership.h"
+#include "clang/Sema/Scope.h"
+#include "clang/Sema/ScopeInfo.h"
+#include "clang/Sema/SemaBase.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/PointerUnion.h"
+#include <optional>
+#include <string>
+#include <utility>
+
+namespace clang {
+
+class SemaOpenMP : public SemaBase {
+public:
+  SemaOpenMP(Sema &S);
+
+  friend class Parser;
+  friend class Sema;
+
+  using DeclGroupPtrTy = OpaquePtr<DeclGroupRef>;
+  using CapturedParamNameType = std::pair<StringRef, QualType>;
+
+  /// Creates a SemaDiagnosticBuilder that emits the diagnostic if the current
+  /// context is "used as device code".
+  ///
+  /// - If CurContext is a `declare target` function or it is known that the
+  /// function is emitted for the device, emits the diagnostics immediately.
+  /// - If CurContext is a non-`declare target` function and we are compiling
+  ///   for the device, creates a diagnostic which is emitted if and when we
+  ///   realize that the function will be codegen'ed.
+  ///
+  /// Example usage:
+  ///
+  ///  // Variable-length arrays are not allowed in NVPTX device code.
+  ///  if (diagIfOpenMPDeviceCode(Loc, diag::err_vla_unsupported))
+  ///    return ExprError();
+  ///  // Otherwise, continue parsing as normal.
+  SemaDiagnosticBuilder diagIfOpenMPDeviceCode(SourceLocation Loc,
+                                               unsigned DiagID,
+                                               const FunctionDecl *FD);
+
+  /// Creates a SemaDiagnosticBuilder that emits the diagnostic if the current
+  /// context is "used as host code".
+  ///
+  /// - If CurContext is a `declare target` function or it is known that the
+  /// function is emitted for the host, emits the diagnostics immediately.
+  /// - If CurContext is a non-host function, just ignore it.
+  ///
+  /// Example usage:
+  ///
+  ///  // Variable-length arrays are not allowed in NVPTX device code.
+  ///  if (diagIfOpenMPHostode(Loc, diag::err_vla_unsupported))
+  ///    return ExprError();
+  ///  // Otherwise, continue parsing as normal.
+  SemaDiagnosticBuilder diagIfOpenMPHostCode(SourceLocation Loc,
+                                             unsigned DiagID,
+                                             const FunctionDecl *FD);
+
+  /// The declarator \p D defines a function in the scope \p S which is nested
+  /// in an `omp begin/end declare variant` scope. In this method we create a
+  /// declaration for \p D and rename \p D according to the OpenMP context
+  /// selector of the surrounding scope. Return all base functions in \p Bases.
+  void ActOnStartOfFunctionDefinitionInOpenMPDeclareVariantScope(
+      Scope *S, Declarator &D, MultiTemplateParamsArg TemplateParameterLists,
+      SmallVectorImpl<FunctionDecl *> &Bases);
+
+  /// Register \p D as specialization of all base functions in \p Bases in the
+  /// current `omp begin/end declare variant` scope.
+  void ActOnFinishedFunctionDefinitionInOpenMPDeclareVariantScope(
+      Decl *D, SmallVectorImpl<FunctionDecl *> &Bases);
+
+  /// Act on \p D, a function definition inside of an `omp [begin/end] assumes`.
+  void ActOnFinishedFunctionDefinitionInOpenMPAssumeScope(Decl *D);
+
+  /// Can we exit an OpenMP declare variant scope at the moment.
+  bool isInOpenMPDeclareVariantScope() const {
+    return !OMPDeclareVariantScopes.empty();
+  }
+
+  ExprResult
+  VerifyPositiveIntegerConstantInClause(Expr *Op, OpenMPClauseKind CKind,
+                                        bool StrictlyPositive = true,
+                                        bool SuppressExprDiags = false);
+
+  /// Given the potential call expression \p Call, determine if there is a
+  /// specialization via the OpenMP declare variant mechanism available. If
+  /// there is, return the specialized call expression, otherwise return the
+  /// original \p Call.
+  ExprResult ActOnOpenMPCall(ExprResult Call, Scope *Scope,
+                             SourceLocation LParenLoc, MultiExprArg ArgExprs,
+                             SourceLocation RParenLoc, Expr *ExecConfig);
+
+  /// Handle a `omp begin declare variant`.
+  void ActOnOpenMPBeginDeclareVariant(SourceLocation Loc, OMPTraitInfo &TI);
+
+  /// Handle a `omp end declare variant`.
+  void ActOnOpenMPEndDeclareVariant();
+
+  /// Function tries to capture lambda's captured variables in the OpenMP region
+  /// before the original lambda is captured.
+  void tryCaptureOpenMPLambdas(ValueDecl *V);
+
+  /// Return true if the provided declaration \a VD should be captured by
+  /// reference.
+  /// \param Level Relative level of nested OpenMP construct for that the check
+  /// is performed.
+  /// \param OpenMPCaptureLevel Capture level within an OpenMP construct.
+  bool isOpenMPCapturedByRef(const ValueDecl *D, unsigned Level,
+                             unsigned OpenMPCaptureLevel) const;
+
+  /// Check if the specified variable is used in one of the private
+  /// clauses (private, firstprivate, lastprivate, reduction etc.) in OpenMP
+  /// constructs.
+  VarDecl *isOpenMPCapturedDecl(ValueDecl *D, bool CheckScopeInfo = false,
+                                unsigned StopAt = 0);
+
+  /// The member expression(this->fd) needs to be rebuilt in the template
+  /// instantiation to generate private copy for OpenMP when default
+  /// clause is used. The function will return true if default
+  /// cluse is used.
+  bool isOpenMPRebuildMemberExpr(ValueDecl *D);
+
+  ExprResult getOpenMPCapturedExpr(VarDecl *Capture, ExprValueKind VK,
+                                   ExprObjectKind OK, SourceLocation Loc);
+
+  /// If the current region is a loop-based region, mark the start of the loop
+  /// construct.
+  void startOpenMPLoop();
+
+  /// If the current region is a range loop-based region, mark the start of the
+  /// loop construct.
+  void startOpenMPCXXRangeFor();
+
+  /// Check if the specified variable is used in 'private' clause.
+  /// \param Level Relative level of nested OpenMP construct for that the check
+  /// is performed.
+  OpenMPClauseKind isOpenMPPrivateDecl(ValueDecl *D, unsigned Level,
+                                       unsigned CapLevel) const;
+
+  /// Sets OpenMP capture kind (OMPC_private, OMPC_firstprivate, OMPC_map etc.)
+  /// for \p FD based on DSA for the provided corresponding captured declaration
+  /// \p D.
+  void setOpenMPCaptureKind(FieldDecl *FD, const ValueDecl *D, unsigned Level);
+
+  /// Check if the specified variable is captured  by 'target' directive.
+  /// \param Level Relative level of nested OpenMP construct for that the check
+  /// is performed.
+  bool isOpenMPTargetCapturedDecl(const ValueDecl *D, unsigned Level,
+                                  unsigned CaptureLevel) const;
+
+  /// Check if the specified global variable must be captured  by outer capture
+  /// regions.
+  /// \param Level Relative level of nested OpenMP construct for that
+  /// the check is performed.
+  bool isOpenMPGlobalCapturedDecl(ValueDecl *D, unsigned Level,
+                                  unsigned CaptureLevel) const;
+
+  ExprResult PerformOpenMPImplicitIntegerConversion(SourceLocation OpLoc,
+                                                    Expr *Op);
+  /// Called on start of new data sharing attribute block.
+  void StartOpenMPDSABlock(OpenMPDirectiveKind K,
+                           const DeclarationNameInfo &DirName, Scope *CurScope,
+                           SourceLocation Loc);
+  /// Start analysis of clauses.
+  void StartOpenMPClause(OpenMPClauseKind K);
+  /// End analysis of clauses.
+  void EndOpenMPClause();
+  /// Called on end of data sharing attribute block.
+  void EndOpenMPDSABlock(Stmt *CurDirective);
+
+  /// Check if the current region is an OpenMP loop region and if it is,
+  /// mark loop control variable, used in \p Init for loop initialization, as
+  /// private by default.
+  /// \param Init First part of the for loop.
+  void ActOnOpenMPLoopInitialization(SourceLocation ForLoc, Stmt *Init);
+
+  /// Called on well-formed '\#pragma omp metadirective' after parsing
+  /// of the  associated statement.
+  StmtResult ActOnOpenMPMetaDirective(ArrayRef<OMPClause *> Clauses,
+                                      Stmt *AStmt, SourceLocation StartLoc,
+                                      SourceLocation EndLoc);
+
+  // OpenMP directives and clauses.
+  /// Called on correct id-expression from the '#pragma omp
+  /// threadprivate'.
+  ExprResult ActOnOpenMPIdExpression(Scope *CurScope, CXXScopeSpec &ScopeSpec,
+                                     const DeclarationNameInfo &Id,
+                                     OpenMPDirectiveKind Kind);
+  /// Called on well-formed '#pragma omp threadprivate'.
+  DeclGroupPtrTy ActOnOpenMPThreadprivateDirective(SourceLocation Loc,
+                                                   ArrayRef<Expr *> VarList);
+  /// Builds a new OpenMPThreadPrivateDecl and checks its correctness.
+  OMPThreadPrivateDecl *CheckOMPThreadPrivateDecl(SourceLocation Loc,
+                                                  ArrayRef<Expr *> VarList);
+  /// Called on well-formed '#pragma omp allocate'.
+  DeclGroupPtrTy ActOnOpenMPAllocateDirective(SourceLocation Loc,
+                                              ArrayRef<Expr *> VarList,
+                                              ArrayRef<OMPClause *> Clauses,
+                                              DeclContext *Owner = nullptr);
+
+  /// Called on well-formed '#pragma omp [begin] assume[s]'.
+  void ActOnOpenMPAssumesDirective(SourceLocation Loc,
+                                   OpenMPDirectiveKind DKind,
+                                   ArrayRef<std::string> Assumptions,
+                                   bool SkippedClauses);
+
+  /// Check if there is an active global `omp begin assumes` directive.
+  bool isInOpenMPAssumeScope() const { return !OMPAssumeScoped.empty(); }
+
+  /// Check if there is an active global `omp assumes` directive.
+  bool hasGlobalOpenMPAssumes() const { return !OMPAssumeGlobal.empty(); }
+
+  /// Called on well-formed '#pragma omp end assumes'.
+  void ActOnOpenMPEndAssumesDirective();
+
+  /// Called on well-formed '#pragma omp requires'.
+  DeclGroupPtrTy ActOnOpenMPRequiresDirective(SourceLocation Loc,
+                                              ArrayRef<OMPClause *> ClauseList);
+  /// Check restrictions on Requires directive
+  OMPRequiresDecl *CheckOMPRequiresDecl(SourceLocation Loc,
+                                        ArrayRef<OMPClause *> Clauses);
+  /// Check if the specified type is allowed to be used in 'omp declare
+  /// reduction' construct.
+  QualType ActOnOpenMPDeclareReductionType(SourceLocation TyLoc,
+                                           TypeResult ParsedType);
+  /// Called on start of '#pragma omp declare reduction'.
+  DeclGroupPtrTy ActOnOpenMPDeclareReductionDirectiveStart(
+      Scope *S, DeclContext *DC, DeclarationName Name,
+      ArrayRef<std::pair<QualType, SourceLocation>> ReductionTypes,
+      AccessSpecifier AS, Decl *PrevDeclInScope = nullptr);
+  /// Initialize declare reduction construct initializer.
+  void ActOnOpenMPDeclareReductionCombinerStart(Scope *S, Decl *D);
+  /// Finish current declare reduction construct initializer.
+  void ActOnOpenMPDeclareReductionCombinerEnd(Decl *D, Expr *Combiner);
+  /// Initialize declare reduction construct initializer.
+  /// \return omp_priv variable.
+  VarDecl *ActOnOpenMPDeclareReductionInitializerStart(Scope *S, Decl *D);
+  /// Finish current declare reduction construct initializer.
+  void ActOnOpenMPDeclareReductionInitializerEnd(Decl *D, Expr *Initializer,
+                                                 VarDecl *OmpPrivParm);
+  /// Called at the end of '#pragma omp declare reduction'.
+  DeclGroupPtrTy ActOnOpenMPDeclareReductionDirectiveEnd(
+      Scope *S, DeclGroupPtrTy DeclReductions, bool IsValid);
+
+  /// Check variable declaration in 'omp declare mapper' construct.
+  TypeResult ActOnOpenMPDeclareMapperVarDecl(Scope *S, Declarator &D);
+  /// Check if the specified type is allowed to be used in 'omp declare
+  /// mapper' construct.
+  QualType ActOnOpenMPDeclareMapperType(SourceLocation TyLoc,
+                                        TypeResult ParsedType);
+  /// Called on start of '#pragma omp declare mapper'.
+  DeclGroupPtrTy ActOnOpenMPDeclareMapperDirective(
+      Scope *S, DeclContext *DC, DeclarationName Name, QualType MapperType,
+      SourceLocation StartLoc, DeclarationName VN, AccessSpecifier AS,
+      Expr *MapperVarRef, ArrayRef<OMPClause *> Clauses,
+      Decl *PrevDeclInScope = nullptr);
+  /// Build the mapper variable of '#pragma omp declare mapper'.
+  ExprResult ActOnOpenMPDeclareMapperDirectiveVarDecl(Scope *S,
+                                                      QualType MapperType,
+                                                      SourceLocation StartLoc,
+                                                      DeclarationName VN);
+  void ActOnOpenMPIteratorVarDecl(VarDecl *VD);
+  bool isOpenMPDeclareMapperVarDeclAllowed(const VarDecl *VD) const;
+  const ValueDecl *getOpenMPDeclareMapperVarName() const;
+
+  struct DeclareTargetContextInfo {
+    struct MapInfo {
+      OMPDeclareTargetDeclAttr::MapTypeTy MT;
+      SourceLocation Loc;
+    };
+    /// Explicitly listed variables and functions in a 'to' or 'link' clause.
+    llvm::DenseMap<NamedDecl *, MapInfo> ExplicitlyMapped;
+
+    /// The 'device_type' as parsed from the clause.
+    OMPDeclareTargetDeclAttr::DevTypeTy DT = OMPDeclareTargetDeclAttr::DT_Any;
+
+    /// The directive kind, `begin declare target` or `declare target`.
+    OpenMPDirectiveKind Kind;
+
+    /// The directive with indirect clause.
+    std::optional<Expr *> Indirect;
+
+    /// The directive location.
+    SourceLocation Loc;
+
+    DeclareTargetContextInfo(OpenMPDirectiveKind Kind, SourceLocation Loc)
+        : Kind(Kind), Loc(Loc) {}
+  };
+
+  /// Called on the start of target region i.e. '#pragma omp declare target'.
+  bool ActOnStartOpenMPDeclareTargetContext(DeclareTargetContextInfo &DTCI);
+
+  /// Called at the end of target region i.e. '#pragma omp end declare target'.
+  const DeclareTargetContextInfo ActOnOpenMPEndDeclareTargetDirective();
+
+  /// Called once a target context is completed, that can be when a
+  /// '#pragma omp end declare target' was encountered or when a
+  /// '#pragma omp declare target' without declaration-definition-seq was
+  /// encountered.
+  void ActOnFinishedOpenMPDeclareTargetContext(DeclareTargetContextInfo &DTCI);
+
+  /// Report unterminated 'omp declare target' or 'omp begin declare target' at
+  /// the end of a compilation unit.
+  void DiagnoseUnterminatedOpenMPDeclareTarget();
+
+  /// Searches for the provided declaration name for OpenMP declare target
+  /// directive.
+  NamedDecl *lookupOpenMPDeclareTargetName(Scope *CurScope,
+                                           CXXScopeSpec &ScopeSpec,
+                                           const DeclarationNameInfo &Id);
+
+  /// Called on correct id-expression from the '#pragma omp declare target'.
+  void ActOnOpenMPDeclareTargetName(NamedDecl *ND, SourceLocation Loc,
+                                    OMPDeclareTargetDeclAttr::MapTypeTy MT,
+                                    DeclareTargetContextInfo &DTCI);
+
+  /// Check declaration inside target region.
+  void
+  checkDeclIsAllowedInOpenMPTarget(Expr *E, Decl *D,
+                                   SourceLocation IdLoc = SourceLocation());
+
+  /// Adds OMPDeclareTargetDeclAttr to referenced variables in declare target
+  /// directive.
+  void ActOnOpenMPDeclareTargetInitializer(Decl *D);
+
+  /// Finishes analysis of the deferred functions calls that may be declared as
+  /// host/nohost during device/host compilation.
+  void finalizeOpenMPDelayedAnalysis(const FunctionDecl *Caller,
+                                     const FunctionDecl *Callee,
+                                     SourceLocation Loc);
+
+  /// Return true if currently in OpenMP task with untied clause context.
+  bool isInOpenMPTaskUntiedContext() const;
+
+  /// Return true inside OpenMP declare target region.
+  bool isInOpenMPDeclareTargetContext() const {
+    return !DeclareTargetNesting.empty();
+  }
+  /// Return true inside OpenMP target region.
+  bool isInOpenMPTargetExecutionDirective() const;
+
+  /// Return the number of captured regions created for an OpenMP directive.
+  static int getOpenMPCaptureLevels(OpenMPDirectiveKind Kind);
+
+  /// Initialization of captured region for OpenMP region.
+  void ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind, Scope *CurScope);
+
+  /// Called for syntactical loops (ForStmt or CXXForRangeStmt) associated to
+  /// an OpenMP loop directive.
+  StmtResult ActOnOpenMPCanonicalLoop(Stmt *AStmt);
+
+  /// Process a canonical OpenMP loop nest that can either be a canonical
+  /// literal loop (ForStmt or CXXForRangeStmt), or the generated loop of an
+  /// OpenMP loop transformation construct.
+  StmtResult ActOnOpenMPLoopnest(Stmt *AStmt);
+
+  /// End of OpenMP region.
+  ///
+  /// \param S Statement associated with the current OpenMP region.
+  /// \param Clauses List of clauses for the current OpenMP region.
+  ///
+  /// \returns Statement for finished OpenMP region.
+  StmtResult ActOnOpenMPRegionEnd(StmtResult S, ArrayRef<OMPClause *> Clauses);
+  StmtResult ActOnOpenMPExecutableDirective(
+      OpenMPDirectiveKind Kind, const DeclarationNameInfo &DirName,
+      OpenMPDirectiveKind CancelRegion, ArrayRef<OMPClause *> Clauses,
+      Stmt *AStmt, SourceLocation StartLoc, SourceLocation EndLoc,
+      OpenMPDirectiveKind PrevMappedDirective = llvm::omp::OMPD_unknown);
+  /// Called on well-formed '\#pragma omp parallel' after parsing
+  /// of the  associated statement.
+  StmtResult ActOnOpenMPParallelDirective(ArrayRef<OMPClause *> Clauses,
+                                          Stmt *AStmt, SourceLocation StartLoc,
+                                          SourceLocation EndLoc);
+  using VarsWithInheritedDSAType =
+      llvm::SmallDenseMap<const ValueDecl *, const Expr *, 4>;
+  /// Called on well-formed '\#pragma omp simd' after parsing
+  /// of the associated statement.
+  StmtResult
+  ActOnOpenMPSimdDirective(ArrayRef<OMPClause *> Clauses, Stmt *AStmt,
+                           SourceLocation StartLoc, SourceLocation EndLoc,
+                           VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '#pragma omp tile' after parsing of its clauses and
+  /// the associated statement.
+  StmtResult ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
+                                      Stmt *AStmt, SourceLocation StartLoc,
+                                      SourceLocation EndLoc);
+  /// Called on well-formed '#pragma omp unroll' after parsing of its clauses
+  /// and the associated statement.
+  StmtResult ActOnOpenMPUnrollDirective(ArrayRef<OMPClause *> Clauses,
+                                        Stmt *AStmt, SourceLocation StartLoc,
+                                        SourceLocation EndLoc);
+  /// Called on well-formed '\#pragma omp for' after parsing
+  /// of the associated statement.
+  StmtResult
+  ActOnOpenMPForDirective(ArrayRef<OMPClause *> Clauses, Stmt *AStmt,
+                          SourceLocation StartLoc, SourceLocation EndLoc,
+                          VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp for simd' after parsing
+  /// of the associated statement.
+  StmtResult
+  ActOnOpenMPForSimdDirective(ArrayRef<OMPClause *> Clauses, Stmt *AStmt,
+                              SourceLocation StartLoc, SourceLocation EndLoc,
+                              VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp sections' after parsing
+  /// of the associated statement.
+  StmtResult ActOnOpenMPSectionsDirective(ArrayRef<OMPClause *> Clauses,
+                                          Stmt *AStmt, SourceLocation StartLoc,
+                                          SourceLocation EndLoc);
+  /// Called on well-formed '\#pragma omp section' after parsing of the
+  /// associated statement.
+  StmtResult ActOnOpenMPSectionDirective(Stmt *AStmt, SourceLocation StartLoc,
+                                         SourceLocation EndLoc);
+  /// Called on well-formed '\#pragma omp scope' after parsing of the
+  /// associated statement.
+  StmtResult ActOnOpenMPScopeDirective(ArrayRef<OMPClause *> Clauses,
+                                       Stmt *AStmt, SourceLocation StartLoc,
+                                       SourceLocation EndLoc);
+  /// Called on well-formed '\#pragma omp single' after parsing of the
+  /// associated statement.
+  StmtResult ActOnOpenMPSingleDirective(ArrayRef<OMPClause *> Clauses,
+                                        Stmt *AStmt, SourceLocation StartLoc,
+                                        SourceLocation EndLoc);
+  /// Called on well-formed '\#pragma omp master' after parsing of the
+  /// associated statement.
+  StmtResult ActOnOpenMPMasterDirective(Stmt *AStmt, SourceLocation StartLoc,
+                                        SourceLocation EndLoc);
+  /// Called on well-formed '\#pragma omp critical' after parsing of the
+  /// associated statement.
+  StmtResult ActOnOpenMPCriticalDirective(const DeclarationNameInfo &DirName,
+                                          ArrayRef<OMPClause *> Clauses,
+                                          Stmt *AStmt, SourceLocation StartLoc,
+                                          SourceLocation EndLoc);
+  /// Called on well-formed '\#pragma omp parallel for' after parsing
+  /// of the  associated statement.
+  StmtResult ActOnOpenMPParallelForDirective(
+      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp parallel for simd' after
+  /// parsing of the  associated statement.
+  StmtResult ActOnOpenMPParallelForSimdDirective(
+      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp parallel master' after
+  /// parsing of the  associated statement.
+  StmtResult ActOnOpenMPParallelMasterDirective(ArrayRef<OMPClause *> Clauses,
+                                                Stmt *AStmt,
+                                                SourceLocation StartLoc,
+                                                SourceLocation EndLoc);
+  /// Called on well-formed '\#pragma omp parallel masked' after
+  /// parsing of the associated statement.
+  StmtResult ActOnOpenMPParallelMaskedDirective(ArrayRef<OMPClause *> Clauses,
+                                                Stmt *AStmt,
+                                                SourceLocation StartLoc,
+                                                SourceLocation EndLoc);
+  /// Called on well-formed '\#pragma omp parallel sections' after
+  /// parsing of the  associated statement.
+  StmtResult ActOnOpenMPParallelSectionsDirective(ArrayRef<OMPClause *> Clauses,
+                                                  Stmt *AStmt,
+                                                  SourceLocation StartLoc,
+                                                  SourceLocation EndLoc);
+  /// Called on well-formed '\#pragma omp task' after parsing of the
+  /// associated statement.
+  StmtResult ActOnOpenMPTaskDirective(ArrayRef<OMPClause *> Clauses,
+                                      Stmt *AStmt, SourceLocation StartLoc,
+                                      SourceLocation EndLoc);
+  /// Called on well-formed '\#pragma omp taskyield'.
+  StmtResult ActOnOpenMPTaskyieldDirective(SourceLocation StartLoc,
+                                           SourceLocation EndLoc);
+  /// Called on well-formed '\#pragma omp error'.
+  /// Error direcitive is allowed in both declared and excutable contexts.
+  /// Adding InExContext to identify which context is called from.
+  StmtResult ActOnOpenMPErrorDirective(ArrayRef<OMPClause *> Clauses,
+                                       SourceLocation StartLoc,
+                                       SourceLocation EndLoc,
+                                       bool InExContext = true);
+  /// Called on well-formed '\#pragma omp barrier'.
+  StmtResult ActOnOpenMPBarrierDirective(SourceLocation StartLoc,
+                                         SourceLocation EndLoc);
+  /// Called on well-formed '\#pragma omp taskwait'.
+  StmtResult ActOnOpenMPTaskwaitDirective(ArrayRef<OMPClause *> Clauses,
+                                          SourceLocation StartLoc,
+                                          SourceLocation EndLoc);
+  /// Called on well-formed '\#pragma omp taskgroup'.
+  StmtResult ActOnOpenMPTaskgroupDirective(ArrayRef<OMPClause *> Clauses,
+                                           Stmt *AStmt, SourceLocation StartLoc,
+                                           SourceLocation EndLoc);
+  /// Called on well-formed '\#pragma omp flush'.
+  StmtResult ActOnOpenMPFlushDirective(ArrayRef<OMPClause *> Clauses,
+                                       SourceLocation StartLoc,
+                                       SourceLocation EndLoc);
+  /// Called on well-formed '\#pragma omp depobj'.
+  StmtResult ActOnOpenMPDepobjDirective(ArrayRef<OMPClause *> Clauses,
+                                        SourceLocation StartLoc,
+                                        SourceLocation EndLoc);
+  /// Called on well-formed '\#pragma omp scan'.
+  StmtResult ActOnOpenMPScanDirective(ArrayRef<OMPClause *> Clauses,
+                                      SourceLocation StartLoc,
+                                      SourceLocation EndLoc);
+  /// Called on well-formed '\#pragma omp ordered' after parsing of the
+  /// associated statement.
+  StmtResult ActOnOpenMPOrderedDirective(ArrayRef<OMPClause *> Clauses,
+                                         Stmt *AStmt, SourceLocation StartLoc,
+                                         SourceLocation EndLoc);
+  /// Called on well-formed '\#pragma omp atomic' after parsing of the
+  /// associated statement.
+  StmtResult ActOnOpenMPAtomicDirective(ArrayRef<OMPClause *> Clauses,
+                                        Stmt *AStmt, SourceLocation StartLoc,
+                                        SourceLocation EndLoc);
+  /// Called on well-formed '\#pragma omp target' after parsing of the
+  /// associated statement.
+  StmtResult ActOnOpenMPTargetDirective(ArrayRef<OMPClause *> Clauses,
+                                        Stmt *AStmt, SourceLocation StartLoc,
+                                        SourceLocation EndLoc);
+  /// Called on well-formed '\#pragma omp target data' after parsing of
+  /// the associated statement.
+  StmtResult ActOnOpenMPTargetDataDirective(ArrayRef<OMPClause *> Clauses,
+                                            Stmt *AStmt,
+                                            SourceLocation StartLoc,
+                                            SourceLocation EndLoc);
+  /// Called on well-formed '\#pragma omp target enter data' after
+  /// parsing of the associated statement.
+  StmtResult ActOnOpenMPTargetEnterDataDirective(ArrayRef<OMPClause *> Clauses,
+                                                 SourceLocation StartLoc,
+                                                 SourceLocation EndLoc,
+                                                 Stmt *AStmt);
+  /// Called on well-formed '\#pragma omp target exit data' after
+  /// parsing of the associated statement.
+  StmtResult ActOnOpenMPTargetExitDataDirective(ArrayRef<OMPClause *> Clauses,
+                                                SourceLocation StartLoc,
+                                                SourceLocation EndLoc,
+                                                Stmt *AStmt);
+  /// Called on well-formed '\#pragma omp target parallel' after
+  /// parsing of the associated statement.
+  StmtResult ActOnOpenMPTargetParallelDirective(ArrayRef<OMPClause *> Clauses,
+                                                Stmt *AStmt,
+                                                SourceLocation StartLoc,
+                                                SourceLocation EndLoc);
+  /// Called on well-formed '\#pragma omp target parallel for' after
+  /// parsing of the  associated statement.
+  StmtResult ActOnOpenMPTargetParallelForDirective(
+      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp teams' after parsing of the
+  /// associated statement.
+  StmtResult ActOnOpenMPTeamsDirective(ArrayRef<OMPClause *> Clauses,
+                                       Stmt *AStmt, SourceLocation StartLoc,
+                                       SourceLocation EndLoc);
+  /// Called on well-formed '\#pragma omp teams loop' after parsing of the
+  /// associated statement.
+  StmtResult ActOnOpenMPTeamsGenericLoopDirective(
+      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp target teams loop' after parsing of
+  /// the associated statement.
+  StmtResult ActOnOpenMPTargetTeamsGenericLoopDirective(
+      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp parallel loop' after parsing of the
+  /// associated statement.
+  StmtResult ActOnOpenMPParallelGenericLoopDirective(
+      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp target parallel loop' after parsing
+  /// of the associated statement.
+  StmtResult ActOnOpenMPTargetParallelGenericLoopDirective(
+      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp cancellation point'.
+  StmtResult
+  ActOnOpenMPCancellationPointDirective(SourceLocation StartLoc,
+                                        SourceLocation EndLoc,
+                                        OpenMPDirectiveKind CancelRegion);
+  /// Called on well-formed '\#pragma omp cancel'.
+  StmtResult ActOnOpenMPCancelDirective(ArrayRef<OMPClause *> Clauses,
+                                        SourceLocation StartLoc,
+                                        SourceLocation EndLoc,
+                                        OpenMPDirectiveKind CancelRegion);
+  /// Called on well-formed '\#pragma omp taskloop' after parsing of the
+  /// associated statement.
+  StmtResult
+  ActOnOpenMPTaskLoopDirective(ArrayRef<OMPClause *> Clauses, Stmt *AStmt,
+                               SourceLocation StartLoc, SourceLocation EndLoc,
+                               VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp taskloop simd' after parsing of
+  /// the associated statement.
+  StmtResult ActOnOpenMPTaskLoopSimdDirective(
+      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp master taskloop' after parsing of the
+  /// associated statement.
+  StmtResult ActOnOpenMPMasterTaskLoopDirective(
+      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp master taskloop simd' after parsing of
+  /// the associated statement.
+  StmtResult ActOnOpenMPMasterTaskLoopSimdDirective(
+      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp parallel master taskloop' after
+  /// parsing of the associated statement.
+  StmtResult ActOnOpenMPParallelMasterTaskLoopDirective(
+      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp parallel master taskloop simd' after
+  /// parsing of the associated statement.
+  StmtResult ActOnOpenMPParallelMasterTaskLoopSimdDirective(
+      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp masked taskloop' after parsing of the
+  /// associated statement.
+  StmtResult ActOnOpenMPMaskedTaskLoopDirective(
+      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp masked taskloop simd' after parsing of
+  /// the associated statement.
+  StmtResult ActOnOpenMPMaskedTaskLoopSimdDirective(
+      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp parallel masked taskloop' after
+  /// parsing of the associated statement.
+  StmtResult ActOnOpenMPParallelMaskedTaskLoopDirective(
+      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp parallel masked taskloop simd' after
+  /// parsing of the associated statement.
+  StmtResult ActOnOpenMPParallelMaskedTaskLoopSimdDirective(
+      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp distribute' after parsing
+  /// of the associated statement.
+  StmtResult
+  ActOnOpenMPDistributeDirective(ArrayRef<OMPClause *> Clauses, Stmt *AStmt,
+                                 SourceLocation StartLoc, SourceLocation EndLoc,
+                                 VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp target update'.
+  StmtResult ActOnOpenMPTargetUpdateDirective(ArrayRef<OMPClause *> Clauses,
+                                              SourceLocation StartLoc,
+                                              SourceLocation EndLoc,
+                                              Stmt *AStmt);
+  /// Called on well-formed '\#pragma omp distribute parallel for' after
+  /// parsing of the associated statement.
+  StmtResult ActOnOpenMPDistributeParallelForDirective(
+      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp distribute parallel for simd'
+  /// after parsing of the associated statement.
+  StmtResult ActOnOpenMPDistributeParallelForSimdDirective(
+      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp distribute simd' after
+  /// parsing of the associated statement.
+  StmtResult ActOnOpenMPDistributeSimdDirective(
+      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp target parallel for simd' after
+  /// parsing of the associated statement.
+  StmtResult ActOnOpenMPTargetParallelForSimdDirective(
+      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp target simd' after parsing of
+  /// the associated statement.
+  StmtResult
+  ActOnOpenMPTargetSimdDirective(ArrayRef<OMPClause *> Clauses, Stmt *AStmt,
+                                 SourceLocation StartLoc, SourceLocation EndLoc,
+                                 VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp teams distribute' after parsing of
+  /// the associated statement.
+  StmtResult ActOnOpenMPTeamsDistributeDirective(
+      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp teams distribute simd' after parsing
+  /// of the associated statement.
+  StmtResult ActOnOpenMPTeamsDistributeSimdDirective(
+      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp teams distribute parallel for simd'
+  /// after parsing of the associated statement.
+  StmtResult ActOnOpenMPTeamsDistributeParallelForSimdDirective(
+      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp teams distribute parallel for'
+  /// after parsing of the associated statement.
+  StmtResult ActOnOpenMPTeamsDistributeParallelForDirective(
+      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp target teams' after parsing of the
+  /// associated statement.
+  StmtResult ActOnOpenMPTargetTeamsDirective(ArrayRef<OMPClause *> Clauses,
+                                             Stmt *AStmt,
+                                             SourceLocation StartLoc,
+                                             SourceLocation EndLoc);
+  /// Called on well-formed '\#pragma omp target teams distribute' after parsing
+  /// of the associated statement.
+  StmtResult ActOnOpenMPTargetTeamsDistributeDirective(
+      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp target teams distribute parallel for'
+  /// after parsing of the associated statement.
+  StmtResult ActOnOpenMPTargetTeamsDistributeParallelForDirective(
+      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp target teams distribute parallel for
+  /// simd' after parsing of the associated statement.
+  StmtResult ActOnOpenMPTargetTeamsDistributeParallelForSimdDirective(
+      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp target teams distribute simd' after
+  /// parsing of the associated statement.
+  StmtResult ActOnOpenMPTargetTeamsDistributeSimdDirective(
+      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
+  /// Called on well-formed '\#pragma omp interop'.
+  StmtResult ActOnOpenMPInteropDirective(ArrayRef<OMPClause *> Clauses,
+                                         SourceLocation StartLoc,
+                                         SourceLocation EndLoc);
+  /// Called on well-formed '\#pragma omp dispatch' after parsing of the
+  // /associated statement.
+  StmtResult ActOnOpenMPDispatchDirective(ArrayRef<OMPClause *> Clauses,
+                                          Stmt *AStmt, SourceLocation StartLoc,
+                                          SourceLocation EndLoc);
+  /// Called on well-formed '\#pragma omp masked' after parsing of the
+  // /associated statement.
+  StmtResult ActOnOpenMPMaskedDirective(ArrayRef<OMPClause *> Clauses,
+                                        Stmt *AStmt, SourceLocation StartLoc,
+                                        SourceLocation EndLoc);
+
+  /// Called on well-formed '\#pragma omp loop' after parsing of the
+  /// associated statement.
+  StmtResult ActOnOpenMPGenericLoopDirective(
+      ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+      SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA);
+
+  /// Checks correctness of linear modifiers.
+  bool CheckOpenMPLinearModifier(OpenMPLinearClauseKind LinKind,
+                                 SourceLocation LinLoc);
+  /// Checks that the specified declaration matches requirements for the linear
+  /// decls.
+  bool CheckOpenMPLinearDecl(const ValueDecl *D, SourceLocation ELoc,
+                             OpenMPLinearClauseKind LinKind, QualType Type,
+                             bool IsDeclareSimd = false);
+
+  /// Called on well-formed '\#pragma omp declare simd' after parsing of
+  /// the associated method/function.
+  DeclGroupPtrTy ActOnOpenMPDeclareSimdDirective(
+      DeclGroupPtrTy DG, OMPDeclareSimdDeclAttr::BranchStateTy BS,
+      Expr *Simdlen, ArrayRef<Expr *> Uniforms, ArrayRef<Expr *> Aligneds,
+      ArrayRef<Expr *> Alignments, ArrayRef<Expr *> Linears,
+      ArrayRef<unsigned> LinModifiers, ArrayRef<Expr *> Steps, SourceRange SR);
+
+  /// Checks '\#pragma omp declare variant' variant function and original
+  /// functions after parsing of the associated method/function.
+  /// \param DG Function declaration to which declare variant directive is
+  /// applied to.
+  /// \param VariantRef Expression that references the variant function, which
+  /// must be used instead of the original one, specified in \p DG.
+  /// \param TI The trait info object representing the match clause.
+  /// \param NumAppendArgs The number of omp_interop_t arguments to account for
+  /// in checking.
+  /// \returns std::nullopt, if the function/variant function are not compatible
+  /// with the pragma, pair of original function/variant ref expression
+  /// otherwise.
+  std::optional<std::pair<FunctionDecl *, Expr *>>
+  checkOpenMPDeclareVariantFunction(DeclGroupPtrTy DG, Expr *VariantRef,
+                                    OMPTraitInfo &TI, unsigned NumAppendArgs,
+                                    SourceRange SR);
+
+  /// Called on well-formed '\#pragma omp declare variant' after parsing of
+  /// the associated method/function.
+  /// \param FD Function declaration to which declare variant directive is
+  /// applied to.
+  /// \param VariantRef Expression that references the variant function, which
+  /// must be used instead of the original one, specified in \p DG.
+  /// \param TI The context traits associated with the function variant.
+  /// \param AdjustArgsNothing The list of 'nothing' arguments.
+  /// \param AdjustArgsNeedDevicePtr The list of 'need_device_ptr' arguments.
+  /// \param AppendArgs The list of 'append_args' arguments.
+  /// \param AdjustArgsLoc The Location of an 'adjust_args' clause.
+  /// \param AppendArgsLoc The Location of an 'append_args' clause.
+  /// \param SR The SourceRange of the 'declare variant' directive.
+  void ActOnOpenMPDeclareVariantDirective(
+      FunctionDecl *FD, Expr *VariantRef, OMPTraitInfo &TI,
+      ArrayRef<Expr *> AdjustArgsNothing,
+      ArrayRef<Expr *> AdjustArgsNeedDevicePtr,
+      ArrayRef<OMPInteropInfo> AppendArgs, SourceLocation AdjustArgsLoc,
+      SourceLocation AppendArgsLoc, SourceRange SR);
+
+  OMPClause *ActOnOpenMPSingleExprClause(OpenMPClauseKind Kind, Expr *Expr,
+                                         SourceLocation StartLoc,
+                                         SourceLocation LParenLoc,
+                                         SourceLocation EndLoc);
+  /// Called on well-formed 'allocator' clause.
+  OMPClause *ActOnOpenMPAllocatorClause(Expr *Allocator,
+                                        SourceLocation StartLoc,
+                                        SourceLocation LParenLoc,
+                                        SourceLocation EndLoc);
+  /// Called on well-formed 'if' clause.
+  OMPClause *ActOnOpenMPIfClause(OpenMPDirectiveKind NameModifier,
+                                 Expr *Condition, SourceLocation StartLoc,
+                                 SourceLocation LParenLoc,
+                                 SourceLocation NameModifierLoc,
+                                 SourceLocation ColonLoc,
+                                 SourceLocation EndLoc);
+  /// Called on well-formed 'final' clause.
+  OMPClause *ActOnOpenMPFinalClause(Expr *Condition, SourceLocation StartLoc,
+                                    SourceLocation LParenLoc,
+                                    SourceLocation EndLoc);
+  /// Called on well-formed 'num_threads' clause.
+  OMPClause *ActOnOpenMPNumThreadsClause(Expr *NumThreads,
+                                         SourceLocation StartLoc,
+                                         SourceLocation LParenLoc,
+                                         SourceLocation EndLoc);
+  /// Called on well-formed 'align' clause.
+  OMPClause *ActOnOpenMPAlignClause(Expr *Alignment, SourceLocation StartLoc,
+                                    SourceLocation LParenLoc,
+                                    SourceLocation EndLoc);
+  /// Called on well-formed 'safelen' clause.
+  OMPClause *ActOnOpenMPSafelenClause(Expr *Length, SourceLocation StartLoc,
+                                      SourceLocation LParenLoc,
+                                      SourceLocation EndLoc);
+  /// Called on well-formed 'simdlen' clause.
+  OMPClause *ActOnOpenMPSimdlenClause(Expr *Length, SourceLocation StartLoc,
+                                      SourceLocation LParenLoc,
+                                      SourceLocation EndLoc);
+  /// Called on well-form 'sizes' clause.
+  OMPClause *ActOnOpenMPSizesClause(ArrayRef<Expr *> SizeExprs,
+                                    SourceLocation StartLoc,
+                                    SourceLocation LParenLoc,
+                                    SourceLocation EndLoc);
+  /// Called on well-form 'full' clauses.
+  OMPClause *ActOnOpenMPFullClause(SourceLocation StartLoc,
+                                   SourceLocation EndLoc);
+  /// Called on well-form 'partial' clauses.
+  OMPClause *ActOnOpenMPPartialClause(Expr *FactorExpr, SourceLocation StartLoc,
+                                      SourceLocation LParenLoc,
+                                      SourceLocation EndLoc);
+  /// Called on well-formed 'collapse' clause.
+  OMPClause *ActOnOpenMPCollapseClause(Expr *NumForLoops,
+                                       SourceLocation StartLoc,
+                                       SourceLocation LParenLoc,
+                                       SourceLocation EndLoc);
+  /// Called on well-formed 'ordered' clause.
+  OMPClause *
+  ActOnOpenMPOrderedClause(SourceLocation StartLoc, SourceLocation EndLoc,
+                           SourceLocation LParenLoc = SourceLocation(),
+                           Expr *NumForLoops = nullptr);
+  /// Called on well-formed 'grainsize' clause.
+  OMPClause *ActOnOpenMPGrainsizeClause(OpenMPGrainsizeClauseModifier Modifier,
+                                        Expr *Size, SourceLocation StartLoc,
+                                        SourceLocation LParenLoc,
+                                        SourceLocation ModifierLoc,
+                                        SourceLocation EndLoc);
+  /// Called on well-formed 'num_tasks' clause.
+  OMPClause *ActOnOpenMPNumTasksClause(OpenMPNumTasksClauseModifier Modifier,
+                                       Expr *NumTasks, SourceLocation StartLoc,
+                                       SourceLocation LParenLoc,
+                                       SourceLocation ModifierLoc,
+                                       SourceLocation EndLoc);
+  /// Called on well-formed 'hint' clause.
+  OMPClause *ActOnOpenMPHintClause(Expr *Hint, SourceLocation StartLoc,
+                                   SourceLocation LParenLoc,
+                                   SourceLocation EndLoc);
+  /// Called on well-formed 'detach' clause.
+  OMPClause *ActOnOpenMPDetachClause(Expr *Evt, SourceLocation StartLoc,
+                                     SourceLocation LParenLoc,
+                                     SourceLocation EndLoc);
+
+  OMPClause *ActOnOpenMPSimpleClause(OpenMPClauseKind Kind, unsigned Argument,
+                                     SourceLocation ArgumentLoc,
+                                     SourceLocation StartLoc,
+                                     SourceLocation LParenLoc,
+                                     SourceLocation EndLoc);
+  /// Called on well-formed 'when' clause.
+  OMPClause *ActOnOpenMPWhenClause(OMPTraitInfo &TI, SourceLocation StartLoc,
+                                   SourceLocation LParenLoc,
+                                   SourceLocation EndLoc);
+  /// Called on well-formed 'default' clause.
+  OMPClause *ActOnOpenMPDefaultClause(llvm::omp::DefaultKind Kind,
+                                      SourceLocation KindLoc,
+                                      SourceLocation StartLoc,
+                                      SourceLocation LParenLoc,
+                                      SourceLocation EndLoc);
+  /// Called on well-formed 'proc_bind' clause.
+  OMPClause *ActOnOpenMPProcBindClause(llvm::omp::ProcBindKind Kind,
+                                       SourceLocation KindLoc,
+                                       SourceLocation StartLoc,
+                                       SourceLocation LParenLoc,
+                                       SourceLocation EndLoc);
+  /// Called on well-formed 'order' clause.
+  OMPClause *ActOnOpenMPOrderClause(OpenMPOrderClauseModifier Modifier,
+                                    OpenMPOrderClauseKind Kind,
+                                    SourceLocation StartLoc,
+                                    SourceLocation LParenLoc,
+                                    SourceLocation MLoc, SourceLocation KindLoc,
+                                    SourceLocation EndLoc);
+  /// Called on well-formed 'update' clause.
+  OMPClause *ActOnOpenMPUpdateClause(OpenMPDependClauseKind Kind,
+                                     SourceLocation KindLoc,
+                                     SourceLocation StartLoc,
+                                     SourceLocation LParenLoc,
+                                     SourceLocation EndLoc);
+
+  OMPClause *ActOnOpenMPSingleExprWithArgClause(
+      OpenMPClauseKind Kind, ArrayRef<unsigned> Arguments, Expr *Expr,
+      SourceLocation StartLoc, SourceLocation LParenLoc,
+      ArrayRef<SourceLocation> ArgumentsLoc, SourceLocation DelimLoc,
+      SourceLocation EndLoc);
+  /// Called on well-formed 'schedule' clause.
+  OMPClause *ActOnOpenMPScheduleClause(
+      OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2,
+      OpenMPScheduleClauseKind Kind, Expr *ChunkSize, SourceLocation StartLoc,
+      SourceLocation LParenLoc, SourceLocation M1Loc, SourceLocation M2Loc,
+      SourceLocation KindLoc, SourceLocation CommaLoc, SourceLocation EndLoc);
+
+  OMPClause *ActOnOpenMPClause(OpenMPClauseKind Kind, SourceLocation StartLoc,
+                               SourceLocation EndLoc);
+  /// Called on well-formed 'nowait' clause.
+  OMPClause *ActOnOpenMPNowaitClause(SourceLocation StartLoc,
+                                     SourceLocation EndLoc);
+  /// Called on well-formed 'untied' clause.
+  OMPClause *ActOnOpenMPUntiedClause(SourceLocation StartLoc,
+                                     SourceLocation EndLoc);
+  /// Called on well-formed 'mergeable' clause.
+  OMPClause *ActOnOpenMPMergeableClause(SourceLocation StartLoc,
+                                        SourceLocation EndLoc);
+  /// Called on well-formed 'read' clause.
+  OMPClause *ActOnOpenMPReadClause(SourceLocation StartLoc,
+                                   SourceLocation EndLoc);
+  /// Called on well-formed 'write' clause.
+  OMPClause *ActOnOpenMPWriteClause(SourceLocation StartLoc,
+                                    SourceLocation EndLoc);
+  /// Called on well-formed 'update' clause.
+  OMPClause *ActOnOpenMPUpdateClause(SourceLocation StartLoc,
+                                     SourceLocation EndLoc);
+  /// Called on well-formed 'capture' clause.
+  OMPClause *ActOnOpenMPCaptureClause(SourceLocation StartLoc,
+                                      SourceLocation EndLoc);
+  /// Called on well-formed 'compare' clause.
+  OMPClause *ActOnOpenMPCompareClause(SourceLocation StartLoc,
+                                      SourceLocation EndLoc);
+  /// Called on well-formed 'fail' clause.
+  OMPClause *ActOnOpenMPFailClause(SourceLocation StartLoc,
+                                   SourceLocation EndLoc);
+  OMPClause *ActOnOpenMPFailClause(OpenMPClauseKind Kind,
+                                   SourceLocation KindLoc,
+                                   SourceLocation StartLoc,
+                                   SourceLocation LParenLoc,
+                                   SourceLocation EndLoc);
+
+  /// Called on well-formed 'seq_cst' clause.
+  OMPClause *ActOnOpenMPSeqCstClause(SourceLocation StartLoc,
+                                     SourceLocation EndLoc);
+  /// Called on well-formed 'acq_rel' clause.
+  OMPClause *ActOnOpenMPAcqRelClause(SourceLocation StartLoc,
+                                     SourceLocation EndLoc);
+  /// Called on well-formed 'acquire' clause.
+  OMPClause *ActOnOpenMPAcquireClause(SourceLocation StartLoc,
+                                      SourceLocation EndLoc);
+  /// Called on well-formed 'release' clause.
+  OMPClause *ActOnOpenMPReleaseClause(SourceLocation StartLoc,
+                                      SourceLocation EndLoc);
+  /// Called on well-formed 'relaxed' clause.
+  OMPClause *ActOnOpenMPRelaxedClause(SourceLocation StartLoc,
+                                      SourceLocation EndLoc);
+  /// Called on well-formed 'weak' clause.
+  OMPClause *ActOnOpenMPWeakClause(SourceLocation StartLoc,
+                                   SourceLocation EndLoc);
+
+  /// Called on well-formed 'init' clause.
+  OMPClause *
+  ActOnOpenMPInitClause(Expr *InteropVar, OMPInteropInfo &InteropInfo,
+                        SourceLocation StartLoc, SourceLocation LParenLoc,
+                        SourceLocation VarLoc, SourceLocation EndLoc);
+
+  /// Called on well-formed 'use' clause.
+  OMPClause *ActOnOpenMPUseClause(Expr *InteropVar, SourceLocation StartLoc,
+                                  SourceLocation LParenLoc,
+                                  SourceLocation VarLoc, SourceLocation EndLoc);
+
+  /// Called on well-formed 'destroy' clause.
+  OMPClause *ActOnOpenMPDestroyClause(Expr *InteropVar, SourceLocation StartLoc,
+                                      SourceLocation LParenLoc,
+                                      SourceLocation VarLoc,
+                                      SourceLocation EndLoc);
+  /// Called on well-formed 'novariants' clause.
+  OMPClause *ActOnOpenMPNovariantsClause(Expr *Condition,
+                                         SourceLocation StartLoc,
+                                         SourceLocation LParenLoc,
+                                         SourceLocation EndLoc);
+  /// Called on well-formed 'nocontext' clause.
+  OMPClause *ActOnOpenMPNocontextClause(Expr *Condition,
+                                        SourceLocation StartLoc,
+                                        SourceLocation LParenLoc,
+                                        SourceLocation EndLoc);
+  /// Called on well-formed 'filter' clause.
+  OMPClause *ActOnOpenMPFilterClause(Expr *ThreadID, SourceLocation StartLoc,
+                                     SourceLocation LParenLoc,
+                                     SourceLocation EndLoc);
+  /// Called on well-formed 'threads' clause.
+  OMPClause *ActOnOpenMPThreadsClause(SourceLocation StartLoc,
+                                      SourceLocation EndLoc);
+  /// Called on well-formed 'simd' clause.
+  OMPClause *ActOnOpenMPSIMDClause(SourceLocation StartLoc,
+                                   SourceLocation EndLoc);
+  /// Called on well-formed 'nogroup' clause.
+  OMPClause *ActOnOpenMPNogroupClause(SourceLocation StartLoc,
+                                      SourceLocation EndLoc);
+  /// Called on well-formed 'unified_address' clause.
+  OMPClause *ActOnOpenMPUnifiedAddressClause(SourceLocation StartLoc,
+                                             SourceLocation EndLoc);
+
+  /// Called on well-formed 'unified_address' clause.
+  OMPClause *ActOnOpenMPUnifiedSharedMemoryClause(SourceLocation StartLoc,
+                                                  SourceLocation EndLoc);
+
+  /// Called on well-formed 'reverse_offload' clause.
+  OMPClause *ActOnOpenMPReverseOffloadClause(SourceLocation StartLoc,
+                                             SourceLocation EndLoc);
+
+  /// Called on well-formed 'dynamic_allocators' clause.
+  OMPClause *ActOnOpenMPDynamicAllocatorsClause(SourceLocation StartLoc,
+                                                SourceLocation EndLoc);
+
+  /// Called on well-formed 'atomic_default_mem_order' clause.
+  OMPClause *ActOnOpenMPAtomicDefaultMemOrderClause(
+      OpenMPAtomicDefaultMemOrderClauseKind Kind, SourceLocation KindLoc,
+      SourceLocation StartLoc, SourceLocation LParenLoc, SourceLocation EndLoc);
+
+  /// Called on well-formed 'at' clause.
+  OMPClause *ActOnOpenMPAtClause(OpenMPAtClauseKind Kind,
+                                 SourceLocation KindLoc,
+                                 SourceLocation StartLoc,
+                                 SourceLocation LParenLoc,
+                                 SourceLocation EndLoc);
+
+  /// Called on well-formed 'severity' clause.
+  OMPClause *ActOnOpenMPSeverityClause(OpenMPSeverityClauseKind Kind,
+                                       SourceLocation KindLoc,
+                                       SourceLocation StartLoc,
+                                       SourceLocation LParenLoc,
+                                       SourceLocation EndLoc);
+
+  /// Called on well-formed 'message' clause.
+  /// passing string for message.
+  OMPClause *ActOnOpenMPMessageClause(Expr *MS, SourceLocation StartLoc,
+                                      SourceLocation LParenLoc,
+                                      SourceLocation EndLoc);
+
+  /// Data used for processing a list of variables in OpenMP clauses.
+  struct OpenMPVarListDataTy final {
+    Expr *DepModOrTailExpr = nullptr;
+    Expr *IteratorExpr = nullptr;
+    SourceLocation ColonLoc;
+    SourceLocation RLoc;
+    CXXScopeSpec ReductionOrMapperIdScopeSpec;
+    DeclarationNameInfo ReductionOrMapperId;
+    int ExtraModifier = -1; ///< Additional modifier for linear, map, depend or
+                            ///< lastprivate clause.
+    SmallVector<OpenMPMapModifierKind, NumberOfOMPMapClauseModifiers>
+        MapTypeModifiers;
+    SmallVector<SourceLocation, NumberOfOMPMapClauseModifiers>
+        MapTypeModifiersLoc;
+    SmallVector<OpenMPMotionModifierKind, NumberOfOMPMotionModifiers>
+        MotionModifiers;
+    SmallVector<SourceLocation, NumberOfOMPMotionModifiers> MotionModifiersLoc;
+    bool IsMapTypeImplicit = false;
+    SourceLocation ExtraModifierLoc;
+    SourceLocation OmpAllMemoryLoc;
+    SourceLocation
+        StepModifierLoc; /// 'step' modifier location for linear clause
+  };
+
+  OMPClause *ActOnOpenMPVarListClause(OpenMPClauseKind Kind,
+                                      ArrayRef<Expr *> Vars,
+                                      const OMPVarListLocTy &Locs,
+                                      OpenMPVarListDataTy &Data);
+  /// Called on well-formed 'inclusive' clause.
+  OMPClause *ActOnOpenMPInclusiveClause(ArrayRef<Expr *> VarList,
+                                        SourceLocation StartLoc,
+                                        SourceLocation LParenLoc,
+                                        SourceLocation EndLoc);
+  /// Called on well-formed 'exclusive' clause.
+  OMPClause *ActOnOpenMPExclusiveClause(ArrayRef<Expr *> VarList,
+                                        SourceLocation StartLoc,
+                                        SourceLocation LParenLoc,
+                                        SourceLocation EndLoc);
+  /// Called on well-formed 'allocate' clause.
+  OMPClause *
+  ActOnOpenMPAllocateClause(Expr *Allocator, ArrayRef<Expr *> VarList,
+                            SourceLocation StartLoc, SourceLocation ColonLoc,
+                            SourceLocation LParenLoc, SourceLocation EndLoc);
+  /// Called on well-formed 'private' clause.
+  OMPClause *ActOnOpenMPPrivateClause(ArrayRef<Expr *> VarList,
+                                      SourceLocation StartLoc,
+                                      SourceLocation LParenLoc,
+                                      SourceLocation EndLoc);
+  /// Called on well-formed 'firstprivate' clause.
+  OMPClause *ActOnOpenMPFirstprivateClause(ArrayRef<Expr *> VarList,
+                                           SourceLocation StartLoc,
+                                           SourceLocation LParenLoc,
+                                           SourceLocation EndLoc);
+  /// Called on well-formed 'lastprivate' clause.
+  OMPClause *ActOnOpenMPLastprivateClause(
+      ArrayRef<Expr *> VarList, OpenMPLastprivateModifier LPKind,
+      SourceLocation LPKindLoc, SourceLocation ColonLoc,
+      SourceLocation StartLoc, SourceLocation LParenLoc, SourceLocation EndLoc);
+  /// Called on well-formed 'shared' clause.
+  OMPClause *ActOnOpenMPSharedClause(ArrayRef<Expr *> VarList,
+                                     SourceLocation StartLoc,
+                                     SourceLocation LParenLoc,
+                                     SourceLocation EndLoc);
+  /// Called on well-formed 'reduction' clause.
+  OMPClause *ActOnOpenMPReductionClause(
+      ArrayRef<Expr *> VarList, OpenMPReductionClauseModifier Modifier,
+      SourceLocation StartLoc, SourceLocation LParenLoc,
+      SourceLocation ModifierLoc, SourceLocation ColonLoc,
+      SourceLocation EndLoc, CXXScopeSpec &ReductionIdScopeSpec,
+      const DeclarationNameInfo &ReductionId,
+      ArrayRef<Expr *> UnresolvedReductions = std::nullopt);
+  /// Called on well-formed 'task_reduction' clause.
+  OMPClause *ActOnOpenMPTaskReductionClause(
+      ArrayRef<Expr *> VarList, SourceLocation StartLoc,
+      SourceLocation LParenLoc, SourceLocation ColonLoc, SourceLocation EndLoc,
+      CXXScopeSpec &ReductionIdScopeSpec,
+      const DeclarationNameInfo &ReductionId,
+      ArrayRef<Expr *> UnresolvedReductions = std::nullopt);
+  /// Called on well-formed 'in_reduction' clause.
+  OMPClause *ActOnOpenMPInReductionClause(
+      ArrayRef<Expr *> VarList, SourceLocation StartLoc,
+      SourceLocation LParenLoc, SourceLocation ColonLoc, SourceLocation EndLoc,
+      CXXScopeSpec &ReductionIdScopeSpec,
+      const DeclarationNameInfo &ReductionId,
+      ArrayRef<Expr *> UnresolvedReductions = std::nullopt);
+  /// Called on well-formed 'linear' clause.
+  OMPClause *ActOnOpenMPLinearClause(
+      ArrayRef<Expr *> VarList, Expr *Step, SourceLocation StartLoc,
+      SourceLocation LParenLoc, OpenMPLinearClauseKind LinKind,
+      SourceLocation LinLoc, SourceLocation ColonLoc,
+      SourceLocation StepModifierLoc, SourceLocation EndLoc);
+  /// Called on well-formed 'aligned' clause.
+  OMPClause *ActOnOpenMPAlignedClause(ArrayRef<Expr *> VarList, Expr *Alignment,
+                                      SourceLocation StartLoc,
+                                      SourceLocation LParenLoc,
+                                      SourceLocation ColonLoc,
+                                      SourceLocation EndLoc);
+  /// Called on well-formed 'copyin' clause.
+  OMPClause *ActOnOpenMPCopyinClause(ArrayRef<Expr *> VarList,
+                                     SourceLocation StartLoc,
+                                     SourceLocation LParenLoc,
+                                     SourceLocation EndLoc);
+  /// Called on well-formed 'copyprivate' clause.
+  OMPClause *ActOnOpenMPCopyprivateClause(ArrayRef<Expr *> VarList,
+                                          SourceLocation StartLoc,
+                                          SourceLocation LParenLoc,
+                                          SourceLocation EndLoc);
+  /// Called on well-formed 'flush' pseudo clause.
+  OMPClause *ActOnOpenMPFlushClause(ArrayRef<Expr *> VarList,
+                                    SourceLocation StartLoc,
+                                    SourceLocation LParenLoc,
+                                    SourceLocation EndLoc);
+  /// Called on well-formed 'depobj' pseudo clause.
+  OMPClause *ActOnOpenMPDepobjClause(Expr *Depobj, SourceLocation StartLoc,
+                                     SourceLocation LParenLoc,
+                                     SourceLocation EndLoc);
+  /// Called on well-formed 'depend' clause.
+  OMPClause *ActOnOpenMPDependClause(const OMPDependClause::DependDataTy &Data,
+                                     Expr *DepModifier,
+                                     ArrayRef<Expr *> VarList,
+                                     SourceLocation StartLoc,
+                                     SourceLocation LParenLoc,
+                                     SourceLocation EndLoc);
+  /// Called on well-formed 'device' clause.
+  OMPClause *ActOnOpenMPDeviceClause(OpenMPDeviceClauseModifier Modifier,
+                                     Expr *Device, SourceLocation StartLoc,
+                                     SourceLocation LParenLoc,
+                                     SourceLocation ModifierLoc,
+                                     SourceLocation EndLoc);
+  /// Called on well-formed 'map' clause.
+  OMPClause *ActOnOpenMPMapClause(
+      Expr *IteratorModifier, ArrayRef<OpenMPMapModifierKind> MapTypeModifiers,
+      ArrayRef<SourceLocation> MapTypeModifiersLoc,
+      CXXScopeSpec &MapperIdScopeSpec, DeclarationNameInfo &MapperId,
+      OpenMPMapClauseKind MapType, bool IsMapTypeImplicit,
+      SourceLocation MapLoc, SourceLocation ColonLoc, ArrayRef<Expr *> VarList,
+      const OMPVarListLocTy &Locs, bool NoDiagnose = false,
+      ArrayRef<Expr *> UnresolvedMappers = std::nullopt);
+  /// Called on well-formed 'num_teams' clause.
+  OMPClause *ActOnOpenMPNumTeamsClause(Expr *NumTeams, SourceLocation StartLoc,
+                                       SourceLocation LParenLoc,
+                                       SourceLocation EndLoc);
+  /// Called on well-formed 'thread_limit' clause.
+  OMPClause *ActOnOpenMPThreadLimitClause(Expr *ThreadLimit,
+                                          SourceLocation StartLoc,
+                                          SourceLocation LParenLoc,
+                                          SourceLocation EndLoc);
+  /// Called on well-formed 'priority' clause.
+  OMPClause *ActOnOpenMPPriorityClause(Expr *Priority, SourceLocation StartLoc,
+                                       SourceLocation LParenLoc,
+                                       SourceLocation EndLoc);
+  /// Called on well-formed 'dist_schedule' clause.
+  OMPClause *ActOnOpenMPDistScheduleClause(
+      OpenMPDistScheduleClauseKind Kind, Expr *ChunkSize,
+      SourceLocation StartLoc, SourceLocation LParenLoc, SourceLocation KindLoc,
+      SourceLocation CommaLoc, SourceLocation EndLoc);
+  /// Called on well-formed 'defaultmap' clause.
+  OMPClause *ActOnOpenMPDefaultmapClause(
+      OpenMPDefaultmapClauseModifier M, OpenMPDefaultmapClauseKind Kind,
+      SourceLocation StartLoc, SourceLocation LParenLoc, SourceLocation MLoc,
+      SourceLocation KindLoc, SourceLocation EndLoc);
+  /// Called on well-formed 'to' clause.
+  OMPClause *
+  ActOnOpenMPToClause(ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
+                      ArrayRef<SourceLocation> MotionModifiersLoc,
+                      CXXScopeSpec &MapperIdScopeSpec,
+                      DeclarationNameInfo &MapperId, SourceLocation ColonLoc,
+                      ArrayRef<Expr *> VarList, const OMPVarListLocTy &Locs,
+                      ArrayRef<Expr *> UnresolvedMappers = std::nullopt);
+  /// Called on well-formed 'from' clause.
+  OMPClause *
+  ActOnOpenMPFromClause(ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
+                        ArrayRef<SourceLocation> MotionModifiersLoc,
+                        CXXScopeSpec &MapperIdScopeSpec,
+                        DeclarationNameInfo &MapperId, SourceLocation ColonLoc,
+                        ArrayRef<Expr *> VarList, const OMPVarListLocTy &Locs,
+                        ArrayRef<Expr *> UnresolvedMappers = std::nullopt);
+  /// Called on well-formed 'use_device_ptr' clause.
+  OMPClause *ActOnOpenMPUseDevicePtrClause(ArrayRef<Expr *> VarList,
+                                           const OMPVarListLocTy &Locs);
+  /// Called on well-formed 'use_device_addr' clause.
+  OMPClause *ActOnOpenMPUseDeviceAddrClause(ArrayRef<Expr *> VarList,
+                                            const OMPVarListLocTy &Locs);
+  /// Called on well-formed 'is_device_ptr' clause.
+  OMPClause *ActOnOpenMPIsDevicePtrClause(ArrayRef<Expr *> VarList,
+                                          const OMPVarListLocTy &Locs);
+  /// Called on well-formed 'has_device_addr' clause.
+  OMPClause *ActOnOpenMPHasDeviceAddrClause(ArrayRef<Expr *> VarList,
+                                            const OMPVarListLocTy &Locs);
+  /// Called on well-formed 'nontemporal' clause.
+  OMPClause *ActOnOpenMPNontemporalClause(ArrayRef<Expr *> VarList,
+                                          SourceLocation StartLoc,
+                                          SourceLocation LParenLoc,
+                                          SourceLocation EndLoc);
+
+  /// Data for list of allocators.
+  struct UsesAllocatorsData {
+    /// Allocator.
+    Expr *Allocator = nullptr;
+    /// Allocator traits.
+    Expr *AllocatorTraits = nullptr;
+    /// Locations of '(' and ')' symbols.
+    SourceLocation LParenLoc, RParenLoc;
+  };
+  /// Called on well-formed 'uses_allocators' clause.
+  OMPClause *ActOnOpenMPUsesAllocatorClause(SourceLocation StartLoc,
+                                            SourceLocation LParenLoc,
+                                            SourceLocation EndLoc,
+                                            ArrayRef<UsesAllocatorsData> Data);
+  /// Called on well-formed 'affinity' clause.
+  OMPClause *ActOnOpenMPAffinityClause(SourceLocation StartLoc,
+                                       SourceLocation LParenLoc,
+                                       SourceLocation ColonLoc,
+                                       SourceLocation EndLoc, Expr *Modifier,
+                                       ArrayRef<Expr *> Locators);
+  /// Called on a well-formed 'bind' clause.
+  OMPClause *ActOnOpenMPBindClause(OpenMPBindClauseKind Kind,
+                                   SourceLocation KindLoc,
+                                   SourceLocation StartLoc,
+                                   SourceLocation LParenLoc,
+                                   SourceLocation EndLoc);
+
+  /// Called on a well-formed 'ompx_dyn_cgroup_mem' clause.
+  OMPClause *ActOnOpenMPXDynCGroupMemClause(Expr *Size, SourceLocation StartLoc,
+                                            SourceLocation LParenLoc,
+                                            SourceLocation EndLoc);
+
+  /// Called on well-formed 'doacross' clause.
+  OMPClause *
+  ActOnOpenMPDoacrossClause(OpenMPDoacrossClauseModifier DepType,
+                            SourceLocation DepLoc, SourceLocation ColonLoc,
+                            ArrayRef<Expr *> VarList, SourceLocation StartLoc,
+                            SourceLocation LParenLoc, SourceLocation EndLoc);
+
+  /// Called on a well-formed 'ompx_attribute' clause.
+  OMPClause *ActOnOpenMPXAttributeClause(ArrayRef<const Attr *> Attrs,
+                                         SourceLocation StartLoc,
+                                         SourceLocation LParenLoc,
+                                         SourceLocation EndLoc);
+
+  /// Called on a well-formed 'ompx_bare' clause.
+  OMPClause *ActOnOpenMPXBareClause(SourceLocation StartLoc,
+                                    SourceLocation EndLoc);
+
+  ExprResult ActOnOMPArraySectionExpr(Expr *Base, SourceLocation LBLoc,
+                                      Expr *LowerBound,
+                                      SourceLocation ColonLocFirst,
+                                      SourceLocation ColonLocSecond,
+                                      Expr *Length, Expr *Stride,
+                                      SourceLocation RBLoc);
+  ExprResult ActOnOMPArrayShapingExpr(Expr *Base, SourceLocation LParenLoc,
+                                      SourceLocation RParenLoc,
+                                      ArrayRef<Expr *> Dims,
+                                      ArrayRef<SourceRange> Brackets);
+
+  /// Data structure for iterator expression.
+  struct OMPIteratorData {
+    IdentifierInfo *DeclIdent = nullptr;
+    SourceLocation DeclIdentLoc;
+    ParsedType Type;
+    OMPIteratorExpr::IteratorRange Range;
+    SourceLocation AssignLoc;
+    SourceLocation ColonLoc;
+    SourceLocation SecColonLoc;
+  };
+
+  ExprResult ActOnOMPIteratorExpr(Scope *S, SourceLocation IteratorKwLoc,
+                                  SourceLocation LLoc, SourceLocation RLoc,
+                                  ArrayRef<OMPIteratorData> Data);
+
+private:
+  void *VarDataSharingAttributesStack;
+
+  /// Number of nested '#pragma omp declare target' directives.
+  SmallVector<DeclareTargetContextInfo, 4> DeclareTargetNesting;
+
+  /// Initialization of data-sharing attributes stack.
+  void InitDataSharingAttributesStack();
+  void DestroyDataSharingAttributesStack();
+
+  /// Returns OpenMP nesting level for current directive.
+  unsigned getOpenMPNestingLevel() const;
+
+  /// Adjusts the function scopes index for the target-based regions.
+  void adjustOpenMPTargetScopeIndex(unsigned &FunctionScopesIndex,
+                                    unsigned Level) const;
+
+  /// Returns the number of scopes associated with the construct on the given
+  /// OpenMP level.
+  int getNumberOfConstructScopes(unsigned Level) const;
+
+  /// Push new OpenMP function region for non-capturing function.
+  void pushOpenMPFunctionRegion();
+
+  /// Pop OpenMP function region for non-capturing function.
+  void popOpenMPFunctionRegion(const sema::FunctionScopeInfo *OldFSI);
+
+  /// Analyzes and checks a loop nest for use by a loop transformation.
+  ///
+  /// \param Kind          The loop transformation directive kind.
+  /// \param NumLoops      How many nested loops the directive is expecting.
+  /// \param AStmt         Associated statement of the transformation directive.
+  /// \param LoopHelpers   [out] The loop analysis result.
+  /// \param Body          [out] The body code nested in \p NumLoops loop.
+  /// \param OriginalInits [out] Collection of statements and declarations that
+  ///                      must have been executed/declared before entering the
+  ///                      loop.
+  ///
+  /// \return Whether there was any error.
+  bool checkTransformableLoopNest(
+      OpenMPDirectiveKind Kind, Stmt *AStmt, int NumLoops,
+      SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
+      Stmt *&Body,
+      SmallVectorImpl<SmallVector<llvm::PointerUnion<Stmt *, Decl *>, 0>>
+          &OriginalInits);
+
+  /// Helper to keep information about the current `omp begin/end declare
+  /// variant` nesting.
+  struct OMPDeclareVariantScope {
+    /// The associated OpenMP context selector.
+    OMPTraitInfo *TI;
+
+    /// The associated OpenMP context selector mangling.
+    std::string NameSuffix;
+
+    OMPDeclareVariantScope(OMPTraitInfo &TI);
+  };
+
+  /// Return the OMPTraitInfo for the surrounding scope, if any.
+  OMPTraitInfo *getOMPTraitInfoForSurroundingScope() {
+    return OMPDeclareVariantScopes.empty() ? nullptr
+                                           : OMPDeclareVariantScopes.back().TI;
+  }
+
+  /// The current `omp begin/end declare variant` scopes.
+  SmallVector<OMPDeclareVariantScope, 4> OMPDeclareVariantScopes;
+
+  /// The current `omp begin/end assumes` scopes.
+  SmallVector<OMPAssumeAttr *, 4> OMPAssumeScoped;
+
+  /// All `omp assumes` we encountered so far.
+  SmallVector<OMPAssumeAttr *, 4> OMPAssumeGlobal;
+
+  /// OMPD_loop is mapped to OMPD_for, OMPD_distribute or OMPD_simd depending
+  /// on the parameter of the bind clause. In the methods for the
+  /// mapped directives, check the parameters of the lastprivate clause.
+  bool checkLastPrivateForMappedDirectives(ArrayRef<OMPClause *> Clauses);
+  /// Depending on the bind clause of OMPD_loop map the directive to new
+  /// directives.
+  ///    1) loop bind(parallel) --> OMPD_for
+  ///    2) loop bind(teams) --> OMPD_distribute
+  ///    3) loop bind(thread) --> OMPD_simd
+  /// This is being handled in Sema instead of Codegen because of the need for
+  /// rigorous semantic checking in the new mapped directives.
+  bool mapLoopConstruct(llvm::SmallVector<OMPClause *> &ClausesWithoutBind,
+                        ArrayRef<OMPClause *> Clauses,
+                        OpenMPBindClauseKind &BindKind,
+                        OpenMPDirectiveKind &Kind,
+                        OpenMPDirectiveKind &PrevMappedDirective,
+                        SourceLocation StartLoc, SourceLocation EndLoc,
+                        const DeclarationNameInfo &DirName,
+                        OpenMPDirectiveKind CancelRegion);
+};
+
+} // namespace clang
+
+#endif // LLVM_CLANG_SEMA_SEMAOPENMP_H
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index 2b934234b7cf..c881b3750777 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -28,6 +28,7 @@
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaDiagnostic.h"
+#include "clang/Sema/SemaOpenMP.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringSwitch.h"
@@ -2383,7 +2384,7 @@ Parser::DeclGroupPtrTy Parser::ParseDeclGroup(ParsingDeclSpec &DS,
       }
 
       if (getLangOpts().OpenMP)
-        Actions.startOpenMPCXXRangeFor();
+        Actions.OpenMP().startOpenMPCXXRangeFor();
       if (Tok.is(tok::l_brace))
         FRI->RangeExpr = ParseBraceInitializer();
       else
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index 473ec9afd601..32d96f81c4c8 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -31,6 +31,7 @@
 #include "clang/Sema/ParsedTemplate.h"
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/SemaCUDA.h"
+#include "clang/Sema/SemaOpenMP.h"
 #include "clang/Sema/SemaSYCL.h"
 #include "clang/Sema/TypoCorrection.h"
 #include "llvm/ADT/SmallVector.h"
@@ -2075,7 +2076,7 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
           // replace this call to ActOnOpenACCArraySectionExpr in the future.
           // Eventually we'll genericize the OPenMPArraySectionExpr type as
           // well.
-          LHS = Actions.ActOnOMPArraySectionExpr(
+          LHS = Actions.OpenMP().ActOnOMPArraySectionExpr(
               LHS.get(), Loc, ArgExprs.empty() ? nullptr : ArgExprs[0],
               ColonLocFirst, ColonLocSecond, Length.get(), Stride.get(), RLoc);
         } else {
@@ -3277,7 +3278,7 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
     if (ErrorFound) {
       Result = ExprError();
     } else if (!Result.isInvalid()) {
-      Result = Actions.ActOnOMPArrayShapingExpr(
+      Result = Actions.OpenMP().ActOnOMPArrayShapingExpr(
           Result.get(), OpenLoc, RParenLoc, OMPDimensions, OMPBracketsRanges);
     }
     return Result;
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index 814126e321d3..480201bc06f6 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -21,6 +21,7 @@
 #include "clang/Parse/RAIIObjectsForParser.h"
 #include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/Scope.h"
+#include "clang/Sema/SemaOpenMP.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/ADT/UniqueVector.h"
@@ -87,7 +88,7 @@ public:
   DeclDirectiveListParserHelper(Parser *P, OpenMPDirectiveKind Kind)
       : P(P), Kind(Kind) {}
   void operator()(CXXScopeSpec &SS, DeclarationNameInfo NameInfo) {
-    ExprResult Res = P->getActions().ActOnOpenMPIdExpression(
+    ExprResult Res = P->getActions().OpenMP().ActOnOpenMPIdExpression(
         P->getCurScope(), SS, NameInfo, Kind);
     if (Res.isUsable())
       Identifiers.push_back(Res.get());
@@ -322,8 +323,8 @@ Parser::ParseOpenMPDeclareReductionDirective(AccessSpecifier AS) {
     SourceRange Range;
     TypeResult TR = ParseTypeName(&Range, DeclaratorContext::Prototype, AS);
     if (TR.isUsable()) {
-      QualType ReductionType =
-          Actions.ActOnOpenMPDeclareReductionType(Range.getBegin(), TR);
+      QualType ReductionType = Actions.OpenMP().ActOnOpenMPDeclareReductionType(
+          Range.getBegin(), TR);
       if (!ReductionType.isNull()) {
         ReductionTypes.push_back(
             std::make_pair(ReductionType, Range.getBegin()));
@@ -363,8 +364,10 @@ Parser::ParseOpenMPDeclareReductionDirective(AccessSpecifier AS) {
     return DeclGroupPtrTy();
   }
 
-  DeclGroupPtrTy DRD = Actions.ActOnOpenMPDeclareReductionDirectiveStart(
-      getCurScope(), Actions.getCurLexicalContext(), Name, ReductionTypes, AS);
+  DeclGroupPtrTy DRD =
+      Actions.OpenMP().ActOnOpenMPDeclareReductionDirectiveStart(
+          getCurScope(), Actions.getCurLexicalContext(), Name, ReductionTypes,
+          AS);
 
   // Parse <combiner> expression and then parse initializer if any for each
   // correct type.
@@ -375,10 +378,11 @@ Parser::ParseOpenMPDeclareReductionDirective(AccessSpecifier AS) {
                                     Scope::CompoundStmtScope |
                                     Scope::OpenMPDirectiveScope);
     // Parse <combiner> expression.
-    Actions.ActOnOpenMPDeclareReductionCombinerStart(getCurScope(), D);
+    Actions.OpenMP().ActOnOpenMPDeclareReductionCombinerStart(getCurScope(), D);
     ExprResult CombinerResult = Actions.ActOnFinishFullExpr(
         ParseExpression().get(), D->getLocation(), /*DiscardedValue*/ false);
-    Actions.ActOnOpenMPDeclareReductionCombinerEnd(D, CombinerResult.get());
+    Actions.OpenMP().ActOnOpenMPDeclareReductionCombinerEnd(
+        D, CombinerResult.get());
 
     if (CombinerResult.isInvalid() && Tok.isNot(tok::r_paren) &&
         Tok.isNot(tok::annot_pragma_openmp_end)) {
@@ -411,8 +415,8 @@ Parser::ParseOpenMPDeclareReductionDirective(AccessSpecifier AS) {
                                         Scope::OpenMPDirectiveScope);
         // Parse expression.
         VarDecl *OmpPrivParm =
-            Actions.ActOnOpenMPDeclareReductionInitializerStart(getCurScope(),
-                                                                D);
+            Actions.OpenMP().ActOnOpenMPDeclareReductionInitializerStart(
+                getCurScope(), D);
         // Check if initializer is omp_priv <init_expr> or something else.
         if (Tok.is(tok::identifier) &&
             Tok.getIdentifierInfo()->isStr("omp_priv")) {
@@ -423,7 +427,7 @@ Parser::ParseOpenMPDeclareReductionDirective(AccessSpecifier AS) {
               ParseAssignmentExpression().get(), D->getLocation(),
               /*DiscardedValue*/ false);
         }
-        Actions.ActOnOpenMPDeclareReductionInitializerEnd(
+        Actions.OpenMP().ActOnOpenMPDeclareReductionInitializerEnd(
             D, InitializerResult.get(), OmpPrivParm);
         if (InitializerResult.isInvalid() && Tok.isNot(tok::r_paren) &&
             Tok.isNot(tok::annot_pragma_openmp_end)) {
@@ -444,8 +448,8 @@ Parser::ParseOpenMPDeclareReductionDirective(AccessSpecifier AS) {
     else
       TPA.Commit();
   }
-  return Actions.ActOnOpenMPDeclareReductionDirectiveEnd(getCurScope(), DRD,
-                                                         IsCorrect);
+  return Actions.OpenMP().ActOnOpenMPDeclareReductionDirectiveEnd(
+      getCurScope(), DRD, IsCorrect);
 }
 
 void Parser::ParseOpenMPReductionInitializerForDecl(VarDecl *OmpPrivParm) {
@@ -569,8 +573,8 @@ Parser::ParseOpenMPDeclareMapperDirective(AccessSpecifier AS) {
   SourceRange Range;
   TypeResult ParsedType = parseOpenMPDeclareMapperVarDecl(Range, VName, AS);
   if (ParsedType.isUsable())
-    MapperType =
-        Actions.ActOnOpenMPDeclareMapperType(Range.getBegin(), ParsedType);
+    MapperType = Actions.OpenMP().ActOnOpenMPDeclareMapperType(Range.getBegin(),
+                                                               ParsedType);
   if (MapperType.isNull())
     IsCorrect = false;
   if (!IsCorrect) {
@@ -591,11 +595,13 @@ Parser::ParseOpenMPDeclareMapperDirective(AccessSpecifier AS) {
   unsigned ScopeFlags = Scope::FnScope | Scope::DeclScope |
                         Scope::CompoundStmtScope | Scope::OpenMPDirectiveScope;
   ParseScope OMPDirectiveScope(this, ScopeFlags);
-  Actions.StartOpenMPDSABlock(OMPD_declare_mapper, DirName, getCurScope(), Loc);
+  Actions.OpenMP().StartOpenMPDSABlock(OMPD_declare_mapper, DirName,
+                                       getCurScope(), Loc);
 
   // Add the mapper variable declaration.
-  ExprResult MapperVarRef = Actions.ActOnOpenMPDeclareMapperDirectiveVarDecl(
-      getCurScope(), MapperType, Range.getBegin(), VName);
+  ExprResult MapperVarRef =
+      Actions.OpenMP().ActOnOpenMPDeclareMapperDirectiveVarDecl(
+          getCurScope(), MapperType, Range.getBegin(), VName);
 
   // Parse map clauses.
   SmallVector<OMPClause *, 6> Clauses;
@@ -603,7 +609,7 @@ Parser::ParseOpenMPDeclareMapperDirective(AccessSpecifier AS) {
     OpenMPClauseKind CKind = Tok.isAnnotation()
                                  ? OMPC_unknown
                                  : getOpenMPClauseKind(PP.getSpelling(Tok));
-    Actions.StartOpenMPClause(CKind);
+    Actions.OpenMP().StartOpenMPClause(CKind);
     OMPClause *Clause =
         ParseOpenMPClause(OMPD_declare_mapper, CKind, Clauses.empty());
     if (Clause)
@@ -613,7 +619,7 @@ Parser::ParseOpenMPDeclareMapperDirective(AccessSpecifier AS) {
     // Skip ',' if any.
     if (Tok.is(tok::comma))
       ConsumeToken();
-    Actions.EndOpenMPClause();
+    Actions.OpenMP().EndOpenMPClause();
   }
   if (Clauses.empty()) {
     Diag(Tok, diag::err_omp_expected_clause)
@@ -622,9 +628,9 @@ Parser::ParseOpenMPDeclareMapperDirective(AccessSpecifier AS) {
   }
 
   // Exit scope.
-  Actions.EndOpenMPDSABlock(nullptr);
+  Actions.OpenMP().EndOpenMPDSABlock(nullptr);
   OMPDirectiveScope.Exit();
-  DeclGroupPtrTy DG = Actions.ActOnOpenMPDeclareMapperDirective(
+  DeclGroupPtrTy DG = Actions.OpenMP().ActOnOpenMPDeclareMapperDirective(
       getCurScope(), Actions.getCurLexicalContext(), MapperId, MapperType,
       Range.getBegin(), VName, AS, MapperVarRef.get(), Clauses);
   if (!IsCorrect)
@@ -652,7 +658,8 @@ TypeResult Parser::parseOpenMPDeclareMapperVarDecl(SourceRange &Range,
   }
   Name = Actions.GetNameForDeclarator(DeclaratorInfo).getName();
 
-  return Actions.ActOnOpenMPDeclareMapperVarDecl(getCurScope(), DeclaratorInfo);
+  return Actions.OpenMP().ActOnOpenMPDeclareMapperVarDecl(getCurScope(),
+                                                          DeclaratorInfo);
 }
 
 namespace {
@@ -748,7 +755,7 @@ static bool parseDeclareSimdClauses(
       OpenMPClauseKind CKind = getOpenMPClauseKind(ClauseName);
       if (CKind == OMPC_uniform || CKind == OMPC_aligned ||
           CKind == OMPC_linear) {
-        Sema::OpenMPVarListDataTy Data;
+        SemaOpenMP::OpenMPVarListDataTy Data;
         SmallVectorImpl<Expr *> *Vars = &Uniforms;
         if (CKind == OMPC_aligned) {
           Vars = &Aligneds;
@@ -768,7 +775,7 @@ static bool parseDeclareSimdClauses(
           assert(0 <= Data.ExtraModifier &&
                  Data.ExtraModifier <= OMPC_LINEAR_unknown &&
                  "Unexpected linear modifier.");
-          if (P.getActions().CheckOpenMPLinearModifier(
+          if (P.getActions().OpenMP().CheckOpenMPLinearModifier(
                   static_cast<OpenMPLinearClauseKind>(Data.ExtraModifier),
                   Data.ExtraModifierLoc))
             Data.ExtraModifier = OMPC_LINEAR_val;
@@ -816,7 +823,7 @@ Parser::ParseOMPDeclareSimdClauses(Parser::DeclGroupPtrTy Ptr,
   SourceLocation EndLoc = ConsumeAnnotationToken();
   if (IsError)
     return Ptr;
-  return Actions.ActOnOpenMPDeclareSimdDirective(
+  return Actions.OpenMP().ActOnOpenMPDeclareSimdDirective(
       Ptr, BS, Simdlen.get(), Uniforms, Aligneds, Alignments, Linears,
       LinModifiers, Steps, SourceRange(Loc, EndLoc));
 }
@@ -1412,7 +1419,8 @@ void Parser::ParseOMPDeclareVariantClauses(Parser::DeclGroupPtrTy Ptr,
     return;
   }
 
-  OMPTraitInfo *ParentTI = Actions.getOMPTraitInfoForSurroundingScope();
+  OMPTraitInfo *ParentTI =
+      Actions.OpenMP().getOMPTraitInfoForSurroundingScope();
   ASTContext &ASTCtx = Actions.getASTContext();
   OMPTraitInfo &TI = ASTCtx.getNewOMPTraitInfo();
   SmallVector<Expr *, 6> AdjustNothing;
@@ -1445,7 +1453,7 @@ void Parser::ParseOMPDeclareVariantClauses(Parser::DeclGroupPtrTy Ptr,
       case OMPC_adjust_args: {
         AdjustArgsLoc = Tok.getLocation();
         ConsumeToken();
-        Sema::OpenMPVarListDataTy Data;
+        SemaOpenMP::OpenMPVarListDataTy Data;
         SmallVector<Expr *> Vars;
         IsError = ParseOpenMPVarList(OMPD_declare_variant, OMPC_adjust_args,
                                      Vars, Data);
@@ -1486,12 +1494,12 @@ void Parser::ParseOMPDeclareVariantClauses(Parser::DeclGroupPtrTy Ptr,
   }
 
   std::optional<std::pair<FunctionDecl *, Expr *>> DeclVarData =
-      Actions.checkOpenMPDeclareVariantFunction(
+      Actions.OpenMP().checkOpenMPDeclareVariantFunction(
           Ptr, AssociatedFunction.get(), TI, AppendArgs.size(),
           SourceRange(Loc, Tok.getLocation()));
 
   if (DeclVarData && !TI.Sets.empty())
-    Actions.ActOnOpenMPDeclareVariantDirective(
+    Actions.OpenMP().ActOnOpenMPDeclareVariantDirective(
         DeclVarData->first, DeclVarData->second, TI, AdjustNothing,
         AdjustNeedDevicePtr, AppendArgs, AdjustArgsLoc, AppendArgsLoc,
         SourceRange(Loc, Tok.getLocation()));
@@ -1642,7 +1650,7 @@ void Parser::ParseOpenMPClauses(OpenMPDirectiveKind DKind,
     OpenMPClauseKind CKind = Tok.isAnnotation()
                                  ? OMPC_unknown
                                  : getOpenMPClauseKind(PP.getSpelling(Tok));
-    Actions.StartOpenMPClause(CKind);
+    Actions.OpenMP().StartOpenMPClause(CKind);
     OMPClause *Clause = ParseOpenMPClause(
         DKind, CKind, !FirstClauses[unsigned(CKind)].getInt());
     SkipUntil(tok::comma, tok::identifier, tok::annot_pragma_openmp_end,
@@ -1651,13 +1659,13 @@ void Parser::ParseOpenMPClauses(OpenMPDirectiveKind DKind,
     if (Clause != nullptr)
       Clauses.push_back(Clause);
     if (Tok.is(tok::annot_pragma_openmp_end)) {
-      Actions.EndOpenMPClause();
+      Actions.OpenMP().EndOpenMPClause();
       break;
     }
     // Skip ',' if any.
     if (Tok.is(tok::comma))
       ConsumeToken();
-    Actions.EndOpenMPClause();
+    Actions.OpenMP().EndOpenMPClause();
   }
 }
 
@@ -1750,12 +1758,13 @@ void Parser::ParseOpenMPAssumesDirective(OpenMPDirectiveKind DKind,
     Assumptions.push_back(Assumption);
   }
 
-  Actions.ActOnOpenMPAssumesDirective(Loc, DKind, Assumptions, SkippedClauses);
+  Actions.OpenMP().ActOnOpenMPAssumesDirective(Loc, DKind, Assumptions,
+                                               SkippedClauses);
 }
 
 void Parser::ParseOpenMPEndAssumesDirective(SourceLocation Loc) {
-  if (Actions.isInOpenMPAssumeScope())
-    Actions.ActOnOpenMPEndAssumesDirective();
+  if (Actions.OpenMP().isInOpenMPAssumeScope())
+    Actions.OpenMP().ActOnOpenMPEndAssumesDirective();
   else
     Diag(Loc, diag::err_expected_begin_assumes);
 }
@@ -1811,7 +1820,7 @@ parseOpenMPSimpleClause(Parser &P, OpenMPClauseKind Kind) {
 }
 
 void Parser::ParseOMPDeclareTargetClauses(
-    Sema::DeclareTargetContextInfo &DTCI) {
+    SemaOpenMP::DeclareTargetContextInfo &DTCI) {
   SourceLocation DeviceTypeLoc;
   bool RequiresToOrLinkOrIndirectClause = false;
   bool HasToOrLinkOrIndirectClause = false;
@@ -1910,11 +1919,11 @@ void Parser::ParseOMPDeclareTargetClauses(
     if (DTCI.Kind == OMPD_declare_target || HasIdentifier) {
       auto &&Callback = [this, MT, &DTCI](CXXScopeSpec &SS,
                                           DeclarationNameInfo NameInfo) {
-        NamedDecl *ND =
-            Actions.lookupOpenMPDeclareTargetName(getCurScope(), SS, NameInfo);
+        NamedDecl *ND = Actions.OpenMP().lookupOpenMPDeclareTargetName(
+            getCurScope(), SS, NameInfo);
         if (!ND)
           return;
-        Sema::DeclareTargetContextInfo::MapInfo MI{MT, NameInfo.getLoc()};
+        SemaOpenMP::DeclareTargetContextInfo::MapInfo MI{MT, NameInfo.getLoc()};
         bool FirstMapping = DTCI.ExplicitlyMapped.try_emplace(ND, MI).second;
         if (!FirstMapping)
           Diag(NameInfo.getLoc(), diag::err_omp_declare_target_multiple)
@@ -2090,8 +2099,8 @@ Parser::DeclGroupPtrTy Parser::ParseOpenMPDeclarativeDirectiveWithExtDecl(
       skipUntilPragmaOpenMPEnd(DKind);
       // Skip the last annot_pragma_openmp_end.
       ConsumeAnnotationToken();
-      return Actions.ActOnOpenMPThreadprivateDirective(Loc,
-                                                       Helper.getIdentifiers());
+      return Actions.OpenMP().ActOnOpenMPThreadprivateDirective(
+          Loc, Helper.getIdentifiers());
     }
     break;
   }
@@ -2109,7 +2118,7 @@ Parser::DeclGroupPtrTy Parser::ParseOpenMPDeclarativeDirectiveWithExtDecl(
           OpenMPClauseKind CKind =
               Tok.isAnnotation() ? OMPC_unknown
                                  : getOpenMPClauseKind(PP.getSpelling(Tok));
-          Actions.StartOpenMPClause(CKind);
+          Actions.OpenMP().StartOpenMPClause(CKind);
           OMPClause *Clause = ParseOpenMPClause(
               OMPD_allocate, CKind, !FirstClauses[unsigned(CKind)].getInt());
           SkipUntil(tok::comma, tok::identifier, tok::annot_pragma_openmp_end,
@@ -2118,20 +2127,20 @@ Parser::DeclGroupPtrTy Parser::ParseOpenMPDeclarativeDirectiveWithExtDecl(
           if (Clause != nullptr)
             Clauses.push_back(Clause);
           if (Tok.is(tok::annot_pragma_openmp_end)) {
-            Actions.EndOpenMPClause();
+            Actions.OpenMP().EndOpenMPClause();
             break;
           }
           // Skip ',' if any.
           if (Tok.is(tok::comma))
             ConsumeToken();
-          Actions.EndOpenMPClause();
+          Actions.OpenMP().EndOpenMPClause();
         }
         skipUntilPragmaOpenMPEnd(DKind);
       }
       // Skip the last annot_pragma_openmp_end.
       ConsumeAnnotationToken();
-      return Actions.ActOnOpenMPAllocateDirective(Loc, Helper.getIdentifiers(),
-                                                  Clauses);
+      return Actions.OpenMP().ActOnOpenMPAllocateDirective(
+          Loc, Helper.getIdentifiers(), Clauses);
     }
     break;
   }
@@ -2150,7 +2159,7 @@ Parser::DeclGroupPtrTy Parser::ParseOpenMPDeclarativeDirectiveWithExtDecl(
       OpenMPClauseKind CKind = Tok.isAnnotation()
                                    ? OMPC_unknown
                                    : getOpenMPClauseKind(PP.getSpelling(Tok));
-      Actions.StartOpenMPClause(CKind);
+      Actions.OpenMP().StartOpenMPClause(CKind);
       OMPClause *Clause = ParseOpenMPClause(
           OMPD_requires, CKind, !FirstClauses[unsigned(CKind)].getInt());
       SkipUntil(tok::comma, tok::identifier, tok::annot_pragma_openmp_end,
@@ -2159,13 +2168,13 @@ Parser::DeclGroupPtrTy Parser::ParseOpenMPDeclarativeDirectiveWithExtDecl(
       if (Clause != nullptr)
         Clauses.push_back(Clause);
       if (Tok.is(tok::annot_pragma_openmp_end)) {
-        Actions.EndOpenMPClause();
+        Actions.OpenMP().EndOpenMPClause();
         break;
       }
       // Skip ',' if any.
       if (Tok.is(tok::comma))
         ConsumeToken();
-      Actions.EndOpenMPClause();
+      Actions.OpenMP().EndOpenMPClause();
     }
     // Consume final annot_pragma_openmp_end
     if (Clauses.empty()) {
@@ -2175,14 +2184,15 @@ Parser::DeclGroupPtrTy Parser::ParseOpenMPDeclarativeDirectiveWithExtDecl(
       return nullptr;
     }
     ConsumeAnnotationToken();
-    return Actions.ActOnOpenMPRequiresDirective(StartLoc, Clauses);
+    return Actions.OpenMP().ActOnOpenMPRequiresDirective(StartLoc, Clauses);
   }
   case OMPD_error: {
     SmallVector<OMPClause *, 1> Clauses;
     SourceLocation StartLoc = ConsumeToken();
     ParseOpenMPClauses(DKind, Clauses, StartLoc);
-    Actions.ActOnOpenMPErrorDirective(Clauses, StartLoc, SourceLocation(),
-                                      /*InExContext = */ false);
+    Actions.OpenMP().ActOnOpenMPErrorDirective(Clauses, StartLoc,
+                                               SourceLocation(),
+                                               /*InExContext = */ false);
     break;
   }
   case OMPD_assumes:
@@ -2217,7 +2227,8 @@ Parser::DeclGroupPtrTy Parser::ParseOpenMPDeclarativeDirectiveWithExtDecl(
     // { #pragma omp end declare variant }
     //
     ConsumeToken();
-    OMPTraitInfo *ParentTI = Actions.getOMPTraitInfoForSurroundingScope();
+    OMPTraitInfo *ParentTI =
+        Actions.OpenMP().getOMPTraitInfoForSurroundingScope();
     ASTContext &ASTCtx = Actions.getASTContext();
     OMPTraitInfo &TI = ASTCtx.getNewOMPTraitInfo();
     if (parseOMPDeclareVariantMatchClause(Loc, TI, ParentTI)) {
@@ -2248,7 +2259,7 @@ Parser::DeclGroupPtrTy Parser::ParseOpenMPDeclarativeDirectiveWithExtDecl(
         /* ConstructTraits */ ArrayRef<llvm::omp::TraitProperty>());
 
     if (isVariantApplicableInContext(VMI, OMPCtx, /* DeviceSetOnly */ true)) {
-      Actions.ActOnOpenMPBeginDeclareVariant(Loc, TI);
+      Actions.OpenMP().ActOnOpenMPBeginDeclareVariant(Loc, TI);
       break;
     }
 
@@ -2275,8 +2286,8 @@ Parser::DeclGroupPtrTy Parser::ParseOpenMPDeclarativeDirectiveWithExtDecl(
     break;
   }
   case OMPD_end_declare_variant: {
-    if (Actions.isInOpenMPDeclareVariantScope())
-      Actions.ActOnOpenMPEndDeclareVariant();
+    if (Actions.OpenMP().isInOpenMPDeclareVariantScope())
+      Actions.OpenMP().ActOnOpenMPEndDeclareVariant();
     else
       Diag(Loc, diag::err_expected_begin_declare_variant);
     ConsumeToken();
@@ -2331,7 +2342,7 @@ Parser::DeclGroupPtrTy Parser::ParseOpenMPDeclarativeDirectiveWithExtDecl(
   case OMPD_declare_target: {
     SourceLocation DTLoc = ConsumeAnyToken();
     bool HasClauses = Tok.isNot(tok::annot_pragma_openmp_end);
-    Sema::DeclareTargetContextInfo DTCI(DKind, DTLoc);
+    SemaOpenMP::DeclareTargetContextInfo DTCI(DKind, DTLoc);
     if (HasClauses)
       ParseOMPDeclareTargetClauses(DTCI);
     bool HasImplicitMappings = DKind == OMPD_begin_declare_target ||
@@ -2342,24 +2353,24 @@ Parser::DeclGroupPtrTy Parser::ParseOpenMPDeclarativeDirectiveWithExtDecl(
     ConsumeAnyToken();
 
     if (HasImplicitMappings) {
-      Actions.ActOnStartOpenMPDeclareTargetContext(DTCI);
+      Actions.OpenMP().ActOnStartOpenMPDeclareTargetContext(DTCI);
       return nullptr;
     }
 
-    Actions.ActOnFinishedOpenMPDeclareTargetContext(DTCI);
+    Actions.OpenMP().ActOnFinishedOpenMPDeclareTargetContext(DTCI);
     llvm::SmallVector<Decl *, 4> Decls;
     for (auto &It : DTCI.ExplicitlyMapped)
       Decls.push_back(It.first);
     return Actions.BuildDeclaratorGroup(Decls);
   }
   case OMPD_end_declare_target: {
-    if (!Actions.isInOpenMPDeclareTargetContext()) {
+    if (!Actions.OpenMP().isInOpenMPDeclareTargetContext()) {
       Diag(Tok, diag::err_omp_unexpected_directive)
           << 1 << getOpenMPDirectiveName(DKind);
       break;
     }
-    const Sema::DeclareTargetContextInfo &DTCI =
-        Actions.ActOnOpenMPEndDeclareTargetDirective();
+    const SemaOpenMP::DeclareTargetContextInfo &DTCI =
+        Actions.OpenMP().ActOnOpenMPEndDeclareTargetDirective();
     ParseOMPEndDeclareTargetDirective(DTCI.Kind, DKind, DTCI.Loc);
     return nullptr;
   }
@@ -2683,7 +2694,7 @@ StmtResult Parser::ParseOpenMPDeclarativeOrExecutableDirective(
     if (!ParseOpenMPSimpleVarList(DKind, Helper,
                                   /*AllowScopeSpecifier=*/false)) {
       skipUntilPragmaOpenMPEnd(DKind);
-      DeclGroupPtrTy Res = Actions.ActOnOpenMPThreadprivateDirective(
+      DeclGroupPtrTy Res = Actions.OpenMP().ActOnOpenMPThreadprivateDirective(
           Loc, Helper.getIdentifiers());
       Directive = Actions.ActOnDeclStmt(Res, Loc, Tok.getLocation());
     }
@@ -2710,7 +2721,7 @@ StmtResult Parser::ParseOpenMPDeclarativeOrExecutableDirective(
           OpenMPClauseKind CKind =
               Tok.isAnnotation() ? OMPC_unknown
                                  : getOpenMPClauseKind(PP.getSpelling(Tok));
-          Actions.StartOpenMPClause(CKind);
+          Actions.OpenMP().StartOpenMPClause(CKind);
           OMPClause *Clause = ParseOpenMPClause(
               OMPD_allocate, CKind, !FirstClauses[unsigned(CKind)].getInt());
           SkipUntil(tok::comma, tok::identifier, tok::annot_pragma_openmp_end,
@@ -2719,17 +2730,17 @@ StmtResult Parser::ParseOpenMPDeclarativeOrExecutableDirective(
           if (Clause != nullptr)
             Clauses.push_back(Clause);
           if (Tok.is(tok::annot_pragma_openmp_end)) {
-            Actions.EndOpenMPClause();
+            Actions.OpenMP().EndOpenMPClause();
             break;
           }
           // Skip ',' if any.
           if (Tok.is(tok::comma))
             ConsumeToken();
-          Actions.EndOpenMPClause();
+          Actions.OpenMP().EndOpenMPClause();
         }
         skipUntilPragmaOpenMPEnd(DKind);
       }
-      DeclGroupPtrTy Res = Actions.ActOnOpenMPAllocateDirective(
+      DeclGroupPtrTy Res = Actions.OpenMP().ActOnOpenMPAllocateDirective(
           Loc, Helper.getIdentifiers(), Clauses);
       Directive = Actions.ActOnDeclStmt(Res, Loc, Tok.getLocation());
     }
@@ -2875,7 +2886,8 @@ StmtResult Parser::ParseOpenMPDeclarativeOrExecutableDirective(
     if (isOpenMPSimdDirective(DKind))
       ScopeFlags |= Scope::OpenMPSimdDirectiveScope;
     ParseScope OMPDirectiveScope(this, ScopeFlags);
-    Actions.StartOpenMPDSABlock(DKind, DirName, Actions.getCurScope(), Loc);
+    Actions.OpenMP().StartOpenMPDSABlock(DKind, DirName, Actions.getCurScope(),
+                                         Loc);
 
     while (Tok.isNot(tok::annot_pragma_openmp_end)) {
       // If we are parsing for a directive within a metadirective, the directive
@@ -2909,7 +2921,7 @@ StmtResult Parser::ParseOpenMPDeclarativeOrExecutableDirective(
       }
       // No more implicit clauses allowed.
       ImplicitClauseAllowed = false;
-      Actions.StartOpenMPClause(CKind);
+      Actions.OpenMP().StartOpenMPClause(CKind);
       HasImplicitClause = false;
       OMPClause *Clause = ParseOpenMPClause(
           DKind, CKind, !FirstClauses[unsigned(CKind)].getInt());
@@ -2922,7 +2934,7 @@ StmtResult Parser::ParseOpenMPDeclarativeOrExecutableDirective(
       // Skip ',' if any.
       if (Tok.is(tok::comma))
         ConsumeToken();
-      Actions.EndOpenMPClause();
+      Actions.OpenMP().EndOpenMPClause();
     }
     // End location of the directive.
     EndLoc = Tok.getLocation();
@@ -2953,7 +2965,7 @@ StmtResult Parser::ParseOpenMPDeclarativeOrExecutableDirective(
     StmtResult AssociatedStmt;
     if (HasAssociatedStatement) {
       // The body is a block scope like in Lambdas and Blocks.
-      Actions.ActOnOpenMPRegionStart(DKind, getCurScope());
+      Actions.OpenMP().ActOnOpenMPRegionStart(DKind, getCurScope());
       // FIXME: We create a bogus CompoundStmt scope to hold the contents of
       // the captured region. Code elsewhere assumes that any FunctionScopeInfo
       // should have at least one compound statement scope within it.
@@ -2964,30 +2976,33 @@ StmtResult Parser::ParseOpenMPDeclarativeOrExecutableDirective(
 
         if (AssociatedStmt.isUsable() && isOpenMPLoopDirective(DKind) &&
             getLangOpts().OpenMPIRBuilder)
-          AssociatedStmt = Actions.ActOnOpenMPLoopnest(AssociatedStmt.get());
+          AssociatedStmt =
+              Actions.OpenMP().ActOnOpenMPLoopnest(AssociatedStmt.get());
       }
-      AssociatedStmt = Actions.ActOnOpenMPRegionEnd(AssociatedStmt, Clauses);
+      AssociatedStmt =
+          Actions.OpenMP().ActOnOpenMPRegionEnd(AssociatedStmt, Clauses);
     } else if (DKind == OMPD_target_update || DKind == OMPD_target_enter_data ||
                DKind == OMPD_target_exit_data) {
-      Actions.ActOnOpenMPRegionStart(DKind, getCurScope());
+      Actions.OpenMP().ActOnOpenMPRegionStart(DKind, getCurScope());
       AssociatedStmt = (Sema::CompoundScopeRAII(Actions),
                         Actions.ActOnCompoundStmt(Loc, Loc, std::nullopt,
                                                   /*isStmtExpr=*/false));
-      AssociatedStmt = Actions.ActOnOpenMPRegionEnd(AssociatedStmt, Clauses);
+      AssociatedStmt =
+          Actions.OpenMP().ActOnOpenMPRegionEnd(AssociatedStmt, Clauses);
     }
-    Directive = Actions.ActOnOpenMPExecutableDirective(
+    Directive = Actions.OpenMP().ActOnOpenMPExecutableDirective(
         DKind, DirName, CancelRegion, Clauses, AssociatedStmt.get(), Loc,
         EndLoc);
 
     // Exit scope.
-    Actions.EndOpenMPDSABlock(Directive.get());
+    Actions.OpenMP().EndOpenMPDSABlock(Directive.get());
     OMPDirectiveScope.Exit();
     break;
   }
   case OMPD_declare_target: {
     SourceLocation DTLoc = ConsumeAnyToken();
     bool HasClauses = Tok.isNot(tok::annot_pragma_openmp_end);
-    Sema::DeclareTargetContextInfo DTCI(DKind, DTLoc);
+    SemaOpenMP::DeclareTargetContextInfo DTCI(DKind, DTLoc);
     if (HasClauses)
       ParseOMPDeclareTargetClauses(DTCI);
     bool HasImplicitMappings =
@@ -3003,7 +3018,7 @@ StmtResult Parser::ParseOpenMPDeclarativeOrExecutableDirective(
     // Skip the last annot_pragma_openmp_end.
     ConsumeAnyToken();
 
-    Actions.ActOnFinishedOpenMPDeclareTargetContext(DTCI);
+    Actions.OpenMP().ActOnFinishedOpenMPDeclareTargetContext(DTCI);
     break;
   }
   case OMPD_declare_simd:
@@ -3118,7 +3133,7 @@ OMPClause *Parser::ParseOpenMPSizesClause() {
 
   T.consumeClose();
 
-  return Actions.ActOnOpenMPSizesClause(
+  return Actions.OpenMP().ActOnOpenMPSizesClause(
       ValExprs, ClauseNameLoc, T.getOpenLocation(), T.getCloseLocation());
 }
 
@@ -3130,7 +3145,7 @@ OMPClause *Parser::ParseOpenMPUsesAllocatorClause(OpenMPDirectiveKind DKind) {
   BalancedDelimiterTracker T(*this, tok::l_paren, tok::annot_pragma_openmp_end);
   if (T.expectAndConsume(diag::err_expected_lparen_after, "uses_allocator"))
     return nullptr;
-  SmallVector<Sema::UsesAllocatorsData, 4> Data;
+  SmallVector<SemaOpenMP::UsesAllocatorsData, 4> Data;
   do {
     CXXScopeSpec SS;
     Token Replacement;
@@ -3144,7 +3159,7 @@ OMPClause *Parser::ParseOpenMPUsesAllocatorClause(OpenMPDirectiveKind DKind) {
                 StopBeforeMatch);
       break;
     }
-    Sema::UsesAllocatorsData &D = Data.emplace_back();
+    SemaOpenMP::UsesAllocatorsData &D = Data.emplace_back();
     D.Allocator = Allocator.get();
     if (Tok.is(tok::l_paren)) {
       BalancedDelimiterTracker T(*this, tok::l_paren,
@@ -3169,8 +3184,8 @@ OMPClause *Parser::ParseOpenMPUsesAllocatorClause(OpenMPDirectiveKind DKind) {
       ConsumeAnyToken();
   } while (Tok.isNot(tok::r_paren) && Tok.isNot(tok::annot_pragma_openmp_end));
   T.consumeClose();
-  return Actions.ActOnOpenMPUsesAllocatorClause(Loc, T.getOpenLocation(),
-                                                T.getCloseLocation(), Data);
+  return Actions.OpenMP().ActOnOpenMPUsesAllocatorClause(
+      Loc, T.getOpenLocation(), T.getCloseLocation(), Data);
 }
 
 /// Parsing of OpenMP clauses.
@@ -3538,15 +3553,16 @@ OMPClause *Parser::ParseOpenMPSingleExprClause(OpenMPClauseKind Kind,
 
   if (ParseOnly)
     return nullptr;
-  return Actions.ActOnOpenMPSingleExprClause(Kind, Val.get(), Loc, LLoc, RLoc);
+  return Actions.OpenMP().ActOnOpenMPSingleExprClause(Kind, Val.get(), Loc,
+                                                      LLoc, RLoc);
 }
 
 /// Parse indirect clause for '#pragma omp declare target' directive.
 ///  'indirect' '[' '(' invoked-by-fptr ')' ']'
 /// where invoked-by-fptr is a constant boolean expression that evaluates to
 /// true or false at compile time.
-bool Parser::ParseOpenMPIndirectClause(Sema::DeclareTargetContextInfo &DTCI,
-                                       bool ParseOnly) {
+bool Parser::ParseOpenMPIndirectClause(
+    SemaOpenMP::DeclareTargetContextInfo &DTCI, bool ParseOnly) {
   SourceLocation Loc = ConsumeToken();
   SourceLocation RLoc;
 
@@ -3721,15 +3737,16 @@ OMPClause *Parser::ParseOpenMPInteropClause(OpenMPClauseKind Kind,
     return nullptr;
 
   if (Kind == OMPC_init)
-    return Actions.ActOnOpenMPInitClause(InteropVarExpr.get(), InteropInfo, Loc,
-                                         T.getOpenLocation(), VarLoc, RLoc);
+    return Actions.OpenMP().ActOnOpenMPInitClause(
+        InteropVarExpr.get(), InteropInfo, Loc, T.getOpenLocation(), VarLoc,
+        RLoc);
   if (Kind == OMPC_use)
-    return Actions.ActOnOpenMPUseClause(InteropVarExpr.get(), Loc,
-                                        T.getOpenLocation(), VarLoc, RLoc);
+    return Actions.OpenMP().ActOnOpenMPUseClause(
+        InteropVarExpr.get(), Loc, T.getOpenLocation(), VarLoc, RLoc);
 
   if (Kind == OMPC_destroy)
-    return Actions.ActOnOpenMPDestroyClause(InteropVarExpr.get(), Loc,
-                                            T.getOpenLocation(), VarLoc, RLoc);
+    return Actions.OpenMP().ActOnOpenMPDestroyClause(
+        InteropVarExpr.get(), Loc, T.getOpenLocation(), VarLoc, RLoc);
 
   llvm_unreachable("Unexpected interop variable clause.");
 }
@@ -3787,8 +3804,8 @@ OMPClause *Parser::ParseOpenMPOMPXAttributesClause(bool ParseOnly) {
     };
   }
 
-  return Actions.ActOnOpenMPXAttributeClause(Attrs, Loc, T.getOpenLocation(),
-                                             T.getCloseLocation());
+  return Actions.OpenMP().ActOnOpenMPXAttributeClause(
+      Attrs, Loc, T.getOpenLocation(), T.getCloseLocation());
 }
 
 /// Parsing of simple OpenMP clauses like 'default' or 'proc_bind'.
@@ -3823,9 +3840,8 @@ OMPClause *Parser::ParseOpenMPSimpleClause(OpenMPClauseKind Kind,
         << getOpenMPClauseName(OMPC_default) << "5.1";
     return nullptr;
   }
-  return Actions.ActOnOpenMPSimpleClause(Kind, Val->Type,
-                                         Val->TypeLoc, Val->LOpen,
-                                         Val->Loc, Val->RLoc);
+  return Actions.OpenMP().ActOnOpenMPSimpleClause(
+      Kind, Val->Type, Val->TypeLoc, Val->LOpen, Val->Loc, Val->RLoc);
 }
 
 /// Parsing of OpenMP clauses like 'ordered'.
@@ -3860,7 +3876,7 @@ OMPClause *Parser::ParseOpenMPClause(OpenMPClauseKind Kind, bool ParseOnly) {
 
   if (ParseOnly)
     return nullptr;
-  return Actions.ActOnOpenMPClause(Kind, Loc, Tok.getLocation());
+  return Actions.OpenMP().ActOnOpenMPClause(Kind, Loc, Tok.getLocation());
 }
 
 /// Parsing of OpenMP clauses with single expressions and some additional
@@ -4118,7 +4134,7 @@ OMPClause *Parser::ParseOpenMPSingleExprWithArgClause(OpenMPDirectiveKind DKind,
 
   if (ParseOnly)
     return nullptr;
-  return Actions.ActOnOpenMPSingleExprWithArgClause(
+  return Actions.OpenMP().ActOnOpenMPSingleExprWithArgClause(
       Kind, Arg, Val.get(), Loc, T.getOpenLocation(), KLoc, DelimLoc, RLoc);
 }
 
@@ -4184,7 +4200,7 @@ static OpenMPMapModifierKind isMapModifier(Parser &P) {
 }
 
 /// Parse the mapper modifier in map, to, and from clauses.
-bool Parser::parseMapperModifier(Sema::OpenMPVarListDataTy &Data) {
+bool Parser::parseMapperModifier(SemaOpenMP::OpenMPVarListDataTy &Data) {
   // Parse '('.
   BalancedDelimiterTracker T(*this, tok::l_paren, tok::colon);
   if (T.expectAndConsume(diag::err_expected_lparen_after, "mapper")) {
@@ -4216,7 +4232,7 @@ bool Parser::parseMapperModifier(Sema::OpenMPVarListDataTy &Data) {
 /// map([ [map-type-modifier[,] [map-type-modifier[,] ...] map-type : ] list)
 /// where, map-type-modifier ::= always | close | mapper(mapper-identifier) |
 /// present
-bool Parser::parseMapTypeModifiers(Sema::OpenMPVarListDataTy &Data) {
+bool Parser::parseMapTypeModifiers(SemaOpenMP::OpenMPVarListDataTy &Data) {
   while (getCurToken().isNot(tok::colon)) {
     OpenMPMapModifierKind TypeModifier = isMapModifier(*this);
     if (TypeModifier == OMPC_MAP_MODIFIER_always ||
@@ -4282,7 +4298,7 @@ static OpenMPMapClauseKind isMapType(Parser &P) {
 /// Parse map-type in map clause.
 /// map([ [map-type-modifier[,] [map-type-modifier[,] ...] map-type : ] list)
 /// where, map-type ::= to | from | tofrom | alloc | release | delete
-static void parseMapType(Parser &P, Sema::OpenMPVarListDataTy &Data) {
+static void parseMapType(Parser &P, SemaOpenMP::OpenMPVarListDataTy &Data) {
   Token Tok = P.getCurToken();
   if (Tok.is(tok::colon)) {
     P.Diag(Tok, diag::err_omp_map_type_missing);
@@ -4306,7 +4322,7 @@ ExprResult Parser::ParseOpenMPIteratorsExpr() {
     return ExprError();
 
   SourceLocation LLoc = T.getOpenLocation();
-  SmallVector<Sema::OMPIteratorData, 4> Data;
+  SmallVector<SemaOpenMP::OMPIteratorData, 4> Data;
   while (Tok.isNot(tok::r_paren) && Tok.isNot(tok::annot_pragma_openmp_end)) {
     // Check if the type parsing is required.
     ParsedType IteratorType;
@@ -4380,7 +4396,7 @@ ExprResult Parser::ParseOpenMPIteratorsExpr() {
     if (Tok.is(tok::comma))
       ConsumeToken();
 
-    Sema::OMPIteratorData &D = Data.emplace_back();
+    SemaOpenMP::OMPIteratorData &D = Data.emplace_back();
     D.DeclIdent = II;
     D.DeclIdentLoc = IdLoc;
     D.Type = IteratorType;
@@ -4397,12 +4413,12 @@ ExprResult Parser::ParseOpenMPIteratorsExpr() {
   if (!T.consumeClose())
     RLoc = T.getCloseLocation();
 
-  return Actions.ActOnOMPIteratorExpr(getCurScope(), IteratorKwLoc, LLoc, RLoc,
-                                      Data);
+  return Actions.OpenMP().ActOnOMPIteratorExpr(getCurScope(), IteratorKwLoc,
+                                               LLoc, RLoc, Data);
 }
 
 bool Parser::ParseOpenMPReservedLocator(OpenMPClauseKind Kind,
-                                        Sema::OpenMPVarListDataTy &Data,
+                                        SemaOpenMP::OpenMPVarListDataTy &Data,
                                         const LangOptions &LangOpts) {
   // Currently the only reserved locator is 'omp_all_memory' which is only
   // allowed on a depend clause.
@@ -4430,7 +4446,7 @@ bool Parser::ParseOpenMPReservedLocator(OpenMPClauseKind Kind,
 
 /// Parse step size expression. Returns true if parsing is successfull,
 /// otherwise returns false.
-static bool parseStepSize(Parser &P, Sema::OpenMPVarListDataTy &Data,
+static bool parseStepSize(Parser &P, SemaOpenMP::OpenMPVarListDataTy &Data,
                           OpenMPClauseKind CKind, SourceLocation ELoc) {
   ExprResult Tail = P.ParseAssignmentExpression();
   Sema &Actions = P.getActions();
@@ -4451,7 +4467,7 @@ static bool parseStepSize(Parser &P, Sema::OpenMPVarListDataTy &Data,
 bool Parser::ParseOpenMPVarList(OpenMPDirectiveKind DKind,
                                 OpenMPClauseKind Kind,
                                 SmallVectorImpl<Expr *> &Vars,
-                                Sema::OpenMPVarListDataTy &Data) {
+                                SemaOpenMP::OpenMPVarListDataTy &Data) {
   UnqualifiedId UnqualifiedReductionId;
   bool InvalidReductionId = false;
   bool IsInvalidMapperModifier = false;
@@ -4961,7 +4977,7 @@ OMPClause *Parser::ParseOpenMPVarListClause(OpenMPDirectiveKind DKind,
   SourceLocation Loc = Tok.getLocation();
   SourceLocation LOpen = ConsumeToken();
   SmallVector<Expr *, 4> Vars;
-  Sema::OpenMPVarListDataTy Data;
+  SemaOpenMP::OpenMPVarListDataTy Data;
 
   if (ParseOpenMPVarList(DKind, Kind, Vars, Data))
     return nullptr;
@@ -4969,5 +4985,5 @@ OMPClause *Parser::ParseOpenMPVarListClause(OpenMPDirectiveKind DKind,
   if (ParseOnly)
     return nullptr;
   OMPVarListLocTy Locs(Loc, LOpen, Data.RLoc);
-  return Actions.ActOnOpenMPVarListClause(Kind, Vars, Locs, Data);
+  return Actions.OpenMP().ActOnOpenMPVarListClause(Kind, Vars, Locs, Data);
 }
diff --git a/clang/lib/Parse/ParseStmt.cpp b/clang/lib/Parse/ParseStmt.cpp
index 76a3fa8f2627..629421c01d17 100644
--- a/clang/lib/Parse/ParseStmt.cpp
+++ b/clang/lib/Parse/ParseStmt.cpp
@@ -22,6 +22,7 @@
 #include "clang/Sema/DeclSpec.h"
 #include "clang/Sema/EnterExpressionEvaluationContext.h"
 #include "clang/Sema/Scope.h"
+#include "clang/Sema/SemaOpenMP.h"
 #include "clang/Sema/TypoCorrection.h"
 #include "llvm/ADT/STLExtras.h"
 #include <optional>
@@ -2301,7 +2302,7 @@ StmtResult Parser::ParseForStatement(SourceLocation *TrailingElseLoc) {
     // In OpenMP loop region loop control variable must be captured and be
     // private. Perform analysis of first part (if any).
     if (getLangOpts().OpenMP && FirstPart.isUsable()) {
-      Actions.ActOnOpenMPLoopInitialization(ForLoc, FirstPart.get());
+      Actions.OpenMP().ActOnOpenMPLoopInitialization(ForLoc, FirstPart.get());
     }
   }
 
diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp
index 8de202f4f7a0..a1e32d391ed0 100644
--- a/clang/lib/Sema/Sema.cpp
+++ b/clang/lib/Sema/Sema.cpp
@@ -46,6 +46,7 @@
 #include "clang/Sema/SemaHLSL.h"
 #include "clang/Sema/SemaInternal.h"
 #include "clang/Sema/SemaOpenACC.h"
+#include "clang/Sema/SemaOpenMP.h"
 #include "clang/Sema/SemaSYCL.h"
 #include "clang/Sema/TemplateDeduction.h"
 #include "clang/Sema/TemplateInstCallback.h"
@@ -203,6 +204,7 @@ Sema::Sema(Preprocessor &pp, ASTContext &ctxt, ASTConsumer &consumer,
       CUDAPtr(std::make_unique<SemaCUDA>(*this)),
       HLSLPtr(std::make_unique<SemaHLSL>(*this)),
       OpenACCPtr(std::make_unique<SemaOpenACC>(*this)),
+      OpenMPPtr(std::make_unique<SemaOpenMP>(*this)),
       SYCLPtr(std::make_unique<SemaSYCL>(*this)),
       MSPointerToMemberRepresentationMethod(
           LangOpts.getMSPointerToMemberRepresentationMethod()),
@@ -226,8 +228,7 @@ Sema::Sema(Preprocessor &pp, ASTContext &ctxt, ASTConsumer &consumer,
       StringWithUTF8StringMethod(nullptr),
       ValueWithBytesObjCTypeMethod(nullptr), NSArrayDecl(nullptr),
       ArrayWithObjectsMethod(nullptr), NSDictionaryDecl(nullptr),
-      DictionaryWithObjectsMethod(nullptr), CodeCompleter(CodeCompleter),
-      VarDataSharingAttributesStack(nullptr) {
+      DictionaryWithObjectsMethod(nullptr), CodeCompleter(CodeCompleter) {
   assert(pp.TUKind == TUKind);
   TUScope = nullptr;
 
@@ -252,7 +253,7 @@ Sema::Sema(Preprocessor &pp, ASTContext &ctxt, ASTConsumer &consumer,
       nullptr, ExpressionEvaluationContextRecord::EK_Other);
 
   // Initialization of data sharing attributes stack for OpenMP
-  InitDataSharingAttributesStack();
+  OpenMP().InitDataSharingAttributesStack();
 
   std::unique_ptr<sema::SemaPPCallbacks> Callbacks =
       std::make_unique<sema::SemaPPCallbacks>();
@@ -501,7 +502,7 @@ Sema::~Sema() {
   threadSafety::threadSafetyCleanup(ThreadSafetyDeclCache);
 
   // Destroys data sharing attributes stack for OpenMP
-  DestroyDataSharingAttributesStack();
+  OpenMP().DestroyDataSharingAttributesStack();
 
   // Detach from the PP callback handler which outlives Sema since it's owned
   // by the preprocessor.
@@ -1159,7 +1160,7 @@ void Sema::ActOnEndOfTranslationUnit() {
 
   DiagnoseUnterminatedPragmaAlignPack();
   DiagnoseUnterminatedPragmaAttribute();
-  DiagnoseUnterminatedOpenMPDeclareTarget();
+  OpenMP().DiagnoseUnterminatedOpenMPDeclareTarget();
 
   // All delayed member exception specs should be checked or we end up accepting
   // incompatible declarations.
@@ -1747,7 +1748,7 @@ public:
     // Finalize analysis of OpenMP-specific constructs.
     if (Caller && S.LangOpts.OpenMP && UsePath.size() == 1 &&
         (ShouldEmitRootNode || InOMPDeviceContext))
-      S.finalizeOpenMPDelayedAnalysis(Caller, FD, Loc);
+      S.OpenMP().finalizeOpenMPDelayedAnalysis(Caller, FD, Loc);
     if (Caller)
       S.CUDA().DeviceKnownEmittedFns[FD] = {Caller, Loc};
     // Always emit deferred diagnostics for the direct users. This does not
@@ -1899,8 +1900,8 @@ Sema::targetDiag(SourceLocation Loc, unsigned DiagID, const FunctionDecl *FD) {
   FD = FD ? FD : getCurFunctionDecl();
   if (LangOpts.OpenMP)
     return LangOpts.OpenMPIsTargetDevice
-               ? diagIfOpenMPDeviceCode(Loc, DiagID, FD)
-               : diagIfOpenMPHostCode(Loc, DiagID, FD);
+               ? OpenMP().diagIfOpenMPDeviceCode(Loc, DiagID, FD)
+               : OpenMP().diagIfOpenMPHostCode(Loc, DiagID, FD);
   if (getLangOpts().CUDA)
     return getLangOpts().CUDAIsDevice ? CUDA().DiagIfDeviceCode(Loc, DiagID)
                                       : CUDA().DiagIfHostCode(Loc, DiagID);
@@ -2131,7 +2132,7 @@ void Sema::PushFunctionScope() {
     FunctionScopes.push_back(new FunctionScopeInfo(getDiagnostics()));
   }
   if (LangOpts.OpenMP)
-    pushOpenMPFunctionRegion();
+    OpenMP().pushOpenMPFunctionRegion();
 }
 
 void Sema::PushBlockScope(Scope *BlockScope, BlockDecl *Block) {
@@ -2251,7 +2252,7 @@ Sema::PopFunctionScopeInfo(const AnalysisBasedWarnings::Policy *WP,
                                PoppedFunctionScopeDeleter(this));
 
   if (LangOpts.OpenMP)
-    popOpenMPFunctionRegion(Scope.get());
+    OpenMP().popOpenMPFunctionRegion(Scope.get());
 
   // Issue any analysis-based warnings.
   if (WP && D)
@@ -2687,7 +2688,9 @@ void Sema::PushCapturedRegionScope(Scope *S, CapturedDecl *CD, RecordDecl *RD,
                                    unsigned OpenMPCaptureLevel) {
   auto *CSI = new CapturedRegionScopeInfo(
       getDiagnostics(), S, CD, RD, CD->getContextParam(), K,
-      (getLangOpts().OpenMP && K == CR_OpenMP) ? getOpenMPNestingLevel() : 0,
+      (getLangOpts().OpenMP && K == CR_OpenMP)
+          ? OpenMP().getOpenMPNestingLevel()
+          : 0,
       OpenMPCaptureLevel);
   CSI->ReturnType = Context.VoidTy;
   FunctionScopes.push_back(CSI);
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 8b3b9d020db5..390da508518e 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -48,6 +48,7 @@
 #include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaHLSL.h"
 #include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/SemaOpenMP.h"
 #include "clang/Sema/Template.h"
 #include "llvm/ADT/STLForwardCompat.h"
 #include "llvm/ADT/SmallString.h"
@@ -6168,11 +6169,12 @@ Decl *Sema::ActOnDeclarator(Scope *S, Declarator &D) {
   // Check if we are in an `omp begin/end declare variant` scope. Handle this
   // declaration only if the `bind_to_declaration` extension is set.
   SmallVector<FunctionDecl *, 4> Bases;
-  if (LangOpts.OpenMP && isInOpenMPDeclareVariantScope())
-    if (getOMPTraitInfoForSurroundingScope()->isExtensionActive(llvm::omp::TraitProperty::
-              implementation_extension_bind_to_declaration))
-    ActOnStartOfFunctionDefinitionInOpenMPDeclareVariantScope(
-        S, D, MultiTemplateParamsArg(), Bases);
+  if (LangOpts.OpenMP && OpenMP().isInOpenMPDeclareVariantScope())
+    if (OpenMP().getOMPTraitInfoForSurroundingScope()->isExtensionActive(
+            llvm::omp::TraitProperty::
+                implementation_extension_bind_to_declaration))
+      OpenMP().ActOnStartOfFunctionDefinitionInOpenMPDeclareVariantScope(
+          S, D, MultiTemplateParamsArg(), Bases);
 
   Decl *Dcl = HandleDeclarator(S, D, MultiTemplateParamsArg());
 
@@ -6181,7 +6183,8 @@ Decl *Sema::ActOnDeclarator(Scope *S, Declarator &D) {
     Dcl->setTopLevelDeclInObjCContainer();
 
   if (!Bases.empty())
-    ActOnFinishedFunctionDefinitionInOpenMPDeclareVariantScope(Dcl, Bases);
+    OpenMP().ActOnFinishedFunctionDefinitionInOpenMPDeclareVariantScope(Dcl,
+                                                                        Bases);
 
   return Dcl;
 }
@@ -6568,8 +6571,8 @@ NamedDecl *Sema::HandleDeclarator(Scope *S, Declarator &D,
   if (New->getDeclName() && AddToScope)
     PushOnScopeChains(New, S);
 
-  if (isInOpenMPDeclareTargetContext())
-    checkDeclIsAllowedInOpenMPTarget(nullptr, New);
+  if (OpenMP().isInOpenMPDeclareTargetContext())
+    OpenMP().checkDeclIsAllowedInOpenMPTarget(nullptr, New);
 
   return New;
 }
@@ -12268,7 +12271,7 @@ bool Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD,
   }
 
   if (LangOpts.OpenMP)
-    ActOnFinishedFunctionDefinitionInOpenMPAssumeScope(NewFD);
+    OpenMP().ActOnFinishedFunctionDefinitionInOpenMPAssumeScope(NewFD);
 
   // Semantic checking for this function declaration (in isolation).
 
@@ -14956,7 +14959,7 @@ Sema::DeclGroupPtrTy Sema::FinalizeDeclaratorGroup(Scope *S, const DeclSpec &DS,
       if (auto *VD = dyn_cast<VarDecl>(D);
           LangOpts.OpenMP && VD && VD->hasAttr<OMPDeclareTargetDeclAttr>() &&
           VD->hasGlobalStorage())
-        ActOnOpenMPDeclareTargetInitializer(D);
+        OpenMP().ActOnOpenMPDeclareTargetInitializer(D);
       // For declarators, there are some additional syntactic-ish checks we need
       // to perform.
       if (auto *DD = dyn_cast<DeclaratorDecl>(D)) {
@@ -15495,8 +15498,8 @@ Sema::ActOnStartOfFunctionDef(Scope *FnBodyScope, Declarator &D,
   // specialization function under the OpenMP context defined as part of the
   // `omp begin declare variant`.
   SmallVector<FunctionDecl *, 4> Bases;
-  if (LangOpts.OpenMP && isInOpenMPDeclareVariantScope())
-    ActOnStartOfFunctionDefinitionInOpenMPDeclareVariantScope(
+  if (LangOpts.OpenMP && OpenMP().isInOpenMPDeclareVariantScope())
+    OpenMP().ActOnStartOfFunctionDefinitionInOpenMPDeclareVariantScope(
         ParentScope, D, TemplateParameterLists, Bases);
 
   D.setFunctionDefinitionKind(FunctionDefinitionKind::Definition);
@@ -15504,7 +15507,8 @@ Sema::ActOnStartOfFunctionDef(Scope *FnBodyScope, Declarator &D,
   Decl *Dcl = ActOnStartOfFunctionDef(FnBodyScope, DP, SkipBody, BodyKind);
 
   if (!Bases.empty())
-    ActOnFinishedFunctionDefinitionInOpenMPDeclareVariantScope(Dcl, Bases);
+    OpenMP().ActOnFinishedFunctionDefinitionInOpenMPDeclareVariantScope(Dcl,
+                                                                        Bases);
 
   return Dcl;
 }
@@ -20651,7 +20655,7 @@ Sema::FunctionEmissionStatus Sema::getEmissionStatus(const FunctionDecl *FD,
         return FunctionEmissionStatus::OMPDiscarded;
     // If we have an explicit value for the device type, or we are in a target
     // declare context, we need to emit all extern and used symbols.
-    if (isInOpenMPDeclareTargetContext() || DevTy)
+    if (OpenMP().isInOpenMPDeclareTargetContext() || DevTy)
       if (IsEmittedForExternalSymbol())
         return FunctionEmissionStatus::Emitted;
     // Device mode only emits what it must, if it wasn't tagged yet and needed,
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index 7669171fea56..8c6bae545bfd 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -44,6 +44,7 @@
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/SemaOpenMP.h"
 #include "clang/Sema/Template.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
@@ -962,8 +963,8 @@ Sema::ActOnDecompositionDeclarator(Scope *S, Declarator &D,
     CurContext->addHiddenDecl(New);
   }
 
-  if (isInOpenMPDeclareTargetContext())
-    checkDeclIsAllowedInOpenMPTarget(nullptr, New);
+  if (OpenMP().isInOpenMPDeclareTargetContext())
+    OpenMP().checkDeclIsAllowedInOpenMPTarget(nullptr, New);
 
   return New;
 }
@@ -18654,8 +18655,8 @@ void Sema::MarkVTableUsed(SourceLocation Loc, CXXRecordDecl *Class,
   // Do not mark as used if compiling for the device outside of the target
   // region.
   if (TUKind != TU_Prefix && LangOpts.OpenMP && LangOpts.OpenMPIsTargetDevice &&
-      !isInOpenMPDeclareTargetContext() &&
-      !isInOpenMPTargetExecutionDirective()) {
+      !OpenMP().isInOpenMPDeclareTargetContext() &&
+      !OpenMP().isInOpenMPTargetExecutionDirective()) {
     if (!DefinitionRequired)
       MarkVirtualMembersReferenced(Loc, Class);
     return;
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 189764cb4b6b..cabffa47c931 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -52,6 +52,7 @@
 #include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaFixItUtils.h"
 #include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/SemaOpenMP.h"
 #include "clang/Sema/Template.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/STLForwardCompat.h"
@@ -360,9 +361,9 @@ bool Sema::DiagnoseUseOfDecl(NamedDecl *D, ArrayRef<SourceLocation> Locs,
   //  at the same location.
   // [OpenMP 5.2] Also allow iterator declared variables.
   if (LangOpts.OpenMP && isa<VarDecl>(D) &&
-      !isOpenMPDeclareMapperVarDeclAllowed(cast<VarDecl>(D))) {
+      !OpenMP().isOpenMPDeclareMapperVarDeclAllowed(cast<VarDecl>(D))) {
     Diag(Loc, diag::err_omp_declare_mapper_wrong_var)
-        << getOpenMPDeclareMapperVarName();
+        << OpenMP().getOpenMPDeclareMapperVarName();
     Diag(D->getLocation(), diag::note_entity_declared_at) << D;
     return true;
   }
@@ -2267,7 +2268,7 @@ NonOdrUseReason Sema::getNonOdrUseReasonInCurrentContext(ValueDecl *D) {
   //   be loaded from the captured.
   if (VarDecl *VD = dyn_cast<VarDecl>(D)) {
     if (VD->getType()->isReferenceType() &&
-        !(getLangOpts().OpenMP && isOpenMPCapturedDecl(D)) &&
+        !(getLangOpts().OpenMP && OpenMP().isOpenMPCapturedDecl(D)) &&
         !isCapturingReferenceToHostVarInCUDADeviceLambda(*this, VD) &&
         VD->isUsableInConstantExpressions(Context))
       return NOUR_Constant;
@@ -5080,9 +5081,10 @@ ExprResult Sema::ActOnArraySubscriptExpr(Scope *S, Expr *base,
 
   if (base && !base->getType().isNull() &&
       base->hasPlaceholderType(BuiltinType::OMPArraySection))
-    return ActOnOMPArraySectionExpr(base, lbLoc, ArgExprs.front(), SourceLocation(),
-                                    SourceLocation(), /*Length*/ nullptr,
-                                    /*Stride=*/nullptr, rbLoc);
+    return OpenMP().ActOnOMPArraySectionExpr(base, lbLoc, ArgExprs.front(),
+                                             SourceLocation(), SourceLocation(),
+                                             /*Length*/ nullptr,
+                                             /*Stride=*/nullptr, rbLoc);
 
   // Since this might be a postfix expression, get rid of ParenListExprs.
   if (isa<ParenListExpr>(base)) {
@@ -5354,558 +5356,6 @@ void Sema::CheckSubscriptAccessOfNoDeref(const ArraySubscriptExpr *E) {
   }
 }
 
-ExprResult Sema::ActOnOMPArraySectionExpr(Expr *Base, SourceLocation LBLoc,
-                                          Expr *LowerBound,
-                                          SourceLocation ColonLocFirst,
-                                          SourceLocation ColonLocSecond,
-                                          Expr *Length, Expr *Stride,
-                                          SourceLocation RBLoc) {
-  if (Base->hasPlaceholderType() &&
-      !Base->hasPlaceholderType(BuiltinType::OMPArraySection)) {
-    ExprResult Result = CheckPlaceholderExpr(Base);
-    if (Result.isInvalid())
-      return ExprError();
-    Base = Result.get();
-  }
-  if (LowerBound && LowerBound->getType()->isNonOverloadPlaceholderType()) {
-    ExprResult Result = CheckPlaceholderExpr(LowerBound);
-    if (Result.isInvalid())
-      return ExprError();
-    Result = DefaultLvalueConversion(Result.get());
-    if (Result.isInvalid())
-      return ExprError();
-    LowerBound = Result.get();
-  }
-  if (Length && Length->getType()->isNonOverloadPlaceholderType()) {
-    ExprResult Result = CheckPlaceholderExpr(Length);
-    if (Result.isInvalid())
-      return ExprError();
-    Result = DefaultLvalueConversion(Result.get());
-    if (Result.isInvalid())
-      return ExprError();
-    Length = Result.get();
-  }
-  if (Stride && Stride->getType()->isNonOverloadPlaceholderType()) {
-    ExprResult Result = CheckPlaceholderExpr(Stride);
-    if (Result.isInvalid())
-      return ExprError();
-    Result = DefaultLvalueConversion(Result.get());
-    if (Result.isInvalid())
-      return ExprError();
-    Stride = Result.get();
-  }
-
-  // Build an unanalyzed expression if either operand is type-dependent.
-  if (Base->isTypeDependent() ||
-      (LowerBound &&
-       (LowerBound->isTypeDependent() || LowerBound->isValueDependent())) ||
-      (Length && (Length->isTypeDependent() || Length->isValueDependent())) ||
-      (Stride && (Stride->isTypeDependent() || Stride->isValueDependent()))) {
-    return new (Context) OMPArraySectionExpr(
-        Base, LowerBound, Length, Stride, Context.DependentTy, VK_LValue,
-        OK_Ordinary, ColonLocFirst, ColonLocSecond, RBLoc);
-  }
-
-  // Perform default conversions.
-  QualType OriginalTy = OMPArraySectionExpr::getBaseOriginalType(Base);
-  QualType ResultTy;
-  if (OriginalTy->isAnyPointerType()) {
-    ResultTy = OriginalTy->getPointeeType();
-  } else if (OriginalTy->isArrayType()) {
-    ResultTy = OriginalTy->getAsArrayTypeUnsafe()->getElementType();
-  } else {
-    return ExprError(
-        Diag(Base->getExprLoc(), diag::err_omp_typecheck_section_value)
-        << Base->getSourceRange());
-  }
-  // C99 6.5.2.1p1
-  if (LowerBound) {
-    auto Res = PerformOpenMPImplicitIntegerConversion(LowerBound->getExprLoc(),
-                                                      LowerBound);
-    if (Res.isInvalid())
-      return ExprError(Diag(LowerBound->getExprLoc(),
-                            diag::err_omp_typecheck_section_not_integer)
-                       << 0 << LowerBound->getSourceRange());
-    LowerBound = Res.get();
-
-    if (LowerBound->getType()->isSpecificBuiltinType(BuiltinType::Char_S) ||
-        LowerBound->getType()->isSpecificBuiltinType(BuiltinType::Char_U))
-      Diag(LowerBound->getExprLoc(), diag::warn_omp_section_is_char)
-          << 0 << LowerBound->getSourceRange();
-  }
-  if (Length) {
-    auto Res =
-        PerformOpenMPImplicitIntegerConversion(Length->getExprLoc(), Length);
-    if (Res.isInvalid())
-      return ExprError(Diag(Length->getExprLoc(),
-                            diag::err_omp_typecheck_section_not_integer)
-                       << 1 << Length->getSourceRange());
-    Length = Res.get();
-
-    if (Length->getType()->isSpecificBuiltinType(BuiltinType::Char_S) ||
-        Length->getType()->isSpecificBuiltinType(BuiltinType::Char_U))
-      Diag(Length->getExprLoc(), diag::warn_omp_section_is_char)
-          << 1 << Length->getSourceRange();
-  }
-  if (Stride) {
-    ExprResult Res =
-        PerformOpenMPImplicitIntegerConversion(Stride->getExprLoc(), Stride);
-    if (Res.isInvalid())
-      return ExprError(Diag(Stride->getExprLoc(),
-                            diag::err_omp_typecheck_section_not_integer)
-                       << 1 << Stride->getSourceRange());
-    Stride = Res.get();
-
-    if (Stride->getType()->isSpecificBuiltinType(BuiltinType::Char_S) ||
-        Stride->getType()->isSpecificBuiltinType(BuiltinType::Char_U))
-      Diag(Stride->getExprLoc(), diag::warn_omp_section_is_char)
-          << 1 << Stride->getSourceRange();
-  }
-
-  // C99 6.5.2.1p1: "shall have type "pointer to *object* type". Similarly,
-  // C++ [expr.sub]p1: The type "T" shall be a completely-defined object
-  // type. Note that functions are not objects, and that (in C99 parlance)
-  // incomplete types are not object types.
-  if (ResultTy->isFunctionType()) {
-    Diag(Base->getExprLoc(), diag::err_omp_section_function_type)
-        << ResultTy << Base->getSourceRange();
-    return ExprError();
-  }
-
-  if (RequireCompleteType(Base->getExprLoc(), ResultTy,
-                          diag::err_omp_section_incomplete_type, Base))
-    return ExprError();
-
-  if (LowerBound && !OriginalTy->isAnyPointerType()) {
-    Expr::EvalResult Result;
-    if (LowerBound->EvaluateAsInt(Result, Context)) {
-      // OpenMP 5.0, [2.1.5 Array Sections]
-      // The array section must be a subset of the original array.
-      llvm::APSInt LowerBoundValue = Result.Val.getInt();
-      if (LowerBoundValue.isNegative()) {
-        Diag(LowerBound->getExprLoc(), diag::err_omp_section_not_subset_of_array)
-            << LowerBound->getSourceRange();
-        return ExprError();
-      }
-    }
-  }
-
-  if (Length) {
-    Expr::EvalResult Result;
-    if (Length->EvaluateAsInt(Result, Context)) {
-      // OpenMP 5.0, [2.1.5 Array Sections]
-      // The length must evaluate to non-negative integers.
-      llvm::APSInt LengthValue = Result.Val.getInt();
-      if (LengthValue.isNegative()) {
-        Diag(Length->getExprLoc(), diag::err_omp_section_length_negative)
-            << toString(LengthValue, /*Radix=*/10, /*Signed=*/true)
-            << Length->getSourceRange();
-        return ExprError();
-      }
-    }
-  } else if (ColonLocFirst.isValid() &&
-             (OriginalTy.isNull() || (!OriginalTy->isConstantArrayType() &&
-                                      !OriginalTy->isVariableArrayType()))) {
-    // OpenMP 5.0, [2.1.5 Array Sections]
-    // When the size of the array dimension is not known, the length must be
-    // specified explicitly.
-    Diag(ColonLocFirst, diag::err_omp_section_length_undefined)
-        << (!OriginalTy.isNull() && OriginalTy->isArrayType());
-    return ExprError();
-  }
-
-  if (Stride) {
-    Expr::EvalResult Result;
-    if (Stride->EvaluateAsInt(Result, Context)) {
-      // OpenMP 5.0, [2.1.5 Array Sections]
-      // The stride must evaluate to a positive integer.
-      llvm::APSInt StrideValue = Result.Val.getInt();
-      if (!StrideValue.isStrictlyPositive()) {
-        Diag(Stride->getExprLoc(), diag::err_omp_section_stride_non_positive)
-            << toString(StrideValue, /*Radix=*/10, /*Signed=*/true)
-            << Stride->getSourceRange();
-        return ExprError();
-      }
-    }
-  }
-
-  if (!Base->hasPlaceholderType(BuiltinType::OMPArraySection)) {
-    ExprResult Result = DefaultFunctionArrayLvalueConversion(Base);
-    if (Result.isInvalid())
-      return ExprError();
-    Base = Result.get();
-  }
-  return new (Context) OMPArraySectionExpr(
-      Base, LowerBound, Length, Stride, Context.OMPArraySectionTy, VK_LValue,
-      OK_Ordinary, ColonLocFirst, ColonLocSecond, RBLoc);
-}
-
-ExprResult Sema::ActOnOMPArrayShapingExpr(Expr *Base, SourceLocation LParenLoc,
-                                          SourceLocation RParenLoc,
-                                          ArrayRef<Expr *> Dims,
-                                          ArrayRef<SourceRange> Brackets) {
-  if (Base->hasPlaceholderType()) {
-    ExprResult Result = CheckPlaceholderExpr(Base);
-    if (Result.isInvalid())
-      return ExprError();
-    Result = DefaultLvalueConversion(Result.get());
-    if (Result.isInvalid())
-      return ExprError();
-    Base = Result.get();
-  }
-  QualType BaseTy = Base->getType();
-  // Delay analysis of the types/expressions if instantiation/specialization is
-  // required.
-  if (!BaseTy->isPointerType() && Base->isTypeDependent())
-    return OMPArrayShapingExpr::Create(Context, Context.DependentTy, Base,
-                                       LParenLoc, RParenLoc, Dims, Brackets);
-  if (!BaseTy->isPointerType() ||
-      (!Base->isTypeDependent() &&
-       BaseTy->getPointeeType()->isIncompleteType()))
-    return ExprError(Diag(Base->getExprLoc(),
-                          diag::err_omp_non_pointer_type_array_shaping_base)
-                     << Base->getSourceRange());
-
-  SmallVector<Expr *, 4> NewDims;
-  bool ErrorFound = false;
-  for (Expr *Dim : Dims) {
-    if (Dim->hasPlaceholderType()) {
-      ExprResult Result = CheckPlaceholderExpr(Dim);
-      if (Result.isInvalid()) {
-        ErrorFound = true;
-        continue;
-      }
-      Result = DefaultLvalueConversion(Result.get());
-      if (Result.isInvalid()) {
-        ErrorFound = true;
-        continue;
-      }
-      Dim = Result.get();
-    }
-    if (!Dim->isTypeDependent()) {
-      ExprResult Result =
-          PerformOpenMPImplicitIntegerConversion(Dim->getExprLoc(), Dim);
-      if (Result.isInvalid()) {
-        ErrorFound = true;
-        Diag(Dim->getExprLoc(), diag::err_omp_typecheck_shaping_not_integer)
-            << Dim->getSourceRange();
-        continue;
-      }
-      Dim = Result.get();
-      Expr::EvalResult EvResult;
-      if (!Dim->isValueDependent() && Dim->EvaluateAsInt(EvResult, Context)) {
-        // OpenMP 5.0, [2.1.4 Array Shaping]
-        // Each si is an integral type expression that must evaluate to a
-        // positive integer.
-        llvm::APSInt Value = EvResult.Val.getInt();
-        if (!Value.isStrictlyPositive()) {
-          Diag(Dim->getExprLoc(), diag::err_omp_shaping_dimension_not_positive)
-              << toString(Value, /*Radix=*/10, /*Signed=*/true)
-              << Dim->getSourceRange();
-          ErrorFound = true;
-          continue;
-        }
-      }
-    }
-    NewDims.push_back(Dim);
-  }
-  if (ErrorFound)
-    return ExprError();
-  return OMPArrayShapingExpr::Create(Context, Context.OMPArrayShapingTy, Base,
-                                     LParenLoc, RParenLoc, NewDims, Brackets);
-}
-
-ExprResult Sema::ActOnOMPIteratorExpr(Scope *S, SourceLocation IteratorKwLoc,
-                                      SourceLocation LLoc, SourceLocation RLoc,
-                                      ArrayRef<OMPIteratorData> Data) {
-  SmallVector<OMPIteratorExpr::IteratorDefinition, 4> ID;
-  bool IsCorrect = true;
-  for (const OMPIteratorData &D : Data) {
-    TypeSourceInfo *TInfo = nullptr;
-    SourceLocation StartLoc;
-    QualType DeclTy;
-    if (!D.Type.getAsOpaquePtr()) {
-      // OpenMP 5.0, 2.1.6 Iterators
-      // In an iterator-specifier, if the iterator-type is not specified then
-      // the type of that iterator is of int type.
-      DeclTy = Context.IntTy;
-      StartLoc = D.DeclIdentLoc;
-    } else {
-      DeclTy = GetTypeFromParser(D.Type, &TInfo);
-      StartLoc = TInfo->getTypeLoc().getBeginLoc();
-    }
-
-    bool IsDeclTyDependent = DeclTy->isDependentType() ||
-                             DeclTy->containsUnexpandedParameterPack() ||
-                             DeclTy->isInstantiationDependentType();
-    if (!IsDeclTyDependent) {
-      if (!DeclTy->isIntegralType(Context) && !DeclTy->isAnyPointerType()) {
-        // OpenMP 5.0, 2.1.6 Iterators, Restrictions, C/C++
-        // The iterator-type must be an integral or pointer type.
-        Diag(StartLoc, diag::err_omp_iterator_not_integral_or_pointer)
-            << DeclTy;
-        IsCorrect = false;
-        continue;
-      }
-      if (DeclTy.isConstant(Context)) {
-        // OpenMP 5.0, 2.1.6 Iterators, Restrictions, C/C++
-        // The iterator-type must not be const qualified.
-        Diag(StartLoc, diag::err_omp_iterator_not_integral_or_pointer)
-            << DeclTy;
-        IsCorrect = false;
-        continue;
-      }
-    }
-
-    // Iterator declaration.
-    assert(D.DeclIdent && "Identifier expected.");
-    // Always try to create iterator declarator to avoid extra error messages
-    // about unknown declarations use.
-    auto *VD = VarDecl::Create(Context, CurContext, StartLoc, D.DeclIdentLoc,
-                               D.DeclIdent, DeclTy, TInfo, SC_None);
-    VD->setImplicit();
-    if (S) {
-      // Check for conflicting previous declaration.
-      DeclarationNameInfo NameInfo(VD->getDeclName(), D.DeclIdentLoc);
-      LookupResult Previous(*this, NameInfo, LookupOrdinaryName,
-                            ForVisibleRedeclaration);
-      Previous.suppressDiagnostics();
-      LookupName(Previous, S);
-
-      FilterLookupForScope(Previous, CurContext, S, /*ConsiderLinkage=*/false,
-                           /*AllowInlineNamespace=*/false);
-      if (!Previous.empty()) {
-        NamedDecl *Old = Previous.getRepresentativeDecl();
-        Diag(D.DeclIdentLoc, diag::err_redefinition) << VD->getDeclName();
-        Diag(Old->getLocation(), diag::note_previous_definition);
-      } else {
-        PushOnScopeChains(VD, S);
-      }
-    } else {
-      CurContext->addDecl(VD);
-    }
-
-    /// Act on the iterator variable declaration.
-    ActOnOpenMPIteratorVarDecl(VD);
-
-    Expr *Begin = D.Range.Begin;
-    if (!IsDeclTyDependent && Begin && !Begin->isTypeDependent()) {
-      ExprResult BeginRes =
-          PerformImplicitConversion(Begin, DeclTy, AA_Converting);
-      Begin = BeginRes.get();
-    }
-    Expr *End = D.Range.End;
-    if (!IsDeclTyDependent && End && !End->isTypeDependent()) {
-      ExprResult EndRes = PerformImplicitConversion(End, DeclTy, AA_Converting);
-      End = EndRes.get();
-    }
-    Expr *Step = D.Range.Step;
-    if (!IsDeclTyDependent && Step && !Step->isTypeDependent()) {
-      if (!Step->getType()->isIntegralType(Context)) {
-        Diag(Step->getExprLoc(), diag::err_omp_iterator_step_not_integral)
-            << Step << Step->getSourceRange();
-        IsCorrect = false;
-        continue;
-      }
-      std::optional<llvm::APSInt> Result =
-          Step->getIntegerConstantExpr(Context);
-      // OpenMP 5.0, 2.1.6 Iterators, Restrictions
-      // If the step expression of a range-specification equals zero, the
-      // behavior is unspecified.
-      if (Result && Result->isZero()) {
-        Diag(Step->getExprLoc(), diag::err_omp_iterator_step_constant_zero)
-            << Step << Step->getSourceRange();
-        IsCorrect = false;
-        continue;
-      }
-    }
-    if (!Begin || !End || !IsCorrect) {
-      IsCorrect = false;
-      continue;
-    }
-    OMPIteratorExpr::IteratorDefinition &IDElem = ID.emplace_back();
-    IDElem.IteratorDecl = VD;
-    IDElem.AssignmentLoc = D.AssignLoc;
-    IDElem.Range.Begin = Begin;
-    IDElem.Range.End = End;
-    IDElem.Range.Step = Step;
-    IDElem.ColonLoc = D.ColonLoc;
-    IDElem.SecondColonLoc = D.SecColonLoc;
-  }
-  if (!IsCorrect) {
-    // Invalidate all created iterator declarations if error is found.
-    for (const OMPIteratorExpr::IteratorDefinition &D : ID) {
-      if (Decl *ID = D.IteratorDecl)
-        ID->setInvalidDecl();
-    }
-    return ExprError();
-  }
-  SmallVector<OMPIteratorHelperData, 4> Helpers;
-  if (!CurContext->isDependentContext()) {
-    // Build number of ityeration for each iteration range.
-    // Ni = ((Stepi > 0) ? ((Endi + Stepi -1 - Begini)/Stepi) :
-    // ((Begini-Stepi-1-Endi) / -Stepi);
-    for (OMPIteratorExpr::IteratorDefinition &D : ID) {
-      // (Endi - Begini)
-      ExprResult Res = CreateBuiltinBinOp(D.AssignmentLoc, BO_Sub, D.Range.End,
-                                          D.Range.Begin);
-      if(!Res.isUsable()) {
-        IsCorrect = false;
-        continue;
-      }
-      ExprResult St, St1;
-      if (D.Range.Step) {
-        St = D.Range.Step;
-        // (Endi - Begini) + Stepi
-        Res = CreateBuiltinBinOp(D.AssignmentLoc, BO_Add, Res.get(), St.get());
-        if (!Res.isUsable()) {
-          IsCorrect = false;
-          continue;
-        }
-        // (Endi - Begini) + Stepi - 1
-        Res =
-            CreateBuiltinBinOp(D.AssignmentLoc, BO_Sub, Res.get(),
-                               ActOnIntegerConstant(D.AssignmentLoc, 1).get());
-        if (!Res.isUsable()) {
-          IsCorrect = false;
-          continue;
-        }
-        // ((Endi - Begini) + Stepi - 1) / Stepi
-        Res = CreateBuiltinBinOp(D.AssignmentLoc, BO_Div, Res.get(), St.get());
-        if (!Res.isUsable()) {
-          IsCorrect = false;
-          continue;
-        }
-        St1 = CreateBuiltinUnaryOp(D.AssignmentLoc, UO_Minus, D.Range.Step);
-        // (Begini - Endi)
-        ExprResult Res1 = CreateBuiltinBinOp(D.AssignmentLoc, BO_Sub,
-                                             D.Range.Begin, D.Range.End);
-        if (!Res1.isUsable()) {
-          IsCorrect = false;
-          continue;
-        }
-        // (Begini - Endi) - Stepi
-        Res1 =
-            CreateBuiltinBinOp(D.AssignmentLoc, BO_Add, Res1.get(), St1.get());
-        if (!Res1.isUsable()) {
-          IsCorrect = false;
-          continue;
-        }
-        // (Begini - Endi) - Stepi - 1
-        Res1 =
-            CreateBuiltinBinOp(D.AssignmentLoc, BO_Sub, Res1.get(),
-                               ActOnIntegerConstant(D.AssignmentLoc, 1).get());
-        if (!Res1.isUsable()) {
-          IsCorrect = false;
-          continue;
-        }
-        // ((Begini - Endi) - Stepi - 1) / (-Stepi)
-        Res1 =
-            CreateBuiltinBinOp(D.AssignmentLoc, BO_Div, Res1.get(), St1.get());
-        if (!Res1.isUsable()) {
-          IsCorrect = false;
-          continue;
-        }
-        // Stepi > 0.
-        ExprResult CmpRes =
-            CreateBuiltinBinOp(D.AssignmentLoc, BO_GT, D.Range.Step,
-                               ActOnIntegerConstant(D.AssignmentLoc, 0).get());
-        if (!CmpRes.isUsable()) {
-          IsCorrect = false;
-          continue;
-        }
-        Res = ActOnConditionalOp(D.AssignmentLoc, D.AssignmentLoc, CmpRes.get(),
-                                 Res.get(), Res1.get());
-        if (!Res.isUsable()) {
-          IsCorrect = false;
-          continue;
-        }
-      }
-      Res = ActOnFinishFullExpr(Res.get(), /*DiscardedValue=*/false);
-      if (!Res.isUsable()) {
-        IsCorrect = false;
-        continue;
-      }
-
-      // Build counter update.
-      // Build counter.
-      auto *CounterVD =
-          VarDecl::Create(Context, CurContext, D.IteratorDecl->getBeginLoc(),
-                          D.IteratorDecl->getBeginLoc(), nullptr,
-                          Res.get()->getType(), nullptr, SC_None);
-      CounterVD->setImplicit();
-      ExprResult RefRes =
-          BuildDeclRefExpr(CounterVD, CounterVD->getType(), VK_LValue,
-                           D.IteratorDecl->getBeginLoc());
-      // Build counter update.
-      // I = Begini + counter * Stepi;
-      ExprResult UpdateRes;
-      if (D.Range.Step) {
-        UpdateRes = CreateBuiltinBinOp(
-            D.AssignmentLoc, BO_Mul,
-            DefaultLvalueConversion(RefRes.get()).get(), St.get());
-      } else {
-        UpdateRes = DefaultLvalueConversion(RefRes.get());
-      }
-      if (!UpdateRes.isUsable()) {
-        IsCorrect = false;
-        continue;
-      }
-      UpdateRes = CreateBuiltinBinOp(D.AssignmentLoc, BO_Add, D.Range.Begin,
-                                     UpdateRes.get());
-      if (!UpdateRes.isUsable()) {
-        IsCorrect = false;
-        continue;
-      }
-      ExprResult VDRes =
-          BuildDeclRefExpr(cast<VarDecl>(D.IteratorDecl),
-                           cast<VarDecl>(D.IteratorDecl)->getType(), VK_LValue,
-                           D.IteratorDecl->getBeginLoc());
-      UpdateRes = CreateBuiltinBinOp(D.AssignmentLoc, BO_Assign, VDRes.get(),
-                                     UpdateRes.get());
-      if (!UpdateRes.isUsable()) {
-        IsCorrect = false;
-        continue;
-      }
-      UpdateRes =
-          ActOnFinishFullExpr(UpdateRes.get(), /*DiscardedValue=*/true);
-      if (!UpdateRes.isUsable()) {
-        IsCorrect = false;
-        continue;
-      }
-      ExprResult CounterUpdateRes =
-          CreateBuiltinUnaryOp(D.AssignmentLoc, UO_PreInc, RefRes.get());
-      if (!CounterUpdateRes.isUsable()) {
-        IsCorrect = false;
-        continue;
-      }
-      CounterUpdateRes =
-          ActOnFinishFullExpr(CounterUpdateRes.get(), /*DiscardedValue=*/true);
-      if (!CounterUpdateRes.isUsable()) {
-        IsCorrect = false;
-        continue;
-      }
-      OMPIteratorHelperData &HD = Helpers.emplace_back();
-      HD.CounterVD = CounterVD;
-      HD.Upper = Res.get();
-      HD.Update = UpdateRes.get();
-      HD.CounterUpdate = CounterUpdateRes.get();
-    }
-  } else {
-    Helpers.assign(ID.size(), {});
-  }
-  if (!IsCorrect) {
-    // Invalidate all created iterator declarations if error is found.
-    for (const OMPIteratorExpr::IteratorDefinition &D : ID) {
-      if (Decl *ID = D.IteratorDecl)
-        ID->setInvalidDecl();
-    }
-    return ExprError();
-  }
-  return OMPIteratorExpr::Create(Context, Context.OMPIteratorTy, IteratorKwLoc,
-                                 LLoc, RLoc, ID, Helpers);
-}
-
 ExprResult
 Sema::CreateBuiltinArraySubscriptExpr(Expr *Base, SourceLocation LLoc,
                                       Expr *Idx, SourceLocation RLoc) {
@@ -7190,8 +6640,8 @@ ExprResult Sema::ActOnCallExpr(Scope *Scope, Expr *Fn, SourceLocation LParenLoc,
   }
 
   if (LangOpts.OpenMP)
-    Call = ActOnOpenMPCall(Call, Scope, LParenLoc, ArgExprs, RParenLoc,
-                           ExecConfig);
+    Call = OpenMP().ActOnOpenMPCall(Call, Scope, LParenLoc, ArgExprs, RParenLoc,
+                                    ExecConfig);
   if (LangOpts.CPlusPlus) {
     if (const auto *CE = dyn_cast<CallExpr>(Call.get()))
       DiagnosedUnqualifiedCallsToStdFunctions(*this, CE);
@@ -19193,7 +18643,7 @@ MarkVarDeclODRUsed(ValueDecl *V, SourceLocation Loc, Sema &SemaRef,
   }
   QualType CaptureType, DeclRefType;
   if (SemaRef.LangOpts.OpenMP)
-    SemaRef.tryCaptureOpenMPLambdas(V);
+    SemaRef.OpenMP().tryCaptureOpenMPLambdas(V);
   SemaRef.tryCaptureVariable(V, Loc, Sema::TryCapture_Implicit,
                              /*EllipsisLoc*/ SourceLocation(),
                              /*BuildAndDiagnose*/ true, CaptureType,
@@ -19474,7 +18924,7 @@ static bool captureInBlock(BlockScopeInfo *BSI, ValueDecl *Var,
 
   const bool HasBlocksAttr = Var->hasAttr<BlocksAttr>();
   if (HasBlocksAttr || CaptureType->isReferenceType() ||
-      (S.getLangOpts().OpenMP && S.isOpenMPCapturedDecl(Var))) {
+      (S.getLangOpts().OpenMP && S.OpenMP().isOpenMPCapturedDecl(Var))) {
     // Block capture by reference does not change the capture or
     // declaration reference types.
     ByRef = true;
@@ -19504,7 +18954,7 @@ static bool captureInCapturedRegion(
     ByRef = (Kind == Sema::TryCapture_ExplicitByRef);
   } else if (S.getLangOpts().OpenMP && RSI->CapRegionKind == CR_OpenMP) {
     // Using an LValue reference type is consistent with Lambdas (see below).
-    if (S.isOpenMPCapturedDecl(Var)) {
+    if (S.OpenMP().isOpenMPCapturedDecl(Var)) {
       bool HasConst = DeclRefType.isConstQualified();
       DeclRefType = DeclRefType.getUnqualifiedType();
       // Don't lose diagnostics about assignments to const.
@@ -19512,11 +18962,11 @@ static bool captureInCapturedRegion(
         DeclRefType.addConst();
     }
     // Do not capture firstprivates in tasks.
-    if (S.isOpenMPPrivateDecl(Var, RSI->OpenMPLevel, RSI->OpenMPCaptureLevel) !=
-        OMPC_unknown)
+    if (S.OpenMP().isOpenMPPrivateDecl(Var, RSI->OpenMPLevel,
+                                       RSI->OpenMPCaptureLevel) != OMPC_unknown)
       return true;
-    ByRef = S.isOpenMPCapturedByRef(Var, RSI->OpenMPLevel,
-                                    RSI->OpenMPCaptureLevel);
+    ByRef = S.OpenMP().isOpenMPCapturedByRef(Var, RSI->OpenMPLevel,
+                                             RSI->OpenMPCaptureLevel);
   }
 
   if (ByRef)
@@ -19777,9 +19227,9 @@ bool Sema::tryCaptureVariable(
   // Capture global variables if it is required to use private copy of this
   // variable.
   bool IsGlobal = !VD->hasLocalStorage();
-  if (IsGlobal &&
-      !(LangOpts.OpenMP && isOpenMPCapturedDecl(Var, /*CheckScopeInfo=*/true,
-                                                MaxFunctionScopesIndex)))
+  if (IsGlobal && !(LangOpts.OpenMP &&
+                    OpenMP().isOpenMPCapturedDecl(Var, /*CheckScopeInfo=*/true,
+                                                  MaxFunctionScopesIndex)))
     return true;
 
   if (isa<VarDecl>(Var))
@@ -19897,7 +19347,7 @@ bool Sema::tryCaptureVariable(
             }
             return true;
           }
-          OpenMPClauseKind IsOpenMPPrivateDecl = isOpenMPPrivateDecl(
+          OpenMPClauseKind IsOpenMPPrivateDecl = OpenMP().isOpenMPPrivateDecl(
               Var, RSI->OpenMPLevel, RSI->OpenMPCaptureLevel);
           // If the variable is private (i.e. not captured) and has variably
           // modified type, we still need to capture the type for correct
@@ -19908,7 +19358,8 @@ bool Sema::tryCaptureVariable(
             QualType QTy = Var->getType();
             if (ParmVarDecl *PVD = dyn_cast_or_null<ParmVarDecl>(Var))
               QTy = PVD->getOriginalType();
-            for (int I = 1, E = getNumberOfConstructScopes(RSI->OpenMPLevel);
+            for (int I = 1,
+                     E = OpenMP().getNumberOfConstructScopes(RSI->OpenMPLevel);
                  I < E; ++I) {
               auto *OuterRSI = cast<CapturedRegionScopeInfo>(
                   FunctionScopes[FunctionScopesIndex - I]);
@@ -19920,18 +19371,19 @@ bool Sema::tryCaptureVariable(
           }
           bool IsTargetCap =
               IsOpenMPPrivateDecl != OMPC_private &&
-              isOpenMPTargetCapturedDecl(Var, RSI->OpenMPLevel,
-                                         RSI->OpenMPCaptureLevel);
+              OpenMP().isOpenMPTargetCapturedDecl(Var, RSI->OpenMPLevel,
+                                                  RSI->OpenMPCaptureLevel);
           // Do not capture global if it is not privatized in outer regions.
           bool IsGlobalCap =
-              IsGlobal && isOpenMPGlobalCapturedDecl(Var, RSI->OpenMPLevel,
-                                                     RSI->OpenMPCaptureLevel);
+              IsGlobal && OpenMP().isOpenMPGlobalCapturedDecl(
+                              Var, RSI->OpenMPLevel, RSI->OpenMPCaptureLevel);
 
           // When we detect target captures we are looking from inside the
           // target region, therefore we need to propagate the capture from the
           // enclosing region. Therefore, the capture is not initially nested.
           if (IsTargetCap)
-            adjustOpenMPTargetScopeIndex(FunctionScopesIndex, RSI->OpenMPLevel);
+            OpenMP().adjustOpenMPTargetScopeIndex(FunctionScopesIndex,
+                                                  RSI->OpenMPLevel);
 
           if (IsTargetCap || IsOpenMPPrivateDecl == OMPC_private ||
               (IsGlobal && !IsGlobalCap)) {
@@ -20753,8 +20205,8 @@ static void
 MarkExprReferenced(Sema &SemaRef, SourceLocation Loc, Decl *D, Expr *E,
                    bool MightBeOdrUse,
                    llvm::DenseMap<const VarDecl *, int> &RefsMinusAssignments) {
-  if (SemaRef.isInOpenMPDeclareTargetContext())
-    SemaRef.checkDeclIsAllowedInOpenMPTarget(E, D);
+  if (SemaRef.OpenMP().isInOpenMPDeclareTargetContext())
+    SemaRef.OpenMP().checkDeclIsAllowedInOpenMPTarget(E, D);
 
   if (VarDecl *Var = dyn_cast<VarDecl>(D)) {
     DoMarkVarDeclReferenced(SemaRef, Loc, Var, E, RefsMinusAssignments);
diff --git a/clang/lib/Sema/SemaExprMember.cpp b/clang/lib/Sema/SemaExprMember.cpp
index 32998ae60eaf..7ea6d733fe5a 100644
--- a/clang/lib/Sema/SemaExprMember.cpp
+++ b/clang/lib/Sema/SemaExprMember.cpp
@@ -9,7 +9,6 @@
 //  This file implements semantic analysis member access expressions.
 //
 //===----------------------------------------------------------------------===//
-#include "clang/Sema/Overload.h"
 #include "clang/AST/ASTLambda.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclObjC.h"
@@ -18,9 +17,11 @@
 #include "clang/AST/ExprObjC.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Sema/Lookup.h"
+#include "clang/Sema/Overload.h"
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/SemaOpenMP.h"
 
 using namespace clang;
 using namespace sema;
@@ -1900,9 +1901,9 @@ Sema::BuildFieldReferenceExpr(Expr *BaseExpr, bool IsArrow,
   if (getLangOpts().OpenMP && IsArrow &&
       !CurContext->isDependentContext() &&
       isa<CXXThisExpr>(Base.get()->IgnoreParenImpCasts())) {
-    if (auto *PrivateCopy = isOpenMPCapturedDecl(Field)) {
-      return getOpenMPCapturedExpr(PrivateCopy, VK, OK,
-                                   MemberNameInfo.getLoc());
+    if (auto *PrivateCopy = OpenMP().isOpenMPCapturedDecl(Field)) {
+      return OpenMP().getOpenMPCapturedExpr(PrivateCopy, VK, OK,
+                                            MemberNameInfo.getLoc());
     }
   }
 
diff --git a/clang/lib/Sema/SemaLambda.cpp b/clang/lib/Sema/SemaLambda.cpp
index 35a51c6c2328..1743afaf1528 100644
--- a/clang/lib/Sema/SemaLambda.cpp
+++ b/clang/lib/Sema/SemaLambda.cpp
@@ -21,6 +21,7 @@
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/SemaOpenMP.h"
 #include "clang/Sema/Template.h"
 #include "llvm/ADT/STLExtras.h"
 #include <optional>
@@ -1398,7 +1399,7 @@ void Sema::ActOnStartOfLambdaDefinition(LambdaIntroducer &Intro,
 
   // OpenMP lambdas might get assumumption attributes.
   if (LangOpts.OpenMP)
-    ActOnFinishedFunctionDefinitionInOpenMPAssumeScope(Method);
+    OpenMP().ActOnFinishedFunctionDefinitionInOpenMPAssumeScope(Method);
 
   handleLambdaNumbering(Class, Method);
 
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index e9efb4721133..d229ef650bcc 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -11,6 +11,7 @@
 ///
 //===----------------------------------------------------------------------===//
 
+#include "clang/Sema/SemaOpenMP.h"
 #include "TreeTransform.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/ASTMutationListener.h"
@@ -33,6 +34,7 @@
 #include "clang/Sema/ParsedAttr.h"
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/ScopeInfo.h"
+#include "clang/Sema/Sema.h"
 #include "clang/Sema/SemaInternal.h"
 #include "llvm/ADT/IndexedMap.h"
 #include "llvm/ADT/PointerEmbeddedInt.h"
@@ -1808,9 +1810,9 @@ const DSAStackTy::DSAVarData DSAStackTy::getTopDSA(ValueDecl *D,
           return DVar;
         }
         const_iterator End = end();
-        if (!SemaRef.isOpenMPCapturedByRef(D,
-                                           std::distance(ParentIterTarget, End),
-                                           /*OpenMPCaptureLevel=*/0)) {
+        if (!SemaRef.OpenMP().isOpenMPCapturedByRef(
+                D, std::distance(ParentIterTarget, End),
+                /*OpenMPCaptureLevel=*/0)) {
           DVar.RefExpr =
               buildDeclRefExpr(SemaRef, VD, D->getType().getNonReferenceType(),
                                IterTarget->ConstructLoc);
@@ -2018,22 +2020,22 @@ bool DSAStackTy::hasDirective(
   return false;
 }
 
-void Sema::InitDataSharingAttributesStack() {
-  VarDataSharingAttributesStack = new DSAStackTy(*this);
+void SemaOpenMP::InitDataSharingAttributesStack() {
+  VarDataSharingAttributesStack = new DSAStackTy(SemaRef);
 }
 
 #define DSAStack static_cast<DSAStackTy *>(VarDataSharingAttributesStack)
 
-void Sema::pushOpenMPFunctionRegion() { DSAStack->pushFunction(); }
+void SemaOpenMP::pushOpenMPFunctionRegion() { DSAStack->pushFunction(); }
 
-void Sema::popOpenMPFunctionRegion(const FunctionScopeInfo *OldFSI) {
+void SemaOpenMP::popOpenMPFunctionRegion(const FunctionScopeInfo *OldFSI) {
   DSAStack->popFunction(OldFSI);
 }
 
 static bool isOpenMPDeviceDelayedContext(Sema &S) {
   assert(S.LangOpts.OpenMP && S.LangOpts.OpenMPIsTargetDevice &&
          "Expected OpenMP device compilation.");
-  return !S.isInOpenMPTargetExecutionDirective();
+  return !S.OpenMP().isInOpenMPTargetExecutionDirective();
 }
 
 namespace {
@@ -2045,20 +2047,20 @@ enum class FunctionEmissionStatus {
 };
 } // anonymous namespace
 
-Sema::SemaDiagnosticBuilder
-Sema::diagIfOpenMPDeviceCode(SourceLocation Loc, unsigned DiagID,
-                             const FunctionDecl *FD) {
-  assert(LangOpts.OpenMP && LangOpts.OpenMPIsTargetDevice &&
+SemaBase::SemaDiagnosticBuilder
+SemaOpenMP::diagIfOpenMPDeviceCode(SourceLocation Loc, unsigned DiagID,
+                                   const FunctionDecl *FD) {
+  assert(getLangOpts().OpenMP && getLangOpts().OpenMPIsTargetDevice &&
          "Expected OpenMP device compilation.");
 
   SemaDiagnosticBuilder::Kind Kind = SemaDiagnosticBuilder::K_Nop;
   if (FD) {
-    FunctionEmissionStatus FES = getEmissionStatus(FD);
+    Sema::FunctionEmissionStatus FES = SemaRef.getEmissionStatus(FD);
     switch (FES) {
-    case FunctionEmissionStatus::Emitted:
+    case Sema::FunctionEmissionStatus::Emitted:
       Kind = SemaDiagnosticBuilder::K_Immediate;
       break;
-    case FunctionEmissionStatus::Unknown:
+    case Sema::FunctionEmissionStatus::Unknown:
       // TODO: We should always delay diagnostics here in case a target
       //       region is in a function we do not emit. However, as the
       //       current diagnostics are associated with the function containing
@@ -2066,48 +2068,48 @@ Sema::diagIfOpenMPDeviceCode(SourceLocation Loc, unsigned DiagID,
       //       on diagnostics for the target region itself. We need to anchor
       //       the diagnostics with the new generated function *or* ensure we
       //       emit diagnostics associated with the surrounding function.
-      Kind = isOpenMPDeviceDelayedContext(*this)
+      Kind = isOpenMPDeviceDelayedContext(SemaRef)
                  ? SemaDiagnosticBuilder::K_Deferred
                  : SemaDiagnosticBuilder::K_Immediate;
       break;
-    case FunctionEmissionStatus::TemplateDiscarded:
-    case FunctionEmissionStatus::OMPDiscarded:
+    case Sema::FunctionEmissionStatus::TemplateDiscarded:
+    case Sema::FunctionEmissionStatus::OMPDiscarded:
       Kind = SemaDiagnosticBuilder::K_Nop;
       break;
-    case FunctionEmissionStatus::CUDADiscarded:
+    case Sema::FunctionEmissionStatus::CUDADiscarded:
       llvm_unreachable("CUDADiscarded unexpected in OpenMP device compilation");
       break;
     }
   }
 
-  return SemaDiagnosticBuilder(Kind, Loc, DiagID, FD, *this);
+  return SemaDiagnosticBuilder(Kind, Loc, DiagID, FD, SemaRef);
 }
 
-Sema::SemaDiagnosticBuilder Sema::diagIfOpenMPHostCode(SourceLocation Loc,
-                                                       unsigned DiagID,
-                                                       const FunctionDecl *FD) {
-  assert(LangOpts.OpenMP && !LangOpts.OpenMPIsTargetDevice &&
+SemaBase::SemaDiagnosticBuilder
+SemaOpenMP::diagIfOpenMPHostCode(SourceLocation Loc, unsigned DiagID,
+                                 const FunctionDecl *FD) {
+  assert(getLangOpts().OpenMP && !getLangOpts().OpenMPIsTargetDevice &&
          "Expected OpenMP host compilation.");
 
   SemaDiagnosticBuilder::Kind Kind = SemaDiagnosticBuilder::K_Nop;
   if (FD) {
-    FunctionEmissionStatus FES = getEmissionStatus(FD);
+    Sema::FunctionEmissionStatus FES = SemaRef.getEmissionStatus(FD);
     switch (FES) {
-    case FunctionEmissionStatus::Emitted:
+    case Sema::FunctionEmissionStatus::Emitted:
       Kind = SemaDiagnosticBuilder::K_Immediate;
       break;
-    case FunctionEmissionStatus::Unknown:
+    case Sema::FunctionEmissionStatus::Unknown:
       Kind = SemaDiagnosticBuilder::K_Deferred;
       break;
-    case FunctionEmissionStatus::TemplateDiscarded:
-    case FunctionEmissionStatus::OMPDiscarded:
-    case FunctionEmissionStatus::CUDADiscarded:
+    case Sema::FunctionEmissionStatus::TemplateDiscarded:
+    case Sema::FunctionEmissionStatus::OMPDiscarded:
+    case Sema::FunctionEmissionStatus::CUDADiscarded:
       Kind = SemaDiagnosticBuilder::K_Nop;
       break;
     }
   }
 
-  return SemaDiagnosticBuilder(Kind, Loc, DiagID, FD, *this);
+  return SemaDiagnosticBuilder(Kind, Loc, DiagID, FD, SemaRef);
 }
 
 static OpenMPDefaultmapClauseKind
@@ -2124,9 +2126,9 @@ getVariableCategoryFromDecl(const LangOptions &LO, const ValueDecl *VD) {
   return OMPC_DEFAULTMAP_aggregate;
 }
 
-bool Sema::isOpenMPCapturedByRef(const ValueDecl *D, unsigned Level,
-                                 unsigned OpenMPCaptureLevel) const {
-  assert(LangOpts.OpenMP && "OpenMP is not allowed");
+bool SemaOpenMP::isOpenMPCapturedByRef(const ValueDecl *D, unsigned Level,
+                                       unsigned OpenMPCaptureLevel) const {
+  assert(getLangOpts().OpenMP && "OpenMP is not allowed");
 
   ASTContext &Ctx = getASTContext();
   bool IsByRef = true;
@@ -2252,7 +2254,7 @@ bool Sema::isOpenMPCapturedByRef(const ValueDecl *D, unsigned Level,
                  !Ty->isAnyPointerType()) ||
                 !Ty->isScalarType() ||
                 DSAStack->isDefaultmapCapturedByRef(
-                    Level, getVariableCategoryFromDecl(LangOpts, D)) ||
+                    Level, getVariableCategoryFromDecl(getLangOpts(), D)) ||
                 DSAStack->hasExplicitDSA(
                     D,
                     [](OpenMPClauseKind K, bool AppliedToPointee) {
@@ -2303,17 +2305,17 @@ bool Sema::isOpenMPCapturedByRef(const ValueDecl *D, unsigned Level,
   return IsByRef;
 }
 
-unsigned Sema::getOpenMPNestingLevel() const {
+unsigned SemaOpenMP::getOpenMPNestingLevel() const {
   assert(getLangOpts().OpenMP);
   return DSAStack->getNestingLevel();
 }
 
-bool Sema::isInOpenMPTaskUntiedContext() const {
+bool SemaOpenMP::isInOpenMPTaskUntiedContext() const {
   return isOpenMPTaskingDirective(DSAStack->getCurrentDirective()) &&
          DSAStack->isUntiedRegion();
 }
 
-bool Sema::isInOpenMPTargetExecutionDirective() const {
+bool SemaOpenMP::isInOpenMPTargetExecutionDirective() const {
   return (isOpenMPTargetExecutionDirective(DSAStack->getCurrentDirective()) &&
           !DSAStack->isClauseParsingMode()) ||
          DSAStack->hasDirective(
@@ -2324,7 +2326,7 @@ bool Sema::isInOpenMPTargetExecutionDirective() const {
              false);
 }
 
-bool Sema::isOpenMPRebuildMemberExpr(ValueDecl *D) {
+bool SemaOpenMP::isOpenMPRebuildMemberExpr(ValueDecl *D) {
   // Only rebuild for Field.
   if (!dyn_cast<FieldDecl>(D))
     return false;
@@ -2347,9 +2349,9 @@ static OMPCapturedExprDecl *buildCaptureDecl(Sema &S, IdentifierInfo *Id,
                                              DeclContext *CurContext,
                                              bool AsExpression);
 
-VarDecl *Sema::isOpenMPCapturedDecl(ValueDecl *D, bool CheckScopeInfo,
-                                    unsigned StopAt) {
-  assert(LangOpts.OpenMP && "OpenMP is not allowed");
+VarDecl *SemaOpenMP::isOpenMPCapturedDecl(ValueDecl *D, bool CheckScopeInfo,
+                                          unsigned StopAt) {
+  assert(getLangOpts().OpenMP && "OpenMP is not allowed");
   D = getCanonicalDecl(D);
 
   auto *VD = dyn_cast<VarDecl>(D);
@@ -2368,7 +2370,8 @@ VarDecl *Sema::isOpenMPCapturedDecl(ValueDecl *D, bool CheckScopeInfo,
   // 'target' we return true so that this global is also mapped to the device.
   //
   if (VD && !VD->hasLocalStorage() &&
-      (getCurCapturedRegion() || getCurBlock() || getCurLambda())) {
+      (SemaRef.getCurCapturedRegion() || SemaRef.getCurBlock() ||
+       SemaRef.getCurLambda())) {
     if (isInOpenMPTargetExecutionDirective()) {
       DSAStackTy::DSAVarData DVarTop =
           DSAStack->getTopDSA(D, DSAStack->isClauseParsingMode());
@@ -2381,8 +2384,9 @@ VarDecl *Sema::isOpenMPCapturedDecl(ValueDecl *D, bool CheckScopeInfo,
         return nullptr;
       CapturedRegionScopeInfo *CSI = nullptr;
       for (FunctionScopeInfo *FSI : llvm::drop_begin(
-               llvm::reverse(FunctionScopes),
-               CheckScopeInfo ? (FunctionScopes.size() - (StopAt + 1)) : 0)) {
+               llvm::reverse(SemaRef.FunctionScopes),
+               CheckScopeInfo ? (SemaRef.FunctionScopes.size() - (StopAt + 1))
+                              : 0)) {
         if (!isa<CapturingScopeInfo>(FSI))
           return nullptr;
         if (auto *RSI = dyn_cast<CapturedRegionScopeInfo>(FSI))
@@ -2401,7 +2405,7 @@ VarDecl *Sema::isOpenMPCapturedDecl(ValueDecl *D, bool CheckScopeInfo,
     if (isInOpenMPDeclareTargetContext()) {
       // Try to mark variable as declare target if it is used in capturing
       // regions.
-      if (LangOpts.OpenMP <= 45 &&
+      if (getLangOpts().OpenMP <= 45 &&
           !OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD))
         checkDeclIsAllowedInOpenMPTarget(nullptr, VD);
       return nullptr;
@@ -2411,7 +2415,7 @@ VarDecl *Sema::isOpenMPCapturedDecl(ValueDecl *D, bool CheckScopeInfo,
   if (CheckScopeInfo) {
     bool OpenMPFound = false;
     for (unsigned I = StopAt + 1; I > 0; --I) {
-      FunctionScopeInfo *FSI = FunctionScopes[I - 1];
+      FunctionScopeInfo *FSI = SemaRef.FunctionScopes[I - 1];
       if (!isa<CapturingScopeInfo>(FSI))
         return nullptr;
       if (auto *RSI = dyn_cast<CapturedRegionScopeInfo>(FSI))
@@ -2476,22 +2480,23 @@ VarDecl *Sema::isOpenMPCapturedDecl(ValueDecl *D, bool CheckScopeInfo,
       VarDecl *VD = DSAStack->getImplicitFDCapExprDecl(FD);
       if (VD)
         return VD;
-      if (getCurrentThisType().isNull())
+      if (SemaRef.getCurrentThisType().isNull())
         return nullptr;
-      Expr *ThisExpr = BuildCXXThisExpr(SourceLocation(), getCurrentThisType(),
-                                        /*IsImplicit=*/true);
+      Expr *ThisExpr = SemaRef.BuildCXXThisExpr(SourceLocation(),
+                                                SemaRef.getCurrentThisType(),
+                                                /*IsImplicit=*/true);
       const CXXScopeSpec CS = CXXScopeSpec();
-      Expr *ME = BuildMemberExpr(ThisExpr, /*IsArrow=*/true, SourceLocation(),
-                                 NestedNameSpecifierLoc(), SourceLocation(), FD,
-                                 DeclAccessPair::make(FD, FD->getAccess()),
-                                 /*HadMultipleCandidates=*/false,
-                                 DeclarationNameInfo(), FD->getType(),
-                                 VK_LValue, OK_Ordinary);
+      Expr *ME = SemaRef.BuildMemberExpr(
+          ThisExpr, /*IsArrow=*/true, SourceLocation(),
+          NestedNameSpecifierLoc(), SourceLocation(), FD,
+          DeclAccessPair::make(FD, FD->getAccess()),
+          /*HadMultipleCandidates=*/false, DeclarationNameInfo(), FD->getType(),
+          VK_LValue, OK_Ordinary);
       OMPCapturedExprDecl *CD = buildCaptureDecl(
-          *this, FD->getIdentifier(), ME, DVarPrivate.CKind != OMPC_private,
-          CurContext->getParent(), /*AsExpression=*/false);
+          SemaRef, FD->getIdentifier(), ME, DVarPrivate.CKind != OMPC_private,
+          SemaRef.CurContext->getParent(), /*AsExpression=*/false);
       DeclRefExpr *VDPrivateRefExpr = buildDeclRefExpr(
-          *this, CD, CD->getType().getNonReferenceType(), SourceLocation());
+          SemaRef, CD, CD->getType().getNonReferenceType(), SourceLocation());
       VD = cast<VarDecl>(VDPrivateRefExpr->getDecl());
       DSAStack->addImplicitDefaultFirstprivateFD(FD, VD);
       return VD;
@@ -2505,28 +2510,28 @@ VarDecl *Sema::isOpenMPCapturedDecl(ValueDecl *D, bool CheckScopeInfo,
   return nullptr;
 }
 
-void Sema::adjustOpenMPTargetScopeIndex(unsigned &FunctionScopesIndex,
-                                        unsigned Level) const {
+void SemaOpenMP::adjustOpenMPTargetScopeIndex(unsigned &FunctionScopesIndex,
+                                              unsigned Level) const {
   FunctionScopesIndex -= getOpenMPCaptureLevels(DSAStack->getDirective(Level));
 }
 
-void Sema::startOpenMPLoop() {
-  assert(LangOpts.OpenMP && "OpenMP must be enabled.");
+void SemaOpenMP::startOpenMPLoop() {
+  assert(getLangOpts().OpenMP && "OpenMP must be enabled.");
   if (isOpenMPLoopDirective(DSAStack->getCurrentDirective()))
     DSAStack->loopInit();
 }
 
-void Sema::startOpenMPCXXRangeFor() {
-  assert(LangOpts.OpenMP && "OpenMP must be enabled.");
+void SemaOpenMP::startOpenMPCXXRangeFor() {
+  assert(getLangOpts().OpenMP && "OpenMP must be enabled.");
   if (isOpenMPLoopDirective(DSAStack->getCurrentDirective())) {
     DSAStack->resetPossibleLoopCounter();
     DSAStack->loopStart();
   }
 }
 
-OpenMPClauseKind Sema::isOpenMPPrivateDecl(ValueDecl *D, unsigned Level,
-                                           unsigned CapLevel) const {
-  assert(LangOpts.OpenMP && "OpenMP is not allowed");
+OpenMPClauseKind SemaOpenMP::isOpenMPPrivateDecl(ValueDecl *D, unsigned Level,
+                                                 unsigned CapLevel) const {
+  assert(getLangOpts().OpenMP && "OpenMP is not allowed");
   if (DSAStack->getCurrentDirective() != OMPD_unknown &&
       (!DSAStack->isClauseParsingMode() ||
        DSAStack->getParentDirective() != OMPD_unknown)) {
@@ -2546,7 +2551,8 @@ OpenMPClauseKind Sema::isOpenMPPrivateDecl(ValueDecl *D, unsigned Level,
   }
   if (DSAStack->hasExplicitDirective(isOpenMPTaskingDirective, Level)) {
     bool IsTriviallyCopyable =
-        D->getType().getNonReferenceType().isTriviallyCopyableType(Context) &&
+        D->getType().getNonReferenceType().isTriviallyCopyableType(
+            getASTContext()) &&
         !D->getType()
              .getNonReferenceType()
              .getCanonicalType()
@@ -2620,9 +2626,9 @@ OpenMPClauseKind Sema::isOpenMPPrivateDecl(ValueDecl *D, unsigned Level,
              : OMPC_unknown;
 }
 
-void Sema::setOpenMPCaptureKind(FieldDecl *FD, const ValueDecl *D,
-                                unsigned Level) {
-  assert(LangOpts.OpenMP && "OpenMP is not allowed");
+void SemaOpenMP::setOpenMPCaptureKind(FieldDecl *FD, const ValueDecl *D,
+                                      unsigned Level) {
+  assert(getLangOpts().OpenMP && "OpenMP is not allowed");
   D = getCanonicalDecl(D);
   OpenMPClauseKind OMPC = OMPC_unknown;
   for (unsigned I = DSAStack->getNestingLevel() + 1; I > Level; --I) {
@@ -2649,18 +2655,19 @@ void Sema::setOpenMPCaptureKind(FieldDecl *FD, const ValueDecl *D,
                                        NewLevel)) {
       OMPC = OMPC_map;
       if (DSAStack->mustBeFirstprivateAtLevel(
-              NewLevel, getVariableCategoryFromDecl(LangOpts, D)))
+              NewLevel, getVariableCategoryFromDecl(getLangOpts(), D)))
         OMPC = OMPC_firstprivate;
       break;
     }
   }
   if (OMPC != OMPC_unknown)
-    FD->addAttr(OMPCaptureKindAttr::CreateImplicit(Context, unsigned(OMPC)));
+    FD->addAttr(
+        OMPCaptureKindAttr::CreateImplicit(getASTContext(), unsigned(OMPC)));
 }
 
-bool Sema::isOpenMPTargetCapturedDecl(const ValueDecl *D, unsigned Level,
-                                      unsigned CaptureLevel) const {
-  assert(LangOpts.OpenMP && "OpenMP is not allowed");
+bool SemaOpenMP::isOpenMPTargetCapturedDecl(const ValueDecl *D, unsigned Level,
+                                            unsigned CaptureLevel) const {
+  assert(getLangOpts().OpenMP && "OpenMP is not allowed");
   // Return true if the current level is no longer enclosed in a target region.
 
   SmallVector<OpenMPDirectiveKind, 4> Regions;
@@ -2672,9 +2679,9 @@ bool Sema::isOpenMPTargetCapturedDecl(const ValueDecl *D, unsigned Level,
          Regions[CaptureLevel] != OMPD_task;
 }
 
-bool Sema::isOpenMPGlobalCapturedDecl(ValueDecl *D, unsigned Level,
-                                      unsigned CaptureLevel) const {
-  assert(LangOpts.OpenMP && "OpenMP is not allowed");
+bool SemaOpenMP::isOpenMPGlobalCapturedDecl(ValueDecl *D, unsigned Level,
+                                            unsigned CaptureLevel) const {
+  assert(getLangOpts().OpenMP && "OpenMP is not allowed");
   // Return true if the current level is no longer enclosed in a target region.
 
   if (const auto *VD = dyn_cast<VarDecl>(D)) {
@@ -2702,37 +2709,37 @@ bool Sema::isOpenMPGlobalCapturedDecl(ValueDecl *D, unsigned Level,
   return true;
 }
 
-void Sema::DestroyDataSharingAttributesStack() { delete DSAStack; }
+void SemaOpenMP::DestroyDataSharingAttributesStack() { delete DSAStack; }
 
-void Sema::ActOnOpenMPBeginDeclareVariant(SourceLocation Loc,
-                                          OMPTraitInfo &TI) {
+void SemaOpenMP::ActOnOpenMPBeginDeclareVariant(SourceLocation Loc,
+                                                OMPTraitInfo &TI) {
   OMPDeclareVariantScopes.push_back(OMPDeclareVariantScope(TI));
 }
 
-void Sema::ActOnOpenMPEndDeclareVariant() {
+void SemaOpenMP::ActOnOpenMPEndDeclareVariant() {
   assert(isInOpenMPDeclareVariantScope() &&
          "Not in OpenMP declare variant scope!");
 
   OMPDeclareVariantScopes.pop_back();
 }
 
-void Sema::finalizeOpenMPDelayedAnalysis(const FunctionDecl *Caller,
-                                         const FunctionDecl *Callee,
-                                         SourceLocation Loc) {
-  assert(LangOpts.OpenMP && "Expected OpenMP compilation mode.");
+void SemaOpenMP::finalizeOpenMPDelayedAnalysis(const FunctionDecl *Caller,
+                                               const FunctionDecl *Callee,
+                                               SourceLocation Loc) {
+  assert(getLangOpts().OpenMP && "Expected OpenMP compilation mode.");
   std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =
       OMPDeclareTargetDeclAttr::getDeviceType(Caller->getMostRecentDecl());
   // Ignore host functions during device analyzis.
-  if (LangOpts.OpenMPIsTargetDevice &&
+  if (getLangOpts().OpenMPIsTargetDevice &&
       (!DevTy || *DevTy == OMPDeclareTargetDeclAttr::DT_Host))
     return;
   // Ignore nohost functions during host analyzis.
-  if (!LangOpts.OpenMPIsTargetDevice && DevTy &&
+  if (!getLangOpts().OpenMPIsTargetDevice && DevTy &&
       *DevTy == OMPDeclareTargetDeclAttr::DT_NoHost)
     return;
   const FunctionDecl *FD = Callee->getMostRecentDecl();
   DevTy = OMPDeclareTargetDeclAttr::getDeviceType(FD);
-  if (LangOpts.OpenMPIsTargetDevice && DevTy &&
+  if (getLangOpts().OpenMPIsTargetDevice && DevTy &&
       *DevTy == OMPDeclareTargetDeclAttr::DT_Host) {
     // Diagnose host function called during device codegen.
     StringRef HostDevTy =
@@ -2743,8 +2750,9 @@ void Sema::finalizeOpenMPDelayedAnalysis(const FunctionDecl *Caller,
         << HostDevTy;
     return;
   }
-  if (!LangOpts.OpenMPIsTargetDevice && !LangOpts.OpenMPOffloadMandatory &&
-      DevTy && *DevTy == OMPDeclareTargetDeclAttr::DT_NoHost) {
+  if (!getLangOpts().OpenMPIsTargetDevice &&
+      !getLangOpts().OpenMPOffloadMandatory && DevTy &&
+      *DevTy == OMPDeclareTargetDeclAttr::DT_NoHost) {
     // In OpenMP 5.2 or later, if the function has a host variant then allow
     // that to be called instead
     auto &&HasHostAttr = [](const FunctionDecl *Callee) {
@@ -2773,21 +2781,21 @@ void Sema::finalizeOpenMPDelayedAnalysis(const FunctionDecl *Caller,
   }
 }
 
-void Sema::StartOpenMPDSABlock(OpenMPDirectiveKind DKind,
-                               const DeclarationNameInfo &DirName,
-                               Scope *CurScope, SourceLocation Loc) {
+void SemaOpenMP::StartOpenMPDSABlock(OpenMPDirectiveKind DKind,
+                                     const DeclarationNameInfo &DirName,
+                                     Scope *CurScope, SourceLocation Loc) {
   DSAStack->push(DKind, DirName, CurScope, Loc);
-  PushExpressionEvaluationContext(
-      ExpressionEvaluationContext::PotentiallyEvaluated);
+  SemaRef.PushExpressionEvaluationContext(
+      Sema::ExpressionEvaluationContext::PotentiallyEvaluated);
 }
 
-void Sema::StartOpenMPClause(OpenMPClauseKind K) {
+void SemaOpenMP::StartOpenMPClause(OpenMPClauseKind K) {
   DSAStack->setClauseParsingMode(K);
 }
 
-void Sema::EndOpenMPClause() {
+void SemaOpenMP::EndOpenMPClause() {
   DSAStack->setClauseParsingMode(/*K=*/OMPC_unknown);
-  CleanupVarDeclMarking();
+  SemaRef.CleanupVarDeclMarking();
 }
 
 static std::pair<ValueDecl *, bool>
@@ -2871,7 +2879,7 @@ static void reportOriginalDsa(Sema &SemaRef, const DSAStackTy *Stack,
                               const DSAStackTy::DSAVarData &DVar,
                               bool IsLoopIterVar = false);
 
-void Sema::EndOpenMPDSABlock(Stmt *CurDirective) {
+void SemaOpenMP::EndOpenMPDSABlock(Stmt *CurDirective) {
   // OpenMP [2.14.3.5, Restrictions, C/C++, p.1]
   //  A variable of class type (or array thereof) that appears in a lastprivate
   //  clause requires an accessible, unambiguous default constructor for the
@@ -2898,15 +2906,15 @@ void Sema::EndOpenMPDSABlock(Stmt *CurDirective) {
             // variable is not added to IdResolver, so the code in the OpenMP
             // region uses original variable for proper diagnostics.
             VarDecl *VDPrivate = buildVarDecl(
-                *this, DE->getExprLoc(), Type.getUnqualifiedType(),
+                SemaRef, DE->getExprLoc(), Type.getUnqualifiedType(),
                 VD->getName(), VD->hasAttrs() ? &VD->getAttrs() : nullptr, DRE);
-            ActOnUninitializedDecl(VDPrivate);
+            SemaRef.ActOnUninitializedDecl(VDPrivate);
             if (VDPrivate->isInvalidDecl()) {
               PrivateCopies.push_back(nullptr);
               continue;
             }
             PrivateCopies.push_back(buildDeclRefExpr(
-                *this, VDPrivate, DE->getType(), DE->getExprLoc()));
+                SemaRef, VDPrivate, DE->getType(), DE->getExprLoc()));
           } else {
             // The variable is also a firstprivate, so initialization sequence
             // for private copy is generated already.
@@ -2924,7 +2932,7 @@ void Sema::EndOpenMPDSABlock(Stmt *CurDirective) {
           SourceLocation ELoc;
           SourceRange ERange;
           Expr *SimpleRefExpr = RefExpr;
-          auto Res = getPrivateItem(*this, SimpleRefExpr, ELoc, ERange);
+          auto Res = getPrivateItem(SemaRef, SimpleRefExpr, ELoc, ERange);
           if (Res.second)
             // It will be analyzed later.
             PrivateRefs.push_back(RefExpr);
@@ -2977,7 +2985,7 @@ void Sema::EndOpenMPDSABlock(Stmt *CurDirective) {
                  diag::err_omp_allocator_used_in_clauses)
                 << D.Allocator->getSourceRange();
             if (DVar.RefExpr)
-              reportOriginalDsa(*this, DSAStack, VD, DVar);
+              reportOriginalDsa(SemaRef, DSAStack, VD, DVar);
             else
               Diag(MapExpr->getExprLoc(), diag::note_used_here)
                   << MapExpr->getSourceRange();
@@ -2987,14 +2995,14 @@ void Sema::EndOpenMPDSABlock(Stmt *CurDirective) {
       }
     }
     // Check allocate clauses.
-    if (!CurContext->isDependentContext())
-      checkAllocateClauses(*this, DSAStack, D->clauses());
-    checkReductionClauses(*this, DSAStack, D->clauses());
+    if (!SemaRef.CurContext->isDependentContext())
+      checkAllocateClauses(SemaRef, DSAStack, D->clauses());
+    checkReductionClauses(SemaRef, DSAStack, D->clauses());
   }
 
   DSAStack->pop();
-  DiscardCleanupsInEvaluationContext();
-  PopExpressionEvaluationContext();
+  SemaRef.DiscardCleanupsInEvaluationContext();
+  SemaRef.PopExpressionEvaluationContext();
 }
 
 static bool FinishOpenMPLinearClause(OMPLinearClause &Clause, DeclRefExpr *IV,
@@ -3047,27 +3055,28 @@ public:
 
 } // namespace
 
-ExprResult Sema::ActOnOpenMPIdExpression(Scope *CurScope,
-                                         CXXScopeSpec &ScopeSpec,
-                                         const DeclarationNameInfo &Id,
-                                         OpenMPDirectiveKind Kind) {
-  LookupResult Lookup(*this, Id, LookupOrdinaryName);
-  LookupParsedName(Lookup, CurScope, &ScopeSpec, true);
+ExprResult SemaOpenMP::ActOnOpenMPIdExpression(Scope *CurScope,
+                                               CXXScopeSpec &ScopeSpec,
+                                               const DeclarationNameInfo &Id,
+                                               OpenMPDirectiveKind Kind) {
+  ASTContext &Context = getASTContext();
+  LookupResult Lookup(SemaRef, Id, Sema::LookupOrdinaryName);
+  SemaRef.LookupParsedName(Lookup, CurScope, &ScopeSpec, true);
 
   if (Lookup.isAmbiguous())
     return ExprError();
 
   VarDecl *VD;
   if (!Lookup.isSingleResult()) {
-    VarDeclFilterCCC CCC(*this);
+    VarDeclFilterCCC CCC(SemaRef);
     if (TypoCorrection Corrected =
-            CorrectTypo(Id, LookupOrdinaryName, CurScope, nullptr, CCC,
-                        CTK_ErrorRecovery)) {
-      diagnoseTypo(Corrected,
-                   PDiag(Lookup.empty()
-                             ? diag::err_undeclared_var_use_suggest
-                             : diag::err_omp_expected_var_arg_suggest)
-                       << Id.getName());
+            SemaRef.CorrectTypo(Id, Sema::LookupOrdinaryName, CurScope, nullptr,
+                                CCC, Sema::CTK_ErrorRecovery)) {
+      SemaRef.diagnoseTypo(
+          Corrected,
+          SemaRef.PDiag(Lookup.empty() ? diag::err_undeclared_var_use_suggest
+                                       : diag::err_omp_expected_var_arg_suggest)
+              << Id.getName());
       VD = Corrected.getCorrectionDeclAs<VarDecl>();
     } else {
       Diag(Id.getLoc(), Lookup.empty() ? diag::err_undeclared_var_use
@@ -3101,7 +3110,7 @@ ExprResult Sema::ActOnOpenMPIdExpression(Scope *CurScope,
   //   A threadprivate directive for file-scope variables must appear outside
   //   any definition or declaration.
   if (CanonicalVD->getDeclContext()->isTranslationUnit() &&
-      !getCurLexicalContext()->isTranslationUnit()) {
+      !SemaRef.getCurLexicalContext()->isTranslationUnit()) {
     Diag(Id.getLoc(), diag::err_omp_var_scope)
         << getOpenMPDirectiveName(Kind) << VD;
     bool IsDecl =
@@ -3116,7 +3125,7 @@ ExprResult Sema::ActOnOpenMPIdExpression(Scope *CurScope,
   //   in the class definition, in the same scope in which the member
   //   variables are declared.
   if (CanonicalVD->isStaticDataMember() &&
-      !CanonicalVD->getDeclContext()->Equals(getCurLexicalContext())) {
+      !CanonicalVD->getDeclContext()->Equals(SemaRef.getCurLexicalContext())) {
     Diag(Id.getLoc(), diag::err_omp_var_scope)
         << getOpenMPDirectiveName(Kind) << VD;
     bool IsDecl =
@@ -3131,8 +3140,9 @@ ExprResult Sema::ActOnOpenMPIdExpression(Scope *CurScope,
   //   outside any definition or declaration other than the namespace
   //   definition itself.
   if (CanonicalVD->getDeclContext()->isNamespace() &&
-      (!getCurLexicalContext()->isFileContext() ||
-       !getCurLexicalContext()->Encloses(CanonicalVD->getDeclContext()))) {
+      (!SemaRef.getCurLexicalContext()->isFileContext() ||
+       !SemaRef.getCurLexicalContext()->Encloses(
+           CanonicalVD->getDeclContext()))) {
     Diag(Id.getLoc(), diag::err_omp_var_scope)
         << getOpenMPDirectiveName(Kind) << VD;
     bool IsDecl =
@@ -3146,7 +3156,7 @@ ExprResult Sema::ActOnOpenMPIdExpression(Scope *CurScope,
   //   A threadprivate directive for static block-scope variables must appear
   //   in the scope of the variable and not in a nested scope.
   if (CanonicalVD->isLocalVarDecl() && CurScope &&
-      !isDeclInScope(ND, getCurLexicalContext(), CurScope)) {
+      !SemaRef.isDeclInScope(ND, SemaRef.getCurLexicalContext(), CurScope)) {
     Diag(Id.getLoc(), diag::err_omp_var_scope)
         << getOpenMPDirectiveName(Kind) << VD;
     bool IsDecl =
@@ -3174,11 +3184,11 @@ ExprResult Sema::ActOnOpenMPIdExpression(Scope *CurScope,
                              Id.getLoc(), ExprType, VK_LValue);
 }
 
-Sema::DeclGroupPtrTy
-Sema::ActOnOpenMPThreadprivateDirective(SourceLocation Loc,
-                                        ArrayRef<Expr *> VarList) {
+SemaOpenMP::DeclGroupPtrTy
+SemaOpenMP::ActOnOpenMPThreadprivateDirective(SourceLocation Loc,
+                                              ArrayRef<Expr *> VarList) {
   if (OMPThreadPrivateDecl *D = CheckOMPThreadPrivateDecl(Loc, VarList)) {
-    CurContext->addDecl(D);
+    SemaRef.CurContext->addDecl(D);
     return DeclGroupPtrTy::make(DeclGroupRef(D));
   }
   return nullptr;
@@ -3215,7 +3225,9 @@ public:
 } // namespace
 
 OMPThreadPrivateDecl *
-Sema::CheckOMPThreadPrivateDecl(SourceLocation Loc, ArrayRef<Expr *> VarList) {
+SemaOpenMP::CheckOMPThreadPrivateDecl(SourceLocation Loc,
+                                      ArrayRef<Expr *> VarList) {
+  ASTContext &Context = getASTContext();
   SmallVector<Expr *, 8> Vars;
   for (Expr *RefExpr : VarList) {
     auto *DE = cast<DeclRefExpr>(RefExpr);
@@ -3235,8 +3247,8 @@ Sema::CheckOMPThreadPrivateDecl(SourceLocation Loc, ArrayRef<Expr *> VarList) {
 
     // OpenMP [2.9.2, Restrictions, C/C++, p.10]
     //   A threadprivate variable must not have an incomplete type.
-    if (RequireCompleteType(ILoc, VD->getType(),
-                            diag::err_omp_threadprivate_incomplete_type)) {
+    if (SemaRef.RequireCompleteType(
+            ILoc, VD->getType(), diag::err_omp_threadprivate_incomplete_type)) {
       continue;
     }
 
@@ -3274,7 +3286,7 @@ Sema::CheckOMPThreadPrivateDecl(SourceLocation Loc, ArrayRef<Expr *> VarList) {
     // Check if initial value of threadprivate variable reference variable with
     // local storage (it is not supported by runtime).
     if (const Expr *Init = VD->getAnyInitializer()) {
-      LocalVarRefChecker Checker(*this);
+      LocalVarRefChecker Checker(SemaRef);
       if (Checker.Visit(Init))
         continue;
     }
@@ -3288,8 +3300,8 @@ Sema::CheckOMPThreadPrivateDecl(SourceLocation Loc, ArrayRef<Expr *> VarList) {
   }
   OMPThreadPrivateDecl *D = nullptr;
   if (!Vars.empty()) {
-    D = OMPThreadPrivateDecl::Create(Context, getCurLexicalContext(), Loc,
-                                     Vars);
+    D = OMPThreadPrivateDecl::Create(Context, SemaRef.getCurLexicalContext(),
+                                     Loc, Vars);
     D->setAccess(AS_public);
   }
   return D;
@@ -3395,10 +3407,9 @@ applyOMPAllocateAttribute(Sema &S, VarDecl *VD,
     ML->DeclarationMarkedOpenMPAllocate(VD, A);
 }
 
-Sema::DeclGroupPtrTy
-Sema::ActOnOpenMPAllocateDirective(SourceLocation Loc, ArrayRef<Expr *> VarList,
-                                   ArrayRef<OMPClause *> Clauses,
-                                   DeclContext *Owner) {
+SemaOpenMP::DeclGroupPtrTy SemaOpenMP::ActOnOpenMPAllocateDirective(
+    SourceLocation Loc, ArrayRef<Expr *> VarList, ArrayRef<OMPClause *> Clauses,
+    DeclContext *Owner) {
   assert(Clauses.size() <= 2 && "Expected at most two clauses.");
   Expr *Alignment = nullptr;
   Expr *Allocator = nullptr;
@@ -3407,9 +3418,9 @@ Sema::ActOnOpenMPAllocateDirective(SourceLocation Loc, ArrayRef<Expr *> VarList,
     // allocate directives that appear in a target region must specify an
     // allocator clause unless a requires directive with the dynamic_allocators
     // clause is present in the same compilation unit.
-    if (LangOpts.OpenMPIsTargetDevice &&
+    if (getLangOpts().OpenMPIsTargetDevice &&
         !DSAStack->hasRequiresDeclWithClause<OMPDynamicAllocatorsClause>())
-      targetDiag(Loc, diag::err_expected_allocator_clause);
+      SemaRef.targetDiag(Loc, diag::err_expected_allocator_clause);
   } else {
     for (const OMPClause *C : Clauses)
       if (const auto *AC = dyn_cast<OMPAllocatorClause>(C))
@@ -3420,7 +3431,7 @@ Sema::ActOnOpenMPAllocateDirective(SourceLocation Loc, ArrayRef<Expr *> VarList,
         llvm_unreachable("Unexpected clause on allocate directive");
   }
   OMPAllocateDeclAttr::AllocatorTypeTy AllocatorKind =
-      getAllocatorKind(*this, DSAStack, Allocator);
+      getAllocatorKind(SemaRef, DSAStack, Allocator);
   SmallVector<Expr *, 8> Vars;
   for (Expr *RefExpr : VarList) {
     auto *DE = cast<DeclRefExpr>(RefExpr);
@@ -3435,7 +3446,7 @@ Sema::ActOnOpenMPAllocateDirective(SourceLocation Loc, ArrayRef<Expr *> VarList,
 
     // If the used several times in the allocate directive, the same allocator
     // must be used.
-    if (checkPreviousOMPAllocateAttribute(*this, DSAStack, RefExpr, VD,
+    if (checkPreviousOMPAllocateAttribute(SemaRef, DSAStack, RefExpr, VD,
                                           AllocatorKind, Allocator))
       continue;
 
@@ -3448,7 +3459,7 @@ Sema::ActOnOpenMPAllocateDirective(SourceLocation Loc, ArrayRef<Expr *> VarList,
         Diag(Allocator->getExprLoc(),
              diag::err_omp_expected_predefined_allocator)
             << Allocator->getSourceRange();
-        bool IsDecl = VD->isThisDeclarationADefinition(Context) ==
+        bool IsDecl = VD->isThisDeclarationADefinition(getASTContext()) ==
                       VarDecl::DeclarationOnly;
         Diag(VD->getLocation(),
              IsDecl ? diag::note_previous_decl : diag::note_defined_here)
@@ -3458,45 +3469,46 @@ Sema::ActOnOpenMPAllocateDirective(SourceLocation Loc, ArrayRef<Expr *> VarList,
     }
 
     Vars.push_back(RefExpr);
-    applyOMPAllocateAttribute(*this, VD, AllocatorKind, Allocator, Alignment,
+    applyOMPAllocateAttribute(SemaRef, VD, AllocatorKind, Allocator, Alignment,
                               DE->getSourceRange());
   }
   if (Vars.empty())
     return nullptr;
   if (!Owner)
-    Owner = getCurLexicalContext();
-  auto *D = OMPAllocateDecl::Create(Context, Owner, Loc, Vars, Clauses);
+    Owner = SemaRef.getCurLexicalContext();
+  auto *D = OMPAllocateDecl::Create(getASTContext(), Owner, Loc, Vars, Clauses);
   D->setAccess(AS_public);
   Owner->addDecl(D);
   return DeclGroupPtrTy::make(DeclGroupRef(D));
 }
 
-Sema::DeclGroupPtrTy
-Sema::ActOnOpenMPRequiresDirective(SourceLocation Loc,
-                                   ArrayRef<OMPClause *> ClauseList) {
+SemaOpenMP::DeclGroupPtrTy
+SemaOpenMP::ActOnOpenMPRequiresDirective(SourceLocation Loc,
+                                         ArrayRef<OMPClause *> ClauseList) {
   OMPRequiresDecl *D = nullptr;
-  if (!CurContext->isFileContext()) {
+  if (!SemaRef.CurContext->isFileContext()) {
     Diag(Loc, diag::err_omp_invalid_scope) << "requires";
   } else {
     D = CheckOMPRequiresDecl(Loc, ClauseList);
     if (D) {
-      CurContext->addDecl(D);
+      SemaRef.CurContext->addDecl(D);
       DSAStack->addRequiresDecl(D);
     }
   }
   return DeclGroupPtrTy::make(DeclGroupRef(D));
 }
 
-void Sema::ActOnOpenMPAssumesDirective(SourceLocation Loc,
-                                       OpenMPDirectiveKind DKind,
-                                       ArrayRef<std::string> Assumptions,
-                                       bool SkippedClauses) {
+void SemaOpenMP::ActOnOpenMPAssumesDirective(SourceLocation Loc,
+                                             OpenMPDirectiveKind DKind,
+                                             ArrayRef<std::string> Assumptions,
+                                             bool SkippedClauses) {
   if (!SkippedClauses && Assumptions.empty())
     Diag(Loc, diag::err_omp_no_clause_for_directive)
         << llvm::omp::getAllAssumeClauseOptions()
         << llvm::omp::getOpenMPDirectiveName(DKind);
 
-  auto *AA = OMPAssumeAttr::Create(Context, llvm::join(Assumptions, ","), Loc);
+  auto *AA =
+      OMPAssumeAttr::Create(getASTContext(), llvm::join(Assumptions, ","), Loc);
   if (DKind == llvm::omp::Directive::OMPD_begin_assumes) {
     OMPAssumeScoped.push_back(AA);
     return;
@@ -3515,7 +3527,7 @@ void Sema::ActOnOpenMPAssumesDirective(SourceLocation Loc,
   // declarations in included headers. To this end, we traverse all existing
   // declaration contexts and annotate function declarations here.
   SmallVector<DeclContext *, 8> DeclContexts;
-  auto *Ctx = CurContext;
+  auto *Ctx = SemaRef.CurContext;
   while (Ctx->getLexicalParent())
     Ctx = Ctx->getLexicalParent();
   DeclContexts.push_back(Ctx);
@@ -3539,13 +3551,14 @@ void Sema::ActOnOpenMPAssumesDirective(SourceLocation Loc,
   }
 }
 
-void Sema::ActOnOpenMPEndAssumesDirective() {
+void SemaOpenMP::ActOnOpenMPEndAssumesDirective() {
   assert(isInOpenMPAssumeScope() && "Not in OpenMP assumes scope!");
   OMPAssumeScoped.pop_back();
 }
 
-OMPRequiresDecl *Sema::CheckOMPRequiresDecl(SourceLocation Loc,
-                                            ArrayRef<OMPClause *> ClauseList) {
+OMPRequiresDecl *
+SemaOpenMP::CheckOMPRequiresDecl(SourceLocation Loc,
+                                 ArrayRef<OMPClause *> ClauseList) {
   /// For target specific clauses, the requires directive cannot be
   /// specified after the handling of any of the target regions in the
   /// current compilation unit.
@@ -3576,8 +3589,8 @@ OMPRequiresDecl *Sema::CheckOMPRequiresDecl(SourceLocation Loc,
   }
 
   if (!DSAStack->hasDuplicateRequiresClause(ClauseList))
-    return OMPRequiresDecl::Create(Context, getCurLexicalContext(), Loc,
-                                   ClauseList);
+    return OMPRequiresDecl::Create(
+        getASTContext(), SemaRef.getCurLexicalContext(), Loc, ClauseList);
   return nullptr;
 }
 
@@ -3695,7 +3708,7 @@ class DSAAttrChecker final : public StmtVisitor<DSAAttrChecker, void> {
   llvm::SmallVector<Expr *, 4> ImplicitMap[DefaultmapKindNum][OMPC_MAP_delete];
   llvm::SmallVector<OpenMPMapModifierKind, NumberOfOMPMapClauseModifiers>
       ImplicitMapModifier[DefaultmapKindNum];
-  Sema::VarsWithInheritedDSAType VarsWithInheritedDSA;
+  SemaOpenMP::VarsWithInheritedDSAType VarsWithInheritedDSA;
   llvm::SmallDenseSet<const ValueDecl *, 4> ImplicitDeclarations;
 
   void VisitSubCaptures(OMPExecutableDirective *S) {
@@ -4161,7 +4174,7 @@ public:
   getImplicitMapModifier(OpenMPDefaultmapClauseKind Kind) const {
     return ImplicitMapModifier[Kind];
   }
-  const Sema::VarsWithInheritedDSAType &getVarsWithInheritedDSA() const {
+  const SemaOpenMP::VarsWithInheritedDSAType &getVarsWithInheritedDSA() const {
     return VarsWithInheritedDSA;
   }
 
@@ -4193,7 +4206,9 @@ static void handleDeclareVariantConstructTrait(DSAStackTy *Stack,
   Stack->handleConstructTrait(Traits, ScopeEntry);
 }
 
-void Sema::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind, Scope *CurScope) {
+void SemaOpenMP::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind,
+                                        Scope *CurScope) {
+  ASTContext &Context = getASTContext();
   switch (DKind) {
   case OMPD_parallel:
   case OMPD_parallel_for:
@@ -4208,13 +4223,13 @@ void Sema::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind, Scope *CurScope) {
     QualType KmpInt32Ty = Context.getIntTypeForBitwidth(32, 1).withConst();
     QualType KmpInt32PtrTy =
         Context.getPointerType(KmpInt32Ty).withConst().withRestrict();
-    Sema::CapturedParamNameType Params[] = {
+    SemaOpenMP::CapturedParamNameType Params[] = {
         std::make_pair(".global_tid.", KmpInt32PtrTy),
         std::make_pair(".bound_tid.", KmpInt32PtrTy),
         std::make_pair(StringRef(), QualType()) // __context with shared vars
     };
-    ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope, CR_OpenMP,
-                             Params);
+    SemaRef.ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope,
+                                     CR_OpenMP, Params);
     break;
   }
   case OMPD_target_teams:
@@ -4232,7 +4247,7 @@ void Sema::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind, Scope *CurScope) {
     FunctionProtoType::ExtProtoInfo EPI;
     EPI.Variadic = true;
     QualType CopyFnType = Context.getFunctionType(Context.VoidTy, Args, EPI);
-    Sema::CapturedParamNameType Params[] = {
+    SemaOpenMP::CapturedParamNameType Params[] = {
         std::make_pair(".global_tid.", KmpInt32Ty),
         std::make_pair(".part_id.", KmpInt32PtrTy),
         std::make_pair(".privates.", VoidPtrTy),
@@ -4242,31 +4257,33 @@ void Sema::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind, Scope *CurScope) {
         std::make_pair(".task_t.", Context.VoidPtrTy.withConst()),
         std::make_pair(StringRef(), QualType()) // __context with shared vars
     };
-    ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope, CR_OpenMP,
-                             Params, /*OpenMPCaptureLevel=*/0);
+    SemaRef.ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope,
+                                     CR_OpenMP, Params,
+                                     /*OpenMPCaptureLevel=*/0);
     // Mark this captured region as inlined, because we don't use outlined
     // function directly.
-    getCurCapturedRegion()->TheCapturedDecl->addAttr(
+    SemaRef.getCurCapturedRegion()->TheCapturedDecl->addAttr(
         AlwaysInlineAttr::CreateImplicit(
             Context, {}, AlwaysInlineAttr::Keyword_forceinline));
-    SmallVector<Sema::CapturedParamNameType, 2> ParamsTarget;
+    SmallVector<SemaOpenMP::CapturedParamNameType, 2> ParamsTarget;
     if (getLangOpts().OpenMPIsTargetDevice)
       ParamsTarget.push_back(std::make_pair(StringRef("dyn_ptr"), VoidPtrTy));
     ParamsTarget.push_back(
         std::make_pair(StringRef(), QualType())); // __context with shared vars;
     // Start a captured region for 'target' with no implicit parameters.
-    ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope, CR_OpenMP,
-                             ParamsTarget,
-                             /*OpenMPCaptureLevel=*/1);
-    Sema::CapturedParamNameType ParamsTeamsOrParallel[] = {
+    SemaRef.ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope,
+                                     CR_OpenMP, ParamsTarget,
+                                     /*OpenMPCaptureLevel=*/1);
+    SemaOpenMP::CapturedParamNameType ParamsTeamsOrParallel[] = {
         std::make_pair(".global_tid.", KmpInt32PtrTy),
         std::make_pair(".bound_tid.", KmpInt32PtrTy),
         std::make_pair(StringRef(), QualType()) // __context with shared vars
     };
     // Start a captured region for 'teams' or 'parallel'.  Both regions have
     // the same implicit parameters.
-    ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope, CR_OpenMP,
-                             ParamsTeamsOrParallel, /*OpenMPCaptureLevel=*/2);
+    SemaRef.ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope,
+                                     CR_OpenMP, ParamsTeamsOrParallel,
+                                     /*OpenMPCaptureLevel=*/2);
     break;
   }
   case OMPD_target:
@@ -4279,7 +4296,7 @@ void Sema::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind, Scope *CurScope) {
     FunctionProtoType::ExtProtoInfo EPI;
     EPI.Variadic = true;
     QualType CopyFnType = Context.getFunctionType(Context.VoidTy, Args, EPI);
-    Sema::CapturedParamNameType Params[] = {
+    SemaOpenMP::CapturedParamNameType Params[] = {
         std::make_pair(".global_tid.", KmpInt32Ty),
         std::make_pair(".part_id.", KmpInt32PtrTy),
         std::make_pair(".privates.", VoidPtrTy),
@@ -4289,21 +4306,22 @@ void Sema::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind, Scope *CurScope) {
         std::make_pair(".task_t.", Context.VoidPtrTy.withConst()),
         std::make_pair(StringRef(), QualType()) // __context with shared vars
     };
-    ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope, CR_OpenMP,
-                             Params, /*OpenMPCaptureLevel=*/0);
+    SemaRef.ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope,
+                                     CR_OpenMP, Params,
+                                     /*OpenMPCaptureLevel=*/0);
     // Mark this captured region as inlined, because we don't use outlined
     // function directly.
-    getCurCapturedRegion()->TheCapturedDecl->addAttr(
+    SemaRef.getCurCapturedRegion()->TheCapturedDecl->addAttr(
         AlwaysInlineAttr::CreateImplicit(
             Context, {}, AlwaysInlineAttr::Keyword_forceinline));
-    SmallVector<Sema::CapturedParamNameType, 2> ParamsTarget;
+    SmallVector<SemaOpenMP::CapturedParamNameType, 2> ParamsTarget;
     if (getLangOpts().OpenMPIsTargetDevice)
       ParamsTarget.push_back(std::make_pair(StringRef("dyn_ptr"), VoidPtrTy));
     ParamsTarget.push_back(
         std::make_pair(StringRef(), QualType())); // __context with shared vars;
-    ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope, CR_OpenMP,
-                             ParamsTarget,
-                             /*OpenMPCaptureLevel=*/1);
+    SemaRef.ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope,
+                                     CR_OpenMP, ParamsTarget,
+                                     /*OpenMPCaptureLevel=*/1);
     break;
   }
   case OMPD_atomic:
@@ -4329,11 +4347,11 @@ void Sema::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind, Scope *CurScope) {
   case OMPD_scope:
   case OMPD_target_data:
   case OMPD_dispatch: {
-    Sema::CapturedParamNameType Params[] = {
+    SemaOpenMP::CapturedParamNameType Params[] = {
         std::make_pair(StringRef(), QualType()) // __context with shared vars
     };
-    ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope, CR_OpenMP,
-                             Params);
+    SemaRef.ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope,
+                                     CR_OpenMP, Params);
     break;
   }
   case OMPD_task: {
@@ -4345,7 +4363,7 @@ void Sema::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind, Scope *CurScope) {
     FunctionProtoType::ExtProtoInfo EPI;
     EPI.Variadic = true;
     QualType CopyFnType = Context.getFunctionType(Context.VoidTy, Args, EPI);
-    Sema::CapturedParamNameType Params[] = {
+    SemaOpenMP::CapturedParamNameType Params[] = {
         std::make_pair(".global_tid.", KmpInt32Ty),
         std::make_pair(".part_id.", KmpInt32PtrTy),
         std::make_pair(".privates.", VoidPtrTy),
@@ -4355,11 +4373,11 @@ void Sema::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind, Scope *CurScope) {
         std::make_pair(".task_t.", Context.VoidPtrTy.withConst()),
         std::make_pair(StringRef(), QualType()) // __context with shared vars
     };
-    ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope, CR_OpenMP,
-                             Params);
+    SemaRef.ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope,
+                                     CR_OpenMP, Params);
     // Mark this captured region as inlined, because we don't use outlined
     // function directly.
-    getCurCapturedRegion()->TheCapturedDecl->addAttr(
+    SemaRef.getCurCapturedRegion()->TheCapturedDecl->addAttr(
         AlwaysInlineAttr::CreateImplicit(
             Context, {}, AlwaysInlineAttr::Keyword_forceinline));
     break;
@@ -4386,7 +4404,7 @@ void Sema::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind, Scope *CurScope) {
     FunctionProtoType::ExtProtoInfo EPI;
     EPI.Variadic = true;
     QualType CopyFnType = Context.getFunctionType(Context.VoidTy, Args, EPI);
-    Sema::CapturedParamNameType Params[] = {
+    SemaOpenMP::CapturedParamNameType Params[] = {
         std::make_pair(".global_tid.", KmpInt32Ty),
         std::make_pair(".part_id.", KmpInt32PtrTy),
         std::make_pair(".privates.", VoidPtrTy),
@@ -4401,11 +4419,11 @@ void Sema::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind, Scope *CurScope) {
         std::make_pair(".reductions.", VoidPtrTy),
         std::make_pair(StringRef(), QualType()) // __context with shared vars
     };
-    ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope, CR_OpenMP,
-                             Params);
+    SemaRef.ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope,
+                                     CR_OpenMP, Params);
     // Mark this captured region as inlined, because we don't use outlined
     // function directly.
-    getCurCapturedRegion()->TheCapturedDecl->addAttr(
+    SemaRef.getCurCapturedRegion()->TheCapturedDecl->addAttr(
         AlwaysInlineAttr::CreateImplicit(
             Context, {}, AlwaysInlineAttr::Keyword_forceinline));
     break;
@@ -4426,19 +4444,20 @@ void Sema::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind, Scope *CurScope) {
     QualType VoidPtrTy = Context.VoidPtrTy.withConst().withRestrict();
     QualType KmpInt32PtrTy =
         Context.getPointerType(KmpInt32Ty).withConst().withRestrict();
-    Sema::CapturedParamNameType ParamsParallel[] = {
+    SemaOpenMP::CapturedParamNameType ParamsParallel[] = {
         std::make_pair(".global_tid.", KmpInt32PtrTy),
         std::make_pair(".bound_tid.", KmpInt32PtrTy),
         std::make_pair(StringRef(), QualType()) // __context with shared vars
     };
     // Start a captured region for 'parallel'.
-    ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope, CR_OpenMP,
-                             ParamsParallel, /*OpenMPCaptureLevel=*/0);
+    SemaRef.ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope,
+                                     CR_OpenMP, ParamsParallel,
+                                     /*OpenMPCaptureLevel=*/0);
     QualType Args[] = {VoidPtrTy};
     FunctionProtoType::ExtProtoInfo EPI;
     EPI.Variadic = true;
     QualType CopyFnType = Context.getFunctionType(Context.VoidTy, Args, EPI);
-    Sema::CapturedParamNameType Params[] = {
+    SemaOpenMP::CapturedParamNameType Params[] = {
         std::make_pair(".global_tid.", KmpInt32Ty),
         std::make_pair(".part_id.", KmpInt32PtrTy),
         std::make_pair(".privates.", VoidPtrTy),
@@ -4453,11 +4472,12 @@ void Sema::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind, Scope *CurScope) {
         std::make_pair(".reductions.", VoidPtrTy),
         std::make_pair(StringRef(), QualType()) // __context with shared vars
     };
-    ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope, CR_OpenMP,
-                             Params, /*OpenMPCaptureLevel=*/1);
+    SemaRef.ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope,
+                                     CR_OpenMP, Params,
+                                     /*OpenMPCaptureLevel=*/1);
     // Mark this captured region as inlined, because we don't use outlined
     // function directly.
-    getCurCapturedRegion()->TheCapturedDecl->addAttr(
+    SemaRef.getCurCapturedRegion()->TheCapturedDecl->addAttr(
         AlwaysInlineAttr::CreateImplicit(
             Context, {}, AlwaysInlineAttr::Keyword_forceinline));
     break;
@@ -4467,15 +4487,15 @@ void Sema::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind, Scope *CurScope) {
     QualType KmpInt32Ty = Context.getIntTypeForBitwidth(32, 1).withConst();
     QualType KmpInt32PtrTy =
         Context.getPointerType(KmpInt32Ty).withConst().withRestrict();
-    Sema::CapturedParamNameType Params[] = {
+    SemaOpenMP::CapturedParamNameType Params[] = {
         std::make_pair(".global_tid.", KmpInt32PtrTy),
         std::make_pair(".bound_tid.", KmpInt32PtrTy),
         std::make_pair(".previous.lb.", Context.getSizeType().withConst()),
         std::make_pair(".previous.ub.", Context.getSizeType().withConst()),
         std::make_pair(StringRef(), QualType()) // __context with shared vars
     };
-    ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope, CR_OpenMP,
-                             Params);
+    SemaRef.ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope,
+                                     CR_OpenMP, Params);
     break;
   }
   // For 'target teams loop', collect all captured regions so codegen can
@@ -4492,7 +4512,7 @@ void Sema::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind, Scope *CurScope) {
     FunctionProtoType::ExtProtoInfo EPI;
     EPI.Variadic = true;
     QualType CopyFnType = Context.getFunctionType(Context.VoidTy, Args, EPI);
-    Sema::CapturedParamNameType Params[] = {
+    SemaOpenMP::CapturedParamNameType Params[] = {
         std::make_pair(".global_tid.", KmpInt32Ty),
         std::make_pair(".part_id.", KmpInt32PtrTy),
         std::make_pair(".privates.", VoidPtrTy),
@@ -4502,32 +4522,35 @@ void Sema::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind, Scope *CurScope) {
         std::make_pair(".task_t.", Context.VoidPtrTy.withConst()),
         std::make_pair(StringRef(), QualType()) // __context with shared vars
     };
-    ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope, CR_OpenMP,
-                             Params, /*OpenMPCaptureLevel=*/0);
+    SemaRef.ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope,
+                                     CR_OpenMP, Params,
+                                     /*OpenMPCaptureLevel=*/0);
     // Mark this captured region as inlined, because we don't use outlined
     // function directly.
-    getCurCapturedRegion()->TheCapturedDecl->addAttr(
+    SemaRef.getCurCapturedRegion()->TheCapturedDecl->addAttr(
         AlwaysInlineAttr::CreateImplicit(
             Context, {}, AlwaysInlineAttr::Keyword_forceinline));
-    SmallVector<Sema::CapturedParamNameType, 2> ParamsTarget;
+    SmallVector<SemaOpenMP::CapturedParamNameType, 2> ParamsTarget;
     if (getLangOpts().OpenMPIsTargetDevice)
       ParamsTarget.push_back(std::make_pair(StringRef("dyn_ptr"), VoidPtrTy));
     ParamsTarget.push_back(
         std::make_pair(StringRef(), QualType())); // __context with shared vars;
     // Start a captured region for 'target' with no implicit parameters.
-    ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope, CR_OpenMP,
-                             ParamsTarget, /*OpenMPCaptureLevel=*/1);
+    SemaRef.ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope,
+                                     CR_OpenMP, ParamsTarget,
+                                     /*OpenMPCaptureLevel=*/1);
 
-    Sema::CapturedParamNameType ParamsTeams[] = {
+    SemaOpenMP::CapturedParamNameType ParamsTeams[] = {
         std::make_pair(".global_tid.", KmpInt32PtrTy),
         std::make_pair(".bound_tid.", KmpInt32PtrTy),
         std::make_pair(StringRef(), QualType()) // __context with shared vars
     };
     // Start a captured region for 'target' with no implicit parameters.
-    ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope, CR_OpenMP,
-                             ParamsTeams, /*OpenMPCaptureLevel=*/2);
+    SemaRef.ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope,
+                                     CR_OpenMP, ParamsTeams,
+                                     /*OpenMPCaptureLevel=*/2);
 
-    Sema::CapturedParamNameType ParamsParallel[] = {
+    SemaOpenMP::CapturedParamNameType ParamsParallel[] = {
         std::make_pair(".global_tid.", KmpInt32PtrTy),
         std::make_pair(".bound_tid.", KmpInt32PtrTy),
         std::make_pair(".previous.lb.", Context.getSizeType().withConst()),
@@ -4536,8 +4559,9 @@ void Sema::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind, Scope *CurScope) {
     };
     // Start a captured region for 'teams' or 'parallel'.  Both regions have
     // the same implicit parameters.
-    ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope, CR_OpenMP,
-                             ParamsParallel, /*OpenMPCaptureLevel=*/3);
+    SemaRef.ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope,
+                                     CR_OpenMP, ParamsParallel,
+                                     /*OpenMPCaptureLevel=*/3);
     break;
   }
 
@@ -4548,16 +4572,17 @@ void Sema::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind, Scope *CurScope) {
     QualType KmpInt32PtrTy =
         Context.getPointerType(KmpInt32Ty).withConst().withRestrict();
 
-    Sema::CapturedParamNameType ParamsTeams[] = {
+    SemaOpenMP::CapturedParamNameType ParamsTeams[] = {
         std::make_pair(".global_tid.", KmpInt32PtrTy),
         std::make_pair(".bound_tid.", KmpInt32PtrTy),
         std::make_pair(StringRef(), QualType()) // __context with shared vars
     };
     // Start a captured region for 'target' with no implicit parameters.
-    ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope, CR_OpenMP,
-                             ParamsTeams, /*OpenMPCaptureLevel=*/0);
+    SemaRef.ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope,
+                                     CR_OpenMP, ParamsTeams,
+                                     /*OpenMPCaptureLevel=*/0);
 
-    Sema::CapturedParamNameType ParamsParallel[] = {
+    SemaOpenMP::CapturedParamNameType ParamsParallel[] = {
         std::make_pair(".global_tid.", KmpInt32PtrTy),
         std::make_pair(".bound_tid.", KmpInt32PtrTy),
         std::make_pair(".previous.lb.", Context.getSizeType().withConst()),
@@ -4566,8 +4591,9 @@ void Sema::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind, Scope *CurScope) {
     };
     // Start a captured region for 'teams' or 'parallel'.  Both regions have
     // the same implicit parameters.
-    ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope, CR_OpenMP,
-                             ParamsParallel, /*OpenMPCaptureLevel=*/1);
+    SemaRef.ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope,
+                                     CR_OpenMP, ParamsParallel,
+                                     /*OpenMPCaptureLevel=*/1);
     break;
   }
   case OMPD_target_update:
@@ -4581,7 +4607,7 @@ void Sema::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind, Scope *CurScope) {
     FunctionProtoType::ExtProtoInfo EPI;
     EPI.Variadic = true;
     QualType CopyFnType = Context.getFunctionType(Context.VoidTy, Args, EPI);
-    Sema::CapturedParamNameType Params[] = {
+    SemaOpenMP::CapturedParamNameType Params[] = {
         std::make_pair(".global_tid.", KmpInt32Ty),
         std::make_pair(".part_id.", KmpInt32PtrTy),
         std::make_pair(".privates.", VoidPtrTy),
@@ -4591,11 +4617,11 @@ void Sema::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind, Scope *CurScope) {
         std::make_pair(".task_t.", Context.VoidPtrTy.withConst()),
         std::make_pair(StringRef(), QualType()) // __context with shared vars
     };
-    ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope, CR_OpenMP,
-                             Params);
+    SemaRef.ActOnCapturedRegionStart(DSAStack->getConstructLoc(), CurScope,
+                                     CR_OpenMP, Params);
     // Mark this captured region as inlined, because we don't use outlined
     // function directly.
-    getCurCapturedRegion()->TheCapturedDecl->addAttr(
+    SemaRef.getCurCapturedRegion()->TheCapturedDecl->addAttr(
         AlwaysInlineAttr::CreateImplicit(
             Context, {}, AlwaysInlineAttr::Keyword_forceinline));
     break;
@@ -4626,15 +4652,15 @@ void Sema::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind, Scope *CurScope) {
   default:
     llvm_unreachable("Unknown OpenMP directive");
   }
-  DSAStack->setContext(CurContext);
+  DSAStack->setContext(SemaRef.CurContext);
   handleDeclareVariantConstructTrait(DSAStack, DKind, /* ScopeEntry */ true);
 }
 
-int Sema::getNumberOfConstructScopes(unsigned Level) const {
+int SemaOpenMP::getNumberOfConstructScopes(unsigned Level) const {
   return getOpenMPCaptureLevels(DSAStack->getDirective(Level));
 }
 
-int Sema::getOpenMPCaptureLevels(OpenMPDirectiveKind DKind) {
+int SemaOpenMP::getOpenMPCaptureLevels(OpenMPDirectiveKind DKind) {
   SmallVector<OpenMPDirectiveKind, 4> CaptureRegions;
   getOpenMPCaptureRegions(CaptureRegions, DKind);
   return CaptureRegions.size();
@@ -4674,7 +4700,7 @@ static OMPCapturedExprDecl *buildCaptureDecl(Sema &S, IdentifierInfo *Id,
 static DeclRefExpr *buildCapture(Sema &S, ValueDecl *D, Expr *CaptureExpr,
                                  bool WithInit) {
   OMPCapturedExprDecl *CD;
-  if (VarDecl *VD = S.isOpenMPCapturedDecl(D))
+  if (VarDecl *VD = S.OpenMP().isOpenMPCapturedDecl(D))
     CD = cast<OMPCapturedExprDecl>(VD);
   else
     CD = buildCaptureDecl(S, D->getIdentifier(), CaptureExpr, WithInit,
@@ -4726,7 +4752,7 @@ public:
       : S(S), ErrorFound(ErrorFound), DKind(DKind) {}
   ~CaptureRegionUnwinderRAII() {
     if (ErrorFound) {
-      int ThisCaptureLevel = S.getOpenMPCaptureLevels(DKind);
+      int ThisCaptureLevel = S.OpenMP().getOpenMPCaptureLevels(DKind);
       while (--ThisCaptureLevel >= 0)
         S.ActOnCapturedRegionError();
     }
@@ -4734,10 +4760,10 @@ public:
 };
 } // namespace
 
-void Sema::tryCaptureOpenMPLambdas(ValueDecl *V) {
+void SemaOpenMP::tryCaptureOpenMPLambdas(ValueDecl *V) {
   // Capture variables captured by reference in lambdas for target-based
   // directives.
-  if (!CurContext->isDependentContext() &&
+  if (!SemaRef.CurContext->isDependentContext() &&
       (isOpenMPTargetExecutionDirective(DSAStack->getCurrentDirective()) ||
        isOpenMPTargetDataManagementDirective(
            DSAStack->getCurrentDirective()))) {
@@ -4757,14 +4783,14 @@ void Sema::tryCaptureOpenMPLambdas(ValueDecl *V) {
           if (LC.getCaptureKind() == LCK_ByRef) {
             VarDecl *VD = cast<VarDecl>(LC.getCapturedVar());
             DeclContext *VDC = VD->getDeclContext();
-            if (!VDC->Encloses(CurContext))
+            if (!VDC->Encloses(SemaRef.CurContext))
               continue;
-            MarkVariableReferenced(LC.getLocation(), VD);
+            SemaRef.MarkVariableReferenced(LC.getLocation(), VD);
           } else if (LC.getCaptureKind() == LCK_This) {
-            QualType ThisTy = getCurrentThisType();
-            if (!ThisTy.isNull() &&
-                Context.typesAreCompatible(ThisTy, ThisCapture->getType()))
-              CheckCXXThisCapture(LC.getLocation());
+            QualType ThisTy = SemaRef.getCurrentThisType();
+            if (!ThisTy.isNull() && getASTContext().typesAreCompatible(
+                                        ThisTy, ThisCapture->getType()))
+              SemaRef.CheckCXXThisCapture(LC.getLocation());
           }
         }
       }
@@ -4804,8 +4830,8 @@ static bool checkOrderedOrderSpecified(Sema &S,
   return false;
 }
 
-StmtResult Sema::ActOnOpenMPRegionEnd(StmtResult S,
-                                      ArrayRef<OMPClause *> Clauses) {
+StmtResult SemaOpenMP::ActOnOpenMPRegionEnd(StmtResult S,
+                                            ArrayRef<OMPClause *> Clauses) {
   handleDeclareVariantConstructTrait(DSAStack, DSAStack->getCurrentDirective(),
                                      /* ScopeEntry */ false);
   if (DSAStack->getCurrentDirective() == OMPD_atomic ||
@@ -4817,7 +4843,7 @@ StmtResult Sema::ActOnOpenMPRegionEnd(StmtResult S,
 
   bool ErrorFound = false;
   CaptureRegionUnwinderRAII CaptureRegionUnwinder(
-      *this, ErrorFound, DSAStack->getCurrentDirective());
+      SemaRef, ErrorFound, DSAStack->getCurrentDirective());
   if (!S.isUsable()) {
     ErrorFound = true;
     return StmtError();
@@ -4831,7 +4857,7 @@ StmtResult Sema::ActOnOpenMPRegionEnd(StmtResult S,
   SmallVector<const OMPClauseWithPreInit *, 4> PICs;
   // This is required for proper codegen.
   for (OMPClause *Clause : Clauses) {
-    if (!LangOpts.OpenMPSimd &&
+    if (!getLangOpts().OpenMPSimd &&
         (isOpenMPTaskingDirective(DSAStack->getCurrentDirective()) ||
          DSAStack->getCurrentDirective() == OMPD_target) &&
         Clause->getClauseKind() == OMPC_in_reduction) {
@@ -4840,7 +4866,7 @@ StmtResult Sema::ActOnOpenMPRegionEnd(StmtResult S,
       auto *IRC = cast<OMPInReductionClause>(Clause);
       for (Expr *E : IRC->taskgroup_descriptors())
         if (E)
-          MarkDeclarationsReferencedInExpr(E);
+          SemaRef.MarkDeclarationsReferencedInExpr(E);
     }
     if (isOpenMPPrivate(Clause->getClauseKind()) ||
         Clause->getClauseKind() == OMPC_copyprivate ||
@@ -4851,7 +4877,7 @@ StmtResult Sema::ActOnOpenMPRegionEnd(StmtResult S,
       // Mark all variables in private list clauses as used in inner region.
       for (Stmt *VarRef : Clause->children()) {
         if (auto *E = cast_or_null<Expr>(VarRef)) {
-          MarkDeclarationsReferencedInExpr(E);
+          SemaRef.MarkDeclarationsReferencedInExpr(E);
         }
       }
       DSAStack->setForceVarCapturing(/*V=*/false);
@@ -4865,7 +4891,7 @@ StmtResult Sema::ActOnOpenMPRegionEnd(StmtResult S,
         PICs.push_back(C);
       if (auto *C = OMPClauseWithPostUpdate::get(Clause)) {
         if (Expr *E = C->getPostUpdateExpr())
-          MarkDeclarationsReferencedInExpr(E);
+          SemaRef.MarkDeclarationsReferencedInExpr(E);
       }
     }
     if (Clause->getClauseKind() == OMPC_schedule)
@@ -4877,7 +4903,7 @@ StmtResult Sema::ActOnOpenMPRegionEnd(StmtResult S,
   }
   // Capture allocator expressions if used.
   for (Expr *E : DSAStack->getInnerAllocators())
-    MarkDeclarationsReferencedInExpr(E);
+    SemaRef.MarkDeclarationsReferencedInExpr(E);
   // OpenMP, 2.7.1 Loop Construct, Restrictions
   // The nonmonotonic modifier cannot be specified if an ordered clause is
   // specified.
@@ -4899,7 +4925,7 @@ StmtResult Sema::ActOnOpenMPRegionEnd(StmtResult S,
   // OpenMP 5.0, 2.9.2 Worksharing-Loop Construct, Restrictions.
   // If an order(concurrent) clause is present, an ordered clause may not appear
   // on the same directive.
-  if (checkOrderedOrderSpecified(*this, Clauses))
+  if (checkOrderedOrderSpecified(SemaRef, Clauses))
     ErrorFound = true;
   if (!LCs.empty() && OC && OC->getNumForLoops()) {
     for (const OMPLinearClause *C : LCs) {
@@ -4936,7 +4962,8 @@ StmtResult Sema::ActOnOpenMPRegionEnd(StmtResult S,
             CaptureRegion == OMPD_unknown) {
           if (auto *DS = cast_or_null<DeclStmt>(C->getPreInitStmt())) {
             for (Decl *D : DS->decls())
-              MarkVariableReferenced(D->getLocation(), cast<VarDecl>(D));
+              SemaRef.MarkVariableReferenced(D->getLocation(),
+                                             cast<VarDecl>(D));
           }
         }
       }
@@ -4950,7 +4977,7 @@ StmtResult Sema::ActOnOpenMPRegionEnd(StmtResult S,
                ++I) {
             OMPUsesAllocatorsClause::Data D = UAC->getAllocatorData(I);
             if (Expr *E = D.AllocatorTraits)
-              MarkDeclarationsReferencedInExpr(E);
+              SemaRef.MarkDeclarationsReferencedInExpr(E);
           }
           continue;
         }
@@ -4965,17 +4992,17 @@ StmtResult Sema::ActOnOpenMPRegionEnd(StmtResult S,
             continue;
           for (Expr *E : RC->copy_array_temps())
             if (E)
-              MarkDeclarationsReferencedInExpr(E);
+              SemaRef.MarkDeclarationsReferencedInExpr(E);
         }
         if (auto *AC = dyn_cast<OMPAlignedClause>(C)) {
           for (Expr *E : AC->varlists())
-            MarkDeclarationsReferencedInExpr(E);
+            SemaRef.MarkDeclarationsReferencedInExpr(E);
         }
       }
     }
     if (++CompletedRegions == CaptureRegions.size())
       DSAStack->setBodyComplete();
-    SR = ActOnCapturedRegionEnd(SR.get());
+    SR = SemaRef.ActOnCapturedRegionEnd(SR.get());
   }
   return SR;
 }
@@ -5782,9 +5809,9 @@ static CapturedStmt *buildLoopVarFunc(Sema &Actions, QualType LoopVarTy,
   // the OpenMPIRBuilder to know additional C/C++ semantics, such as how to
   // invoke a copy constructor.
   QualType TargetParamTy = Ctx.getLValueReferenceType(LoopVarTy);
-  Sema::CapturedParamNameType Params[] = {{"LoopVar", TargetParamTy},
-                                          {"Logical", LogicalTy},
-                                          {StringRef(), QualType()}};
+  SemaOpenMP::CapturedParamNameType Params[] = {{"LoopVar", TargetParamTy},
+                                                {"Logical", LogicalTy},
+                                                {StringRef(), QualType()}};
   Actions.ActOnCapturedRegionStart({}, nullptr, CR_Default, Params);
 
   // Capture the initial iterator which represents the LoopVar value at the
@@ -5835,7 +5862,7 @@ static CapturedStmt *buildLoopVarFunc(Sema &Actions, QualType LoopVarTy,
       AssertSuccess(Actions.ActOnCapturedRegionEnd(Body)));
 }
 
-StmtResult Sema::ActOnOpenMPCanonicalLoop(Stmt *AStmt) {
+StmtResult SemaOpenMP::ActOnOpenMPCanonicalLoop(Stmt *AStmt) {
   ASTContext &Ctx = getASTContext();
 
   // Extract the common elements of ForStmt and CXXForRangeStmt:
@@ -5946,8 +5973,8 @@ StmtResult Sema::ActOnOpenMPCanonicalLoop(Stmt *AStmt) {
     if (IncBin->getOpcode() == BO_AddAssign) {
       Step = IncBin->getRHS();
     } else if (IncBin->getOpcode() == BO_SubAssign) {
-      Step =
-          AssertSuccess(BuildUnaryOp(nullptr, {}, UO_Minus, IncBin->getRHS()));
+      Step = AssertSuccess(
+          SemaRef.BuildUnaryOp(nullptr, {}, UO_Minus, IncBin->getRHS()));
     } else
       llvm_unreachable("unhandled binary increment operator");
   } else if (auto *CondCXXOp = dyn_cast<CXXOperatorCallExpr>(Inc)) {
@@ -5965,7 +5992,7 @@ StmtResult Sema::ActOnOpenMPCanonicalLoop(Stmt *AStmt) {
       break;
     case OO_MinusEqual:
       Step = AssertSuccess(
-          BuildUnaryOp(nullptr, {}, UO_Minus, CondCXXOp->getArg(1)));
+          SemaRef.BuildUnaryOp(nullptr, {}, UO_Minus, CondCXXOp->getArg(1)));
       break;
     default:
       llvm_unreachable("unhandled overloaded increment operator");
@@ -5974,16 +6001,17 @@ StmtResult Sema::ActOnOpenMPCanonicalLoop(Stmt *AStmt) {
     llvm_unreachable("unknown increment expression");
 
   CapturedStmt *DistanceFunc =
-      buildDistanceFunc(*this, LogicalTy, CondRel, LHS, RHS, Step);
+      buildDistanceFunc(SemaRef, LogicalTy, CondRel, LHS, RHS, Step);
   CapturedStmt *LoopVarFunc = buildLoopVarFunc(
-      *this, LVTy, LogicalTy, CounterRef, Step, isa<CXXForRangeStmt>(AStmt));
-  DeclRefExpr *LVRef = BuildDeclRefExpr(LUVDecl, LUVDecl->getType(), VK_LValue,
-                                        {}, nullptr, nullptr, {}, nullptr);
+      SemaRef, LVTy, LogicalTy, CounterRef, Step, isa<CXXForRangeStmt>(AStmt));
+  DeclRefExpr *LVRef =
+      SemaRef.BuildDeclRefExpr(LUVDecl, LUVDecl->getType(), VK_LValue, {},
+                               nullptr, nullptr, {}, nullptr);
   return OMPCanonicalLoop::create(getASTContext(), AStmt, DistanceFunc,
                                   LoopVarFunc, LVRef);
 }
 
-StmtResult Sema::ActOnOpenMPLoopnest(Stmt *AStmt) {
+StmtResult SemaOpenMP::ActOnOpenMPLoopnest(Stmt *AStmt) {
   // Handle a literal loop.
   if (isa<ForStmt>(AStmt) || isa<CXXForRangeStmt>(AStmt))
     return ActOnOpenMPCanonicalLoop(AStmt);
@@ -6128,7 +6156,7 @@ processImplicitMapsWithDefaultMappers(Sema &S, DSAStackTy *Stack,
       continue;
     CXXScopeSpec MapperIdScopeSpec;
     DeclarationNameInfo MapperId;
-    if (OMPClause *NewClause = S.ActOnOpenMPMapClause(
+    if (OMPClause *NewClause = S.OpenMP().ActOnOpenMPMapClause(
             nullptr, C->getMapTypeModifiers(), C->getMapTypeModifiersLoc(),
             MapperIdScopeSpec, MapperId, C->getMapType(),
             /*IsMapTypeImplicit=*/true, SourceLocation(), SourceLocation(),
@@ -6210,14 +6238,12 @@ static bool teamsLoopCanBeParallelFor(Stmt *AStmt, Sema &SemaRef) {
   return Checker.teamsLoopCanBeParallelFor();
 }
 
-bool Sema::mapLoopConstruct(llvm::SmallVector<OMPClause *> &ClausesWithoutBind,
-                            ArrayRef<OMPClause *> Clauses,
-                            OpenMPBindClauseKind &BindKind,
-                            OpenMPDirectiveKind &Kind,
-                            OpenMPDirectiveKind &PrevMappedDirective,
-                            SourceLocation StartLoc, SourceLocation EndLoc,
-                            const DeclarationNameInfo &DirName,
-                            OpenMPDirectiveKind CancelRegion) {
+bool SemaOpenMP::mapLoopConstruct(
+    llvm::SmallVector<OMPClause *> &ClausesWithoutBind,
+    ArrayRef<OMPClause *> Clauses, OpenMPBindClauseKind &BindKind,
+    OpenMPDirectiveKind &Kind, OpenMPDirectiveKind &PrevMappedDirective,
+    SourceLocation StartLoc, SourceLocation EndLoc,
+    const DeclarationNameInfo &DirName, OpenMPDirectiveKind CancelRegion) {
 
   bool UseClausesWithoutBind = false;
 
@@ -6299,7 +6325,7 @@ bool Sema::mapLoopConstruct(llvm::SmallVector<OMPClause *> &ClausesWithoutBind,
   return UseClausesWithoutBind;
 }
 
-StmtResult Sema::ActOnOpenMPExecutableDirective(
+StmtResult SemaOpenMP::ActOnOpenMPExecutableDirective(
     OpenMPDirectiveKind Kind, const DeclarationNameInfo &DirName,
     OpenMPDirectiveKind CancelRegion, ArrayRef<OMPClause *> Clauses,
     Stmt *AStmt, SourceLocation StartLoc, SourceLocation EndLoc,
@@ -6324,8 +6350,8 @@ StmtResult Sema::ActOnOpenMPExecutableDirective(
   }
 
   // First check CancelRegion which is then used in checkNestingOfRegions.
-  if (checkCancelRegion(*this, Kind, CancelRegion, StartLoc) ||
-      checkNestingOfRegions(*this, DSAStack, DK, DirName, CancelRegion,
+  if (checkCancelRegion(SemaRef, Kind, CancelRegion, StartLoc) ||
+      checkNestingOfRegions(SemaRef, DSAStack, DK, DirName, CancelRegion,
                             BindKind, StartLoc)) {
     return StmtError();
   }
@@ -6344,13 +6370,14 @@ StmtResult Sema::ActOnOpenMPExecutableDirective(
   } else {
     ClausesWithImplicit.append(Clauses.begin(), Clauses.end());
   }
-  if (AStmt && !CurContext->isDependentContext() && Kind != OMPD_atomic &&
-      Kind != OMPD_critical && Kind != OMPD_section && Kind != OMPD_master &&
-      Kind != OMPD_masked && !isOpenMPLoopTransformationDirective(Kind)) {
+  if (AStmt && !SemaRef.CurContext->isDependentContext() &&
+      Kind != OMPD_atomic && Kind != OMPD_critical && Kind != OMPD_section &&
+      Kind != OMPD_master && Kind != OMPD_masked &&
+      !isOpenMPLoopTransformationDirective(Kind)) {
     assert(isa<CapturedStmt>(AStmt) && "Captured statement expected");
 
     // Check default data sharing attributes for referenced variables.
-    DSAAttrChecker DSAChecker(DSAStack, *this, cast<CapturedStmt>(AStmt));
+    DSAAttrChecker DSAChecker(DSAStack, SemaRef, cast<CapturedStmt>(AStmt));
     int ThisCaptureLevel = getOpenMPCaptureLevels(Kind);
     Stmt *S = AStmt;
     while (--ThisCaptureLevel >= 0)
@@ -6490,8 +6517,8 @@ StmtResult Sema::ActOnOpenMPExecutableDirective(
     }
     // Build expressions for implicit maps of data members with 'default'
     // mappers.
-    if (LangOpts.OpenMP >= 50)
-      processImplicitMapsWithDefaultMappers(*this, DSAStack,
+    if (getLangOpts().OpenMP >= 50)
+      processImplicitMapsWithDefaultMappers(SemaRef, DSAStack,
                                             ClausesWithImplicit);
   }
 
@@ -6505,7 +6532,7 @@ StmtResult Sema::ActOnOpenMPExecutableDirective(
   case OMPD_simd:
     Res = ActOnOpenMPSimdDirective(ClausesWithImplicit, AStmt, StartLoc, EndLoc,
                                    VarsWithInheritedDSA);
-    if (LangOpts.OpenMP >= 50)
+    if (getLangOpts().OpenMP >= 50)
       AllowedNameModifiers.push_back(OMPD_simd);
     break;
   case OMPD_tile:
@@ -6523,7 +6550,7 @@ StmtResult Sema::ActOnOpenMPExecutableDirective(
   case OMPD_for_simd:
     Res = ActOnOpenMPForSimdDirective(ClausesWithImplicit, AStmt, StartLoc,
                                       EndLoc, VarsWithInheritedDSA);
-    if (LangOpts.OpenMP >= 50)
+    if (getLangOpts().OpenMP >= 50)
       AllowedNameModifiers.push_back(OMPD_simd);
     break;
   case OMPD_sections:
@@ -6561,7 +6588,7 @@ StmtResult Sema::ActOnOpenMPExecutableDirective(
     Res = ActOnOpenMPParallelForSimdDirective(
         ClausesWithImplicit, AStmt, StartLoc, EndLoc, VarsWithInheritedDSA);
     AllowedNameModifiers.push_back(OMPD_parallel);
-    if (LangOpts.OpenMP >= 50)
+    if (getLangOpts().OpenMP >= 50)
       AllowedNameModifiers.push_back(OMPD_simd);
     break;
   case OMPD_scope:
@@ -6698,7 +6725,7 @@ StmtResult Sema::ActOnOpenMPExecutableDirective(
     Res = ActOnOpenMPTaskLoopSimdDirective(ClausesWithImplicit, AStmt, StartLoc,
                                            EndLoc, VarsWithInheritedDSA);
     AllowedNameModifiers.push_back(OMPD_taskloop);
-    if (LangOpts.OpenMP >= 50)
+    if (getLangOpts().OpenMP >= 50)
       AllowedNameModifiers.push_back(OMPD_simd);
     break;
   case OMPD_master_taskloop:
@@ -6715,13 +6742,13 @@ StmtResult Sema::ActOnOpenMPExecutableDirective(
     Res = ActOnOpenMPMasterTaskLoopSimdDirective(
         ClausesWithImplicit, AStmt, StartLoc, EndLoc, VarsWithInheritedDSA);
     AllowedNameModifiers.push_back(OMPD_taskloop);
-    if (LangOpts.OpenMP >= 50)
+    if (getLangOpts().OpenMP >= 50)
       AllowedNameModifiers.push_back(OMPD_simd);
     break;
   case OMPD_masked_taskloop_simd:
     Res = ActOnOpenMPMaskedTaskLoopSimdDirective(
         ClausesWithImplicit, AStmt, StartLoc, EndLoc, VarsWithInheritedDSA);
-    if (LangOpts.OpenMP >= 51) {
+    if (getLangOpts().OpenMP >= 51) {
       AllowedNameModifiers.push_back(OMPD_taskloop);
       AllowedNameModifiers.push_back(OMPD_simd);
     }
@@ -6735,7 +6762,7 @@ StmtResult Sema::ActOnOpenMPExecutableDirective(
   case OMPD_parallel_masked_taskloop:
     Res = ActOnOpenMPParallelMaskedTaskLoopDirective(
         ClausesWithImplicit, AStmt, StartLoc, EndLoc, VarsWithInheritedDSA);
-    if (LangOpts.OpenMP >= 51) {
+    if (getLangOpts().OpenMP >= 51) {
       AllowedNameModifiers.push_back(OMPD_taskloop);
       AllowedNameModifiers.push_back(OMPD_parallel);
     }
@@ -6745,13 +6772,13 @@ StmtResult Sema::ActOnOpenMPExecutableDirective(
         ClausesWithImplicit, AStmt, StartLoc, EndLoc, VarsWithInheritedDSA);
     AllowedNameModifiers.push_back(OMPD_taskloop);
     AllowedNameModifiers.push_back(OMPD_parallel);
-    if (LangOpts.OpenMP >= 50)
+    if (getLangOpts().OpenMP >= 50)
       AllowedNameModifiers.push_back(OMPD_simd);
     break;
   case OMPD_parallel_masked_taskloop_simd:
     Res = ActOnOpenMPParallelMaskedTaskLoopSimdDirective(
         ClausesWithImplicit, AStmt, StartLoc, EndLoc, VarsWithInheritedDSA);
-    if (LangOpts.OpenMP >= 51) {
+    if (getLangOpts().OpenMP >= 51) {
       AllowedNameModifiers.push_back(OMPD_taskloop);
       AllowedNameModifiers.push_back(OMPD_parallel);
       AllowedNameModifiers.push_back(OMPD_simd);
@@ -6775,13 +6802,13 @@ StmtResult Sema::ActOnOpenMPExecutableDirective(
     Res = ActOnOpenMPDistributeParallelForSimdDirective(
         ClausesWithImplicit, AStmt, StartLoc, EndLoc, VarsWithInheritedDSA);
     AllowedNameModifiers.push_back(OMPD_parallel);
-    if (LangOpts.OpenMP >= 50)
+    if (getLangOpts().OpenMP >= 50)
       AllowedNameModifiers.push_back(OMPD_simd);
     break;
   case OMPD_distribute_simd:
     Res = ActOnOpenMPDistributeSimdDirective(
         ClausesWithImplicit, AStmt, StartLoc, EndLoc, VarsWithInheritedDSA);
-    if (LangOpts.OpenMP >= 50)
+    if (getLangOpts().OpenMP >= 50)
       AllowedNameModifiers.push_back(OMPD_simd);
     break;
   case OMPD_target_parallel_for_simd:
@@ -6789,14 +6816,14 @@ StmtResult Sema::ActOnOpenMPExecutableDirective(
         ClausesWithImplicit, AStmt, StartLoc, EndLoc, VarsWithInheritedDSA);
     AllowedNameModifiers.push_back(OMPD_target);
     AllowedNameModifiers.push_back(OMPD_parallel);
-    if (LangOpts.OpenMP >= 50)
+    if (getLangOpts().OpenMP >= 50)
       AllowedNameModifiers.push_back(OMPD_simd);
     break;
   case OMPD_target_simd:
     Res = ActOnOpenMPTargetSimdDirective(ClausesWithImplicit, AStmt, StartLoc,
                                          EndLoc, VarsWithInheritedDSA);
     AllowedNameModifiers.push_back(OMPD_target);
-    if (LangOpts.OpenMP >= 50)
+    if (getLangOpts().OpenMP >= 50)
       AllowedNameModifiers.push_back(OMPD_simd);
     break;
   case OMPD_teams_distribute:
@@ -6806,14 +6833,14 @@ StmtResult Sema::ActOnOpenMPExecutableDirective(
   case OMPD_teams_distribute_simd:
     Res = ActOnOpenMPTeamsDistributeSimdDirective(
         ClausesWithImplicit, AStmt, StartLoc, EndLoc, VarsWithInheritedDSA);
-    if (LangOpts.OpenMP >= 50)
+    if (getLangOpts().OpenMP >= 50)
       AllowedNameModifiers.push_back(OMPD_simd);
     break;
   case OMPD_teams_distribute_parallel_for_simd:
     Res = ActOnOpenMPTeamsDistributeParallelForSimdDirective(
         ClausesWithImplicit, AStmt, StartLoc, EndLoc, VarsWithInheritedDSA);
     AllowedNameModifiers.push_back(OMPD_parallel);
-    if (LangOpts.OpenMP >= 50)
+    if (getLangOpts().OpenMP >= 50)
       AllowedNameModifiers.push_back(OMPD_simd);
     break;
   case OMPD_teams_distribute_parallel_for:
@@ -6842,14 +6869,14 @@ StmtResult Sema::ActOnOpenMPExecutableDirective(
         ClausesWithImplicit, AStmt, StartLoc, EndLoc, VarsWithInheritedDSA);
     AllowedNameModifiers.push_back(OMPD_target);
     AllowedNameModifiers.push_back(OMPD_parallel);
-    if (LangOpts.OpenMP >= 50)
+    if (getLangOpts().OpenMP >= 50)
       AllowedNameModifiers.push_back(OMPD_simd);
     break;
   case OMPD_target_teams_distribute_simd:
     Res = ActOnOpenMPTargetTeamsDistributeSimdDirective(
         ClausesWithImplicit, AStmt, StartLoc, EndLoc, VarsWithInheritedDSA);
     AllowedNameModifiers.push_back(OMPD_target);
-    if (LangOpts.OpenMP >= 50)
+    if (getLangOpts().OpenMP >= 50)
       AllowedNameModifiers.push_back(OMPD_simd);
     break;
   case OMPD_interop:
@@ -6906,7 +6933,7 @@ StmtResult Sema::ActOnOpenMPExecutableDirective(
   if (DSAStack->getDefaultDSA() == DSA_none ||
       DSAStack->getDefaultDSA() == DSA_private ||
       DSAStack->getDefaultDSA() == DSA_firstprivate) {
-    DSAAttrChecker DSAChecker(DSAStack, *this, nullptr);
+    DSAAttrChecker DSAChecker(DSAStack, SemaRef, nullptr);
     for (OMPClause *C : Clauses) {
       switch (C->getClauseKind()) {
       case OMPC_num_threads:
@@ -7043,13 +7070,13 @@ StmtResult Sema::ActOnOpenMPExecutableDirective(
   }
 
   if (!AllowedNameModifiers.empty())
-    ErrorFound = checkIfClauses(*this, Kind, Clauses, AllowedNameModifiers) ||
+    ErrorFound = checkIfClauses(SemaRef, Kind, Clauses, AllowedNameModifiers) ||
                  ErrorFound;
 
   if (ErrorFound)
     return StmtError();
 
-  if (!CurContext->isDependentContext() &&
+  if (!SemaRef.CurContext->isDependentContext() &&
       isOpenMPTargetExecutionDirective(Kind) &&
       !(DSAStack->hasRequiresDeclWithClause<OMPUnifiedSharedMemoryClause>() ||
         DSAStack->hasRequiresDeclWithClause<OMPUnifiedAddressClause>() ||
@@ -7062,7 +7089,7 @@ StmtResult Sema::ActOnOpenMPExecutableDirective(
   return Res;
 }
 
-Sema::DeclGroupPtrTy Sema::ActOnOpenMPDeclareSimdDirective(
+SemaOpenMP::DeclGroupPtrTy SemaOpenMP::ActOnOpenMPDeclareSimdDirective(
     DeclGroupPtrTy DG, OMPDeclareSimdDeclAttr::BranchStateTy BS, Expr *Simdlen,
     ArrayRef<Expr *> Uniforms, ArrayRef<Expr *> Aligneds,
     ArrayRef<Expr *> Alignments, ArrayRef<Expr *> Linears,
@@ -7297,13 +7324,15 @@ Sema::DeclGroupPtrTy Sema::ActOnOpenMPDeclareSimdDirective(
       NewStep = PerformOpenMPImplicitIntegerConversion(Step->getExprLoc(), Step)
                     .get();
       if (NewStep)
-        NewStep =
-            VerifyIntegerConstantExpression(NewStep, /*FIXME*/ AllowFold).get();
+        NewStep = SemaRef
+                      .VerifyIntegerConstantExpression(
+                          NewStep, /*FIXME*/ Sema::AllowFold)
+                      .get();
     }
     NewSteps.push_back(NewStep);
   }
   auto *NewAttr = OMPDeclareSimdDeclAttr::CreateImplicit(
-      Context, BS, SL.get(), const_cast<Expr **>(Uniforms.data()),
+      getASTContext(), BS, SL.get(), const_cast<Expr **>(Uniforms.data()),
       Uniforms.size(), const_cast<Expr **>(Aligneds.data()), Aligneds.size(),
       const_cast<Expr **>(NewAligns.data()), NewAligns.size(),
       const_cast<Expr **>(Linears.data()), Linears.size(),
@@ -7336,7 +7365,7 @@ static void setPrototype(Sema &S, FunctionDecl *FD, FunctionDecl *FDWithProto,
   FD->setParams(Params);
 }
 
-void Sema::ActOnFinishedFunctionDefinitionInOpenMPAssumeScope(Decl *D) {
+void SemaOpenMP::ActOnFinishedFunctionDefinitionInOpenMPAssumeScope(Decl *D) {
   if (D->isInvalidDecl())
     return;
   FunctionDecl *FD = nullptr;
@@ -7349,7 +7378,7 @@ void Sema::ActOnFinishedFunctionDefinitionInOpenMPAssumeScope(Decl *D) {
   // If we are instantiating templates we do *not* apply scoped assumptions but
   // only global ones. We apply scoped assumption to the template definition
   // though.
-  if (!inTemplateInstantiation()) {
+  if (!SemaRef.inTemplateInstantiation()) {
     for (OMPAssumeAttr *AA : OMPAssumeScoped)
       FD->addAttr(AA);
   }
@@ -7357,10 +7386,10 @@ void Sema::ActOnFinishedFunctionDefinitionInOpenMPAssumeScope(Decl *D) {
     FD->addAttr(AA);
 }
 
-Sema::OMPDeclareVariantScope::OMPDeclareVariantScope(OMPTraitInfo &TI)
+SemaOpenMP::OMPDeclareVariantScope::OMPDeclareVariantScope(OMPTraitInfo &TI)
     : TI(&TI), NameSuffix(TI.getMangledName()) {}
 
-void Sema::ActOnStartOfFunctionDefinitionInOpenMPDeclareVariantScope(
+void SemaOpenMP::ActOnStartOfFunctionDefinitionInOpenMPDeclareVariantScope(
     Scope *S, Declarator &D, MultiTemplateParamsArg TemplateParamLists,
     SmallVectorImpl<FunctionDecl *> &Bases) {
   if (!D.getIdentifier())
@@ -7376,11 +7405,11 @@ void Sema::ActOnStartOfFunctionDefinitionInOpenMPDeclareVariantScope(
     return;
 
   const IdentifierInfo *BaseII = D.getIdentifier();
-  LookupResult Lookup(*this, DeclarationName(BaseII), D.getIdentifierLoc(),
-                      LookupOrdinaryName);
-  LookupParsedName(Lookup, S, &D.getCXXScopeSpec());
+  LookupResult Lookup(SemaRef, DeclarationName(BaseII), D.getIdentifierLoc(),
+                      Sema::LookupOrdinaryName);
+  SemaRef.LookupParsedName(Lookup, S, &D.getCXXScopeSpec());
 
-  TypeSourceInfo *TInfo = GetTypeForDeclarator(D);
+  TypeSourceInfo *TInfo = SemaRef.GetTypeForDeclarator(D);
   QualType FType = TInfo->getType();
 
   bool IsConstexpr =
@@ -7409,7 +7438,7 @@ void Sema::ActOnStartOfFunctionDefinitionInOpenMPDeclareVariantScope(
 
     QualType UDeclTy = UDecl->getType();
     if (!UDeclTy->isDependentType()) {
-      QualType NewType = Context.mergeFunctionTypes(
+      QualType NewType = getASTContext().mergeFunctionTypes(
           FType, UDeclTy, /* OfBlockPointer */ false,
           /* Unqualified */ false, /* AllowCXX */ true);
       if (NewType.isNull())
@@ -7425,7 +7454,7 @@ void Sema::ActOnStartOfFunctionDefinitionInOpenMPDeclareVariantScope(
   // If no base was found we create a declaration that we use as base.
   if (Bases.empty() && UseImplicitBase) {
     D.setFunctionDefinitionKind(FunctionDefinitionKind::Declaration);
-    Decl *BaseD = HandleDeclarator(S, D, TemplateParamLists);
+    Decl *BaseD = SemaRef.HandleDeclarator(S, D, TemplateParamLists);
     BaseD->setImplicit(true);
     if (auto *BaseTemplD = dyn_cast<FunctionTemplateDecl>(BaseD))
       Bases.push_back(BaseTemplD->getTemplatedDecl());
@@ -7437,18 +7466,18 @@ void Sema::ActOnStartOfFunctionDefinitionInOpenMPDeclareVariantScope(
   MangledName += D.getIdentifier()->getName();
   MangledName += getOpenMPVariantManglingSeparatorStr();
   MangledName += DVScope.NameSuffix;
-  IdentifierInfo &VariantII = Context.Idents.get(MangledName);
+  IdentifierInfo &VariantII = getASTContext().Idents.get(MangledName);
 
   VariantII.setMangledOpenMPVariantName(true);
   D.SetIdentifier(&VariantII, D.getBeginLoc());
 }
 
-void Sema::ActOnFinishedFunctionDefinitionInOpenMPDeclareVariantScope(
+void SemaOpenMP::ActOnFinishedFunctionDefinitionInOpenMPDeclareVariantScope(
     Decl *D, SmallVectorImpl<FunctionDecl *> &Bases) {
   // Do not mark function as is used to prevent its emission if this is the
   // only place where it is used.
   EnterExpressionEvaluationContext Unevaluated(
-      *this, Sema::ExpressionEvaluationContext::Unevaluated);
+      SemaRef, Sema::ExpressionEvaluationContext::Unevaluated);
 
   FunctionDecl *FD = nullptr;
   if (auto *UTemplDecl = dyn_cast<FunctionTemplateDecl>(D))
@@ -7456,14 +7485,14 @@ void Sema::ActOnFinishedFunctionDefinitionInOpenMPDeclareVariantScope(
   else
     FD = cast<FunctionDecl>(D);
   auto *VariantFuncRef = DeclRefExpr::Create(
-      Context, NestedNameSpecifierLoc(), SourceLocation(), FD,
+      getASTContext(), NestedNameSpecifierLoc(), SourceLocation(), FD,
       /* RefersToEnclosingVariableOrCapture */ false,
       /* NameLoc */ FD->getLocation(), FD->getType(),
       ExprValueKind::VK_PRValue);
 
   OMPDeclareVariantScope &DVScope = OMPDeclareVariantScopes.back();
   auto *OMPDeclareVariantA = OMPDeclareVariantAttr::CreateImplicit(
-      Context, VariantFuncRef, DVScope.TI,
+      getASTContext(), VariantFuncRef, DVScope.TI,
       /*NothingArgs=*/nullptr, /*NothingArgsSize=*/0,
       /*NeedDevicePtrArgs=*/nullptr, /*NeedDevicePtrArgsSize=*/0,
       /*AppendArgs=*/nullptr, /*AppendArgsSize=*/0);
@@ -7471,10 +7500,11 @@ void Sema::ActOnFinishedFunctionDefinitionInOpenMPDeclareVariantScope(
     BaseFD->addAttr(OMPDeclareVariantA);
 }
 
-ExprResult Sema::ActOnOpenMPCall(ExprResult Call, Scope *Scope,
-                                 SourceLocation LParenLoc,
-                                 MultiExprArg ArgExprs,
-                                 SourceLocation RParenLoc, Expr *ExecConfig) {
+ExprResult SemaOpenMP::ActOnOpenMPCall(ExprResult Call, Scope *Scope,
+                                       SourceLocation LParenLoc,
+                                       MultiExprArg ArgExprs,
+                                       SourceLocation RParenLoc,
+                                       Expr *ExecConfig) {
   // The common case is a regular call we do not want to specialize at all. Try
   // to make that case fast by bailing early.
   CallExpr *CE = dyn_cast<CallExpr>(Call.get());
@@ -7485,7 +7515,7 @@ ExprResult Sema::ActOnOpenMPCall(ExprResult Call, Scope *Scope,
   if (!CalleeFnDecl)
     return Call;
 
-  if (LangOpts.OpenMP >= 51 && CalleeFnDecl->getIdentifier() &&
+  if (getLangOpts().OpenMP >= 51 && CalleeFnDecl->getIdentifier() &&
       CalleeFnDecl->getName().starts_with_insensitive("omp_")) {
     // checking for any calls inside an Order region
     if (Scope && Scope->isOpenMPOrderClauseScope())
@@ -7504,7 +7534,8 @@ ExprResult Sema::ActOnOpenMPCall(ExprResult Call, Scope *Scope,
         << ISATrait;
   };
   TargetOMPContext OMPCtx(Context, std::move(DiagUnknownTrait),
-                          getCurFunctionDecl(), DSAStack->getConstructTraits());
+                          SemaRef.getCurFunctionDecl(),
+                          DSAStack->getConstructTraits());
 
   QualType CalleeFnType = CalleeFnDecl->getType();
 
@@ -7549,7 +7580,7 @@ ExprResult Sema::ActOnOpenMPCall(ExprResult Call, Scope *Scope,
       // different type than the base function. This is intended and OK but if
       // we cannot create a call the difference is not in the "implementation
       // defined range" we allow.
-      Sema::TentativeAnalysisScope Trap(*this);
+      Sema::TentativeAnalysisScope Trap(SemaRef);
 
       if (auto *SpecializedMethod = dyn_cast<CXXMethodDecl>(BestDecl)) {
         auto *MemberCall = dyn_cast<CXXMemberCallExpr>(CE);
@@ -7558,12 +7589,12 @@ ExprResult Sema::ActOnOpenMPCall(ExprResult Call, Scope *Scope,
             /* IsArrow */ false, SpecializedMethod, Context.BoundMemberTy,
             MemberCall->getValueKind(), MemberCall->getObjectKind());
       }
-      NewCall = BuildCallExpr(Scope, BestExpr, LParenLoc, ArgExprs, RParenLoc,
-                              ExecConfig);
+      NewCall = SemaRef.BuildCallExpr(Scope, BestExpr, LParenLoc, ArgExprs,
+                                      RParenLoc, ExecConfig);
       if (NewCall.isUsable()) {
         if (CallExpr *NCE = dyn_cast<CallExpr>(NewCall.get())) {
           FunctionDecl *NewCalleeFnDecl = NCE->getDirectCallee();
-          QualType NewType = Context.mergeFunctionTypes(
+          QualType NewType = getASTContext().mergeFunctionTypes(
               CalleeFnType, NewCalleeFnDecl->getType(),
               /* OfBlockPointer */ false,
               /* Unqualified */ false, /* AllowCXX */ true);
@@ -7581,14 +7612,16 @@ ExprResult Sema::ActOnOpenMPCall(ExprResult Call, Scope *Scope,
 
   if (!NewCall.isUsable())
     return Call;
-  return PseudoObjectExpr::Create(Context, CE, {NewCall.get()}, 0);
+  return PseudoObjectExpr::Create(getASTContext(), CE, {NewCall.get()}, 0);
 }
 
 std::optional<std::pair<FunctionDecl *, Expr *>>
-Sema::checkOpenMPDeclareVariantFunction(Sema::DeclGroupPtrTy DG,
-                                        Expr *VariantRef, OMPTraitInfo &TI,
-                                        unsigned NumAppendArgs,
-                                        SourceRange SR) {
+SemaOpenMP::checkOpenMPDeclareVariantFunction(SemaOpenMP::DeclGroupPtrTy DG,
+                                              Expr *VariantRef,
+                                              OMPTraitInfo &TI,
+                                              unsigned NumAppendArgs,
+                                              SourceRange SR) {
+  ASTContext &Context = getASTContext();
   if (!DG || DG.get().isNull())
     return std::nullopt;
 
@@ -7631,7 +7664,7 @@ Sema::checkOpenMPDeclareVariantFunction(Sema::DeclGroupPtrTy DG,
   // Check if the function was emitted already.
   const FunctionDecl *Definition;
   if (!FD->isThisDeclarationADefinition() && FD->isDefined(Definition) &&
-      (LangOpts.EmitAllDecls || Context.DeclMustBeEmitted(Definition)))
+      (getLangOpts().EmitAllDecls || Context.DeclMustBeEmitted(Definition)))
     Diag(SR.getBegin(), diag::warn_omp_declare_variant_after_emitted)
         << FD->getLocation();
 
@@ -7654,7 +7687,7 @@ Sema::checkOpenMPDeclareVariantFunction(Sema::DeclGroupPtrTy DG,
   // Deal with non-constant score and user condition expressions.
   auto HandleNonConstantScoresAndConditions = [this](Expr *&E,
                                                      bool IsScore) -> bool {
-    if (!E || E->isIntegerConstantExpr(Context))
+    if (!E || E->isIntegerConstantExpr(getASTContext()))
       return false;
 
     if (IsScore) {
@@ -7686,9 +7719,9 @@ Sema::checkOpenMPDeclareVariantFunction(Sema::DeclGroupPtrTy DG,
     // Adjust the function type to account for an extra omp_interop_t for each
     // specified in the append_args clause.
     const TypeDecl *TD = nullptr;
-    LookupResult Result(*this, &Context.Idents.get("omp_interop_t"),
+    LookupResult Result(SemaRef, &Context.Idents.get("omp_interop_t"),
                         SR.getBegin(), Sema::LookupOrdinaryName);
-    if (LookupName(Result, getCurScope())) {
+    if (SemaRef.LookupName(Result, SemaRef.getCurScope())) {
       NamedDecl *ND = Result.getFoundDecl();
       TD = dyn_cast_or_null<TypeDecl>(ND);
     }
@@ -7711,7 +7744,7 @@ Sema::checkOpenMPDeclareVariantFunction(Sema::DeclGroupPtrTy DG,
   // Convert VariantRef expression to the type of the original function to
   // resolve possible conflicts.
   ExprResult VariantRefCast = VariantRef;
-  if (LangOpts.CPlusPlus) {
+  if (getLangOpts().CPlusPlus) {
     QualType FnPtrType;
     auto *Method = dyn_cast<CXXMethodDecl>(FD);
     if (Method && !Method->isStatic()) {
@@ -7722,9 +7755,9 @@ Sema::checkOpenMPDeclareVariantFunction(Sema::DeclGroupPtrTy DG,
       {
         // Build adrr_of unary op to correctly handle type checks for member
         // functions.
-        Sema::TentativeAnalysisScope Trap(*this);
-        ER = CreateBuiltinUnaryOp(VariantRef->getBeginLoc(), UO_AddrOf,
-                                  VariantRef);
+        Sema::TentativeAnalysisScope Trap(SemaRef);
+        ER = SemaRef.CreateBuiltinUnaryOp(VariantRef->getBeginLoc(), UO_AddrOf,
+                                          VariantRef);
       }
       if (!ER.isUsable()) {
         Diag(VariantRef->getExprLoc(), diag::err_omp_function_expected)
@@ -7737,9 +7770,9 @@ Sema::checkOpenMPDeclareVariantFunction(Sema::DeclGroupPtrTy DG,
     }
     QualType VarianPtrType = Context.getPointerType(VariantRef->getType());
     if (VarianPtrType.getUnqualifiedType() != FnPtrType.getUnqualifiedType()) {
-      ImplicitConversionSequence ICS = TryImplicitConversion(
+      ImplicitConversionSequence ICS = SemaRef.TryImplicitConversion(
           VariantRef, FnPtrType.getUnqualifiedType(),
-          /*SuppressUserConversions=*/false, AllowedExplicit::None,
+          /*SuppressUserConversions=*/false, Sema::AllowedExplicit::None,
           /*InOverloadResolution=*/false,
           /*CStyle=*/false,
           /*AllowObjCWritebackConversion=*/false);
@@ -7751,8 +7784,8 @@ Sema::checkOpenMPDeclareVariantFunction(Sema::DeclGroupPtrTy DG,
             << (NumAppendArgs ? 1 : 0) << VariantRef->getSourceRange();
         return std::nullopt;
       }
-      VariantRefCast = PerformImplicitConversion(
-          VariantRef, FnPtrType.getUnqualifiedType(), AA_Converting);
+      VariantRefCast = SemaRef.PerformImplicitConversion(
+          VariantRef, FnPtrType.getUnqualifiedType(), Sema::AA_Converting);
       if (!VariantRefCast.isUsable())
         return std::nullopt;
     }
@@ -7765,7 +7798,7 @@ Sema::checkOpenMPDeclareVariantFunction(Sema::DeclGroupPtrTy DG,
     }
   }
 
-  ExprResult ER = CheckPlaceholderExpr(VariantRefCast.get());
+  ExprResult ER = SemaRef.CheckPlaceholderExpr(VariantRefCast.get());
   if (!ER.isUsable() ||
       !ER.get()->IgnoreParenImpCasts()->getType()->isFunctionType()) {
     Diag(VariantRef->getExprLoc(), diag::err_omp_function_expected)
@@ -7795,7 +7828,7 @@ Sema::checkOpenMPDeclareVariantFunction(Sema::DeclGroupPtrTy DG,
   }
 
   // Check if function types are compatible in C.
-  if (!LangOpts.CPlusPlus) {
+  if (!getLangOpts().CPlusPlus) {
     QualType NewType =
         Context.mergeFunctionTypes(AdjustedFnType, NewFD->getType());
     if (NewType.isNull()) {
@@ -7807,9 +7840,9 @@ Sema::checkOpenMPDeclareVariantFunction(Sema::DeclGroupPtrTy DG,
     }
     if (NewType->isFunctionProtoType()) {
       if (FD->getType()->isFunctionNoProtoType())
-        setPrototype(*this, FD, NewFD, NewType);
+        setPrototype(SemaRef, FD, NewFD, NewType);
       else if (NewFD->getType()->isFunctionNoProtoType())
-        setPrototype(*this, NewFD, FD, NewType);
+        setPrototype(SemaRef, NewFD, FD, NewType);
     }
   }
 
@@ -7872,15 +7905,15 @@ Sema::checkOpenMPDeclareVariantFunction(Sema::DeclGroupPtrTy DG,
   }
 
   // Check general compatibility.
-  if (areMultiversionVariantFunctionsCompatible(
+  if (SemaRef.areMultiversionVariantFunctionsCompatible(
           FD, NewFD, PartialDiagnostic::NullDiagnostic(),
           PartialDiagnosticAt(SourceLocation(),
                               PartialDiagnostic::NullDiagnostic()),
           PartialDiagnosticAt(
               VariantRef->getExprLoc(),
-              PDiag(diag::err_omp_declare_variant_doesnt_support)),
+              SemaRef.PDiag(diag::err_omp_declare_variant_doesnt_support)),
           PartialDiagnosticAt(VariantRef->getExprLoc(),
-                              PDiag(diag::err_omp_declare_variant_diff)
+                              SemaRef.PDiag(diag::err_omp_declare_variant_diff)
                                   << FD->getLocation()),
           /*TemplatesSupported=*/true, /*ConstexprSupported=*/false,
           /*CLinkageMayDiffer=*/true))
@@ -7888,7 +7921,7 @@ Sema::checkOpenMPDeclareVariantFunction(Sema::DeclGroupPtrTy DG,
   return std::make_pair(FD, cast<Expr>(DRE));
 }
 
-void Sema::ActOnOpenMPDeclareVariantDirective(
+void SemaOpenMP::ActOnOpenMPDeclareVariantDirective(
     FunctionDecl *FD, Expr *VariantRef, OMPTraitInfo &TI,
     ArrayRef<Expr *> AdjustArgsNothing,
     ArrayRef<Expr *> AdjustArgsNeedDevicePtr,
@@ -7906,7 +7939,7 @@ void Sema::ActOnOpenMPDeclareVariantDirective(
 
   if (!AllAdjustArgs.empty() || !AppendArgs.empty()) {
     VariantMatchInfo VMI;
-    TI.getAsVariantMatchInfo(Context, VMI);
+    TI.getAsVariantMatchInfo(getASTContext(), VMI);
     if (!llvm::is_contained(
             VMI.ConstructTraits,
             llvm::omp::TraitProperty::construct_dispatch_dispatch)) {
@@ -7949,18 +7982,18 @@ void Sema::ActOnOpenMPDeclareVariantDirective(
   }
 
   auto *NewAttr = OMPDeclareVariantAttr::CreateImplicit(
-      Context, VariantRef, &TI, const_cast<Expr **>(AdjustArgsNothing.data()),
-      AdjustArgsNothing.size(),
+      getASTContext(), VariantRef, &TI,
+      const_cast<Expr **>(AdjustArgsNothing.data()), AdjustArgsNothing.size(),
       const_cast<Expr **>(AdjustArgsNeedDevicePtr.data()),
       AdjustArgsNeedDevicePtr.size(),
       const_cast<OMPInteropInfo *>(AppendArgs.data()), AppendArgs.size(), SR);
   FD->addAttr(NewAttr);
 }
 
-StmtResult Sema::ActOnOpenMPParallelDirective(ArrayRef<OMPClause *> Clauses,
-                                              Stmt *AStmt,
-                                              SourceLocation StartLoc,
-                                              SourceLocation EndLoc) {
+StmtResult
+SemaOpenMP::ActOnOpenMPParallelDirective(ArrayRef<OMPClause *> Clauses,
+                                         Stmt *AStmt, SourceLocation StartLoc,
+                                         SourceLocation EndLoc) {
   if (!AStmt)
     return StmtError();
 
@@ -7972,11 +8005,11 @@ StmtResult Sema::ActOnOpenMPParallelDirective(ArrayRef<OMPClause *> Clauses,
   // longjmp() and throw() must not violate the entry/exit criteria.
   CS->getCapturedDecl()->setNothrow();
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
 
-  return OMPParallelDirective::Create(Context, StartLoc, EndLoc, Clauses, AStmt,
-                                      DSAStack->getTaskgroupReductionRef(),
-                                      DSAStack->isCancelRegion());
+  return OMPParallelDirective::Create(
+      getASTContext(), StartLoc, EndLoc, Clauses, AStmt,
+      DSAStack->getTaskgroupReductionRef(), DSAStack->isCancelRegion());
 }
 
 namespace {
@@ -8226,7 +8259,7 @@ bool OpenMPIterationSpaceChecker::setStep(Expr *NewStep, bool Subtract) {
   if (!NewStep->isValueDependent()) {
     // Check that the step is integer expression.
     SourceLocation StepLoc = NewStep->getBeginLoc();
-    ExprResult Val = SemaRef.PerformOpenMPImplicitIntegerConversion(
+    ExprResult Val = SemaRef.OpenMP().PerformOpenMPImplicitIntegerConversion(
         StepLoc, getExprAsWritten(NewStep));
     if (Val.isInvalid())
       return true;
@@ -9248,7 +9281,7 @@ DeclRefExpr *OpenMPIterationSpaceChecker::buildCounterVar(
     DSAStackTy &DSA) const {
   auto *VD = dyn_cast<VarDecl>(LCDecl);
   if (!VD) {
-    VD = SemaRef.isOpenMPCapturedDecl(LCDecl);
+    VD = SemaRef.OpenMP().isOpenMPCapturedDecl(LCDecl);
     DeclRefExpr *Ref = buildDeclRefExpr(
         SemaRef, VD, VD->getType().getNonReferenceType(), DefaultLoc);
     const DSAStackTy::DSAVarData Data =
@@ -9321,14 +9354,15 @@ Expr *OpenMPIterationSpaceChecker::buildOrderedLoopData(
 }
 } // namespace
 
-void Sema::ActOnOpenMPLoopInitialization(SourceLocation ForLoc, Stmt *Init) {
+void SemaOpenMP::ActOnOpenMPLoopInitialization(SourceLocation ForLoc,
+                                               Stmt *Init) {
   assert(getLangOpts().OpenMP && "OpenMP is not active.");
   assert(Init && "Expected loop in canonical form.");
   unsigned AssociatedLoops = DSAStack->getAssociatedLoops();
   if (AssociatedLoops > 0 &&
       isOpenMPLoopDirective(DSAStack->getCurrentDirective())) {
     DSAStack->loopStart();
-    OpenMPIterationSpaceChecker ISC(*this, /*SupportsNonRectangular=*/true,
+    OpenMPIterationSpaceChecker ISC(SemaRef, /*SupportsNonRectangular=*/true,
                                     *DSAStack, ForLoc);
     if (!ISC.checkAndSetInit(Init, /*EmitDiags=*/false)) {
       if (ValueDecl *D = ISC.getLoopDecl()) {
@@ -9338,7 +9372,7 @@ void Sema::ActOnOpenMPLoopInitialization(SourceLocation ForLoc, Stmt *Init) {
           if (VarDecl *Private = isOpenMPCapturedDecl(D)) {
             VD = Private;
           } else {
-            PrivateRef = buildCapture(*this, D, ISC.getLoopDeclRefExpr(),
+            PrivateRef = buildCapture(SemaRef, D, ISC.getLoopDeclRefExpr(),
                                       /*WithInit=*/false);
             VD = cast<VarDecl>(PrivateRef->getDecl());
           }
@@ -9348,10 +9382,10 @@ void Sema::ActOnOpenMPLoopInitialization(SourceLocation ForLoc, Stmt *Init) {
         if (LD != D->getCanonicalDecl()) {
           DSAStack->resetPossibleLoopCounter();
           if (auto *Var = dyn_cast_or_null<VarDecl>(LD))
-            MarkDeclarationsReferencedInExpr(
-                buildDeclRefExpr(*this, const_cast<VarDecl *>(Var),
-                                 Var->getType().getNonLValueExprType(Context),
-                                 ForLoc, /*RefersToCapture=*/true));
+            SemaRef.MarkDeclarationsReferencedInExpr(buildDeclRefExpr(
+                SemaRef, const_cast<VarDecl *>(Var),
+                Var->getType().getNonLValueExprType(getASTContext()), ForLoc,
+                /*RefersToCapture=*/true));
         }
         OpenMPDirectiveKind DKind = DSAStack->getCurrentDirective();
         // OpenMP [2.14.1.1, Data-sharing Attribute Rules for Variables
@@ -9372,8 +9406,8 @@ void Sema::ActOnOpenMPLoopInitialization(SourceLocation ForLoc, Stmt *Init) {
                 : OMPC_private;
         if (((isOpenMPSimdDirective(DKind) && DVar.CKind != OMPC_unknown &&
               DVar.CKind != PredeterminedCKind && DVar.RefExpr &&
-              (LangOpts.OpenMP <= 45 || (DVar.CKind != OMPC_lastprivate &&
-                                         DVar.CKind != OMPC_private))) ||
+              (getLangOpts().OpenMP <= 45 || (DVar.CKind != OMPC_lastprivate &&
+                                              DVar.CKind != OMPC_private))) ||
              ((isOpenMPWorksharingDirective(DKind) || DKind == OMPD_taskloop ||
                DKind == OMPD_master_taskloop || DKind == OMPD_masked_taskloop ||
                DKind == OMPD_parallel_master_taskloop ||
@@ -9388,7 +9422,7 @@ void Sema::ActOnOpenMPLoopInitialization(SourceLocation ForLoc, Stmt *Init) {
               << getOpenMPClauseName(PredeterminedCKind);
           if (DVar.RefExpr == nullptr)
             DVar.CKind = PredeterminedCKind;
-          reportOriginalDsa(*this, DSAStack, D, DVar,
+          reportOriginalDsa(SemaRef, DSAStack, D, DVar,
                             /*IsLoopIterVar=*/true);
         } else if (LoopDeclRefExpr) {
           // Make the loop iteration variable private (for worksharing
@@ -9428,7 +9462,7 @@ static bool checkOpenMPIterationSpace(
     unsigned CurrentNestedLoopCount, unsigned NestedLoopCount,
     unsigned TotalNestedLoopCount, Expr *CollapseLoopCountExpr,
     Expr *OrderedLoopCountExpr,
-    Sema::VarsWithInheritedDSAType &VarsWithImplicitDSA,
+    SemaOpenMP::VarsWithInheritedDSAType &VarsWithImplicitDSA,
     llvm::MutableArrayRef<LoopIterationSpace> ResultIterSpaces,
     llvm::MapVector<const Expr *, DeclRefExpr *> &Captures) {
   bool SupportsNonRectangular = !isOpenMPLoopTransformationDirective(DKind);
@@ -9817,7 +9851,7 @@ static unsigned
 checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
                 Expr *OrderedLoopCountExpr, Stmt *AStmt, Sema &SemaRef,
                 DSAStackTy &DSA,
-                Sema::VarsWithInheritedDSAType &VarsWithImplicitDSA,
+                SemaOpenMP::VarsWithInheritedDSAType &VarsWithImplicitDSA,
                 OMPLoopBasedDirective::HelperExprs &Built) {
   unsigned NestedLoopCount = 1;
   bool SupportsNonPerfectlyNested = (SemaRef.LangOpts.OpenMP >= 50) &&
@@ -10566,7 +10600,8 @@ static bool checkGenericLoopLastprivate(Sema &S, ArrayRef<OMPClause *> Clauses,
                                         OpenMPDirectiveKind K,
                                         DSAStackTy *Stack);
 
-bool Sema::checkLastPrivateForMappedDirectives(ArrayRef<OMPClause *> Clauses) {
+bool SemaOpenMP::checkLastPrivateForMappedDirectives(
+    ArrayRef<OMPClause *> Clauses) {
 
   // Check for syntax of lastprivate
   // Param of the lastprivate have different meanings in the mapped directives
@@ -10574,16 +10609,15 @@ bool Sema::checkLastPrivateForMappedDirectives(ArrayRef<OMPClause *> Clauses) {
   //      "omp for"  lastprivate vars must be shared
   if (getLangOpts().OpenMP >= 50 &&
       DSAStack->getMappedDirective() == OMPD_loop &&
-      checkGenericLoopLastprivate(*this, Clauses, OMPD_loop, DSAStack)) {
+      checkGenericLoopLastprivate(SemaRef, Clauses, OMPD_loop, DSAStack)) {
     return false;
   }
   return true;
 }
 
-StmtResult
-Sema::ActOnOpenMPSimdDirective(ArrayRef<OMPClause *> Clauses, Stmt *AStmt,
-                               SourceLocation StartLoc, SourceLocation EndLoc,
-                               VarsWithInheritedDSAType &VarsWithImplicitDSA) {
+StmtResult SemaOpenMP::ActOnOpenMPSimdDirective(
+    ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+    SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
     return StmtError();
 
@@ -10596,38 +10630,37 @@ Sema::ActOnOpenMPSimdDirective(ArrayRef<OMPClause *> Clauses, Stmt *AStmt,
   // define the nested loops number.
   unsigned NestedLoopCount = checkOpenMPLoop(
       OMPD_simd, getCollapseNumberExpr(Clauses), getOrderedNumberExpr(Clauses),
-      AStmt, *this, *DSAStack, VarsWithImplicitDSA, B);
+      AStmt, SemaRef, *DSAStack, VarsWithImplicitDSA, B);
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp simd loop exprs were not built");
 
-  if (!CurContext->isDependentContext()) {
+  if (!SemaRef.CurContext->isDependentContext()) {
     // Finalize the clauses that need pre-built expressions for CodeGen.
     for (OMPClause *C : Clauses) {
       if (auto *LC = dyn_cast<OMPLinearClause>(C))
         if (FinishOpenMPLinearClause(*LC, cast<DeclRefExpr>(B.IterationVarRef),
-                                     B.NumIterations, *this, CurScope,
-                                     DSAStack))
+                                     B.NumIterations, SemaRef,
+                                     SemaRef.getCurScope(), DSAStack))
           return StmtError();
     }
   }
 
-  if (checkSimdlenSafelenSpecified(*this, Clauses))
+  if (checkSimdlenSafelenSpecified(SemaRef, Clauses))
     return StmtError();
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
   auto *SimdDirective = OMPSimdDirective::Create(
-      Context, StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B,
+      getASTContext(), StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B,
       DSAStack->getMappedDirective());
   return SimdDirective;
 }
 
-StmtResult
-Sema::ActOnOpenMPForDirective(ArrayRef<OMPClause *> Clauses, Stmt *AStmt,
-                              SourceLocation StartLoc, SourceLocation EndLoc,
-                              VarsWithInheritedDSAType &VarsWithImplicitDSA) {
+StmtResult SemaOpenMP::ActOnOpenMPForDirective(
+    ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+    SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
     return StmtError();
 
@@ -10640,32 +10673,32 @@ Sema::ActOnOpenMPForDirective(ArrayRef<OMPClause *> Clauses, Stmt *AStmt,
   // define the nested loops number.
   unsigned NestedLoopCount = checkOpenMPLoop(
       OMPD_for, getCollapseNumberExpr(Clauses), getOrderedNumberExpr(Clauses),
-      AStmt, *this, *DSAStack, VarsWithImplicitDSA, B);
+      AStmt, SemaRef, *DSAStack, VarsWithImplicitDSA, B);
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp for loop exprs were not built");
 
-  if (!CurContext->isDependentContext()) {
+  if (!SemaRef.CurContext->isDependentContext()) {
     // Finalize the clauses that need pre-built expressions for CodeGen.
     for (OMPClause *C : Clauses) {
       if (auto *LC = dyn_cast<OMPLinearClause>(C))
         if (FinishOpenMPLinearClause(*LC, cast<DeclRefExpr>(B.IterationVarRef),
-                                     B.NumIterations, *this, CurScope,
-                                     DSAStack))
+                                     B.NumIterations, SemaRef,
+                                     SemaRef.getCurScope(), DSAStack))
           return StmtError();
     }
   }
 
   auto *ForDirective = OMPForDirective::Create(
-      Context, StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B,
+      getASTContext(), StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B,
       DSAStack->getTaskgroupReductionRef(), DSAStack->isCancelRegion(),
       DSAStack->getMappedDirective());
   return ForDirective;
 }
 
-StmtResult Sema::ActOnOpenMPForSimdDirective(
+StmtResult SemaOpenMP::ActOnOpenMPForSimdDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -10677,37 +10710,37 @@ StmtResult Sema::ActOnOpenMPForSimdDirective(
   // define the nested loops number.
   unsigned NestedLoopCount =
       checkOpenMPLoop(OMPD_for_simd, getCollapseNumberExpr(Clauses),
-                      getOrderedNumberExpr(Clauses), AStmt, *this, *DSAStack,
+                      getOrderedNumberExpr(Clauses), AStmt, SemaRef, *DSAStack,
                       VarsWithImplicitDSA, B);
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp for simd loop exprs were not built");
 
-  if (!CurContext->isDependentContext()) {
+  if (!SemaRef.CurContext->isDependentContext()) {
     // Finalize the clauses that need pre-built expressions for CodeGen.
     for (OMPClause *C : Clauses) {
       if (auto *LC = dyn_cast<OMPLinearClause>(C))
         if (FinishOpenMPLinearClause(*LC, cast<DeclRefExpr>(B.IterationVarRef),
-                                     B.NumIterations, *this, CurScope,
-                                     DSAStack))
+                                     B.NumIterations, SemaRef,
+                                     SemaRef.getCurScope(), DSAStack))
           return StmtError();
     }
   }
 
-  if (checkSimdlenSafelenSpecified(*this, Clauses))
+  if (checkSimdlenSafelenSpecified(SemaRef, Clauses))
     return StmtError();
 
-  setFunctionHasBranchProtectedScope();
-  return OMPForSimdDirective::Create(Context, StartLoc, EndLoc, NestedLoopCount,
-                                     Clauses, AStmt, B);
+  SemaRef.setFunctionHasBranchProtectedScope();
+  return OMPForSimdDirective::Create(getASTContext(), StartLoc, EndLoc,
+                                     NestedLoopCount, Clauses, AStmt, B);
 }
 
-StmtResult Sema::ActOnOpenMPSectionsDirective(ArrayRef<OMPClause *> Clauses,
-                                              Stmt *AStmt,
-                                              SourceLocation StartLoc,
-                                              SourceLocation EndLoc) {
+StmtResult
+SemaOpenMP::ActOnOpenMPSectionsDirective(ArrayRef<OMPClause *> Clauses,
+                                         Stmt *AStmt, SourceLocation StartLoc,
+                                         SourceLocation EndLoc) {
   if (!AStmt)
     return StmtError();
 
@@ -10736,23 +10769,23 @@ StmtResult Sema::ActOnOpenMPSectionsDirective(ArrayRef<OMPClause *> Clauses,
     return StmtError();
   }
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
 
-  return OMPSectionsDirective::Create(Context, StartLoc, EndLoc, Clauses, AStmt,
-                                      DSAStack->getTaskgroupReductionRef(),
-                                      DSAStack->isCancelRegion());
+  return OMPSectionsDirective::Create(
+      getASTContext(), StartLoc, EndLoc, Clauses, AStmt,
+      DSAStack->getTaskgroupReductionRef(), DSAStack->isCancelRegion());
 }
 
-StmtResult Sema::ActOnOpenMPSectionDirective(Stmt *AStmt,
-                                             SourceLocation StartLoc,
-                                             SourceLocation EndLoc) {
+StmtResult SemaOpenMP::ActOnOpenMPSectionDirective(Stmt *AStmt,
+                                                   SourceLocation StartLoc,
+                                                   SourceLocation EndLoc) {
   if (!AStmt)
     return StmtError();
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
   DSAStack->setParentCancelRegion(DSAStack->isCancelRegion());
 
-  return OMPSectionDirective::Create(Context, StartLoc, EndLoc, AStmt,
+  return OMPSectionDirective::Create(getASTContext(), StartLoc, EndLoc, AStmt,
                                      DSAStack->isCancelRegion());
 }
 
@@ -10764,10 +10797,10 @@ static Expr *getDirectCallExpr(Expr *E) {
   return nullptr;
 }
 
-StmtResult Sema::ActOnOpenMPDispatchDirective(ArrayRef<OMPClause *> Clauses,
-                                              Stmt *AStmt,
-                                              SourceLocation StartLoc,
-                                              SourceLocation EndLoc) {
+StmtResult
+SemaOpenMP::ActOnOpenMPDispatchDirective(ArrayRef<OMPClause *> Clauses,
+                                         Stmt *AStmt, SourceLocation StartLoc,
+                                         SourceLocation EndLoc) {
   if (!AStmt)
     return StmtError();
 
@@ -10780,7 +10813,7 @@ StmtResult Sema::ActOnOpenMPDispatchDirective(ArrayRef<OMPClause *> Clauses,
 
   SourceLocation TargetCallLoc;
 
-  if (!CurContext->isDependentContext()) {
+  if (!SemaRef.CurContext->isDependentContext()) {
     Expr *TargetCall = nullptr;
 
     auto *E = dyn_cast<Expr>(S);
@@ -10808,10 +10841,10 @@ StmtResult Sema::ActOnOpenMPDispatchDirective(ArrayRef<OMPClause *> Clauses,
     TargetCallLoc = TargetCall->getExprLoc();
   }
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
 
-  return OMPDispatchDirective::Create(Context, StartLoc, EndLoc, Clauses, AStmt,
-                                      TargetCallLoc);
+  return OMPDispatchDirective::Create(getASTContext(), StartLoc, EndLoc,
+                                      Clauses, AStmt, TargetCallLoc);
 }
 
 static bool checkGenericLoopLastprivate(Sema &S, ArrayRef<OMPClause *> Clauses,
@@ -10839,7 +10872,7 @@ static bool checkGenericLoopLastprivate(Sema &S, ArrayRef<OMPClause *> Clauses,
   return ErrorFound;
 }
 
-StmtResult Sema::ActOnOpenMPGenericLoopDirective(
+StmtResult SemaOpenMP::ActOnOpenMPGenericLoopDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -10848,7 +10881,7 @@ StmtResult Sema::ActOnOpenMPGenericLoopDirective(
   // OpenMP 5.1 [2.11.7, loop construct, Restrictions]
   // A list item may not appear in a lastprivate clause unless it is the
   // loop iteration variable of a loop that is associated with the construct.
-  if (checkGenericLoopLastprivate(*this, Clauses, OMPD_loop, DSAStack))
+  if (checkGenericLoopLastprivate(SemaRef, Clauses, OMPD_loop, DSAStack))
     return StmtError();
 
   auto *CS = cast<CapturedStmt>(AStmt);
@@ -10863,19 +10896,19 @@ StmtResult Sema::ActOnOpenMPGenericLoopDirective(
   // In presence of clause 'collapse', it will define the nested loops number.
   unsigned NestedLoopCount = checkOpenMPLoop(
       OMPD_loop, getCollapseNumberExpr(Clauses), getOrderedNumberExpr(Clauses),
-      AStmt, *this, *DSAStack, VarsWithImplicitDSA, B);
+      AStmt, SemaRef, *DSAStack, VarsWithImplicitDSA, B);
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp loop exprs were not built");
 
-  setFunctionHasBranchProtectedScope();
-  return OMPGenericLoopDirective::Create(Context, StartLoc, EndLoc,
+  SemaRef.setFunctionHasBranchProtectedScope();
+  return OMPGenericLoopDirective::Create(getASTContext(), StartLoc, EndLoc,
                                          NestedLoopCount, Clauses, AStmt, B);
 }
 
-StmtResult Sema::ActOnOpenMPTeamsGenericLoopDirective(
+StmtResult SemaOpenMP::ActOnOpenMPTeamsGenericLoopDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -10884,7 +10917,7 @@ StmtResult Sema::ActOnOpenMPTeamsGenericLoopDirective(
   // OpenMP 5.1 [2.11.7, loop construct, Restrictions]
   // A list item may not appear in a lastprivate clause unless it is the
   // loop iteration variable of a loop that is associated with the construct.
-  if (checkGenericLoopLastprivate(*this, Clauses, OMPD_teams_loop, DSAStack))
+  if (checkGenericLoopLastprivate(SemaRef, Clauses, OMPD_teams_loop, DSAStack))
     return StmtError();
 
   auto *CS = cast<CapturedStmt>(AStmt);
@@ -10909,22 +10942,22 @@ StmtResult Sema::ActOnOpenMPTeamsGenericLoopDirective(
   // In presence of clause 'collapse', it will define the nested loops number.
   unsigned NestedLoopCount =
       checkOpenMPLoop(OMPD_teams_loop, getCollapseNumberExpr(Clauses),
-                      /*OrderedLoopCountExpr=*/nullptr, CS, *this, *DSAStack,
+                      /*OrderedLoopCountExpr=*/nullptr, CS, SemaRef, *DSAStack,
                       VarsWithImplicitDSA, B);
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp loop exprs were not built");
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
   DSAStack->setParentTeamsRegionLoc(StartLoc);
 
   return OMPTeamsGenericLoopDirective::Create(
-      Context, StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
+      getASTContext(), StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
 }
 
-StmtResult Sema::ActOnOpenMPTargetTeamsGenericLoopDirective(
+StmtResult SemaOpenMP::ActOnOpenMPTargetTeamsGenericLoopDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -10933,7 +10966,7 @@ StmtResult Sema::ActOnOpenMPTargetTeamsGenericLoopDirective(
   // OpenMP 5.1 [2.11.7, loop construct, Restrictions]
   // A list item may not appear in a lastprivate clause unless it is the
   // loop iteration variable of a loop that is associated with the construct.
-  if (checkGenericLoopLastprivate(*this, Clauses, OMPD_target_teams_loop,
+  if (checkGenericLoopLastprivate(SemaRef, Clauses, OMPD_target_teams_loop,
                                   DSAStack))
     return StmtError();
 
@@ -10959,22 +10992,22 @@ StmtResult Sema::ActOnOpenMPTargetTeamsGenericLoopDirective(
   // In presence of clause 'collapse', it will define the nested loops number.
   unsigned NestedLoopCount =
       checkOpenMPLoop(OMPD_target_teams_loop, getCollapseNumberExpr(Clauses),
-                      /*OrderedLoopCountExpr=*/nullptr, CS, *this, *DSAStack,
+                      /*OrderedLoopCountExpr=*/nullptr, CS, SemaRef, *DSAStack,
                       VarsWithImplicitDSA, B);
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp loop exprs were not built");
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
 
   return OMPTargetTeamsGenericLoopDirective::Create(
-      Context, StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B,
-      teamsLoopCanBeParallelFor(AStmt, *this));
+      getASTContext(), StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B,
+      teamsLoopCanBeParallelFor(AStmt, SemaRef));
 }
 
-StmtResult Sema::ActOnOpenMPParallelGenericLoopDirective(
+StmtResult SemaOpenMP::ActOnOpenMPParallelGenericLoopDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -10983,7 +11016,8 @@ StmtResult Sema::ActOnOpenMPParallelGenericLoopDirective(
   // OpenMP 5.1 [2.11.7, loop construct, Restrictions]
   // A list item may not appear in a lastprivate clause unless it is the
   // loop iteration variable of a loop that is associated with the construct.
-  if (checkGenericLoopLastprivate(*this, Clauses, OMPD_parallel_loop, DSAStack))
+  if (checkGenericLoopLastprivate(SemaRef, Clauses, OMPD_parallel_loop,
+                                  DSAStack))
     return StmtError();
 
   auto *CS = cast<CapturedStmt>(AStmt);
@@ -11008,21 +11042,21 @@ StmtResult Sema::ActOnOpenMPParallelGenericLoopDirective(
   // In presence of clause 'collapse', it will define the nested loops number.
   unsigned NestedLoopCount =
       checkOpenMPLoop(OMPD_parallel_loop, getCollapseNumberExpr(Clauses),
-                      /*OrderedLoopCountExpr=*/nullptr, CS, *this, *DSAStack,
+                      /*OrderedLoopCountExpr=*/nullptr, CS, SemaRef, *DSAStack,
                       VarsWithImplicitDSA, B);
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp loop exprs were not built");
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
 
   return OMPParallelGenericLoopDirective::Create(
-      Context, StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
+      getASTContext(), StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
 }
 
-StmtResult Sema::ActOnOpenMPTargetParallelGenericLoopDirective(
+StmtResult SemaOpenMP::ActOnOpenMPTargetParallelGenericLoopDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -11031,7 +11065,7 @@ StmtResult Sema::ActOnOpenMPTargetParallelGenericLoopDirective(
   // OpenMP 5.1 [2.11.7, loop construct, Restrictions]
   // A list item may not appear in a lastprivate clause unless it is the
   // loop iteration variable of a loop that is associated with the construct.
-  if (checkGenericLoopLastprivate(*this, Clauses, OMPD_target_parallel_loop,
+  if (checkGenericLoopLastprivate(SemaRef, Clauses, OMPD_target_parallel_loop,
                                   DSAStack))
     return StmtError();
 
@@ -11057,30 +11091,30 @@ StmtResult Sema::ActOnOpenMPTargetParallelGenericLoopDirective(
   // In presence of clause 'collapse', it will define the nested loops number.
   unsigned NestedLoopCount =
       checkOpenMPLoop(OMPD_target_parallel_loop, getCollapseNumberExpr(Clauses),
-                      /*OrderedLoopCountExpr=*/nullptr, CS, *this, *DSAStack,
+                      /*OrderedLoopCountExpr=*/nullptr, CS, SemaRef, *DSAStack,
                       VarsWithImplicitDSA, B);
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp loop exprs were not built");
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
 
   return OMPTargetParallelGenericLoopDirective::Create(
-      Context, StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
+      getASTContext(), StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
 }
 
-StmtResult Sema::ActOnOpenMPSingleDirective(ArrayRef<OMPClause *> Clauses,
-                                            Stmt *AStmt,
-                                            SourceLocation StartLoc,
-                                            SourceLocation EndLoc) {
+StmtResult SemaOpenMP::ActOnOpenMPSingleDirective(ArrayRef<OMPClause *> Clauses,
+                                                  Stmt *AStmt,
+                                                  SourceLocation StartLoc,
+                                                  SourceLocation EndLoc) {
   if (!AStmt)
     return StmtError();
 
   assert(isa<CapturedStmt>(AStmt) && "Captured statement expected");
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
 
   // OpenMP [2.7.3, single Construct, Restrictions]
   // The copyprivate clause must not be used with the nowait clause.
@@ -11099,33 +11133,35 @@ StmtResult Sema::ActOnOpenMPSingleDirective(ArrayRef<OMPClause *> Clauses,
     }
   }
 
-  return OMPSingleDirective::Create(Context, StartLoc, EndLoc, Clauses, AStmt);
+  return OMPSingleDirective::Create(getASTContext(), StartLoc, EndLoc, Clauses,
+                                    AStmt);
 }
 
-StmtResult Sema::ActOnOpenMPMasterDirective(Stmt *AStmt,
-                                            SourceLocation StartLoc,
-                                            SourceLocation EndLoc) {
+StmtResult SemaOpenMP::ActOnOpenMPMasterDirective(Stmt *AStmt,
+                                                  SourceLocation StartLoc,
+                                                  SourceLocation EndLoc) {
   if (!AStmt)
     return StmtError();
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
 
-  return OMPMasterDirective::Create(Context, StartLoc, EndLoc, AStmt);
+  return OMPMasterDirective::Create(getASTContext(), StartLoc, EndLoc, AStmt);
 }
 
-StmtResult Sema::ActOnOpenMPMaskedDirective(ArrayRef<OMPClause *> Clauses,
-                                            Stmt *AStmt,
-                                            SourceLocation StartLoc,
-                                            SourceLocation EndLoc) {
+StmtResult SemaOpenMP::ActOnOpenMPMaskedDirective(ArrayRef<OMPClause *> Clauses,
+                                                  Stmt *AStmt,
+                                                  SourceLocation StartLoc,
+                                                  SourceLocation EndLoc) {
   if (!AStmt)
     return StmtError();
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
 
-  return OMPMaskedDirective::Create(Context, StartLoc, EndLoc, Clauses, AStmt);
+  return OMPMaskedDirective::Create(getASTContext(), StartLoc, EndLoc, Clauses,
+                                    AStmt);
 }
 
-StmtResult Sema::ActOnOpenMPCriticalDirective(
+StmtResult SemaOpenMP::ActOnOpenMPCriticalDirective(
     const DeclarationNameInfo &DirName, ArrayRef<OMPClause *> Clauses,
     Stmt *AStmt, SourceLocation StartLoc, SourceLocation EndLoc) {
   if (!AStmt)
@@ -11146,7 +11182,7 @@ StmtResult Sema::ActOnOpenMPCriticalDirective(
           E->isInstantiationDependent()) {
         DependentHint = true;
       } else {
-        Hint = E->EvaluateKnownConstInt(Context);
+        Hint = E->EvaluateKnownConstInt(getASTContext());
         HintLoc = C->getBeginLoc();
       }
     }
@@ -11165,7 +11201,7 @@ StmtResult Sema::ActOnOpenMPCriticalDirective(
       if (const auto *C = Pair.first->getSingleClause<OMPHintClause>()) {
         Diag(C->getBeginLoc(), diag::note_omp_critical_hint_here)
             << 1
-            << toString(C->getHint()->EvaluateKnownConstInt(Context),
+            << toString(C->getHint()->EvaluateKnownConstInt(getASTContext()),
                         /*Radix=*/10, /*Signed=*/false);
       } else {
         Diag(Pair.first->getBeginLoc(), diag::note_omp_critical_no_hint) << 1;
@@ -11173,16 +11209,16 @@ StmtResult Sema::ActOnOpenMPCriticalDirective(
     }
   }
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
 
-  auto *Dir = OMPCriticalDirective::Create(Context, DirName, StartLoc, EndLoc,
-                                           Clauses, AStmt);
+  auto *Dir = OMPCriticalDirective::Create(getASTContext(), DirName, StartLoc,
+                                           EndLoc, Clauses, AStmt);
   if (!Pair.first && DirName.getName() && !DependentHint)
     DSAStack->addCriticalWithHint(Dir, Hint);
   return Dir;
 }
 
-StmtResult Sema::ActOnOpenMPParallelForDirective(
+StmtResult SemaOpenMP::ActOnOpenMPParallelForDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -11201,32 +11237,32 @@ StmtResult Sema::ActOnOpenMPParallelForDirective(
   // define the nested loops number.
   unsigned NestedLoopCount =
       checkOpenMPLoop(OMPD_parallel_for, getCollapseNumberExpr(Clauses),
-                      getOrderedNumberExpr(Clauses), AStmt, *this, *DSAStack,
+                      getOrderedNumberExpr(Clauses), AStmt, SemaRef, *DSAStack,
                       VarsWithImplicitDSA, B);
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp parallel for loop exprs were not built");
 
-  if (!CurContext->isDependentContext()) {
+  if (!SemaRef.CurContext->isDependentContext()) {
     // Finalize the clauses that need pre-built expressions for CodeGen.
     for (OMPClause *C : Clauses) {
       if (auto *LC = dyn_cast<OMPLinearClause>(C))
         if (FinishOpenMPLinearClause(*LC, cast<DeclRefExpr>(B.IterationVarRef),
-                                     B.NumIterations, *this, CurScope,
-                                     DSAStack))
+                                     B.NumIterations, SemaRef,
+                                     SemaRef.getCurScope(), DSAStack))
           return StmtError();
     }
   }
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
   return OMPParallelForDirective::Create(
-      Context, StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B,
+      getASTContext(), StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B,
       DSAStack->getTaskgroupReductionRef(), DSAStack->isCancelRegion());
 }
 
-StmtResult Sema::ActOnOpenMPParallelForSimdDirective(
+StmtResult SemaOpenMP::ActOnOpenMPParallelForSimdDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -11245,34 +11281,33 @@ StmtResult Sema::ActOnOpenMPParallelForSimdDirective(
   // define the nested loops number.
   unsigned NestedLoopCount =
       checkOpenMPLoop(OMPD_parallel_for_simd, getCollapseNumberExpr(Clauses),
-                      getOrderedNumberExpr(Clauses), AStmt, *this, *DSAStack,
+                      getOrderedNumberExpr(Clauses), AStmt, SemaRef, *DSAStack,
                       VarsWithImplicitDSA, B);
   if (NestedLoopCount == 0)
     return StmtError();
 
-  if (!CurContext->isDependentContext()) {
+  if (!SemaRef.CurContext->isDependentContext()) {
     // Finalize the clauses that need pre-built expressions for CodeGen.
     for (OMPClause *C : Clauses) {
       if (auto *LC = dyn_cast<OMPLinearClause>(C))
         if (FinishOpenMPLinearClause(*LC, cast<DeclRefExpr>(B.IterationVarRef),
-                                     B.NumIterations, *this, CurScope,
-                                     DSAStack))
+                                     B.NumIterations, SemaRef,
+                                     SemaRef.getCurScope(), DSAStack))
           return StmtError();
     }
   }
 
-  if (checkSimdlenSafelenSpecified(*this, Clauses))
+  if (checkSimdlenSafelenSpecified(SemaRef, Clauses))
     return StmtError();
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
   return OMPParallelForSimdDirective::Create(
-      Context, StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
+      getASTContext(), StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
 }
 
-StmtResult
-Sema::ActOnOpenMPParallelMasterDirective(ArrayRef<OMPClause *> Clauses,
-                                         Stmt *AStmt, SourceLocation StartLoc,
-                                         SourceLocation EndLoc) {
+StmtResult SemaOpenMP::ActOnOpenMPParallelMasterDirective(
+    ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+    SourceLocation EndLoc) {
   if (!AStmt)
     return StmtError();
 
@@ -11285,17 +11320,16 @@ Sema::ActOnOpenMPParallelMasterDirective(ArrayRef<OMPClause *> Clauses,
   // longjmp() and throw() must not violate the entry/exit criteria.
   CS->getCapturedDecl()->setNothrow();
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
 
   return OMPParallelMasterDirective::Create(
-      Context, StartLoc, EndLoc, Clauses, AStmt,
+      getASTContext(), StartLoc, EndLoc, Clauses, AStmt,
       DSAStack->getTaskgroupReductionRef());
 }
 
-StmtResult
-Sema::ActOnOpenMPParallelMaskedDirective(ArrayRef<OMPClause *> Clauses,
-                                         Stmt *AStmt, SourceLocation StartLoc,
-                                         SourceLocation EndLoc) {
+StmtResult SemaOpenMP::ActOnOpenMPParallelMaskedDirective(
+    ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+    SourceLocation EndLoc) {
   if (!AStmt)
     return StmtError();
 
@@ -11308,17 +11342,16 @@ Sema::ActOnOpenMPParallelMaskedDirective(ArrayRef<OMPClause *> Clauses,
   // longjmp() and throw() must not violate the entry/exit criteria.
   CS->getCapturedDecl()->setNothrow();
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
 
   return OMPParallelMaskedDirective::Create(
-      Context, StartLoc, EndLoc, Clauses, AStmt,
+      getASTContext(), StartLoc, EndLoc, Clauses, AStmt,
       DSAStack->getTaskgroupReductionRef());
 }
 
-StmtResult
-Sema::ActOnOpenMPParallelSectionsDirective(ArrayRef<OMPClause *> Clauses,
-                                           Stmt *AStmt, SourceLocation StartLoc,
-                                           SourceLocation EndLoc) {
+StmtResult SemaOpenMP::ActOnOpenMPParallelSectionsDirective(
+    ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+    SourceLocation EndLoc) {
   if (!AStmt)
     return StmtError();
 
@@ -11348,10 +11381,10 @@ Sema::ActOnOpenMPParallelSectionsDirective(ArrayRef<OMPClause *> Clauses,
     return StmtError();
   }
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
 
   return OMPParallelSectionsDirective::Create(
-      Context, StartLoc, EndLoc, Clauses, AStmt,
+      getASTContext(), StartLoc, EndLoc, Clauses, AStmt,
       DSAStack->getTaskgroupReductionRef(), DSAStack->isCancelRegion());
 }
 
@@ -11378,16 +11411,17 @@ static bool checkMutuallyExclusiveClauses(
   return ErrorFound;
 }
 
-StmtResult Sema::ActOnOpenMPTaskDirective(ArrayRef<OMPClause *> Clauses,
-                                          Stmt *AStmt, SourceLocation StartLoc,
-                                          SourceLocation EndLoc) {
+StmtResult SemaOpenMP::ActOnOpenMPTaskDirective(ArrayRef<OMPClause *> Clauses,
+                                                Stmt *AStmt,
+                                                SourceLocation StartLoc,
+                                                SourceLocation EndLoc) {
   if (!AStmt)
     return StmtError();
 
   // OpenMP 5.0, 2.10.1 task Construct
   // If a detach clause appears on the directive, then a mergeable clause cannot
   // appear on the same directive.
-  if (checkMutuallyExclusiveClauses(*this, Clauses,
+  if (checkMutuallyExclusiveClauses(SemaRef, Clauses,
                                     {OMPC_detach, OMPC_mergeable}))
     return StmtError();
 
@@ -11399,26 +11433,26 @@ StmtResult Sema::ActOnOpenMPTaskDirective(ArrayRef<OMPClause *> Clauses,
   // longjmp() and throw() must not violate the entry/exit criteria.
   CS->getCapturedDecl()->setNothrow();
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
 
-  return OMPTaskDirective::Create(Context, StartLoc, EndLoc, Clauses, AStmt,
-                                  DSAStack->isCancelRegion());
+  return OMPTaskDirective::Create(getASTContext(), StartLoc, EndLoc, Clauses,
+                                  AStmt, DSAStack->isCancelRegion());
 }
 
-StmtResult Sema::ActOnOpenMPTaskyieldDirective(SourceLocation StartLoc,
-                                               SourceLocation EndLoc) {
-  return OMPTaskyieldDirective::Create(Context, StartLoc, EndLoc);
+StmtResult SemaOpenMP::ActOnOpenMPTaskyieldDirective(SourceLocation StartLoc,
+                                                     SourceLocation EndLoc) {
+  return OMPTaskyieldDirective::Create(getASTContext(), StartLoc, EndLoc);
 }
 
-StmtResult Sema::ActOnOpenMPBarrierDirective(SourceLocation StartLoc,
-                                             SourceLocation EndLoc) {
-  return OMPBarrierDirective::Create(Context, StartLoc, EndLoc);
+StmtResult SemaOpenMP::ActOnOpenMPBarrierDirective(SourceLocation StartLoc,
+                                                   SourceLocation EndLoc) {
+  return OMPBarrierDirective::Create(getASTContext(), StartLoc, EndLoc);
 }
 
-StmtResult Sema::ActOnOpenMPErrorDirective(ArrayRef<OMPClause *> Clauses,
-                                           SourceLocation StartLoc,
-                                           SourceLocation EndLoc,
-                                           bool InExContext) {
+StmtResult SemaOpenMP::ActOnOpenMPErrorDirective(ArrayRef<OMPClause *> Clauses,
+                                                 SourceLocation StartLoc,
+                                                 SourceLocation EndLoc,
+                                                 bool InExContext) {
   const OMPAtClause *AtC =
       OMPExecutableDirective::getSingleClause<OMPAtClause>(Clauses);
 
@@ -11443,12 +11477,13 @@ StmtResult Sema::ActOnOpenMPErrorDirective(ArrayRef<OMPClause *> Clauses,
     if (!SeverityC || SeverityC->getSeverityKind() != OMPC_SEVERITY_warning)
       return StmtError();
   }
-  return OMPErrorDirective::Create(Context, StartLoc, EndLoc, Clauses);
+  return OMPErrorDirective::Create(getASTContext(), StartLoc, EndLoc, Clauses);
 }
 
-StmtResult Sema::ActOnOpenMPTaskwaitDirective(ArrayRef<OMPClause *> Clauses,
-                                              SourceLocation StartLoc,
-                                              SourceLocation EndLoc) {
+StmtResult
+SemaOpenMP::ActOnOpenMPTaskwaitDirective(ArrayRef<OMPClause *> Clauses,
+                                         SourceLocation StartLoc,
+                                         SourceLocation EndLoc) {
   const OMPNowaitClause *NowaitC =
       OMPExecutableDirective::getSingleClause<OMPNowaitClause>(Clauses);
   bool HasDependC =
@@ -11459,28 +11494,29 @@ StmtResult Sema::ActOnOpenMPTaskwaitDirective(ArrayRef<OMPClause *> Clauses,
     return StmtError();
   }
 
-  return OMPTaskwaitDirective::Create(Context, StartLoc, EndLoc, Clauses);
+  return OMPTaskwaitDirective::Create(getASTContext(), StartLoc, EndLoc,
+                                      Clauses);
 }
 
-StmtResult Sema::ActOnOpenMPTaskgroupDirective(ArrayRef<OMPClause *> Clauses,
-                                               Stmt *AStmt,
-                                               SourceLocation StartLoc,
-                                               SourceLocation EndLoc) {
+StmtResult
+SemaOpenMP::ActOnOpenMPTaskgroupDirective(ArrayRef<OMPClause *> Clauses,
+                                          Stmt *AStmt, SourceLocation StartLoc,
+                                          SourceLocation EndLoc) {
   if (!AStmt)
     return StmtError();
 
   assert(isa<CapturedStmt>(AStmt) && "Captured statement expected");
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
 
-  return OMPTaskgroupDirective::Create(Context, StartLoc, EndLoc, Clauses,
-                                       AStmt,
+  return OMPTaskgroupDirective::Create(getASTContext(), StartLoc, EndLoc,
+                                       Clauses, AStmt,
                                        DSAStack->getTaskgroupReductionRef());
 }
 
-StmtResult Sema::ActOnOpenMPFlushDirective(ArrayRef<OMPClause *> Clauses,
-                                           SourceLocation StartLoc,
-                                           SourceLocation EndLoc) {
+StmtResult SemaOpenMP::ActOnOpenMPFlushDirective(ArrayRef<OMPClause *> Clauses,
+                                                 SourceLocation StartLoc,
+                                                 SourceLocation EndLoc) {
   OMPFlushClause *FC = nullptr;
   OMPClause *OrderClause = nullptr;
   for (OMPClause *C : Clauses) {
@@ -11514,12 +11550,12 @@ StmtResult Sema::ActOnOpenMPFlushDirective(ArrayRef<OMPClause *> Clauses,
         << getOpenMPClauseName(OrderClause->getClauseKind());
     return StmtError();
   }
-  return OMPFlushDirective::Create(Context, StartLoc, EndLoc, Clauses);
+  return OMPFlushDirective::Create(getASTContext(), StartLoc, EndLoc, Clauses);
 }
 
-StmtResult Sema::ActOnOpenMPDepobjDirective(ArrayRef<OMPClause *> Clauses,
-                                            SourceLocation StartLoc,
-                                            SourceLocation EndLoc) {
+StmtResult SemaOpenMP::ActOnOpenMPDepobjDirective(ArrayRef<OMPClause *> Clauses,
+                                                  SourceLocation StartLoc,
+                                                  SourceLocation EndLoc) {
   if (Clauses.empty()) {
     Diag(StartLoc, diag::err_omp_depobj_expected);
     return StmtError();
@@ -11536,12 +11572,12 @@ StmtResult Sema::ActOnOpenMPDepobjDirective(ArrayRef<OMPClause *> Clauses,
     Diag(Clauses[0]->getEndLoc(), diag::err_omp_depobj_single_clause_expected);
     return StmtError();
   }
-  return OMPDepobjDirective::Create(Context, StartLoc, EndLoc, Clauses);
+  return OMPDepobjDirective::Create(getASTContext(), StartLoc, EndLoc, Clauses);
 }
 
-StmtResult Sema::ActOnOpenMPScanDirective(ArrayRef<OMPClause *> Clauses,
-                                          SourceLocation StartLoc,
-                                          SourceLocation EndLoc) {
+StmtResult SemaOpenMP::ActOnOpenMPScanDirective(ArrayRef<OMPClause *> Clauses,
+                                                SourceLocation StartLoc,
+                                                SourceLocation EndLoc) {
   // Check that exactly one clause is specified.
   if (Clauses.size() != 1) {
     Diag(Clauses.empty() ? EndLoc : Clauses[1]->getBeginLoc(),
@@ -11566,13 +11602,13 @@ StmtResult Sema::ActOnOpenMPScanDirective(ArrayRef<OMPClause *> Clauses,
     return StmtError();
   }
   DSAStack->setParentHasScanDirective(StartLoc);
-  return OMPScanDirective::Create(Context, StartLoc, EndLoc, Clauses);
+  return OMPScanDirective::Create(getASTContext(), StartLoc, EndLoc, Clauses);
 }
 
-StmtResult Sema::ActOnOpenMPOrderedDirective(ArrayRef<OMPClause *> Clauses,
-                                             Stmt *AStmt,
-                                             SourceLocation StartLoc,
-                                             SourceLocation EndLoc) {
+StmtResult
+SemaOpenMP::ActOnOpenMPOrderedDirective(ArrayRef<OMPClause *> Clauses,
+                                        Stmt *AStmt, SourceLocation StartLoc,
+                                        SourceLocation EndLoc) {
   const OMPClause *DependFound = nullptr;
   const OMPClause *DependSourceClause = nullptr;
   const OMPClause *DependSinkClause = nullptr;
@@ -11631,7 +11667,7 @@ StmtResult Sema::ActOnOpenMPOrderedDirective(ArrayRef<OMPClause *> Clauses,
     // An ordered construct with the simd clause is the only OpenMP construct
     // that can appear in the simd region.
     Diag(StartLoc, diag::err_omp_prohibited_region_simd)
-        << (LangOpts.OpenMP >= 50 ? 1 : 0);
+        << (getLangOpts().OpenMP >= 50 ? 1 : 0);
     ErrorFound = true;
   } else if ((DependFound || DoacrossFound) && (TC || SC)) {
     SourceLocation Loc =
@@ -11678,10 +11714,11 @@ StmtResult Sema::ActOnOpenMPOrderedDirective(ArrayRef<OMPClause *> Clauses,
   if (AStmt) {
     assert(isa<CapturedStmt>(AStmt) && "Captured statement expected");
 
-    setFunctionHasBranchProtectedScope();
+    SemaRef.setFunctionHasBranchProtectedScope();
   }
 
-  return OMPOrderedDirective::Create(Context, StartLoc, EndLoc, Clauses, AStmt);
+  return OMPOrderedDirective::Create(getASTContext(), StartLoc, EndLoc, Clauses,
+                                     AStmt);
 }
 
 namespace {
@@ -12739,10 +12776,11 @@ bool OpenMPAtomicCompareCaptureChecker::checkStmt(Stmt *S,
 }
 } // namespace
 
-StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef<OMPClause *> Clauses,
-                                            Stmt *AStmt,
-                                            SourceLocation StartLoc,
-                                            SourceLocation EndLoc) {
+StmtResult SemaOpenMP::ActOnOpenMPAtomicDirective(ArrayRef<OMPClause *> Clauses,
+                                                  Stmt *AStmt,
+                                                  SourceLocation StartLoc,
+                                                  SourceLocation EndLoc) {
+  ASTContext &Context = getASTContext();
   // Register location of the first atomic directive.
   DSAStack->addAtomicDirectiveLoc(StartLoc);
   if (!AStmt)
@@ -12945,7 +12983,7 @@ StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef<OMPClause *> Clauses,
           << ErrorFound << NoteRange;
       return StmtError();
     }
-    if (CurContext->isDependentContext())
+    if (SemaRef.CurContext->isDependentContext())
       V = X = nullptr;
   } else if (AtomicKind == OMPC_write) {
     enum {
@@ -13007,7 +13045,7 @@ StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef<OMPClause *> Clauses,
           << ErrorFound << NoteRange;
       return StmtError();
     }
-    if (CurContext->isDependentContext())
+    if (SemaRef.CurContext->isDependentContext())
       E = X = nullptr;
   } else if (AtomicKind == OMPC_update || AtomicKind == OMPC_unknown) {
     // If clause is update:
@@ -13018,7 +13056,7 @@ StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef<OMPClause *> Clauses,
     //  x binop= expr;
     //  x = x binop expr;
     //  x = expr binop x;
-    OpenMPAtomicUpdateChecker Checker(*this);
+    OpenMPAtomicUpdateChecker Checker(SemaRef);
     if (Checker.checkStatement(
             Body,
             (AtomicKind == OMPC_update)
@@ -13026,7 +13064,7 @@ StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef<OMPClause *> Clauses,
                 : diag::err_omp_atomic_not_expression_statement,
             diag::note_omp_atomic_update))
       return StmtError();
-    if (!CurContext->isDependentContext()) {
+    if (!SemaRef.CurContext->isDependentContext()) {
       E = Checker.getExpr();
       X = Checker.getX();
       UE = Checker.getUpdateExpr();
@@ -13056,7 +13094,7 @@ StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef<OMPClause *> Clauses,
       if (AtomicBinOp && AtomicBinOp->getOpcode() == BO_Assign) {
         V = AtomicBinOp->getLHS();
         Body = AtomicBinOp->getRHS()->IgnoreParenImpCasts();
-        OpenMPAtomicUpdateChecker Checker(*this);
+        OpenMPAtomicUpdateChecker Checker(SemaRef);
         if (Checker.checkStatement(
                 Body, diag::err_omp_atomic_capture_not_expression_statement,
                 diag::note_omp_atomic_update))
@@ -13081,7 +13119,7 @@ StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef<OMPClause *> Clauses,
         Diag(NoteLoc, diag::note_omp_atomic_capture) << ErrorFound << NoteRange;
         return StmtError();
       }
-      if (CurContext->isDependentContext())
+      if (SemaRef.CurContext->isDependentContext())
         UE = V = E = X = nullptr;
     } else {
       // If clause is a capture:
@@ -13110,14 +13148,14 @@ StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef<OMPClause *> Clauses,
           if (auto *EWC = dyn_cast<ExprWithCleanups>(Second))
             Second = EWC->getSubExpr()->IgnoreParenImpCasts();
           // Need to find what subexpression is 'v' and what is 'x'.
-          OpenMPAtomicUpdateChecker Checker(*this);
+          OpenMPAtomicUpdateChecker Checker(SemaRef);
           bool IsUpdateExprFound = !Checker.checkStatement(Second);
           BinaryOperator *BinOp = nullptr;
           if (IsUpdateExprFound) {
             BinOp = dyn_cast<BinaryOperator>(First);
             IsUpdateExprFound = BinOp && BinOp->getOpcode() == BO_Assign;
           }
-          if (IsUpdateExprFound && !CurContext->isDependentContext()) {
+          if (IsUpdateExprFound && !SemaRef.CurContext->isDependentContext()) {
             //  { v = x; x++; }
             //  { v = x; x--; }
             //  { v = x; ++x; }
@@ -13147,7 +13185,8 @@ StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef<OMPClause *> Clauses,
               BinOp = dyn_cast<BinaryOperator>(Second);
               IsUpdateExprFound = BinOp && BinOp->getOpcode() == BO_Assign;
             }
-            if (IsUpdateExprFound && !CurContext->isDependentContext()) {
+            if (IsUpdateExprFound &&
+                !SemaRef.CurContext->isDependentContext()) {
               //  { x++; v = x; }
               //  { x--; v = x; }
               //  { ++x; v = x; }
@@ -13244,12 +13283,12 @@ StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef<OMPClause *> Clauses,
       Diag(NoteLoc, diag::note_omp_atomic_capture) << ErrorFound << NoteRange;
       return StmtError();
     }
-    if (CurContext->isDependentContext())
+    if (SemaRef.CurContext->isDependentContext())
       UE = V = E = X = nullptr;
   } else if (AtomicKind == OMPC_compare) {
     if (IsCompareCapture) {
       OpenMPAtomicCompareCaptureChecker::ErrorInfoTy ErrorInfo;
-      OpenMPAtomicCompareCaptureChecker Checker(*this);
+      OpenMPAtomicCompareCaptureChecker Checker(SemaRef);
       if (!Checker.checkStmt(Body, ErrorInfo)) {
         Diag(ErrorInfo.ErrorLoc, diag::err_omp_atomic_compare_capture)
             << ErrorInfo.ErrorRange;
@@ -13269,7 +13308,7 @@ StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef<OMPClause *> Clauses,
       IsPostfixUpdate = Checker.isPostfixUpdate();
     } else {
       OpenMPAtomicCompareChecker::ErrorInfoTy ErrorInfo;
-      OpenMPAtomicCompareChecker Checker(*this);
+      OpenMPAtomicCompareChecker Checker(SemaRef);
       if (!Checker.checkStmt(Body, ErrorInfo)) {
         Diag(ErrorInfo.ErrorLoc, diag::err_omp_atomic_compare)
             << ErrorInfo.ErrorRange;
@@ -13307,17 +13346,17 @@ StmtResult Sema::ActOnOpenMPAtomicDirective(ArrayRef<OMPClause *> Clauses,
     }
   }
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
 
   return OMPAtomicDirective::Create(
       Context, StartLoc, EndLoc, Clauses, AStmt,
       {X, V, R, E, UE, D, CE, IsXLHSInRHSPart, IsPostfixUpdate, IsFailOnly});
 }
 
-StmtResult Sema::ActOnOpenMPTargetDirective(ArrayRef<OMPClause *> Clauses,
-                                            Stmt *AStmt,
-                                            SourceLocation StartLoc,
-                                            SourceLocation EndLoc) {
+StmtResult SemaOpenMP::ActOnOpenMPTargetDirective(ArrayRef<OMPClause *> Clauses,
+                                                  Stmt *AStmt,
+                                                  SourceLocation StartLoc,
+                                                  SourceLocation EndLoc) {
   if (!AStmt)
     return StmtError();
 
@@ -13374,15 +13413,15 @@ StmtResult Sema::ActOnOpenMPTargetDirective(ArrayRef<OMPClause *> Clauses,
     }
   }
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
 
-  return OMPTargetDirective::Create(Context, StartLoc, EndLoc, Clauses, AStmt);
+  return OMPTargetDirective::Create(getASTContext(), StartLoc, EndLoc, Clauses,
+                                    AStmt);
 }
 
-StmtResult
-Sema::ActOnOpenMPTargetParallelDirective(ArrayRef<OMPClause *> Clauses,
-                                         Stmt *AStmt, SourceLocation StartLoc,
-                                         SourceLocation EndLoc) {
+StmtResult SemaOpenMP::ActOnOpenMPTargetParallelDirective(
+    ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+    SourceLocation EndLoc) {
   if (!AStmt)
     return StmtError();
 
@@ -13404,14 +13443,14 @@ Sema::ActOnOpenMPTargetParallelDirective(ArrayRef<OMPClause *> Clauses,
     CS->getCapturedDecl()->setNothrow();
   }
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
 
   return OMPTargetParallelDirective::Create(
-      Context, StartLoc, EndLoc, Clauses, AStmt,
+      getASTContext(), StartLoc, EndLoc, Clauses, AStmt,
       DSAStack->getTaskgroupReductionRef(), DSAStack->isCancelRegion());
 }
 
-StmtResult Sema::ActOnOpenMPTargetParallelForDirective(
+StmtResult SemaOpenMP::ActOnOpenMPTargetParallelForDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -13440,28 +13479,28 @@ StmtResult Sema::ActOnOpenMPTargetParallelForDirective(
   // define the nested loops number.
   unsigned NestedLoopCount =
       checkOpenMPLoop(OMPD_target_parallel_for, getCollapseNumberExpr(Clauses),
-                      getOrderedNumberExpr(Clauses), CS, *this, *DSAStack,
+                      getOrderedNumberExpr(Clauses), CS, SemaRef, *DSAStack,
                       VarsWithImplicitDSA, B);
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp target parallel for loop exprs were not built");
 
-  if (!CurContext->isDependentContext()) {
+  if (!SemaRef.CurContext->isDependentContext()) {
     // Finalize the clauses that need pre-built expressions for CodeGen.
     for (OMPClause *C : Clauses) {
       if (auto *LC = dyn_cast<OMPLinearClause>(C))
         if (FinishOpenMPLinearClause(*LC, cast<DeclRefExpr>(B.IterationVarRef),
-                                     B.NumIterations, *this, CurScope,
-                                     DSAStack))
+                                     B.NumIterations, SemaRef,
+                                     SemaRef.getCurScope(), DSAStack))
           return StmtError();
     }
   }
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
   return OMPTargetParallelForDirective::Create(
-      Context, StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B,
+      getASTContext(), StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B,
       DSAStack->getTaskgroupReductionRef(), DSAStack->isCancelRegion());
 }
 
@@ -13498,10 +13537,10 @@ static bool isClauseMappable(ArrayRef<OMPClause *> Clauses) {
   return true;
 }
 
-StmtResult Sema::ActOnOpenMPTargetDataDirective(ArrayRef<OMPClause *> Clauses,
-                                                Stmt *AStmt,
-                                                SourceLocation StartLoc,
-                                                SourceLocation EndLoc) {
+StmtResult
+SemaOpenMP::ActOnOpenMPTargetDataDirective(ArrayRef<OMPClause *> Clauses,
+                                           Stmt *AStmt, SourceLocation StartLoc,
+                                           SourceLocation EndLoc) {
   if (!AStmt)
     return StmtError();
 
@@ -13511,9 +13550,10 @@ StmtResult Sema::ActOnOpenMPTargetDataDirective(ArrayRef<OMPClause *> Clauses,
   // At least one map, use_device_addr or use_device_ptr clause must appear on
   // the directive.
   if (!hasClauses(Clauses, OMPC_map, OMPC_use_device_ptr) &&
-      (LangOpts.OpenMP < 50 || !hasClauses(Clauses, OMPC_use_device_addr))) {
+      (getLangOpts().OpenMP < 50 ||
+       !hasClauses(Clauses, OMPC_use_device_addr))) {
     StringRef Expected;
-    if (LangOpts.OpenMP < 50)
+    if (getLangOpts().OpenMP < 50)
       Expected = "'map' or 'use_device_ptr'";
     else
       Expected = "'map', 'use_device_ptr', or 'use_device_addr'";
@@ -13522,16 +13562,15 @@ StmtResult Sema::ActOnOpenMPTargetDataDirective(ArrayRef<OMPClause *> Clauses,
     return StmtError();
   }
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
 
-  return OMPTargetDataDirective::Create(Context, StartLoc, EndLoc, Clauses,
-                                        AStmt);
+  return OMPTargetDataDirective::Create(getASTContext(), StartLoc, EndLoc,
+                                        Clauses, AStmt);
 }
 
-StmtResult
-Sema::ActOnOpenMPTargetEnterDataDirective(ArrayRef<OMPClause *> Clauses,
-                                          SourceLocation StartLoc,
-                                          SourceLocation EndLoc, Stmt *AStmt) {
+StmtResult SemaOpenMP::ActOnOpenMPTargetEnterDataDirective(
+    ArrayRef<OMPClause *> Clauses, SourceLocation StartLoc,
+    SourceLocation EndLoc, Stmt *AStmt) {
   if (!AStmt)
     return StmtError();
 
@@ -13561,14 +13600,13 @@ Sema::ActOnOpenMPTargetEnterDataDirective(ArrayRef<OMPClause *> Clauses,
     return StmtError();
   }
 
-  return OMPTargetEnterDataDirective::Create(Context, StartLoc, EndLoc, Clauses,
-                                             AStmt);
+  return OMPTargetEnterDataDirective::Create(getASTContext(), StartLoc, EndLoc,
+                                             Clauses, AStmt);
 }
 
-StmtResult
-Sema::ActOnOpenMPTargetExitDataDirective(ArrayRef<OMPClause *> Clauses,
-                                         SourceLocation StartLoc,
-                                         SourceLocation EndLoc, Stmt *AStmt) {
+StmtResult SemaOpenMP::ActOnOpenMPTargetExitDataDirective(
+    ArrayRef<OMPClause *> Clauses, SourceLocation StartLoc,
+    SourceLocation EndLoc, Stmt *AStmt) {
   if (!AStmt)
     return StmtError();
 
@@ -13598,14 +13636,13 @@ Sema::ActOnOpenMPTargetExitDataDirective(ArrayRef<OMPClause *> Clauses,
     return StmtError();
   }
 
-  return OMPTargetExitDataDirective::Create(Context, StartLoc, EndLoc, Clauses,
-                                            AStmt);
+  return OMPTargetExitDataDirective::Create(getASTContext(), StartLoc, EndLoc,
+                                            Clauses, AStmt);
 }
 
-StmtResult Sema::ActOnOpenMPTargetUpdateDirective(ArrayRef<OMPClause *> Clauses,
-                                                  SourceLocation StartLoc,
-                                                  SourceLocation EndLoc,
-                                                  Stmt *AStmt) {
+StmtResult SemaOpenMP::ActOnOpenMPTargetUpdateDirective(
+    ArrayRef<OMPClause *> Clauses, SourceLocation StartLoc,
+    SourceLocation EndLoc, Stmt *AStmt) {
   if (!AStmt)
     return StmtError();
 
@@ -13637,13 +13674,14 @@ StmtResult Sema::ActOnOpenMPTargetUpdateDirective(ArrayRef<OMPClause *> Clauses,
     return StmtError();
   }
 
-  return OMPTargetUpdateDirective::Create(Context, StartLoc, EndLoc, Clauses,
-                                          AStmt);
+  return OMPTargetUpdateDirective::Create(getASTContext(), StartLoc, EndLoc,
+                                          Clauses, AStmt);
 }
 
-StmtResult Sema::ActOnOpenMPTeamsDirective(ArrayRef<OMPClause *> Clauses,
-                                           Stmt *AStmt, SourceLocation StartLoc,
-                                           SourceLocation EndLoc) {
+StmtResult SemaOpenMP::ActOnOpenMPTeamsDirective(ArrayRef<OMPClause *> Clauses,
+                                                 Stmt *AStmt,
+                                                 SourceLocation StartLoc,
+                                                 SourceLocation EndLoc) {
   if (!AStmt)
     return StmtError();
 
@@ -13659,17 +13697,17 @@ StmtResult Sema::ActOnOpenMPTeamsDirective(ArrayRef<OMPClause *> Clauses,
   // longjmp() and throw() must not violate the entry/exit criteria.
   CS->getCapturedDecl()->setNothrow();
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
 
   DSAStack->setParentTeamsRegionLoc(StartLoc);
 
-  return OMPTeamsDirective::Create(Context, StartLoc, EndLoc, Clauses, AStmt);
+  return OMPTeamsDirective::Create(getASTContext(), StartLoc, EndLoc, Clauses,
+                                   AStmt);
 }
 
-StmtResult
-Sema::ActOnOpenMPCancellationPointDirective(SourceLocation StartLoc,
-                                            SourceLocation EndLoc,
-                                            OpenMPDirectiveKind CancelRegion) {
+StmtResult SemaOpenMP::ActOnOpenMPCancellationPointDirective(
+    SourceLocation StartLoc, SourceLocation EndLoc,
+    OpenMPDirectiveKind CancelRegion) {
   if (DSAStack->isParentNowaitRegion()) {
     Diag(StartLoc, diag::err_omp_parent_cancel_region_nowait) << 0;
     return StmtError();
@@ -13678,14 +13716,13 @@ Sema::ActOnOpenMPCancellationPointDirective(SourceLocation StartLoc,
     Diag(StartLoc, diag::err_omp_parent_cancel_region_ordered) << 0;
     return StmtError();
   }
-  return OMPCancellationPointDirective::Create(Context, StartLoc, EndLoc,
-                                               CancelRegion);
+  return OMPCancellationPointDirective::Create(getASTContext(), StartLoc,
+                                               EndLoc, CancelRegion);
 }
 
-StmtResult Sema::ActOnOpenMPCancelDirective(ArrayRef<OMPClause *> Clauses,
-                                            SourceLocation StartLoc,
-                                            SourceLocation EndLoc,
-                                            OpenMPDirectiveKind CancelRegion) {
+StmtResult SemaOpenMP::ActOnOpenMPCancelDirective(
+    ArrayRef<OMPClause *> Clauses, SourceLocation StartLoc,
+    SourceLocation EndLoc, OpenMPDirectiveKind CancelRegion) {
   if (DSAStack->isParentNowaitRegion()) {
     Diag(StartLoc, diag::err_omp_parent_cancel_region_nowait) << 1;
     return StmtError();
@@ -13695,7 +13732,7 @@ StmtResult Sema::ActOnOpenMPCancelDirective(ArrayRef<OMPClause *> Clauses,
     return StmtError();
   }
   DSAStack->setParentCancelRegion(/*Cancel=*/true);
-  return OMPCancelDirective::Create(Context, StartLoc, EndLoc, Clauses,
+  return OMPCancelDirective::Create(getASTContext(), StartLoc, EndLoc, Clauses,
                                     CancelRegion);
 }
 
@@ -13726,7 +13763,7 @@ static bool checkReductionClauseWithNogroup(Sema &S,
   return false;
 }
 
-StmtResult Sema::ActOnOpenMPTaskLoopDirective(
+StmtResult SemaOpenMP::ActOnOpenMPTaskLoopDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -13738,33 +13775,33 @@ StmtResult Sema::ActOnOpenMPTaskLoopDirective(
   // define the nested loops number.
   unsigned NestedLoopCount =
       checkOpenMPLoop(OMPD_taskloop, getCollapseNumberExpr(Clauses),
-                      /*OrderedLoopCountExpr=*/nullptr, AStmt, *this, *DSAStack,
-                      VarsWithImplicitDSA, B);
+                      /*OrderedLoopCountExpr=*/nullptr, AStmt, SemaRef,
+                      *DSAStack, VarsWithImplicitDSA, B);
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp for loop exprs were not built");
 
   // OpenMP, [2.9.2 taskloop Construct, Restrictions]
   // The grainsize clause and num_tasks clause are mutually exclusive and may
   // not appear on the same taskloop directive.
-  if (checkMutuallyExclusiveClauses(*this, Clauses,
+  if (checkMutuallyExclusiveClauses(SemaRef, Clauses,
                                     {OMPC_grainsize, OMPC_num_tasks}))
     return StmtError();
   // OpenMP, [2.9.2 taskloop Construct, Restrictions]
   // If a reduction clause is present on the taskloop directive, the nogroup
   // clause must not be specified.
-  if (checkReductionClauseWithNogroup(*this, Clauses))
+  if (checkReductionClauseWithNogroup(SemaRef, Clauses))
     return StmtError();
 
-  setFunctionHasBranchProtectedScope();
-  return OMPTaskLoopDirective::Create(Context, StartLoc, EndLoc,
+  SemaRef.setFunctionHasBranchProtectedScope();
+  return OMPTaskLoopDirective::Create(getASTContext(), StartLoc, EndLoc,
                                       NestedLoopCount, Clauses, AStmt, B,
                                       DSAStack->isCancelRegion());
 }
 
-StmtResult Sema::ActOnOpenMPTaskLoopSimdDirective(
+StmtResult SemaOpenMP::ActOnOpenMPTaskLoopSimdDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -13776,21 +13813,21 @@ StmtResult Sema::ActOnOpenMPTaskLoopSimdDirective(
   // define the nested loops number.
   unsigned NestedLoopCount =
       checkOpenMPLoop(OMPD_taskloop_simd, getCollapseNumberExpr(Clauses),
-                      /*OrderedLoopCountExpr=*/nullptr, AStmt, *this, *DSAStack,
-                      VarsWithImplicitDSA, B);
+                      /*OrderedLoopCountExpr=*/nullptr, AStmt, SemaRef,
+                      *DSAStack, VarsWithImplicitDSA, B);
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp for loop exprs were not built");
 
-  if (!CurContext->isDependentContext()) {
+  if (!SemaRef.CurContext->isDependentContext()) {
     // Finalize the clauses that need pre-built expressions for CodeGen.
     for (OMPClause *C : Clauses) {
       if (auto *LC = dyn_cast<OMPLinearClause>(C))
         if (FinishOpenMPLinearClause(*LC, cast<DeclRefExpr>(B.IterationVarRef),
-                                     B.NumIterations, *this, CurScope,
-                                     DSAStack))
+                                     B.NumIterations, SemaRef,
+                                     SemaRef.getCurScope(), DSAStack))
           return StmtError();
     }
   }
@@ -13798,23 +13835,23 @@ StmtResult Sema::ActOnOpenMPTaskLoopSimdDirective(
   // OpenMP, [2.9.2 taskloop Construct, Restrictions]
   // The grainsize clause and num_tasks clause are mutually exclusive and may
   // not appear on the same taskloop directive.
-  if (checkMutuallyExclusiveClauses(*this, Clauses,
+  if (checkMutuallyExclusiveClauses(SemaRef, Clauses,
                                     {OMPC_grainsize, OMPC_num_tasks}))
     return StmtError();
   // OpenMP, [2.9.2 taskloop Construct, Restrictions]
   // If a reduction clause is present on the taskloop directive, the nogroup
   // clause must not be specified.
-  if (checkReductionClauseWithNogroup(*this, Clauses))
+  if (checkReductionClauseWithNogroup(SemaRef, Clauses))
     return StmtError();
-  if (checkSimdlenSafelenSpecified(*this, Clauses))
+  if (checkSimdlenSafelenSpecified(SemaRef, Clauses))
     return StmtError();
 
-  setFunctionHasBranchProtectedScope();
-  return OMPTaskLoopSimdDirective::Create(Context, StartLoc, EndLoc,
+  SemaRef.setFunctionHasBranchProtectedScope();
+  return OMPTaskLoopSimdDirective::Create(getASTContext(), StartLoc, EndLoc,
                                           NestedLoopCount, Clauses, AStmt, B);
 }
 
-StmtResult Sema::ActOnOpenMPMasterTaskLoopDirective(
+StmtResult SemaOpenMP::ActOnOpenMPMasterTaskLoopDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -13826,33 +13863,33 @@ StmtResult Sema::ActOnOpenMPMasterTaskLoopDirective(
   // define the nested loops number.
   unsigned NestedLoopCount =
       checkOpenMPLoop(OMPD_master_taskloop, getCollapseNumberExpr(Clauses),
-                      /*OrderedLoopCountExpr=*/nullptr, AStmt, *this, *DSAStack,
-                      VarsWithImplicitDSA, B);
+                      /*OrderedLoopCountExpr=*/nullptr, AStmt, SemaRef,
+                      *DSAStack, VarsWithImplicitDSA, B);
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp for loop exprs were not built");
 
   // OpenMP, [2.9.2 taskloop Construct, Restrictions]
   // The grainsize clause and num_tasks clause are mutually exclusive and may
   // not appear on the same taskloop directive.
-  if (checkMutuallyExclusiveClauses(*this, Clauses,
+  if (checkMutuallyExclusiveClauses(SemaRef, Clauses,
                                     {OMPC_grainsize, OMPC_num_tasks}))
     return StmtError();
   // OpenMP, [2.9.2 taskloop Construct, Restrictions]
   // If a reduction clause is present on the taskloop directive, the nogroup
   // clause must not be specified.
-  if (checkReductionClauseWithNogroup(*this, Clauses))
+  if (checkReductionClauseWithNogroup(SemaRef, Clauses))
     return StmtError();
 
-  setFunctionHasBranchProtectedScope();
-  return OMPMasterTaskLoopDirective::Create(Context, StartLoc, EndLoc,
+  SemaRef.setFunctionHasBranchProtectedScope();
+  return OMPMasterTaskLoopDirective::Create(getASTContext(), StartLoc, EndLoc,
                                             NestedLoopCount, Clauses, AStmt, B,
                                             DSAStack->isCancelRegion());
 }
 
-StmtResult Sema::ActOnOpenMPMaskedTaskLoopDirective(
+StmtResult SemaOpenMP::ActOnOpenMPMaskedTaskLoopDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -13864,33 +13901,33 @@ StmtResult Sema::ActOnOpenMPMaskedTaskLoopDirective(
   // define the nested loops number.
   unsigned NestedLoopCount =
       checkOpenMPLoop(OMPD_masked_taskloop, getCollapseNumberExpr(Clauses),
-                      /*OrderedLoopCountExpr=*/nullptr, AStmt, *this, *DSAStack,
-                      VarsWithImplicitDSA, B);
+                      /*OrderedLoopCountExpr=*/nullptr, AStmt, SemaRef,
+                      *DSAStack, VarsWithImplicitDSA, B);
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp for loop exprs were not built");
 
   // OpenMP, [2.9.2 taskloop Construct, Restrictions]
   // The grainsize clause and num_tasks clause are mutually exclusive and may
   // not appear on the same taskloop directive.
-  if (checkMutuallyExclusiveClauses(*this, Clauses,
+  if (checkMutuallyExclusiveClauses(SemaRef, Clauses,
                                     {OMPC_grainsize, OMPC_num_tasks}))
     return StmtError();
   // OpenMP, [2.9.2 taskloop Construct, Restrictions]
   // If a reduction clause is present on the taskloop directive, the nogroup
   // clause must not be specified.
-  if (checkReductionClauseWithNogroup(*this, Clauses))
+  if (checkReductionClauseWithNogroup(SemaRef, Clauses))
     return StmtError();
 
-  setFunctionHasBranchProtectedScope();
-  return OMPMaskedTaskLoopDirective::Create(Context, StartLoc, EndLoc,
+  SemaRef.setFunctionHasBranchProtectedScope();
+  return OMPMaskedTaskLoopDirective::Create(getASTContext(), StartLoc, EndLoc,
                                             NestedLoopCount, Clauses, AStmt, B,
                                             DSAStack->isCancelRegion());
 }
 
-StmtResult Sema::ActOnOpenMPMasterTaskLoopSimdDirective(
+StmtResult SemaOpenMP::ActOnOpenMPMasterTaskLoopSimdDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -13902,21 +13939,21 @@ StmtResult Sema::ActOnOpenMPMasterTaskLoopSimdDirective(
   // define the nested loops number.
   unsigned NestedLoopCount =
       checkOpenMPLoop(OMPD_master_taskloop_simd, getCollapseNumberExpr(Clauses),
-                      /*OrderedLoopCountExpr=*/nullptr, AStmt, *this, *DSAStack,
-                      VarsWithImplicitDSA, B);
+                      /*OrderedLoopCountExpr=*/nullptr, AStmt, SemaRef,
+                      *DSAStack, VarsWithImplicitDSA, B);
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp for loop exprs were not built");
 
-  if (!CurContext->isDependentContext()) {
+  if (!SemaRef.CurContext->isDependentContext()) {
     // Finalize the clauses that need pre-built expressions for CodeGen.
     for (OMPClause *C : Clauses) {
       if (auto *LC = dyn_cast<OMPLinearClause>(C))
         if (FinishOpenMPLinearClause(*LC, cast<DeclRefExpr>(B.IterationVarRef),
-                                     B.NumIterations, *this, CurScope,
-                                     DSAStack))
+                                     B.NumIterations, SemaRef,
+                                     SemaRef.getCurScope(), DSAStack))
           return StmtError();
     }
   }
@@ -13924,23 +13961,23 @@ StmtResult Sema::ActOnOpenMPMasterTaskLoopSimdDirective(
   // OpenMP, [2.9.2 taskloop Construct, Restrictions]
   // The grainsize clause and num_tasks clause are mutually exclusive and may
   // not appear on the same taskloop directive.
-  if (checkMutuallyExclusiveClauses(*this, Clauses,
+  if (checkMutuallyExclusiveClauses(SemaRef, Clauses,
                                     {OMPC_grainsize, OMPC_num_tasks}))
     return StmtError();
   // OpenMP, [2.9.2 taskloop Construct, Restrictions]
   // If a reduction clause is present on the taskloop directive, the nogroup
   // clause must not be specified.
-  if (checkReductionClauseWithNogroup(*this, Clauses))
+  if (checkReductionClauseWithNogroup(SemaRef, Clauses))
     return StmtError();
-  if (checkSimdlenSafelenSpecified(*this, Clauses))
+  if (checkSimdlenSafelenSpecified(SemaRef, Clauses))
     return StmtError();
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
   return OMPMasterTaskLoopSimdDirective::Create(
-      Context, StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
+      getASTContext(), StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
 }
 
-StmtResult Sema::ActOnOpenMPMaskedTaskLoopSimdDirective(
+StmtResult SemaOpenMP::ActOnOpenMPMaskedTaskLoopSimdDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -13952,21 +13989,21 @@ StmtResult Sema::ActOnOpenMPMaskedTaskLoopSimdDirective(
   // define the nested loops number.
   unsigned NestedLoopCount =
       checkOpenMPLoop(OMPD_masked_taskloop_simd, getCollapseNumberExpr(Clauses),
-                      /*OrderedLoopCountExpr=*/nullptr, AStmt, *this, *DSAStack,
-                      VarsWithImplicitDSA, B);
+                      /*OrderedLoopCountExpr=*/nullptr, AStmt, SemaRef,
+                      *DSAStack, VarsWithImplicitDSA, B);
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp for loop exprs were not built");
 
-  if (!CurContext->isDependentContext()) {
+  if (!SemaRef.CurContext->isDependentContext()) {
     // Finalize the clauses that need pre-built expressions for CodeGen.
     for (OMPClause *C : Clauses) {
       if (auto *LC = dyn_cast<OMPLinearClause>(C))
         if (FinishOpenMPLinearClause(*LC, cast<DeclRefExpr>(B.IterationVarRef),
-                                     B.NumIterations, *this, CurScope,
-                                     DSAStack))
+                                     B.NumIterations, SemaRef,
+                                     SemaRef.getCurScope(), DSAStack))
           return StmtError();
     }
   }
@@ -13974,23 +14011,23 @@ StmtResult Sema::ActOnOpenMPMaskedTaskLoopSimdDirective(
   // OpenMP, [2.9.2 taskloop Construct, Restrictions]
   // The grainsize clause and num_tasks clause are mutually exclusive and may
   // not appear on the same taskloop directive.
-  if (checkMutuallyExclusiveClauses(*this, Clauses,
+  if (checkMutuallyExclusiveClauses(SemaRef, Clauses,
                                     {OMPC_grainsize, OMPC_num_tasks}))
     return StmtError();
   // OpenMP, [2.9.2 taskloop Construct, Restrictions]
   // If a reduction clause is present on the taskloop directive, the nogroup
   // clause must not be specified.
-  if (checkReductionClauseWithNogroup(*this, Clauses))
+  if (checkReductionClauseWithNogroup(SemaRef, Clauses))
     return StmtError();
-  if (checkSimdlenSafelenSpecified(*this, Clauses))
+  if (checkSimdlenSafelenSpecified(SemaRef, Clauses))
     return StmtError();
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
   return OMPMaskedTaskLoopSimdDirective::Create(
-      Context, StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
+      getASTContext(), StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
 }
 
-StmtResult Sema::ActOnOpenMPParallelMasterTaskLoopDirective(
+StmtResult SemaOpenMP::ActOnOpenMPParallelMasterTaskLoopDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -14021,33 +14058,33 @@ StmtResult Sema::ActOnOpenMPParallelMasterTaskLoopDirective(
   // define the nested loops number.
   unsigned NestedLoopCount = checkOpenMPLoop(
       OMPD_parallel_master_taskloop, getCollapseNumberExpr(Clauses),
-      /*OrderedLoopCountExpr=*/nullptr, CS, *this, *DSAStack,
+      /*OrderedLoopCountExpr=*/nullptr, CS, SemaRef, *DSAStack,
       VarsWithImplicitDSA, B);
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp for loop exprs were not built");
 
   // OpenMP, [2.9.2 taskloop Construct, Restrictions]
   // The grainsize clause and num_tasks clause are mutually exclusive and may
   // not appear on the same taskloop directive.
-  if (checkMutuallyExclusiveClauses(*this, Clauses,
+  if (checkMutuallyExclusiveClauses(SemaRef, Clauses,
                                     {OMPC_grainsize, OMPC_num_tasks}))
     return StmtError();
   // OpenMP, [2.9.2 taskloop Construct, Restrictions]
   // If a reduction clause is present on the taskloop directive, the nogroup
   // clause must not be specified.
-  if (checkReductionClauseWithNogroup(*this, Clauses))
+  if (checkReductionClauseWithNogroup(SemaRef, Clauses))
     return StmtError();
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
   return OMPParallelMasterTaskLoopDirective::Create(
-      Context, StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B,
+      getASTContext(), StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B,
       DSAStack->isCancelRegion());
 }
 
-StmtResult Sema::ActOnOpenMPParallelMaskedTaskLoopDirective(
+StmtResult SemaOpenMP::ActOnOpenMPParallelMaskedTaskLoopDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -14078,33 +14115,33 @@ StmtResult Sema::ActOnOpenMPParallelMaskedTaskLoopDirective(
   // define the nested loops number.
   unsigned NestedLoopCount = checkOpenMPLoop(
       OMPD_parallel_masked_taskloop, getCollapseNumberExpr(Clauses),
-      /*OrderedLoopCountExpr=*/nullptr, CS, *this, *DSAStack,
+      /*OrderedLoopCountExpr=*/nullptr, CS, SemaRef, *DSAStack,
       VarsWithImplicitDSA, B);
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp for loop exprs were not built");
 
   // OpenMP, [2.9.2 taskloop Construct, Restrictions]
   // The grainsize clause and num_tasks clause are mutually exclusive and may
   // not appear on the same taskloop directive.
-  if (checkMutuallyExclusiveClauses(*this, Clauses,
+  if (checkMutuallyExclusiveClauses(SemaRef, Clauses,
                                     {OMPC_grainsize, OMPC_num_tasks}))
     return StmtError();
   // OpenMP, [2.9.2 taskloop Construct, Restrictions]
   // If a reduction clause is present on the taskloop directive, the nogroup
   // clause must not be specified.
-  if (checkReductionClauseWithNogroup(*this, Clauses))
+  if (checkReductionClauseWithNogroup(SemaRef, Clauses))
     return StmtError();
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
   return OMPParallelMaskedTaskLoopDirective::Create(
-      Context, StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B,
+      getASTContext(), StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B,
       DSAStack->isCancelRegion());
 }
 
-StmtResult Sema::ActOnOpenMPParallelMasterTaskLoopSimdDirective(
+StmtResult SemaOpenMP::ActOnOpenMPParallelMasterTaskLoopSimdDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -14135,21 +14172,21 @@ StmtResult Sema::ActOnOpenMPParallelMasterTaskLoopSimdDirective(
   // define the nested loops number.
   unsigned NestedLoopCount = checkOpenMPLoop(
       OMPD_parallel_master_taskloop_simd, getCollapseNumberExpr(Clauses),
-      /*OrderedLoopCountExpr=*/nullptr, CS, *this, *DSAStack,
+      /*OrderedLoopCountExpr=*/nullptr, CS, SemaRef, *DSAStack,
       VarsWithImplicitDSA, B);
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp for loop exprs were not built");
 
-  if (!CurContext->isDependentContext()) {
+  if (!SemaRef.CurContext->isDependentContext()) {
     // Finalize the clauses that need pre-built expressions for CodeGen.
     for (OMPClause *C : Clauses) {
       if (auto *LC = dyn_cast<OMPLinearClause>(C))
         if (FinishOpenMPLinearClause(*LC, cast<DeclRefExpr>(B.IterationVarRef),
-                                     B.NumIterations, *this, CurScope,
-                                     DSAStack))
+                                     B.NumIterations, SemaRef,
+                                     SemaRef.getCurScope(), DSAStack))
           return StmtError();
     }
   }
@@ -14157,23 +14194,23 @@ StmtResult Sema::ActOnOpenMPParallelMasterTaskLoopSimdDirective(
   // OpenMP, [2.9.2 taskloop Construct, Restrictions]
   // The grainsize clause and num_tasks clause are mutually exclusive and may
   // not appear on the same taskloop directive.
-  if (checkMutuallyExclusiveClauses(*this, Clauses,
+  if (checkMutuallyExclusiveClauses(SemaRef, Clauses,
                                     {OMPC_grainsize, OMPC_num_tasks}))
     return StmtError();
   // OpenMP, [2.9.2 taskloop Construct, Restrictions]
   // If a reduction clause is present on the taskloop directive, the nogroup
   // clause must not be specified.
-  if (checkReductionClauseWithNogroup(*this, Clauses))
+  if (checkReductionClauseWithNogroup(SemaRef, Clauses))
     return StmtError();
-  if (checkSimdlenSafelenSpecified(*this, Clauses))
+  if (checkSimdlenSafelenSpecified(SemaRef, Clauses))
     return StmtError();
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
   return OMPParallelMasterTaskLoopSimdDirective::Create(
-      Context, StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
+      getASTContext(), StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
 }
 
-StmtResult Sema::ActOnOpenMPParallelMaskedTaskLoopSimdDirective(
+StmtResult SemaOpenMP::ActOnOpenMPParallelMaskedTaskLoopSimdDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -14204,21 +14241,21 @@ StmtResult Sema::ActOnOpenMPParallelMaskedTaskLoopSimdDirective(
   // define the nested loops number.
   unsigned NestedLoopCount = checkOpenMPLoop(
       OMPD_parallel_masked_taskloop_simd, getCollapseNumberExpr(Clauses),
-      /*OrderedLoopCountExpr=*/nullptr, CS, *this, *DSAStack,
+      /*OrderedLoopCountExpr=*/nullptr, CS, SemaRef, *DSAStack,
       VarsWithImplicitDSA, B);
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp for loop exprs were not built");
 
-  if (!CurContext->isDependentContext()) {
+  if (!SemaRef.CurContext->isDependentContext()) {
     // Finalize the clauses that need pre-built expressions for CodeGen.
     for (OMPClause *C : Clauses) {
       if (auto *LC = dyn_cast<OMPLinearClause>(C))
         if (FinishOpenMPLinearClause(*LC, cast<DeclRefExpr>(B.IterationVarRef),
-                                     B.NumIterations, *this, CurScope,
-                                     DSAStack))
+                                     B.NumIterations, SemaRef,
+                                     SemaRef.getCurScope(), DSAStack))
           return StmtError();
     }
   }
@@ -14226,23 +14263,23 @@ StmtResult Sema::ActOnOpenMPParallelMaskedTaskLoopSimdDirective(
   // OpenMP, [2.9.2 taskloop Construct, Restrictions]
   // The grainsize clause and num_tasks clause are mutually exclusive and may
   // not appear on the same taskloop directive.
-  if (checkMutuallyExclusiveClauses(*this, Clauses,
+  if (checkMutuallyExclusiveClauses(SemaRef, Clauses,
                                     {OMPC_grainsize, OMPC_num_tasks}))
     return StmtError();
   // OpenMP, [2.9.2 taskloop Construct, Restrictions]
   // If a reduction clause is present on the taskloop directive, the nogroup
   // clause must not be specified.
-  if (checkReductionClauseWithNogroup(*this, Clauses))
+  if (checkReductionClauseWithNogroup(SemaRef, Clauses))
     return StmtError();
-  if (checkSimdlenSafelenSpecified(*this, Clauses))
+  if (checkSimdlenSafelenSpecified(SemaRef, Clauses))
     return StmtError();
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
   return OMPParallelMaskedTaskLoopSimdDirective::Create(
-      Context, StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
+      getASTContext(), StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
 }
 
-StmtResult Sema::ActOnOpenMPDistributeDirective(
+StmtResult SemaOpenMP::ActOnOpenMPDistributeDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -14258,21 +14295,21 @@ StmtResult Sema::ActOnOpenMPDistributeDirective(
   unsigned NestedLoopCount =
       checkOpenMPLoop(OMPD_distribute, getCollapseNumberExpr(Clauses),
                       nullptr /*ordered not a clause on distribute*/, AStmt,
-                      *this, *DSAStack, VarsWithImplicitDSA, B);
+                      SemaRef, *DSAStack, VarsWithImplicitDSA, B);
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp for loop exprs were not built");
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
   auto *DistributeDirective = OMPDistributeDirective::Create(
-      Context, StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B,
+      getASTContext(), StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B,
       DSAStack->getMappedDirective());
   return DistributeDirective;
 }
 
-StmtResult Sema::ActOnOpenMPDistributeParallelForDirective(
+StmtResult SemaOpenMP::ActOnOpenMPDistributeParallelForDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -14302,21 +14339,21 @@ StmtResult Sema::ActOnOpenMPDistributeParallelForDirective(
   // define the nested loops number.
   unsigned NestedLoopCount = checkOpenMPLoop(
       OMPD_distribute_parallel_for, getCollapseNumberExpr(Clauses),
-      nullptr /*ordered not a clause on distribute*/, CS, *this, *DSAStack,
+      nullptr /*ordered not a clause on distribute*/, CS, SemaRef, *DSAStack,
       VarsWithImplicitDSA, B);
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp for loop exprs were not built");
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
   return OMPDistributeParallelForDirective::Create(
-      Context, StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B,
+      getASTContext(), StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B,
       DSAStack->getTaskgroupReductionRef(), DSAStack->isCancelRegion());
 }
 
-StmtResult Sema::ActOnOpenMPDistributeParallelForSimdDirective(
+StmtResult SemaOpenMP::ActOnOpenMPDistributeParallelForSimdDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -14346,34 +14383,34 @@ StmtResult Sema::ActOnOpenMPDistributeParallelForSimdDirective(
   // define the nested loops number.
   unsigned NestedLoopCount = checkOpenMPLoop(
       OMPD_distribute_parallel_for_simd, getCollapseNumberExpr(Clauses),
-      nullptr /*ordered not a clause on distribute*/, CS, *this, *DSAStack,
+      nullptr /*ordered not a clause on distribute*/, CS, SemaRef, *DSAStack,
       VarsWithImplicitDSA, B);
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp for loop exprs were not built");
 
-  if (!CurContext->isDependentContext()) {
+  if (!SemaRef.CurContext->isDependentContext()) {
     // Finalize the clauses that need pre-built expressions for CodeGen.
     for (OMPClause *C : Clauses) {
       if (auto *LC = dyn_cast<OMPLinearClause>(C))
         if (FinishOpenMPLinearClause(*LC, cast<DeclRefExpr>(B.IterationVarRef),
-                                     B.NumIterations, *this, CurScope,
-                                     DSAStack))
+                                     B.NumIterations, SemaRef,
+                                     SemaRef.getCurScope(), DSAStack))
           return StmtError();
     }
   }
 
-  if (checkSimdlenSafelenSpecified(*this, Clauses))
+  if (checkSimdlenSafelenSpecified(SemaRef, Clauses))
     return StmtError();
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
   return OMPDistributeParallelForSimdDirective::Create(
-      Context, StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
+      getASTContext(), StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
 }
 
-StmtResult Sema::ActOnOpenMPDistributeSimdDirective(
+StmtResult SemaOpenMP::ActOnOpenMPDistributeSimdDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -14402,34 +14439,34 @@ StmtResult Sema::ActOnOpenMPDistributeSimdDirective(
   // define the nested loops number.
   unsigned NestedLoopCount =
       checkOpenMPLoop(OMPD_distribute_simd, getCollapseNumberExpr(Clauses),
-                      nullptr /*ordered not a clause on distribute*/, CS, *this,
-                      *DSAStack, VarsWithImplicitDSA, B);
+                      nullptr /*ordered not a clause on distribute*/, CS,
+                      SemaRef, *DSAStack, VarsWithImplicitDSA, B);
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp for loop exprs were not built");
 
-  if (!CurContext->isDependentContext()) {
+  if (!SemaRef.CurContext->isDependentContext()) {
     // Finalize the clauses that need pre-built expressions for CodeGen.
     for (OMPClause *C : Clauses) {
       if (auto *LC = dyn_cast<OMPLinearClause>(C))
         if (FinishOpenMPLinearClause(*LC, cast<DeclRefExpr>(B.IterationVarRef),
-                                     B.NumIterations, *this, CurScope,
-                                     DSAStack))
+                                     B.NumIterations, SemaRef,
+                                     SemaRef.getCurScope(), DSAStack))
           return StmtError();
     }
   }
 
-  if (checkSimdlenSafelenSpecified(*this, Clauses))
+  if (checkSimdlenSafelenSpecified(SemaRef, Clauses))
     return StmtError();
 
-  setFunctionHasBranchProtectedScope();
-  return OMPDistributeSimdDirective::Create(Context, StartLoc, EndLoc,
+  SemaRef.setFunctionHasBranchProtectedScope();
+  return OMPDistributeSimdDirective::Create(getASTContext(), StartLoc, EndLoc,
                                             NestedLoopCount, Clauses, AStmt, B);
 }
 
-StmtResult Sema::ActOnOpenMPTargetParallelForSimdDirective(
+StmtResult SemaOpenMP::ActOnOpenMPTargetParallelForSimdDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -14459,33 +14496,33 @@ StmtResult Sema::ActOnOpenMPTargetParallelForSimdDirective(
   // define the nested loops number.
   unsigned NestedLoopCount = checkOpenMPLoop(
       OMPD_target_parallel_for_simd, getCollapseNumberExpr(Clauses),
-      getOrderedNumberExpr(Clauses), CS, *this, *DSAStack, VarsWithImplicitDSA,
-      B);
+      getOrderedNumberExpr(Clauses), CS, SemaRef, *DSAStack,
+      VarsWithImplicitDSA, B);
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp target parallel for simd loop exprs were not built");
 
-  if (!CurContext->isDependentContext()) {
+  if (!SemaRef.CurContext->isDependentContext()) {
     // Finalize the clauses that need pre-built expressions for CodeGen.
     for (OMPClause *C : Clauses) {
       if (auto *LC = dyn_cast<OMPLinearClause>(C))
         if (FinishOpenMPLinearClause(*LC, cast<DeclRefExpr>(B.IterationVarRef),
-                                     B.NumIterations, *this, CurScope,
-                                     DSAStack))
+                                     B.NumIterations, SemaRef,
+                                     SemaRef.getCurScope(), DSAStack))
           return StmtError();
     }
   }
-  if (checkSimdlenSafelenSpecified(*this, Clauses))
+  if (checkSimdlenSafelenSpecified(SemaRef, Clauses))
     return StmtError();
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
   return OMPTargetParallelForSimdDirective::Create(
-      Context, StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
+      getASTContext(), StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
 }
 
-StmtResult Sema::ActOnOpenMPTargetSimdDirective(
+StmtResult SemaOpenMP::ActOnOpenMPTargetSimdDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -14514,34 +14551,34 @@ StmtResult Sema::ActOnOpenMPTargetSimdDirective(
   // nested loops number.
   unsigned NestedLoopCount =
       checkOpenMPLoop(OMPD_target_simd, getCollapseNumberExpr(Clauses),
-                      getOrderedNumberExpr(Clauses), CS, *this, *DSAStack,
+                      getOrderedNumberExpr(Clauses), CS, SemaRef, *DSAStack,
                       VarsWithImplicitDSA, B);
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp target simd loop exprs were not built");
 
-  if (!CurContext->isDependentContext()) {
+  if (!SemaRef.CurContext->isDependentContext()) {
     // Finalize the clauses that need pre-built expressions for CodeGen.
     for (OMPClause *C : Clauses) {
       if (auto *LC = dyn_cast<OMPLinearClause>(C))
         if (FinishOpenMPLinearClause(*LC, cast<DeclRefExpr>(B.IterationVarRef),
-                                     B.NumIterations, *this, CurScope,
-                                     DSAStack))
+                                     B.NumIterations, SemaRef,
+                                     SemaRef.getCurScope(), DSAStack))
           return StmtError();
     }
   }
 
-  if (checkSimdlenSafelenSpecified(*this, Clauses))
+  if (checkSimdlenSafelenSpecified(SemaRef, Clauses))
     return StmtError();
 
-  setFunctionHasBranchProtectedScope();
-  return OMPTargetSimdDirective::Create(Context, StartLoc, EndLoc,
+  SemaRef.setFunctionHasBranchProtectedScope();
+  return OMPTargetSimdDirective::Create(getASTContext(), StartLoc, EndLoc,
                                         NestedLoopCount, Clauses, AStmt, B);
 }
 
-StmtResult Sema::ActOnOpenMPTeamsDistributeDirective(
+StmtResult SemaOpenMP::ActOnOpenMPTeamsDistributeDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -14570,23 +14607,23 @@ StmtResult Sema::ActOnOpenMPTeamsDistributeDirective(
   // define the nested loops number.
   unsigned NestedLoopCount =
       checkOpenMPLoop(OMPD_teams_distribute, getCollapseNumberExpr(Clauses),
-                      nullptr /*ordered not a clause on distribute*/, CS, *this,
-                      *DSAStack, VarsWithImplicitDSA, B);
+                      nullptr /*ordered not a clause on distribute*/, CS,
+                      SemaRef, *DSAStack, VarsWithImplicitDSA, B);
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp teams distribute loop exprs were not built");
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
 
   DSAStack->setParentTeamsRegionLoc(StartLoc);
 
   return OMPTeamsDistributeDirective::Create(
-      Context, StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
+      getASTContext(), StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
 }
 
-StmtResult Sema::ActOnOpenMPTeamsDistributeSimdDirective(
+StmtResult SemaOpenMP::ActOnOpenMPTeamsDistributeSimdDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -14616,38 +14653,38 @@ StmtResult Sema::ActOnOpenMPTeamsDistributeSimdDirective(
   // define the nested loops number.
   unsigned NestedLoopCount = checkOpenMPLoop(
       OMPD_teams_distribute_simd, getCollapseNumberExpr(Clauses),
-      nullptr /*ordered not a clause on distribute*/, CS, *this, *DSAStack,
+      nullptr /*ordered not a clause on distribute*/, CS, SemaRef, *DSAStack,
       VarsWithImplicitDSA, B);
 
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp teams distribute simd loop exprs were not built");
 
-  if (!CurContext->isDependentContext()) {
+  if (!SemaRef.CurContext->isDependentContext()) {
     // Finalize the clauses that need pre-built expressions for CodeGen.
     for (OMPClause *C : Clauses) {
       if (auto *LC = dyn_cast<OMPLinearClause>(C))
         if (FinishOpenMPLinearClause(*LC, cast<DeclRefExpr>(B.IterationVarRef),
-                                     B.NumIterations, *this, CurScope,
-                                     DSAStack))
+                                     B.NumIterations, SemaRef,
+                                     SemaRef.getCurScope(), DSAStack))
           return StmtError();
     }
   }
 
-  if (checkSimdlenSafelenSpecified(*this, Clauses))
+  if (checkSimdlenSafelenSpecified(SemaRef, Clauses))
     return StmtError();
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
 
   DSAStack->setParentTeamsRegionLoc(StartLoc);
 
   return OMPTeamsDistributeSimdDirective::Create(
-      Context, StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
+      getASTContext(), StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
 }
 
-StmtResult Sema::ActOnOpenMPTeamsDistributeParallelForSimdDirective(
+StmtResult SemaOpenMP::ActOnOpenMPTeamsDistributeParallelForSimdDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -14678,38 +14715,38 @@ StmtResult Sema::ActOnOpenMPTeamsDistributeParallelForSimdDirective(
   // define the nested loops number.
   unsigned NestedLoopCount = checkOpenMPLoop(
       OMPD_teams_distribute_parallel_for_simd, getCollapseNumberExpr(Clauses),
-      nullptr /*ordered not a clause on distribute*/, CS, *this, *DSAStack,
+      nullptr /*ordered not a clause on distribute*/, CS, SemaRef, *DSAStack,
       VarsWithImplicitDSA, B);
 
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp for loop exprs were not built");
 
-  if (!CurContext->isDependentContext()) {
+  if (!SemaRef.CurContext->isDependentContext()) {
     // Finalize the clauses that need pre-built expressions for CodeGen.
     for (OMPClause *C : Clauses) {
       if (auto *LC = dyn_cast<OMPLinearClause>(C))
         if (FinishOpenMPLinearClause(*LC, cast<DeclRefExpr>(B.IterationVarRef),
-                                     B.NumIterations, *this, CurScope,
-                                     DSAStack))
+                                     B.NumIterations, SemaRef,
+                                     SemaRef.getCurScope(), DSAStack))
           return StmtError();
     }
   }
 
-  if (checkSimdlenSafelenSpecified(*this, Clauses))
+  if (checkSimdlenSafelenSpecified(SemaRef, Clauses))
     return StmtError();
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
 
   DSAStack->setParentTeamsRegionLoc(StartLoc);
 
   return OMPTeamsDistributeParallelForSimdDirective::Create(
-      Context, StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
+      getASTContext(), StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
 }
 
-StmtResult Sema::ActOnOpenMPTeamsDistributeParallelForDirective(
+StmtResult SemaOpenMP::ActOnOpenMPTeamsDistributeParallelForDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -14740,28 +14777,27 @@ StmtResult Sema::ActOnOpenMPTeamsDistributeParallelForDirective(
   // define the nested loops number.
   unsigned NestedLoopCount = checkOpenMPLoop(
       OMPD_teams_distribute_parallel_for, getCollapseNumberExpr(Clauses),
-      nullptr /*ordered not a clause on distribute*/, CS, *this, *DSAStack,
+      nullptr /*ordered not a clause on distribute*/, CS, SemaRef, *DSAStack,
       VarsWithImplicitDSA, B);
 
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp for loop exprs were not built");
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
 
   DSAStack->setParentTeamsRegionLoc(StartLoc);
 
   return OMPTeamsDistributeParallelForDirective::Create(
-      Context, StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B,
+      getASTContext(), StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B,
       DSAStack->getTaskgroupReductionRef(), DSAStack->isCancelRegion());
 }
 
-StmtResult Sema::ActOnOpenMPTargetTeamsDirective(ArrayRef<OMPClause *> Clauses,
-                                                 Stmt *AStmt,
-                                                 SourceLocation StartLoc,
-                                                 SourceLocation EndLoc) {
+StmtResult SemaOpenMP::ActOnOpenMPTargetTeamsDirective(
+    ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
+    SourceLocation EndLoc) {
   if (!AStmt)
     return StmtError();
 
@@ -14783,7 +14819,7 @@ StmtResult Sema::ActOnOpenMPTargetTeamsDirective(ArrayRef<OMPClause *> Clauses,
     // longjmp() and throw() must not violate the entry/exit criteria.
     CS->getCapturedDecl()->setNothrow();
   }
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
 
   const OMPClause *BareClause = nullptr;
   bool HasThreadLimitAndNumTeamsClause = hasClauses(Clauses, OMPC_num_teams) &&
@@ -14798,11 +14834,11 @@ StmtResult Sema::ActOnOpenMPTargetTeamsDirective(ArrayRef<OMPClause *> Clauses,
     return StmtError();
   }
 
-  return OMPTargetTeamsDirective::Create(Context, StartLoc, EndLoc, Clauses,
-                                         AStmt);
+  return OMPTargetTeamsDirective::Create(getASTContext(), StartLoc, EndLoc,
+                                         Clauses, AStmt);
 }
 
-StmtResult Sema::ActOnOpenMPTargetTeamsDistributeDirective(
+StmtResult SemaOpenMP::ActOnOpenMPTargetTeamsDistributeDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -14832,20 +14868,20 @@ StmtResult Sema::ActOnOpenMPTargetTeamsDistributeDirective(
   // define the nested loops number.
   unsigned NestedLoopCount = checkOpenMPLoop(
       OMPD_target_teams_distribute, getCollapseNumberExpr(Clauses),
-      nullptr /*ordered not a clause on distribute*/, CS, *this, *DSAStack,
+      nullptr /*ordered not a clause on distribute*/, CS, SemaRef, *DSAStack,
       VarsWithImplicitDSA, B);
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp target teams distribute loop exprs were not built");
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
   return OMPTargetTeamsDistributeDirective::Create(
-      Context, StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
+      getASTContext(), StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
 }
 
-StmtResult Sema::ActOnOpenMPTargetTeamsDistributeParallelForDirective(
+StmtResult SemaOpenMP::ActOnOpenMPTargetTeamsDistributeParallelForDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -14875,32 +14911,32 @@ StmtResult Sema::ActOnOpenMPTargetTeamsDistributeParallelForDirective(
   // define the nested loops number.
   unsigned NestedLoopCount = checkOpenMPLoop(
       OMPD_target_teams_distribute_parallel_for, getCollapseNumberExpr(Clauses),
-      nullptr /*ordered not a clause on distribute*/, CS, *this, *DSAStack,
+      nullptr /*ordered not a clause on distribute*/, CS, SemaRef, *DSAStack,
       VarsWithImplicitDSA, B);
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp target teams distribute parallel for loop exprs were not built");
 
-  if (!CurContext->isDependentContext()) {
+  if (!SemaRef.CurContext->isDependentContext()) {
     // Finalize the clauses that need pre-built expressions for CodeGen.
     for (OMPClause *C : Clauses) {
       if (auto *LC = dyn_cast<OMPLinearClause>(C))
         if (FinishOpenMPLinearClause(*LC, cast<DeclRefExpr>(B.IterationVarRef),
-                                     B.NumIterations, *this, CurScope,
-                                     DSAStack))
+                                     B.NumIterations, SemaRef,
+                                     SemaRef.getCurScope(), DSAStack))
           return StmtError();
     }
   }
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
   return OMPTargetTeamsDistributeParallelForDirective::Create(
-      Context, StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B,
+      getASTContext(), StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B,
       DSAStack->getTaskgroupReductionRef(), DSAStack->isCancelRegion());
 }
 
-StmtResult Sema::ActOnOpenMPTargetTeamsDistributeParallelForSimdDirective(
+StmtResult SemaOpenMP::ActOnOpenMPTargetTeamsDistributeParallelForSimdDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -14931,35 +14967,35 @@ StmtResult Sema::ActOnOpenMPTargetTeamsDistributeParallelForSimdDirective(
   unsigned NestedLoopCount =
       checkOpenMPLoop(OMPD_target_teams_distribute_parallel_for_simd,
                       getCollapseNumberExpr(Clauses),
-                      nullptr /*ordered not a clause on distribute*/, CS, *this,
-                      *DSAStack, VarsWithImplicitDSA, B);
+                      nullptr /*ordered not a clause on distribute*/, CS,
+                      SemaRef, *DSAStack, VarsWithImplicitDSA, B);
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp target teams distribute parallel for simd loop exprs were not "
          "built");
 
-  if (!CurContext->isDependentContext()) {
+  if (!SemaRef.CurContext->isDependentContext()) {
     // Finalize the clauses that need pre-built expressions for CodeGen.
     for (OMPClause *C : Clauses) {
       if (auto *LC = dyn_cast<OMPLinearClause>(C))
         if (FinishOpenMPLinearClause(*LC, cast<DeclRefExpr>(B.IterationVarRef),
-                                     B.NumIterations, *this, CurScope,
-                                     DSAStack))
+                                     B.NumIterations, SemaRef,
+                                     SemaRef.getCurScope(), DSAStack))
           return StmtError();
     }
   }
 
-  if (checkSimdlenSafelenSpecified(*this, Clauses))
+  if (checkSimdlenSafelenSpecified(SemaRef, Clauses))
     return StmtError();
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
   return OMPTargetTeamsDistributeParallelForSimdDirective::Create(
-      Context, StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
+      getASTContext(), StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
 }
 
-StmtResult Sema::ActOnOpenMPTargetTeamsDistributeSimdDirective(
+StmtResult SemaOpenMP::ActOnOpenMPTargetTeamsDistributeSimdDirective(
     ArrayRef<OMPClause *> Clauses, Stmt *AStmt, SourceLocation StartLoc,
     SourceLocation EndLoc, VarsWithInheritedDSAType &VarsWithImplicitDSA) {
   if (!AStmt)
@@ -14989,34 +15025,34 @@ StmtResult Sema::ActOnOpenMPTargetTeamsDistributeSimdDirective(
   // define the nested loops number.
   unsigned NestedLoopCount = checkOpenMPLoop(
       OMPD_target_teams_distribute_simd, getCollapseNumberExpr(Clauses),
-      nullptr /*ordered not a clause on distribute*/, CS, *this, *DSAStack,
+      nullptr /*ordered not a clause on distribute*/, CS, SemaRef, *DSAStack,
       VarsWithImplicitDSA, B);
   if (NestedLoopCount == 0)
     return StmtError();
 
-  assert((CurContext->isDependentContext() || B.builtAll()) &&
+  assert((SemaRef.CurContext->isDependentContext() || B.builtAll()) &&
          "omp target teams distribute simd loop exprs were not built");
 
-  if (!CurContext->isDependentContext()) {
+  if (!SemaRef.CurContext->isDependentContext()) {
     // Finalize the clauses that need pre-built expressions for CodeGen.
     for (OMPClause *C : Clauses) {
       if (auto *LC = dyn_cast<OMPLinearClause>(C))
         if (FinishOpenMPLinearClause(*LC, cast<DeclRefExpr>(B.IterationVarRef),
-                                     B.NumIterations, *this, CurScope,
-                                     DSAStack))
+                                     B.NumIterations, SemaRef,
+                                     SemaRef.getCurScope(), DSAStack))
           return StmtError();
     }
   }
 
-  if (checkSimdlenSafelenSpecified(*this, Clauses))
+  if (checkSimdlenSafelenSpecified(SemaRef, Clauses))
     return StmtError();
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
   return OMPTargetTeamsDistributeSimdDirective::Create(
-      Context, StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
+      getASTContext(), StartLoc, EndLoc, NestedLoopCount, Clauses, AStmt, B);
 }
 
-bool Sema::checkTransformableLoopNest(
+bool SemaOpenMP::checkTransformableLoopNest(
     OpenMPDirectiveKind Kind, Stmt *AStmt, int NumLoops,
     SmallVectorImpl<OMPLoopBasedDirective::HelperExprs> &LoopHelpers,
     Stmt *&Body,
@@ -15029,7 +15065,7 @@ bool Sema::checkTransformableLoopNest(
                                                         Stmt *CurStmt) {
         VarsWithInheritedDSAType TmpDSA;
         unsigned SingleNumLoops =
-            checkOpenMPLoop(Kind, nullptr, nullptr, CurStmt, *this, *DSAStack,
+            checkOpenMPLoop(Kind, nullptr, nullptr, CurStmt, SemaRef, *DSAStack,
                             TmpDSA, LoopHelpers[Cnt]);
         if (SingleNumLoops == 0)
           return true;
@@ -15065,9 +15101,11 @@ bool Sema::checkTransformableLoopNest(
   return Result;
 }
 
-StmtResult Sema::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
-                                          Stmt *AStmt, SourceLocation StartLoc,
-                                          SourceLocation EndLoc) {
+StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
+                                                Stmt *AStmt,
+                                                SourceLocation StartLoc,
+                                                SourceLocation EndLoc) {
+  ASTContext &Context = getASTContext();
   auto SizesClauses =
       OMPExecutableDirective::getClausesOfKind<OMPSizesClause>(Clauses);
   if (SizesClauses.empty()) {
@@ -15091,7 +15129,7 @@ StmtResult Sema::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     return StmtError();
 
   // Delay tiling to when template is completely instantiated.
-  if (CurContext->isDependentContext())
+  if (SemaRef.CurContext->isDependentContext())
     return OMPTileDirective::Create(Context, StartLoc, EndLoc, Clauses,
                                     NumLoops, AStmt, nullptr, nullptr);
 
@@ -15117,7 +15155,7 @@ StmtResult Sema::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
       std::string FloorCntName =
           (Twine(".floor_") + llvm::utostr(I) + ".iv." + OrigVarName).str();
       VarDecl *FloorCntDecl =
-          buildVarDecl(*this, {}, CntTy, FloorCntName, nullptr, OrigCntVar);
+          buildVarDecl(SemaRef, {}, CntTy, FloorCntName, nullptr, OrigCntVar);
       FloorIndVars[I] = FloorCntDecl;
     }
 
@@ -15130,7 +15168,8 @@ StmtResult Sema::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
       // used by the expressions to derive the original iteration variable's
       // value from the logical iteration number.
       auto *TileCntDecl = cast<VarDecl>(IterVarRef->getDecl());
-      TileCntDecl->setDeclName(&PP.getIdentifierTable().get(TileCntName));
+      TileCntDecl->setDeclName(
+          &SemaRef.PP.getIdentifierTable().get(TileCntName));
       TileIndVars[I] = TileCntDecl;
     }
     for (auto &P : OriginalInits[I]) {
@@ -15159,17 +15198,18 @@ StmtResult Sema::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     auto *OrigCntVar = cast<DeclRefExpr>(LoopHelper.Counters[0]);
     QualType CntTy = OrigCntVar->getType();
     Expr *DimTileSize = SizesClause->getSizesRefs()[I];
-    Scope *CurScope = getCurScope();
+    Scope *CurScope = SemaRef.getCurScope();
 
     // Commonly used variables.
-    DeclRefExpr *TileIV = buildDeclRefExpr(*this, TileIndVars[I], CntTy,
+    DeclRefExpr *TileIV = buildDeclRefExpr(SemaRef, TileIndVars[I], CntTy,
                                            OrigCntVar->getExprLoc());
-    DeclRefExpr *FloorIV = buildDeclRefExpr(*this, FloorIndVars[I], CntTy,
+    DeclRefExpr *FloorIV = buildDeclRefExpr(SemaRef, FloorIndVars[I], CntTy,
                                             OrigCntVar->getExprLoc());
 
     // For init-statement: auto .tile.iv = .floor.iv
-    AddInitializerToDecl(TileIndVars[I], DefaultLvalueConversion(FloorIV).get(),
-                         /*DirectInit=*/false);
+    SemaRef.AddInitializerToDecl(TileIndVars[I],
+                                 SemaRef.DefaultLvalueConversion(FloorIV).get(),
+                                 /*DirectInit=*/false);
     Decl *CounterDecl = TileIndVars[I];
     StmtResult InitStmt = new (Context)
         DeclStmt(DeclGroupRef::Create(Context, &CounterDecl, 1),
@@ -15179,28 +15219,29 @@ StmtResult Sema::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
 
     // For cond-expression: .tile.iv < min(.floor.iv + DimTileSize,
     // NumIterations)
-    ExprResult EndOfTile = BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(),
-                                      BO_Add, FloorIV, DimTileSize);
+    ExprResult EndOfTile = SemaRef.BuildBinOp(
+        CurScope, LoopHelper.Cond->getExprLoc(), BO_Add, FloorIV, DimTileSize);
     if (!EndOfTile.isUsable())
       return StmtError();
     ExprResult IsPartialTile =
-        BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
-                   NumIterations, EndOfTile.get());
+        SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
+                           NumIterations, EndOfTile.get());
     if (!IsPartialTile.isUsable())
       return StmtError();
-    ExprResult MinTileAndIterSpace = ActOnConditionalOp(
+    ExprResult MinTileAndIterSpace = SemaRef.ActOnConditionalOp(
         LoopHelper.Cond->getBeginLoc(), LoopHelper.Cond->getEndLoc(),
         IsPartialTile.get(), NumIterations, EndOfTile.get());
     if (!MinTileAndIterSpace.isUsable())
       return StmtError();
-    ExprResult CondExpr = BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(),
-                                     BO_LT, TileIV, MinTileAndIterSpace.get());
+    ExprResult CondExpr =
+        SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
+                           TileIV, MinTileAndIterSpace.get());
     if (!CondExpr.isUsable())
       return StmtError();
 
     // For incr-statement: ++.tile.iv
-    ExprResult IncrStmt =
-        BuildUnaryOp(CurScope, LoopHelper.Inc->getExprLoc(), UO_PreInc, TileIV);
+    ExprResult IncrStmt = SemaRef.BuildUnaryOp(
+        CurScope, LoopHelper.Inc->getExprLoc(), UO_PreInc, TileIV);
     if (!IncrStmt.isUsable())
       return StmtError();
 
@@ -15235,16 +15276,16 @@ StmtResult Sema::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     DeclRefExpr *OrigCntVar = cast<DeclRefExpr>(LoopHelper.Counters[0]);
     QualType CntTy = OrigCntVar->getType();
     Expr *DimTileSize = SizesClause->getSizesRefs()[I];
-    Scope *CurScope = getCurScope();
+    Scope *CurScope = SemaRef.getCurScope();
 
     // Commonly used variables.
-    DeclRefExpr *FloorIV = buildDeclRefExpr(*this, FloorIndVars[I], CntTy,
+    DeclRefExpr *FloorIV = buildDeclRefExpr(SemaRef, FloorIndVars[I], CntTy,
                                             OrigCntVar->getExprLoc());
 
     // For init-statement: auto .floor.iv = 0
-    AddInitializerToDecl(
+    SemaRef.AddInitializerToDecl(
         FloorIndVars[I],
-        ActOnIntegerConstant(LoopHelper.Init->getExprLoc(), 0).get(),
+        SemaRef.ActOnIntegerConstant(LoopHelper.Init->getExprLoc(), 0).get(),
         /*DirectInit=*/false);
     Decl *CounterDecl = FloorIndVars[I];
     StmtResult InitStmt = new (Context)
@@ -15254,14 +15295,15 @@ StmtResult Sema::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
       return StmtError();
 
     // For cond-expression: .floor.iv < NumIterations
-    ExprResult CondExpr = BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(),
-                                     BO_LT, FloorIV, NumIterations);
+    ExprResult CondExpr = SemaRef.BuildBinOp(
+        CurScope, LoopHelper.Cond->getExprLoc(), BO_LT, FloorIV, NumIterations);
     if (!CondExpr.isUsable())
       return StmtError();
 
     // For incr-statement: .floor.iv += DimTileSize
-    ExprResult IncrStmt = BuildBinOp(CurScope, LoopHelper.Inc->getExprLoc(),
-                                     BO_AddAssign, FloorIV, DimTileSize);
+    ExprResult IncrStmt =
+        SemaRef.BuildBinOp(CurScope, LoopHelper.Inc->getExprLoc(), BO_AddAssign,
+                           FloorIV, DimTileSize);
     if (!IncrStmt.isUsable())
       return StmtError();
 
@@ -15276,15 +15318,18 @@ StmtResult Sema::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
                                   buildPreInits(Context, PreInits));
 }
 
-StmtResult Sema::ActOnOpenMPUnrollDirective(ArrayRef<OMPClause *> Clauses,
-                                            Stmt *AStmt,
-                                            SourceLocation StartLoc,
-                                            SourceLocation EndLoc) {
+StmtResult SemaOpenMP::ActOnOpenMPUnrollDirective(ArrayRef<OMPClause *> Clauses,
+                                                  Stmt *AStmt,
+                                                  SourceLocation StartLoc,
+                                                  SourceLocation EndLoc) {
+  ASTContext &Context = getASTContext();
+  Scope *CurScope = SemaRef.getCurScope();
   // Empty statement should only be possible if there already was an error.
   if (!AStmt)
     return StmtError();
 
-  if (checkMutuallyExclusiveClauses(*this, Clauses, {OMPC_partial, OMPC_full}))
+  if (checkMutuallyExclusiveClauses(SemaRef, Clauses,
+                                    {OMPC_partial, OMPC_full}))
     return StmtError();
 
   const OMPFullClause *FullClause =
@@ -15307,7 +15352,7 @@ StmtResult Sema::ActOnOpenMPUnrollDirective(ArrayRef<OMPClause *> Clauses,
   unsigned NumGeneratedLoops = PartialClause ? 1 : 0;
 
   // Delay unrolling to when template is completely instantiated.
-  if (CurContext->isDependentContext())
+  if (SemaRef.CurContext->isDependentContext())
     return OMPUnrollDirective::Create(Context, StartLoc, EndLoc, Clauses, AStmt,
                                       NumGeneratedLoops, nullptr, nullptr);
 
@@ -15412,8 +15457,8 @@ StmtResult Sema::ActOnOpenMPUnrollDirective(ArrayRef<OMPClause *> Clauses,
   assert(Factor > 0 && "Expected positive unroll factor");
   auto MakeFactorExpr = [this, Factor, IVTy, FactorLoc]() {
     return IntegerLiteral::Create(
-        Context, llvm::APInt(Context.getIntWidth(IVTy), Factor), IVTy,
-        FactorLoc);
+        getASTContext(), llvm::APInt(getASTContext().getIntWidth(IVTy), Factor),
+        IVTy, FactorLoc);
   };
 
   // Iteration variable SourceLocations.
@@ -15430,30 +15475,31 @@ StmtResult Sema::ActOnOpenMPUnrollDirective(ArrayRef<OMPClause *> Clauses,
 
   // Create the iteration variable for the unrolled loop.
   VarDecl *OuterIVDecl =
-      buildVarDecl(*this, {}, IVTy, OuterIVName, nullptr, OrigVar);
+      buildVarDecl(SemaRef, {}, IVTy, OuterIVName, nullptr, OrigVar);
   auto MakeOuterRef = [this, OuterIVDecl, IVTy, OrigVarLoc]() {
-    return buildDeclRefExpr(*this, OuterIVDecl, IVTy, OrigVarLoc);
+    return buildDeclRefExpr(SemaRef, OuterIVDecl, IVTy, OrigVarLoc);
   };
 
   // Iteration variable for the inner loop: Reuse the iteration variable created
   // by checkOpenMPLoop.
   auto *InnerIVDecl = cast<VarDecl>(IterationVarRef->getDecl());
-  InnerIVDecl->setDeclName(&PP.getIdentifierTable().get(InnerIVName));
+  InnerIVDecl->setDeclName(&SemaRef.PP.getIdentifierTable().get(InnerIVName));
   auto MakeInnerRef = [this, InnerIVDecl, IVTy, OrigVarLoc]() {
-    return buildDeclRefExpr(*this, InnerIVDecl, IVTy, OrigVarLoc);
+    return buildDeclRefExpr(SemaRef, InnerIVDecl, IVTy, OrigVarLoc);
   };
 
   // Make a copy of the NumIterations expression for each use: By the AST
   // constraints, every expression object in a DeclContext must be unique.
-  CaptureVars CopyTransformer(*this);
+  CaptureVars CopyTransformer(SemaRef);
   auto MakeNumIterations = [&CopyTransformer, &LoopHelper]() -> Expr * {
     return AssertSuccess(
         CopyTransformer.TransformExpr(LoopHelper.NumIterations));
   };
 
   // Inner For init-statement: auto .unroll_inner.iv = .unrolled.iv
-  ExprResult LValueConv = DefaultLvalueConversion(MakeOuterRef());
-  AddInitializerToDecl(InnerIVDecl, LValueConv.get(), /*DirectInit=*/false);
+  ExprResult LValueConv = SemaRef.DefaultLvalueConversion(MakeOuterRef());
+  SemaRef.AddInitializerToDecl(InnerIVDecl, LValueConv.get(),
+                               /*DirectInit=*/false);
   StmtResult InnerInit = new (Context)
       DeclStmt(DeclGroupRef(InnerIVDecl), OrigVarLocBegin, OrigVarLocEnd);
   if (!InnerInit.isUsable())
@@ -15466,28 +15512,30 @@ StmtResult Sema::ActOnOpenMPUnrollDirective(ArrayRef<OMPClause *> Clauses,
   // \endcode
   // This conjunction of two conditions allows ScalarEvolution to derive the
   // maximum trip count of the inner loop.
-  ExprResult EndOfTile = BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(),
-                                    BO_Add, MakeOuterRef(), MakeFactorExpr());
+  ExprResult EndOfTile =
+      SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_Add,
+                         MakeOuterRef(), MakeFactorExpr());
   if (!EndOfTile.isUsable())
     return StmtError();
-  ExprResult InnerCond1 = BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(),
-                                     BO_LT, MakeInnerRef(), EndOfTile.get());
+  ExprResult InnerCond1 =
+      SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
+                         MakeInnerRef(), EndOfTile.get());
   if (!InnerCond1.isUsable())
     return StmtError();
   ExprResult InnerCond2 =
-      BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT, MakeInnerRef(),
-                 MakeNumIterations());
+      SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
+                         MakeInnerRef(), MakeNumIterations());
   if (!InnerCond2.isUsable())
     return StmtError();
   ExprResult InnerCond =
-      BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LAnd,
-                 InnerCond1.get(), InnerCond2.get());
+      SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LAnd,
+                         InnerCond1.get(), InnerCond2.get());
   if (!InnerCond.isUsable())
     return StmtError();
 
   // Inner For incr-statement: ++.unroll_inner.iv
-  ExprResult InnerIncr = BuildUnaryOp(CurScope, LoopHelper.Inc->getExprLoc(),
-                                      UO_PreInc, MakeInnerRef());
+  ExprResult InnerIncr = SemaRef.BuildUnaryOp(
+      CurScope, LoopHelper.Inc->getExprLoc(), UO_PreInc, MakeInnerRef());
   if (!InnerIncr.isUsable())
     return StmtError();
 
@@ -15496,7 +15544,7 @@ StmtResult Sema::ActOnOpenMPUnrollDirective(ArrayRef<OMPClause *> Clauses,
   InnerBodyStmts.append(LoopHelper.Updates.begin(), LoopHelper.Updates.end());
   InnerBodyStmts.push_back(Body);
   CompoundStmt *InnerBody =
-      CompoundStmt::Create(Context, InnerBodyStmts, FPOptionsOverride(),
+      CompoundStmt::Create(getASTContext(), InnerBodyStmts, FPOptionsOverride(),
                            Body->getBeginLoc(), Body->getEndLoc());
   ForStmt *InnerFor = new (Context)
       ForStmt(Context, InnerInit.get(), InnerCond.get(), nullptr,
@@ -15518,12 +15566,13 @@ StmtResult Sema::ActOnOpenMPUnrollDirective(ArrayRef<OMPClause *> Clauses,
   LoopHintAttr *UnrollHintAttr =
       LoopHintAttr::CreateImplicit(Context, LoopHintAttr::UnrollCount,
                                    LoopHintAttr::Numeric, MakeFactorExpr());
-  AttributedStmt *InnerUnrolled =
-      AttributedStmt::Create(Context, StartLoc, {UnrollHintAttr}, InnerFor);
+  AttributedStmt *InnerUnrolled = AttributedStmt::Create(
+      getASTContext(), StartLoc, {UnrollHintAttr}, InnerFor);
 
   // Outer For init-statement: auto .unrolled.iv = 0
-  AddInitializerToDecl(
-      OuterIVDecl, ActOnIntegerConstant(LoopHelper.Init->getExprLoc(), 0).get(),
+  SemaRef.AddInitializerToDecl(
+      OuterIVDecl,
+      SemaRef.ActOnIntegerConstant(LoopHelper.Init->getExprLoc(), 0).get(),
       /*DirectInit=*/false);
   StmtResult OuterInit = new (Context)
       DeclStmt(DeclGroupRef(OuterIVDecl), OrigVarLocBegin, OrigVarLocEnd);
@@ -15532,15 +15581,15 @@ StmtResult Sema::ActOnOpenMPUnrollDirective(ArrayRef<OMPClause *> Clauses,
 
   // Outer For cond-expression: .unrolled.iv < NumIterations
   ExprResult OuterConde =
-      BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT, MakeOuterRef(),
-                 MakeNumIterations());
+      SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
+                         MakeOuterRef(), MakeNumIterations());
   if (!OuterConde.isUsable())
     return StmtError();
 
   // Outer For incr-statement: .unrolled.iv += Factor
   ExprResult OuterIncr =
-      BuildBinOp(CurScope, LoopHelper.Inc->getExprLoc(), BO_AddAssign,
-                 MakeOuterRef(), MakeFactorExpr());
+      SemaRef.BuildBinOp(CurScope, LoopHelper.Inc->getExprLoc(), BO_AddAssign,
+                         MakeOuterRef(), MakeFactorExpr());
   if (!OuterIncr.isUsable())
     return StmtError();
 
@@ -15555,10 +15604,11 @@ StmtResult Sema::ActOnOpenMPUnrollDirective(ArrayRef<OMPClause *> Clauses,
                                     buildPreInits(Context, PreInits));
 }
 
-OMPClause *Sema::ActOnOpenMPSingleExprClause(OpenMPClauseKind Kind, Expr *Expr,
-                                             SourceLocation StartLoc,
-                                             SourceLocation LParenLoc,
-                                             SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPSingleExprClause(OpenMPClauseKind Kind,
+                                                   Expr *Expr,
+                                                   SourceLocation StartLoc,
+                                                   SourceLocation LParenLoc,
+                                                   SourceLocation EndLoc) {
   OMPClause *Res = nullptr;
   switch (Kind) {
   case OMPC_final:
@@ -16646,19 +16696,17 @@ static OpenMPDirectiveKind getOpenMPCaptureRegionForClause(
   return CaptureRegion;
 }
 
-OMPClause *Sema::ActOnOpenMPIfClause(OpenMPDirectiveKind NameModifier,
-                                     Expr *Condition, SourceLocation StartLoc,
-                                     SourceLocation LParenLoc,
-                                     SourceLocation NameModifierLoc,
-                                     SourceLocation ColonLoc,
-                                     SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPIfClause(
+    OpenMPDirectiveKind NameModifier, Expr *Condition, SourceLocation StartLoc,
+    SourceLocation LParenLoc, SourceLocation NameModifierLoc,
+    SourceLocation ColonLoc, SourceLocation EndLoc) {
   Expr *ValExpr = Condition;
   Stmt *HelperValStmt = nullptr;
   OpenMPDirectiveKind CaptureRegion = OMPD_unknown;
   if (!Condition->isValueDependent() && !Condition->isTypeDependent() &&
       !Condition->isInstantiationDependent() &&
       !Condition->containsUnexpandedParameterPack()) {
-    ExprResult Val = CheckBooleanCondition(StartLoc, Condition);
+    ExprResult Val = SemaRef.CheckBooleanCondition(StartLoc, Condition);
     if (Val.isInvalid())
       return nullptr;
 
@@ -16666,57 +16714,60 @@ OMPClause *Sema::ActOnOpenMPIfClause(OpenMPDirectiveKind NameModifier,
 
     OpenMPDirectiveKind DKind = DSAStack->getCurrentDirective();
     CaptureRegion = getOpenMPCaptureRegionForClause(
-        DKind, OMPC_if, LangOpts.OpenMP, NameModifier);
-    if (CaptureRegion != OMPD_unknown && !CurContext->isDependentContext()) {
-      ValExpr = MakeFullExpr(ValExpr).get();
+        DKind, OMPC_if, getLangOpts().OpenMP, NameModifier);
+    if (CaptureRegion != OMPD_unknown &&
+        !SemaRef.CurContext->isDependentContext()) {
+      ValExpr = SemaRef.MakeFullExpr(ValExpr).get();
       llvm::MapVector<const Expr *, DeclRefExpr *> Captures;
-      ValExpr = tryBuildCapture(*this, ValExpr, Captures).get();
-      HelperValStmt = buildPreInits(Context, Captures);
+      ValExpr = tryBuildCapture(SemaRef, ValExpr, Captures).get();
+      HelperValStmt = buildPreInits(getASTContext(), Captures);
     }
   }
 
-  return new (Context)
+  return new (getASTContext())
       OMPIfClause(NameModifier, ValExpr, HelperValStmt, CaptureRegion, StartLoc,
                   LParenLoc, NameModifierLoc, ColonLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPFinalClause(Expr *Condition,
-                                        SourceLocation StartLoc,
-                                        SourceLocation LParenLoc,
-                                        SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPFinalClause(Expr *Condition,
+                                              SourceLocation StartLoc,
+                                              SourceLocation LParenLoc,
+                                              SourceLocation EndLoc) {
   Expr *ValExpr = Condition;
   Stmt *HelperValStmt = nullptr;
   OpenMPDirectiveKind CaptureRegion = OMPD_unknown;
   if (!Condition->isValueDependent() && !Condition->isTypeDependent() &&
       !Condition->isInstantiationDependent() &&
       !Condition->containsUnexpandedParameterPack()) {
-    ExprResult Val = CheckBooleanCondition(StartLoc, Condition);
+    ExprResult Val = SemaRef.CheckBooleanCondition(StartLoc, Condition);
     if (Val.isInvalid())
       return nullptr;
 
-    ValExpr = MakeFullExpr(Val.get()).get();
+    ValExpr = SemaRef.MakeFullExpr(Val.get()).get();
 
     OpenMPDirectiveKind DKind = DSAStack->getCurrentDirective();
-    CaptureRegion =
-        getOpenMPCaptureRegionForClause(DKind, OMPC_final, LangOpts.OpenMP);
-    if (CaptureRegion != OMPD_unknown && !CurContext->isDependentContext()) {
-      ValExpr = MakeFullExpr(ValExpr).get();
+    CaptureRegion = getOpenMPCaptureRegionForClause(DKind, OMPC_final,
+                                                    getLangOpts().OpenMP);
+    if (CaptureRegion != OMPD_unknown &&
+        !SemaRef.CurContext->isDependentContext()) {
+      ValExpr = SemaRef.MakeFullExpr(ValExpr).get();
       llvm::MapVector<const Expr *, DeclRefExpr *> Captures;
-      ValExpr = tryBuildCapture(*this, ValExpr, Captures).get();
-      HelperValStmt = buildPreInits(Context, Captures);
+      ValExpr = tryBuildCapture(SemaRef, ValExpr, Captures).get();
+      HelperValStmt = buildPreInits(getASTContext(), Captures);
     }
   }
 
-  return new (Context) OMPFinalClause(ValExpr, HelperValStmt, CaptureRegion,
-                                      StartLoc, LParenLoc, EndLoc);
+  return new (getASTContext()) OMPFinalClause(
+      ValExpr, HelperValStmt, CaptureRegion, StartLoc, LParenLoc, EndLoc);
 }
 
-ExprResult Sema::PerformOpenMPImplicitIntegerConversion(SourceLocation Loc,
-                                                        Expr *Op) {
+ExprResult
+SemaOpenMP::PerformOpenMPImplicitIntegerConversion(SourceLocation Loc,
+                                                   Expr *Op) {
   if (!Op)
     return ExprError();
 
-  class IntConvertDiagnoser : public ICEConvertDiagnoser {
+  class IntConvertDiagnoser : public Sema::ICEConvertDiagnoser {
   public:
     IntConvertDiagnoser()
         : ICEConvertDiagnoser(/*AllowScopedEnumerations*/ false, false, true) {}
@@ -16752,7 +16803,7 @@ ExprResult Sema::PerformOpenMPImplicitIntegerConversion(SourceLocation Loc,
       llvm_unreachable("conversion functions are permitted");
     }
   } ConvertDiagnoser;
-  return PerformContextualImplicitConversion(Loc, Op, ConvertDiagnoser);
+  return SemaRef.PerformContextualImplicitConversion(Loc, Op, ConvertDiagnoser);
 }
 
 static bool
@@ -16765,7 +16816,7 @@ isNonNegativeIntegerValue(Expr *&ValExpr, Sema &SemaRef, OpenMPClauseKind CKind,
       !ValExpr->isInstantiationDependent()) {
     SourceLocation Loc = ValExpr->getExprLoc();
     ExprResult Value =
-        SemaRef.PerformOpenMPImplicitIntegerConversion(Loc, ValExpr);
+        SemaRef.OpenMP().PerformOpenMPImplicitIntegerConversion(Loc, ValExpr);
     if (Value.isInvalid())
       return false;
 
@@ -16797,37 +16848,37 @@ isNonNegativeIntegerValue(Expr *&ValExpr, Sema &SemaRef, OpenMPClauseKind CKind,
   return true;
 }
 
-OMPClause *Sema::ActOnOpenMPNumThreadsClause(Expr *NumThreads,
-                                             SourceLocation StartLoc,
-                                             SourceLocation LParenLoc,
-                                             SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPNumThreadsClause(Expr *NumThreads,
+                                                   SourceLocation StartLoc,
+                                                   SourceLocation LParenLoc,
+                                                   SourceLocation EndLoc) {
   Expr *ValExpr = NumThreads;
   Stmt *HelperValStmt = nullptr;
 
   // OpenMP [2.5, Restrictions]
   //  The num_threads expression must evaluate to a positive integer value.
-  if (!isNonNegativeIntegerValue(ValExpr, *this, OMPC_num_threads,
+  if (!isNonNegativeIntegerValue(ValExpr, SemaRef, OMPC_num_threads,
                                  /*StrictlyPositive=*/true))
     return nullptr;
 
   OpenMPDirectiveKind DKind = DSAStack->getCurrentDirective();
-  OpenMPDirectiveKind CaptureRegion =
-      getOpenMPCaptureRegionForClause(DKind, OMPC_num_threads, LangOpts.OpenMP);
-  if (CaptureRegion != OMPD_unknown && !CurContext->isDependentContext()) {
-    ValExpr = MakeFullExpr(ValExpr).get();
+  OpenMPDirectiveKind CaptureRegion = getOpenMPCaptureRegionForClause(
+      DKind, OMPC_num_threads, getLangOpts().OpenMP);
+  if (CaptureRegion != OMPD_unknown &&
+      !SemaRef.CurContext->isDependentContext()) {
+    ValExpr = SemaRef.MakeFullExpr(ValExpr).get();
     llvm::MapVector<const Expr *, DeclRefExpr *> Captures;
-    ValExpr = tryBuildCapture(*this, ValExpr, Captures).get();
-    HelperValStmt = buildPreInits(Context, Captures);
+    ValExpr = tryBuildCapture(SemaRef, ValExpr, Captures).get();
+    HelperValStmt = buildPreInits(getASTContext(), Captures);
   }
 
-  return new (Context) OMPNumThreadsClause(
+  return new (getASTContext()) OMPNumThreadsClause(
       ValExpr, HelperValStmt, CaptureRegion, StartLoc, LParenLoc, EndLoc);
 }
 
-ExprResult Sema::VerifyPositiveIntegerConstantInClause(Expr *E,
-                                                       OpenMPClauseKind CKind,
-                                                       bool StrictlyPositive,
-                                                       bool SuppressExprDiags) {
+ExprResult SemaOpenMP::VerifyPositiveIntegerConstantInClause(
+    Expr *E, OpenMPClauseKind CKind, bool StrictlyPositive,
+    bool SuppressExprDiags) {
   if (!E)
     return ExprError();
   if (E->isValueDependent() || E->isTypeDependent() ||
@@ -16841,14 +16892,16 @@ ExprResult Sema::VerifyPositiveIntegerConstantInClause(Expr *E,
     // expression.
     struct SuppressedDiagnoser : public Sema::VerifyICEDiagnoser {
       SuppressedDiagnoser() : VerifyICEDiagnoser(/*Suppress=*/true) {}
-      Sema::SemaDiagnosticBuilder diagnoseNotICE(Sema &S,
-                                                 SourceLocation Loc) override {
+      SemaBase::SemaDiagnosticBuilder
+      diagnoseNotICE(Sema &S, SourceLocation Loc) override {
         llvm_unreachable("Diagnostic suppressed");
       }
     } Diagnoser;
-    ICE = VerifyIntegerConstantExpression(E, &Result, Diagnoser, AllowFold);
+    ICE = SemaRef.VerifyIntegerConstantExpression(E, &Result, Diagnoser,
+                                                  Sema::AllowFold);
   } else {
-    ICE = VerifyIntegerConstantExpression(E, &Result, /*FIXME*/ AllowFold);
+    ICE = SemaRef.VerifyIntegerConstantExpression(E, &Result,
+                                                  /*FIXME*/ Sema::AllowFold);
   }
   if (ICE.isInvalid())
     return ExprError();
@@ -16872,29 +16925,31 @@ ExprResult Sema::VerifyPositiveIntegerConstantInClause(Expr *E,
   return ICE;
 }
 
-OMPClause *Sema::ActOnOpenMPSafelenClause(Expr *Len, SourceLocation StartLoc,
-                                          SourceLocation LParenLoc,
-                                          SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPSafelenClause(Expr *Len,
+                                                SourceLocation StartLoc,
+                                                SourceLocation LParenLoc,
+                                                SourceLocation EndLoc) {
   // OpenMP [2.8.1, simd construct, Description]
   // The parameter of the safelen clause must be a constant
   // positive integer expression.
   ExprResult Safelen = VerifyPositiveIntegerConstantInClause(Len, OMPC_safelen);
   if (Safelen.isInvalid())
     return nullptr;
-  return new (Context)
+  return new (getASTContext())
       OMPSafelenClause(Safelen.get(), StartLoc, LParenLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPSimdlenClause(Expr *Len, SourceLocation StartLoc,
-                                          SourceLocation LParenLoc,
-                                          SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPSimdlenClause(Expr *Len,
+                                                SourceLocation StartLoc,
+                                                SourceLocation LParenLoc,
+                                                SourceLocation EndLoc) {
   // OpenMP [2.8.1, simd construct, Description]
   // The parameter of the simdlen clause must be a constant
   // positive integer expression.
   ExprResult Simdlen = VerifyPositiveIntegerConstantInClause(Len, OMPC_simdlen);
   if (Simdlen.isInvalid())
     return nullptr;
-  return new (Context)
+  return new (getASTContext())
       OMPSimdlenClause(Simdlen.get(), StartLoc, LParenLoc, EndLoc);
 }
 
@@ -16954,31 +17009,32 @@ static bool findOMPAllocatorHandleT(Sema &S, SourceLocation Loc,
   return true;
 }
 
-OMPClause *Sema::ActOnOpenMPAllocatorClause(Expr *A, SourceLocation StartLoc,
-                                            SourceLocation LParenLoc,
-                                            SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPAllocatorClause(Expr *A,
+                                                  SourceLocation StartLoc,
+                                                  SourceLocation LParenLoc,
+                                                  SourceLocation EndLoc) {
   // OpenMP [2.11.3, allocate Directive, Description]
   // allocator is an expression of omp_allocator_handle_t type.
-  if (!findOMPAllocatorHandleT(*this, A->getExprLoc(), DSAStack))
+  if (!findOMPAllocatorHandleT(SemaRef, A->getExprLoc(), DSAStack))
     return nullptr;
 
-  ExprResult Allocator = DefaultLvalueConversion(A);
+  ExprResult Allocator = SemaRef.DefaultLvalueConversion(A);
   if (Allocator.isInvalid())
     return nullptr;
-  Allocator = PerformImplicitConversion(Allocator.get(),
-                                        DSAStack->getOMPAllocatorHandleT(),
-                                        Sema::AA_Initializing,
-                                        /*AllowExplicit=*/true);
+  Allocator = SemaRef.PerformImplicitConversion(
+      Allocator.get(), DSAStack->getOMPAllocatorHandleT(),
+      Sema::AA_Initializing,
+      /*AllowExplicit=*/true);
   if (Allocator.isInvalid())
     return nullptr;
-  return new (Context)
+  return new (getASTContext())
       OMPAllocatorClause(Allocator.get(), StartLoc, LParenLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPCollapseClause(Expr *NumForLoops,
-                                           SourceLocation StartLoc,
-                                           SourceLocation LParenLoc,
-                                           SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPCollapseClause(Expr *NumForLoops,
+                                                 SourceLocation StartLoc,
+                                                 SourceLocation LParenLoc,
+                                                 SourceLocation EndLoc) {
   // OpenMP [2.7.1, loop construct, Description]
   // OpenMP [2.8.1, simd construct, Description]
   // OpenMP [2.9.6, distribute construct, Description]
@@ -16988,14 +17044,14 @@ OMPClause *Sema::ActOnOpenMPCollapseClause(Expr *NumForLoops,
       VerifyPositiveIntegerConstantInClause(NumForLoops, OMPC_collapse);
   if (NumForLoopsResult.isInvalid())
     return nullptr;
-  return new (Context)
+  return new (getASTContext())
       OMPCollapseClause(NumForLoopsResult.get(), StartLoc, LParenLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPOrderedClause(SourceLocation StartLoc,
-                                          SourceLocation EndLoc,
-                                          SourceLocation LParenLoc,
-                                          Expr *NumForLoops) {
+OMPClause *SemaOpenMP::ActOnOpenMPOrderedClause(SourceLocation StartLoc,
+                                                SourceLocation EndLoc,
+                                                SourceLocation LParenLoc,
+                                                Expr *NumForLoops) {
   // OpenMP [2.7.1, loop construct, Description]
   // OpenMP [2.8.1, simd construct, Description]
   // OpenMP [2.9.6, distribute construct, Description]
@@ -17010,14 +17066,15 @@ OMPClause *Sema::ActOnOpenMPOrderedClause(SourceLocation StartLoc,
   } else {
     NumForLoops = nullptr;
   }
-  auto *Clause = OMPOrderedClause::Create(
-      Context, NumForLoops, NumForLoops ? DSAStack->getAssociatedLoops() : 0,
-      StartLoc, LParenLoc, EndLoc);
+  auto *Clause =
+      OMPOrderedClause::Create(getASTContext(), NumForLoops,
+                               NumForLoops ? DSAStack->getAssociatedLoops() : 0,
+                               StartLoc, LParenLoc, EndLoc);
   DSAStack->setOrderedRegion(/*IsOrdered=*/true, NumForLoops, Clause);
   return Clause;
 }
 
-OMPClause *Sema::ActOnOpenMPSimpleClause(
+OMPClause *SemaOpenMP::ActOnOpenMPSimpleClause(
     OpenMPClauseKind Kind, unsigned Argument, SourceLocation ArgumentLoc,
     SourceLocation StartLoc, SourceLocation LParenLoc, SourceLocation EndLoc) {
   OMPClause *Res = nullptr;
@@ -17159,11 +17216,11 @@ getListOfPossibleValues(OpenMPClauseKind K, unsigned First, unsigned Last,
   return std::string(Out.str());
 }
 
-OMPClause *Sema::ActOnOpenMPDefaultClause(DefaultKind Kind,
-                                          SourceLocation KindKwLoc,
-                                          SourceLocation StartLoc,
-                                          SourceLocation LParenLoc,
-                                          SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPDefaultClause(DefaultKind Kind,
+                                                SourceLocation KindKwLoc,
+                                                SourceLocation StartLoc,
+                                                SourceLocation LParenLoc,
+                                                SourceLocation EndLoc) {
   if (Kind == OMP_DEFAULT_unknown) {
     Diag(KindKwLoc, diag::err_omp_unexpected_clause_value)
         << getListOfPossibleValues(OMPC_default, /*First=*/0,
@@ -17189,39 +17246,39 @@ OMPClause *Sema::ActOnOpenMPDefaultClause(DefaultKind Kind,
     llvm_unreachable("DSA unexpected in OpenMP default clause");
   }
 
-  return new (Context)
+  return new (getASTContext())
       OMPDefaultClause(Kind, KindKwLoc, StartLoc, LParenLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPProcBindClause(ProcBindKind Kind,
-                                           SourceLocation KindKwLoc,
-                                           SourceLocation StartLoc,
-                                           SourceLocation LParenLoc,
-                                           SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPProcBindClause(ProcBindKind Kind,
+                                                 SourceLocation KindKwLoc,
+                                                 SourceLocation StartLoc,
+                                                 SourceLocation LParenLoc,
+                                                 SourceLocation EndLoc) {
   if (Kind == OMP_PROC_BIND_unknown) {
     Diag(KindKwLoc, diag::err_omp_unexpected_clause_value)
         << getListOfPossibleValues(OMPC_proc_bind,
                                    /*First=*/unsigned(OMP_PROC_BIND_master),
                                    /*Last=*/
-                                   unsigned(LangOpts.OpenMP > 50
+                                   unsigned(getLangOpts().OpenMP > 50
                                                 ? OMP_PROC_BIND_primary
                                                 : OMP_PROC_BIND_spread) +
                                        1)
         << getOpenMPClauseName(OMPC_proc_bind);
     return nullptr;
   }
-  if (Kind == OMP_PROC_BIND_primary && LangOpts.OpenMP < 51)
+  if (Kind == OMP_PROC_BIND_primary && getLangOpts().OpenMP < 51)
     Diag(KindKwLoc, diag::err_omp_unexpected_clause_value)
         << getListOfPossibleValues(OMPC_proc_bind,
                                    /*First=*/unsigned(OMP_PROC_BIND_master),
                                    /*Last=*/
                                    unsigned(OMP_PROC_BIND_spread) + 1)
         << getOpenMPClauseName(OMPC_proc_bind);
-  return new (Context)
+  return new (getASTContext())
       OMPProcBindClause(Kind, KindKwLoc, StartLoc, LParenLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPAtomicDefaultMemOrderClause(
+OMPClause *SemaOpenMP::ActOnOpenMPAtomicDefaultMemOrderClause(
     OpenMPAtomicDefaultMemOrderClauseKind Kind, SourceLocation KindKwLoc,
     SourceLocation StartLoc, SourceLocation LParenLoc, SourceLocation EndLoc) {
   if (Kind == OMPC_ATOMIC_DEFAULT_MEM_ORDER_unknown) {
@@ -17232,15 +17289,15 @@ OMPClause *Sema::ActOnOpenMPAtomicDefaultMemOrderClause(
         << getOpenMPClauseName(OMPC_atomic_default_mem_order);
     return nullptr;
   }
-  return new (Context) OMPAtomicDefaultMemOrderClause(Kind, KindKwLoc, StartLoc,
-                                                      LParenLoc, EndLoc);
+  return new (getASTContext()) OMPAtomicDefaultMemOrderClause(
+      Kind, KindKwLoc, StartLoc, LParenLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPAtClause(OpenMPAtClauseKind Kind,
-                                     SourceLocation KindKwLoc,
-                                     SourceLocation StartLoc,
-                                     SourceLocation LParenLoc,
-                                     SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPAtClause(OpenMPAtClauseKind Kind,
+                                           SourceLocation KindKwLoc,
+                                           SourceLocation StartLoc,
+                                           SourceLocation LParenLoc,
+                                           SourceLocation EndLoc) {
   if (Kind == OMPC_AT_unknown) {
     Diag(KindKwLoc, diag::err_omp_unexpected_clause_value)
         << getListOfPossibleValues(OMPC_at, /*First=*/0,
@@ -17248,15 +17305,15 @@ OMPClause *Sema::ActOnOpenMPAtClause(OpenMPAtClauseKind Kind,
         << getOpenMPClauseName(OMPC_at);
     return nullptr;
   }
-  return new (Context)
+  return new (getASTContext())
       OMPAtClause(Kind, KindKwLoc, StartLoc, LParenLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPSeverityClause(OpenMPSeverityClauseKind Kind,
-                                           SourceLocation KindKwLoc,
-                                           SourceLocation StartLoc,
-                                           SourceLocation LParenLoc,
-                                           SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPSeverityClause(OpenMPSeverityClauseKind Kind,
+                                                 SourceLocation KindKwLoc,
+                                                 SourceLocation StartLoc,
+                                                 SourceLocation LParenLoc,
+                                                 SourceLocation EndLoc) {
   if (Kind == OMPC_SEVERITY_unknown) {
     Diag(KindKwLoc, diag::err_omp_unexpected_clause_value)
         << getListOfPossibleValues(OMPC_severity, /*First=*/0,
@@ -17264,28 +17321,30 @@ OMPClause *Sema::ActOnOpenMPSeverityClause(OpenMPSeverityClauseKind Kind,
         << getOpenMPClauseName(OMPC_severity);
     return nullptr;
   }
-  return new (Context)
+  return new (getASTContext())
       OMPSeverityClause(Kind, KindKwLoc, StartLoc, LParenLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPMessageClause(Expr *ME, SourceLocation StartLoc,
-                                          SourceLocation LParenLoc,
-                                          SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPMessageClause(Expr *ME,
+                                                SourceLocation StartLoc,
+                                                SourceLocation LParenLoc,
+                                                SourceLocation EndLoc) {
   assert(ME && "NULL expr in Message clause");
   if (!isa<StringLiteral>(ME)) {
     Diag(ME->getBeginLoc(), diag::warn_clause_expected_string)
         << getOpenMPClauseName(OMPC_message);
     return nullptr;
   }
-  return new (Context) OMPMessageClause(ME, StartLoc, LParenLoc, EndLoc);
+  return new (getASTContext())
+      OMPMessageClause(ME, StartLoc, LParenLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPOrderClause(
+OMPClause *SemaOpenMP::ActOnOpenMPOrderClause(
     OpenMPOrderClauseModifier Modifier, OpenMPOrderClauseKind Kind,
     SourceLocation StartLoc, SourceLocation LParenLoc, SourceLocation MLoc,
     SourceLocation KindLoc, SourceLocation EndLoc) {
   if (Kind != OMPC_ORDER_concurrent ||
-      (LangOpts.OpenMP < 51 && MLoc.isValid())) {
+      (getLangOpts().OpenMP < 51 && MLoc.isValid())) {
     // Kind should be concurrent,
     // Modifiers introduced in OpenMP 5.1
     static_assert(OMPC_ORDER_unknown > 0,
@@ -17298,7 +17357,7 @@ OMPClause *Sema::ActOnOpenMPOrderClause(
         << getOpenMPClauseName(OMPC_order);
     return nullptr;
   }
-  if (LangOpts.OpenMP >= 51) {
+  if (getLangOpts().OpenMP >= 51) {
     if (Modifier == OMPC_ORDER_MODIFIER_unknown && MLoc.isValid()) {
       Diag(MLoc, diag::err_omp_unexpected_clause_value)
           << getListOfPossibleValues(OMPC_order,
@@ -17315,21 +17374,21 @@ OMPClause *Sema::ActOnOpenMPOrderClause(
       }
     }
   }
-  return new (Context) OMPOrderClause(Kind, KindLoc, StartLoc, LParenLoc,
-                                      EndLoc, Modifier, MLoc);
+  return new (getASTContext()) OMPOrderClause(
+      Kind, KindLoc, StartLoc, LParenLoc, EndLoc, Modifier, MLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPUpdateClause(OpenMPDependClauseKind Kind,
-                                         SourceLocation KindKwLoc,
-                                         SourceLocation StartLoc,
-                                         SourceLocation LParenLoc,
-                                         SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPUpdateClause(OpenMPDependClauseKind Kind,
+                                               SourceLocation KindKwLoc,
+                                               SourceLocation StartLoc,
+                                               SourceLocation LParenLoc,
+                                               SourceLocation EndLoc) {
   if (Kind == OMPC_DEPEND_unknown || Kind == OMPC_DEPEND_source ||
       Kind == OMPC_DEPEND_sink || Kind == OMPC_DEPEND_depobj) {
     SmallVector<unsigned> Except = {
         OMPC_DEPEND_source, OMPC_DEPEND_sink, OMPC_DEPEND_depobj,
         OMPC_DEPEND_outallmemory, OMPC_DEPEND_inoutallmemory};
-    if (LangOpts.OpenMP < 51)
+    if (getLangOpts().OpenMP < 51)
       Except.push_back(OMPC_DEPEND_inoutset);
     Diag(KindKwLoc, diag::err_omp_unexpected_clause_value)
         << getListOfPossibleValues(OMPC_depend, /*First=*/0,
@@ -17337,14 +17396,14 @@ OMPClause *Sema::ActOnOpenMPUpdateClause(OpenMPDependClauseKind Kind,
         << getOpenMPClauseName(OMPC_update);
     return nullptr;
   }
-  return OMPUpdateClause::Create(Context, StartLoc, LParenLoc, KindKwLoc, Kind,
-                                 EndLoc);
+  return OMPUpdateClause::Create(getASTContext(), StartLoc, LParenLoc,
+                                 KindKwLoc, Kind, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPSizesClause(ArrayRef<Expr *> SizeExprs,
-                                        SourceLocation StartLoc,
-                                        SourceLocation LParenLoc,
-                                        SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPSizesClause(ArrayRef<Expr *> SizeExprs,
+                                              SourceLocation StartLoc,
+                                              SourceLocation LParenLoc,
+                                              SourceLocation EndLoc) {
   for (Expr *SizeExpr : SizeExprs) {
     ExprResult NumForLoopsResult = VerifyPositiveIntegerConstantInClause(
         SizeExpr, OMPC_sizes, /*StrictlyPositive=*/true);
@@ -17353,19 +17412,19 @@ OMPClause *Sema::ActOnOpenMPSizesClause(ArrayRef<Expr *> SizeExprs,
   }
 
   DSAStack->setAssociatedLoops(SizeExprs.size());
-  return OMPSizesClause::Create(Context, StartLoc, LParenLoc, EndLoc,
+  return OMPSizesClause::Create(getASTContext(), StartLoc, LParenLoc, EndLoc,
                                 SizeExprs);
 }
 
-OMPClause *Sema::ActOnOpenMPFullClause(SourceLocation StartLoc,
-                                       SourceLocation EndLoc) {
-  return OMPFullClause::Create(Context, StartLoc, EndLoc);
+OMPClause *SemaOpenMP::ActOnOpenMPFullClause(SourceLocation StartLoc,
+                                             SourceLocation EndLoc) {
+  return OMPFullClause::Create(getASTContext(), StartLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPPartialClause(Expr *FactorExpr,
-                                          SourceLocation StartLoc,
-                                          SourceLocation LParenLoc,
-                                          SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPPartialClause(Expr *FactorExpr,
+                                                SourceLocation StartLoc,
+                                                SourceLocation LParenLoc,
+                                                SourceLocation EndLoc) {
   if (FactorExpr) {
     // If an argument is specified, it must be a constant (or an unevaluated
     // template expression).
@@ -17376,22 +17435,22 @@ OMPClause *Sema::ActOnOpenMPPartialClause(Expr *FactorExpr,
     FactorExpr = FactorResult.get();
   }
 
-  return OMPPartialClause::Create(Context, StartLoc, LParenLoc, EndLoc,
+  return OMPPartialClause::Create(getASTContext(), StartLoc, LParenLoc, EndLoc,
                                   FactorExpr);
 }
 
-OMPClause *Sema::ActOnOpenMPAlignClause(Expr *A, SourceLocation StartLoc,
-                                        SourceLocation LParenLoc,
-                                        SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPAlignClause(Expr *A, SourceLocation StartLoc,
+                                              SourceLocation LParenLoc,
+                                              SourceLocation EndLoc) {
   ExprResult AlignVal;
   AlignVal = VerifyPositiveIntegerConstantInClause(A, OMPC_align);
   if (AlignVal.isInvalid())
     return nullptr;
-  return OMPAlignClause::Create(Context, AlignVal.get(), StartLoc, LParenLoc,
-                                EndLoc);
+  return OMPAlignClause::Create(getASTContext(), AlignVal.get(), StartLoc,
+                                LParenLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPSingleExprWithArgClause(
+OMPClause *SemaOpenMP::ActOnOpenMPSingleExprWithArgClause(
     OpenMPClauseKind Kind, ArrayRef<unsigned> Argument, Expr *Expr,
     SourceLocation StartLoc, SourceLocation LParenLoc,
     ArrayRef<SourceLocation> ArgumentLoc, SourceLocation DelimLoc,
@@ -17559,13 +17618,13 @@ static bool checkScheduleModifiers(Sema &S, OpenMPScheduleClauseModifier M1,
   return false;
 }
 
-OMPClause *Sema::ActOnOpenMPScheduleClause(
+OMPClause *SemaOpenMP::ActOnOpenMPScheduleClause(
     OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2,
     OpenMPScheduleClauseKind Kind, Expr *ChunkSize, SourceLocation StartLoc,
     SourceLocation LParenLoc, SourceLocation M1Loc, SourceLocation M2Loc,
     SourceLocation KindLoc, SourceLocation CommaLoc, SourceLocation EndLoc) {
-  if (checkScheduleModifiers(*this, M1, M2, M1Loc, M2Loc) ||
-      checkScheduleModifiers(*this, M2, M1, M2Loc, M1Loc))
+  if (checkScheduleModifiers(SemaRef, M1, M2, M1Loc, M2Loc) ||
+      checkScheduleModifiers(SemaRef, M2, M1, M2Loc, M1Loc))
     return nullptr;
   // OpenMP, 2.7.1, Loop Construct, Restrictions
   // Either the monotonic modifier or the nonmonotonic modifier can be specified
@@ -17599,7 +17658,7 @@ OMPClause *Sema::ActOnOpenMPScheduleClause(
   // The nonmonotonic modifier can only be specified with schedule(dynamic) or
   // schedule(guided).
   // OpenMP 5.0 does not have this restriction.
-  if (LangOpts.OpenMP < 50 &&
+  if (getLangOpts().OpenMP < 50 &&
       (M1 == OMPC_SCHEDULE_MODIFIER_nonmonotonic ||
        M2 == OMPC_SCHEDULE_MODIFIER_nonmonotonic) &&
       Kind != OMPC_SCHEDULE_dynamic && Kind != OMPC_SCHEDULE_guided) {
@@ -17625,7 +17684,7 @@ OMPClause *Sema::ActOnOpenMPScheduleClause(
       //  chunk_size must be a loop invariant integer expression with a positive
       //  value.
       if (std::optional<llvm::APSInt> Result =
-              ValExpr->getIntegerConstantExpr(Context)) {
+              ValExpr->getIntegerConstantExpr(getASTContext())) {
         if (Result->isSigned() && !Result->isStrictlyPositive()) {
           Diag(ChunkSizeLoc, diag::err_omp_negative_expression_in_clause)
               << "schedule" << 1 << ChunkSize->getSourceRange();
@@ -17633,24 +17692,24 @@ OMPClause *Sema::ActOnOpenMPScheduleClause(
         }
       } else if (getOpenMPCaptureRegionForClause(
                      DSAStack->getCurrentDirective(), OMPC_schedule,
-                     LangOpts.OpenMP) != OMPD_unknown &&
-                 !CurContext->isDependentContext()) {
-        ValExpr = MakeFullExpr(ValExpr).get();
+                     getLangOpts().OpenMP) != OMPD_unknown &&
+                 !SemaRef.CurContext->isDependentContext()) {
+        ValExpr = SemaRef.MakeFullExpr(ValExpr).get();
         llvm::MapVector<const Expr *, DeclRefExpr *> Captures;
-        ValExpr = tryBuildCapture(*this, ValExpr, Captures).get();
-        HelperValStmt = buildPreInits(Context, Captures);
+        ValExpr = tryBuildCapture(SemaRef, ValExpr, Captures).get();
+        HelperValStmt = buildPreInits(getASTContext(), Captures);
       }
     }
   }
 
-  return new (Context)
+  return new (getASTContext())
       OMPScheduleClause(StartLoc, LParenLoc, KindLoc, CommaLoc, EndLoc, Kind,
                         ValExpr, HelperValStmt, M1, M1Loc, M2, M2Loc);
 }
 
-OMPClause *Sema::ActOnOpenMPClause(OpenMPClauseKind Kind,
-                                   SourceLocation StartLoc,
-                                   SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPClause(OpenMPClauseKind Kind,
+                                         SourceLocation StartLoc,
+                                         SourceLocation EndLoc) {
   OMPClause *Res = nullptr;
   switch (Kind) {
   case OMPC_ordered:
@@ -17804,134 +17863,138 @@ OMPClause *Sema::ActOnOpenMPClause(OpenMPClauseKind Kind,
   return Res;
 }
 
-OMPClause *Sema::ActOnOpenMPNowaitClause(SourceLocation StartLoc,
-                                         SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPNowaitClause(SourceLocation StartLoc,
+                                               SourceLocation EndLoc) {
   DSAStack->setNowaitRegion();
-  return new (Context) OMPNowaitClause(StartLoc, EndLoc);
+  return new (getASTContext()) OMPNowaitClause(StartLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPUntiedClause(SourceLocation StartLoc,
-                                         SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPUntiedClause(SourceLocation StartLoc,
+                                               SourceLocation EndLoc) {
   DSAStack->setUntiedRegion();
-  return new (Context) OMPUntiedClause(StartLoc, EndLoc);
+  return new (getASTContext()) OMPUntiedClause(StartLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPMergeableClause(SourceLocation StartLoc,
-                                            SourceLocation EndLoc) {
-  return new (Context) OMPMergeableClause(StartLoc, EndLoc);
+OMPClause *SemaOpenMP::ActOnOpenMPMergeableClause(SourceLocation StartLoc,
+                                                  SourceLocation EndLoc) {
+  return new (getASTContext()) OMPMergeableClause(StartLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPReadClause(SourceLocation StartLoc,
-                                       SourceLocation EndLoc) {
-  return new (Context) OMPReadClause(StartLoc, EndLoc);
+OMPClause *SemaOpenMP::ActOnOpenMPReadClause(SourceLocation StartLoc,
+                                             SourceLocation EndLoc) {
+  return new (getASTContext()) OMPReadClause(StartLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPWriteClause(SourceLocation StartLoc,
-                                        SourceLocation EndLoc) {
-  return new (Context) OMPWriteClause(StartLoc, EndLoc);
+OMPClause *SemaOpenMP::ActOnOpenMPWriteClause(SourceLocation StartLoc,
+                                              SourceLocation EndLoc) {
+  return new (getASTContext()) OMPWriteClause(StartLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPUpdateClause(SourceLocation StartLoc,
-                                         SourceLocation EndLoc) {
-  return OMPUpdateClause::Create(Context, StartLoc, EndLoc);
+OMPClause *SemaOpenMP::ActOnOpenMPUpdateClause(SourceLocation StartLoc,
+                                               SourceLocation EndLoc) {
+  return OMPUpdateClause::Create(getASTContext(), StartLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPCaptureClause(SourceLocation StartLoc,
-                                          SourceLocation EndLoc) {
-  return new (Context) OMPCaptureClause(StartLoc, EndLoc);
+OMPClause *SemaOpenMP::ActOnOpenMPCaptureClause(SourceLocation StartLoc,
+                                                SourceLocation EndLoc) {
+  return new (getASTContext()) OMPCaptureClause(StartLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPCompareClause(SourceLocation StartLoc,
-                                          SourceLocation EndLoc) {
-  return new (Context) OMPCompareClause(StartLoc, EndLoc);
+OMPClause *SemaOpenMP::ActOnOpenMPCompareClause(SourceLocation StartLoc,
+                                                SourceLocation EndLoc) {
+  return new (getASTContext()) OMPCompareClause(StartLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPFailClause(SourceLocation StartLoc,
-                                       SourceLocation EndLoc) {
-  return new (Context) OMPFailClause(StartLoc, EndLoc);
+OMPClause *SemaOpenMP::ActOnOpenMPFailClause(SourceLocation StartLoc,
+                                             SourceLocation EndLoc) {
+  return new (getASTContext()) OMPFailClause(StartLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPFailClause(
-      OpenMPClauseKind Parameter, SourceLocation KindLoc,
-      SourceLocation StartLoc, SourceLocation LParenLoc,
-      SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPFailClause(OpenMPClauseKind Parameter,
+                                             SourceLocation KindLoc,
+                                             SourceLocation StartLoc,
+                                             SourceLocation LParenLoc,
+                                             SourceLocation EndLoc) {
 
   if (!checkFailClauseParameter(Parameter)) {
     Diag(KindLoc, diag::err_omp_atomic_fail_wrong_or_no_clauses);
     return nullptr;
   }
-  return new (Context)
+  return new (getASTContext())
       OMPFailClause(Parameter, KindLoc, StartLoc, LParenLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPSeqCstClause(SourceLocation StartLoc,
-                                         SourceLocation EndLoc) {
-  return new (Context) OMPSeqCstClause(StartLoc, EndLoc);
+OMPClause *SemaOpenMP::ActOnOpenMPSeqCstClause(SourceLocation StartLoc,
+                                               SourceLocation EndLoc) {
+  return new (getASTContext()) OMPSeqCstClause(StartLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPAcqRelClause(SourceLocation StartLoc,
-                                         SourceLocation EndLoc) {
-  return new (Context) OMPAcqRelClause(StartLoc, EndLoc);
+OMPClause *SemaOpenMP::ActOnOpenMPAcqRelClause(SourceLocation StartLoc,
+                                               SourceLocation EndLoc) {
+  return new (getASTContext()) OMPAcqRelClause(StartLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPAcquireClause(SourceLocation StartLoc,
-                                          SourceLocation EndLoc) {
-  return new (Context) OMPAcquireClause(StartLoc, EndLoc);
+OMPClause *SemaOpenMP::ActOnOpenMPAcquireClause(SourceLocation StartLoc,
+                                                SourceLocation EndLoc) {
+  return new (getASTContext()) OMPAcquireClause(StartLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPReleaseClause(SourceLocation StartLoc,
-                                          SourceLocation EndLoc) {
-  return new (Context) OMPReleaseClause(StartLoc, EndLoc);
+OMPClause *SemaOpenMP::ActOnOpenMPReleaseClause(SourceLocation StartLoc,
+                                                SourceLocation EndLoc) {
+  return new (getASTContext()) OMPReleaseClause(StartLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPRelaxedClause(SourceLocation StartLoc,
-                                          SourceLocation EndLoc) {
-  return new (Context) OMPRelaxedClause(StartLoc, EndLoc);
+OMPClause *SemaOpenMP::ActOnOpenMPRelaxedClause(SourceLocation StartLoc,
+                                                SourceLocation EndLoc) {
+  return new (getASTContext()) OMPRelaxedClause(StartLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPWeakClause(SourceLocation StartLoc,
-                                       SourceLocation EndLoc) {
-  return new (Context) OMPWeakClause(StartLoc, EndLoc);
+OMPClause *SemaOpenMP::ActOnOpenMPWeakClause(SourceLocation StartLoc,
+                                             SourceLocation EndLoc) {
+  return new (getASTContext()) OMPWeakClause(StartLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPThreadsClause(SourceLocation StartLoc,
-                                          SourceLocation EndLoc) {
-  return new (Context) OMPThreadsClause(StartLoc, EndLoc);
+OMPClause *SemaOpenMP::ActOnOpenMPThreadsClause(SourceLocation StartLoc,
+                                                SourceLocation EndLoc) {
+  return new (getASTContext()) OMPThreadsClause(StartLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPSIMDClause(SourceLocation StartLoc,
-                                       SourceLocation EndLoc) {
-  return new (Context) OMPSIMDClause(StartLoc, EndLoc);
+OMPClause *SemaOpenMP::ActOnOpenMPSIMDClause(SourceLocation StartLoc,
+                                             SourceLocation EndLoc) {
+  return new (getASTContext()) OMPSIMDClause(StartLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPNogroupClause(SourceLocation StartLoc,
-                                          SourceLocation EndLoc) {
-  return new (Context) OMPNogroupClause(StartLoc, EndLoc);
+OMPClause *SemaOpenMP::ActOnOpenMPNogroupClause(SourceLocation StartLoc,
+                                                SourceLocation EndLoc) {
+  return new (getASTContext()) OMPNogroupClause(StartLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPUnifiedAddressClause(SourceLocation StartLoc,
-                                                 SourceLocation EndLoc) {
-  return new (Context) OMPUnifiedAddressClause(StartLoc, EndLoc);
+OMPClause *SemaOpenMP::ActOnOpenMPUnifiedAddressClause(SourceLocation StartLoc,
+                                                       SourceLocation EndLoc) {
+  return new (getASTContext()) OMPUnifiedAddressClause(StartLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPUnifiedSharedMemoryClause(SourceLocation StartLoc,
-                                                      SourceLocation EndLoc) {
-  return new (Context) OMPUnifiedSharedMemoryClause(StartLoc, EndLoc);
+OMPClause *
+SemaOpenMP::ActOnOpenMPUnifiedSharedMemoryClause(SourceLocation StartLoc,
+                                                 SourceLocation EndLoc) {
+  return new (getASTContext()) OMPUnifiedSharedMemoryClause(StartLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPReverseOffloadClause(SourceLocation StartLoc,
-                                                 SourceLocation EndLoc) {
-  return new (Context) OMPReverseOffloadClause(StartLoc, EndLoc);
+OMPClause *SemaOpenMP::ActOnOpenMPReverseOffloadClause(SourceLocation StartLoc,
+                                                       SourceLocation EndLoc) {
+  return new (getASTContext()) OMPReverseOffloadClause(StartLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPDynamicAllocatorsClause(SourceLocation StartLoc,
-                                                    SourceLocation EndLoc) {
-  return new (Context) OMPDynamicAllocatorsClause(StartLoc, EndLoc);
+OMPClause *
+SemaOpenMP::ActOnOpenMPDynamicAllocatorsClause(SourceLocation StartLoc,
+                                               SourceLocation EndLoc) {
+  return new (getASTContext()) OMPDynamicAllocatorsClause(StartLoc, EndLoc);
 }
 
-StmtResult Sema::ActOnOpenMPInteropDirective(ArrayRef<OMPClause *> Clauses,
-                                             SourceLocation StartLoc,
-                                             SourceLocation EndLoc) {
+StmtResult
+SemaOpenMP::ActOnOpenMPInteropDirective(ArrayRef<OMPClause *> Clauses,
+                                        SourceLocation StartLoc,
+                                        SourceLocation EndLoc) {
 
   // OpenMP 5.1 [2.15.1, interop Construct, Restrictions]
   // At least one action-clause must appear on a directive.
@@ -17981,13 +18044,13 @@ StmtResult Sema::ActOnOpenMPInteropDirective(ArrayRef<OMPClause *> Clauses,
 
     if (ClauseKind == OMPC_init) {
       auto *E = cast<OMPInitClause>(C)->getInteropVar();
-      DeclResult = getPrivateItem(*this, E, ELoc, ERange);
+      DeclResult = getPrivateItem(SemaRef, E, ELoc, ERange);
     } else if (ClauseKind == OMPC_use) {
       auto *E = cast<OMPUseClause>(C)->getInteropVar();
-      DeclResult = getPrivateItem(*this, E, ELoc, ERange);
+      DeclResult = getPrivateItem(SemaRef, E, ELoc, ERange);
     } else if (ClauseKind == OMPC_destroy) {
       auto *E = cast<OMPDestroyClause>(C)->getInteropVar();
-      DeclResult = getPrivateItem(*this, E, ELoc, ERange);
+      DeclResult = getPrivateItem(SemaRef, E, ELoc, ERange);
     }
 
     if (DeclResult.first) {
@@ -17999,7 +18062,8 @@ StmtResult Sema::ActOnOpenMPInteropDirective(ArrayRef<OMPClause *> Clauses,
     }
   }
 
-  return OMPInteropDirective::Create(Context, StartLoc, EndLoc, Clauses);
+  return OMPInteropDirective::Create(getASTContext(), StartLoc, EndLoc,
+                                     Clauses);
 }
 
 static bool isValidInteropVariable(Sema &SemaRef, Expr *InteropVarExpr,
@@ -18059,12 +18123,11 @@ static bool isValidInteropVariable(Sema &SemaRef, Expr *InteropVarExpr,
   return true;
 }
 
-OMPClause *
-Sema::ActOnOpenMPInitClause(Expr *InteropVar, OMPInteropInfo &InteropInfo,
-                            SourceLocation StartLoc, SourceLocation LParenLoc,
-                            SourceLocation VarLoc, SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPInitClause(
+    Expr *InteropVar, OMPInteropInfo &InteropInfo, SourceLocation StartLoc,
+    SourceLocation LParenLoc, SourceLocation VarLoc, SourceLocation EndLoc) {
 
-  if (!isValidInteropVariable(*this, InteropVar, VarLoc, OMPC_init))
+  if (!isValidInteropVariable(SemaRef, InteropVar, VarLoc, OMPC_init))
     return nullptr;
 
   // Check prefer_type values.  These foreign-runtime-id values are either
@@ -18073,7 +18136,7 @@ Sema::ActOnOpenMPInitClause(Expr *InteropVar, OMPInteropInfo &InteropInfo,
     if (E->isValueDependent() || E->isTypeDependent() ||
         E->isInstantiationDependent() || E->containsUnexpandedParameterPack())
       continue;
-    if (E->isIntegerConstantExpr(Context))
+    if (E->isIntegerConstantExpr(getASTContext()))
       continue;
     if (isa<StringLiteral>(E))
       continue;
@@ -18081,28 +18144,29 @@ Sema::ActOnOpenMPInitClause(Expr *InteropVar, OMPInteropInfo &InteropInfo,
     return nullptr;
   }
 
-  return OMPInitClause::Create(Context, InteropVar, InteropInfo, StartLoc,
-                               LParenLoc, VarLoc, EndLoc);
+  return OMPInitClause::Create(getASTContext(), InteropVar, InteropInfo,
+                               StartLoc, LParenLoc, VarLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPUseClause(Expr *InteropVar, SourceLocation StartLoc,
-                                      SourceLocation LParenLoc,
-                                      SourceLocation VarLoc,
-                                      SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPUseClause(Expr *InteropVar,
+                                            SourceLocation StartLoc,
+                                            SourceLocation LParenLoc,
+                                            SourceLocation VarLoc,
+                                            SourceLocation EndLoc) {
 
-  if (!isValidInteropVariable(*this, InteropVar, VarLoc, OMPC_use))
+  if (!isValidInteropVariable(SemaRef, InteropVar, VarLoc, OMPC_use))
     return nullptr;
 
-  return new (Context)
+  return new (getASTContext())
       OMPUseClause(InteropVar, StartLoc, LParenLoc, VarLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPDestroyClause(Expr *InteropVar,
-                                          SourceLocation StartLoc,
-                                          SourceLocation LParenLoc,
-                                          SourceLocation VarLoc,
-                                          SourceLocation EndLoc) {
-  if (!InteropVar && LangOpts.OpenMP >= 52 &&
+OMPClause *SemaOpenMP::ActOnOpenMPDestroyClause(Expr *InteropVar,
+                                                SourceLocation StartLoc,
+                                                SourceLocation LParenLoc,
+                                                SourceLocation VarLoc,
+                                                SourceLocation EndLoc) {
+  if (!InteropVar && getLangOpts().OpenMP >= 52 &&
       DSAStack->getCurrentDirective() == OMPD_depobj) {
     Diag(StartLoc, diag::err_omp_expected_clause_argument)
         << getOpenMPClauseName(OMPC_destroy)
@@ -18110,100 +18174,103 @@ OMPClause *Sema::ActOnOpenMPDestroyClause(Expr *InteropVar,
     return nullptr;
   }
   if (InteropVar &&
-      !isValidInteropVariable(*this, InteropVar, VarLoc, OMPC_destroy))
+      !isValidInteropVariable(SemaRef, InteropVar, VarLoc, OMPC_destroy))
     return nullptr;
 
-  return new (Context)
+  return new (getASTContext())
       OMPDestroyClause(InteropVar, StartLoc, LParenLoc, VarLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPNovariantsClause(Expr *Condition,
-                                             SourceLocation StartLoc,
-                                             SourceLocation LParenLoc,
-                                             SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPNovariantsClause(Expr *Condition,
+                                                   SourceLocation StartLoc,
+                                                   SourceLocation LParenLoc,
+                                                   SourceLocation EndLoc) {
   Expr *ValExpr = Condition;
   Stmt *HelperValStmt = nullptr;
   OpenMPDirectiveKind CaptureRegion = OMPD_unknown;
   if (!Condition->isValueDependent() && !Condition->isTypeDependent() &&
       !Condition->isInstantiationDependent() &&
       !Condition->containsUnexpandedParameterPack()) {
-    ExprResult Val = CheckBooleanCondition(StartLoc, Condition);
+    ExprResult Val = SemaRef.CheckBooleanCondition(StartLoc, Condition);
     if (Val.isInvalid())
       return nullptr;
 
-    ValExpr = MakeFullExpr(Val.get()).get();
+    ValExpr = SemaRef.MakeFullExpr(Val.get()).get();
 
     OpenMPDirectiveKind DKind = DSAStack->getCurrentDirective();
     CaptureRegion = getOpenMPCaptureRegionForClause(DKind, OMPC_novariants,
-                                                    LangOpts.OpenMP);
-    if (CaptureRegion != OMPD_unknown && !CurContext->isDependentContext()) {
-      ValExpr = MakeFullExpr(ValExpr).get();
+                                                    getLangOpts().OpenMP);
+    if (CaptureRegion != OMPD_unknown &&
+        !SemaRef.CurContext->isDependentContext()) {
+      ValExpr = SemaRef.MakeFullExpr(ValExpr).get();
       llvm::MapVector<const Expr *, DeclRefExpr *> Captures;
-      ValExpr = tryBuildCapture(*this, ValExpr, Captures).get();
-      HelperValStmt = buildPreInits(Context, Captures);
+      ValExpr = tryBuildCapture(SemaRef, ValExpr, Captures).get();
+      HelperValStmt = buildPreInits(getASTContext(), Captures);
     }
   }
 
-  return new (Context) OMPNovariantsClause(
+  return new (getASTContext()) OMPNovariantsClause(
       ValExpr, HelperValStmt, CaptureRegion, StartLoc, LParenLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPNocontextClause(Expr *Condition,
-                                            SourceLocation StartLoc,
-                                            SourceLocation LParenLoc,
-                                            SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPNocontextClause(Expr *Condition,
+                                                  SourceLocation StartLoc,
+                                                  SourceLocation LParenLoc,
+                                                  SourceLocation EndLoc) {
   Expr *ValExpr = Condition;
   Stmt *HelperValStmt = nullptr;
   OpenMPDirectiveKind CaptureRegion = OMPD_unknown;
   if (!Condition->isValueDependent() && !Condition->isTypeDependent() &&
       !Condition->isInstantiationDependent() &&
       !Condition->containsUnexpandedParameterPack()) {
-    ExprResult Val = CheckBooleanCondition(StartLoc, Condition);
+    ExprResult Val = SemaRef.CheckBooleanCondition(StartLoc, Condition);
     if (Val.isInvalid())
       return nullptr;
 
-    ValExpr = MakeFullExpr(Val.get()).get();
+    ValExpr = SemaRef.MakeFullExpr(Val.get()).get();
 
     OpenMPDirectiveKind DKind = DSAStack->getCurrentDirective();
-    CaptureRegion =
-        getOpenMPCaptureRegionForClause(DKind, OMPC_nocontext, LangOpts.OpenMP);
-    if (CaptureRegion != OMPD_unknown && !CurContext->isDependentContext()) {
-      ValExpr = MakeFullExpr(ValExpr).get();
+    CaptureRegion = getOpenMPCaptureRegionForClause(DKind, OMPC_nocontext,
+                                                    getLangOpts().OpenMP);
+    if (CaptureRegion != OMPD_unknown &&
+        !SemaRef.CurContext->isDependentContext()) {
+      ValExpr = SemaRef.MakeFullExpr(ValExpr).get();
       llvm::MapVector<const Expr *, DeclRefExpr *> Captures;
-      ValExpr = tryBuildCapture(*this, ValExpr, Captures).get();
-      HelperValStmt = buildPreInits(Context, Captures);
+      ValExpr = tryBuildCapture(SemaRef, ValExpr, Captures).get();
+      HelperValStmt = buildPreInits(getASTContext(), Captures);
     }
   }
 
-  return new (Context) OMPNocontextClause(ValExpr, HelperValStmt, CaptureRegion,
-                                          StartLoc, LParenLoc, EndLoc);
+  return new (getASTContext()) OMPNocontextClause(
+      ValExpr, HelperValStmt, CaptureRegion, StartLoc, LParenLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPFilterClause(Expr *ThreadID,
-                                         SourceLocation StartLoc,
-                                         SourceLocation LParenLoc,
-                                         SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPFilterClause(Expr *ThreadID,
+                                               SourceLocation StartLoc,
+                                               SourceLocation LParenLoc,
+                                               SourceLocation EndLoc) {
   Expr *ValExpr = ThreadID;
   Stmt *HelperValStmt = nullptr;
 
   OpenMPDirectiveKind DKind = DSAStack->getCurrentDirective();
   OpenMPDirectiveKind CaptureRegion =
-      getOpenMPCaptureRegionForClause(DKind, OMPC_filter, LangOpts.OpenMP);
-  if (CaptureRegion != OMPD_unknown && !CurContext->isDependentContext()) {
-    ValExpr = MakeFullExpr(ValExpr).get();
+      getOpenMPCaptureRegionForClause(DKind, OMPC_filter, getLangOpts().OpenMP);
+  if (CaptureRegion != OMPD_unknown &&
+      !SemaRef.CurContext->isDependentContext()) {
+    ValExpr = SemaRef.MakeFullExpr(ValExpr).get();
     llvm::MapVector<const Expr *, DeclRefExpr *> Captures;
-    ValExpr = tryBuildCapture(*this, ValExpr, Captures).get();
-    HelperValStmt = buildPreInits(Context, Captures);
+    ValExpr = tryBuildCapture(SemaRef, ValExpr, Captures).get();
+    HelperValStmt = buildPreInits(getASTContext(), Captures);
   }
 
-  return new (Context) OMPFilterClause(ValExpr, HelperValStmt, CaptureRegion,
-                                       StartLoc, LParenLoc, EndLoc);
+  return new (getASTContext()) OMPFilterClause(
+      ValExpr, HelperValStmt, CaptureRegion, StartLoc, LParenLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPVarListClause(OpenMPClauseKind Kind,
-                                          ArrayRef<Expr *> VarList,
-                                          const OMPVarListLocTy &Locs,
-                                          OpenMPVarListDataTy &Data) {
+OMPClause *SemaOpenMP::ActOnOpenMPVarListClause(OpenMPClauseKind Kind,
+                                                ArrayRef<Expr *> VarList,
+                                                const OMPVarListLocTy &Locs,
+                                                OpenMPVarListDataTy &Data) {
   SourceLocation StartLoc = Locs.StartLoc;
   SourceLocation LParenLoc = Locs.LParenLoc;
   SourceLocation EndLoc = Locs.EndLoc;
@@ -18395,29 +18462,30 @@ OMPClause *Sema::ActOnOpenMPVarListClause(OpenMPClauseKind Kind,
   return Res;
 }
 
-ExprResult Sema::getOpenMPCapturedExpr(VarDecl *Capture, ExprValueKind VK,
-                                       ExprObjectKind OK, SourceLocation Loc) {
-  ExprResult Res = BuildDeclRefExpr(
+ExprResult SemaOpenMP::getOpenMPCapturedExpr(VarDecl *Capture, ExprValueKind VK,
+                                             ExprObjectKind OK,
+                                             SourceLocation Loc) {
+  ExprResult Res = SemaRef.BuildDeclRefExpr(
       Capture, Capture->getType().getNonReferenceType(), VK_LValue, Loc);
   if (!Res.isUsable())
     return ExprError();
   if (OK == OK_Ordinary && !getLangOpts().CPlusPlus) {
-    Res = CreateBuiltinUnaryOp(Loc, UO_Deref, Res.get());
+    Res = SemaRef.CreateBuiltinUnaryOp(Loc, UO_Deref, Res.get());
     if (!Res.isUsable())
       return ExprError();
   }
   if (VK != VK_LValue && Res.get()->isGLValue()) {
-    Res = DefaultLvalueConversion(Res.get());
+    Res = SemaRef.DefaultLvalueConversion(Res.get());
     if (!Res.isUsable())
       return ExprError();
   }
   return Res;
 }
 
-OMPClause *Sema::ActOnOpenMPPrivateClause(ArrayRef<Expr *> VarList,
-                                          SourceLocation StartLoc,
-                                          SourceLocation LParenLoc,
-                                          SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPPrivateClause(ArrayRef<Expr *> VarList,
+                                                SourceLocation StartLoc,
+                                                SourceLocation LParenLoc,
+                                                SourceLocation EndLoc) {
   SmallVector<Expr *, 8> Vars;
   SmallVector<Expr *, 8> PrivateCopies;
   bool IsImplicitClause =
@@ -18427,7 +18495,7 @@ OMPClause *Sema::ActOnOpenMPPrivateClause(ArrayRef<Expr *> VarList,
     SourceLocation ELoc;
     SourceRange ERange;
     Expr *SimpleRefExpr = RefExpr;
-    auto Res = getPrivateItem(*this, SimpleRefExpr, ELoc, ERange);
+    auto Res = getPrivateItem(SemaRef, SimpleRefExpr, ELoc, ERange);
     if (Res.second) {
       // It will be analyzed later.
       Vars.push_back(RefExpr);
@@ -18443,7 +18511,8 @@ OMPClause *Sema::ActOnOpenMPPrivateClause(ArrayRef<Expr *> VarList,
     // OpenMP [2.9.3.3, Restrictions, C/C++, p.3]
     //  A variable that appears in a private clause must not have an incomplete
     //  type or a reference type.
-    if (RequireCompleteType(ELoc, Type, diag::err_omp_private_incomplete_type))
+    if (SemaRef.RequireCompleteType(ELoc, Type,
+                                    diag::err_omp_private_incomplete_type))
       continue;
     Type = Type.getNonReferenceType();
 
@@ -18455,7 +18524,7 @@ OMPClause *Sema::ActOnOpenMPPrivateClause(ArrayRef<Expr *> VarList,
     // OpenMP 3.1 [2.9.3.3, private clause, Restrictions]
     // A variable that appears in a private clause must not have a
     // const-qualified type unless it is of class type with a mutable member.
-    if (rejectConstNotMutableType(*this, D, Type, OMPC_private, ELoc))
+    if (rejectConstNotMutableType(SemaRef, D, Type, OMPC_private, ELoc))
       continue;
 
     // OpenMP [2.9.1.1, Data-sharing Attribute Rules for Variables Referenced
@@ -18469,7 +18538,7 @@ OMPClause *Sema::ActOnOpenMPPrivateClause(ArrayRef<Expr *> VarList,
     if (DVar.CKind != OMPC_unknown && DVar.CKind != OMPC_private) {
       Diag(ELoc, diag::err_omp_wrong_dsa) << getOpenMPClauseName(DVar.CKind)
                                           << getOpenMPClauseName(OMPC_private);
-      reportOriginalDsa(*this, DSAStack, D, DVar);
+      reportOriginalDsa(SemaRef, DSAStack, D, DVar);
       continue;
     }
 
@@ -18480,7 +18549,7 @@ OMPClause *Sema::ActOnOpenMPPrivateClause(ArrayRef<Expr *> VarList,
       Diag(ELoc, diag::err_omp_variably_modified_type_not_supported)
           << getOpenMPClauseName(OMPC_private) << Type
           << getOpenMPDirectiveName(CurrDir);
-      bool IsDecl = !VD || VD->isThisDeclarationADefinition(Context) ==
+      bool IsDecl = !VD || VD->isThisDeclarationADefinition(getASTContext()) ==
                                VarDecl::DeclarationOnly;
       Diag(D->getLocation(),
            IsDecl ? diag::note_previous_decl : diag::note_defined_here)
@@ -18496,7 +18565,8 @@ OMPClause *Sema::ActOnOpenMPPrivateClause(ArrayRef<Expr *> VarList,
     // A list item cannot appear in both a map clause and a data-sharing
     // attribute clause on the same construct unless the construct is a
     // combined construct.
-    if ((LangOpts.OpenMP <= 45 && isOpenMPTargetExecutionDirective(CurrDir)) ||
+    if ((getLangOpts().OpenMP <= 45 &&
+         isOpenMPTargetExecutionDirective(CurrDir)) ||
         CurrDir == OMPD_target) {
       OpenMPClauseKind ConflictKind;
       if (DSAStack->checkMappableExprComponentListsForDecl(
@@ -18510,7 +18580,7 @@ OMPClause *Sema::ActOnOpenMPPrivateClause(ArrayRef<Expr *> VarList,
             << getOpenMPClauseName(OMPC_private)
             << getOpenMPClauseName(ConflictKind)
             << getOpenMPDirectiveName(CurrDir);
-        reportOriginalDsa(*this, DSAStack, D, DVar);
+        reportOriginalDsa(SemaRef, DSAStack, D, DVar);
         continue;
       }
     }
@@ -18526,28 +18596,28 @@ OMPClause *Sema::ActOnOpenMPPrivateClause(ArrayRef<Expr *> VarList,
     // proper diagnostics.
     Type = Type.getUnqualifiedType();
     VarDecl *VDPrivate =
-        buildVarDecl(*this, ELoc, Type, D->getName(),
+        buildVarDecl(SemaRef, ELoc, Type, D->getName(),
                      D->hasAttrs() ? &D->getAttrs() : nullptr,
                      VD ? cast<DeclRefExpr>(SimpleRefExpr) : nullptr);
-    ActOnUninitializedDecl(VDPrivate);
+    SemaRef.ActOnUninitializedDecl(VDPrivate);
     if (VDPrivate->isInvalidDecl())
       continue;
     DeclRefExpr *VDPrivateRefExpr = buildDeclRefExpr(
-        *this, VDPrivate, RefExpr->getType().getUnqualifiedType(), ELoc);
+        SemaRef, VDPrivate, RefExpr->getType().getUnqualifiedType(), ELoc);
 
     DeclRefExpr *Ref = nullptr;
-    if (!VD && !CurContext->isDependentContext()) {
+    if (!VD && !SemaRef.CurContext->isDependentContext()) {
       auto *FD = dyn_cast<FieldDecl>(D);
       VarDecl *VD = FD ? DSAStack->getImplicitFDCapExprDecl(FD) : nullptr;
       if (VD)
-        Ref = buildDeclRefExpr(*this, VD, VD->getType().getNonReferenceType(),
+        Ref = buildDeclRefExpr(SemaRef, VD, VD->getType().getNonReferenceType(),
                                RefExpr->getExprLoc());
       else
-        Ref = buildCapture(*this, D, SimpleRefExpr, /*WithInit=*/false);
+        Ref = buildCapture(SemaRef, D, SimpleRefExpr, /*WithInit=*/false);
     }
     if (!IsImplicitClause)
       DSAStack->addDSA(D, RefExpr->IgnoreParens(), OMPC_private, Ref);
-    Vars.push_back((VD || CurContext->isDependentContext())
+    Vars.push_back((VD || SemaRef.CurContext->isDependentContext())
                        ? RefExpr->IgnoreParens()
                        : Ref);
     PrivateCopies.push_back(VDPrivateRefExpr);
@@ -18556,14 +18626,14 @@ OMPClause *Sema::ActOnOpenMPPrivateClause(ArrayRef<Expr *> VarList,
   if (Vars.empty())
     return nullptr;
 
-  return OMPPrivateClause::Create(Context, StartLoc, LParenLoc, EndLoc, Vars,
-                                  PrivateCopies);
+  return OMPPrivateClause::Create(getASTContext(), StartLoc, LParenLoc, EndLoc,
+                                  Vars, PrivateCopies);
 }
 
-OMPClause *Sema::ActOnOpenMPFirstprivateClause(ArrayRef<Expr *> VarList,
-                                               SourceLocation StartLoc,
-                                               SourceLocation LParenLoc,
-                                               SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPFirstprivateClause(ArrayRef<Expr *> VarList,
+                                                     SourceLocation StartLoc,
+                                                     SourceLocation LParenLoc,
+                                                     SourceLocation EndLoc) {
   SmallVector<Expr *, 8> Vars;
   SmallVector<Expr *, 8> PrivateCopies;
   SmallVector<Expr *, 8> Inits;
@@ -18577,7 +18647,7 @@ OMPClause *Sema::ActOnOpenMPFirstprivateClause(ArrayRef<Expr *> VarList,
     SourceLocation ELoc;
     SourceRange ERange;
     Expr *SimpleRefExpr = RefExpr;
-    auto Res = getPrivateItem(*this, SimpleRefExpr, ELoc, ERange);
+    auto Res = getPrivateItem(SemaRef, SimpleRefExpr, ELoc, ERange);
     if (Res.second) {
       // It will be analyzed later.
       Vars.push_back(RefExpr);
@@ -18595,8 +18665,8 @@ OMPClause *Sema::ActOnOpenMPFirstprivateClause(ArrayRef<Expr *> VarList,
     // OpenMP [2.9.3.3, Restrictions, C/C++, p.3]
     //  A variable that appears in a private clause must not have an incomplete
     //  type or a reference type.
-    if (RequireCompleteType(ELoc, Type,
-                            diag::err_omp_firstprivate_incomplete_type))
+    if (SemaRef.RequireCompleteType(ELoc, Type,
+                                    diag::err_omp_firstprivate_incomplete_type))
       continue;
     Type = Type.getNonReferenceType();
 
@@ -18604,7 +18674,8 @@ OMPClause *Sema::ActOnOpenMPFirstprivateClause(ArrayRef<Expr *> VarList,
     //  A variable of class type (or array thereof) that appears in a private
     //  clause requires an accessible, unambiguous copy constructor for the
     //  class type.
-    QualType ElemType = Context.getBaseElementType(Type).getNonReferenceType();
+    QualType ElemType =
+        getASTContext().getBaseElementType(Type).getNonReferenceType();
 
     // If an implicit firstprivate variable found it was checked already.
     DSAStackTy::DSAVarData TopDVar;
@@ -18613,7 +18684,7 @@ OMPClause *Sema::ActOnOpenMPFirstprivateClause(ArrayRef<Expr *> VarList,
           DSAStack->getTopDSA(D, /*FromParent=*/false);
       TopDVar = DVar;
       OpenMPDirectiveKind CurrDir = DSAStack->getCurrentDirective();
-      bool IsConstant = ElemType.isConstant(Context);
+      bool IsConstant = ElemType.isConstant(getASTContext());
       // OpenMP [2.4.13, Data-sharing Attribute Clauses]
       //  A list item that specifies a given variable may not appear in more
       // than one clause on the same directive, except that a variable may be
@@ -18628,7 +18699,7 @@ OMPClause *Sema::ActOnOpenMPFirstprivateClause(ArrayRef<Expr *> VarList,
         Diag(ELoc, diag::err_omp_wrong_dsa)
             << getOpenMPClauseName(DVar.CKind)
             << getOpenMPClauseName(OMPC_firstprivate);
-        reportOriginalDsa(*this, DSAStack, D, DVar);
+        reportOriginalDsa(SemaRef, DSAStack, D, DVar);
         continue;
       }
 
@@ -18648,7 +18719,7 @@ OMPClause *Sema::ActOnOpenMPFirstprivateClause(ArrayRef<Expr *> VarList,
         Diag(ELoc, diag::err_omp_wrong_dsa)
             << getOpenMPClauseName(DVar.CKind)
             << getOpenMPClauseName(OMPC_firstprivate);
-        reportOriginalDsa(*this, DSAStack, D, DVar);
+        reportOriginalDsa(SemaRef, DSAStack, D, DVar);
         continue;
       }
 
@@ -18679,7 +18750,7 @@ OMPClause *Sema::ActOnOpenMPFirstprivateClause(ArrayRef<Expr *> VarList,
           Diag(ELoc, diag::err_omp_required_access)
               << getOpenMPClauseName(OMPC_firstprivate)
               << getOpenMPClauseName(OMPC_shared);
-          reportOriginalDsa(*this, DSAStack, D, DVar);
+          reportOriginalDsa(SemaRef, DSAStack, D, DVar);
           continue;
         }
       }
@@ -18712,7 +18783,7 @@ OMPClause *Sema::ActOnOpenMPFirstprivateClause(ArrayRef<Expr *> VarList,
              isOpenMPTeamsDirective(DVar.DKind))) {
           Diag(ELoc, diag::err_omp_parallel_reduction_in_task_firstprivate)
               << getOpenMPDirectiveName(DVar.DKind);
-          reportOriginalDsa(*this, DSAStack, D, DVar);
+          reportOriginalDsa(SemaRef, DSAStack, D, DVar);
           continue;
         }
       }
@@ -18725,7 +18796,7 @@ OMPClause *Sema::ActOnOpenMPFirstprivateClause(ArrayRef<Expr *> VarList,
       // A list item cannot appear in both a map clause and a data-sharing
       // attribute clause on the same construct unless the construct is a
       // combined construct.
-      if ((LangOpts.OpenMP <= 45 &&
+      if ((getLangOpts().OpenMP <= 45 &&
            isOpenMPTargetExecutionDirective(CurrDir)) ||
           CurrDir == OMPD_target) {
         OpenMPClauseKind ConflictKind;
@@ -18741,7 +18812,7 @@ OMPClause *Sema::ActOnOpenMPFirstprivateClause(ArrayRef<Expr *> VarList,
               << getOpenMPClauseName(OMPC_firstprivate)
               << getOpenMPClauseName(ConflictKind)
               << getOpenMPDirectiveName(DSAStack->getCurrentDirective());
-          reportOriginalDsa(*this, DSAStack, D, DVar);
+          reportOriginalDsa(SemaRef, DSAStack, D, DVar);
           continue;
         }
       }
@@ -18753,7 +18824,7 @@ OMPClause *Sema::ActOnOpenMPFirstprivateClause(ArrayRef<Expr *> VarList,
       Diag(ELoc, diag::err_omp_variably_modified_type_not_supported)
           << getOpenMPClauseName(OMPC_firstprivate) << Type
           << getOpenMPDirectiveName(DSAStack->getCurrentDirective());
-      bool IsDecl = !VD || VD->isThisDeclarationADefinition(Context) ==
+      bool IsDecl = !VD || VD->isThisDeclarationADefinition(getASTContext()) ==
                                VarDecl::DeclarationOnly;
       Diag(D->getLocation(),
            IsDecl ? diag::note_previous_decl : diag::note_defined_here)
@@ -18763,7 +18834,7 @@ OMPClause *Sema::ActOnOpenMPFirstprivateClause(ArrayRef<Expr *> VarList,
 
     Type = Type.getUnqualifiedType();
     VarDecl *VDPrivate =
-        buildVarDecl(*this, ELoc, Type, D->getName(),
+        buildVarDecl(SemaRef, ELoc, Type, D->getName(),
                      D->hasAttrs() ? &D->getAttrs() : nullptr,
                      VD ? cast<DeclRefExpr>(SimpleRefExpr) : nullptr);
     // Generate helper private variable and initialize it with the value of the
@@ -18776,32 +18847,32 @@ OMPClause *Sema::ActOnOpenMPFirstprivateClause(ArrayRef<Expr *> VarList,
     // original array element in CodeGen.
     if (Type->isArrayType()) {
       VarDecl *VDInit =
-          buildVarDecl(*this, RefExpr->getExprLoc(), ElemType, D->getName());
-      VDInitRefExpr = buildDeclRefExpr(*this, VDInit, ElemType, ELoc);
-      Expr *Init = DefaultLvalueConversion(VDInitRefExpr).get();
+          buildVarDecl(SemaRef, RefExpr->getExprLoc(), ElemType, D->getName());
+      VDInitRefExpr = buildDeclRefExpr(SemaRef, VDInit, ElemType, ELoc);
+      Expr *Init = SemaRef.DefaultLvalueConversion(VDInitRefExpr).get();
       ElemType = ElemType.getUnqualifiedType();
-      VarDecl *VDInitTemp = buildVarDecl(*this, RefExpr->getExprLoc(), ElemType,
-                                         ".firstprivate.temp");
+      VarDecl *VDInitTemp = buildVarDecl(SemaRef, RefExpr->getExprLoc(),
+                                         ElemType, ".firstprivate.temp");
       InitializedEntity Entity =
           InitializedEntity::InitializeVariable(VDInitTemp);
       InitializationKind Kind = InitializationKind::CreateCopy(ELoc, ELoc);
 
-      InitializationSequence InitSeq(*this, Entity, Kind, Init);
-      ExprResult Result = InitSeq.Perform(*this, Entity, Kind, Init);
+      InitializationSequence InitSeq(SemaRef, Entity, Kind, Init);
+      ExprResult Result = InitSeq.Perform(SemaRef, Entity, Kind, Init);
       if (Result.isInvalid())
         VDPrivate->setInvalidDecl();
       else
         VDPrivate->setInit(Result.getAs<Expr>());
       // Remove temp variable declaration.
-      Context.Deallocate(VDInitTemp);
+      getASTContext().Deallocate(VDInitTemp);
     } else {
-      VarDecl *VDInit = buildVarDecl(*this, RefExpr->getExprLoc(), Type,
+      VarDecl *VDInit = buildVarDecl(SemaRef, RefExpr->getExprLoc(), Type,
                                      ".firstprivate.temp");
-      VDInitRefExpr = buildDeclRefExpr(*this, VDInit, RefExpr->getType(),
+      VDInitRefExpr = buildDeclRefExpr(SemaRef, VDInit, RefExpr->getType(),
                                        RefExpr->getExprLoc());
-      AddInitializerToDecl(VDPrivate,
-                           DefaultLvalueConversion(VDInitRefExpr).get(),
-                           /*DirectInit=*/false);
+      SemaRef.AddInitializerToDecl(
+          VDPrivate, SemaRef.DefaultLvalueConversion(VDInitRefExpr).get(),
+          /*DirectInit=*/false);
     }
     if (VDPrivate->isInvalidDecl()) {
       if (IsImplicitClause) {
@@ -18810,29 +18881,30 @@ OMPClause *Sema::ActOnOpenMPFirstprivateClause(ArrayRef<Expr *> VarList,
       }
       continue;
     }
-    CurContext->addDecl(VDPrivate);
+    SemaRef.CurContext->addDecl(VDPrivate);
     DeclRefExpr *VDPrivateRefExpr = buildDeclRefExpr(
-        *this, VDPrivate, RefExpr->getType().getUnqualifiedType(),
+        SemaRef, VDPrivate, RefExpr->getType().getUnqualifiedType(),
         RefExpr->getExprLoc());
     DeclRefExpr *Ref = nullptr;
-    if (!VD && !CurContext->isDependentContext()) {
+    if (!VD && !SemaRef.CurContext->isDependentContext()) {
       if (TopDVar.CKind == OMPC_lastprivate) {
         Ref = TopDVar.PrivateCopy;
       } else {
         auto *FD = dyn_cast<FieldDecl>(D);
         VarDecl *VD = FD ? DSAStack->getImplicitFDCapExprDecl(FD) : nullptr;
         if (VD)
-          Ref = buildDeclRefExpr(*this, VD, VD->getType().getNonReferenceType(),
-                                 RefExpr->getExprLoc());
+          Ref =
+              buildDeclRefExpr(SemaRef, VD, VD->getType().getNonReferenceType(),
+                               RefExpr->getExprLoc());
         else
-          Ref = buildCapture(*this, D, SimpleRefExpr, /*WithInit=*/true);
+          Ref = buildCapture(SemaRef, D, SimpleRefExpr, /*WithInit=*/true);
         if (VD || !isOpenMPCapturedDecl(D))
           ExprCaptures.push_back(Ref->getDecl());
       }
     }
     if (!IsImplicitClause)
       DSAStack->addDSA(D, RefExpr->IgnoreParens(), OMPC_firstprivate, Ref);
-    Vars.push_back((VD || CurContext->isDependentContext())
+    Vars.push_back((VD || SemaRef.CurContext->isDependentContext())
                        ? RefExpr->IgnoreParens()
                        : Ref);
     PrivateCopies.push_back(VDPrivateRefExpr);
@@ -18842,12 +18914,12 @@ OMPClause *Sema::ActOnOpenMPFirstprivateClause(ArrayRef<Expr *> VarList,
   if (Vars.empty())
     return nullptr;
 
-  return OMPFirstprivateClause::Create(Context, StartLoc, LParenLoc, EndLoc,
-                                       Vars, PrivateCopies, Inits,
-                                       buildPreInits(Context, ExprCaptures));
+  return OMPFirstprivateClause::Create(
+      getASTContext(), StartLoc, LParenLoc, EndLoc, Vars, PrivateCopies, Inits,
+      buildPreInits(getASTContext(), ExprCaptures));
 }
 
-OMPClause *Sema::ActOnOpenMPLastprivateClause(
+OMPClause *SemaOpenMP::ActOnOpenMPLastprivateClause(
     ArrayRef<Expr *> VarList, OpenMPLastprivateModifier LPKind,
     SourceLocation LPKindLoc, SourceLocation ColonLoc, SourceLocation StartLoc,
     SourceLocation LParenLoc, SourceLocation EndLoc) {
@@ -18871,7 +18943,7 @@ OMPClause *Sema::ActOnOpenMPLastprivateClause(
     SourceLocation ELoc;
     SourceRange ERange;
     Expr *SimpleRefExpr = RefExpr;
-    auto Res = getPrivateItem(*this, SimpleRefExpr, ELoc, ERange);
+    auto Res = getPrivateItem(SemaRef, SimpleRefExpr, ELoc, ERange);
     if (Res.second) {
       // It will be analyzed later.
       Vars.push_back(RefExpr);
@@ -18889,8 +18961,8 @@ OMPClause *Sema::ActOnOpenMPLastprivateClause(
     // OpenMP [2.14.3.5, Restrictions, C/C++, p.2]
     //  A variable that appears in a lastprivate clause must not have an
     //  incomplete type or a reference type.
-    if (RequireCompleteType(ELoc, Type,
-                            diag::err_omp_lastprivate_incomplete_type))
+    if (SemaRef.RequireCompleteType(ELoc, Type,
+                                    diag::err_omp_lastprivate_incomplete_type))
       continue;
     Type = Type.getNonReferenceType();
 
@@ -18902,7 +18974,7 @@ OMPClause *Sema::ActOnOpenMPLastprivateClause(
     // OpenMP 3.1 [2.9.3.5, lastprivate clause, Restrictions]
     // A variable that appears in a lastprivate clause must not have a
     // const-qualified type unless it is of class type with a mutable member.
-    if (rejectConstNotMutableType(*this, D, Type, OMPC_lastprivate, ELoc))
+    if (rejectConstNotMutableType(SemaRef, D, Type, OMPC_lastprivate, ELoc))
       continue;
 
     // OpenMP 5.0 [2.19.4.5 lastprivate Clause, Restrictions]
@@ -18910,7 +18982,7 @@ OMPClause *Sema::ActOnOpenMPLastprivateClause(
     // modifier must be a scalar variable.
     if (LPKind == OMPC_LASTPRIVATE_conditional && !Type->isScalarType()) {
       Diag(ELoc, diag::err_omp_lastprivate_conditional_non_scalar);
-      bool IsDecl = !VD || VD->isThisDeclarationADefinition(Context) ==
+      bool IsDecl = !VD || VD->isThisDeclarationADefinition(getASTContext()) ==
                                VarDecl::DeclarationOnly;
       Diag(D->getLocation(),
            IsDecl ? diag::note_previous_decl : diag::note_defined_here)
@@ -18935,7 +19007,7 @@ OMPClause *Sema::ActOnOpenMPLastprivateClause(
       Diag(ELoc, diag::err_omp_wrong_dsa)
           << getOpenMPClauseName(DVar.CKind)
           << getOpenMPClauseName(OMPC_lastprivate);
-      reportOriginalDsa(*this, DSAStack, D, DVar);
+      reportOriginalDsa(SemaRef, DSAStack, D, DVar);
       continue;
     }
 
@@ -18954,7 +19026,7 @@ OMPClause *Sema::ActOnOpenMPLastprivateClause(
         Diag(ELoc, diag::err_omp_required_access)
             << getOpenMPClauseName(OMPC_lastprivate)
             << getOpenMPClauseName(OMPC_shared);
-        reportOriginalDsa(*this, DSAStack, D, DVar);
+        reportOriginalDsa(SemaRef, DSAStack, D, DVar);
         continue;
       }
     }
@@ -18967,53 +19039,53 @@ OMPClause *Sema::ActOnOpenMPLastprivateClause(
     //  A variable of class type (or array thereof) that appears in a
     //  lastprivate clause requires an accessible, unambiguous copy assignment
     //  operator for the class type.
-    Type = Context.getBaseElementType(Type).getNonReferenceType();
-    VarDecl *SrcVD = buildVarDecl(*this, ERange.getBegin(),
+    Type = getASTContext().getBaseElementType(Type).getNonReferenceType();
+    VarDecl *SrcVD = buildVarDecl(SemaRef, ERange.getBegin(),
                                   Type.getUnqualifiedType(), ".lastprivate.src",
                                   D->hasAttrs() ? &D->getAttrs() : nullptr);
     DeclRefExpr *PseudoSrcExpr =
-        buildDeclRefExpr(*this, SrcVD, Type.getUnqualifiedType(), ELoc);
+        buildDeclRefExpr(SemaRef, SrcVD, Type.getUnqualifiedType(), ELoc);
     VarDecl *DstVD =
-        buildVarDecl(*this, ERange.getBegin(), Type, ".lastprivate.dst",
+        buildVarDecl(SemaRef, ERange.getBegin(), Type, ".lastprivate.dst",
                      D->hasAttrs() ? &D->getAttrs() : nullptr);
-    DeclRefExpr *PseudoDstExpr = buildDeclRefExpr(*this, DstVD, Type, ELoc);
+    DeclRefExpr *PseudoDstExpr = buildDeclRefExpr(SemaRef, DstVD, Type, ELoc);
     // For arrays generate assignment operation for single element and replace
     // it by the original array element in CodeGen.
-    ExprResult AssignmentOp = BuildBinOp(/*S=*/nullptr, ELoc, BO_Assign,
-                                         PseudoDstExpr, PseudoSrcExpr);
+    ExprResult AssignmentOp = SemaRef.BuildBinOp(/*S=*/nullptr, ELoc, BO_Assign,
+                                                 PseudoDstExpr, PseudoSrcExpr);
     if (AssignmentOp.isInvalid())
       continue;
-    AssignmentOp =
-        ActOnFinishFullExpr(AssignmentOp.get(), ELoc, /*DiscardedValue*/ false);
+    AssignmentOp = SemaRef.ActOnFinishFullExpr(AssignmentOp.get(), ELoc,
+                                               /*DiscardedValue*/ false);
     if (AssignmentOp.isInvalid())
       continue;
 
     DeclRefExpr *Ref = nullptr;
-    if (!VD && !CurContext->isDependentContext()) {
+    if (!VD && !SemaRef.CurContext->isDependentContext()) {
       if (TopDVar.CKind == OMPC_firstprivate) {
         Ref = TopDVar.PrivateCopy;
       } else {
-        Ref = buildCapture(*this, D, SimpleRefExpr, /*WithInit=*/false);
+        Ref = buildCapture(SemaRef, D, SimpleRefExpr, /*WithInit=*/false);
         if (!isOpenMPCapturedDecl(D))
           ExprCaptures.push_back(Ref->getDecl());
       }
       if ((TopDVar.CKind == OMPC_firstprivate && !TopDVar.PrivateCopy) ||
           (!isOpenMPCapturedDecl(D) &&
            Ref->getDecl()->hasAttr<OMPCaptureNoInitAttr>())) {
-        ExprResult RefRes = DefaultLvalueConversion(Ref);
+        ExprResult RefRes = SemaRef.DefaultLvalueConversion(Ref);
         if (!RefRes.isUsable())
           continue;
         ExprResult PostUpdateRes =
-            BuildBinOp(DSAStack->getCurScope(), ELoc, BO_Assign, SimpleRefExpr,
-                       RefRes.get());
+            SemaRef.BuildBinOp(DSAStack->getCurScope(), ELoc, BO_Assign,
+                               SimpleRefExpr, RefRes.get());
         if (!PostUpdateRes.isUsable())
           continue;
         ExprPostUpdates.push_back(
-            IgnoredValueConversions(PostUpdateRes.get()).get());
+            SemaRef.IgnoredValueConversions(PostUpdateRes.get()).get());
       }
     }
     DSAStack->addDSA(D, RefExpr->IgnoreParens(), OMPC_lastprivate, Ref);
-    Vars.push_back((VD || CurContext->isDependentContext())
+    Vars.push_back((VD || SemaRef.CurContext->isDependentContext())
                        ? RefExpr->IgnoreParens()
                        : Ref);
     SrcExprs.push_back(PseudoSrcExpr);
@@ -19024,24 +19096,24 @@ OMPClause *Sema::ActOnOpenMPLastprivateClause(
   if (Vars.empty())
     return nullptr;
 
-  return OMPLastprivateClause::Create(Context, StartLoc, LParenLoc, EndLoc,
-                                      Vars, SrcExprs, DstExprs, AssignmentOps,
-                                      LPKind, LPKindLoc, ColonLoc,
-                                      buildPreInits(Context, ExprCaptures),
-                                      buildPostUpdate(*this, ExprPostUpdates));
+  return OMPLastprivateClause::Create(
+      getASTContext(), StartLoc, LParenLoc, EndLoc, Vars, SrcExprs, DstExprs,
+      AssignmentOps, LPKind, LPKindLoc, ColonLoc,
+      buildPreInits(getASTContext(), ExprCaptures),
+      buildPostUpdate(SemaRef, ExprPostUpdates));
 }
 
-OMPClause *Sema::ActOnOpenMPSharedClause(ArrayRef<Expr *> VarList,
-                                         SourceLocation StartLoc,
-                                         SourceLocation LParenLoc,
-                                         SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPSharedClause(ArrayRef<Expr *> VarList,
+                                               SourceLocation StartLoc,
+                                               SourceLocation LParenLoc,
+                                               SourceLocation EndLoc) {
   SmallVector<Expr *, 8> Vars;
   for (Expr *RefExpr : VarList) {
     assert(RefExpr && "NULL expr in OpenMP lastprivate clause.");
     SourceLocation ELoc;
     SourceRange ERange;
     Expr *SimpleRefExpr = RefExpr;
-    auto Res = getPrivateItem(*this, SimpleRefExpr, ELoc, ERange);
+    auto Res = getPrivateItem(SemaRef, SimpleRefExpr, ELoc, ERange);
     if (Res.second) {
       // It will be analyzed later.
       Vars.push_back(RefExpr);
@@ -19063,15 +19135,16 @@ OMPClause *Sema::ActOnOpenMPSharedClause(ArrayRef<Expr *> VarList,
         DVar.RefExpr) {
       Diag(ELoc, diag::err_omp_wrong_dsa) << getOpenMPClauseName(DVar.CKind)
                                           << getOpenMPClauseName(OMPC_shared);
-      reportOriginalDsa(*this, DSAStack, D, DVar);
+      reportOriginalDsa(SemaRef, DSAStack, D, DVar);
       continue;
     }
 
     DeclRefExpr *Ref = nullptr;
-    if (!VD && isOpenMPCapturedDecl(D) && !CurContext->isDependentContext())
-      Ref = buildCapture(*this, D, SimpleRefExpr, /*WithInit=*/true);
+    if (!VD && isOpenMPCapturedDecl(D) &&
+        !SemaRef.CurContext->isDependentContext())
+      Ref = buildCapture(SemaRef, D, SimpleRefExpr, /*WithInit=*/true);
     DSAStack->addDSA(D, RefExpr->IgnoreParens(), OMPC_shared, Ref);
-    Vars.push_back((VD || !Ref || CurContext->isDependentContext())
+    Vars.push_back((VD || !Ref || SemaRef.CurContext->isDependentContext())
                        ? RefExpr->IgnoreParens()
                        : Ref);
   }
@@ -19079,7 +19152,8 @@ OMPClause *Sema::ActOnOpenMPSharedClause(ArrayRef<Expr *> VarList,
   if (Vars.empty())
     return nullptr;
 
-  return OMPSharedClause::Create(Context, StartLoc, LParenLoc, EndLoc, Vars);
+  return OMPSharedClause::Create(getASTContext(), StartLoc, LParenLoc, EndLoc,
+                                 Vars);
 }
 
 namespace {
@@ -20200,7 +20274,7 @@ static bool actOnOMPReductionKindClause(
       } else {
         VarsExpr = Ref = buildCapture(S, D, SimpleRefExpr, /*WithInit=*/false);
       }
-      if (!S.isOpenMPCapturedDecl(D)) {
+      if (!S.OpenMP().isOpenMPCapturedDecl(D)) {
         RD.ExprCaptures.emplace_back(Ref->getDecl());
         if (Ref->getDecl()->hasAttr<OMPCaptureNoInitAttr>()) {
           ExprResult RefRes = S.DefaultLvalueConversion(Ref);
@@ -20250,7 +20324,7 @@ static bool actOnOMPReductionKindClause(
   return RD.Vars.empty();
 }
 
-OMPClause *Sema::ActOnOpenMPReductionClause(
+OMPClause *SemaOpenMP::ActOnOpenMPReductionClause(
     ArrayRef<Expr *> VarList, OpenMPReductionClauseModifier Modifier,
     SourceLocation StartLoc, SourceLocation LParenLoc,
     SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc,
@@ -20279,77 +20353,80 @@ OMPClause *Sema::ActOnOpenMPReductionClause(
   }
 
   ReductionData RD(VarList.size(), Modifier);
-  if (actOnOMPReductionKindClause(*this, DSAStack, OMPC_reduction, VarList,
+  if (actOnOMPReductionKindClause(SemaRef, DSAStack, OMPC_reduction, VarList,
                                   StartLoc, LParenLoc, ColonLoc, EndLoc,
                                   ReductionIdScopeSpec, ReductionId,
                                   UnresolvedReductions, RD))
     return nullptr;
 
   return OMPReductionClause::Create(
-      Context, StartLoc, LParenLoc, ModifierLoc, ColonLoc, EndLoc, Modifier,
-      RD.Vars, ReductionIdScopeSpec.getWithLocInContext(Context), ReductionId,
+      getASTContext(), StartLoc, LParenLoc, ModifierLoc, ColonLoc, EndLoc,
+      Modifier, RD.Vars,
+      ReductionIdScopeSpec.getWithLocInContext(getASTContext()), ReductionId,
       RD.Privates, RD.LHSs, RD.RHSs, RD.ReductionOps, RD.InscanCopyOps,
       RD.InscanCopyArrayTemps, RD.InscanCopyArrayElems,
-      buildPreInits(Context, RD.ExprCaptures),
-      buildPostUpdate(*this, RD.ExprPostUpdates));
+      buildPreInits(getASTContext(), RD.ExprCaptures),
+      buildPostUpdate(SemaRef, RD.ExprPostUpdates));
 }
 
-OMPClause *Sema::ActOnOpenMPTaskReductionClause(
+OMPClause *SemaOpenMP::ActOnOpenMPTaskReductionClause(
     ArrayRef<Expr *> VarList, SourceLocation StartLoc, SourceLocation LParenLoc,
     SourceLocation ColonLoc, SourceLocation EndLoc,
     CXXScopeSpec &ReductionIdScopeSpec, const DeclarationNameInfo &ReductionId,
     ArrayRef<Expr *> UnresolvedReductions) {
   ReductionData RD(VarList.size());
-  if (actOnOMPReductionKindClause(*this, DSAStack, OMPC_task_reduction, VarList,
-                                  StartLoc, LParenLoc, ColonLoc, EndLoc,
-                                  ReductionIdScopeSpec, ReductionId,
+  if (actOnOMPReductionKindClause(SemaRef, DSAStack, OMPC_task_reduction,
+                                  VarList, StartLoc, LParenLoc, ColonLoc,
+                                  EndLoc, ReductionIdScopeSpec, ReductionId,
                                   UnresolvedReductions, RD))
     return nullptr;
 
   return OMPTaskReductionClause::Create(
-      Context, StartLoc, LParenLoc, ColonLoc, EndLoc, RD.Vars,
-      ReductionIdScopeSpec.getWithLocInContext(Context), ReductionId,
+      getASTContext(), StartLoc, LParenLoc, ColonLoc, EndLoc, RD.Vars,
+      ReductionIdScopeSpec.getWithLocInContext(getASTContext()), ReductionId,
       RD.Privates, RD.LHSs, RD.RHSs, RD.ReductionOps,
-      buildPreInits(Context, RD.ExprCaptures),
-      buildPostUpdate(*this, RD.ExprPostUpdates));
+      buildPreInits(getASTContext(), RD.ExprCaptures),
+      buildPostUpdate(SemaRef, RD.ExprPostUpdates));
 }
 
-OMPClause *Sema::ActOnOpenMPInReductionClause(
+OMPClause *SemaOpenMP::ActOnOpenMPInReductionClause(
     ArrayRef<Expr *> VarList, SourceLocation StartLoc, SourceLocation LParenLoc,
     SourceLocation ColonLoc, SourceLocation EndLoc,
     CXXScopeSpec &ReductionIdScopeSpec, const DeclarationNameInfo &ReductionId,
     ArrayRef<Expr *> UnresolvedReductions) {
   ReductionData RD(VarList.size());
-  if (actOnOMPReductionKindClause(*this, DSAStack, OMPC_in_reduction, VarList,
+  if (actOnOMPReductionKindClause(SemaRef, DSAStack, OMPC_in_reduction, VarList,
                                   StartLoc, LParenLoc, ColonLoc, EndLoc,
                                   ReductionIdScopeSpec, ReductionId,
                                   UnresolvedReductions, RD))
     return nullptr;
 
   return OMPInReductionClause::Create(
-      Context, StartLoc, LParenLoc, ColonLoc, EndLoc, RD.Vars,
-      ReductionIdScopeSpec.getWithLocInContext(Context), ReductionId,
+      getASTContext(), StartLoc, LParenLoc, ColonLoc, EndLoc, RD.Vars,
+      ReductionIdScopeSpec.getWithLocInContext(getASTContext()), ReductionId,
       RD.Privates, RD.LHSs, RD.RHSs, RD.ReductionOps, RD.TaskgroupDescriptors,
-      buildPreInits(Context, RD.ExprCaptures),
-      buildPostUpdate(*this, RD.ExprPostUpdates));
+      buildPreInits(getASTContext(), RD.ExprCaptures),
+      buildPostUpdate(SemaRef, RD.ExprPostUpdates));
 }
 
-bool Sema::CheckOpenMPLinearModifier(OpenMPLinearClauseKind LinKind,
-                                     SourceLocation LinLoc) {
-  if ((!LangOpts.CPlusPlus && LinKind != OMPC_LINEAR_val) ||
+bool SemaOpenMP::CheckOpenMPLinearModifier(OpenMPLinearClauseKind LinKind,
+                                           SourceLocation LinLoc) {
+  if ((!getLangOpts().CPlusPlus && LinKind != OMPC_LINEAR_val) ||
       LinKind == OMPC_LINEAR_unknown || LinKind == OMPC_LINEAR_step) {
-    Diag(LinLoc, diag::err_omp_wrong_linear_modifier) << LangOpts.CPlusPlus;
+    Diag(LinLoc, diag::err_omp_wrong_linear_modifier)
+        << getLangOpts().CPlusPlus;
     return true;
   }
   return false;
 }
 
-bool Sema::CheckOpenMPLinearDecl(const ValueDecl *D, SourceLocation ELoc,
-                                 OpenMPLinearClauseKind LinKind, QualType Type,
-                                 bool IsDeclareSimd) {
+bool SemaOpenMP::CheckOpenMPLinearDecl(const ValueDecl *D, SourceLocation ELoc,
+                                       OpenMPLinearClauseKind LinKind,
+                                       QualType Type, bool IsDeclareSimd) {
   const auto *VD = dyn_cast_or_null<VarDecl>(D);
   // A variable must not have an incomplete type or a reference type.
-  if (RequireCompleteType(ELoc, Type, diag::err_omp_linear_incomplete_type))
+  if (SemaRef.RequireCompleteType(ELoc, Type,
+                                  diag::err_omp_linear_incomplete_type))
     return true;
   if ((LinKind == OMPC_LINEAR_uval || LinKind == OMPC_LINEAR_ref) &&
       !Type->isReferenceType()) {
@@ -20365,17 +20442,17 @@ bool Sema::CheckOpenMPLinearDecl(const ValueDecl *D, SourceLocation ELoc,
   // not apply to the firstprivate clause, nor to the linear clause on
   // declarative directives (like declare simd).
   if (!IsDeclareSimd &&
-      rejectConstNotMutableType(*this, D, Type, OMPC_linear, ELoc))
+      rejectConstNotMutableType(SemaRef, D, Type, OMPC_linear, ELoc))
     return true;
 
   // A list item must be of integral or pointer type.
   Type = Type.getUnqualifiedType().getCanonicalType();
   const auto *Ty = Type.getTypePtrOrNull();
   if (!Ty || (LinKind != OMPC_LINEAR_ref && !Ty->isDependentType() &&
-              !Ty->isIntegralType(Context) && !Ty->isPointerType())) {
+              !Ty->isIntegralType(getASTContext()) && !Ty->isPointerType())) {
     Diag(ELoc, diag::err_omp_linear_expected_int_or_ptr) << Type;
     if (D) {
-      bool IsDecl = !VD || VD->isThisDeclarationADefinition(Context) ==
+      bool IsDecl = !VD || VD->isThisDeclarationADefinition(getASTContext()) ==
                                VarDecl::DeclarationOnly;
       Diag(D->getLocation(),
            IsDecl ? diag::note_previous_decl : diag::note_defined_here)
@@ -20386,7 +20463,7 @@ bool Sema::CheckOpenMPLinearDecl(const ValueDecl *D, SourceLocation ELoc,
   return false;
 }
 
-OMPClause *Sema::ActOnOpenMPLinearClause(
+OMPClause *SemaOpenMP::ActOnOpenMPLinearClause(
     ArrayRef<Expr *> VarList, Expr *Step, SourceLocation StartLoc,
     SourceLocation LParenLoc, OpenMPLinearClauseKind LinKind,
     SourceLocation LinLoc, SourceLocation ColonLoc,
@@ -20409,7 +20486,7 @@ OMPClause *Sema::ActOnOpenMPLinearClause(
     SourceLocation ELoc;
     SourceRange ERange;
     Expr *SimpleRefExpr = RefExpr;
-    auto Res = getPrivateItem(*this, SimpleRefExpr, ELoc, ERange);
+    auto Res = getPrivateItem(SemaRef, SimpleRefExpr, ELoc, ERange);
     if (Res.second) {
       // It will be analyzed later.
       Vars.push_back(RefExpr);
@@ -20431,7 +20508,7 @@ OMPClause *Sema::ActOnOpenMPLinearClause(
     if (DVar.RefExpr) {
       Diag(ELoc, diag::err_omp_wrong_dsa) << getOpenMPClauseName(DVar.CKind)
                                           << getOpenMPClauseName(OMPC_linear);
-      reportOriginalDsa(*this, DSAStack, D, DVar);
+      reportOriginalDsa(SemaRef, DSAStack, D, DVar);
       continue;
     }
 
@@ -20441,29 +20518,29 @@ OMPClause *Sema::ActOnOpenMPLinearClause(
 
     // Build private copy of original var.
     VarDecl *Private =
-        buildVarDecl(*this, ELoc, Type, D->getName(),
+        buildVarDecl(SemaRef, ELoc, Type, D->getName(),
                      D->hasAttrs() ? &D->getAttrs() : nullptr,
                      VD ? cast<DeclRefExpr>(SimpleRefExpr) : nullptr);
-    DeclRefExpr *PrivateRef = buildDeclRefExpr(*this, Private, Type, ELoc);
+    DeclRefExpr *PrivateRef = buildDeclRefExpr(SemaRef, Private, Type, ELoc);
     // Build var to save initial value.
-    VarDecl *Init = buildVarDecl(*this, ELoc, Type, ".linear.start");
+    VarDecl *Init = buildVarDecl(SemaRef, ELoc, Type, ".linear.start");
     Expr *InitExpr;
     DeclRefExpr *Ref = nullptr;
-    if (!VD && !CurContext->isDependentContext()) {
-      Ref = buildCapture(*this, D, SimpleRefExpr, /*WithInit=*/false);
+    if (!VD && !SemaRef.CurContext->isDependentContext()) {
+      Ref = buildCapture(SemaRef, D, SimpleRefExpr, /*WithInit=*/false);
       if (!isOpenMPCapturedDecl(D)) {
         ExprCaptures.push_back(Ref->getDecl());
         if (Ref->getDecl()->hasAttr<OMPCaptureNoInitAttr>()) {
-          ExprResult RefRes = DefaultLvalueConversion(Ref);
+          ExprResult RefRes = SemaRef.DefaultLvalueConversion(Ref);
           if (!RefRes.isUsable())
             continue;
           ExprResult PostUpdateRes =
-              BuildBinOp(DSAStack->getCurScope(), ELoc, BO_Assign,
-                         SimpleRefExpr, RefRes.get());
+              SemaRef.BuildBinOp(DSAStack->getCurScope(), ELoc, BO_Assign,
+                                 SimpleRefExpr, RefRes.get());
           if (!PostUpdateRes.isUsable())
             continue;
           ExprPostUpdates.push_back(
-              IgnoredValueConversions(PostUpdateRes.get()).get());
+              SemaRef.IgnoredValueConversions(PostUpdateRes.get()).get());
         }
       }
     }
@@ -20471,12 +20548,13 @@ OMPClause *Sema::ActOnOpenMPLinearClause(
       InitExpr = VD ? VD->getInit() : SimpleRefExpr;
     else
       InitExpr = VD ? SimpleRefExpr : Ref;
-    AddInitializerToDecl(Init, DefaultLvalueConversion(InitExpr).get(),
-                         /*DirectInit=*/false);
-    DeclRefExpr *InitRef = buildDeclRefExpr(*this, Init, Type, ELoc);
+    SemaRef.AddInitializerToDecl(
+        Init, SemaRef.DefaultLvalueConversion(InitExpr).get(),
+        /*DirectInit=*/false);
+    DeclRefExpr *InitRef = buildDeclRefExpr(SemaRef, Init, Type, ELoc);
 
     DSAStack->addDSA(D, RefExpr->IgnoreParens(), OMPC_linear, Ref);
-    Vars.push_back((VD || CurContext->isDependentContext())
+    Vars.push_back((VD || SemaRef.CurContext->isDependentContext())
                        ? RefExpr->IgnoreParens()
                        : Ref);
     Privates.push_back(PrivateRef);
@@ -20499,17 +20577,18 @@ OMPClause *Sema::ActOnOpenMPLinearClause(
 
     // Build var to save the step value.
     VarDecl *SaveVar =
-        buildVarDecl(*this, StepLoc, StepExpr->getType(), ".linear.step");
+        buildVarDecl(SemaRef, StepLoc, StepExpr->getType(), ".linear.step");
     ExprResult SaveRef =
-        buildDeclRefExpr(*this, SaveVar, StepExpr->getType(), StepLoc);
-    ExprResult CalcStep =
-        BuildBinOp(CurScope, StepLoc, BO_Assign, SaveRef.get(), StepExpr);
-    CalcStep = ActOnFinishFullExpr(CalcStep.get(), /*DiscardedValue*/ false);
+        buildDeclRefExpr(SemaRef, SaveVar, StepExpr->getType(), StepLoc);
+    ExprResult CalcStep = SemaRef.BuildBinOp(
+        SemaRef.getCurScope(), StepLoc, BO_Assign, SaveRef.get(), StepExpr);
+    CalcStep =
+        SemaRef.ActOnFinishFullExpr(CalcStep.get(), /*DiscardedValue*/ false);
 
     // Warn about zero linear step (it would be probably better specified as
     // making corresponding variables 'const').
     if (std::optional<llvm::APSInt> Result =
-            StepExpr->getIntegerConstantExpr(Context)) {
+            StepExpr->getIntegerConstantExpr(getASTContext())) {
       if (!Result->isNegative() && !Result->isStrictlyPositive())
         Diag(StepLoc, diag::warn_omp_linear_step_zero)
             << Vars[0] << (Vars.size() > 1);
@@ -20520,11 +20599,11 @@ OMPClause *Sema::ActOnOpenMPLinearClause(
     }
   }
 
-  return OMPLinearClause::Create(Context, StartLoc, LParenLoc, LinKind, LinLoc,
-                                 ColonLoc, StepModifierLoc, EndLoc, Vars,
-                                 Privates, Inits, StepExpr, CalcStepExpr,
-                                 buildPreInits(Context, ExprCaptures),
-                                 buildPostUpdate(*this, ExprPostUpdates));
+  return OMPLinearClause::Create(getASTContext(), StartLoc, LParenLoc, LinKind,
+                                 LinLoc, ColonLoc, StepModifierLoc, EndLoc,
+                                 Vars, Privates, Inits, StepExpr, CalcStepExpr,
+                                 buildPreInits(getASTContext(), ExprCaptures),
+                                 buildPostUpdate(SemaRef, ExprPostUpdates));
 }
 
 static bool FinishOpenMPLinearClause(OMPLinearClause &Clause, DeclRefExpr *IV,
@@ -20630,7 +20709,7 @@ static bool FinishOpenMPLinearClause(OMPLinearClause &Clause, DeclRefExpr *IV,
   return HasErrors;
 }
 
-OMPClause *Sema::ActOnOpenMPAlignedClause(
+OMPClause *SemaOpenMP::ActOnOpenMPAlignedClause(
     ArrayRef<Expr *> VarList, Expr *Alignment, SourceLocation StartLoc,
     SourceLocation LParenLoc, SourceLocation ColonLoc, SourceLocation EndLoc) {
   SmallVector<Expr *, 8> Vars;
@@ -20639,7 +20718,7 @@ OMPClause *Sema::ActOnOpenMPAlignedClause(
     SourceLocation ELoc;
     SourceRange ERange;
     Expr *SimpleRefExpr = RefExpr;
-    auto Res = getPrivateItem(*this, SimpleRefExpr, ELoc, ERange);
+    auto Res = getPrivateItem(SemaRef, SimpleRefExpr, ELoc, ERange);
     if (Res.second) {
       // It will be analyzed later.
       Vars.push_back(RefExpr);
@@ -20659,7 +20738,7 @@ OMPClause *Sema::ActOnOpenMPAlignedClause(
     if (!Ty || (!Ty->isArrayType() && !Ty->isPointerType())) {
       Diag(ELoc, diag::err_omp_aligned_expected_array_or_ptr)
           << QType << getLangOpts().CPlusPlus << ERange;
-      bool IsDecl = !VD || VD->isThisDeclarationADefinition(Context) ==
+      bool IsDecl = !VD || VD->isThisDeclarationADefinition(getASTContext()) ==
                                VarDecl::DeclarationOnly;
       Diag(D->getLocation(),
            IsDecl ? diag::note_previous_decl : diag::note_defined_here)
@@ -20679,9 +20758,10 @@ OMPClause *Sema::ActOnOpenMPAlignedClause(
 
     DeclRefExpr *Ref = nullptr;
     if (!VD && isOpenMPCapturedDecl(D))
-      Ref = buildCapture(*this, D, SimpleRefExpr, /*WithInit=*/true);
-    Vars.push_back(DefaultFunctionArrayConversion(
-                       (VD || !Ref) ? RefExpr->IgnoreParens() : Ref)
+      Ref = buildCapture(SemaRef, D, SimpleRefExpr, /*WithInit=*/true);
+    Vars.push_back(SemaRef
+                       .DefaultFunctionArrayConversion(
+                           (VD || !Ref) ? RefExpr->IgnoreParens() : Ref)
                        .get());
   }
 
@@ -20700,14 +20780,14 @@ OMPClause *Sema::ActOnOpenMPAlignedClause(
   if (Vars.empty())
     return nullptr;
 
-  return OMPAlignedClause::Create(Context, StartLoc, LParenLoc, ColonLoc,
-                                  EndLoc, Vars, Alignment);
+  return OMPAlignedClause::Create(getASTContext(), StartLoc, LParenLoc,
+                                  ColonLoc, EndLoc, Vars, Alignment);
 }
 
-OMPClause *Sema::ActOnOpenMPCopyinClause(ArrayRef<Expr *> VarList,
-                                         SourceLocation StartLoc,
-                                         SourceLocation LParenLoc,
-                                         SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPCopyinClause(ArrayRef<Expr *> VarList,
+                                               SourceLocation StartLoc,
+                                               SourceLocation LParenLoc,
+                                               SourceLocation EndLoc) {
   SmallVector<Expr *, 8> Vars;
   SmallVector<Expr *, 8> SrcExprs;
   SmallVector<Expr *, 8> DstExprs;
@@ -20761,26 +20841,28 @@ OMPClause *Sema::ActOnOpenMPCopyinClause(ArrayRef<Expr *> VarList,
     //  A variable of class type (or array thereof) that appears in a
     //  copyin clause requires an accessible, unambiguous copy assignment
     //  operator for the class type.
-    QualType ElemType = Context.getBaseElementType(Type).getNonReferenceType();
+    QualType ElemType =
+        getASTContext().getBaseElementType(Type).getNonReferenceType();
     VarDecl *SrcVD =
-        buildVarDecl(*this, DE->getBeginLoc(), ElemType.getUnqualifiedType(),
+        buildVarDecl(SemaRef, DE->getBeginLoc(), ElemType.getUnqualifiedType(),
                      ".copyin.src", VD->hasAttrs() ? &VD->getAttrs() : nullptr);
     DeclRefExpr *PseudoSrcExpr = buildDeclRefExpr(
-        *this, SrcVD, ElemType.getUnqualifiedType(), DE->getExprLoc());
+        SemaRef, SrcVD, ElemType.getUnqualifiedType(), DE->getExprLoc());
     VarDecl *DstVD =
-        buildVarDecl(*this, DE->getBeginLoc(), ElemType, ".copyin.dst",
+        buildVarDecl(SemaRef, DE->getBeginLoc(), ElemType, ".copyin.dst",
                      VD->hasAttrs() ? &VD->getAttrs() : nullptr);
     DeclRefExpr *PseudoDstExpr =
-        buildDeclRefExpr(*this, DstVD, ElemType, DE->getExprLoc());
+        buildDeclRefExpr(SemaRef, DstVD, ElemType, DE->getExprLoc());
     // For arrays generate assignment operation for single element and replace
     // it by the original array element in CodeGen.
     ExprResult AssignmentOp =
-        BuildBinOp(/*S=*/nullptr, DE->getExprLoc(), BO_Assign, PseudoDstExpr,
-                   PseudoSrcExpr);
+        SemaRef.BuildBinOp(/*S=*/nullptr, DE->getExprLoc(), BO_Assign,
+                           PseudoDstExpr, PseudoSrcExpr);
     if (AssignmentOp.isInvalid())
       continue;
-    AssignmentOp = ActOnFinishFullExpr(AssignmentOp.get(), DE->getExprLoc(),
-                                       /*DiscardedValue*/ false);
+    AssignmentOp =
+        SemaRef.ActOnFinishFullExpr(AssignmentOp.get(), DE->getExprLoc(),
+                                    /*DiscardedValue*/ false);
     if (AssignmentOp.isInvalid())
       continue;
 
@@ -20794,14 +20876,14 @@ OMPClause *Sema::ActOnOpenMPCopyinClause(ArrayRef<Expr *> VarList,
   if (Vars.empty())
     return nullptr;
 
-  return OMPCopyinClause::Create(Context, StartLoc, LParenLoc, EndLoc, Vars,
-                                 SrcExprs, DstExprs, AssignmentOps);
+  return OMPCopyinClause::Create(getASTContext(), StartLoc, LParenLoc, EndLoc,
+                                 Vars, SrcExprs, DstExprs, AssignmentOps);
 }
 
-OMPClause *Sema::ActOnOpenMPCopyprivateClause(ArrayRef<Expr *> VarList,
-                                              SourceLocation StartLoc,
-                                              SourceLocation LParenLoc,
-                                              SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPCopyprivateClause(ArrayRef<Expr *> VarList,
+                                                    SourceLocation StartLoc,
+                                                    SourceLocation LParenLoc,
+                                                    SourceLocation EndLoc) {
   SmallVector<Expr *, 8> Vars;
   SmallVector<Expr *, 8> SrcExprs;
   SmallVector<Expr *, 8> DstExprs;
@@ -20811,7 +20893,7 @@ OMPClause *Sema::ActOnOpenMPCopyprivateClause(ArrayRef<Expr *> VarList,
     SourceLocation ELoc;
     SourceRange ERange;
     Expr *SimpleRefExpr = RefExpr;
-    auto Res = getPrivateItem(*this, SimpleRefExpr, ELoc, ERange);
+    auto Res = getPrivateItem(SemaRef, SimpleRefExpr, ELoc, ERange);
     if (Res.second) {
       // It will be analyzed later.
       Vars.push_back(RefExpr);
@@ -20837,7 +20919,7 @@ OMPClause *Sema::ActOnOpenMPCopyprivateClause(ArrayRef<Expr *> VarList,
         Diag(ELoc, diag::err_omp_wrong_dsa)
             << getOpenMPClauseName(DVar.CKind)
             << getOpenMPClauseName(OMPC_copyprivate);
-        reportOriginalDsa(*this, DSAStack, D, DVar);
+        reportOriginalDsa(SemaRef, DSAStack, D, DVar);
         continue;
       }
 
@@ -20850,7 +20932,7 @@ OMPClause *Sema::ActOnOpenMPCopyprivateClause(ArrayRef<Expr *> VarList,
           Diag(ELoc, diag::err_omp_required_access)
               << getOpenMPClauseName(OMPC_copyprivate)
               << "threadprivate or private in the enclosing context";
-          reportOriginalDsa(*this, DSAStack, D, DVar);
+          reportOriginalDsa(SemaRef, DSAStack, D, DVar);
           continue;
         }
       }
@@ -20861,7 +20943,7 @@ OMPClause *Sema::ActOnOpenMPCopyprivateClause(ArrayRef<Expr *> VarList,
       Diag(ELoc, diag::err_omp_variably_modified_type_not_supported)
           << getOpenMPClauseName(OMPC_copyprivate) << Type
           << getOpenMPDirectiveName(DSAStack->getCurrentDirective());
-      bool IsDecl = !VD || VD->isThisDeclarationADefinition(Context) ==
+      bool IsDecl = !VD || VD->isThisDeclarationADefinition(getASTContext()) ==
                                VarDecl::DeclarationOnly;
       Diag(D->getLocation(),
            IsDecl ? diag::note_previous_decl : diag::note_defined_here)
@@ -20873,22 +20955,23 @@ OMPClause *Sema::ActOnOpenMPCopyprivateClause(ArrayRef<Expr *> VarList,
     //  A variable of class type (or array thereof) that appears in a
     //  copyin clause requires an accessible, unambiguous copy assignment
     //  operator for the class type.
-    Type = Context.getBaseElementType(Type.getNonReferenceType())
+    Type = getASTContext()
+               .getBaseElementType(Type.getNonReferenceType())
                .getUnqualifiedType();
     VarDecl *SrcVD =
-        buildVarDecl(*this, RefExpr->getBeginLoc(), Type, ".copyprivate.src",
+        buildVarDecl(SemaRef, RefExpr->getBeginLoc(), Type, ".copyprivate.src",
                      D->hasAttrs() ? &D->getAttrs() : nullptr);
-    DeclRefExpr *PseudoSrcExpr = buildDeclRefExpr(*this, SrcVD, Type, ELoc);
+    DeclRefExpr *PseudoSrcExpr = buildDeclRefExpr(SemaRef, SrcVD, Type, ELoc);
     VarDecl *DstVD =
-        buildVarDecl(*this, RefExpr->getBeginLoc(), Type, ".copyprivate.dst",
+        buildVarDecl(SemaRef, RefExpr->getBeginLoc(), Type, ".copyprivate.dst",
                      D->hasAttrs() ? &D->getAttrs() : nullptr);
-    DeclRefExpr *PseudoDstExpr = buildDeclRefExpr(*this, DstVD, Type, ELoc);
-    ExprResult AssignmentOp = BuildBinOp(
+    DeclRefExpr *PseudoDstExpr = buildDeclRefExpr(SemaRef, DstVD, Type, ELoc);
+    ExprResult AssignmentOp = SemaRef.BuildBinOp(
         DSAStack->getCurScope(), ELoc, BO_Assign, PseudoDstExpr, PseudoSrcExpr);
     if (AssignmentOp.isInvalid())
       continue;
-    AssignmentOp =
-        ActOnFinishFullExpr(AssignmentOp.get(), ELoc, /*DiscardedValue*/ false);
+    AssignmentOp = SemaRef.ActOnFinishFullExpr(AssignmentOp.get(), ELoc,
+                                               /*DiscardedValue*/ false);
     if (AssignmentOp.isInvalid())
       continue;
 
@@ -20897,7 +20980,7 @@ OMPClause *Sema::ActOnOpenMPCopyprivateClause(ArrayRef<Expr *> VarList,
     assert(VD || isOpenMPCapturedDecl(D));
     Vars.push_back(
         VD ? RefExpr->IgnoreParens()
-           : buildCapture(*this, D, SimpleRefExpr, /*WithInit=*/false));
+           : buildCapture(SemaRef, D, SimpleRefExpr, /*WithInit=*/false));
     SrcExprs.push_back(PseudoSrcExpr);
     DstExprs.push_back(PseudoDstExpr);
     AssignmentOps.push_back(AssignmentOp.get());
@@ -20906,18 +20989,20 @@ OMPClause *Sema::ActOnOpenMPCopyprivateClause(ArrayRef<Expr *> VarList,
   if (Vars.empty())
     return nullptr;
 
-  return OMPCopyprivateClause::Create(Context, StartLoc, LParenLoc, EndLoc,
-                                      Vars, SrcExprs, DstExprs, AssignmentOps);
+  return OMPCopyprivateClause::Create(getASTContext(), StartLoc, LParenLoc,
+                                      EndLoc, Vars, SrcExprs, DstExprs,
+                                      AssignmentOps);
 }
 
-OMPClause *Sema::ActOnOpenMPFlushClause(ArrayRef<Expr *> VarList,
-                                        SourceLocation StartLoc,
-                                        SourceLocation LParenLoc,
-                                        SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPFlushClause(ArrayRef<Expr *> VarList,
+                                              SourceLocation StartLoc,
+                                              SourceLocation LParenLoc,
+                                              SourceLocation EndLoc) {
   if (VarList.empty())
     return nullptr;
 
-  return OMPFlushClause::Create(Context, StartLoc, LParenLoc, EndLoc, VarList);
+  return OMPFlushClause::Create(getASTContext(), StartLoc, LParenLoc, EndLoc,
+                                VarList);
 }
 
 /// Tries to find omp_depend_t. type.
@@ -20937,22 +21022,23 @@ static bool findOMPDependT(Sema &S, SourceLocation Loc, DSAStackTy *Stack,
   return true;
 }
 
-OMPClause *Sema::ActOnOpenMPDepobjClause(Expr *Depobj, SourceLocation StartLoc,
-                                         SourceLocation LParenLoc,
-                                         SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPDepobjClause(Expr *Depobj,
+                                               SourceLocation StartLoc,
+                                               SourceLocation LParenLoc,
+                                               SourceLocation EndLoc) {
   if (!Depobj)
     return nullptr;
 
-  bool OMPDependTFound = findOMPDependT(*this, StartLoc, DSAStack);
+  bool OMPDependTFound = findOMPDependT(SemaRef, StartLoc, DSAStack);
 
   // OpenMP 5.0, 2.17.10.1 depobj Construct
   // depobj is an lvalue expression of type omp_depend_t.
   if (!Depobj->isTypeDependent() && !Depobj->isValueDependent() &&
       !Depobj->isInstantiationDependent() &&
       !Depobj->containsUnexpandedParameterPack() &&
-      (OMPDependTFound &&
-       !Context.typesAreCompatible(DSAStack->getOMPDependT(), Depobj->getType(),
-                                   /*CompareUnqualified=*/true))) {
+      (OMPDependTFound && !getASTContext().typesAreCompatible(
+                              DSAStack->getOMPDependT(), Depobj->getType(),
+                              /*CompareUnqualified=*/true))) {
     Diag(Depobj->getExprLoc(), diag::err_omp_expected_omp_depend_t_lvalue)
         << 0 << Depobj->getType() << Depobj->getSourceRange();
   }
@@ -20962,7 +21048,8 @@ OMPClause *Sema::ActOnOpenMPDepobjClause(Expr *Depobj, SourceLocation StartLoc,
         << 1 << Depobj->getSourceRange();
   }
 
-  return OMPDepobjClause::Create(Context, StartLoc, LParenLoc, EndLoc, Depobj);
+  return OMPDepobjClause::Create(getASTContext(), StartLoc, LParenLoc, EndLoc,
+                                 Depobj);
 }
 
 namespace {
@@ -21062,8 +21149,9 @@ ProcessOpenMPDoacrossClauseCommon(Sema &SemaRef, bool IsSource,
         continue;
       }
       if (RHS) {
-        ExprResult RHSRes = SemaRef.VerifyPositiveIntegerConstantInClause(
-            RHS, OMPC_depend, /*StrictlyPositive=*/false);
+        ExprResult RHSRes =
+            SemaRef.OpenMP().VerifyPositiveIntegerConstantInClause(
+                RHS, OMPC_depend, /*StrictlyPositive=*/false);
         if (RHSRes.isInvalid())
           continue;
       }
@@ -21094,11 +21182,10 @@ ProcessOpenMPDoacrossClauseCommon(Sema &SemaRef, bool IsSource,
   return {Vars, OpsOffs, TotalDepCount};
 }
 
-OMPClause *
-Sema::ActOnOpenMPDependClause(const OMPDependClause::DependDataTy &Data,
-                              Expr *DepModifier, ArrayRef<Expr *> VarList,
-                              SourceLocation StartLoc, SourceLocation LParenLoc,
-                              SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPDependClause(
+    const OMPDependClause::DependDataTy &Data, Expr *DepModifier,
+    ArrayRef<Expr *> VarList, SourceLocation StartLoc, SourceLocation LParenLoc,
+    SourceLocation EndLoc) {
   OpenMPDependClauseKind DepKind = Data.DepKind;
   SourceLocation DepLoc = Data.DepLoc;
   if (DSAStack->getCurrentDirective() == OMPD_ordered &&
@@ -21116,17 +21203,18 @@ Sema::ActOnOpenMPDependClause(const OMPDependClause::DependDataTy &Data,
        DSAStack->getCurrentDirective() == OMPD_depobj) &&
       (DepKind == OMPC_DEPEND_unknown || DepKind == OMPC_DEPEND_source ||
        DepKind == OMPC_DEPEND_sink ||
-       ((LangOpts.OpenMP < 50 ||
+       ((getLangOpts().OpenMP < 50 ||
          DSAStack->getCurrentDirective() == OMPD_depobj) &&
         DepKind == OMPC_DEPEND_depobj))) {
     SmallVector<unsigned, 6> Except = {OMPC_DEPEND_source, OMPC_DEPEND_sink,
                                        OMPC_DEPEND_outallmemory,
                                        OMPC_DEPEND_inoutallmemory};
-    if (LangOpts.OpenMP < 50 || DSAStack->getCurrentDirective() == OMPD_depobj)
+    if (getLangOpts().OpenMP < 50 ||
+        DSAStack->getCurrentDirective() == OMPD_depobj)
       Except.push_back(OMPC_DEPEND_depobj);
-    if (LangOpts.OpenMP < 51)
+    if (getLangOpts().OpenMP < 51)
       Except.push_back(OMPC_DEPEND_inoutset);
-    std::string Expected = (LangOpts.OpenMP >= 50 && !DepModifier)
+    std::string Expected = (getLangOpts().OpenMP >= 50 && !DepModifier)
                                ? "depend modifier(iterator) or "
                                : "";
     Diag(DepLoc, diag::err_omp_unexpected_clause_value)
@@ -21152,7 +21240,7 @@ Sema::ActOnOpenMPDependClause(const OMPDependClause::DependDataTy &Data,
 
   if (DepKind == OMPC_DEPEND_sink || DepKind == OMPC_DEPEND_source) {
     DoacrossDataInfoTy VarOffset = ProcessOpenMPDoacrossClauseCommon(
-        *this, DepKind == OMPC_DEPEND_source, VarList, DSAStack, EndLoc);
+        SemaRef, DepKind == OMPC_DEPEND_source, VarList, DSAStack, EndLoc);
     Vars = VarOffset.Vars;
     OpsOffs = VarOffset.OpsOffs;
     TotalDepCount = VarOffset.TotalDepCount;
@@ -21168,9 +21256,9 @@ Sema::ActOnOpenMPDependClause(const OMPDependClause::DependDataTy &Data,
       SourceLocation ELoc = RefExpr->getExprLoc();
       Expr *SimpleExpr = RefExpr->IgnoreParenCasts();
       if (DepKind != OMPC_DEPEND_sink && DepKind != OMPC_DEPEND_source) {
-        bool OMPDependTFound = LangOpts.OpenMP >= 50;
+        bool OMPDependTFound = getLangOpts().OpenMP >= 50;
         if (OMPDependTFound)
-          OMPDependTFound = findOMPDependT(*this, StartLoc, DSAStack,
+          OMPDependTFound = findOMPDependT(SemaRef, StartLoc, DSAStack,
                                            DepKind == OMPC_DEPEND_depobj);
         if (DepKind == OMPC_DEPEND_depobj) {
           // OpenMP 5.0, 2.17.11 depend Clause, Restrictions, C/C++
@@ -21180,8 +21268,8 @@ Sema::ActOnOpenMPDependClause(const OMPDependClause::DependDataTy &Data,
               !RefExpr->isInstantiationDependent() &&
               !RefExpr->containsUnexpandedParameterPack() &&
               (OMPDependTFound &&
-               !Context.hasSameUnqualifiedType(DSAStack->getOMPDependT(),
-                                               RefExpr->getType()))) {
+               !getASTContext().hasSameUnqualifiedType(
+                   DSAStack->getOMPDependT(), RefExpr->getType()))) {
             Diag(ELoc, diag::err_omp_expected_omp_depend_t_lvalue)
                 << 0 << RefExpr->getType() << RefExpr->getSourceRange();
             continue;
@@ -21212,7 +21300,7 @@ Sema::ActOnOpenMPDependClause(const OMPDependClause::DependDataTy &Data,
             const Expr *Length = OASE->getLength();
             Expr::EvalResult Result;
             if (Length && !Length->isValueDependent() &&
-                Length->EvaluateAsInt(Result, Context) &&
+                Length->EvaluateAsInt(Result, getASTContext()) &&
                 Result.Val.getInt().isZero()) {
               Diag(ELoc,
                    diag::err_omp_depend_zero_length_array_section_not_allowed)
@@ -21232,8 +21320,9 @@ Sema::ActOnOpenMPDependClause(const OMPDependClause::DependDataTy &Data,
                (OMPDependTFound && DSAStack->getOMPDependT().getTypePtr() ==
                                        ExprTy.getTypePtr()))) {
             Diag(ELoc, diag::err_omp_expected_addressable_lvalue_or_array_item)
-                << (LangOpts.OpenMP >= 50 ? 1 : 0)
-                << (LangOpts.OpenMP >= 50 ? 1 : 0) << RefExpr->getSourceRange();
+                << (getLangOpts().OpenMP >= 50 ? 1 : 0)
+                << (getLangOpts().OpenMP >= 50 ? 1 : 0)
+                << RefExpr->getSourceRange();
             continue;
           }
 
@@ -21245,22 +21334,24 @@ Sema::ActOnOpenMPDependClause(const OMPDependClause::DependDataTy &Data,
                    ->isPointerType() &&
               !ASE->getBase()->getType().getNonReferenceType()->isArrayType()) {
             Diag(ELoc, diag::err_omp_expected_addressable_lvalue_or_array_item)
-                << (LangOpts.OpenMP >= 50 ? 1 : 0)
-                << (LangOpts.OpenMP >= 50 ? 1 : 0) << RefExpr->getSourceRange();
+                << (getLangOpts().OpenMP >= 50 ? 1 : 0)
+                << (getLangOpts().OpenMP >= 50 ? 1 : 0)
+                << RefExpr->getSourceRange();
             continue;
           }
 
           ExprResult Res;
           {
-            Sema::TentativeAnalysisScope Trap(*this);
-            Res = CreateBuiltinUnaryOp(ELoc, UO_AddrOf,
-                                       RefExpr->IgnoreParenImpCasts());
+            Sema::TentativeAnalysisScope Trap(SemaRef);
+            Res = SemaRef.CreateBuiltinUnaryOp(ELoc, UO_AddrOf,
+                                               RefExpr->IgnoreParenImpCasts());
           }
           if (!Res.isUsable() && !isa<OMPArraySectionExpr>(SimpleExpr) &&
               !isa<OMPArrayShapingExpr>(SimpleExpr)) {
             Diag(ELoc, diag::err_omp_expected_addressable_lvalue_or_array_item)
-                << (LangOpts.OpenMP >= 50 ? 1 : 0)
-                << (LangOpts.OpenMP >= 50 ? 1 : 0) << RefExpr->getSourceRange();
+                << (getLangOpts().OpenMP >= 50 ? 1 : 0)
+                << (getLangOpts().OpenMP >= 50 ? 1 : 0)
+                << RefExpr->getSourceRange();
             continue;
           }
         }
@@ -21275,7 +21366,7 @@ Sema::ActOnOpenMPDependClause(const OMPDependClause::DependDataTy &Data,
     return nullptr;
 
   auto *C = OMPDependClause::Create(
-      Context, StartLoc, LParenLoc, EndLoc,
+      getASTContext(), StartLoc, LParenLoc, EndLoc,
       {DepKind, DepLoc, Data.ColonLoc, Data.OmpAllMemoryLoc}, DepModifier, Vars,
       TotalDepCount.getZExtValue());
   if ((DepKind == OMPC_DEPEND_sink || DepKind == OMPC_DEPEND_source) &&
@@ -21284,12 +21375,11 @@ Sema::ActOnOpenMPDependClause(const OMPDependClause::DependDataTy &Data,
   return C;
 }
 
-OMPClause *Sema::ActOnOpenMPDeviceClause(OpenMPDeviceClauseModifier Modifier,
-                                         Expr *Device, SourceLocation StartLoc,
-                                         SourceLocation LParenLoc,
-                                         SourceLocation ModifierLoc,
-                                         SourceLocation EndLoc) {
-  assert((ModifierLoc.isInvalid() || LangOpts.OpenMP >= 50) &&
+OMPClause *SemaOpenMP::ActOnOpenMPDeviceClause(
+    OpenMPDeviceClauseModifier Modifier, Expr *Device, SourceLocation StartLoc,
+    SourceLocation LParenLoc, SourceLocation ModifierLoc,
+    SourceLocation EndLoc) {
+  assert((ModifierLoc.isInvalid() || getLangOpts().OpenMP >= 50) &&
          "Unexpected device modifier in OpenMP < 50.");
 
   bool ErrorFound = false;
@@ -21306,7 +21396,7 @@ OMPClause *Sema::ActOnOpenMPDeviceClause(OpenMPDeviceClauseModifier Modifier,
 
   // OpenMP [2.9.1, Restrictions]
   // The device expression must evaluate to a non-negative integer value.
-  ErrorFound = !isNonNegativeIntegerValue(ValExpr, *this, OMPC_device,
+  ErrorFound = !isNonNegativeIntegerValue(ValExpr, SemaRef, OMPC_device,
                                           /*StrictlyPositive=*/false) ||
                ErrorFound;
   if (ErrorFound)
@@ -21317,7 +21407,7 @@ OMPClause *Sema::ActOnOpenMPDeviceClause(OpenMPDeviceClauseModifier Modifier,
   // the reverse_offload clause must be specified.
   if (Modifier == OMPC_DEVICE_ancestor) {
     if (!DSAStack->hasRequiresDeclWithClause<OMPReverseOffloadClause>()) {
-      targetDiag(
+      SemaRef.targetDiag(
           StartLoc,
           diag::err_omp_device_ancestor_without_requires_reverse_offload);
       ErrorFound = true;
@@ -21326,15 +21416,16 @@ OMPClause *Sema::ActOnOpenMPDeviceClause(OpenMPDeviceClauseModifier Modifier,
 
   OpenMPDirectiveKind DKind = DSAStack->getCurrentDirective();
   OpenMPDirectiveKind CaptureRegion =
-      getOpenMPCaptureRegionForClause(DKind, OMPC_device, LangOpts.OpenMP);
-  if (CaptureRegion != OMPD_unknown && !CurContext->isDependentContext()) {
-    ValExpr = MakeFullExpr(ValExpr).get();
+      getOpenMPCaptureRegionForClause(DKind, OMPC_device, getLangOpts().OpenMP);
+  if (CaptureRegion != OMPD_unknown &&
+      !SemaRef.CurContext->isDependentContext()) {
+    ValExpr = SemaRef.MakeFullExpr(ValExpr).get();
     llvm::MapVector<const Expr *, DeclRefExpr *> Captures;
-    ValExpr = tryBuildCapture(*this, ValExpr, Captures).get();
-    HelperValStmt = buildPreInits(Context, Captures);
+    ValExpr = tryBuildCapture(SemaRef, ValExpr, Captures).get();
+    HelperValStmt = buildPreInits(getASTContext(), Captures);
   }
 
-  return new (Context)
+  return new (getASTContext())
       OMPDeviceClause(Modifier, ValExpr, HelperValStmt, CaptureRegion, StartLoc,
                       LParenLoc, ModifierLoc, EndLoc);
 }
@@ -22527,7 +22618,7 @@ static void checkMappableExpressionList(
   }
 }
 
-OMPClause *Sema::ActOnOpenMPMapClause(
+OMPClause *SemaOpenMP::ActOnOpenMPMapClause(
     Expr *IteratorModifier, ArrayRef<OpenMPMapModifierKind> MapTypeModifiers,
     ArrayRef<SourceLocation> MapTypeModifiersLoc,
     CXXScopeSpec &MapperIdScopeSpec, DeclarationNameInfo &MapperId,
@@ -22562,7 +22653,7 @@ OMPClause *Sema::ActOnOpenMPMapClause(
   }
 
   MappableVarListInfo MVLI(VarList);
-  checkMappableExpressionList(*this, DSAStack, OMPC_map, MVLI, Locs.StartLoc,
+  checkMappableExpressionList(SemaRef, DSAStack, OMPC_map, MVLI, Locs.StartLoc,
                               MapperIdScopeSpec, MapperId, UnresolvedMappers,
                               MapType, Modifiers, IsMapTypeImplicit,
                               NoDiagnose);
@@ -22570,17 +22661,17 @@ OMPClause *Sema::ActOnOpenMPMapClause(
   // We need to produce a map clause even if we don't have variables so that
   // other diagnostics related with non-existing map clauses are accurate.
   return OMPMapClause::Create(
-      Context, Locs, MVLI.ProcessedVarList, MVLI.VarBaseDeclarations,
+      getASTContext(), Locs, MVLI.ProcessedVarList, MVLI.VarBaseDeclarations,
       MVLI.VarComponents, MVLI.UDMapperList, IteratorModifier, Modifiers,
-      ModifiersLoc, MapperIdScopeSpec.getWithLocInContext(Context), MapperId,
-      MapType, IsMapTypeImplicit, MapLoc);
+      ModifiersLoc, MapperIdScopeSpec.getWithLocInContext(getASTContext()),
+      MapperId, MapType, IsMapTypeImplicit, MapLoc);
 }
 
-QualType Sema::ActOnOpenMPDeclareReductionType(SourceLocation TyLoc,
-                                               TypeResult ParsedType) {
+QualType SemaOpenMP::ActOnOpenMPDeclareReductionType(SourceLocation TyLoc,
+                                                     TypeResult ParsedType) {
   assert(ParsedType.isUsable());
 
-  QualType ReductionType = GetTypeFromParser(ParsedType.get());
+  QualType ReductionType = SemaRef.GetTypeFromParser(ParsedType.get());
   if (ReductionType.isNull())
     return QualType();
 
@@ -22608,15 +22699,17 @@ QualType Sema::ActOnOpenMPDeclareReductionType(SourceLocation TyLoc,
   return ReductionType;
 }
 
-Sema::DeclGroupPtrTy Sema::ActOnOpenMPDeclareReductionDirectiveStart(
+SemaOpenMP::DeclGroupPtrTy
+SemaOpenMP::ActOnOpenMPDeclareReductionDirectiveStart(
     Scope *S, DeclContext *DC, DeclarationName Name,
     ArrayRef<std::pair<QualType, SourceLocation>> ReductionTypes,
     AccessSpecifier AS, Decl *PrevDeclInScope) {
   SmallVector<Decl *, 8> Decls;
   Decls.reserve(ReductionTypes.size());
 
-  LookupResult Lookup(*this, Name, SourceLocation(), LookupOMPReductionName,
-                      forRedeclarationInCurContext());
+  LookupResult Lookup(SemaRef, Name, SourceLocation(),
+                      Sema::LookupOMPReductionName,
+                      SemaRef.forRedeclarationInCurContext());
   // [OpenMP 4.0], 2.15 declare reduction Directive, Restrictions
   // A reduction-identifier may not be re-declared in the current scope for the
   // same type or for a type that is compatible according to the base language
@@ -22627,12 +22720,12 @@ Sema::DeclGroupPtrTy Sema::ActOnOpenMPDeclareReductionDirectiveStart(
   if (S != nullptr) {
     // Find previous declaration with the same name not referenced in other
     // declarations.
-    FunctionScopeInfo *ParentFn = getEnclosingFunction();
+    FunctionScopeInfo *ParentFn = SemaRef.getEnclosingFunction();
     InCompoundScope =
         (ParentFn != nullptr) && !ParentFn->CompoundScopes.empty();
-    LookupName(Lookup, S);
-    FilterLookupForScope(Lookup, DC, S, /*ConsiderLinkage=*/false,
-                         /*AllowInlineNamespace=*/false);
+    SemaRef.LookupName(Lookup, S);
+    SemaRef.FilterLookupForScope(Lookup, DC, S, /*ConsiderLinkage=*/false,
+                                 /*AllowInlineNamespace=*/false);
     llvm::DenseMap<OMPDeclareReductionDecl *, bool> UsedAsPrevious;
     LookupResult::Filter Filter = Lookup.makeFilter();
     while (Filter.hasNext()) {
@@ -22675,8 +22768,8 @@ Sema::DeclGroupPtrTy Sema::ActOnOpenMPDeclareReductionDirectiveStart(
       Invalid = true;
     }
     PreviousRedeclTypes[TyData.first.getCanonicalType()] = TyData.second;
-    auto *DRD = OMPDeclareReductionDecl::Create(Context, DC, TyData.second,
-                                                Name, TyData.first, PrevDRD);
+    auto *DRD = OMPDeclareReductionDecl::Create(
+        getASTContext(), DC, TyData.second, Name, TyData.first, PrevDRD);
     DC->addDecl(DRD);
     DRD->setAccess(AS);
     Decls.push_back(DRD);
@@ -22687,24 +22780,24 @@ Sema::DeclGroupPtrTy Sema::ActOnOpenMPDeclareReductionDirectiveStart(
   }
 
   return DeclGroupPtrTy::make(
-      DeclGroupRef::Create(Context, Decls.begin(), Decls.size()));
+      DeclGroupRef::Create(getASTContext(), Decls.begin(), Decls.size()));
 }
 
-void Sema::ActOnOpenMPDeclareReductionCombinerStart(Scope *S, Decl *D) {
+void SemaOpenMP::ActOnOpenMPDeclareReductionCombinerStart(Scope *S, Decl *D) {
   auto *DRD = cast<OMPDeclareReductionDecl>(D);
 
   // Enter new function scope.
-  PushFunctionScope();
-  setFunctionHasBranchProtectedScope();
-  getCurFunction()->setHasOMPDeclareReductionCombiner();
+  SemaRef.PushFunctionScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
+  SemaRef.getCurFunction()->setHasOMPDeclareReductionCombiner();
 
   if (S != nullptr)
-    PushDeclContext(S, DRD);
+    SemaRef.PushDeclContext(S, DRD);
   else
-    CurContext = DRD;
+    SemaRef.CurContext = DRD;
 
-  PushExpressionEvaluationContext(
-      ExpressionEvaluationContext::PotentiallyEvaluated);
+  SemaRef.PushExpressionEvaluationContext(
+      Sema::ExpressionEvaluationContext::PotentiallyEvaluated);
 
   QualType ReductionType = DRD->getType();
   // Create 'T* omp_parm;T omp_in;'. All references to 'omp_in' will
@@ -22714,7 +22807,7 @@ void Sema::ActOnOpenMPDeclareReductionCombinerStart(Scope *S, Decl *D) {
   // pointers.
   // Create 'T omp_in;' variable.
   VarDecl *OmpInParm =
-      buildVarDecl(*this, D->getLocation(), ReductionType, "omp_in");
+      buildVarDecl(SemaRef, D->getLocation(), ReductionType, "omp_in");
   // Create 'T* omp_parm;T omp_out;'. All references to 'omp_out' will
   // be replaced by '*omp_parm' during codegen. This required because 'omp_out'
   // uses semantics of argument handles by value, but it should be passed by
@@ -22722,28 +22815,29 @@ void Sema::ActOnOpenMPDeclareReductionCombinerStart(Scope *S, Decl *D) {
   // pointers.
   // Create 'T omp_out;' variable.
   VarDecl *OmpOutParm =
-      buildVarDecl(*this, D->getLocation(), ReductionType, "omp_out");
+      buildVarDecl(SemaRef, D->getLocation(), ReductionType, "omp_out");
   if (S != nullptr) {
-    PushOnScopeChains(OmpInParm, S);
-    PushOnScopeChains(OmpOutParm, S);
+    SemaRef.PushOnScopeChains(OmpInParm, S);
+    SemaRef.PushOnScopeChains(OmpOutParm, S);
   } else {
     DRD->addDecl(OmpInParm);
     DRD->addDecl(OmpOutParm);
   }
   Expr *InE =
-      ::buildDeclRefExpr(*this, OmpInParm, ReductionType, D->getLocation());
+      ::buildDeclRefExpr(SemaRef, OmpInParm, ReductionType, D->getLocation());
   Expr *OutE =
-      ::buildDeclRefExpr(*this, OmpOutParm, ReductionType, D->getLocation());
+      ::buildDeclRefExpr(SemaRef, OmpOutParm, ReductionType, D->getLocation());
   DRD->setCombinerData(InE, OutE);
 }
 
-void Sema::ActOnOpenMPDeclareReductionCombinerEnd(Decl *D, Expr *Combiner) {
+void SemaOpenMP::ActOnOpenMPDeclareReductionCombinerEnd(Decl *D,
+                                                        Expr *Combiner) {
   auto *DRD = cast<OMPDeclareReductionDecl>(D);
-  DiscardCleanupsInEvaluationContext();
-  PopExpressionEvaluationContext();
+  SemaRef.DiscardCleanupsInEvaluationContext();
+  SemaRef.PopExpressionEvaluationContext();
 
-  PopDeclContext();
-  PopFunctionScopeInfo();
+  SemaRef.PopDeclContext();
+  SemaRef.PopFunctionScopeInfo();
 
   if (Combiner != nullptr)
     DRD->setCombiner(Combiner);
@@ -22751,20 +22845,21 @@ void Sema::ActOnOpenMPDeclareReductionCombinerEnd(Decl *D, Expr *Combiner) {
     DRD->setInvalidDecl();
 }
 
-VarDecl *Sema::ActOnOpenMPDeclareReductionInitializerStart(Scope *S, Decl *D) {
+VarDecl *SemaOpenMP::ActOnOpenMPDeclareReductionInitializerStart(Scope *S,
+                                                                 Decl *D) {
   auto *DRD = cast<OMPDeclareReductionDecl>(D);
 
   // Enter new function scope.
-  PushFunctionScope();
-  setFunctionHasBranchProtectedScope();
+  SemaRef.PushFunctionScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
 
   if (S != nullptr)
-    PushDeclContext(S, DRD);
+    SemaRef.PushDeclContext(S, DRD);
   else
-    CurContext = DRD;
+    SemaRef.CurContext = DRD;
 
-  PushExpressionEvaluationContext(
-      ExpressionEvaluationContext::PotentiallyEvaluated);
+  SemaRef.PushExpressionEvaluationContext(
+      Sema::ExpressionEvaluationContext::PotentiallyEvaluated);
 
   QualType ReductionType = DRD->getType();
   // Create 'T* omp_parm;T omp_priv;'. All references to 'omp_priv' will
@@ -22774,7 +22869,7 @@ VarDecl *Sema::ActOnOpenMPDeclareReductionInitializerStart(Scope *S, Decl *D) {
   // pointers.
   // Create 'T omp_priv;' variable.
   VarDecl *OmpPrivParm =
-      buildVarDecl(*this, D->getLocation(), ReductionType, "omp_priv");
+      buildVarDecl(SemaRef, D->getLocation(), ReductionType, "omp_priv");
   // Create 'T* omp_parm;T omp_orig;'. All references to 'omp_orig' will
   // be replaced by '*omp_parm' during codegen. This required because 'omp_orig'
   // uses semantics of argument handles by value, but it should be passed by
@@ -22782,30 +22877,30 @@ VarDecl *Sema::ActOnOpenMPDeclareReductionInitializerStart(Scope *S, Decl *D) {
   // pointers.
   // Create 'T omp_orig;' variable.
   VarDecl *OmpOrigParm =
-      buildVarDecl(*this, D->getLocation(), ReductionType, "omp_orig");
+      buildVarDecl(SemaRef, D->getLocation(), ReductionType, "omp_orig");
   if (S != nullptr) {
-    PushOnScopeChains(OmpPrivParm, S);
-    PushOnScopeChains(OmpOrigParm, S);
+    SemaRef.PushOnScopeChains(OmpPrivParm, S);
+    SemaRef.PushOnScopeChains(OmpOrigParm, S);
   } else {
     DRD->addDecl(OmpPrivParm);
     DRD->addDecl(OmpOrigParm);
   }
   Expr *OrigE =
-      ::buildDeclRefExpr(*this, OmpOrigParm, ReductionType, D->getLocation());
+      ::buildDeclRefExpr(SemaRef, OmpOrigParm, ReductionType, D->getLocation());
   Expr *PrivE =
-      ::buildDeclRefExpr(*this, OmpPrivParm, ReductionType, D->getLocation());
+      ::buildDeclRefExpr(SemaRef, OmpPrivParm, ReductionType, D->getLocation());
   DRD->setInitializerData(OrigE, PrivE);
   return OmpPrivParm;
 }
 
-void Sema::ActOnOpenMPDeclareReductionInitializerEnd(Decl *D, Expr *Initializer,
-                                                     VarDecl *OmpPrivParm) {
+void SemaOpenMP::ActOnOpenMPDeclareReductionInitializerEnd(
+    Decl *D, Expr *Initializer, VarDecl *OmpPrivParm) {
   auto *DRD = cast<OMPDeclareReductionDecl>(D);
-  DiscardCleanupsInEvaluationContext();
-  PopExpressionEvaluationContext();
+  SemaRef.DiscardCleanupsInEvaluationContext();
+  SemaRef.PopExpressionEvaluationContext();
 
-  PopDeclContext();
-  PopFunctionScopeInfo();
+  SemaRef.PopDeclContext();
+  SemaRef.PopFunctionScopeInfo();
 
   if (Initializer != nullptr) {
     DRD->setInitializer(Initializer, OMPDeclareReductionInitKind::Call);
@@ -22819,13 +22914,13 @@ void Sema::ActOnOpenMPDeclareReductionInitializerEnd(Decl *D, Expr *Initializer,
   }
 }
 
-Sema::DeclGroupPtrTy Sema::ActOnOpenMPDeclareReductionDirectiveEnd(
+SemaOpenMP::DeclGroupPtrTy SemaOpenMP::ActOnOpenMPDeclareReductionDirectiveEnd(
     Scope *S, DeclGroupPtrTy DeclReductions, bool IsValid) {
   for (Decl *D : DeclReductions.get()) {
     if (IsValid) {
       if (S)
-        PushOnScopeChains(cast<OMPDeclareReductionDecl>(D), S,
-                          /*AddToContext=*/false);
+        SemaRef.PushOnScopeChains(cast<OMPDeclareReductionDecl>(D), S,
+                                  /*AddToContext=*/false);
     } else {
       D->setInvalidDecl();
     }
@@ -22833,25 +22928,26 @@ Sema::DeclGroupPtrTy Sema::ActOnOpenMPDeclareReductionDirectiveEnd(
   return DeclReductions;
 }
 
-TypeResult Sema::ActOnOpenMPDeclareMapperVarDecl(Scope *S, Declarator &D) {
-  TypeSourceInfo *TInfo = GetTypeForDeclarator(D);
+TypeResult SemaOpenMP::ActOnOpenMPDeclareMapperVarDecl(Scope *S,
+                                                       Declarator &D) {
+  TypeSourceInfo *TInfo = SemaRef.GetTypeForDeclarator(D);
   QualType T = TInfo->getType();
   if (D.isInvalidType())
     return true;
 
   if (getLangOpts().CPlusPlus) {
     // Check that there are no default arguments (C++ only).
-    CheckExtraCXXDefaultArguments(D);
+    SemaRef.CheckExtraCXXDefaultArguments(D);
   }
 
-  return CreateParsedType(T, TInfo);
+  return SemaRef.CreateParsedType(T, TInfo);
 }
 
-QualType Sema::ActOnOpenMPDeclareMapperType(SourceLocation TyLoc,
-                                            TypeResult ParsedType) {
+QualType SemaOpenMP::ActOnOpenMPDeclareMapperType(SourceLocation TyLoc,
+                                                  TypeResult ParsedType) {
   assert(ParsedType.isUsable() && "Expect usable parsed mapper type");
 
-  QualType MapperType = GetTypeFromParser(ParsedType.get());
+  QualType MapperType = SemaRef.GetTypeFromParser(ParsedType.get());
   assert(!MapperType.isNull() && "Expect valid mapper type");
 
   // [OpenMP 5.0], 2.19.7.3 declare mapper Directive, Restrictions
@@ -22863,12 +22959,13 @@ QualType Sema::ActOnOpenMPDeclareMapperType(SourceLocation TyLoc,
   return MapperType;
 }
 
-Sema::DeclGroupPtrTy Sema::ActOnOpenMPDeclareMapperDirective(
+SemaOpenMP::DeclGroupPtrTy SemaOpenMP::ActOnOpenMPDeclareMapperDirective(
     Scope *S, DeclContext *DC, DeclarationName Name, QualType MapperType,
     SourceLocation StartLoc, DeclarationName VN, AccessSpecifier AS,
     Expr *MapperVarRef, ArrayRef<OMPClause *> Clauses, Decl *PrevDeclInScope) {
-  LookupResult Lookup(*this, Name, SourceLocation(), LookupOMPMapperName,
-                      forRedeclarationInCurContext());
+  LookupResult Lookup(SemaRef, Name, SourceLocation(),
+                      Sema::LookupOMPMapperName,
+                      SemaRef.forRedeclarationInCurContext());
   // [OpenMP 5.0], 2.19.7.3 declare mapper Directive, Restrictions
   //  A mapper-identifier may not be redeclared in the current scope for the
   //  same type or for a type that is compatible according to the base language
@@ -22879,12 +22976,12 @@ Sema::DeclGroupPtrTy Sema::ActOnOpenMPDeclareMapperDirective(
   if (S != nullptr) {
     // Find previous declaration with the same name not referenced in other
     // declarations.
-    FunctionScopeInfo *ParentFn = getEnclosingFunction();
+    FunctionScopeInfo *ParentFn = SemaRef.getEnclosingFunction();
     InCompoundScope =
         (ParentFn != nullptr) && !ParentFn->CompoundScopes.empty();
-    LookupName(Lookup, S);
-    FilterLookupForScope(Lookup, DC, S, /*ConsiderLinkage=*/false,
-                         /*AllowInlineNamespace=*/false);
+    SemaRef.LookupName(Lookup, S);
+    SemaRef.FilterLookupForScope(Lookup, DC, S, /*ConsiderLinkage=*/false,
+                                 /*AllowInlineNamespace=*/false);
     llvm::DenseMap<OMPDeclareMapperDecl *, bool> UsedAsPrevious;
     LookupResult::Filter Filter = Lookup.makeFilter();
     while (Filter.hasNext()) {
@@ -22929,13 +23026,14 @@ Sema::DeclGroupPtrTy Sema::ActOnOpenMPDeclareMapperDirective(
   // mappers.
   SmallVector<OMPClause *, 4> ClausesWithImplicit(Clauses.begin(),
                                                   Clauses.end());
-  if (LangOpts.OpenMP >= 50)
-    processImplicitMapsWithDefaultMappers(*this, DSAStack, ClausesWithImplicit);
-  auto *DMD =
-      OMPDeclareMapperDecl::Create(Context, DC, StartLoc, Name, MapperType, VN,
-                                   ClausesWithImplicit, PrevDMD);
+  if (getLangOpts().OpenMP >= 50)
+    processImplicitMapsWithDefaultMappers(SemaRef, DSAStack,
+                                          ClausesWithImplicit);
+  auto *DMD = OMPDeclareMapperDecl::Create(getASTContext(), DC, StartLoc, Name,
+                                           MapperType, VN, ClausesWithImplicit,
+                                           PrevDMD);
   if (S)
-    PushOnScopeChains(DMD, S);
+    SemaRef.PushOnScopeChains(DMD, S);
   else
     DC->addDecl(DMD);
   DMD->setAccess(AS);
@@ -22951,105 +23049,106 @@ Sema::DeclGroupPtrTy Sema::ActOnOpenMPDeclareMapperDirective(
   return DeclGroupPtrTy::make(DeclGroupRef(DMD));
 }
 
-ExprResult
-Sema::ActOnOpenMPDeclareMapperDirectiveVarDecl(Scope *S, QualType MapperType,
-                                               SourceLocation StartLoc,
-                                               DeclarationName VN) {
+ExprResult SemaOpenMP::ActOnOpenMPDeclareMapperDirectiveVarDecl(
+    Scope *S, QualType MapperType, SourceLocation StartLoc,
+    DeclarationName VN) {
   TypeSourceInfo *TInfo =
-      Context.getTrivialTypeSourceInfo(MapperType, StartLoc);
-  auto *VD = VarDecl::Create(Context, Context.getTranslationUnitDecl(),
-                             StartLoc, StartLoc, VN.getAsIdentifierInfo(),
-                             MapperType, TInfo, SC_None);
+      getASTContext().getTrivialTypeSourceInfo(MapperType, StartLoc);
+  auto *VD = VarDecl::Create(
+      getASTContext(), getASTContext().getTranslationUnitDecl(), StartLoc,
+      StartLoc, VN.getAsIdentifierInfo(), MapperType, TInfo, SC_None);
   if (S)
-    PushOnScopeChains(VD, S, /*AddToContext=*/false);
-  Expr *E = buildDeclRefExpr(*this, VD, MapperType, StartLoc);
+    SemaRef.PushOnScopeChains(VD, S, /*AddToContext=*/false);
+  Expr *E = buildDeclRefExpr(SemaRef, VD, MapperType, StartLoc);
   DSAStack->addDeclareMapperVarRef(E);
   return E;
 }
 
-void Sema::ActOnOpenMPIteratorVarDecl(VarDecl *VD) {
+void SemaOpenMP::ActOnOpenMPIteratorVarDecl(VarDecl *VD) {
   if (DSAStack->getDeclareMapperVarRef())
     DSAStack->addIteratorVarDecl(VD);
 }
 
-bool Sema::isOpenMPDeclareMapperVarDeclAllowed(const VarDecl *VD) const {
-  assert(LangOpts.OpenMP && "Expected OpenMP mode.");
+bool SemaOpenMP::isOpenMPDeclareMapperVarDeclAllowed(const VarDecl *VD) const {
+  assert(getLangOpts().OpenMP && "Expected OpenMP mode.");
   const Expr *Ref = DSAStack->getDeclareMapperVarRef();
   if (const auto *DRE = cast_or_null<DeclRefExpr>(Ref)) {
     if (VD->getCanonicalDecl() == DRE->getDecl()->getCanonicalDecl())
       return true;
-    if (VD->isUsableInConstantExpressions(Context))
+    if (VD->isUsableInConstantExpressions(getASTContext()))
       return true;
-    if (LangOpts.OpenMP >= 52 && DSAStack->isIteratorVarDecl(VD))
+    if (getLangOpts().OpenMP >= 52 && DSAStack->isIteratorVarDecl(VD))
       return true;
     return false;
   }
   return true;
 }
 
-const ValueDecl *Sema::getOpenMPDeclareMapperVarName() const {
-  assert(LangOpts.OpenMP && "Expected OpenMP mode.");
+const ValueDecl *SemaOpenMP::getOpenMPDeclareMapperVarName() const {
+  assert(getLangOpts().OpenMP && "Expected OpenMP mode.");
   return cast<DeclRefExpr>(DSAStack->getDeclareMapperVarRef())->getDecl();
 }
 
-OMPClause *Sema::ActOnOpenMPNumTeamsClause(Expr *NumTeams,
-                                           SourceLocation StartLoc,
-                                           SourceLocation LParenLoc,
-                                           SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPNumTeamsClause(Expr *NumTeams,
+                                                 SourceLocation StartLoc,
+                                                 SourceLocation LParenLoc,
+                                                 SourceLocation EndLoc) {
   Expr *ValExpr = NumTeams;
   Stmt *HelperValStmt = nullptr;
 
   // OpenMP [teams Constrcut, Restrictions]
   // The num_teams expression must evaluate to a positive integer value.
-  if (!isNonNegativeIntegerValue(ValExpr, *this, OMPC_num_teams,
+  if (!isNonNegativeIntegerValue(ValExpr, SemaRef, OMPC_num_teams,
                                  /*StrictlyPositive=*/true))
     return nullptr;
 
   OpenMPDirectiveKind DKind = DSAStack->getCurrentDirective();
-  OpenMPDirectiveKind CaptureRegion =
-      getOpenMPCaptureRegionForClause(DKind, OMPC_num_teams, LangOpts.OpenMP);
-  if (CaptureRegion != OMPD_unknown && !CurContext->isDependentContext()) {
-    ValExpr = MakeFullExpr(ValExpr).get();
+  OpenMPDirectiveKind CaptureRegion = getOpenMPCaptureRegionForClause(
+      DKind, OMPC_num_teams, getLangOpts().OpenMP);
+  if (CaptureRegion != OMPD_unknown &&
+      !SemaRef.CurContext->isDependentContext()) {
+    ValExpr = SemaRef.MakeFullExpr(ValExpr).get();
     llvm::MapVector<const Expr *, DeclRefExpr *> Captures;
-    ValExpr = tryBuildCapture(*this, ValExpr, Captures).get();
-    HelperValStmt = buildPreInits(Context, Captures);
+    ValExpr = tryBuildCapture(SemaRef, ValExpr, Captures).get();
+    HelperValStmt = buildPreInits(getASTContext(), Captures);
   }
 
-  return new (Context) OMPNumTeamsClause(ValExpr, HelperValStmt, CaptureRegion,
-                                         StartLoc, LParenLoc, EndLoc);
+  return new (getASTContext()) OMPNumTeamsClause(
+      ValExpr, HelperValStmt, CaptureRegion, StartLoc, LParenLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPThreadLimitClause(Expr *ThreadLimit,
-                                              SourceLocation StartLoc,
-                                              SourceLocation LParenLoc,
-                                              SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPThreadLimitClause(Expr *ThreadLimit,
+                                                    SourceLocation StartLoc,
+                                                    SourceLocation LParenLoc,
+                                                    SourceLocation EndLoc) {
   Expr *ValExpr = ThreadLimit;
   Stmt *HelperValStmt = nullptr;
 
   // OpenMP [teams Constrcut, Restrictions]
   // The thread_limit expression must evaluate to a positive integer value.
-  if (!isNonNegativeIntegerValue(ValExpr, *this, OMPC_thread_limit,
+  if (!isNonNegativeIntegerValue(ValExpr, SemaRef, OMPC_thread_limit,
                                  /*StrictlyPositive=*/true))
     return nullptr;
 
   OpenMPDirectiveKind DKind = DSAStack->getCurrentDirective();
   OpenMPDirectiveKind CaptureRegion = getOpenMPCaptureRegionForClause(
-      DKind, OMPC_thread_limit, LangOpts.OpenMP);
-  if (CaptureRegion != OMPD_unknown && !CurContext->isDependentContext()) {
-    ValExpr = MakeFullExpr(ValExpr).get();
+      DKind, OMPC_thread_limit, getLangOpts().OpenMP);
+  if (CaptureRegion != OMPD_unknown &&
+      !SemaRef.CurContext->isDependentContext()) {
+    ValExpr = SemaRef.MakeFullExpr(ValExpr).get();
     llvm::MapVector<const Expr *, DeclRefExpr *> Captures;
-    ValExpr = tryBuildCapture(*this, ValExpr, Captures).get();
-    HelperValStmt = buildPreInits(Context, Captures);
+    ValExpr = tryBuildCapture(SemaRef, ValExpr, Captures).get();
+    HelperValStmt = buildPreInits(getASTContext(), Captures);
   }
 
-  return new (Context) OMPThreadLimitClause(
+  return new (getASTContext()) OMPThreadLimitClause(
       ValExpr, HelperValStmt, CaptureRegion, StartLoc, LParenLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPPriorityClause(Expr *Priority,
-                                           SourceLocation StartLoc,
-                                           SourceLocation LParenLoc,
-                                           SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPPriorityClause(Expr *Priority,
+                                                 SourceLocation StartLoc,
+                                                 SourceLocation LParenLoc,
+                                                 SourceLocation EndLoc) {
   Expr *ValExpr = Priority;
   Stmt *HelperValStmt = nullptr;
   OpenMPDirectiveKind CaptureRegion = OMPD_unknown;
@@ -23057,20 +23156,20 @@ OMPClause *Sema::ActOnOpenMPPriorityClause(Expr *Priority,
   // OpenMP [2.9.1, task Constrcut]
   // The priority-value is a non-negative numerical scalar expression.
   if (!isNonNegativeIntegerValue(
-          ValExpr, *this, OMPC_priority,
+          ValExpr, SemaRef, OMPC_priority,
           /*StrictlyPositive=*/false, /*BuildCapture=*/true,
           DSAStack->getCurrentDirective(), &CaptureRegion, &HelperValStmt))
     return nullptr;
 
-  return new (Context) OMPPriorityClause(ValExpr, HelperValStmt, CaptureRegion,
-                                         StartLoc, LParenLoc, EndLoc);
+  return new (getASTContext()) OMPPriorityClause(
+      ValExpr, HelperValStmt, CaptureRegion, StartLoc, LParenLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPGrainsizeClause(
+OMPClause *SemaOpenMP::ActOnOpenMPGrainsizeClause(
     OpenMPGrainsizeClauseModifier Modifier, Expr *Grainsize,
     SourceLocation StartLoc, SourceLocation LParenLoc,
     SourceLocation ModifierLoc, SourceLocation EndLoc) {
-  assert((ModifierLoc.isInvalid() || LangOpts.OpenMP >= 51) &&
+  assert((ModifierLoc.isInvalid() || getLangOpts().OpenMP >= 51) &&
          "Unexpected grainsize modifier in OpenMP < 51.");
 
   if (ModifierLoc.isValid() && Modifier == OMPC_GRAINSIZE_unknown) {
@@ -23088,23 +23187,23 @@ OMPClause *Sema::ActOnOpenMPGrainsizeClause(
   // OpenMP [2.9.2, taskloop Constrcut]
   // The parameter of the grainsize clause must be a positive integer
   // expression.
-  if (!isNonNegativeIntegerValue(ValExpr, *this, OMPC_grainsize,
+  if (!isNonNegativeIntegerValue(ValExpr, SemaRef, OMPC_grainsize,
                                  /*StrictlyPositive=*/true,
                                  /*BuildCapture=*/true,
                                  DSAStack->getCurrentDirective(),
                                  &CaptureRegion, &HelperValStmt))
     return nullptr;
 
-  return new (Context)
+  return new (getASTContext())
       OMPGrainsizeClause(Modifier, ValExpr, HelperValStmt, CaptureRegion,
                          StartLoc, LParenLoc, ModifierLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPNumTasksClause(
+OMPClause *SemaOpenMP::ActOnOpenMPNumTasksClause(
     OpenMPNumTasksClauseModifier Modifier, Expr *NumTasks,
     SourceLocation StartLoc, SourceLocation LParenLoc,
     SourceLocation ModifierLoc, SourceLocation EndLoc) {
-  assert((ModifierLoc.isInvalid() || LangOpts.OpenMP >= 51) &&
+  assert((ModifierLoc.isInvalid() || getLangOpts().OpenMP >= 51) &&
          "Unexpected num_tasks modifier in OpenMP < 51.");
 
   if (ModifierLoc.isValid() && Modifier == OMPC_NUMTASKS_unknown) {
@@ -23123,19 +23222,20 @@ OMPClause *Sema::ActOnOpenMPNumTasksClause(
   // The parameter of the num_tasks clause must be a positive integer
   // expression.
   if (!isNonNegativeIntegerValue(
-          ValExpr, *this, OMPC_num_tasks,
+          ValExpr, SemaRef, OMPC_num_tasks,
           /*StrictlyPositive=*/true, /*BuildCapture=*/true,
           DSAStack->getCurrentDirective(), &CaptureRegion, &HelperValStmt))
     return nullptr;
 
-  return new (Context)
+  return new (getASTContext())
       OMPNumTasksClause(Modifier, ValExpr, HelperValStmt, CaptureRegion,
                         StartLoc, LParenLoc, ModifierLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPHintClause(Expr *Hint, SourceLocation StartLoc,
-                                       SourceLocation LParenLoc,
-                                       SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPHintClause(Expr *Hint,
+                                             SourceLocation StartLoc,
+                                             SourceLocation LParenLoc,
+                                             SourceLocation EndLoc) {
   // OpenMP [2.13.2, critical construct, Description]
   // ... where hint-expression is an integer constant expression that evaluates
   // to a valid lock hint.
@@ -23143,7 +23243,7 @@ OMPClause *Sema::ActOnOpenMPHintClause(Expr *Hint, SourceLocation StartLoc,
       VerifyPositiveIntegerConstantInClause(Hint, OMPC_hint, false);
   if (HintExpr.isInvalid())
     return nullptr;
-  return new (Context)
+  return new (getASTContext())
       OMPHintClause(HintExpr.get(), StartLoc, LParenLoc, EndLoc);
 }
 
@@ -23163,13 +23263,14 @@ static bool findOMPEventHandleT(Sema &S, SourceLocation Loc,
   return true;
 }
 
-OMPClause *Sema::ActOnOpenMPDetachClause(Expr *Evt, SourceLocation StartLoc,
-                                         SourceLocation LParenLoc,
-                                         SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPDetachClause(Expr *Evt,
+                                               SourceLocation StartLoc,
+                                               SourceLocation LParenLoc,
+                                               SourceLocation EndLoc) {
   if (!Evt->isValueDependent() && !Evt->isTypeDependent() &&
       !Evt->isInstantiationDependent() &&
       !Evt->containsUnexpandedParameterPack()) {
-    if (!findOMPEventHandleT(*this, Evt->getExprLoc(), DSAStack))
+    if (!findOMPEventHandleT(SemaRef, Evt->getExprLoc(), DSAStack))
       return nullptr;
     // OpenMP 5.0, 2.10.1 task Construct.
     // event-handle is a variable of the omp_event_handle_t type.
@@ -23185,9 +23286,9 @@ OMPClause *Sema::ActOnOpenMPDetachClause(Expr *Evt, SourceLocation StartLoc,
           << "omp_event_handle_t" << 0 << Evt->getSourceRange();
       return nullptr;
     }
-    if (!Context.hasSameUnqualifiedType(DSAStack->getOMPEventHandleT(),
-                                        VD->getType()) ||
-        VD->getType().isConstant(Context)) {
+    if (!getASTContext().hasSameUnqualifiedType(DSAStack->getOMPEventHandleT(),
+                                                VD->getType()) ||
+        VD->getType().isConstant(getASTContext())) {
       Diag(Evt->getExprLoc(), diag::err_omp_var_expected)
           << "omp_event_handle_t" << 1 << VD->getType()
           << Evt->getSourceRange();
@@ -23202,15 +23303,16 @@ OMPClause *Sema::ActOnOpenMPDetachClause(Expr *Evt, SourceLocation StartLoc,
       Diag(Evt->getExprLoc(), diag::err_omp_wrong_dsa)
           << getOpenMPClauseName(DVar.CKind)
           << getOpenMPClauseName(OMPC_firstprivate);
-      reportOriginalDsa(*this, DSAStack, VD, DVar);
+      reportOriginalDsa(SemaRef, DSAStack, VD, DVar);
       return nullptr;
     }
   }
 
-  return new (Context) OMPDetachClause(Evt, StartLoc, LParenLoc, EndLoc);
+  return new (getASTContext())
+      OMPDetachClause(Evt, StartLoc, LParenLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPDistScheduleClause(
+OMPClause *SemaOpenMP::ActOnOpenMPDistScheduleClause(
     OpenMPDistScheduleClauseKind Kind, Expr *ChunkSize, SourceLocation StartLoc,
     SourceLocation LParenLoc, SourceLocation KindLoc, SourceLocation CommaLoc,
     SourceLocation EndLoc) {
@@ -23241,7 +23343,7 @@ OMPClause *Sema::ActOnOpenMPDistScheduleClause(
       //  chunk_size must be a loop invariant integer expression with a positive
       //  value.
       if (std::optional<llvm::APSInt> Result =
-              ValExpr->getIntegerConstantExpr(Context)) {
+              ValExpr->getIntegerConstantExpr(getASTContext())) {
         if (Result->isSigned() && !Result->isStrictlyPositive()) {
           Diag(ChunkSizeLoc, diag::err_omp_negative_expression_in_clause)
               << "dist_schedule" << ChunkSize->getSourceRange();
@@ -23249,22 +23351,22 @@ OMPClause *Sema::ActOnOpenMPDistScheduleClause(
         }
       } else if (getOpenMPCaptureRegionForClause(
                      DSAStack->getCurrentDirective(), OMPC_dist_schedule,
-                     LangOpts.OpenMP) != OMPD_unknown &&
-                 !CurContext->isDependentContext()) {
-        ValExpr = MakeFullExpr(ValExpr).get();
+                     getLangOpts().OpenMP) != OMPD_unknown &&
+                 !SemaRef.CurContext->isDependentContext()) {
+        ValExpr = SemaRef.MakeFullExpr(ValExpr).get();
         llvm::MapVector<const Expr *, DeclRefExpr *> Captures;
-        ValExpr = tryBuildCapture(*this, ValExpr, Captures).get();
-        HelperValStmt = buildPreInits(Context, Captures);
+        ValExpr = tryBuildCapture(SemaRef, ValExpr, Captures).get();
+        HelperValStmt = buildPreInits(getASTContext(), Captures);
       }
     }
   }
 
-  return new (Context)
+  return new (getASTContext())
       OMPDistScheduleClause(StartLoc, LParenLoc, KindLoc, CommaLoc, EndLoc,
                             Kind, ValExpr, HelperValStmt);
 }
 
-OMPClause *Sema::ActOnOpenMPDefaultmapClause(
+OMPClause *SemaOpenMP::ActOnOpenMPDefaultmapClause(
     OpenMPDefaultmapClauseModifier M, OpenMPDefaultmapClauseKind Kind,
     SourceLocation StartLoc, SourceLocation LParenLoc, SourceLocation MLoc,
     SourceLocation KindLoc, SourceLocation EndLoc) {
@@ -23291,10 +23393,10 @@ OMPClause *Sema::ActOnOpenMPDefaultmapClause(
   } else {
     bool isDefaultmapModifier = (M != OMPC_DEFAULTMAP_MODIFIER_unknown);
     bool isDefaultmapKind = (Kind != OMPC_DEFAULTMAP_unknown) ||
-                            (LangOpts.OpenMP >= 50 && KindLoc.isInvalid());
+                            (getLangOpts().OpenMP >= 50 && KindLoc.isInvalid());
     if (!isDefaultmapKind || !isDefaultmapModifier) {
       StringRef KindValue = "'scalar', 'aggregate', 'pointer'";
-      if (LangOpts.OpenMP == 50) {
+      if (getLangOpts().OpenMP == 50) {
         StringRef ModifierValue = "'alloc', 'from', 'to', 'tofrom', "
                                   "'firstprivate', 'none', 'default'";
         if (!isDefaultmapKind && isDefaultmapModifier) {
@@ -23346,13 +23448,13 @@ OMPClause *Sema::ActOnOpenMPDefaultmapClause(
     DSAStack->setDefaultDMAAttr(M, Kind, StartLoc);
   }
 
-  return new (Context)
+  return new (getASTContext())
       OMPDefaultmapClause(StartLoc, LParenLoc, MLoc, KindLoc, EndLoc, Kind, M);
 }
 
-bool Sema::ActOnStartOpenMPDeclareTargetContext(
+bool SemaOpenMP::ActOnStartOpenMPDeclareTargetContext(
     DeclareTargetContextInfo &DTCI) {
-  DeclContext *CurLexicalContext = getCurLexicalContext();
+  DeclContext *CurLexicalContext = SemaRef.getCurLexicalContext();
   if (!CurLexicalContext->isFileContext() &&
       !CurLexicalContext->isExternCContext() &&
       !CurLexicalContext->isExternCXXContext() &&
@@ -23372,20 +23474,20 @@ bool Sema::ActOnStartOpenMPDeclareTargetContext(
   return true;
 }
 
-const Sema::DeclareTargetContextInfo
-Sema::ActOnOpenMPEndDeclareTargetDirective() {
+const SemaOpenMP::DeclareTargetContextInfo
+SemaOpenMP::ActOnOpenMPEndDeclareTargetDirective() {
   assert(!DeclareTargetNesting.empty() &&
          "check isInOpenMPDeclareTargetContext() first!");
   return DeclareTargetNesting.pop_back_val();
 }
 
-void Sema::ActOnFinishedOpenMPDeclareTargetContext(
+void SemaOpenMP::ActOnFinishedOpenMPDeclareTargetContext(
     DeclareTargetContextInfo &DTCI) {
   for (auto &It : DTCI.ExplicitlyMapped)
     ActOnOpenMPDeclareTargetName(It.first, It.second.Loc, It.second.MT, DTCI);
 }
 
-void Sema::DiagnoseUnterminatedOpenMPDeclareTarget() {
+void SemaOpenMP::DiagnoseUnterminatedOpenMPDeclareTarget() {
   if (DeclareTargetNesting.empty())
     return;
   DeclareTargetContextInfo &DTCI = DeclareTargetNesting.back();
@@ -23393,23 +23495,23 @@ void Sema::DiagnoseUnterminatedOpenMPDeclareTarget() {
       << getOpenMPDirectiveName(DTCI.Kind);
 }
 
-NamedDecl *Sema::lookupOpenMPDeclareTargetName(Scope *CurScope,
-                                               CXXScopeSpec &ScopeSpec,
-                                               const DeclarationNameInfo &Id) {
-  LookupResult Lookup(*this, Id, LookupOrdinaryName);
-  LookupParsedName(Lookup, CurScope, &ScopeSpec, true);
+NamedDecl *SemaOpenMP::lookupOpenMPDeclareTargetName(
+    Scope *CurScope, CXXScopeSpec &ScopeSpec, const DeclarationNameInfo &Id) {
+  LookupResult Lookup(SemaRef, Id, Sema::LookupOrdinaryName);
+  SemaRef.LookupParsedName(Lookup, CurScope, &ScopeSpec, true);
 
   if (Lookup.isAmbiguous())
     return nullptr;
   Lookup.suppressDiagnostics();
 
   if (!Lookup.isSingleResult()) {
-    VarOrFuncDeclFilterCCC CCC(*this);
+    VarOrFuncDeclFilterCCC CCC(SemaRef);
     if (TypoCorrection Corrected =
-            CorrectTypo(Id, LookupOrdinaryName, CurScope, nullptr, CCC,
-                        CTK_ErrorRecovery)) {
-      diagnoseTypo(Corrected, PDiag(diag::err_undeclared_var_use_suggest)
-                                  << Id.getName());
+            SemaRef.CorrectTypo(Id, Sema::LookupOrdinaryName, CurScope, nullptr,
+                                CCC, Sema::CTK_ErrorRecovery)) {
+      SemaRef.diagnoseTypo(Corrected,
+                           SemaRef.PDiag(diag::err_undeclared_var_use_suggest)
+                               << Id.getName());
       checkDeclIsAllowedInOpenMPTarget(nullptr, Corrected.getCorrectionDecl());
       return nullptr;
     }
@@ -23427,9 +23529,9 @@ NamedDecl *Sema::lookupOpenMPDeclareTargetName(Scope *CurScope,
   return ND;
 }
 
-void Sema::ActOnOpenMPDeclareTargetName(NamedDecl *ND, SourceLocation Loc,
-                                        OMPDeclareTargetDeclAttr::MapTypeTy MT,
-                                        DeclareTargetContextInfo &DTCI) {
+void SemaOpenMP::ActOnOpenMPDeclareTargetName(
+    NamedDecl *ND, SourceLocation Loc, OMPDeclareTargetDeclAttr::MapTypeTy MT,
+    DeclareTargetContextInfo &DTCI) {
   assert((isa<VarDecl>(ND) || isa<FunctionDecl>(ND) ||
           isa<FunctionTemplateDecl>(ND)) &&
          "Expected variable, function or function template.");
@@ -23445,7 +23547,7 @@ void Sema::ActOnOpenMPDeclareTargetName(NamedDecl *ND, SourceLocation Loc,
   }
   // Diagnose marking after use as it may lead to incorrect diagnosis and
   // codegen.
-  if (LangOpts.OpenMP >= 50 &&
+  if (getLangOpts().OpenMP >= 50 &&
       (ND->isUsed(/*CheckUsedAttr=*/false) || ND->isReferenced()))
     Diag(Loc, diag::warn_omp_declare_target_after_first_use);
 
@@ -23484,14 +23586,14 @@ void Sema::ActOnOpenMPDeclareTargetName(NamedDecl *ND, SourceLocation Loc,
       IsIndirect = true;
   }
   auto *A = OMPDeclareTargetDeclAttr::CreateImplicit(
-      Context, MT, DTCI.DT, IndirectE, IsIndirect, Level,
+      getASTContext(), MT, DTCI.DT, IndirectE, IsIndirect, Level,
       SourceRange(Loc, Loc));
   ND->addAttr(A);
-  if (ASTMutationListener *ML = Context.getASTMutationListener())
+  if (ASTMutationListener *ML = getASTContext().getASTMutationListener())
     ML->DeclarationMarkedOpenMPDeclareTarget(ND, A);
   checkDeclIsAllowedInOpenMPTarget(nullptr, ND, Loc);
   if (auto *VD = dyn_cast<VarDecl>(ND);
-      LangOpts.OpenMP && VD && VD->hasAttr<OMPDeclareTargetDeclAttr>() &&
+      getLangOpts().OpenMP && VD && VD->hasAttr<OMPDeclareTargetDeclAttr>() &&
       VD->hasGlobalStorage())
     ActOnOpenMPDeclareTargetInitializer(ND);
 }
@@ -23535,8 +23637,8 @@ static bool checkValueDeclInTarget(SourceLocation SL, SourceRange SR,
                            /*FullCheck=*/false);
 }
 
-void Sema::checkDeclIsAllowedInOpenMPTarget(Expr *E, Decl *D,
-                                            SourceLocation IdLoc) {
+void SemaOpenMP::checkDeclIsAllowedInOpenMPTarget(Expr *E, Decl *D,
+                                                  SourceLocation IdLoc) {
   if (!D || D->isInvalidDecl())
     return;
   SourceRange SR = E ? E->getSourceRange() : D->getSourceRange();
@@ -23550,7 +23652,7 @@ void Sema::checkDeclIsAllowedInOpenMPTarget(Expr *E, Decl *D,
     // directive.
     if (DSAStack->isThreadPrivate(VD)) {
       Diag(SL, diag::err_omp_threadprivate_in_target);
-      reportOriginalDsa(*this, DSAStack, VD, DSAStack->getTopDSA(VD, false));
+      reportOriginalDsa(SemaRef, DSAStack, VD, DSAStack->getTopDSA(VD, false));
       return;
     }
   }
@@ -23569,7 +23671,7 @@ void Sema::checkDeclIsAllowedInOpenMPTarget(Expr *E, Decl *D,
     // Problem if any with var declared with incomplete type will be reported
     // as normal, so no need to check it here.
     if ((E || !VD->getType()->isIncompleteType()) &&
-        !checkValueDeclInTarget(SL, SR, *this, DSAStack, VD))
+        !checkValueDeclInTarget(SL, SR, SemaRef, DSAStack, VD))
       return;
     if (!E && isInOpenMPDeclareTargetContext()) {
       // Checking declaration inside declare target region.
@@ -23589,13 +23691,13 @@ void Sema::checkDeclIsAllowedInOpenMPTarget(Expr *E, Decl *D,
             IsIndirect = true;
         }
         auto *A = OMPDeclareTargetDeclAttr::CreateImplicit(
-            Context,
+            getASTContext(),
             getLangOpts().OpenMP >= 52 ? OMPDeclareTargetDeclAttr::MT_Enter
                                        : OMPDeclareTargetDeclAttr::MT_To,
             DTCI.DT, IndirectE, IsIndirect, Level,
             SourceRange(DTCI.Loc, DTCI.Loc));
         D->addAttr(A);
-        if (ASTMutationListener *ML = Context.getASTMutationListener())
+        if (ASTMutationListener *ML = getASTContext().getASTMutationListener())
           ML->DeclarationMarkedOpenMPDeclareTarget(D, A);
       }
       return;
@@ -23603,7 +23705,7 @@ void Sema::checkDeclIsAllowedInOpenMPTarget(Expr *E, Decl *D,
   }
   if (!E)
     return;
-  checkDeclInTargetContext(E->getExprLoc(), E->getSourceRange(), *this, D);
+  checkDeclInTargetContext(E->getExprLoc(), E->getSourceRange(), SemaRef, D);
 }
 
 /// This class visits every VarDecl that the initializer references and adds
@@ -23649,13 +23751,13 @@ public:
 /// Adding OMPDeclareTargetDeclAttr to variables with static storage
 /// duration that are referenced in the initializer expression list of
 /// variables with static storage duration in declare target directive.
-void Sema::ActOnOpenMPDeclareTargetInitializer(Decl *TargetDecl) {
+void SemaOpenMP::ActOnOpenMPDeclareTargetInitializer(Decl *TargetDecl) {
   GlobalDeclRefChecker Checker;
   if (isa<VarDecl>(TargetDecl))
     Checker.declareTargetInitializer(TargetDecl);
 }
 
-OMPClause *Sema::ActOnOpenMPToClause(
+OMPClause *SemaOpenMP::ActOnOpenMPToClause(
     ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
     ArrayRef<SourceLocation> MotionModifiersLoc,
     CXXScopeSpec &MapperIdScopeSpec, DeclarationNameInfo &MapperId,
@@ -23681,18 +23783,18 @@ OMPClause *Sema::ActOnOpenMPToClause(
   }
 
   MappableVarListInfo MVLI(VarList);
-  checkMappableExpressionList(*this, DSAStack, OMPC_to, MVLI, Locs.StartLoc,
+  checkMappableExpressionList(SemaRef, DSAStack, OMPC_to, MVLI, Locs.StartLoc,
                               MapperIdScopeSpec, MapperId, UnresolvedMappers);
   if (MVLI.ProcessedVarList.empty())
     return nullptr;
 
   return OMPToClause::Create(
-      Context, Locs, MVLI.ProcessedVarList, MVLI.VarBaseDeclarations,
+      getASTContext(), Locs, MVLI.ProcessedVarList, MVLI.VarBaseDeclarations,
       MVLI.VarComponents, MVLI.UDMapperList, Modifiers, ModifiersLoc,
-      MapperIdScopeSpec.getWithLocInContext(Context), MapperId);
+      MapperIdScopeSpec.getWithLocInContext(getASTContext()), MapperId);
 }
 
-OMPClause *Sema::ActOnOpenMPFromClause(
+OMPClause *SemaOpenMP::ActOnOpenMPFromClause(
     ArrayRef<OpenMPMotionModifierKind> MotionModifiers,
     ArrayRef<SourceLocation> MotionModifiersLoc,
     CXXScopeSpec &MapperIdScopeSpec, DeclarationNameInfo &MapperId,
@@ -23718,19 +23820,20 @@ OMPClause *Sema::ActOnOpenMPFromClause(
   }
 
   MappableVarListInfo MVLI(VarList);
-  checkMappableExpressionList(*this, DSAStack, OMPC_from, MVLI, Locs.StartLoc,
+  checkMappableExpressionList(SemaRef, DSAStack, OMPC_from, MVLI, Locs.StartLoc,
                               MapperIdScopeSpec, MapperId, UnresolvedMappers);
   if (MVLI.ProcessedVarList.empty())
     return nullptr;
 
   return OMPFromClause::Create(
-      Context, Locs, MVLI.ProcessedVarList, MVLI.VarBaseDeclarations,
+      getASTContext(), Locs, MVLI.ProcessedVarList, MVLI.VarBaseDeclarations,
       MVLI.VarComponents, MVLI.UDMapperList, Modifiers, ModifiersLoc,
-      MapperIdScopeSpec.getWithLocInContext(Context), MapperId);
+      MapperIdScopeSpec.getWithLocInContext(getASTContext()), MapperId);
 }
 
-OMPClause *Sema::ActOnOpenMPUseDevicePtrClause(ArrayRef<Expr *> VarList,
-                                               const OMPVarListLocTy &Locs) {
+OMPClause *
+SemaOpenMP::ActOnOpenMPUseDevicePtrClause(ArrayRef<Expr *> VarList,
+                                          const OMPVarListLocTy &Locs) {
   MappableVarListInfo MVLI(VarList);
   SmallVector<Expr *, 8> PrivateCopies;
   SmallVector<Expr *, 8> Inits;
@@ -23740,7 +23843,7 @@ OMPClause *Sema::ActOnOpenMPUseDevicePtrClause(ArrayRef<Expr *> VarList,
     SourceLocation ELoc;
     SourceRange ERange;
     Expr *SimpleRefExpr = RefExpr;
-    auto Res = getPrivateItem(*this, SimpleRefExpr, ELoc, ERange);
+    auto Res = getPrivateItem(SemaRef, SimpleRefExpr, ELoc, ERange);
     if (Res.second) {
       // It will be analyzed later.
       MVLI.ProcessedVarList.push_back(RefExpr);
@@ -23765,30 +23868,30 @@ OMPClause *Sema::ActOnOpenMPUseDevicePtrClause(ArrayRef<Expr *> VarList,
 
     // Build the private variable and the expression that refers to it.
     auto VDPrivate =
-        buildVarDecl(*this, ELoc, Type, D->getName(),
+        buildVarDecl(SemaRef, ELoc, Type, D->getName(),
                      D->hasAttrs() ? &D->getAttrs() : nullptr,
                      VD ? cast<DeclRefExpr>(SimpleRefExpr) : nullptr);
     if (VDPrivate->isInvalidDecl())
       continue;
 
-    CurContext->addDecl(VDPrivate);
+    SemaRef.CurContext->addDecl(VDPrivate);
     DeclRefExpr *VDPrivateRefExpr = buildDeclRefExpr(
-        *this, VDPrivate, RefExpr->getType().getUnqualifiedType(), ELoc);
+        SemaRef, VDPrivate, RefExpr->getType().getUnqualifiedType(), ELoc);
 
     // Add temporary variable to initialize the private copy of the pointer.
     VarDecl *VDInit =
-        buildVarDecl(*this, RefExpr->getExprLoc(), Type, ".devptr.temp");
+        buildVarDecl(SemaRef, RefExpr->getExprLoc(), Type, ".devptr.temp");
     DeclRefExpr *VDInitRefExpr = buildDeclRefExpr(
-        *this, VDInit, RefExpr->getType(), RefExpr->getExprLoc());
-    AddInitializerToDecl(VDPrivate,
-                         DefaultLvalueConversion(VDInitRefExpr).get(),
-                         /*DirectInit=*/false);
+        SemaRef, VDInit, RefExpr->getType(), RefExpr->getExprLoc());
+    SemaRef.AddInitializerToDecl(
+        VDPrivate, SemaRef.DefaultLvalueConversion(VDInitRefExpr).get(),
+        /*DirectInit=*/false);
 
     // If required, build a capture to implement the privatization initialized
     // with the current list item value.
     DeclRefExpr *Ref = nullptr;
     if (!VD)
-      Ref = buildCapture(*this, D, SimpleRefExpr, /*WithInit=*/true);
+      Ref = buildCapture(SemaRef, D, SimpleRefExpr, /*WithInit=*/true);
     MVLI.ProcessedVarList.push_back(VD ? RefExpr->IgnoreParens() : Ref);
     PrivateCopies.push_back(VDPrivateRefExpr);
     Inits.push_back(VDInitRefExpr);
@@ -23810,12 +23913,13 @@ OMPClause *Sema::ActOnOpenMPUseDevicePtrClause(ArrayRef<Expr *> VarList,
     return nullptr;
 
   return OMPUseDevicePtrClause::Create(
-      Context, Locs, MVLI.ProcessedVarList, PrivateCopies, Inits,
+      getASTContext(), Locs, MVLI.ProcessedVarList, PrivateCopies, Inits,
       MVLI.VarBaseDeclarations, MVLI.VarComponents);
 }
 
-OMPClause *Sema::ActOnOpenMPUseDeviceAddrClause(ArrayRef<Expr *> VarList,
-                                                const OMPVarListLocTy &Locs) {
+OMPClause *
+SemaOpenMP::ActOnOpenMPUseDeviceAddrClause(ArrayRef<Expr *> VarList,
+                                           const OMPVarListLocTy &Locs) {
   MappableVarListInfo MVLI(VarList);
 
   for (Expr *RefExpr : VarList) {
@@ -23823,7 +23927,7 @@ OMPClause *Sema::ActOnOpenMPUseDeviceAddrClause(ArrayRef<Expr *> VarList,
     SourceLocation ELoc;
     SourceRange ERange;
     Expr *SimpleRefExpr = RefExpr;
-    auto Res = getPrivateItem(*this, SimpleRefExpr, ELoc, ERange,
+    auto Res = getPrivateItem(SemaRef, SimpleRefExpr, ELoc, ERange,
                               /*AllowArraySection=*/true);
     if (Res.second) {
       // It will be analyzed later.
@@ -23838,7 +23942,7 @@ OMPClause *Sema::ActOnOpenMPUseDeviceAddrClause(ArrayRef<Expr *> VarList,
     // with the current list item value.
     DeclRefExpr *Ref = nullptr;
     if (!VD)
-      Ref = buildCapture(*this, D, SimpleRefExpr, /*WithInit=*/true);
+      Ref = buildCapture(SemaRef, D, SimpleRefExpr, /*WithInit=*/true);
     MVLI.ProcessedVarList.push_back(VD ? RefExpr->IgnoreParens() : Ref);
 
     // We need to add a data sharing attribute for this variable to make sure it
@@ -23853,7 +23957,8 @@ OMPClause *Sema::ActOnOpenMPUseDeviceAddrClause(ArrayRef<Expr *> VarList,
     Expr *Component = SimpleRefExpr;
     if (VD && (isa<OMPArraySectionExpr>(RefExpr->IgnoreParenImpCasts()) ||
                isa<ArraySubscriptExpr>(RefExpr->IgnoreParenImpCasts())))
-      Component = DefaultFunctionArrayLvalueConversion(SimpleRefExpr).get();
+      Component =
+          SemaRef.DefaultFunctionArrayLvalueConversion(SimpleRefExpr).get();
     MVLI.VarComponents.back().emplace_back(Component, D,
                                            /*IsNonContiguous=*/false);
   }
@@ -23861,20 +23966,21 @@ OMPClause *Sema::ActOnOpenMPUseDeviceAddrClause(ArrayRef<Expr *> VarList,
   if (MVLI.ProcessedVarList.empty())
     return nullptr;
 
-  return OMPUseDeviceAddrClause::Create(Context, Locs, MVLI.ProcessedVarList,
-                                        MVLI.VarBaseDeclarations,
-                                        MVLI.VarComponents);
+  return OMPUseDeviceAddrClause::Create(
+      getASTContext(), Locs, MVLI.ProcessedVarList, MVLI.VarBaseDeclarations,
+      MVLI.VarComponents);
 }
 
-OMPClause *Sema::ActOnOpenMPIsDevicePtrClause(ArrayRef<Expr *> VarList,
-                                              const OMPVarListLocTy &Locs) {
+OMPClause *
+SemaOpenMP::ActOnOpenMPIsDevicePtrClause(ArrayRef<Expr *> VarList,
+                                         const OMPVarListLocTy &Locs) {
   MappableVarListInfo MVLI(VarList);
   for (Expr *RefExpr : VarList) {
     assert(RefExpr && "NULL expr in OpenMP is_device_ptr clause.");
     SourceLocation ELoc;
     SourceRange ERange;
     Expr *SimpleRefExpr = RefExpr;
-    auto Res = getPrivateItem(*this, SimpleRefExpr, ELoc, ERange);
+    auto Res = getPrivateItem(SemaRef, SimpleRefExpr, ELoc, ERange);
     if (Res.second) {
       // It will be analyzed later.
       MVLI.ProcessedVarList.push_back(RefExpr);
@@ -23900,7 +24006,7 @@ OMPClause *Sema::ActOnOpenMPIsDevicePtrClause(ArrayRef<Expr *> VarList,
           << getOpenMPClauseName(DVar.CKind)
           << getOpenMPClauseName(OMPC_is_device_ptr)
           << getOpenMPDirectiveName(DSAStack->getCurrentDirective());
-      reportOriginalDsa(*this, DSAStack, D, DVar);
+      reportOriginalDsa(SemaRef, DSAStack, D, DVar);
       continue;
     }
 
@@ -23944,20 +24050,21 @@ OMPClause *Sema::ActOnOpenMPIsDevicePtrClause(ArrayRef<Expr *> VarList,
   if (MVLI.ProcessedVarList.empty())
     return nullptr;
 
-  return OMPIsDevicePtrClause::Create(Context, Locs, MVLI.ProcessedVarList,
-                                      MVLI.VarBaseDeclarations,
-                                      MVLI.VarComponents);
+  return OMPIsDevicePtrClause::Create(
+      getASTContext(), Locs, MVLI.ProcessedVarList, MVLI.VarBaseDeclarations,
+      MVLI.VarComponents);
 }
 
-OMPClause *Sema::ActOnOpenMPHasDeviceAddrClause(ArrayRef<Expr *> VarList,
-                                                const OMPVarListLocTy &Locs) {
+OMPClause *
+SemaOpenMP::ActOnOpenMPHasDeviceAddrClause(ArrayRef<Expr *> VarList,
+                                           const OMPVarListLocTy &Locs) {
   MappableVarListInfo MVLI(VarList);
   for (Expr *RefExpr : VarList) {
     assert(RefExpr && "NULL expr in OpenMP has_device_addr clause.");
     SourceLocation ELoc;
     SourceRange ERange;
     Expr *SimpleRefExpr = RefExpr;
-    auto Res = getPrivateItem(*this, SimpleRefExpr, ELoc, ERange,
+    auto Res = getPrivateItem(SemaRef, SimpleRefExpr, ELoc, ERange,
                               /*AllowArraySection=*/true);
     if (Res.second) {
       // It will be analyzed later.
@@ -23975,7 +24082,7 @@ OMPClause *Sema::ActOnOpenMPHasDeviceAddrClause(ArrayRef<Expr *> VarList,
           << getOpenMPClauseName(DVar.CKind)
           << getOpenMPClauseName(OMPC_has_device_addr)
           << getOpenMPDirectiveName(DSAStack->getCurrentDirective());
-      reportOriginalDsa(*this, DSAStack, D, DVar);
+      reportOriginalDsa(SemaRef, DSAStack, D, DVar);
       continue;
     }
 
@@ -24000,16 +24107,17 @@ OMPClause *Sema::ActOnOpenMPHasDeviceAddrClause(ArrayRef<Expr *> VarList,
     auto *VD = dyn_cast<VarDecl>(D);
     if (VD && (isa<OMPArraySectionExpr>(RefExpr->IgnoreParenImpCasts()) ||
                isa<ArraySubscriptExpr>(RefExpr->IgnoreParenImpCasts())))
-      Component = DefaultFunctionArrayLvalueConversion(SimpleRefExpr).get();
+      Component =
+          SemaRef.DefaultFunctionArrayLvalueConversion(SimpleRefExpr).get();
     OMPClauseMappableExprCommon::MappableComponent MC(
         Component, D, /*IsNonContiguous=*/false);
     DSAStack->addMappableExpressionComponents(
         D, MC, /*WhereFoundClauseKind=*/OMPC_has_device_addr);
 
     // Record the expression we've just processed.
-    if (!VD && !CurContext->isDependentContext()) {
+    if (!VD && !SemaRef.CurContext->isDependentContext()) {
       DeclRefExpr *Ref =
-          buildCapture(*this, D, SimpleRefExpr, /*WithInit=*/true);
+          buildCapture(SemaRef, D, SimpleRefExpr, /*WithInit=*/true);
       assert(Ref && "has_device_addr capture failed");
       MVLI.ProcessedVarList.push_back(Ref);
     } else
@@ -24030,27 +24138,27 @@ OMPClause *Sema::ActOnOpenMPHasDeviceAddrClause(ArrayRef<Expr *> VarList,
   if (MVLI.ProcessedVarList.empty())
     return nullptr;
 
-  return OMPHasDeviceAddrClause::Create(Context, Locs, MVLI.ProcessedVarList,
-                                        MVLI.VarBaseDeclarations,
-                                        MVLI.VarComponents);
+  return OMPHasDeviceAddrClause::Create(
+      getASTContext(), Locs, MVLI.ProcessedVarList, MVLI.VarBaseDeclarations,
+      MVLI.VarComponents);
 }
 
-OMPClause *Sema::ActOnOpenMPAllocateClause(
+OMPClause *SemaOpenMP::ActOnOpenMPAllocateClause(
     Expr *Allocator, ArrayRef<Expr *> VarList, SourceLocation StartLoc,
     SourceLocation ColonLoc, SourceLocation LParenLoc, SourceLocation EndLoc) {
   if (Allocator) {
     // OpenMP [2.11.4 allocate Clause, Description]
     // allocator is an expression of omp_allocator_handle_t type.
-    if (!findOMPAllocatorHandleT(*this, Allocator->getExprLoc(), DSAStack))
+    if (!findOMPAllocatorHandleT(SemaRef, Allocator->getExprLoc(), DSAStack))
       return nullptr;
 
-    ExprResult AllocatorRes = DefaultLvalueConversion(Allocator);
+    ExprResult AllocatorRes = SemaRef.DefaultLvalueConversion(Allocator);
     if (AllocatorRes.isInvalid())
       return nullptr;
-    AllocatorRes = PerformImplicitConversion(AllocatorRes.get(),
-                                             DSAStack->getOMPAllocatorHandleT(),
-                                             Sema::AA_Initializing,
-                                             /*AllowExplicit=*/true);
+    AllocatorRes = SemaRef.PerformImplicitConversion(
+        AllocatorRes.get(), DSAStack->getOMPAllocatorHandleT(),
+        Sema::AA_Initializing,
+        /*AllowExplicit=*/true);
     if (AllocatorRes.isInvalid())
       return nullptr;
     Allocator = AllocatorRes.get();
@@ -24060,9 +24168,9 @@ OMPClause *Sema::ActOnOpenMPAllocateClause(
     // target region must specify an allocator expression unless a requires
     // directive with the dynamic_allocators clause is present in the same
     // compilation unit.
-    if (LangOpts.OpenMPIsTargetDevice &&
+    if (getLangOpts().OpenMPIsTargetDevice &&
         !DSAStack->hasRequiresDeclWithClause<OMPDynamicAllocatorsClause>())
-      targetDiag(StartLoc, diag::err_expected_allocator_expression);
+      SemaRef.targetDiag(StartLoc, diag::err_expected_allocator_expression);
   }
   // Analyze and build list of variables.
   SmallVector<Expr *, 8> Vars;
@@ -24071,7 +24179,7 @@ OMPClause *Sema::ActOnOpenMPAllocateClause(
     SourceLocation ELoc;
     SourceRange ERange;
     Expr *SimpleRefExpr = RefExpr;
-    auto Res = getPrivateItem(*this, SimpleRefExpr, ELoc, ERange);
+    auto Res = getPrivateItem(SemaRef, SimpleRefExpr, ELoc, ERange);
     if (Res.second) {
       // It will be analyzed later.
       Vars.push_back(RefExpr);
@@ -24082,9 +24190,9 @@ OMPClause *Sema::ActOnOpenMPAllocateClause(
 
     auto *VD = dyn_cast<VarDecl>(D);
     DeclRefExpr *Ref = nullptr;
-    if (!VD && !CurContext->isDependentContext())
-      Ref = buildCapture(*this, D, SimpleRefExpr, /*WithInit=*/false);
-    Vars.push_back((VD || CurContext->isDependentContext())
+    if (!VD && !SemaRef.CurContext->isDependentContext())
+      Ref = buildCapture(SemaRef, D, SimpleRefExpr, /*WithInit=*/false);
+    Vars.push_back((VD || SemaRef.CurContext->isDependentContext())
                        ? RefExpr->IgnoreParens()
                        : Ref);
   }
@@ -24094,21 +24202,21 @@ OMPClause *Sema::ActOnOpenMPAllocateClause(
 
   if (Allocator)
     DSAStack->addInnerAllocatorExpr(Allocator);
-  return OMPAllocateClause::Create(Context, StartLoc, LParenLoc, Allocator,
-                                   ColonLoc, EndLoc, Vars);
+  return OMPAllocateClause::Create(getASTContext(), StartLoc, LParenLoc,
+                                   Allocator, ColonLoc, EndLoc, Vars);
 }
 
-OMPClause *Sema::ActOnOpenMPNontemporalClause(ArrayRef<Expr *> VarList,
-                                              SourceLocation StartLoc,
-                                              SourceLocation LParenLoc,
-                                              SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPNontemporalClause(ArrayRef<Expr *> VarList,
+                                                    SourceLocation StartLoc,
+                                                    SourceLocation LParenLoc,
+                                                    SourceLocation EndLoc) {
   SmallVector<Expr *, 8> Vars;
   for (Expr *RefExpr : VarList) {
     assert(RefExpr && "NULL expr in OpenMP nontemporal clause.");
     SourceLocation ELoc;
     SourceRange ERange;
     Expr *SimpleRefExpr = RefExpr;
-    auto Res = getPrivateItem(*this, SimpleRefExpr, ELoc, ERange);
+    auto Res = getPrivateItem(SemaRef, SimpleRefExpr, ELoc, ERange);
     if (Res.second)
       // It will be analyzed later.
       Vars.push_back(RefExpr);
@@ -24133,32 +24241,34 @@ OMPClause *Sema::ActOnOpenMPNontemporalClause(ArrayRef<Expr *> VarList,
   if (Vars.empty())
     return nullptr;
 
-  return OMPNontemporalClause::Create(Context, StartLoc, LParenLoc, EndLoc,
-                                      Vars);
+  return OMPNontemporalClause::Create(getASTContext(), StartLoc, LParenLoc,
+                                      EndLoc, Vars);
 }
 
-StmtResult Sema::ActOnOpenMPScopeDirective(ArrayRef<OMPClause *> Clauses,
-                                           Stmt *AStmt, SourceLocation StartLoc,
-                                           SourceLocation EndLoc) {
+StmtResult SemaOpenMP::ActOnOpenMPScopeDirective(ArrayRef<OMPClause *> Clauses,
+                                                 Stmt *AStmt,
+                                                 SourceLocation StartLoc,
+                                                 SourceLocation EndLoc) {
   if (!AStmt)
     return StmtError();
 
-  setFunctionHasBranchProtectedScope();
+  SemaRef.setFunctionHasBranchProtectedScope();
 
-  return OMPScopeDirective::Create(Context, StartLoc, EndLoc, Clauses, AStmt);
+  return OMPScopeDirective::Create(getASTContext(), StartLoc, EndLoc, Clauses,
+                                   AStmt);
 }
 
-OMPClause *Sema::ActOnOpenMPInclusiveClause(ArrayRef<Expr *> VarList,
-                                            SourceLocation StartLoc,
-                                            SourceLocation LParenLoc,
-                                            SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPInclusiveClause(ArrayRef<Expr *> VarList,
+                                                  SourceLocation StartLoc,
+                                                  SourceLocation LParenLoc,
+                                                  SourceLocation EndLoc) {
   SmallVector<Expr *, 8> Vars;
   for (Expr *RefExpr : VarList) {
     assert(RefExpr && "NULL expr in OpenMP nontemporal clause.");
     SourceLocation ELoc;
     SourceRange ERange;
     Expr *SimpleRefExpr = RefExpr;
-    auto Res = getPrivateItem(*this, SimpleRefExpr, ELoc, ERange,
+    auto Res = getPrivateItem(SemaRef, SimpleRefExpr, ELoc, ERange,
                               /*AllowArraySection=*/true);
     if (Res.second)
       // It will be analyzed later.
@@ -24185,20 +24295,21 @@ OMPClause *Sema::ActOnOpenMPInclusiveClause(ArrayRef<Expr *> VarList,
   if (Vars.empty())
     return nullptr;
 
-  return OMPInclusiveClause::Create(Context, StartLoc, LParenLoc, EndLoc, Vars);
+  return OMPInclusiveClause::Create(getASTContext(), StartLoc, LParenLoc,
+                                    EndLoc, Vars);
 }
 
-OMPClause *Sema::ActOnOpenMPExclusiveClause(ArrayRef<Expr *> VarList,
-                                            SourceLocation StartLoc,
-                                            SourceLocation LParenLoc,
-                                            SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPExclusiveClause(ArrayRef<Expr *> VarList,
+                                                  SourceLocation StartLoc,
+                                                  SourceLocation LParenLoc,
+                                                  SourceLocation EndLoc) {
   SmallVector<Expr *, 8> Vars;
   for (Expr *RefExpr : VarList) {
     assert(RefExpr && "NULL expr in OpenMP nontemporal clause.");
     SourceLocation ELoc;
     SourceRange ERange;
     Expr *SimpleRefExpr = RefExpr;
-    auto Res = getPrivateItem(*this, SimpleRefExpr, ELoc, ERange,
+    auto Res = getPrivateItem(SemaRef, SimpleRefExpr, ELoc, ERange,
                               /*AllowArraySection=*/true);
     if (Res.second)
       // It will be analyzed later.
@@ -24228,7 +24339,8 @@ OMPClause *Sema::ActOnOpenMPExclusiveClause(ArrayRef<Expr *> VarList,
   if (Vars.empty())
     return nullptr;
 
-  return OMPExclusiveClause::Create(Context, StartLoc, LParenLoc, EndLoc, Vars);
+  return OMPExclusiveClause::Create(getASTContext(), StartLoc, LParenLoc,
+                                    EndLoc, Vars);
 }
 
 /// Tries to find omp_alloctrait_t type.
@@ -24246,19 +24358,20 @@ static bool findOMPAlloctraitT(Sema &S, SourceLocation Loc, DSAStackTy *Stack) {
   return true;
 }
 
-OMPClause *Sema::ActOnOpenMPUsesAllocatorClause(
+OMPClause *SemaOpenMP::ActOnOpenMPUsesAllocatorClause(
     SourceLocation StartLoc, SourceLocation LParenLoc, SourceLocation EndLoc,
     ArrayRef<UsesAllocatorsData> Data) {
+  ASTContext &Context = getASTContext();
   // OpenMP [2.12.5, target Construct]
   // allocator is an identifier of omp_allocator_handle_t type.
-  if (!findOMPAllocatorHandleT(*this, StartLoc, DSAStack))
+  if (!findOMPAllocatorHandleT(SemaRef, StartLoc, DSAStack))
     return nullptr;
   // OpenMP [2.12.5, target Construct]
   // allocator-traits-array is an identifier of const omp_alloctrait_t * type.
   if (llvm::any_of(
           Data,
           [](const UsesAllocatorsData &D) { return D.AllocatorTraits; }) &&
-      !findOMPAlloctraitT(*this, StartLoc, DSAStack))
+      !findOMPAlloctraitT(SemaRef, StartLoc, DSAStack))
     return nullptr;
   llvm::SmallPtrSet<CanonicalDeclPtr<Decl>, 4> PredefinedAllocators;
   for (int I = 0; I < OMPAllocateDeclAttr::OMPUserDefinedMemAlloc; ++I) {
@@ -24266,8 +24379,8 @@ OMPClause *Sema::ActOnOpenMPUsesAllocatorClause(
     StringRef Allocator =
         OMPAllocateDeclAttr::ConvertAllocatorTypeTyToStr(AllocatorKind);
     DeclarationName AllocatorName = &Context.Idents.get(Allocator);
-    PredefinedAllocators.insert(LookupSingleName(
-        TUScope, AllocatorName, StartLoc, Sema::LookupAnyName));
+    PredefinedAllocators.insert(SemaRef.LookupSingleName(
+        SemaRef.TUScope, AllocatorName, StartLoc, Sema::LookupAnyName));
   }
 
   SmallVector<OMPUsesAllocatorsClause::Data, 4> NewData;
@@ -24284,7 +24397,7 @@ OMPClause *Sema::ActOnOpenMPUsesAllocatorClause(
       bool IsPredefinedAllocator = false;
       if (DRE) {
         OMPAllocateDeclAttr::AllocatorTypeTy AllocatorTy =
-            getAllocatorKind(*this, DSAStack, AllocatorExpr);
+            getAllocatorKind(SemaRef, DSAStack, AllocatorExpr);
         IsPredefinedAllocator =
             AllocatorTy !=
             OMPAllocateDeclAttr::AllocatorTypeTy::OMPUserDefinedMemAlloc;
@@ -24329,7 +24442,7 @@ OMPClause *Sema::ActOnOpenMPUsesAllocatorClause(
       }
       // No allocator traits - just convert it to rvalue.
       if (!D.AllocatorTraits)
-        AllocatorExpr = DefaultLvalueConversion(AllocatorExpr).get();
+        AllocatorExpr = SemaRef.DefaultLvalueConversion(AllocatorExpr).get();
       DSAStack->addUsesAllocatorsDecl(
           DRE->getDecl(),
           IsPredefinedAllocator
@@ -24376,11 +24489,11 @@ OMPClause *Sema::ActOnOpenMPUsesAllocatorClause(
     NewD.LParenLoc = D.LParenLoc;
     NewD.RParenLoc = D.RParenLoc;
   }
-  return OMPUsesAllocatorsClause::Create(Context, StartLoc, LParenLoc, EndLoc,
-                                         NewData);
+  return OMPUsesAllocatorsClause::Create(getASTContext(), StartLoc, LParenLoc,
+                                         EndLoc, NewData);
 }
 
-OMPClause *Sema::ActOnOpenMPAffinityClause(
+OMPClause *SemaOpenMP::ActOnOpenMPAffinityClause(
     SourceLocation StartLoc, SourceLocation LParenLoc, SourceLocation ColonLoc,
     SourceLocation EndLoc, Expr *Modifier, ArrayRef<Expr *> Locators) {
   SmallVector<Expr *, 8> Vars;
@@ -24403,8 +24516,8 @@ OMPClause *Sema::ActOnOpenMPAffinityClause(
 
     ExprResult Res;
     {
-      Sema::TentativeAnalysisScope Trap(*this);
-      Res = CreateBuiltinUnaryOp(ELoc, UO_AddrOf, SimpleExpr);
+      Sema::TentativeAnalysisScope Trap(SemaRef);
+      Res = SemaRef.CreateBuiltinUnaryOp(ELoc, UO_AddrOf, SimpleExpr);
     }
     if (!Res.isUsable() && !isa<OMPArraySectionExpr>(SimpleExpr) &&
         !isa<OMPArrayShapingExpr>(SimpleExpr)) {
@@ -24415,15 +24528,15 @@ OMPClause *Sema::ActOnOpenMPAffinityClause(
     Vars.push_back(SimpleExpr);
   }
 
-  return OMPAffinityClause::Create(Context, StartLoc, LParenLoc, ColonLoc,
-                                   EndLoc, Modifier, Vars);
+  return OMPAffinityClause::Create(getASTContext(), StartLoc, LParenLoc,
+                                   ColonLoc, EndLoc, Modifier, Vars);
 }
 
-OMPClause *Sema::ActOnOpenMPBindClause(OpenMPBindClauseKind Kind,
-                                       SourceLocation KindLoc,
-                                       SourceLocation StartLoc,
-                                       SourceLocation LParenLoc,
-                                       SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPBindClause(OpenMPBindClauseKind Kind,
+                                             SourceLocation KindLoc,
+                                             SourceLocation StartLoc,
+                                             SourceLocation LParenLoc,
+                                             SourceLocation EndLoc) {
   if (Kind == OMPC_BIND_unknown) {
     Diag(KindLoc, diag::err_omp_unexpected_clause_value)
         << getListOfPossibleValues(OMPC_bind, /*First=*/0,
@@ -24432,39 +24545,40 @@ OMPClause *Sema::ActOnOpenMPBindClause(OpenMPBindClauseKind Kind,
     return nullptr;
   }
 
-  return OMPBindClause::Create(Context, Kind, KindLoc, StartLoc, LParenLoc,
-                               EndLoc);
+  return OMPBindClause::Create(getASTContext(), Kind, KindLoc, StartLoc,
+                               LParenLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPXDynCGroupMemClause(Expr *Size,
-                                                SourceLocation StartLoc,
-                                                SourceLocation LParenLoc,
-                                                SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPXDynCGroupMemClause(Expr *Size,
+                                                      SourceLocation StartLoc,
+                                                      SourceLocation LParenLoc,
+                                                      SourceLocation EndLoc) {
   Expr *ValExpr = Size;
   Stmt *HelperValStmt = nullptr;
 
   // OpenMP [2.5, Restrictions]
   //  The ompx_dyn_cgroup_mem expression must evaluate to a positive integer
   //  value.
-  if (!isNonNegativeIntegerValue(ValExpr, *this, OMPC_ompx_dyn_cgroup_mem,
+  if (!isNonNegativeIntegerValue(ValExpr, SemaRef, OMPC_ompx_dyn_cgroup_mem,
                                  /*StrictlyPositive=*/false))
     return nullptr;
 
   OpenMPDirectiveKind DKind = DSAStack->getCurrentDirective();
   OpenMPDirectiveKind CaptureRegion = getOpenMPCaptureRegionForClause(
-      DKind, OMPC_ompx_dyn_cgroup_mem, LangOpts.OpenMP);
-  if (CaptureRegion != OMPD_unknown && !CurContext->isDependentContext()) {
-    ValExpr = MakeFullExpr(ValExpr).get();
+      DKind, OMPC_ompx_dyn_cgroup_mem, getLangOpts().OpenMP);
+  if (CaptureRegion != OMPD_unknown &&
+      !SemaRef.CurContext->isDependentContext()) {
+    ValExpr = SemaRef.MakeFullExpr(ValExpr).get();
     llvm::MapVector<const Expr *, DeclRefExpr *> Captures;
-    ValExpr = tryBuildCapture(*this, ValExpr, Captures).get();
-    HelperValStmt = buildPreInits(Context, Captures);
+    ValExpr = tryBuildCapture(SemaRef, ValExpr, Captures).get();
+    HelperValStmt = buildPreInits(getASTContext(), Captures);
   }
 
-  return new (Context) OMPXDynCGroupMemClause(
+  return new (getASTContext()) OMPXDynCGroupMemClause(
       ValExpr, HelperValStmt, CaptureRegion, StartLoc, LParenLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPDoacrossClause(
+OMPClause *SemaOpenMP::ActOnOpenMPDoacrossClause(
     OpenMPDoacrossClauseModifier DepType, SourceLocation DepLoc,
     SourceLocation ColonLoc, ArrayRef<Expr *> VarList, SourceLocation StartLoc,
     SourceLocation LParenLoc, SourceLocation EndLoc) {
@@ -24483,7 +24597,7 @@ OMPClause *Sema::ActOnOpenMPDoacrossClause(
   DSAStackTy::OperatorOffsetTy OpsOffs;
   llvm::APSInt TotalDepCount(/*BitWidth=*/32);
   DoacrossDataInfoTy VarOffset = ProcessOpenMPDoacrossClauseCommon(
-      *this,
+      SemaRef,
       DepType == OMPC_DOACROSS_source ||
           DepType == OMPC_DOACROSS_source_omp_cur_iteration ||
           DepType == OMPC_DOACROSS_sink_omp_cur_iteration,
@@ -24491,22 +24605,587 @@ OMPClause *Sema::ActOnOpenMPDoacrossClause(
   Vars = VarOffset.Vars;
   OpsOffs = VarOffset.OpsOffs;
   TotalDepCount = VarOffset.TotalDepCount;
-  auto *C = OMPDoacrossClause::Create(Context, StartLoc, LParenLoc, EndLoc,
-                                      DepType, DepLoc, ColonLoc, Vars,
+  auto *C = OMPDoacrossClause::Create(getASTContext(), StartLoc, LParenLoc,
+                                      EndLoc, DepType, DepLoc, ColonLoc, Vars,
                                       TotalDepCount.getZExtValue());
   if (DSAStack->isParentOrderedRegion())
     DSAStack->addDoacrossDependClause(C, OpsOffs);
   return C;
 }
 
-OMPClause *Sema::ActOnOpenMPXAttributeClause(ArrayRef<const Attr *> Attrs,
-                                             SourceLocation StartLoc,
-                                             SourceLocation LParenLoc,
-                                             SourceLocation EndLoc) {
-  return new (Context) OMPXAttributeClause(Attrs, StartLoc, LParenLoc, EndLoc);
+OMPClause *SemaOpenMP::ActOnOpenMPXAttributeClause(ArrayRef<const Attr *> Attrs,
+                                                   SourceLocation StartLoc,
+                                                   SourceLocation LParenLoc,
+                                                   SourceLocation EndLoc) {
+  return new (getASTContext())
+      OMPXAttributeClause(Attrs, StartLoc, LParenLoc, EndLoc);
 }
 
-OMPClause *Sema::ActOnOpenMPXBareClause(SourceLocation StartLoc,
-                                        SourceLocation EndLoc) {
-  return new (Context) OMPXBareClause(StartLoc, EndLoc);
+OMPClause *SemaOpenMP::ActOnOpenMPXBareClause(SourceLocation StartLoc,
+                                              SourceLocation EndLoc) {
+  return new (getASTContext()) OMPXBareClause(StartLoc, EndLoc);
+}
+
+ExprResult SemaOpenMP::ActOnOMPArraySectionExpr(
+    Expr *Base, SourceLocation LBLoc, Expr *LowerBound,
+    SourceLocation ColonLocFirst, SourceLocation ColonLocSecond, Expr *Length,
+    Expr *Stride, SourceLocation RBLoc) {
+  ASTContext &Context = getASTContext();
+  if (Base->hasPlaceholderType() &&
+      !Base->hasPlaceholderType(BuiltinType::OMPArraySection)) {
+    ExprResult Result = SemaRef.CheckPlaceholderExpr(Base);
+    if (Result.isInvalid())
+      return ExprError();
+    Base = Result.get();
+  }
+  if (LowerBound && LowerBound->getType()->isNonOverloadPlaceholderType()) {
+    ExprResult Result = SemaRef.CheckPlaceholderExpr(LowerBound);
+    if (Result.isInvalid())
+      return ExprError();
+    Result = SemaRef.DefaultLvalueConversion(Result.get());
+    if (Result.isInvalid())
+      return ExprError();
+    LowerBound = Result.get();
+  }
+  if (Length && Length->getType()->isNonOverloadPlaceholderType()) {
+    ExprResult Result = SemaRef.CheckPlaceholderExpr(Length);
+    if (Result.isInvalid())
+      return ExprError();
+    Result = SemaRef.DefaultLvalueConversion(Result.get());
+    if (Result.isInvalid())
+      return ExprError();
+    Length = Result.get();
+  }
+  if (Stride && Stride->getType()->isNonOverloadPlaceholderType()) {
+    ExprResult Result = SemaRef.CheckPlaceholderExpr(Stride);
+    if (Result.isInvalid())
+      return ExprError();
+    Result = SemaRef.DefaultLvalueConversion(Result.get());
+    if (Result.isInvalid())
+      return ExprError();
+    Stride = Result.get();
+  }
+
+  // Build an unanalyzed expression if either operand is type-dependent.
+  if (Base->isTypeDependent() ||
+      (LowerBound &&
+       (LowerBound->isTypeDependent() || LowerBound->isValueDependent())) ||
+      (Length && (Length->isTypeDependent() || Length->isValueDependent())) ||
+      (Stride && (Stride->isTypeDependent() || Stride->isValueDependent()))) {
+    return new (Context) OMPArraySectionExpr(
+        Base, LowerBound, Length, Stride, Context.DependentTy, VK_LValue,
+        OK_Ordinary, ColonLocFirst, ColonLocSecond, RBLoc);
+  }
+
+  // Perform default conversions.
+  QualType OriginalTy = OMPArraySectionExpr::getBaseOriginalType(Base);
+  QualType ResultTy;
+  if (OriginalTy->isAnyPointerType()) {
+    ResultTy = OriginalTy->getPointeeType();
+  } else if (OriginalTy->isArrayType()) {
+    ResultTy = OriginalTy->getAsArrayTypeUnsafe()->getElementType();
+  } else {
+    return ExprError(
+        Diag(Base->getExprLoc(), diag::err_omp_typecheck_section_value)
+        << Base->getSourceRange());
+  }
+  // C99 6.5.2.1p1
+  if (LowerBound) {
+    auto Res = PerformOpenMPImplicitIntegerConversion(LowerBound->getExprLoc(),
+                                                      LowerBound);
+    if (Res.isInvalid())
+      return ExprError(Diag(LowerBound->getExprLoc(),
+                            diag::err_omp_typecheck_section_not_integer)
+                       << 0 << LowerBound->getSourceRange());
+    LowerBound = Res.get();
+
+    if (LowerBound->getType()->isSpecificBuiltinType(BuiltinType::Char_S) ||
+        LowerBound->getType()->isSpecificBuiltinType(BuiltinType::Char_U))
+      Diag(LowerBound->getExprLoc(), diag::warn_omp_section_is_char)
+          << 0 << LowerBound->getSourceRange();
+  }
+  if (Length) {
+    auto Res =
+        PerformOpenMPImplicitIntegerConversion(Length->getExprLoc(), Length);
+    if (Res.isInvalid())
+      return ExprError(Diag(Length->getExprLoc(),
+                            diag::err_omp_typecheck_section_not_integer)
+                       << 1 << Length->getSourceRange());
+    Length = Res.get();
+
+    if (Length->getType()->isSpecificBuiltinType(BuiltinType::Char_S) ||
+        Length->getType()->isSpecificBuiltinType(BuiltinType::Char_U))
+      Diag(Length->getExprLoc(), diag::warn_omp_section_is_char)
+          << 1 << Length->getSourceRange();
+  }
+  if (Stride) {
+    ExprResult Res =
+        PerformOpenMPImplicitIntegerConversion(Stride->getExprLoc(), Stride);
+    if (Res.isInvalid())
+      return ExprError(Diag(Stride->getExprLoc(),
+                            diag::err_omp_typecheck_section_not_integer)
+                       << 1 << Stride->getSourceRange());
+    Stride = Res.get();
+
+    if (Stride->getType()->isSpecificBuiltinType(BuiltinType::Char_S) ||
+        Stride->getType()->isSpecificBuiltinType(BuiltinType::Char_U))
+      Diag(Stride->getExprLoc(), diag::warn_omp_section_is_char)
+          << 1 << Stride->getSourceRange();
+  }
+
+  // C99 6.5.2.1p1: "shall have type "pointer to *object* type". Similarly,
+  // C++ [expr.sub]p1: The type "T" shall be a completely-defined object
+  // type. Note that functions are not objects, and that (in C99 parlance)
+  // incomplete types are not object types.
+  if (ResultTy->isFunctionType()) {
+    Diag(Base->getExprLoc(), diag::err_omp_section_function_type)
+        << ResultTy << Base->getSourceRange();
+    return ExprError();
+  }
+
+  if (SemaRef.RequireCompleteType(Base->getExprLoc(), ResultTy,
+                                  diag::err_omp_section_incomplete_type, Base))
+    return ExprError();
+
+  if (LowerBound && !OriginalTy->isAnyPointerType()) {
+    Expr::EvalResult Result;
+    if (LowerBound->EvaluateAsInt(Result, Context)) {
+      // OpenMP 5.0, [2.1.5 Array Sections]
+      // The array section must be a subset of the original array.
+      llvm::APSInt LowerBoundValue = Result.Val.getInt();
+      if (LowerBoundValue.isNegative()) {
+        Diag(LowerBound->getExprLoc(),
+             diag::err_omp_section_not_subset_of_array)
+            << LowerBound->getSourceRange();
+        return ExprError();
+      }
+    }
+  }
+
+  if (Length) {
+    Expr::EvalResult Result;
+    if (Length->EvaluateAsInt(Result, Context)) {
+      // OpenMP 5.0, [2.1.5 Array Sections]
+      // The length must evaluate to non-negative integers.
+      llvm::APSInt LengthValue = Result.Val.getInt();
+      if (LengthValue.isNegative()) {
+        Diag(Length->getExprLoc(), diag::err_omp_section_length_negative)
+            << toString(LengthValue, /*Radix=*/10, /*Signed=*/true)
+            << Length->getSourceRange();
+        return ExprError();
+      }
+    }
+  } else if (ColonLocFirst.isValid() &&
+             (OriginalTy.isNull() || (!OriginalTy->isConstantArrayType() &&
+                                      !OriginalTy->isVariableArrayType()))) {
+    // OpenMP 5.0, [2.1.5 Array Sections]
+    // When the size of the array dimension is not known, the length must be
+    // specified explicitly.
+    Diag(ColonLocFirst, diag::err_omp_section_length_undefined)
+        << (!OriginalTy.isNull() && OriginalTy->isArrayType());
+    return ExprError();
+  }
+
+  if (Stride) {
+    Expr::EvalResult Result;
+    if (Stride->EvaluateAsInt(Result, Context)) {
+      // OpenMP 5.0, [2.1.5 Array Sections]
+      // The stride must evaluate to a positive integer.
+      llvm::APSInt StrideValue = Result.Val.getInt();
+      if (!StrideValue.isStrictlyPositive()) {
+        Diag(Stride->getExprLoc(), diag::err_omp_section_stride_non_positive)
+            << toString(StrideValue, /*Radix=*/10, /*Signed=*/true)
+            << Stride->getSourceRange();
+        return ExprError();
+      }
+    }
+  }
+
+  if (!Base->hasPlaceholderType(BuiltinType::OMPArraySection)) {
+    ExprResult Result = SemaRef.DefaultFunctionArrayLvalueConversion(Base);
+    if (Result.isInvalid())
+      return ExprError();
+    Base = Result.get();
+  }
+  return new (Context) OMPArraySectionExpr(
+      Base, LowerBound, Length, Stride, Context.OMPArraySectionTy, VK_LValue,
+      OK_Ordinary, ColonLocFirst, ColonLocSecond, RBLoc);
+}
+
+ExprResult SemaOpenMP::ActOnOMPArrayShapingExpr(
+    Expr *Base, SourceLocation LParenLoc, SourceLocation RParenLoc,
+    ArrayRef<Expr *> Dims, ArrayRef<SourceRange> Brackets) {
+  ASTContext &Context = getASTContext();
+  if (Base->hasPlaceholderType()) {
+    ExprResult Result = SemaRef.CheckPlaceholderExpr(Base);
+    if (Result.isInvalid())
+      return ExprError();
+    Result = SemaRef.DefaultLvalueConversion(Result.get());
+    if (Result.isInvalid())
+      return ExprError();
+    Base = Result.get();
+  }
+  QualType BaseTy = Base->getType();
+  // Delay analysis of the types/expressions if instantiation/specialization is
+  // required.
+  if (!BaseTy->isPointerType() && Base->isTypeDependent())
+    return OMPArrayShapingExpr::Create(Context, Context.DependentTy, Base,
+                                       LParenLoc, RParenLoc, Dims, Brackets);
+  if (!BaseTy->isPointerType() ||
+      (!Base->isTypeDependent() &&
+       BaseTy->getPointeeType()->isIncompleteType()))
+    return ExprError(Diag(Base->getExprLoc(),
+                          diag::err_omp_non_pointer_type_array_shaping_base)
+                     << Base->getSourceRange());
+
+  SmallVector<Expr *, 4> NewDims;
+  bool ErrorFound = false;
+  for (Expr *Dim : Dims) {
+    if (Dim->hasPlaceholderType()) {
+      ExprResult Result = SemaRef.CheckPlaceholderExpr(Dim);
+      if (Result.isInvalid()) {
+        ErrorFound = true;
+        continue;
+      }
+      Result = SemaRef.DefaultLvalueConversion(Result.get());
+      if (Result.isInvalid()) {
+        ErrorFound = true;
+        continue;
+      }
+      Dim = Result.get();
+    }
+    if (!Dim->isTypeDependent()) {
+      ExprResult Result =
+          PerformOpenMPImplicitIntegerConversion(Dim->getExprLoc(), Dim);
+      if (Result.isInvalid()) {
+        ErrorFound = true;
+        Diag(Dim->getExprLoc(), diag::err_omp_typecheck_shaping_not_integer)
+            << Dim->getSourceRange();
+        continue;
+      }
+      Dim = Result.get();
+      Expr::EvalResult EvResult;
+      if (!Dim->isValueDependent() && Dim->EvaluateAsInt(EvResult, Context)) {
+        // OpenMP 5.0, [2.1.4 Array Shaping]
+        // Each si is an integral type expression that must evaluate to a
+        // positive integer.
+        llvm::APSInt Value = EvResult.Val.getInt();
+        if (!Value.isStrictlyPositive()) {
+          Diag(Dim->getExprLoc(), diag::err_omp_shaping_dimension_not_positive)
+              << toString(Value, /*Radix=*/10, /*Signed=*/true)
+              << Dim->getSourceRange();
+          ErrorFound = true;
+          continue;
+        }
+      }
+    }
+    NewDims.push_back(Dim);
+  }
+  if (ErrorFound)
+    return ExprError();
+  return OMPArrayShapingExpr::Create(Context, Context.OMPArrayShapingTy, Base,
+                                     LParenLoc, RParenLoc, NewDims, Brackets);
 }
+
+ExprResult SemaOpenMP::ActOnOMPIteratorExpr(Scope *S,
+                                            SourceLocation IteratorKwLoc,
+                                            SourceLocation LLoc,
+                                            SourceLocation RLoc,
+                                            ArrayRef<OMPIteratorData> Data) {
+  ASTContext &Context = getASTContext();
+  SmallVector<OMPIteratorExpr::IteratorDefinition, 4> ID;
+  bool IsCorrect = true;
+  for (const OMPIteratorData &D : Data) {
+    TypeSourceInfo *TInfo = nullptr;
+    SourceLocation StartLoc;
+    QualType DeclTy;
+    if (!D.Type.getAsOpaquePtr()) {
+      // OpenMP 5.0, 2.1.6 Iterators
+      // In an iterator-specifier, if the iterator-type is not specified then
+      // the type of that iterator is of int type.
+      DeclTy = Context.IntTy;
+      StartLoc = D.DeclIdentLoc;
+    } else {
+      DeclTy = Sema::GetTypeFromParser(D.Type, &TInfo);
+      StartLoc = TInfo->getTypeLoc().getBeginLoc();
+    }
+
+    bool IsDeclTyDependent = DeclTy->isDependentType() ||
+                             DeclTy->containsUnexpandedParameterPack() ||
+                             DeclTy->isInstantiationDependentType();
+    if (!IsDeclTyDependent) {
+      if (!DeclTy->isIntegralType(Context) && !DeclTy->isAnyPointerType()) {
+        // OpenMP 5.0, 2.1.6 Iterators, Restrictions, C/C++
+        // The iterator-type must be an integral or pointer type.
+        Diag(StartLoc, diag::err_omp_iterator_not_integral_or_pointer)
+            << DeclTy;
+        IsCorrect = false;
+        continue;
+      }
+      if (DeclTy.isConstant(Context)) {
+        // OpenMP 5.0, 2.1.6 Iterators, Restrictions, C/C++
+        // The iterator-type must not be const qualified.
+        Diag(StartLoc, diag::err_omp_iterator_not_integral_or_pointer)
+            << DeclTy;
+        IsCorrect = false;
+        continue;
+      }
+    }
+
+    // Iterator declaration.
+    assert(D.DeclIdent && "Identifier expected.");
+    // Always try to create iterator declarator to avoid extra error messages
+    // about unknown declarations use.
+    auto *VD =
+        VarDecl::Create(Context, SemaRef.CurContext, StartLoc, D.DeclIdentLoc,
+                        D.DeclIdent, DeclTy, TInfo, SC_None);
+    VD->setImplicit();
+    if (S) {
+      // Check for conflicting previous declaration.
+      DeclarationNameInfo NameInfo(VD->getDeclName(), D.DeclIdentLoc);
+      LookupResult Previous(SemaRef, NameInfo, Sema::LookupOrdinaryName,
+                            Sema::ForVisibleRedeclaration);
+      Previous.suppressDiagnostics();
+      SemaRef.LookupName(Previous, S);
+
+      SemaRef.FilterLookupForScope(Previous, SemaRef.CurContext, S,
+                                   /*ConsiderLinkage=*/false,
+                                   /*AllowInlineNamespace=*/false);
+      if (!Previous.empty()) {
+        NamedDecl *Old = Previous.getRepresentativeDecl();
+        Diag(D.DeclIdentLoc, diag::err_redefinition) << VD->getDeclName();
+        Diag(Old->getLocation(), diag::note_previous_definition);
+      } else {
+        SemaRef.PushOnScopeChains(VD, S);
+      }
+    } else {
+      SemaRef.CurContext->addDecl(VD);
+    }
+
+    /// Act on the iterator variable declaration.
+    ActOnOpenMPIteratorVarDecl(VD);
+
+    Expr *Begin = D.Range.Begin;
+    if (!IsDeclTyDependent && Begin && !Begin->isTypeDependent()) {
+      ExprResult BeginRes =
+          SemaRef.PerformImplicitConversion(Begin, DeclTy, Sema::AA_Converting);
+      Begin = BeginRes.get();
+    }
+    Expr *End = D.Range.End;
+    if (!IsDeclTyDependent && End && !End->isTypeDependent()) {
+      ExprResult EndRes =
+          SemaRef.PerformImplicitConversion(End, DeclTy, Sema::AA_Converting);
+      End = EndRes.get();
+    }
+    Expr *Step = D.Range.Step;
+    if (!IsDeclTyDependent && Step && !Step->isTypeDependent()) {
+      if (!Step->getType()->isIntegralType(Context)) {
+        Diag(Step->getExprLoc(), diag::err_omp_iterator_step_not_integral)
+            << Step << Step->getSourceRange();
+        IsCorrect = false;
+        continue;
+      }
+      std::optional<llvm::APSInt> Result =
+          Step->getIntegerConstantExpr(Context);
+      // OpenMP 5.0, 2.1.6 Iterators, Restrictions
+      // If the step expression of a range-specification equals zero, the
+      // behavior is unspecified.
+      if (Result && Result->isZero()) {
+        Diag(Step->getExprLoc(), diag::err_omp_iterator_step_constant_zero)
+            << Step << Step->getSourceRange();
+        IsCorrect = false;
+        continue;
+      }
+    }
+    if (!Begin || !End || !IsCorrect) {
+      IsCorrect = false;
+      continue;
+    }
+    OMPIteratorExpr::IteratorDefinition &IDElem = ID.emplace_back();
+    IDElem.IteratorDecl = VD;
+    IDElem.AssignmentLoc = D.AssignLoc;
+    IDElem.Range.Begin = Begin;
+    IDElem.Range.End = End;
+    IDElem.Range.Step = Step;
+    IDElem.ColonLoc = D.ColonLoc;
+    IDElem.SecondColonLoc = D.SecColonLoc;
+  }
+  if (!IsCorrect) {
+    // Invalidate all created iterator declarations if error is found.
+    for (const OMPIteratorExpr::IteratorDefinition &D : ID) {
+      if (Decl *ID = D.IteratorDecl)
+        ID->setInvalidDecl();
+    }
+    return ExprError();
+  }
+  SmallVector<OMPIteratorHelperData, 4> Helpers;
+  if (!SemaRef.CurContext->isDependentContext()) {
+    // Build number of ityeration for each iteration range.
+    // Ni = ((Stepi > 0) ? ((Endi + Stepi -1 - Begini)/Stepi) :
+    // ((Begini-Stepi-1-Endi) / -Stepi);
+    for (OMPIteratorExpr::IteratorDefinition &D : ID) {
+      // (Endi - Begini)
+      ExprResult Res = SemaRef.CreateBuiltinBinOp(D.AssignmentLoc, BO_Sub,
+                                                  D.Range.End, D.Range.Begin);
+      if (!Res.isUsable()) {
+        IsCorrect = false;
+        continue;
+      }
+      ExprResult St, St1;
+      if (D.Range.Step) {
+        St = D.Range.Step;
+        // (Endi - Begini) + Stepi
+        Res = SemaRef.CreateBuiltinBinOp(D.AssignmentLoc, BO_Add, Res.get(),
+                                         St.get());
+        if (!Res.isUsable()) {
+          IsCorrect = false;
+          continue;
+        }
+        // (Endi - Begini) + Stepi - 1
+        Res = SemaRef.CreateBuiltinBinOp(
+            D.AssignmentLoc, BO_Sub, Res.get(),
+            SemaRef.ActOnIntegerConstant(D.AssignmentLoc, 1).get());
+        if (!Res.isUsable()) {
+          IsCorrect = false;
+          continue;
+        }
+        // ((Endi - Begini) + Stepi - 1) / Stepi
+        Res = SemaRef.CreateBuiltinBinOp(D.AssignmentLoc, BO_Div, Res.get(),
+                                         St.get());
+        if (!Res.isUsable()) {
+          IsCorrect = false;
+          continue;
+        }
+        St1 = SemaRef.CreateBuiltinUnaryOp(D.AssignmentLoc, UO_Minus,
+                                           D.Range.Step);
+        // (Begini - Endi)
+        ExprResult Res1 = SemaRef.CreateBuiltinBinOp(
+            D.AssignmentLoc, BO_Sub, D.Range.Begin, D.Range.End);
+        if (!Res1.isUsable()) {
+          IsCorrect = false;
+          continue;
+        }
+        // (Begini - Endi) - Stepi
+        Res1 = SemaRef.CreateBuiltinBinOp(D.AssignmentLoc, BO_Add, Res1.get(),
+                                          St1.get());
+        if (!Res1.isUsable()) {
+          IsCorrect = false;
+          continue;
+        }
+        // (Begini - Endi) - Stepi - 1
+        Res1 = SemaRef.CreateBuiltinBinOp(
+            D.AssignmentLoc, BO_Sub, Res1.get(),
+            SemaRef.ActOnIntegerConstant(D.AssignmentLoc, 1).get());
+        if (!Res1.isUsable()) {
+          IsCorrect = false;
+          continue;
+        }
+        // ((Begini - Endi) - Stepi - 1) / (-Stepi)
+        Res1 = SemaRef.CreateBuiltinBinOp(D.AssignmentLoc, BO_Div, Res1.get(),
+                                          St1.get());
+        if (!Res1.isUsable()) {
+          IsCorrect = false;
+          continue;
+        }
+        // Stepi > 0.
+        ExprResult CmpRes = SemaRef.CreateBuiltinBinOp(
+            D.AssignmentLoc, BO_GT, D.Range.Step,
+            SemaRef.ActOnIntegerConstant(D.AssignmentLoc, 0).get());
+        if (!CmpRes.isUsable()) {
+          IsCorrect = false;
+          continue;
+        }
+        Res = SemaRef.ActOnConditionalOp(D.AssignmentLoc, D.AssignmentLoc,
+                                         CmpRes.get(), Res.get(), Res1.get());
+        if (!Res.isUsable()) {
+          IsCorrect = false;
+          continue;
+        }
+      }
+      Res = SemaRef.ActOnFinishFullExpr(Res.get(), /*DiscardedValue=*/false);
+      if (!Res.isUsable()) {
+        IsCorrect = false;
+        continue;
+      }
+
+      // Build counter update.
+      // Build counter.
+      auto *CounterVD = VarDecl::Create(Context, SemaRef.CurContext,
+                                        D.IteratorDecl->getBeginLoc(),
+                                        D.IteratorDecl->getBeginLoc(), nullptr,
+                                        Res.get()->getType(), nullptr, SC_None);
+      CounterVD->setImplicit();
+      ExprResult RefRes =
+          SemaRef.BuildDeclRefExpr(CounterVD, CounterVD->getType(), VK_LValue,
+                                   D.IteratorDecl->getBeginLoc());
+      // Build counter update.
+      // I = Begini + counter * Stepi;
+      ExprResult UpdateRes;
+      if (D.Range.Step) {
+        UpdateRes = SemaRef.CreateBuiltinBinOp(
+            D.AssignmentLoc, BO_Mul,
+            SemaRef.DefaultLvalueConversion(RefRes.get()).get(), St.get());
+      } else {
+        UpdateRes = SemaRef.DefaultLvalueConversion(RefRes.get());
+      }
+      if (!UpdateRes.isUsable()) {
+        IsCorrect = false;
+        continue;
+      }
+      UpdateRes = SemaRef.CreateBuiltinBinOp(D.AssignmentLoc, BO_Add,
+                                             D.Range.Begin, UpdateRes.get());
+      if (!UpdateRes.isUsable()) {
+        IsCorrect = false;
+        continue;
+      }
+      ExprResult VDRes =
+          SemaRef.BuildDeclRefExpr(cast<VarDecl>(D.IteratorDecl),
+                                   cast<VarDecl>(D.IteratorDecl)->getType(),
+                                   VK_LValue, D.IteratorDecl->getBeginLoc());
+      UpdateRes = SemaRef.CreateBuiltinBinOp(D.AssignmentLoc, BO_Assign,
+                                             VDRes.get(), UpdateRes.get());
+      if (!UpdateRes.isUsable()) {
+        IsCorrect = false;
+        continue;
+      }
+      UpdateRes =
+          SemaRef.ActOnFinishFullExpr(UpdateRes.get(), /*DiscardedValue=*/true);
+      if (!UpdateRes.isUsable()) {
+        IsCorrect = false;
+        continue;
+      }
+      ExprResult CounterUpdateRes = SemaRef.CreateBuiltinUnaryOp(
+          D.AssignmentLoc, UO_PreInc, RefRes.get());
+      if (!CounterUpdateRes.isUsable()) {
+        IsCorrect = false;
+        continue;
+      }
+      CounterUpdateRes = SemaRef.ActOnFinishFullExpr(CounterUpdateRes.get(),
+                                                     /*DiscardedValue=*/true);
+      if (!CounterUpdateRes.isUsable()) {
+        IsCorrect = false;
+        continue;
+      }
+      OMPIteratorHelperData &HD = Helpers.emplace_back();
+      HD.CounterVD = CounterVD;
+      HD.Upper = Res.get();
+      HD.Update = UpdateRes.get();
+      HD.CounterUpdate = CounterUpdateRes.get();
+    }
+  } else {
+    Helpers.assign(ID.size(), {});
+  }
+  if (!IsCorrect) {
+    // Invalidate all created iterator declarations if error is found.
+    for (const OMPIteratorExpr::IteratorDefinition &D : ID) {
+      if (Decl *ID = D.IteratorDecl)
+        ID->setInvalidDecl();
+    }
+    return ExprError();
+  }
+  return OMPIteratorExpr::Create(Context, Context.OMPIteratorTy, IteratorKwLoc,
+                                 LLoc, RLoc, ID, Helpers);
+}
+
+SemaOpenMP::SemaOpenMP(Sema &S)
+    : SemaBase(S), VarDataSharingAttributesStack(nullptr) {}
diff --git a/clang/lib/Sema/SemaStmt.cpp b/clang/lib/Sema/SemaStmt.cpp
index d28c24cfdfd3..a7b33f0db047 100644
--- a/clang/lib/Sema/SemaStmt.cpp
+++ b/clang/lib/Sema/SemaStmt.cpp
@@ -35,6 +35,7 @@
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/SemaOpenMP.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/STLExtras.h"
@@ -3097,7 +3098,7 @@ StmtResult Sema::BuildCXXForRangeStmt(
   // In OpenMP loop region loop control variable must be private. Perform
   // analysis of first part (if any).
   if (getLangOpts().OpenMP >= 50 && BeginDeclStmt.isUsable())
-    ActOnOpenMPLoopInitialization(ForLoc, BeginDeclStmt.get());
+    OpenMP().ActOnOpenMPLoopInitialization(ForLoc, BeginDeclStmt.get());
 
   return new (Context) CXXForRangeStmt(
       InitStmt, RangeDS, cast_or_null<DeclStmt>(BeginDeclStmt.get()),
@@ -4822,7 +4823,8 @@ buildCapturedStmtCaptureList(Sema &S, CapturedRegionScopeInfo *RSI,
       assert(Cap.isVariableCapture() && "unknown kind of capture");
 
       if (S.getLangOpts().OpenMP && RSI->CapRegionKind == CR_OpenMP)
-        S.setOpenMPCaptureKind(Field, Cap.getVariable(), RSI->OpenMPLevel);
+        S.OpenMP().setOpenMPCaptureKind(Field, Cap.getVariable(),
+                                        RSI->OpenMPLevel);
 
       Captures.push_back(CapturedStmt::Capture(
           Cap.getLocation(),
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index c45a8d1408ff..6d359c5a9a02 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -28,6 +28,7 @@
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/SemaOpenMP.h"
 #include "clang/Sema/Template.h"
 #include "clang/Sema/TemplateInstCallback.h"
 #include "llvm/Support/TimeProfiler.h"
@@ -399,7 +400,7 @@ static void instantiateOMPDeclareSimdDeclAttr(
     ++SI;
   }
   LinModifiers.append(Attr.modifiers_begin(), Attr.modifiers_end());
-  (void)S.ActOnOpenMPDeclareSimdDirective(
+  (void)S.OpenMP().ActOnOpenMPDeclareSimdDirective(
       S.ConvertDeclToDeclGroup(New), Attr.getBranchState(), Simdlen.get(),
       Uniforms, Aligneds, Alignments, Linears, LinModifiers, Steps,
       Attr.getRange());
@@ -476,9 +477,9 @@ static void instantiateOMPDeclareVariantAttr(
   // Check function/variant ref for `omp declare variant` but not for `omp
   // begin declare variant` (which use implicit attributes).
   std::optional<std::pair<FunctionDecl *, Expr *>> DeclVarData =
-      S.checkOpenMPDeclareVariantFunction(S.ConvertDeclToDeclGroup(New), E, TI,
-                                          Attr.appendArgs_size(),
-                                          Attr.getRange());
+      S.OpenMP().checkOpenMPDeclareVariantFunction(
+          S.ConvertDeclToDeclGroup(New), E, TI, Attr.appendArgs_size(),
+          Attr.getRange());
 
   if (!DeclVarData)
     return;
@@ -539,7 +540,7 @@ static void instantiateOMPDeclareVariantAttr(
     AppendArgs.emplace_back(II.IsTarget, II.IsTargetSync);
   }
 
-  S.ActOnOpenMPDeclareVariantDirective(
+  S.OpenMP().ActOnOpenMPDeclareVariantDirective(
       FD, E, TI, NothingExprs, NeedDevicePtrExprs, AppendArgs, SourceLocation(),
       SourceLocation(), Attr.getRange());
 }
@@ -3587,7 +3588,7 @@ Decl *TemplateDeclInstantiator::VisitOMPThreadPrivateDecl(
   }
 
   OMPThreadPrivateDecl *TD =
-    SemaRef.CheckOMPThreadPrivateDecl(D->getLocation(), Vars);
+      SemaRef.OpenMP().CheckOMPThreadPrivateDecl(D->getLocation(), Vars);
 
   TD->setAccess(AS_public);
   Owner->addDecl(TD);
@@ -3610,14 +3611,14 @@ Decl *TemplateDeclInstantiator::VisitOMPAllocateDecl(OMPAllocateDecl *D) {
       ExprResult NewE = SemaRef.SubstExpr(AC->getAllocator(), TemplateArgs);
       if (!NewE.isUsable())
         continue;
-      IC = SemaRef.ActOnOpenMPAllocatorClause(
+      IC = SemaRef.OpenMP().ActOnOpenMPAllocatorClause(
           NewE.get(), AC->getBeginLoc(), AC->getLParenLoc(), AC->getEndLoc());
     } else if (auto *AC = dyn_cast<OMPAlignClause>(C)) {
       ExprResult NewE = SemaRef.SubstExpr(AC->getAlignment(), TemplateArgs);
       if (!NewE.isUsable())
         continue;
-      IC = SemaRef.ActOnOpenMPAlignClause(NewE.get(), AC->getBeginLoc(),
-                                          AC->getLParenLoc(), AC->getEndLoc());
+      IC = SemaRef.OpenMP().ActOnOpenMPAlignClause(
+          NewE.get(), AC->getBeginLoc(), AC->getLParenLoc(), AC->getEndLoc());
       // If align clause value ends up being invalid, this can end up null.
       if (!IC)
         continue;
@@ -3625,7 +3626,7 @@ Decl *TemplateDeclInstantiator::VisitOMPAllocateDecl(OMPAllocateDecl *D) {
     Clauses.push_back(IC);
   }
 
-  Sema::DeclGroupPtrTy Res = SemaRef.ActOnOpenMPAllocateDirective(
+  Sema::DeclGroupPtrTy Res = SemaRef.OpenMP().ActOnOpenMPAllocateDirective(
       D->getLocation(), Vars, Clauses, Owner);
   if (Res.get().isNull())
     return nullptr;
@@ -3646,7 +3647,7 @@ Decl *TemplateDeclInstantiator::VisitOMPDeclareReductionDecl(
       D->getType()->containsUnexpandedParameterPack();
   QualType SubstReductionType;
   if (RequiresInstantiation) {
-    SubstReductionType = SemaRef.ActOnOpenMPDeclareReductionType(
+    SubstReductionType = SemaRef.OpenMP().ActOnOpenMPDeclareReductionType(
         D->getLocation(),
         ParsedType::make(SemaRef.SubstType(
             D->getType(), TemplateArgs, D->getLocation(), DeclarationName())));
@@ -3667,7 +3668,7 @@ Decl *TemplateDeclInstantiator::VisitOMPDeclareReductionDecl(
         SemaRef.CurrentInstantiationScope->findInstantiationOf(PrevDeclInScope)
             ->get<Decl *>());
   }
-  auto DRD = SemaRef.ActOnOpenMPDeclareReductionDirectiveStart(
+  auto DRD = SemaRef.OpenMP().ActOnOpenMPDeclareReductionDirectiveStart(
       /*S=*/nullptr, Owner, D->getDeclName(), ReductionTypes, D->getAccess(),
       PrevDeclInScope);
   auto *NewDRD = cast<OMPDeclareReductionDecl>(DRD.get().getSingleDecl());
@@ -3676,7 +3677,7 @@ Decl *TemplateDeclInstantiator::VisitOMPDeclareReductionDecl(
   Expr *SubstInitializer = nullptr;
   // Combiners instantiation sequence.
   if (Combiner) {
-    SemaRef.ActOnOpenMPDeclareReductionCombinerStart(
+    SemaRef.OpenMP().ActOnOpenMPDeclareReductionCombinerStart(
         /*S=*/nullptr, NewDRD);
     SemaRef.CurrentInstantiationScope->InstantiatedLocal(
         cast<DeclRefExpr>(D->getCombinerIn())->getDecl(),
@@ -3688,12 +3689,14 @@ Decl *TemplateDeclInstantiator::VisitOMPDeclareReductionDecl(
     Sema::CXXThisScopeRAII ThisScope(SemaRef, ThisContext, Qualifiers(),
                                      ThisContext);
     SubstCombiner = SemaRef.SubstExpr(Combiner, TemplateArgs).get();
-    SemaRef.ActOnOpenMPDeclareReductionCombinerEnd(NewDRD, SubstCombiner);
+    SemaRef.OpenMP().ActOnOpenMPDeclareReductionCombinerEnd(NewDRD,
+                                                            SubstCombiner);
   }
   // Initializers instantiation sequence.
   if (Init) {
-    VarDecl *OmpPrivParm = SemaRef.ActOnOpenMPDeclareReductionInitializerStart(
-        /*S=*/nullptr, NewDRD);
+    VarDecl *OmpPrivParm =
+        SemaRef.OpenMP().ActOnOpenMPDeclareReductionInitializerStart(
+            /*S=*/nullptr, NewDRD);
     SemaRef.CurrentInstantiationScope->InstantiatedLocal(
         cast<DeclRefExpr>(D->getInitOrig())->getDecl(),
         cast<DeclRefExpr>(NewDRD->getInitOrig())->getDecl());
@@ -3710,8 +3713,8 @@ Decl *TemplateDeclInstantiator::VisitOMPDeclareReductionDecl(
         SemaRef.InstantiateVariableInitializer(OmpPrivParm, OldPrivParm,
                                                TemplateArgs);
     }
-    SemaRef.ActOnOpenMPDeclareReductionInitializerEnd(NewDRD, SubstInitializer,
-                                                      OmpPrivParm);
+    SemaRef.OpenMP().ActOnOpenMPDeclareReductionInitializerEnd(
+        NewDRD, SubstInitializer, OmpPrivParm);
   }
   IsCorrect = IsCorrect && SubstCombiner &&
               (!Init ||
@@ -3720,7 +3723,7 @@ Decl *TemplateDeclInstantiator::VisitOMPDeclareReductionDecl(
                (D->getInitializerKind() != OMPDeclareReductionInitKind::Call &&
                 !SubstInitializer));
 
-  (void)SemaRef.ActOnOpenMPDeclareReductionDirectiveEnd(
+  (void)SemaRef.OpenMP().ActOnOpenMPDeclareReductionDirectiveEnd(
       /*S=*/nullptr, DRD, IsCorrect && !D->isInvalidDecl());
 
   return NewDRD;
@@ -3736,7 +3739,7 @@ TemplateDeclInstantiator::VisitOMPDeclareMapperDecl(OMPDeclareMapperDecl *D) {
   QualType SubstMapperTy;
   DeclarationName VN = D->getVarName();
   if (RequiresInstantiation) {
-    SubstMapperTy = SemaRef.ActOnOpenMPDeclareMapperType(
+    SubstMapperTy = SemaRef.OpenMP().ActOnOpenMPDeclareMapperType(
         D->getLocation(),
         ParsedType::make(SemaRef.SubstType(D->getType(), TemplateArgs,
                                            D->getLocation(), VN)));
@@ -3756,11 +3759,12 @@ TemplateDeclInstantiator::VisitOMPDeclareMapperDecl(OMPDeclareMapperDecl *D) {
   SmallVector<OMPClause *, 6> Clauses;
   // Instantiate the mapper variable.
   DeclarationNameInfo DirName;
-  SemaRef.StartOpenMPDSABlock(llvm::omp::OMPD_declare_mapper, DirName,
-                              /*S=*/nullptr,
-                              (*D->clauselist_begin())->getBeginLoc());
-  ExprResult MapperVarRef = SemaRef.ActOnOpenMPDeclareMapperDirectiveVarDecl(
-      /*S=*/nullptr, SubstMapperTy, D->getLocation(), VN);
+  SemaRef.OpenMP().StartOpenMPDSABlock(llvm::omp::OMPD_declare_mapper, DirName,
+                                       /*S=*/nullptr,
+                                       (*D->clauselist_begin())->getBeginLoc());
+  ExprResult MapperVarRef =
+      SemaRef.OpenMP().ActOnOpenMPDeclareMapperDirectiveVarDecl(
+          /*S=*/nullptr, SubstMapperTy, D->getLocation(), VN);
   SemaRef.CurrentInstantiationScope->InstantiatedLocal(
       cast<DeclRefExpr>(D->getMapperVarRef())->getDecl(),
       cast<DeclRefExpr>(MapperVarRef.get())->getDecl());
@@ -3790,17 +3794,17 @@ TemplateDeclInstantiator::VisitOMPDeclareMapperDecl(OMPDeclareMapperDecl *D) {
         SemaRef.SubstDeclarationNameInfo(OldC->getMapperIdInfo(), TemplateArgs);
     OMPVarListLocTy Locs(OldC->getBeginLoc(), OldC->getLParenLoc(),
                          OldC->getEndLoc());
-    OMPClause *NewC = SemaRef.ActOnOpenMPMapClause(
+    OMPClause *NewC = SemaRef.OpenMP().ActOnOpenMPMapClause(
         OldC->getIteratorModifier(), OldC->getMapTypeModifiers(),
         OldC->getMapTypeModifiersLoc(), SS, NewNameInfo, OldC->getMapType(),
         OldC->isImplicitMapType(), OldC->getMapLoc(), OldC->getColonLoc(),
         NewVars, Locs);
     Clauses.push_back(NewC);
   }
-  SemaRef.EndOpenMPDSABlock(nullptr);
+  SemaRef.OpenMP().EndOpenMPDSABlock(nullptr);
   if (!IsCorrect)
     return nullptr;
-  Sema::DeclGroupPtrTy DG = SemaRef.ActOnOpenMPDeclareMapperDirective(
+  Sema::DeclGroupPtrTy DG = SemaRef.OpenMP().ActOnOpenMPDeclareMapperDirective(
       /*S=*/nullptr, Owner, D->getDeclName(), SubstMapperTy, D->getLocation(),
       VN, D->getAccess(), MapperVarRef.get(), Clauses, PrevDeclInScope);
   Decl *NewDMD = DG.get().getSingleDecl();
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 404c4e8e31b5..1b31df8d97fb 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -35,6 +35,7 @@
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/SemaCUDA.h"
 #include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/SemaOpenMP.h"
 #include "clang/Sema/Template.h"
 #include "clang/Sema/TemplateInstCallback.h"
 #include "llvm/ADT/ArrayRef.h"
@@ -2640,7 +2641,7 @@ QualType Sema::BuildArrayType(QualType T, ArraySizeModifier ASM,
   } else if (isSFINAEContext()) {
     VLADiag = diag::err_vla_in_sfinae;
     VLAIsError = true;
-  } else if (getLangOpts().OpenMP && isInOpenMPTaskUntiedContext()) {
+  } else if (getLangOpts().OpenMP && OpenMP().isInOpenMPTaskUntiedContext()) {
     VLADiag = diag::err_openmp_vla_in_task_untied;
     VLAIsError = true;
   } else if (getLangOpts().CPlusPlus) {
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 9d15f3eacbb0..acc2d7ff9d42 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -40,6 +40,7 @@
 #include "clang/Sema/SemaDiagnostic.h"
 #include "clang/Sema/SemaInternal.h"
 #include "clang/Sema/SemaOpenACC.h"
+#include "clang/Sema/SemaOpenMP.h"
 #include "clang/Sema/SemaSYCL.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -1656,7 +1657,7 @@ public:
   /// Ensures that the outermost loop in @p LoopStmt is wrapped by a
   /// OMPCanonicalLoop.
   StmtResult RebuildOMPCanonicalLoop(Stmt *LoopStmt) {
-    return getSema().ActOnOpenMPCanonicalLoop(LoopStmt);
+    return getSema().OpenMP().ActOnOpenMPCanonicalLoop(LoopStmt);
   }
 
   /// Build a new OpenMP executable directive.
@@ -1669,7 +1670,7 @@ public:
       Stmt *AStmt, SourceLocation StartLoc, SourceLocation EndLoc,
       OpenMPDirectiveKind PrevMappedDirective = OMPD_unknown) {
 
-    return getSema().ActOnOpenMPExecutableDirective(
+    return getSema().OpenMP().ActOnOpenMPExecutableDirective(
         Kind, DirName, CancelRegion, Clauses, AStmt, StartLoc, EndLoc,
         PrevMappedDirective);
   }
@@ -1684,9 +1685,9 @@ public:
                                 SourceLocation NameModifierLoc,
                                 SourceLocation ColonLoc,
                                 SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPIfClause(NameModifier, Condition, StartLoc,
-                                         LParenLoc, NameModifierLoc, ColonLoc,
-                                         EndLoc);
+    return getSema().OpenMP().ActOnOpenMPIfClause(
+        NameModifier, Condition, StartLoc, LParenLoc, NameModifierLoc, ColonLoc,
+        EndLoc);
   }
 
   /// Build a new OpenMP 'final' clause.
@@ -1696,8 +1697,8 @@ public:
   OMPClause *RebuildOMPFinalClause(Expr *Condition, SourceLocation StartLoc,
                                    SourceLocation LParenLoc,
                                    SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPFinalClause(Condition, StartLoc, LParenLoc,
-                                            EndLoc);
+    return getSema().OpenMP().ActOnOpenMPFinalClause(Condition, StartLoc,
+                                                     LParenLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'num_threads' clause.
@@ -1708,8 +1709,8 @@ public:
                                         SourceLocation StartLoc,
                                         SourceLocation LParenLoc,
                                         SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPNumThreadsClause(NumThreads, StartLoc,
-                                                 LParenLoc, EndLoc);
+    return getSema().OpenMP().ActOnOpenMPNumThreadsClause(NumThreads, StartLoc,
+                                                          LParenLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'safelen' clause.
@@ -1719,7 +1720,8 @@ public:
   OMPClause *RebuildOMPSafelenClause(Expr *Len, SourceLocation StartLoc,
                                      SourceLocation LParenLoc,
                                      SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPSafelenClause(Len, StartLoc, LParenLoc, EndLoc);
+    return getSema().OpenMP().ActOnOpenMPSafelenClause(Len, StartLoc, LParenLoc,
+                                                       EndLoc);
   }
 
   /// Build a new OpenMP 'simdlen' clause.
@@ -1729,28 +1731,30 @@ public:
   OMPClause *RebuildOMPSimdlenClause(Expr *Len, SourceLocation StartLoc,
                                      SourceLocation LParenLoc,
                                      SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPSimdlenClause(Len, StartLoc, LParenLoc, EndLoc);
+    return getSema().OpenMP().ActOnOpenMPSimdlenClause(Len, StartLoc, LParenLoc,
+                                                       EndLoc);
   }
 
   OMPClause *RebuildOMPSizesClause(ArrayRef<Expr *> Sizes,
                                    SourceLocation StartLoc,
                                    SourceLocation LParenLoc,
                                    SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPSizesClause(Sizes, StartLoc, LParenLoc, EndLoc);
+    return getSema().OpenMP().ActOnOpenMPSizesClause(Sizes, StartLoc, LParenLoc,
+                                                     EndLoc);
   }
 
   /// Build a new OpenMP 'full' clause.
   OMPClause *RebuildOMPFullClause(SourceLocation StartLoc,
                                   SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPFullClause(StartLoc, EndLoc);
+    return getSema().OpenMP().ActOnOpenMPFullClause(StartLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'partial' clause.
   OMPClause *RebuildOMPPartialClause(Expr *Factor, SourceLocation StartLoc,
                                      SourceLocation LParenLoc,
                                      SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPPartialClause(Factor, StartLoc, LParenLoc,
-                                              EndLoc);
+    return getSema().OpenMP().ActOnOpenMPPartialClause(Factor, StartLoc,
+                                                       LParenLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'allocator' clause.
@@ -1760,7 +1764,8 @@ public:
   OMPClause *RebuildOMPAllocatorClause(Expr *A, SourceLocation StartLoc,
                                        SourceLocation LParenLoc,
                                        SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPAllocatorClause(A, StartLoc, LParenLoc, EndLoc);
+    return getSema().OpenMP().ActOnOpenMPAllocatorClause(A, StartLoc, LParenLoc,
+                                                         EndLoc);
   }
 
   /// Build a new OpenMP 'collapse' clause.
@@ -1770,8 +1775,8 @@ public:
   OMPClause *RebuildOMPCollapseClause(Expr *Num, SourceLocation StartLoc,
                                       SourceLocation LParenLoc,
                                       SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPCollapseClause(Num, StartLoc, LParenLoc,
-                                               EndLoc);
+    return getSema().OpenMP().ActOnOpenMPCollapseClause(Num, StartLoc,
+                                                        LParenLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'default' clause.
@@ -1782,8 +1787,8 @@ public:
                                      SourceLocation StartLoc,
                                      SourceLocation LParenLoc,
                                      SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPDefaultClause(Kind, KindKwLoc,
-                                              StartLoc, LParenLoc, EndLoc);
+    return getSema().OpenMP().ActOnOpenMPDefaultClause(
+        Kind, KindKwLoc, StartLoc, LParenLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'proc_bind' clause.
@@ -1795,8 +1800,8 @@ public:
                                       SourceLocation StartLoc,
                                       SourceLocation LParenLoc,
                                       SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPProcBindClause(Kind, KindKwLoc,
-                                               StartLoc, LParenLoc, EndLoc);
+    return getSema().OpenMP().ActOnOpenMPProcBindClause(
+        Kind, KindKwLoc, StartLoc, LParenLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'schedule' clause.
@@ -1808,7 +1813,7 @@ public:
       OpenMPScheduleClauseKind Kind, Expr *ChunkSize, SourceLocation StartLoc,
       SourceLocation LParenLoc, SourceLocation M1Loc, SourceLocation M2Loc,
       SourceLocation KindLoc, SourceLocation CommaLoc, SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPScheduleClause(
+    return getSema().OpenMP().ActOnOpenMPScheduleClause(
         M1, M2, Kind, ChunkSize, StartLoc, LParenLoc, M1Loc, M2Loc, KindLoc,
         CommaLoc, EndLoc);
   }
@@ -1820,7 +1825,8 @@ public:
   OMPClause *RebuildOMPOrderedClause(SourceLocation StartLoc,
                                      SourceLocation EndLoc,
                                      SourceLocation LParenLoc, Expr *Num) {
-    return getSema().ActOnOpenMPOrderedClause(StartLoc, EndLoc, LParenLoc, Num);
+    return getSema().OpenMP().ActOnOpenMPOrderedClause(StartLoc, EndLoc,
+                                                       LParenLoc, Num);
   }
 
   /// Build a new OpenMP 'private' clause.
@@ -1831,8 +1837,8 @@ public:
                                      SourceLocation StartLoc,
                                      SourceLocation LParenLoc,
                                      SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPPrivateClause(VarList, StartLoc, LParenLoc,
-                                              EndLoc);
+    return getSema().OpenMP().ActOnOpenMPPrivateClause(VarList, StartLoc,
+                                                       LParenLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'firstprivate' clause.
@@ -1843,8 +1849,8 @@ public:
                                           SourceLocation StartLoc,
                                           SourceLocation LParenLoc,
                                           SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPFirstprivateClause(VarList, StartLoc, LParenLoc,
-                                                   EndLoc);
+    return getSema().OpenMP().ActOnOpenMPFirstprivateClause(VarList, StartLoc,
+                                                            LParenLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'lastprivate' clause.
@@ -1858,7 +1864,7 @@ public:
                                          SourceLocation StartLoc,
                                          SourceLocation LParenLoc,
                                          SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPLastprivateClause(
+    return getSema().OpenMP().ActOnOpenMPLastprivateClause(
         VarList, LPKind, LPKindLoc, ColonLoc, StartLoc, LParenLoc, EndLoc);
   }
 
@@ -1870,8 +1876,8 @@ public:
                                     SourceLocation StartLoc,
                                     SourceLocation LParenLoc,
                                     SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPSharedClause(VarList, StartLoc, LParenLoc,
-                                             EndLoc);
+    return getSema().OpenMP().ActOnOpenMPSharedClause(VarList, StartLoc,
+                                                      LParenLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'reduction' clause.
@@ -1885,7 +1891,7 @@ public:
       SourceLocation EndLoc, CXXScopeSpec &ReductionIdScopeSpec,
       const DeclarationNameInfo &ReductionId,
       ArrayRef<Expr *> UnresolvedReductions) {
-    return getSema().ActOnOpenMPReductionClause(
+    return getSema().OpenMP().ActOnOpenMPReductionClause(
         VarList, Modifier, StartLoc, LParenLoc, ModifierLoc, ColonLoc, EndLoc,
         ReductionIdScopeSpec, ReductionId, UnresolvedReductions);
   }
@@ -1900,7 +1906,7 @@ public:
       CXXScopeSpec &ReductionIdScopeSpec,
       const DeclarationNameInfo &ReductionId,
       ArrayRef<Expr *> UnresolvedReductions) {
-    return getSema().ActOnOpenMPTaskReductionClause(
+    return getSema().OpenMP().ActOnOpenMPTaskReductionClause(
         VarList, StartLoc, LParenLoc, ColonLoc, EndLoc, ReductionIdScopeSpec,
         ReductionId, UnresolvedReductions);
   }
@@ -1916,7 +1922,7 @@ public:
                               CXXScopeSpec &ReductionIdScopeSpec,
                               const DeclarationNameInfo &ReductionId,
                               ArrayRef<Expr *> UnresolvedReductions) {
-    return getSema().ActOnOpenMPInReductionClause(
+    return getSema().OpenMP().ActOnOpenMPInReductionClause(
         VarList, StartLoc, LParenLoc, ColonLoc, EndLoc, ReductionIdScopeSpec,
         ReductionId, UnresolvedReductions);
   }
@@ -1930,9 +1936,9 @@ public:
       SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier,
       SourceLocation ModifierLoc, SourceLocation ColonLoc,
       SourceLocation StepModifierLoc, SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPLinearClause(VarList, Step, StartLoc, LParenLoc,
-                                             Modifier, ModifierLoc, ColonLoc,
-                                             StepModifierLoc, EndLoc);
+    return getSema().OpenMP().ActOnOpenMPLinearClause(
+        VarList, Step, StartLoc, LParenLoc, Modifier, ModifierLoc, ColonLoc,
+        StepModifierLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'aligned' clause.
@@ -1944,8 +1950,8 @@ public:
                                      SourceLocation LParenLoc,
                                      SourceLocation ColonLoc,
                                      SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPAlignedClause(VarList, Alignment, StartLoc,
-                                              LParenLoc, ColonLoc, EndLoc);
+    return getSema().OpenMP().ActOnOpenMPAlignedClause(
+        VarList, Alignment, StartLoc, LParenLoc, ColonLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'copyin' clause.
@@ -1956,8 +1962,8 @@ public:
                                     SourceLocation StartLoc,
                                     SourceLocation LParenLoc,
                                     SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPCopyinClause(VarList, StartLoc, LParenLoc,
-                                             EndLoc);
+    return getSema().OpenMP().ActOnOpenMPCopyinClause(VarList, StartLoc,
+                                                      LParenLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'copyprivate' clause.
@@ -1968,8 +1974,8 @@ public:
                                          SourceLocation StartLoc,
                                          SourceLocation LParenLoc,
                                          SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPCopyprivateClause(VarList, StartLoc, LParenLoc,
-                                                  EndLoc);
+    return getSema().OpenMP().ActOnOpenMPCopyprivateClause(VarList, StartLoc,
+                                                           LParenLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'flush' pseudo clause.
@@ -1980,8 +1986,8 @@ public:
                                    SourceLocation StartLoc,
                                    SourceLocation LParenLoc,
                                    SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPFlushClause(VarList, StartLoc, LParenLoc,
-                                            EndLoc);
+    return getSema().OpenMP().ActOnOpenMPFlushClause(VarList, StartLoc,
+                                                     LParenLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'depobj' pseudo clause.
@@ -1991,8 +1997,8 @@ public:
   OMPClause *RebuildOMPDepobjClause(Expr *Depobj, SourceLocation StartLoc,
                                     SourceLocation LParenLoc,
                                     SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPDepobjClause(Depobj, StartLoc, LParenLoc,
-                                             EndLoc);
+    return getSema().OpenMP().ActOnOpenMPDepobjClause(Depobj, StartLoc,
+                                                      LParenLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'depend' pseudo clause.
@@ -2004,8 +2010,8 @@ public:
                                     SourceLocation StartLoc,
                                     SourceLocation LParenLoc,
                                     SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPDependClause(Data, DepModifier, VarList,
-                                             StartLoc, LParenLoc, EndLoc);
+    return getSema().OpenMP().ActOnOpenMPDependClause(
+        Data, DepModifier, VarList, StartLoc, LParenLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'device' clause.
@@ -2017,8 +2023,8 @@ public:
                                     SourceLocation LParenLoc,
                                     SourceLocation ModifierLoc,
                                     SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPDeviceClause(Modifier, Device, StartLoc,
-                                             LParenLoc, ModifierLoc, EndLoc);
+    return getSema().OpenMP().ActOnOpenMPDeviceClause(
+        Modifier, Device, StartLoc, LParenLoc, ModifierLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'map' clause.
@@ -2032,7 +2038,7 @@ public:
       OpenMPMapClauseKind MapType, bool IsMapTypeImplicit,
       SourceLocation MapLoc, SourceLocation ColonLoc, ArrayRef<Expr *> VarList,
       const OMPVarListLocTy &Locs, ArrayRef<Expr *> UnresolvedMappers) {
-    return getSema().ActOnOpenMPMapClause(
+    return getSema().OpenMP().ActOnOpenMPMapClause(
         IteratorModifier, MapTypeModifiers, MapTypeModifiersLoc,
         MapperIdScopeSpec, MapperId, MapType, IsMapTypeImplicit, MapLoc,
         ColonLoc, VarList, Locs,
@@ -2048,8 +2054,8 @@ public:
                                       SourceLocation LParenLoc,
                                       SourceLocation ColonLoc,
                                       SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPAllocateClause(Allocate, VarList, StartLoc,
-                                               LParenLoc, ColonLoc, EndLoc);
+    return getSema().OpenMP().ActOnOpenMPAllocateClause(
+        Allocate, VarList, StartLoc, LParenLoc, ColonLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'num_teams' clause.
@@ -2059,8 +2065,8 @@ public:
   OMPClause *RebuildOMPNumTeamsClause(Expr *NumTeams, SourceLocation StartLoc,
                                       SourceLocation LParenLoc,
                                       SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPNumTeamsClause(NumTeams, StartLoc, LParenLoc,
-                                               EndLoc);
+    return getSema().OpenMP().ActOnOpenMPNumTeamsClause(NumTeams, StartLoc,
+                                                        LParenLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'thread_limit' clause.
@@ -2071,8 +2077,8 @@ public:
                                          SourceLocation StartLoc,
                                          SourceLocation LParenLoc,
                                          SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPThreadLimitClause(ThreadLimit, StartLoc,
-                                                  LParenLoc, EndLoc);
+    return getSema().OpenMP().ActOnOpenMPThreadLimitClause(
+        ThreadLimit, StartLoc, LParenLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'priority' clause.
@@ -2082,8 +2088,8 @@ public:
   OMPClause *RebuildOMPPriorityClause(Expr *Priority, SourceLocation StartLoc,
                                       SourceLocation LParenLoc,
                                       SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPPriorityClause(Priority, StartLoc, LParenLoc,
-                                               EndLoc);
+    return getSema().OpenMP().ActOnOpenMPPriorityClause(Priority, StartLoc,
+                                                        LParenLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'grainsize' clause.
@@ -2095,8 +2101,8 @@ public:
                                        SourceLocation LParenLoc,
                                        SourceLocation ModifierLoc,
                                        SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPGrainsizeClause(Modifier, Device, StartLoc,
-                                                LParenLoc, ModifierLoc, EndLoc);
+    return getSema().OpenMP().ActOnOpenMPGrainsizeClause(
+        Modifier, Device, StartLoc, LParenLoc, ModifierLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'num_tasks' clause.
@@ -2108,8 +2114,8 @@ public:
                                       SourceLocation LParenLoc,
                                       SourceLocation ModifierLoc,
                                       SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPNumTasksClause(Modifier, NumTasks, StartLoc,
-                                               LParenLoc, ModifierLoc, EndLoc);
+    return getSema().OpenMP().ActOnOpenMPNumTasksClause(
+        Modifier, NumTasks, StartLoc, LParenLoc, ModifierLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'hint' clause.
@@ -2119,7 +2125,8 @@ public:
   OMPClause *RebuildOMPHintClause(Expr *Hint, SourceLocation StartLoc,
                                   SourceLocation LParenLoc,
                                   SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPHintClause(Hint, StartLoc, LParenLoc, EndLoc);
+    return getSema().OpenMP().ActOnOpenMPHintClause(Hint, StartLoc, LParenLoc,
+                                                    EndLoc);
   }
 
   /// Build a new OpenMP 'detach' clause.
@@ -2129,7 +2136,8 @@ public:
   OMPClause *RebuildOMPDetachClause(Expr *Evt, SourceLocation StartLoc,
                                     SourceLocation LParenLoc,
                                     SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPDetachClause(Evt, StartLoc, LParenLoc, EndLoc);
+    return getSema().OpenMP().ActOnOpenMPDetachClause(Evt, StartLoc, LParenLoc,
+                                                      EndLoc);
   }
 
   /// Build a new OpenMP 'dist_schedule' clause.
@@ -2141,7 +2149,7 @@ public:
                                Expr *ChunkSize, SourceLocation StartLoc,
                                SourceLocation LParenLoc, SourceLocation KindLoc,
                                SourceLocation CommaLoc, SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPDistScheduleClause(
+    return getSema().OpenMP().ActOnOpenMPDistScheduleClause(
         Kind, ChunkSize, StartLoc, LParenLoc, KindLoc, CommaLoc, EndLoc);
   }
 
@@ -2156,9 +2164,9 @@ public:
                      DeclarationNameInfo &MapperId, SourceLocation ColonLoc,
                      ArrayRef<Expr *> VarList, const OMPVarListLocTy &Locs,
                      ArrayRef<Expr *> UnresolvedMappers) {
-    return getSema().ActOnOpenMPToClause(MotionModifiers, MotionModifiersLoc,
-                                         MapperIdScopeSpec, MapperId, ColonLoc,
-                                         VarList, Locs, UnresolvedMappers);
+    return getSema().OpenMP().ActOnOpenMPToClause(
+        MotionModifiers, MotionModifiersLoc, MapperIdScopeSpec, MapperId,
+        ColonLoc, VarList, Locs, UnresolvedMappers);
   }
 
   /// Build a new OpenMP 'from' clause.
@@ -2172,7 +2180,7 @@ public:
                        DeclarationNameInfo &MapperId, SourceLocation ColonLoc,
                        ArrayRef<Expr *> VarList, const OMPVarListLocTy &Locs,
                        ArrayRef<Expr *> UnresolvedMappers) {
-    return getSema().ActOnOpenMPFromClause(
+    return getSema().OpenMP().ActOnOpenMPFromClause(
         MotionModifiers, MotionModifiersLoc, MapperIdScopeSpec, MapperId,
         ColonLoc, VarList, Locs, UnresolvedMappers);
   }
@@ -2183,7 +2191,7 @@ public:
   /// Subclasses may override this routine to provide different behavior.
   OMPClause *RebuildOMPUseDevicePtrClause(ArrayRef<Expr *> VarList,
                                           const OMPVarListLocTy &Locs) {
-    return getSema().ActOnOpenMPUseDevicePtrClause(VarList, Locs);
+    return getSema().OpenMP().ActOnOpenMPUseDevicePtrClause(VarList, Locs);
   }
 
   /// Build a new OpenMP 'use_device_addr' clause.
@@ -2192,7 +2200,7 @@ public:
   /// Subclasses may override this routine to provide different behavior.
   OMPClause *RebuildOMPUseDeviceAddrClause(ArrayRef<Expr *> VarList,
                                            const OMPVarListLocTy &Locs) {
-    return getSema().ActOnOpenMPUseDeviceAddrClause(VarList, Locs);
+    return getSema().OpenMP().ActOnOpenMPUseDeviceAddrClause(VarList, Locs);
   }
 
   /// Build a new OpenMP 'is_device_ptr' clause.
@@ -2201,7 +2209,7 @@ public:
   /// Subclasses may override this routine to provide different behavior.
   OMPClause *RebuildOMPIsDevicePtrClause(ArrayRef<Expr *> VarList,
                                          const OMPVarListLocTy &Locs) {
-    return getSema().ActOnOpenMPIsDevicePtrClause(VarList, Locs);
+    return getSema().OpenMP().ActOnOpenMPIsDevicePtrClause(VarList, Locs);
   }
 
   /// Build a new OpenMP 'has_device_addr' clause.
@@ -2210,7 +2218,7 @@ public:
   /// Subclasses may override this routine to provide different behavior.
   OMPClause *RebuildOMPHasDeviceAddrClause(ArrayRef<Expr *> VarList,
                                            const OMPVarListLocTy &Locs) {
-    return getSema().ActOnOpenMPHasDeviceAddrClause(VarList, Locs);
+    return getSema().OpenMP().ActOnOpenMPHasDeviceAddrClause(VarList, Locs);
   }
 
   /// Build a new OpenMP 'defaultmap' clause.
@@ -2224,8 +2232,8 @@ public:
                                         SourceLocation MLoc,
                                         SourceLocation KindLoc,
                                         SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPDefaultmapClause(M, Kind, StartLoc, LParenLoc,
-                                                 MLoc, KindLoc, EndLoc);
+    return getSema().OpenMP().ActOnOpenMPDefaultmapClause(
+        M, Kind, StartLoc, LParenLoc, MLoc, KindLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'nontemporal' clause.
@@ -2236,8 +2244,8 @@ public:
                                          SourceLocation StartLoc,
                                          SourceLocation LParenLoc,
                                          SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPNontemporalClause(VarList, StartLoc, LParenLoc,
-                                                  EndLoc);
+    return getSema().OpenMP().ActOnOpenMPNontemporalClause(VarList, StartLoc,
+                                                           LParenLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'inclusive' clause.
@@ -2248,8 +2256,8 @@ public:
                                        SourceLocation StartLoc,
                                        SourceLocation LParenLoc,
                                        SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPInclusiveClause(VarList, StartLoc, LParenLoc,
-                                                EndLoc);
+    return getSema().OpenMP().ActOnOpenMPInclusiveClause(VarList, StartLoc,
+                                                         LParenLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'exclusive' clause.
@@ -2260,8 +2268,8 @@ public:
                                        SourceLocation StartLoc,
                                        SourceLocation LParenLoc,
                                        SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPExclusiveClause(VarList, StartLoc, LParenLoc,
-                                                EndLoc);
+    return getSema().OpenMP().ActOnOpenMPExclusiveClause(VarList, StartLoc,
+                                                         LParenLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'uses_allocators' clause.
@@ -2269,10 +2277,10 @@ public:
   /// By default, performs semantic analysis to build the new OpenMP clause.
   /// Subclasses may override this routine to provide different behavior.
   OMPClause *RebuildOMPUsesAllocatorsClause(
-      ArrayRef<Sema::UsesAllocatorsData> Data, SourceLocation StartLoc,
+      ArrayRef<SemaOpenMP::UsesAllocatorsData> Data, SourceLocation StartLoc,
       SourceLocation LParenLoc, SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPUsesAllocatorClause(StartLoc, LParenLoc, EndLoc,
-                                                    Data);
+    return getSema().OpenMP().ActOnOpenMPUsesAllocatorClause(
+        StartLoc, LParenLoc, EndLoc, Data);
   }
 
   /// Build a new OpenMP 'affinity' clause.
@@ -2284,8 +2292,8 @@ public:
                                       SourceLocation ColonLoc,
                                       SourceLocation EndLoc, Expr *Modifier,
                                       ArrayRef<Expr *> Locators) {
-    return getSema().ActOnOpenMPAffinityClause(StartLoc, LParenLoc, ColonLoc,
-                                               EndLoc, Modifier, Locators);
+    return getSema().OpenMP().ActOnOpenMPAffinityClause(
+        StartLoc, LParenLoc, ColonLoc, EndLoc, Modifier, Locators);
   }
 
   /// Build a new OpenMP 'order' clause.
@@ -2296,8 +2304,8 @@ public:
       OpenMPOrderClauseKind Kind, SourceLocation KindKwLoc,
       SourceLocation StartLoc, SourceLocation LParenLoc, SourceLocation EndLoc,
       OpenMPOrderClauseModifier Modifier, SourceLocation ModifierKwLoc) {
-    return getSema().ActOnOpenMPOrderClause(Modifier, Kind, StartLoc, LParenLoc,
-                                            ModifierKwLoc, KindKwLoc, EndLoc);
+    return getSema().OpenMP().ActOnOpenMPOrderClause(
+        Modifier, Kind, StartLoc, LParenLoc, ModifierKwLoc, KindKwLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'init' clause.
@@ -2309,8 +2317,8 @@ public:
                                   SourceLocation LParenLoc,
                                   SourceLocation VarLoc,
                                   SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPInitClause(InteropVar, InteropInfo, StartLoc,
-                                           LParenLoc, VarLoc, EndLoc);
+    return getSema().OpenMP().ActOnOpenMPInitClause(
+        InteropVar, InteropInfo, StartLoc, LParenLoc, VarLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'use' clause.
@@ -2320,8 +2328,8 @@ public:
   OMPClause *RebuildOMPUseClause(Expr *InteropVar, SourceLocation StartLoc,
                                  SourceLocation LParenLoc,
                                  SourceLocation VarLoc, SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPUseClause(InteropVar, StartLoc, LParenLoc,
-                                          VarLoc, EndLoc);
+    return getSema().OpenMP().ActOnOpenMPUseClause(InteropVar, StartLoc,
+                                                   LParenLoc, VarLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'destroy' clause.
@@ -2332,8 +2340,8 @@ public:
                                      SourceLocation LParenLoc,
                                      SourceLocation VarLoc,
                                      SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPDestroyClause(InteropVar, StartLoc, LParenLoc,
-                                              VarLoc, EndLoc);
+    return getSema().OpenMP().ActOnOpenMPDestroyClause(
+        InteropVar, StartLoc, LParenLoc, VarLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'novariants' clause.
@@ -2344,8 +2352,8 @@ public:
                                         SourceLocation StartLoc,
                                         SourceLocation LParenLoc,
                                         SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPNovariantsClause(Condition, StartLoc, LParenLoc,
-                                                 EndLoc);
+    return getSema().OpenMP().ActOnOpenMPNovariantsClause(Condition, StartLoc,
+                                                          LParenLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'nocontext' clause.
@@ -2355,8 +2363,8 @@ public:
   OMPClause *RebuildOMPNocontextClause(Expr *Condition, SourceLocation StartLoc,
                                        SourceLocation LParenLoc,
                                        SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPNocontextClause(Condition, StartLoc, LParenLoc,
-                                                EndLoc);
+    return getSema().OpenMP().ActOnOpenMPNocontextClause(Condition, StartLoc,
+                                                         LParenLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'filter' clause.
@@ -2366,8 +2374,8 @@ public:
   OMPClause *RebuildOMPFilterClause(Expr *ThreadID, SourceLocation StartLoc,
                                     SourceLocation LParenLoc,
                                     SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPFilterClause(ThreadID, StartLoc, LParenLoc,
-                                             EndLoc);
+    return getSema().OpenMP().ActOnOpenMPFilterClause(ThreadID, StartLoc,
+                                                      LParenLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'bind' clause.
@@ -2379,8 +2387,8 @@ public:
                                   SourceLocation StartLoc,
                                   SourceLocation LParenLoc,
                                   SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPBindClause(Kind, KindLoc, StartLoc, LParenLoc,
-                                           EndLoc);
+    return getSema().OpenMP().ActOnOpenMPBindClause(Kind, KindLoc, StartLoc,
+                                                    LParenLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'ompx_dyn_cgroup_mem' clause.
@@ -2390,8 +2398,8 @@ public:
   OMPClause *RebuildOMPXDynCGroupMemClause(Expr *Size, SourceLocation StartLoc,
                                            SourceLocation LParenLoc,
                                            SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPXDynCGroupMemClause(Size, StartLoc, LParenLoc,
-                                                    EndLoc);
+    return getSema().OpenMP().ActOnOpenMPXDynCGroupMemClause(Size, StartLoc,
+                                                             LParenLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'ompx_attribute' clause.
@@ -2402,8 +2410,8 @@ public:
                                         SourceLocation StartLoc,
                                         SourceLocation LParenLoc,
                                         SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPXAttributeClause(Attrs, StartLoc, LParenLoc,
-                                                 EndLoc);
+    return getSema().OpenMP().ActOnOpenMPXAttributeClause(Attrs, StartLoc,
+                                                          LParenLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'ompx_bare' clause.
@@ -2412,7 +2420,7 @@ public:
   /// Subclasses may override this routine to provide different behavior.
   OMPClause *RebuildOMPXBareClause(SourceLocation StartLoc,
                                    SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPXBareClause(StartLoc, EndLoc);
+    return getSema().OpenMP().ActOnOpenMPXBareClause(StartLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'align' clause.
@@ -2422,7 +2430,8 @@ public:
   OMPClause *RebuildOMPAlignClause(Expr *A, SourceLocation StartLoc,
                                    SourceLocation LParenLoc,
                                    SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPAlignClause(A, StartLoc, LParenLoc, EndLoc);
+    return getSema().OpenMP().ActOnOpenMPAlignClause(A, StartLoc, LParenLoc,
+                                                     EndLoc);
   }
 
   /// Build a new OpenMP 'at' clause.
@@ -2433,8 +2442,8 @@ public:
                                 SourceLocation StartLoc,
                                 SourceLocation LParenLoc,
                                 SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPAtClause(Kind, KwLoc, StartLoc, LParenLoc,
-                                         EndLoc);
+    return getSema().OpenMP().ActOnOpenMPAtClause(Kind, KwLoc, StartLoc,
+                                                  LParenLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'severity' clause.
@@ -2446,8 +2455,8 @@ public:
                                       SourceLocation StartLoc,
                                       SourceLocation LParenLoc,
                                       SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPSeverityClause(Kind, KwLoc, StartLoc, LParenLoc,
-                                               EndLoc);
+    return getSema().OpenMP().ActOnOpenMPSeverityClause(Kind, KwLoc, StartLoc,
+                                                        LParenLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'message' clause.
@@ -2457,7 +2466,8 @@ public:
   OMPClause *RebuildOMPMessageClause(Expr *MS, SourceLocation StartLoc,
                                      SourceLocation LParenLoc,
                                      SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPMessageClause(MS, StartLoc, LParenLoc, EndLoc);
+    return getSema().OpenMP().ActOnOpenMPMessageClause(MS, StartLoc, LParenLoc,
+                                                       EndLoc);
   }
 
   /// Build a new OpenMP 'doacross' clause.
@@ -2469,7 +2479,7 @@ public:
                            SourceLocation DepLoc, SourceLocation ColonLoc,
                            ArrayRef<Expr *> VarList, SourceLocation StartLoc,
                            SourceLocation LParenLoc, SourceLocation EndLoc) {
-    return getSema().ActOnOpenMPDoacrossClause(
+    return getSema().OpenMP().ActOnOpenMPDoacrossClause(
         DepType, DepLoc, ColonLoc, VarList, StartLoc, LParenLoc, EndLoc);
   }
 
@@ -2777,9 +2787,9 @@ public:
                                         SourceLocation ColonLocSecond,
                                         Expr *Length, Expr *Stride,
                                         SourceLocation RBracketLoc) {
-    return getSema().ActOnOMPArraySectionExpr(Base, LBracketLoc, LowerBound,
-                                              ColonLocFirst, ColonLocSecond,
-                                              Length, Stride, RBracketLoc);
+    return getSema().OpenMP().ActOnOMPArraySectionExpr(
+        Base, LBracketLoc, LowerBound, ColonLocFirst, ColonLocSecond, Length,
+        Stride, RBracketLoc);
   }
 
   /// Build a new array shaping expression.
@@ -2790,19 +2800,20 @@ public:
                                         SourceLocation RParenLoc,
                                         ArrayRef<Expr *> Dims,
                                         ArrayRef<SourceRange> BracketsRanges) {
-    return getSema().ActOnOMPArrayShapingExpr(Base, LParenLoc, RParenLoc, Dims,
-                                              BracketsRanges);
+    return getSema().OpenMP().ActOnOMPArrayShapingExpr(
+        Base, LParenLoc, RParenLoc, Dims, BracketsRanges);
   }
 
   /// Build a new iterator expression.
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  ExprResult RebuildOMPIteratorExpr(
-      SourceLocation IteratorKwLoc, SourceLocation LLoc, SourceLocation RLoc,
-      ArrayRef<Sema::OMPIteratorData> Data) {
-    return getSema().ActOnOMPIteratorExpr(/*Scope=*/nullptr, IteratorKwLoc,
-                                          LLoc, RLoc, Data);
+  ExprResult
+  RebuildOMPIteratorExpr(SourceLocation IteratorKwLoc, SourceLocation LLoc,
+                         SourceLocation RLoc,
+                         ArrayRef<SemaOpenMP::OMPIteratorData> Data) {
+    return getSema().OpenMP().ActOnOMPIteratorExpr(
+        /*Scope=*/nullptr, IteratorKwLoc, LLoc, RLoc, Data);
   }
 
   /// Build a new call expression.
@@ -8060,7 +8071,7 @@ template<typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformForStmt(ForStmt *S) {
   if (getSema().getLangOpts().OpenMP)
-    getSema().startOpenMPLoop();
+    getSema().OpenMP().startOpenMPLoop();
 
   // Transform the initialization statement
   StmtResult Init = getDerived().TransformStmt(S->getInit());
@@ -8070,7 +8081,8 @@ TreeTransform<Derived>::TransformForStmt(ForStmt *S) {
   // In OpenMP loop region loop control variable must be captured and be
   // private. Perform analysis of first part (if any).
   if (getSema().getLangOpts().OpenMP && Init.isUsable())
-    getSema().ActOnOpenMPLoopInitialization(S->getForLoc(), Init.get());
+    getSema().OpenMP().ActOnOpenMPLoopInitialization(S->getForLoc(),
+                                                     Init.get());
 
   // Transform the condition
   Sema::ConditionResult Cond = getDerived().TransformCondition(
@@ -9029,9 +9041,9 @@ StmtResult TreeTransform<Derived>::TransformOMPExecutableDirective(
   for (ArrayRef<OMPClause *>::iterator I = Clauses.begin(), E = Clauses.end();
        I != E; ++I) {
     if (*I) {
-      getDerived().getSema().StartOpenMPClause((*I)->getClauseKind());
+      getDerived().getSema().OpenMP().StartOpenMPClause((*I)->getClauseKind());
       OMPClause *Clause = getDerived().TransformOMPClause(*I);
-      getDerived().getSema().EndOpenMPClause();
+      getDerived().getSema().OpenMP().EndOpenMPClause();
       if (Clause)
         TClauses.push_back(Clause);
     } else {
@@ -9040,8 +9052,9 @@ StmtResult TreeTransform<Derived>::TransformOMPExecutableDirective(
   }
   StmtResult AssociatedStmt;
   if (D->hasAssociatedStmt() && D->getAssociatedStmt()) {
-    getDerived().getSema().ActOnOpenMPRegionStart(D->getDirectiveKind(),
-                                                  /*CurScope=*/nullptr);
+    getDerived().getSema().OpenMP().ActOnOpenMPRegionStart(
+        D->getDirectiveKind(),
+        /*CurScope=*/nullptr);
     StmtResult Body;
     {
       Sema::CompoundScopeRAII CompoundScope(getSema());
@@ -9059,7 +9072,7 @@ StmtResult TreeTransform<Derived>::TransformOMPExecutableDirective(
         Body = getDerived().RebuildOMPCanonicalLoop(Body.get());
     }
     AssociatedStmt =
-        getDerived().getSema().ActOnOpenMPRegionEnd(Body, TClauses);
+        getDerived().getSema().OpenMP().ActOnOpenMPRegionEnd(Body, TClauses);
     if (AssociatedStmt.isInvalid()) {
       return StmtError();
     }
@@ -9100,10 +9113,10 @@ template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPParallelDirective(OMPParallelDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_parallel, DirName, nullptr,
-                                             D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_parallel, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9111,10 +9124,10 @@ template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPSimdDirective(OMPSimdDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_simd, DirName, nullptr,
-                                             D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_simd, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9122,10 +9135,10 @@ template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPTileDirective(OMPTileDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(D->getDirectiveKind(), DirName,
-                                             nullptr, D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      D->getDirectiveKind(), DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9133,10 +9146,10 @@ template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPUnrollDirective(OMPUnrollDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(D->getDirectiveKind(), DirName,
-                                             nullptr, D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      D->getDirectiveKind(), DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9144,10 +9157,10 @@ template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPForDirective(OMPForDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_for, DirName, nullptr,
-                                             D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_for, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9155,10 +9168,10 @@ template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPForSimdDirective(OMPForSimdDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_for_simd, DirName, nullptr,
-                                             D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_for_simd, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9166,10 +9179,10 @@ template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPSectionsDirective(OMPSectionsDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_sections, DirName, nullptr,
-                                             D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_sections, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9177,10 +9190,10 @@ template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPSectionDirective(OMPSectionDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_section, DirName, nullptr,
-                                             D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_section, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9188,10 +9201,10 @@ template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPScopeDirective(OMPScopeDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_scope, DirName, nullptr,
-                                             D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_scope, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9199,10 +9212,10 @@ template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPSingleDirective(OMPSingleDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_single, DirName, nullptr,
-                                             D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_single, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9210,20 +9223,20 @@ template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPMasterDirective(OMPMasterDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_master, DirName, nullptr,
-                                             D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_master, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
 template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPCriticalDirective(OMPCriticalDirective *D) {
-  getDerived().getSema().StartOpenMPDSABlock(
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
       OMPD_critical, D->getDirectiveName(), nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9231,10 +9244,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPParallelForDirective(
     OMPParallelForDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_parallel_for, DirName,
-                                             nullptr, D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_parallel_for, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9242,10 +9255,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPParallelForSimdDirective(
     OMPParallelForSimdDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_parallel_for_simd, DirName,
-                                             nullptr, D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_parallel_for_simd, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9253,10 +9266,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPParallelMasterDirective(
     OMPParallelMasterDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_parallel_master, DirName,
-                                             nullptr, D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_parallel_master, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9264,10 +9277,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPParallelMaskedDirective(
     OMPParallelMaskedDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_parallel_masked, DirName,
-                                             nullptr, D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_parallel_masked, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9275,10 +9288,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPParallelSectionsDirective(
     OMPParallelSectionsDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_parallel_sections, DirName,
-                                             nullptr, D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_parallel_sections, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9286,10 +9299,10 @@ template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPTaskDirective(OMPTaskDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_task, DirName, nullptr,
-                                             D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_task, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9297,10 +9310,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPTaskyieldDirective(
     OMPTaskyieldDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_taskyield, DirName, nullptr,
-                                             D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_taskyield, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9308,10 +9321,10 @@ template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPBarrierDirective(OMPBarrierDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_barrier, DirName, nullptr,
-                                             D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_barrier, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9319,10 +9332,10 @@ template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPTaskwaitDirective(OMPTaskwaitDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_taskwait, DirName, nullptr,
-                                             D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_taskwait, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9330,10 +9343,10 @@ template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPErrorDirective(OMPErrorDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_error, DirName, nullptr,
-                                             D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_error, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9341,10 +9354,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPTaskgroupDirective(
     OMPTaskgroupDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_taskgroup, DirName, nullptr,
-                                             D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_taskgroup, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9352,10 +9365,10 @@ template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPFlushDirective(OMPFlushDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_flush, DirName, nullptr,
-                                             D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_flush, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9363,10 +9376,10 @@ template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPDepobjDirective(OMPDepobjDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_depobj, DirName, nullptr,
-                                             D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_depobj, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9374,10 +9387,10 @@ template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPScanDirective(OMPScanDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_scan, DirName, nullptr,
-                                             D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_scan, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9385,10 +9398,10 @@ template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPOrderedDirective(OMPOrderedDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_ordered, DirName, nullptr,
-                                             D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_ordered, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9396,10 +9409,10 @@ template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPAtomicDirective(OMPAtomicDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_atomic, DirName, nullptr,
-                                             D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_atomic, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9407,10 +9420,10 @@ template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPTargetDirective(OMPTargetDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_target, DirName, nullptr,
-                                             D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_target, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9418,10 +9431,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPTargetDataDirective(
     OMPTargetDataDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_target_data, DirName, nullptr,
-                                             D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_target_data, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9429,10 +9442,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPTargetEnterDataDirective(
     OMPTargetEnterDataDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_target_enter_data, DirName,
-                                             nullptr, D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_target_enter_data, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9440,10 +9453,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPTargetExitDataDirective(
     OMPTargetExitDataDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_target_exit_data, DirName,
-                                             nullptr, D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_target_exit_data, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9451,10 +9464,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPTargetParallelDirective(
     OMPTargetParallelDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_target_parallel, DirName,
-                                             nullptr, D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_target_parallel, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9462,10 +9475,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPTargetParallelForDirective(
     OMPTargetParallelForDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_target_parallel_for, DirName,
-                                             nullptr, D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_target_parallel_for, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9473,10 +9486,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPTargetUpdateDirective(
     OMPTargetUpdateDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_target_update, DirName,
-                                             nullptr, D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_target_update, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9484,10 +9497,10 @@ template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPTeamsDirective(OMPTeamsDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_teams, DirName, nullptr,
-                                             D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_teams, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9495,10 +9508,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPCancellationPointDirective(
     OMPCancellationPointDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_cancellation_point, DirName,
-                                             nullptr, D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_cancellation_point, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9506,10 +9519,10 @@ template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPCancelDirective(OMPCancelDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_cancel, DirName, nullptr,
-                                             D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_cancel, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9517,10 +9530,10 @@ template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPTaskLoopDirective(OMPTaskLoopDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_taskloop, DirName, nullptr,
-                                             D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_taskloop, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9528,10 +9541,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPTaskLoopSimdDirective(
     OMPTaskLoopSimdDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_taskloop_simd, DirName,
-                                             nullptr, D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_taskloop_simd, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9539,10 +9552,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPMasterTaskLoopDirective(
     OMPMasterTaskLoopDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_master_taskloop, DirName,
-                                             nullptr, D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_master_taskloop, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9550,10 +9563,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPMaskedTaskLoopDirective(
     OMPMaskedTaskLoopDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_masked_taskloop, DirName,
-                                             nullptr, D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_masked_taskloop, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9561,10 +9574,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPMasterTaskLoopSimdDirective(
     OMPMasterTaskLoopSimdDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_master_taskloop_simd, DirName,
-                                             nullptr, D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_master_taskloop_simd, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9572,10 +9585,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPMaskedTaskLoopSimdDirective(
     OMPMaskedTaskLoopSimdDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_masked_taskloop_simd, DirName,
-                                             nullptr, D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_masked_taskloop_simd, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9583,10 +9596,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPParallelMasterTaskLoopDirective(
     OMPParallelMasterTaskLoopDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
       OMPD_parallel_master_taskloop, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9594,10 +9607,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPParallelMaskedTaskLoopDirective(
     OMPParallelMaskedTaskLoopDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
       OMPD_parallel_masked_taskloop, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9606,10 +9619,10 @@ StmtResult
 TreeTransform<Derived>::TransformOMPParallelMasterTaskLoopSimdDirective(
     OMPParallelMasterTaskLoopSimdDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
       OMPD_parallel_master_taskloop_simd, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9618,10 +9631,10 @@ StmtResult
 TreeTransform<Derived>::TransformOMPParallelMaskedTaskLoopSimdDirective(
     OMPParallelMaskedTaskLoopSimdDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
       OMPD_parallel_masked_taskloop_simd, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9629,10 +9642,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPDistributeDirective(
     OMPDistributeDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_distribute, DirName, nullptr,
-                                             D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_distribute, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9640,10 +9653,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPDistributeParallelForDirective(
     OMPDistributeParallelForDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
       OMPD_distribute_parallel_for, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9652,10 +9665,10 @@ StmtResult
 TreeTransform<Derived>::TransformOMPDistributeParallelForSimdDirective(
     OMPDistributeParallelForSimdDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
       OMPD_distribute_parallel_for_simd, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9663,10 +9676,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPDistributeSimdDirective(
     OMPDistributeSimdDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_distribute_simd, DirName,
-                                             nullptr, D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_distribute_simd, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9674,10 +9687,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPTargetParallelForSimdDirective(
     OMPTargetParallelForSimdDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
       OMPD_target_parallel_for_simd, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9685,10 +9698,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPTargetSimdDirective(
     OMPTargetSimdDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_target_simd, DirName, nullptr,
-                                             D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_target_simd, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9696,10 +9709,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPTeamsDistributeDirective(
     OMPTeamsDistributeDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_teams_distribute, DirName,
-                                             nullptr, D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_teams_distribute, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9707,10 +9720,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPTeamsDistributeSimdDirective(
     OMPTeamsDistributeSimdDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
       OMPD_teams_distribute_simd, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9718,11 +9731,11 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPTeamsDistributeParallelForSimdDirective(
     OMPTeamsDistributeParallelForSimdDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
       OMPD_teams_distribute_parallel_for_simd, DirName, nullptr,
       D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9730,10 +9743,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPTeamsDistributeParallelForDirective(
     OMPTeamsDistributeParallelForDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
       OMPD_teams_distribute_parallel_for, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9741,10 +9754,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPTargetTeamsDirective(
     OMPTargetTeamsDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_target_teams, DirName,
-                                             nullptr, D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_target_teams, DirName, nullptr, D->getBeginLoc());
   auto Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9752,10 +9765,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPTargetTeamsDistributeDirective(
     OMPTargetTeamsDistributeDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
       OMPD_target_teams_distribute, DirName, nullptr, D->getBeginLoc());
   auto Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9764,11 +9777,11 @@ StmtResult
 TreeTransform<Derived>::TransformOMPTargetTeamsDistributeParallelForDirective(
     OMPTargetTeamsDistributeParallelForDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
       OMPD_target_teams_distribute_parallel_for, DirName, nullptr,
       D->getBeginLoc());
   auto Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9777,11 +9790,11 @@ StmtResult TreeTransform<Derived>::
     TransformOMPTargetTeamsDistributeParallelForSimdDirective(
         OMPTargetTeamsDistributeParallelForSimdDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
       OMPD_target_teams_distribute_parallel_for_simd, DirName, nullptr,
       D->getBeginLoc());
   auto Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9790,10 +9803,10 @@ StmtResult
 TreeTransform<Derived>::TransformOMPTargetTeamsDistributeSimdDirective(
     OMPTargetTeamsDistributeSimdDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
       OMPD_target_teams_distribute_simd, DirName, nullptr, D->getBeginLoc());
   auto Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9801,10 +9814,10 @@ template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPInteropDirective(OMPInteropDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_interop, DirName, nullptr,
-                                             D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_interop, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9812,10 +9825,10 @@ template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPDispatchDirective(OMPDispatchDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_dispatch, DirName, nullptr,
-                                             D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_dispatch, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9823,10 +9836,10 @@ template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPMaskedDirective(OMPMaskedDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_masked, DirName, nullptr,
-                                             D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_masked, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9834,10 +9847,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPGenericLoopDirective(
     OMPGenericLoopDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_loop, DirName, nullptr,
-                                             D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_loop, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9845,10 +9858,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPTeamsGenericLoopDirective(
     OMPTeamsGenericLoopDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_teams_loop, DirName, nullptr,
-                                             D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_teams_loop, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9856,10 +9869,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPTargetTeamsGenericLoopDirective(
     OMPTargetTeamsGenericLoopDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_target_teams_loop, DirName,
-                                             nullptr, D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_target_teams_loop, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9867,10 +9880,10 @@ template <typename Derived>
 StmtResult TreeTransform<Derived>::TransformOMPParallelGenericLoopDirective(
     OMPParallelGenericLoopDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_parallel_loop, DirName,
-                                             nullptr, D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_parallel_loop, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -9879,10 +9892,10 @@ StmtResult
 TreeTransform<Derived>::TransformOMPTargetParallelGenericLoopDirective(
     OMPTargetParallelGenericLoopDirective *D) {
   DeclarationNameInfo DirName;
-  getDerived().getSema().StartOpenMPDSABlock(OMPD_target_parallel_loop, DirName,
-                                             nullptr, D->getBeginLoc());
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_target_parallel_loop, DirName, nullptr, D->getBeginLoc());
   StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
-  getDerived().getSema().EndOpenMPDSABlock(Res.get());
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
   return Res;
 }
 
@@ -10972,7 +10985,7 @@ TreeTransform<Derived>::TransformOMPExclusiveClause(OMPExclusiveClause *C) {
 template <typename Derived>
 OMPClause *TreeTransform<Derived>::TransformOMPUsesAllocatorsClause(
     OMPUsesAllocatorsClause *C) {
-  SmallVector<Sema::UsesAllocatorsData, 16> Data;
+  SmallVector<SemaOpenMP::UsesAllocatorsData, 16> Data;
   Data.reserve(C->getNumberOfAllocators());
   for (unsigned I = 0, E = C->getNumberOfAllocators(); I < E; ++I) {
     OMPUsesAllocatorsClause::Data D = C->getAllocatorData(I);
@@ -10985,7 +10998,7 @@ OMPClause *TreeTransform<Derived>::TransformOMPUsesAllocatorsClause(
       if (AllocatorTraits.isInvalid())
         continue;
     }
-    Sema::UsesAllocatorsData &NewD = Data.emplace_back();
+    SemaOpenMP::UsesAllocatorsData &NewD = Data.emplace_back();
     NewD.Allocator = Allocator.get();
     NewD.AllocatorTraits = AllocatorTraits.get();
     NewD.LParenLoc = D.LParenLoc;
@@ -11667,7 +11680,7 @@ template <typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformOMPIteratorExpr(OMPIteratorExpr *E) {
   unsigned NumIterators = E->numOfIterators();
-  SmallVector<Sema::OMPIteratorData, 4> Data(NumIterators);
+  SmallVector<SemaOpenMP::OMPIteratorData, 4> Data(NumIterators);
 
   bool ErrorFound = false;
   bool NeedToRebuild = getDerived().AlwaysRebuild();
@@ -11802,7 +11815,8 @@ TreeTransform<Derived>::TransformMemberExpr(MemberExpr *E) {
     // Skip for member expression of (this->f), rebuilt thisi->f is needed
     // for Openmp where the field need to be privatizized in the case.
     if (!(isa<CXXThisExpr>(E->getBase()) &&
-          getSema().isOpenMPRebuildMemberExpr(cast<ValueDecl>(Member)))) {
+          getSema().OpenMP().isOpenMPRebuildMemberExpr(
+              cast<ValueDecl>(Member)))) {
       // Mark it referenced in the new context regardless.
       // FIXME: this is a bit instantiation-specific.
       SemaRef.MarkMemberReferenced(E);
-- 
GitLab


From 9e7aab951ffba0211193ceb435c6b49e4e19ac24 Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Tue, 16 Apr 2024 07:43:13 -0500
Subject: [PATCH 397/464] [CUDA] Rename SM_32 to SM_32_ to work around AIX
 headers (#88779)

Summary:
AIX headers define this, so we need to work around it. In the future
this will be removed but for now we should just rename it to avoid these
issues.
---
 clang/include/clang/Basic/Cuda.h         | 8 +++-----
 clang/lib/Basic/Cuda.cpp                 | 6 +++---
 clang/lib/Basic/Targets/NVPTX.cpp        | 2 +-
 clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp | 2 +-
 4 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/clang/include/clang/Basic/Cuda.h b/clang/include/clang/Basic/Cuda.h
index 38f30543a0f6..ba0e4465a0f5 100644
--- a/clang/include/clang/Basic/Cuda.h
+++ b/clang/include/clang/Basic/Cuda.h
@@ -50,17 +50,15 @@ const char *CudaVersionToString(CudaVersion V);
 // Input is "Major.Minor"
 CudaVersion CudaStringToVersion(const llvm::Twine &S);
 
-// We have a name conflict with sys/mac.h on AIX
-#ifdef SM_32
-#undef SM_32
-#endif
 enum class CudaArch {
   UNUSED,
   UNKNOWN,
+  // TODO: Deprecate and remove GPU architectures older than sm_52.
   SM_20,
   SM_21,
   SM_30,
-  SM_32,
+  // This has a name conflict with sys/mac.h on AIX, rename it as a workaround.
+  SM_32_,
   SM_35,
   SM_37,
   SM_50,
diff --git a/clang/lib/Basic/Cuda.cpp b/clang/lib/Basic/Cuda.cpp
index 1b1da6a1356f..113483db5729 100644
--- a/clang/lib/Basic/Cuda.cpp
+++ b/clang/lib/Basic/Cuda.cpp
@@ -86,7 +86,7 @@ static const CudaArchToStringMap arch_names[] = {
     // clang-format off
     {CudaArch::UNUSED, "", ""},
     SM2(20, "compute_20"), SM2(21, "compute_20"), // Fermi
-    SM(30), SM(32), SM(35), SM(37),  // Kepler
+    SM(30), {CudaArch::SM_32_, "sm_32", "compute_32"}, SM(35), SM(37),  // Kepler
     SM(50), SM(52), SM(53),          // Maxwell
     SM(60), SM(61), SM(62),          // Pascal
     SM(70), SM(72),                  // Volta
@@ -186,7 +186,7 @@ CudaVersion MinVersionForCudaArch(CudaArch A) {
   case CudaArch::SM_20:
   case CudaArch::SM_21:
   case CudaArch::SM_30:
-  case CudaArch::SM_32:
+  case CudaArch::SM_32_:
   case CudaArch::SM_35:
   case CudaArch::SM_37:
   case CudaArch::SM_50:
@@ -231,7 +231,7 @@ CudaVersion MaxVersionForCudaArch(CudaArch A) {
   case CudaArch::SM_21:
     return CudaVersion::CUDA_80;
   case CudaArch::SM_30:
-  case CudaArch::SM_32:
+  case CudaArch::SM_32_:
     return CudaVersion::CUDA_102;
   case CudaArch::SM_35:
   case CudaArch::SM_37:
diff --git a/clang/lib/Basic/Targets/NVPTX.cpp b/clang/lib/Basic/Targets/NVPTX.cpp
index b47c399fef60..8ad9e6e5f589 100644
--- a/clang/lib/Basic/Targets/NVPTX.cpp
+++ b/clang/lib/Basic/Targets/NVPTX.cpp
@@ -239,7 +239,7 @@ void NVPTXTargetInfo::getTargetDefines(const LangOptions &Opts,
         return "210";
       case CudaArch::SM_30:
         return "300";
-      case CudaArch::SM_32:
+      case CudaArch::SM_32_:
         return "320";
       case CudaArch::SM_35:
         return "350";
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
index 59ba03c6b862..eb716520e5ff 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
@@ -3466,7 +3466,7 @@ void CGOpenMPRuntimeGPU::processRequiresDirective(
       case CudaArch::SM_20:
       case CudaArch::SM_21:
       case CudaArch::SM_30:
-      case CudaArch::SM_32:
+      case CudaArch::SM_32_:
       case CudaArch::SM_35:
       case CudaArch::SM_37:
       case CudaArch::SM_50:
-- 
GitLab


From e7fb49c24e4be4780ee4df9829980c5e8ddd511e Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron@aaronballman.com>
Date: Tue, 16 Apr 2024 09:00:57 -0400
Subject: [PATCH 398/464] Switch release notes links to using markup for github
 issues; NFC

---
 clang/docs/ReleaseNotes.rst | 43 +++++++++++++------------------------
 1 file changed, 15 insertions(+), 28 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 255d2cc04404..db90db6fa4ab 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -104,8 +104,7 @@ C++20 Feature Support
 
 - Clang now implements [module.import]p7 fully. Clang now will import module
   units transitively for the module units coming from the same module of the
-  current module units.
-  Fixes `#84002 <https://github.com/llvm/llvm-project/issues/84002>`_.
+  current module units. Fixes #GH84002
 
 - Initial support for class template argument deduction (CTAD) for type alias
   templates (`P1814R0 <https://wg21.link/p1814r0>`_).
@@ -135,8 +134,7 @@ C++2c Feature Support
 Resolutions to C++ Defect Reports
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 - Substitute template parameter pack, when it is not explicitly specified
-  in the template parameters, but is deduced from a previous argument.
-  (`#78449: <https://github.com/llvm/llvm-project/issues/78449>`_).
+  in the template parameters, but is deduced from a previous argument. (#GH78449)
 
 - Type qualifications are now ignored when evaluating layout compatibility
   of two types.
@@ -176,8 +174,7 @@ C23 Feature Support
 
 - Clang now generates predefined macros of the form ``__TYPE_FMTB__`` and
   ``__TYPE_FMTb__`` (e.g., ``__UINT_FAST64_FMTB__``) in C23 mode for use with
-  macros typically exposed from ``<inttypes.h>``, such as ``PRIb8``.
-  (`#81896: <https://github.com/llvm/llvm-project/issues/81896>`_).
+  macros typically exposed from ``<inttypes.h>``, such as ``PRIb8``. (#GH81896)
 
 - Clang now supports `N3018 The constexpr specifier for object definitions`
   <https://www.open-std.org/jtc1/sc22/wg14/www/docs/n3018.htm>`_.
@@ -215,7 +212,7 @@ New Compiler Flags
 
 - ``-Wmissing-designated-field-initializers``, grouped under ``-Wmissing-field-initializers``.
   This diagnostic can be disabled to make ``-Wmissing-field-initializers`` behave
-  like it did before Clang 18.x. Fixes (`#56628 <https://github.com/llvm/llvm-project/issues/68933>`_)
+  like it did before Clang 18.x. Fixes #GH56628
 
 Deprecated Compiler Flags
 -------------------------
@@ -254,8 +251,7 @@ Removed Compiler Flags
 
 - The ``-freroll-loops`` flag has been removed. It had no effect since Clang 13.
 - ``-m[no-]unaligned-access`` is removed for RISC-V and LoongArch.
-  ``-m[no-]strict-align``, also supported by GCC, should be used instead.
-  (`#85350 <https://github.com/llvm/llvm-project/pull/85350>`_.)
+  ``-m[no-]strict-align``, also supported by GCC, should be used instead. (#GH85350)
 
 Attribute Changes in Clang
 --------------------------
@@ -325,8 +321,7 @@ Improvements to Clang's diagnostics
   Fixes #GH82512.
 
 - Clang now provides improved warnings for the ``cleanup`` attribute to detect misuse scenarios,
-  such as attempting to call ``free`` on an unallocated object. Fixes
-  `#79443 <https://github.com/llvm/llvm-project/issues/79443>`_.
+  such as attempting to call ``free`` on an unallocated object. Fixes #GH79443.
 
 - Clang no longer warns when the ``bitand`` operator is used with boolean
   operands, distinguishing it from potential typographical errors or unintended
@@ -372,11 +367,10 @@ Improvements to Clang's time-trace
 Bug Fixes in This Version
 -------------------------
 - Clang's ``-Wundefined-func-template`` no longer warns on pure virtual
-  functions.
-  (`#74016 <https://github.com/llvm/llvm-project/issues/74016>`_)
+  functions. (#GH74016)
 
 - Fixed missing warnings when comparing mismatched enumeration constants
-  in C (`#29217 <https://github.com/llvm/llvm-project/issues/29217>`).
+  in C (#GH29217)
 
 - Clang now accepts elaborated-type-specifiers that explicitly specialize
   a member class template for an implicit instantiation of a class template.
@@ -415,7 +409,7 @@ Bug Fixes in This Version
   type only rather than to the complex type (e.g. ``_Complex float / int`` is now evaluated
   as ``_Complex float / float`` rather than ``_Complex float / _Complex float``), as mandated
   by the C standard. This significantly improves codegen of `*` and `/` especially.
-  Fixes (`#31205 <https://github.com/llvm/llvm-project/issues/31205>`_).
+  Fixes #GH31205.
 
 - Fixes an assertion failure on invalid code when trying to define member
   functions in lambdas.
@@ -464,8 +458,7 @@ Bug Fixes to C++ Support
 - Fix a crash when trying to call a varargs function that also has an explicit object parameter. (#GH80971)
 - Fixed a bug where abbreviated function templates would append their invented template parameters to
   an empty template parameter lists.
-- Fix parsing of abominable function types inside type traits.
-  Fixes (`#77585 <https://github.com/llvm/llvm-project/issues/77585>`_)
+- Fix parsing of abominable function types inside type traits. Fixes #GH77585
 - Clang now classifies aggregate initialization in C++17 and newer as constant
   or non-constant more accurately. Previously, only a subset of the initializer
   elements were considered, misclassifying some initializers as constant. Partially fixes
@@ -506,9 +499,7 @@ Bug Fixes to C++ Support
 - Fix a bug where overload resolution falsely reported an ambiguity when it was comparing
   a member-function against a non member function or a member-function with an
   explicit object parameter against a member function with no explicit object parameter
-  when one of the function had more specialized templates.
-  Fixes (`#82509 <https://github.com/llvm/llvm-project/issues/82509>`_)
-  and (`#74494 <https://github.com/llvm/llvm-project/issues/74494>`_)
+  when one of the function had more specialized templates. Fixes #GH82509 and #GH74494
 - Clang now supports direct lambda calls inside of a type alias template declarations.
   This addresses (#GH70601), (#GH76674), (#GH79555), (#GH81145) and (#GH82104).
 - Allow access to a public template alias declaration that refers to friend's
@@ -530,8 +521,7 @@ Bug Fixes to C++ Support
 - Fixed a bug that prevented member function templates of class templates declared with a deduced return type
   from being explicitly specialized for a given implicit instantiation of the class template.
 
-- Fix crash when inheriting from a cv-qualified type. Fixes:
-  (`#35603 <https://github.com/llvm/llvm-project/issues/35603>`_)
+- Fix crash when inheriting from a cv-qualified type. Fixes #GH35603
 - Fix a crash when the using enum declaration uses an anonymous enumeration. Fixes (#GH86790).
 - Handled an edge case in ``getFullyPackExpandedSize`` so that we now avoid a false-positive diagnostic. (#GH84220)
 - Clang now correctly tracks type dependence of by-value captures in lambdas with an explicit
@@ -539,8 +529,7 @@ Bug Fixes to C++ Support
   Fixes (#GH70604), (#GH79754), (#GH84163), (#GH84425), (#GH86054), (#GH86398), and (#GH86399).
 - Fix a crash when deducing ``auto`` from an invalid dereference (#GH88329).
 - Fix a crash in requires expression with templated base class member function. Fixes (#GH84020).
-- placement new initializes typedef array with correct size
-  (`#GH41441 <https://github.com/llvm/llvm-project/issues/41441>`_)
+- Placement new initializes typedef array with correct size (#GH41441)
 
 Bug Fixes to AST Handling
 ^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -554,8 +543,7 @@ Miscellaneous Clang Crashes Fixed
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 - Do not attempt to dump the layout of dependent types or invalid declarations
-  when ``-fdump-record-layouts-complete`` is passed.
-  Fixes (`#83684 <https://github.com/llvm/llvm-project/issues/83684>`_).
+  when ``-fdump-record-layouts-complete`` is passed. Fixes #GH83684.
 
 OpenACC Specific Changes
 ------------------------
@@ -605,8 +593,7 @@ Windows Support
   would only be included if AVX was enabled at compile time. This was done to work
   around include times from MSVC STL including ``intrin.h`` under clang-cl.
   Clang-cl now provides ``intrin0.h`` for MSVC STL and therefore all intrinsic
-  features without requiring enablement at compile time.
-  Fixes: (`#53520 <https://github.com/llvm/llvm-project/issues/53520>`_)
+  features without requiring enablement at compile time. Fixes #GH53520
 
 - Improved compile times with MSVC STL. MSVC provides ``intrin0.h`` which is a
   header that only includes intrinsics that are used by MSVC STL to avoid the
-- 
GitLab


From 9d8be2408768912dc113a342050049231e4fc8d1 Mon Sep 17 00:00:00 2001
From: Utkarsh Saxena <usx@google.com>
Date: Tue, 16 Apr 2024 15:30:32 +0200
Subject: [PATCH 399/464] Revert "[codegen] Emit missing cleanups for stmt-expr
 and coro suspensions" and related commits (#88884)

The original change caused widespread breakages in msan/ubsan tests and
causes `use-after-free`. Most likely we are adding more cleanups than
necessary.
---
 clang/lib/CodeGen/CGCall.cpp                  |  13 +-
 clang/lib/CodeGen/CGCleanup.cpp               |  49 +--
 clang/lib/CodeGen/CGCleanup.h                 |  57 +--
 clang/lib/CodeGen/CGDecl.cpp                  |  61 +--
 clang/lib/CodeGen/CGExpr.cpp                  |  12 +-
 clang/lib/CodeGen/CGExprAgg.cpp               |  87 ++--
 clang/lib/CodeGen/CGExprCXX.cpp               |  38 +-
 clang/lib/CodeGen/CodeGenFunction.cpp         |   6 -
 clang/lib/CodeGen/CodeGenFunction.h           |  99 +----
 .../CodeGenCXX/control-flow-in-stmt-expr.cpp  | 409 ------------------
 .../coro-suspend-cleanups.cpp                 |  93 ----
 11 files changed, 128 insertions(+), 796 deletions(-)
 delete mode 100644 clang/test/CodeGenCXX/control-flow-in-stmt-expr.cpp
 delete mode 100644 clang/test/CodeGenCoroutines/coro-suspend-cleanups.cpp

diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index 0c860a3ccbd2..7a0bc6fa77b8 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -4694,11 +4694,11 @@ void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
     AggValueSlot Slot = args.isUsingInAlloca()
         ? createPlaceholderSlot(*this, type) : CreateAggTemp(type, "agg.tmp");
 
-    bool DestroyedInCallee = true, NeedsCleanup = true;
+    bool DestroyedInCallee = true, NeedsEHCleanup = true;
     if (const auto *RD = type->getAsCXXRecordDecl())
       DestroyedInCallee = RD->hasNonTrivialDestructor();
     else
-      NeedsCleanup = type.isDestructedType();
+      NeedsEHCleanup = needsEHCleanup(type.isDestructedType());
 
     if (DestroyedInCallee)
       Slot.setExternallyDestructed();
@@ -4707,15 +4707,14 @@ void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
     RValue RV = Slot.asRValue();
     args.add(RV, type);
 
-    if (DestroyedInCallee && NeedsCleanup) {
+    if (DestroyedInCallee && NeedsEHCleanup) {
       // Create a no-op GEP between the placeholder and the cleanup so we can
       // RAUW it successfully.  It also serves as a marker of the first
       // instruction where the cleanup is active.
-      pushFullExprCleanup<DestroyUnpassedArg>(NormalAndEHCleanup,
-                                              Slot.getAddress(), type);
+      pushFullExprCleanup<DestroyUnpassedArg>(EHCleanup, Slot.getAddress(),
+                                              type);
       // This unreachable is a temporary marker which will be removed later.
-      llvm::Instruction *IsActive =
-          Builder.CreateFlagLoad(llvm::Constant::getNullValue(Int8PtrTy));
+      llvm::Instruction *IsActive = Builder.CreateUnreachable();
       args.addArgCleanupDeactivation(EHStack.stable_begin(), IsActive);
     }
     return;
diff --git a/clang/lib/CodeGen/CGCleanup.cpp b/clang/lib/CodeGen/CGCleanup.cpp
index 8683f19d9da2..e6f8e6873004 100644
--- a/clang/lib/CodeGen/CGCleanup.cpp
+++ b/clang/lib/CodeGen/CGCleanup.cpp
@@ -634,19 +634,12 @@ static void destroyOptimisticNormalEntry(CodeGenFunction &CGF,
 /// Pops a cleanup block.  If the block includes a normal cleanup, the
 /// current insertion point is threaded through the cleanup, as are
 /// any branch fixups on the cleanup.
-void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough,
-                                      bool ForDeactivation) {
+void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough) {
   assert(!EHStack.empty() && "cleanup stack is empty!");
   assert(isa<EHCleanupScope>(*EHStack.begin()) && "top not a cleanup!");
   EHCleanupScope &Scope = cast<EHCleanupScope>(*EHStack.begin());
   assert(Scope.getFixupDepth() <= EHStack.getNumBranchFixups());
 
-  // If we are deactivating a normal cleanup, we need to pretend that the
-  // fallthrough is unreachable. We restore this IP before returning.
-  CGBuilderTy::InsertPoint NormalDeactivateOrigIP;
-  if (ForDeactivation && (Scope.isNormalCleanup() || !getLangOpts().EHAsynch)) {
-    NormalDeactivateOrigIP = Builder.saveAndClearIP();
-  }
   // Remember activation information.
   bool IsActive = Scope.isActive();
   Address NormalActiveFlag =
@@ -674,8 +667,7 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough,
 
   // - whether there's a fallthrough
   llvm::BasicBlock *FallthroughSource = Builder.GetInsertBlock();
-  bool HasFallthrough =
-      FallthroughSource != nullptr && (IsActive || HasExistingBranches);
+  bool HasFallthrough = (FallthroughSource != nullptr && IsActive);
 
   // Branch-through fall-throughs leave the insertion point set to the
   // end of the last cleanup, which points to the current scope.  The
@@ -700,11 +692,7 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough,
 
   // If we have a prebranched fallthrough into an inactive normal
   // cleanup, rewrite it so that it leads to the appropriate place.
-  if (Scope.isNormalCleanup() && HasPrebranchedFallthrough &&
-      !RequiresNormalCleanup) {
-    // FIXME: Come up with a program which would need forwarding prebranched
-    // fallthrough and add tests. Otherwise delete this and assert against it.
-    assert(!IsActive);
+  if (Scope.isNormalCleanup() && HasPrebranchedFallthrough && !IsActive) {
     llvm::BasicBlock *prebranchDest;
 
     // If the prebranch is semantically branching through the next
@@ -736,8 +724,6 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough,
     EHStack.popCleanup(); // safe because there are no fixups
     assert(EHStack.getNumBranchFixups() == 0 ||
            EHStack.hasNormalCleanups());
-    if (NormalDeactivateOrigIP.isSet())
-      Builder.restoreIP(NormalDeactivateOrigIP);
     return;
   }
 
@@ -774,19 +760,11 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough,
   if (!RequiresNormalCleanup) {
     // Mark CPP scope end for passed-by-value Arg temp
     //   per Windows ABI which is "normally" Cleanup in callee
-    if (IsEHa && getInvokeDest()) {
-      // If we are deactivating a normal cleanup then we don't have a
-      // fallthrough. Restore original IP to emit CPP scope ends in the correct
-      // block.
-      if (NormalDeactivateOrigIP.isSet())
-        Builder.restoreIP(NormalDeactivateOrigIP);
-      if (Personality.isMSVCXXPersonality() && Builder.GetInsertBlock())
+    if (IsEHa && getInvokeDest() && Builder.GetInsertBlock()) {
+      if (Personality.isMSVCXXPersonality())
         EmitSehCppScopeEnd();
-      if (NormalDeactivateOrigIP.isSet())
-        NormalDeactivateOrigIP = Builder.saveAndClearIP();
     }
     destroyOptimisticNormalEntry(*this, Scope);
-    Scope.MarkEmitted();
     EHStack.popCleanup();
   } else {
     // If we have a fallthrough and no other need for the cleanup,
@@ -803,7 +781,6 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough,
       }
 
       destroyOptimisticNormalEntry(*this, Scope);
-      Scope.MarkEmitted();
       EHStack.popCleanup();
 
       EmitCleanup(*this, Fn, cleanupFlags, NormalActiveFlag);
@@ -939,7 +916,6 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough,
       }
 
       // IV.  Pop the cleanup and emit it.
-      Scope.MarkEmitted();
       EHStack.popCleanup();
       assert(EHStack.hasNormalCleanups() == HasEnclosingCleanups);
 
@@ -1008,8 +984,6 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough,
     }
   }
 
-  if (NormalDeactivateOrigIP.isSet())
-    Builder.restoreIP(NormalDeactivateOrigIP);
   assert(EHStack.hasNormalCleanups() || EHStack.getNumBranchFixups() == 0);
 
   // Emit the EH cleanup if required.
@@ -1299,8 +1273,17 @@ void CodeGenFunction::DeactivateCleanupBlock(EHScopeStack::stable_iterator C,
   // to the current RunCleanupsScope.
   if (C == EHStack.stable_begin() &&
       CurrentCleanupScopeDepth.strictlyEncloses(C)) {
-    PopCleanupBlock(/*FallthroughIsBranchThrough=*/false,
-                    /*ForDeactivation=*/true);
+    // Per comment below, checking EHAsynch is not really necessary
+    // it's there to assure zero-impact w/o EHAsynch option
+    if (!Scope.isNormalCleanup() && getLangOpts().EHAsynch) {
+      PopCleanupBlock();
+    } else {
+      // If it's a normal cleanup, we need to pretend that the
+      // fallthrough is unreachable.
+      CGBuilderTy::InsertPoint SavedIP = Builder.saveAndClearIP();
+      PopCleanupBlock();
+      Builder.restoreIP(SavedIP);
+    }
     return;
   }
 
diff --git a/clang/lib/CodeGen/CGCleanup.h b/clang/lib/CodeGen/CGCleanup.h
index c73c97146abc..03e4a29d7b3d 100644
--- a/clang/lib/CodeGen/CGCleanup.h
+++ b/clang/lib/CodeGen/CGCleanup.h
@@ -16,11 +16,8 @@
 #include "EHScopeStack.h"
 
 #include "Address.h"
-#include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/IR/Instruction.h"
 
 namespace llvm {
 class BasicBlock;
@@ -269,51 +266,6 @@ class alignas(8) EHCleanupScope : public EHScope {
   };
   mutable struct ExtInfo *ExtInfo;
 
-  /// Erases auxillary allocas and their usages for an unused cleanup.
-  /// Cleanups should mark these allocas as 'used' if the cleanup is
-  /// emitted, otherwise these instructions would be erased.
-  struct AuxillaryAllocas {
-    SmallVector<llvm::Instruction *, 1> AuxAllocas;
-    bool used = false;
-
-    // Records a potentially unused instruction to be erased later.
-    void Add(llvm::AllocaInst *Alloca) { AuxAllocas.push_back(Alloca); }
-
-    // Mark all recorded instructions as used. These will not be erased later.
-    void MarkUsed() {
-      used = true;
-      AuxAllocas.clear();
-    }
-
-    ~AuxillaryAllocas() {
-      if (used)
-        return;
-      llvm::SetVector<llvm::Instruction *> Uses;
-      for (auto *Inst : llvm::reverse(AuxAllocas))
-        CollectUses(Inst, Uses);
-      // Delete uses in the reverse order of insertion.
-      for (auto *I : llvm::reverse(Uses))
-        I->eraseFromParent();
-    }
-
-  private:
-    void CollectUses(llvm::Instruction *I,
-                     llvm::SetVector<llvm::Instruction *> &Uses) {
-      if (!I || !Uses.insert(I))
-        return;
-      for (auto *User : I->users())
-        CollectUses(cast<llvm::Instruction>(User), Uses);
-    }
-  };
-  mutable struct AuxillaryAllocas *AuxAllocas;
-
-  AuxillaryAllocas &getAuxillaryAllocas() {
-    if (!AuxAllocas) {
-      AuxAllocas = new struct AuxillaryAllocas();
-    }
-    return *AuxAllocas;
-  }
-
   /// The number of fixups required by enclosing scopes (not including
   /// this one).  If this is the top cleanup scope, all the fixups
   /// from this index onwards belong to this scope.
@@ -346,7 +298,7 @@ public:
                  EHScopeStack::stable_iterator enclosingEH)
       : EHScope(EHScope::Cleanup, enclosingEH),
         EnclosingNormal(enclosingNormal), NormalBlock(nullptr),
-        ActiveFlag(Address::invalid()), ExtInfo(nullptr), AuxAllocas(nullptr),
+        ActiveFlag(Address::invalid()), ExtInfo(nullptr),
         FixupDepth(fixupDepth) {
     CleanupBits.IsNormalCleanup = isNormal;
     CleanupBits.IsEHCleanup = isEH;
@@ -360,15 +312,8 @@ public:
   }
 
   void Destroy() {
-    if (AuxAllocas)
-      delete AuxAllocas;
     delete ExtInfo;
   }
-  void AddAuxAllocas(llvm::SmallVector<llvm::AllocaInst *> Allocas) {
-    for (auto *Alloca : Allocas)
-      getAuxillaryAllocas().Add(Alloca);
-  }
-  void MarkEmitted() { getAuxillaryAllocas().MarkUsed(); }
   // Objects of EHCleanupScope are not destructed. Use Destroy().
   ~EHCleanupScope() = delete;
 
diff --git a/clang/lib/CodeGen/CGDecl.cpp b/clang/lib/CodeGen/CGDecl.cpp
index 3f05ebb561da..ce6d6d895607 100644
--- a/clang/lib/CodeGen/CGDecl.cpp
+++ b/clang/lib/CodeGen/CGDecl.cpp
@@ -19,7 +19,6 @@
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
 #include "ConstantEmitter.h"
-#include "EHScopeStack.h"
 #include "PatternInit.h"
 #include "TargetInfo.h"
 #include "clang/AST/ASTContext.h"
@@ -2202,27 +2201,6 @@ void CodeGenFunction::pushDestroy(CleanupKind cleanupKind, Address addr,
                                      destroyer, useEHCleanupForArray);
 }
 
-// Pushes a destroy and defers its deactivation until its
-// CleanupDeactivationScope is exited.
-void CodeGenFunction::pushDestroyAndDeferDeactivation(
-    QualType::DestructionKind dtorKind, Address addr, QualType type) {
-  assert(dtorKind && "cannot push destructor for trivial type");
-
-  CleanupKind cleanupKind = getCleanupKind(dtorKind);
-  pushDestroyAndDeferDeactivation(
-      cleanupKind, addr, type, getDestroyer(dtorKind), cleanupKind & EHCleanup);
-}
-
-void CodeGenFunction::pushDestroyAndDeferDeactivation(
-    CleanupKind cleanupKind, Address addr, QualType type, Destroyer *destroyer,
-    bool useEHCleanupForArray) {
-  llvm::Instruction *DominatingIP =
-      Builder.CreateFlagLoad(llvm::Constant::getNullValue(Int8PtrTy));
-  pushDestroy(cleanupKind, addr, type, destroyer, useEHCleanupForArray);
-  DeferredDeactivationCleanupStack.push_back(
-      {EHStack.stable_begin(), DominatingIP});
-}
-
 void CodeGenFunction::pushStackRestore(CleanupKind Kind, Address SPMem) {
   EHStack.pushCleanup<CallStackRestore>(Kind, SPMem);
 }
@@ -2239,19 +2217,16 @@ void CodeGenFunction::pushLifetimeExtendedDestroy(CleanupKind cleanupKind,
   // If we're not in a conditional branch, we don't need to bother generating a
   // conditional cleanup.
   if (!isInConditionalBranch()) {
+    // Push an EH-only cleanup for the object now.
     // FIXME: When popping normal cleanups, we need to keep this EH cleanup
     // around in case a temporary's destructor throws an exception.
+    if (cleanupKind & EHCleanup)
+      EHStack.pushCleanup<DestroyObject>(
+          static_cast<CleanupKind>(cleanupKind & ~NormalCleanup), addr, type,
+          destroyer, useEHCleanupForArray);
 
-    // Add the cleanup to the EHStack. After the full-expr, this would be
-    // deactivated before being popped from the stack.
-    pushDestroyAndDeferDeactivation(cleanupKind, addr, type, destroyer,
-                                    useEHCleanupForArray);
-
-    // Since this is lifetime-extended, push it once again to the EHStack after
-    // the full expression.
     return pushCleanupAfterFullExprWithActiveFlag<DestroyObject>(
-        cleanupKind, Address::invalid(), addr, type, destroyer,
-        useEHCleanupForArray);
+        cleanupKind, Address::invalid(), addr, type, destroyer, useEHCleanupForArray);
   }
 
   // Otherwise, we should only destroy the object if it's been initialized.
@@ -2266,12 +2241,13 @@ void CodeGenFunction::pushLifetimeExtendedDestroy(CleanupKind cleanupKind,
   Address ActiveFlag = createCleanupActiveFlag();
   SavedType SavedAddr = saveValueInCond(addr);
 
-  pushCleanupAndDeferDeactivation<ConditionalCleanupType>(
-      cleanupKind, SavedAddr, type, destroyer, useEHCleanupForArray);
-  initFullExprCleanupWithFlag(ActiveFlag);
+  if (cleanupKind & EHCleanup) {
+    EHStack.pushCleanup<ConditionalCleanupType>(
+        static_cast<CleanupKind>(cleanupKind & ~NormalCleanup), SavedAddr, type,
+        destroyer, useEHCleanupForArray);
+    initFullExprCleanupWithFlag(ActiveFlag);
+  }
 
-  // Since this is lifetime-extended, push it once again to the EHStack after
-  // the full expression.
   pushCleanupAfterFullExprWithActiveFlag<ConditionalCleanupType>(
       cleanupKind, ActiveFlag, SavedAddr, type, destroyer,
       useEHCleanupForArray);
@@ -2466,9 +2442,9 @@ namespace {
   };
 } // end anonymous namespace
 
-/// pushIrregularPartialArrayCleanup - Push a NormalAndEHCleanup to
-/// destroy already-constructed elements of the given array.  The cleanup may be
-/// popped with DeactivateCleanupBlock or PopCleanupBlock.
+/// pushIrregularPartialArrayCleanup - Push an EH cleanup to destroy
+/// already-constructed elements of the given array.  The cleanup
+/// may be popped with DeactivateCleanupBlock or PopCleanupBlock.
 ///
 /// \param elementType - the immediate element type of the array;
 ///   possibly still an array type
@@ -2477,9 +2453,10 @@ void CodeGenFunction::pushIrregularPartialArrayCleanup(llvm::Value *arrayBegin,
                                                        QualType elementType,
                                                        CharUnits elementAlign,
                                                        Destroyer *destroyer) {
-  pushFullExprCleanup<IrregularPartialArrayDestroy>(
-      NormalAndEHCleanup, arrayBegin, arrayEndPointer, elementType,
-      elementAlign, destroyer);
+  pushFullExprCleanup<IrregularPartialArrayDestroy>(EHCleanup,
+                                                    arrayBegin, arrayEndPointer,
+                                                    elementType, elementAlign,
+                                                    destroyer);
 }
 
 /// pushRegularPartialArrayCleanup - Push an EH cleanup to destroy
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index c85a339f5e3f..cf696a1c9f56 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -115,16 +115,10 @@ RawAddress CodeGenFunction::CreateTempAlloca(llvm::Type *Ty, CharUnits Align,
 llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(llvm::Type *Ty,
                                                     const Twine &Name,
                                                     llvm::Value *ArraySize) {
-  llvm::AllocaInst *Alloca;
   if (ArraySize)
-    Alloca = Builder.CreateAlloca(Ty, ArraySize, Name);
-  else
-    Alloca = new llvm::AllocaInst(Ty, CGM.getDataLayout().getAllocaAddrSpace(),
-                                  ArraySize, Name, AllocaInsertPt);
-  if (Allocas) {
-    Allocas->Add(Alloca);
-  }
-  return Alloca;
+    return Builder.CreateAlloca(Ty, ArraySize, Name);
+  return new llvm::AllocaInst(Ty, CGM.getDataLayout().getAllocaAddrSpace(),
+                              ArraySize, Name, AllocaInsertPt);
 }
 
 /// CreateDefaultAlignTempAlloca - This creates an alloca with the
diff --git a/clang/lib/CodeGen/CGExprAgg.cpp b/clang/lib/CodeGen/CGExprAgg.cpp
index 560a9e2c5ead..1b9287ea2393 100644
--- a/clang/lib/CodeGen/CGExprAgg.cpp
+++ b/clang/lib/CodeGen/CGExprAgg.cpp
@@ -15,7 +15,6 @@
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
 #include "ConstantEmitter.h"
-#include "EHScopeStack.h"
 #include "TargetInfo.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Attr.h"
@@ -25,7 +24,6 @@
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/GlobalVariable.h"
-#include "llvm/IR/Instruction.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
 using namespace clang;
@@ -560,27 +558,24 @@ void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType,
   // For that, we'll need an EH cleanup.
   QualType::DestructionKind dtorKind = elementType.isDestructedType();
   Address endOfInit = Address::invalid();
-  CodeGenFunction::CleanupDeactivationScope deactivation(CGF);
-
-  if (dtorKind) {
-    CodeGenFunction::AllocaTrackerRAII allocaTracker(CGF);
+  EHScopeStack::stable_iterator cleanup;
+  llvm::Instruction *cleanupDominator = nullptr;
+  if (CGF.needsEHCleanup(dtorKind)) {
     // In principle we could tell the cleanup where we are more
     // directly, but the control flow can get so varied here that it
     // would actually be quite complex.  Therefore we go through an
     // alloca.
-    llvm::Instruction *dominatingIP =
-        Builder.CreateFlagLoad(llvm::ConstantInt::getNullValue(CGF.Int8PtrTy));
     endOfInit = CGF.CreateTempAlloca(begin->getType(), CGF.getPointerAlign(),
                                      "arrayinit.endOfInit");
-    Builder.CreateStore(begin, endOfInit);
+    cleanupDominator = Builder.CreateStore(begin, endOfInit);
     CGF.pushIrregularPartialArrayCleanup(begin, endOfInit, elementType,
                                          elementAlign,
                                          CGF.getDestroyer(dtorKind));
-    cast<EHCleanupScope>(*CGF.EHStack.find(CGF.EHStack.stable_begin()))
-        .AddAuxAllocas(allocaTracker.Take());
+    cleanup = CGF.EHStack.stable_begin();
 
-    CGF.DeferredDeactivationCleanupStack.push_back(
-        {CGF.EHStack.stable_begin(), dominatingIP});
+  // Otherwise, remember that we didn't need a cleanup.
+  } else {
+    dtorKind = QualType::DK_none;
   }
 
   llvm::Value *one = llvm::ConstantInt::get(CGF.SizeTy, 1);
@@ -676,6 +671,9 @@ void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType,
 
     CGF.EmitBlock(endBB);
   }
+
+  // Leave the partial-array cleanup if we entered one.
+  if (dtorKind) CGF.DeactivateCleanupBlock(cleanup, cleanupDominator);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1376,8 +1374,9 @@ AggExprEmitter::VisitLambdaExpr(LambdaExpr *E) {
   LValue SlotLV = CGF.MakeAddrLValue(Slot.getAddress(), E->getType());
 
   // We'll need to enter cleanup scopes in case any of the element
-  // initializers throws an exception or contains branch out of the expressions.
-  CodeGenFunction::CleanupDeactivationScope scope(CGF);
+  // initializers throws an exception.
+  SmallVector<EHScopeStack::stable_iterator, 16> Cleanups;
+  llvm::Instruction *CleanupDominator = nullptr;
 
   CXXRecordDecl::field_iterator CurField = E->getLambdaClass()->field_begin();
   for (LambdaExpr::const_capture_init_iterator i = E->capture_init_begin(),
@@ -1396,12 +1395,28 @@ AggExprEmitter::VisitLambdaExpr(LambdaExpr *E) {
     if (QualType::DestructionKind DtorKind =
             CurField->getType().isDestructedType()) {
       assert(LV.isSimple());
-      if (DtorKind)
-        CGF.pushDestroyAndDeferDeactivation(
-            NormalAndEHCleanup, LV.getAddress(CGF), CurField->getType(),
-            CGF.getDestroyer(DtorKind), false);
+      if (CGF.needsEHCleanup(DtorKind)) {
+        if (!CleanupDominator)
+          CleanupDominator = CGF.Builder.CreateAlignedLoad(
+              CGF.Int8Ty,
+              llvm::Constant::getNullValue(CGF.Int8PtrTy),
+              CharUnits::One()); // placeholder
+
+        CGF.pushDestroy(EHCleanup, LV.getAddress(CGF), CurField->getType(),
+                        CGF.getDestroyer(DtorKind), false);
+        Cleanups.push_back(CGF.EHStack.stable_begin());
+      }
     }
   }
+
+  // Deactivate all the partial cleanups in reverse order, which
+  // generally means popping them.
+  for (unsigned i = Cleanups.size(); i != 0; --i)
+    CGF.DeactivateCleanupBlock(Cleanups[i-1], CleanupDominator);
+
+  // Destroy the placeholder if we made one.
+  if (CleanupDominator)
+    CleanupDominator->eraseFromParent();
 }
 
 void AggExprEmitter::VisitExprWithCleanups(ExprWithCleanups *E) {
@@ -1690,7 +1705,14 @@ void AggExprEmitter::VisitCXXParenListOrInitListExpr(
   // We'll need to enter cleanup scopes in case any of the element
   // initializers throws an exception.
   SmallVector<EHScopeStack::stable_iterator, 16> cleanups;
-  CodeGenFunction::CleanupDeactivationScope DeactivateCleanups(CGF);
+  llvm::Instruction *cleanupDominator = nullptr;
+  auto addCleanup = [&](const EHScopeStack::stable_iterator &cleanup) {
+    cleanups.push_back(cleanup);
+    if (!cleanupDominator) // create placeholder once needed
+      cleanupDominator = CGF.Builder.CreateAlignedLoad(
+          CGF.Int8Ty, llvm::Constant::getNullValue(CGF.Int8PtrTy),
+          CharUnits::One());
+  };
 
   unsigned curInitIndex = 0;
 
@@ -1713,8 +1735,10 @@ void AggExprEmitter::VisitCXXParenListOrInitListExpr(
       CGF.EmitAggExpr(InitExprs[curInitIndex++], AggSlot);
 
       if (QualType::DestructionKind dtorKind =
-              Base.getType().isDestructedType())
-        CGF.pushDestroyAndDeferDeactivation(dtorKind, V, Base.getType());
+              Base.getType().isDestructedType()) {
+        CGF.pushDestroy(dtorKind, V, Base.getType());
+        addCleanup(CGF.EHStack.stable_begin());
+      }
     }
   }
 
@@ -1789,10 +1813,10 @@ void AggExprEmitter::VisitCXXParenListOrInitListExpr(
     if (QualType::DestructionKind dtorKind
           = field->getType().isDestructedType()) {
       assert(LV.isSimple());
-      if (dtorKind) {
-        CGF.pushDestroyAndDeferDeactivation(
-            NormalAndEHCleanup, LV.getAddress(CGF), field->getType(),
-            CGF.getDestroyer(dtorKind), false);
+      if (CGF.needsEHCleanup(dtorKind)) {
+        CGF.pushDestroy(EHCleanup, LV.getAddress(CGF), field->getType(),
+                        CGF.getDestroyer(dtorKind), false);
+        addCleanup(CGF.EHStack.stable_begin());
         pushedCleanup = true;
       }
     }
@@ -1805,6 +1829,17 @@ void AggExprEmitter::VisitCXXParenListOrInitListExpr(
         if (GEP->use_empty())
           GEP->eraseFromParent();
   }
+
+  // Deactivate all the partial cleanups in reverse order, which
+  // generally means popping them.
+  assert((cleanupDominator || cleanups.empty()) &&
+         "Missing cleanupDominator before deactivating cleanup blocks");
+  for (unsigned i = cleanups.size(); i != 0; --i)
+    CGF.DeactivateCleanupBlock(cleanups[i-1], cleanupDominator);
+
+  // Destroy the placeholder if we made one.
+  if (cleanupDominator)
+    cleanupDominator->eraseFromParent();
 }
 
 void AggExprEmitter::VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E,
diff --git a/clang/lib/CodeGen/CGExprCXX.cpp b/clang/lib/CodeGen/CGExprCXX.cpp
index a88b29b326bb..a4fb673284ce 100644
--- a/clang/lib/CodeGen/CGExprCXX.cpp
+++ b/clang/lib/CodeGen/CGExprCXX.cpp
@@ -1008,8 +1008,8 @@ void CodeGenFunction::EmitNewArrayInitializer(
   const Expr *Init = E->getInitializer();
   Address EndOfInit = Address::invalid();
   QualType::DestructionKind DtorKind = ElementType.isDestructedType();
-  CleanupDeactivationScope deactivation(*this);
-  bool pushedCleanup = false;
+  EHScopeStack::stable_iterator Cleanup;
+  llvm::Instruction *CleanupDominator = nullptr;
 
   CharUnits ElementSize = getContext().getTypeSizeInChars(ElementType);
   CharUnits ElementAlign =
@@ -1105,24 +1105,19 @@ void CodeGenFunction::EmitNewArrayInitializer(
     }
 
     // Enter a partial-destruction Cleanup if necessary.
-    if (DtorKind) {
-      AllocaTrackerRAII AllocaTracker(*this);
+    if (needsEHCleanup(DtorKind)) {
       // In principle we could tell the Cleanup where we are more
       // directly, but the control flow can get so varied here that it
       // would actually be quite complex.  Therefore we go through an
       // alloca.
-      llvm::Instruction *DominatingIP =
-          Builder.CreateFlagLoad(llvm::ConstantInt::getNullValue(Int8PtrTy));
       EndOfInit = CreateTempAlloca(BeginPtr.getType(), getPointerAlign(),
                                    "array.init.end");
+      CleanupDominator =
+          Builder.CreateStore(BeginPtr.emitRawPointer(*this), EndOfInit);
       pushIrregularPartialArrayCleanup(BeginPtr.emitRawPointer(*this),
                                        EndOfInit, ElementType, ElementAlign,
                                        getDestroyer(DtorKind));
-      cast<EHCleanupScope>(*EHStack.find(EHStack.stable_begin()))
-          .AddAuxAllocas(AllocaTracker.Take());
-      DeferredDeactivationCleanupStack.push_back(
-          {EHStack.stable_begin(), DominatingIP});
-      pushedCleanup = true;
+      Cleanup = EHStack.stable_begin();
     }
 
     CharUnits StartAlign = CurPtr.getAlignment();
@@ -1169,6 +1164,9 @@ void CodeGenFunction::EmitNewArrayInitializer(
   // initialization.
   llvm::ConstantInt *ConstNum = dyn_cast<llvm::ConstantInt>(NumElements);
   if (ConstNum && ConstNum->getZExtValue() <= InitListElements) {
+    // If there was a Cleanup, deactivate it.
+    if (CleanupDominator)
+      DeactivateCleanupBlock(Cleanup, CleanupDominator);
     return;
   }
 
@@ -1283,14 +1281,13 @@ void CodeGenFunction::EmitNewArrayInitializer(
     Builder.CreateStore(CurPtr.emitRawPointer(*this), EndOfInit);
 
   // Enter a partial-destruction Cleanup if necessary.
-  if (!pushedCleanup && needsEHCleanup(DtorKind)) {
-    llvm::Instruction *DominatingIP =
-        Builder.CreateFlagLoad(llvm::ConstantInt::getNullValue(Int8PtrTy));
-    pushRegularPartialArrayCleanup(BeginPtr.emitRawPointer(*this),
-                                   CurPtr.emitRawPointer(*this), ElementType,
+  if (!CleanupDominator && needsEHCleanup(DtorKind)) {
+    llvm::Value *BeginPtrRaw = BeginPtr.emitRawPointer(*this);
+    llvm::Value *CurPtrRaw = CurPtr.emitRawPointer(*this);
+    pushRegularPartialArrayCleanup(BeginPtrRaw, CurPtrRaw, ElementType,
                                    ElementAlign, getDestroyer(DtorKind));
-    DeferredDeactivationCleanupStack.push_back(
-        {EHStack.stable_begin(), DominatingIP});
+    Cleanup = EHStack.stable_begin();
+    CleanupDominator = Builder.CreateUnreachable();
   }
 
   // Emit the initializer into this element.
@@ -1298,7 +1295,10 @@ void CodeGenFunction::EmitNewArrayInitializer(
                           AggValueSlot::DoesNotOverlap);
 
   // Leave the Cleanup if we entered one.
-  deactivation.ForceDeactivate();
+  if (CleanupDominator) {
+    DeactivateCleanupBlock(Cleanup, CleanupDominator);
+    CleanupDominator->eraseFromParent();
+  }
 
   // Advance to the next element by adjusting the pointer type as necessary.
   llvm::Value *NextPtr = Builder.CreateConstInBoundsGEP1_32(
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 87766a758311..86a6ddd80cc1 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -91,8 +91,6 @@ CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext)
 
 CodeGenFunction::~CodeGenFunction() {
   assert(LifetimeExtendedCleanupStack.empty() && "failed to emit a cleanup");
-  assert(DeferredDeactivationCleanupStack.empty() &&
-         "missed to deactivate a cleanup");
 
   if (getLangOpts().OpenMP && CurFn)
     CGM.getOpenMPRuntime().functionFinished(*this);
@@ -348,10 +346,6 @@ static void EmitIfUsed(CodeGenFunction &CGF, llvm::BasicBlock *BB) {
 void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
   assert(BreakContinueStack.empty() &&
          "mismatched push/pop in break/continue stack!");
-  assert(LifetimeExtendedCleanupStack.empty() &&
-         "mismatched push/pop of cleanups in EHStack!");
-  assert(DeferredDeactivationCleanupStack.empty() &&
-         "mismatched activate/deactivate of cleanups!");
 
   bool OnlySimpleReturnStmts = NumSimpleReturnExprs > 0
     && NumSimpleReturnExprs == NumReturnExprs
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index d99188671f1f..ff1873325d40 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -39,7 +39,6 @@
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
-#include "llvm/IR/Instructions.h"
 #include "llvm/IR/ValueHandle.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Transforms/Utils/SanitizerStats.h"
@@ -671,51 +670,6 @@ public:
 
   EHScopeStack EHStack;
   llvm::SmallVector<char, 256> LifetimeExtendedCleanupStack;
-
-  // A stack of cleanups which were added to EHStack but have to be deactivated
-  // later before being popped or emitted. These are usually deactivated on
-  // exiting a `CleanupDeactivationScope` scope. For instance, after a
-  // full-expr.
-  //
-  // These are specially useful for correctly emitting cleanups while
-  // encountering branches out of expression (through stmt-expr or coroutine
-  // suspensions).
-  struct DeferredDeactivateCleanup {
-    EHScopeStack::stable_iterator Cleanup;
-    llvm::Instruction *DominatingIP;
-  };
-  llvm::SmallVector<DeferredDeactivateCleanup> DeferredDeactivationCleanupStack;
-
-  // Enters a new scope for capturing cleanups which are deferred to be
-  // deactivated, all of which will be deactivated once the scope is exited.
-  struct CleanupDeactivationScope {
-    CodeGenFunction &CGF;
-    size_t OldDeactivateCleanupStackSize;
-    bool Deactivated;
-    CleanupDeactivationScope(CodeGenFunction &CGF)
-        : CGF(CGF), OldDeactivateCleanupStackSize(
-                        CGF.DeferredDeactivationCleanupStack.size()),
-          Deactivated(false) {}
-
-    void ForceDeactivate() {
-      assert(!Deactivated && "Deactivating already deactivated scope");
-      auto &Stack = CGF.DeferredDeactivationCleanupStack;
-      for (size_t I = Stack.size(); I > OldDeactivateCleanupStackSize; I--) {
-        CGF.DeactivateCleanupBlock(Stack[I - 1].Cleanup,
-                                   Stack[I - 1].DominatingIP);
-        Stack[I - 1].DominatingIP->eraseFromParent();
-      }
-      Stack.resize(OldDeactivateCleanupStackSize);
-      Deactivated = true;
-    }
-
-    ~CleanupDeactivationScope() {
-      if (Deactivated)
-        return;
-      ForceDeactivate();
-    }
-  };
-
   llvm::SmallVector<const JumpDest *, 2> SEHTryEpilogueStack;
 
   llvm::Instruction *CurrentFuncletPad = nullptr;
@@ -921,19 +875,6 @@ public:
       new (Buffer + sizeof(Header) + sizeof(T)) RawAddress(ActiveFlag);
   }
 
-  // Push a cleanup onto EHStack and deactivate it later. It is usually
-  // deactivated when exiting a `CleanupDeactivationScope` (for example: after a
-  // full expression).
-  template <class T, class... As>
-  void pushCleanupAndDeferDeactivation(CleanupKind Kind, As... A) {
-    // Placeholder dominating IP for this cleanup.
-    llvm::Instruction *DominatingIP =
-        Builder.CreateFlagLoad(llvm::Constant::getNullValue(Int8PtrTy));
-    EHStack.pushCleanup<T>(Kind, A...);
-    DeferredDeactivationCleanupStack.push_back(
-        {EHStack.stable_begin(), DominatingIP});
-  }
-
   /// Set up the last cleanup that was pushed as a conditional
   /// full-expression cleanup.
   void initFullExprCleanup() {
@@ -957,8 +898,7 @@ public:
 
   /// PopCleanupBlock - Will pop the cleanup entry on the stack and
   /// process all branch fixups.
-  void PopCleanupBlock(bool FallThroughIsBranchThrough = false,
-                       bool ForDeactivation = false);
+  void PopCleanupBlock(bool FallThroughIsBranchThrough = false);
 
   /// DeactivateCleanupBlock - Deactivates the given cleanup block.
   /// The block cannot be reactivated.  Pops it if it's the top of the
@@ -986,7 +926,6 @@ public:
   class RunCleanupsScope {
     EHScopeStack::stable_iterator CleanupStackDepth, OldCleanupScopeDepth;
     size_t LifetimeExtendedCleanupStackSize;
-    CleanupDeactivationScope DeactivateCleanups;
     bool OldDidCallStackSave;
   protected:
     bool PerformCleanup;
@@ -1001,7 +940,8 @@ public:
   public:
     /// Enter a new cleanup scope.
     explicit RunCleanupsScope(CodeGenFunction &CGF)
-        : DeactivateCleanups(CGF), PerformCleanup(true), CGF(CGF) {
+      : PerformCleanup(true), CGF(CGF)
+    {
       CleanupStackDepth = CGF.EHStack.stable_begin();
       LifetimeExtendedCleanupStackSize =
           CGF.LifetimeExtendedCleanupStack.size();
@@ -1031,7 +971,6 @@ public:
     void ForceCleanup(std::initializer_list<llvm::Value**> ValuesToReload = {}) {
       assert(PerformCleanup && "Already forced cleanup");
       CGF.DidCallStackSave = OldDidCallStackSave;
-      DeactivateCleanups.ForceDeactivate();
       CGF.PopCleanupBlocks(CleanupStackDepth, LifetimeExtendedCleanupStackSize,
                            ValuesToReload);
       PerformCleanup = false;
@@ -2221,11 +2160,6 @@ public:
                      Address addr, QualType type);
   void pushDestroy(CleanupKind kind, Address addr, QualType type,
                    Destroyer *destroyer, bool useEHCleanupForArray);
-  void pushDestroyAndDeferDeactivation(QualType::DestructionKind dtorKind,
-                                       Address addr, QualType type);
-  void pushDestroyAndDeferDeactivation(CleanupKind cleanupKind, Address addr,
-                                       QualType type, Destroyer *destroyer,
-                                       bool useEHCleanupForArray);
   void pushLifetimeExtendedDestroy(CleanupKind kind, Address addr,
                                    QualType type, Destroyer *destroyer,
                                    bool useEHCleanupForArray);
@@ -2764,33 +2698,6 @@ public:
                             TBAAAccessInfo *TBAAInfo = nullptr);
   LValue EmitLoadOfPointerLValue(Address Ptr, const PointerType *PtrTy);
 
-private:
-  struct AllocaTracker {
-    void Add(llvm::AllocaInst *I) { Allocas.push_back(I); }
-    llvm::SmallVector<llvm::AllocaInst *> Take() { return std::move(Allocas); }
-
-  private:
-    llvm::SmallVector<llvm::AllocaInst *> Allocas;
-  };
-  AllocaTracker *Allocas = nullptr;
-
-public:
-  // Captures all the allocas created during the scope of its RAII object.
-  struct AllocaTrackerRAII {
-    AllocaTrackerRAII(CodeGenFunction &CGF)
-        : CGF(CGF), OldTracker(CGF.Allocas) {
-      CGF.Allocas = &Tracker;
-    }
-    ~AllocaTrackerRAII() { CGF.Allocas = OldTracker; }
-
-    llvm::SmallVector<llvm::AllocaInst *> Take() { return Tracker.Take(); }
-
-  private:
-    CodeGenFunction &CGF;
-    AllocaTracker *OldTracker;
-    AllocaTracker Tracker;
-  };
-
   /// CreateTempAlloca - This creates an alloca and inserts it into the entry
   /// block if \p ArraySize is nullptr, otherwise inserts it at the current
   /// insertion point of the builder. The caller is responsible for setting an
diff --git a/clang/test/CodeGenCXX/control-flow-in-stmt-expr.cpp b/clang/test/CodeGenCXX/control-flow-in-stmt-expr.cpp
deleted file mode 100644
index 0a51b0e4121c..000000000000
--- a/clang/test/CodeGenCXX/control-flow-in-stmt-expr.cpp
+++ /dev/null
@@ -1,409 +0,0 @@
-// RUN: %clang_cc1 --std=c++20 -fexceptions -triple x86_64-linux-gnu -emit-llvm %s -o - | FileCheck -check-prefixes=EH %s
-// RUN: %clang_cc1 --std=c++20 -triple x86_64-linux-gnu -emit-llvm %s -o - | FileCheck -check-prefixes=NOEH,CHECK %s
-
-struct Printy {
-  Printy(const char *name) : name(name) {}
-  ~Printy() {}
-  const char *name;
-};
-
-int foo() { return 2; }
-
-struct Printies {
-  Printy a;
-  Printy b;
-  Printy c;
-};
-
-void ParenInit() {
-  // CHECK-LABEL: define dso_local void @_Z9ParenInitv()
-  // CHECK: [[CLEANUP_DEST:%.+]] = alloca i32, align 4
-  Printies ps(Printy("a"), 
-              // CHECK: call void @_ZN6PrintyC1EPKc
-              ({
-                if (foo()) return;
-                // CHECK:     if.then:
-                // CHECK-NEXT:   store i32 1, ptr [[CLEANUP_DEST]], align 4
-                // CHECK-NEXT:   br label %cleanup
-                Printy("b");
-                // CHECK:     if.end:
-                // CHECK-NEXT:  call void @_ZN6PrintyC1EPKc
-              }),
-              ({
-                if (foo()) return;
-                // CHECK:     if.then{{.*}}:
-                // CHECK-NEXT:  store i32 1, ptr [[CLEANUP_DEST]], align 4
-                // CHECK-NEXT:  call void @_ZN6PrintyD1Ev
-                // CHECK-NEXT:  br label %cleanup
-                Printy("c");
-                // CHECK:     if.end{{.*}}:
-                // CHECK-NEXT:  call void @_ZN6PrintyC1EPKc
-                // CHECK-NEXT:  call void @_ZN8PrintiesD1Ev
-                // CHECK-NEXT:  br label %return
-              }));
-  // CHECK:     cleanup:
-  // CHECK-NEXT:  call void @_ZN6PrintyD1Ev
-  // CHECK-NEXT:  br label %return
-}
-
-void break_in_stmt_expr() {
-  // Verify that the "break" in "if.then".calls dtor before jumping to "for.end".
-
-  // CHECK-LABEL: define dso_local void @_Z18break_in_stmt_exprv()
-  Printies p{Printy("a"), 
-            // CHECK: call void @_ZN6PrintyC1EPKc
-            ({
-                for (;;) {
-                    Printies ps{
-                      Printy("b"), 
-                      // CHECK: for.cond:
-                      // CHECK:   call void @_ZN6PrintyC1EPKc
-                      ({
-                        if (foo()) {
-                          break;
-                          // CHECK:       if.then:
-                          // CHECK-NEXT:    call void @_ZN6PrintyD1Ev
-                          // CHECK-NEXT:    br label %for.end
-                        }
-                        Printy("c");
-                        // CHECK:       if.end:
-                        // CHECK-NEXT:    call void @_ZN6PrintyC1EPKc
-                      }),
-                      Printy("d")};
-                      // CHECK:           call void @_ZN6PrintyC1EPKc
-                      // CHECK-NEXT:      call void @_ZN8PrintiesD1Ev
-                      // CHECK-NEXT:      br label %for.cond
-                }
-                Printy("e");
-  // CHECK:       for.end:
-  // CHECK-NEXT:    call void @_ZN6PrintyC1EPKc
-              }),
-              Printy("f")};
-  // CHECK:         call void @_ZN6PrintyC1EPKc
-  // CHECK-NEXT:    call void @_ZN8PrintiesD1Ev
-}
-
-void goto_in_stmt_expr() {
-  // Verify that:
-  //  - correct branch fixups for deactivated normal cleanups are generated correctly.
-
-  // CHECK-LABEL: define dso_local void @_Z17goto_in_stmt_exprv()
-  // CHECK: [[CLEANUP_DEST_SLOT:%cleanup.dest.slot.*]] = alloca i32, align 4
-  {
-    Printies p1{Printy("a"), // CHECK: call void @_ZN6PrintyC1EPKc
-                ({
-                  {
-                    Printies p2{Printy("b"),
-                                // CHECK: call void @_ZN6PrintyC1EPKc
-                                ({
-                                  if (foo() == 1) {
-                                    goto in;
-                                    // CHECK:       if.then:
-                                    // CHECK-NEXT:    store i32 2, ptr [[CLEANUP_DEST_SLOT]], align 4
-                                    // CHECK-NEXT:    br label %[[CLEANUP1:.+]]
-                                  }
-                                  if (foo() == 2) {
-                                    goto out;
-                                    // CHECK:       if.then{{.*}}:
-                                    // CHECK-NEXT:    store i32 3, ptr [[CLEANUP_DEST_SLOT]], align 4
-                                    // CHECK-NEXT:    br label %[[CLEANUP1]]
-                                  }
-                                  Printy("c");
-                                  // CHECK:       if.end{{.*}}:
-                                  // CHECK-NEXT:    call void @_ZN6PrintyC1EPKc
-                                }),
-                                Printy("d")};
-                                // CHECK:           call void @_ZN6PrintyC1EPKc
-                                // CHECK-NEXT:      call void @_ZN8PrintiesD1Ev
-                                // CHECK-NEXT:      br label %in
-
-                  }
-                in:
-                  Printy("e");
-                // CHECK:       in:                                               ; preds = %if.end{{.*}}, %[[CLEANUP1]]
-                // CHECK-NEXT:    call void @_ZN6PrintyC1EPKc
-                }),
-                Printy("f")};
-                // CHECK:         call void @_ZN6PrintyC1EPKc
-                // CHECK-NEXT:    call void @_ZN8PrintiesD1Ev
-                // CHECK-NEXT:    br label %out
-  }
-out:
-  return;
-  // CHECK:       out:
-  // CHECK-NEXT:    ret void
-
-  // CHECK:       [[CLEANUP1]]:                                          ; preds = %if.then{{.*}}, %if.then
-  // CHECK-NEXT:    call void @_ZN6PrintyD1Ev
-  // CHECK-NEXT:    %cleanup.dest = load i32, ptr [[CLEANUP_DEST_SLOT]], align 4
-  // CHECK-NEXT:    switch i32 %cleanup.dest, label %[[CLEANUP2:.+]] [
-  // CHECK-NEXT:      i32 2, label %in
-  // CHECK-NEXT:    ]
-
-  // CHECK:       [[CLEANUP2]]:                                         ; preds = %[[CLEANUP1]]
-  // CHECK-NEXT:    call void @_ZN6PrintyD1Ev
-  // CHECK-NEXT:    %cleanup.dest{{.*}} = load i32, ptr [[CLEANUP_DEST_SLOT]], align 4
-  // CHECK-NEXT:    switch i32 %cleanup.dest{{.*}}, label %unreachable [
-  // CHECK-NEXT:      i32 3, label %out
-  // CHECK-NEXT:    ]
-}
-
-void ArrayInit() {
-  // Printy arr[4] = {ctorA, ctorB, stmt-exprC, stmt-exprD};
-  // Verify that:
-  //  - We do the necessary stores for array cleanups (endOfInit and last constructed element).
-  //  - We update the array init element correctly for ctorA, ctorB and stmt-exprC.
-  //  - stmt-exprC and stmt-exprD share the array body dtor code (see %cleanup).
-
-  // CHECK-LABEL: define dso_local void @_Z9ArrayInitv()
-  // CHECK: %arrayinit.endOfInit = alloca ptr, align 8
-  // CHECK: %cleanup.dest.slot = alloca i32, align 4
-  // CHECK: %arrayinit.begin = getelementptr inbounds [4 x %struct.Printy], ptr %arr, i64 0, i64 0
-  // CHECK: store ptr %arrayinit.begin, ptr %arrayinit.endOfInit, align 8
-  Printy arr[4] = {
-    Printy("a"),
-    // CHECK: call void @_ZN6PrintyC1EPKc(ptr noundef nonnull align 8 dereferenceable(8) %arrayinit.begin, ptr noundef @.str)
-    // CHECK: [[ARRAYINIT_ELEMENT1:%.+]] = getelementptr inbounds %struct.Printy, ptr %arrayinit.begin, i64 1
-    // CHECK: store ptr [[ARRAYINIT_ELEMENT1]], ptr %arrayinit.endOfInit, align 8
-    Printy("b"),
-    // CHECK: call void @_ZN6PrintyC1EPKc(ptr noundef nonnull align 8 dereferenceable(8) [[ARRAYINIT_ELEMENT1]], ptr noundef @.str.1)
-    // CHECK: [[ARRAYINIT_ELEMENT2:%.+]] = getelementptr inbounds %struct.Printy, ptr [[ARRAYINIT_ELEMENT1]], i64 1
-    // CHECK: store ptr [[ARRAYINIT_ELEMENT2]], ptr %arrayinit.endOfInit, align 8
-    ({
-    // CHECK: br i1 {{.*}}, label %if.then, label %if.end
-      if (foo()) {
-        return;
-      // CHECK:       if.then:
-      // CHECK-NEXT:    store i32 1, ptr %cleanup.dest.slot, align 4
-      // CHECK-NEXT:    br label %cleanup
-      }
-      // CHECK:       if.end:
-      Printy("c");
-      // CHECK-NEXT:    call void @_ZN6PrintyC1EPKc
-      // CHECK-NEXT:    %arrayinit.element2 = getelementptr inbounds %struct.Printy, ptr %arrayinit.element1, i64 1
-      // CHECK-NEXT:    store ptr %arrayinit.element2, ptr %arrayinit.endOfInit, align 8
-    }),
-    ({
-    // CHECK: br i1 {{%.+}} label %[[IF_THEN2:.+]], label %[[IF_END2:.+]]
-      if (foo()) {
-        return;
-      // CHECK:       [[IF_THEN2]]:
-      // CHECK-NEXT:    store i32 1, ptr %cleanup.dest.slot, align 4
-      // CHECK-NEXT:    br label %cleanup
-      }
-      // CHECK:       [[IF_END2]]:
-      Printy("d");
-      // CHECK-NEXT:    call void @_ZN6PrintyC1EPKc
-      // CHECK-NEXT:    %array.begin = getelementptr inbounds [4 x %struct.Printy], ptr %arr, i32 0, i32 0
-      // CHECK-NEXT:    %0 = getelementptr inbounds %struct.Printy, ptr %array.begin, i64 4
-      // CHECK-NEXT:    br label %[[ARRAY_DESTROY_BODY1:.+]]
-  }),
-  };
-
-  // CHECK:       [[ARRAY_DESTROY_BODY1]]:
-  // CHECK-NEXT:    %arraydestroy.elementPast{{.*}} = phi ptr [ %0, %[[IF_END2]] ], [ %arraydestroy.element{{.*}}, %[[ARRAY_DESTROY_BODY1]] ]
-  // CHECK-NEXT:    %arraydestroy.element{{.*}} = getelementptr inbounds %struct.Printy, ptr %arraydestroy.elementPast{{.*}}, i64 -1
-  // CHECK-NEXT:    call void @_ZN6PrintyD1Ev
-  // CHECK-NEXT:    %arraydestroy.done{{.*}} = icmp eq ptr %arraydestroy.element{{.*}}, %array.begin
-  // CHECK-NEXT:    br i1 %arraydestroy.done{{.*}}, label %[[ARRAY_DESTROY_DONE1:.+]], label %[[ARRAY_DESTROY_BODY1]]
-
-  // CHECK:       [[ARRAY_DESTROY_DONE1]]:
-  // CHECK-NEXT:    ret void
-
-  // CHECK:       cleanup:
-  // CHECK-NEXT:    %1 = load ptr, ptr %arrayinit.endOfInit, align 8
-  // CHECK-NEXT:    %arraydestroy.isempty = icmp eq ptr %arrayinit.begin, %1
-  // CHECK-NEXT:    br i1 %arraydestroy.isempty, label %[[ARRAY_DESTROY_DONE2:.+]], label %[[ARRAY_DESTROY_BODY2:.+]]
-
-  // CHECK:       [[ARRAY_DESTROY_BODY2]]:
-  // CHECK-NEXT:    %arraydestroy.elementPast = phi ptr [ %1, %cleanup ], [ %arraydestroy.element, %[[ARRAY_DESTROY_BODY2]] ]
-  // CHECK-NEXT:    %arraydestroy.element = getelementptr inbounds %struct.Printy, ptr %arraydestroy.elementPast, i64 -1
-  // CHECK-NEXT:    call void @_ZN6PrintyD1Ev(ptr noundef nonnull align 8 dereferenceable(8) %arraydestroy.element)
-  // CHECK-NEXT:    %arraydestroy.done = icmp eq ptr %arraydestroy.element, %arrayinit.begin
-  // CHECK-NEXT:    br i1 %arraydestroy.done, label %[[ARRAY_DESTROY_DONE2]], label %[[ARRAY_DESTROY_BODY2]]
-
-  // CHECK:       [[ARRAY_DESTROY_DONE2]]:
-  // CHECK-NEXT:    br label %[[ARRAY_DESTROY_DONE1]]
-}
-
-void ArraySubobjects() {
-  struct S {
-    Printy arr1[2];
-    Printy arr2[2];
-    Printy p;
-  };
-  // CHECK-LABEL: define dso_local void @_Z15ArraySubobjectsv()
-  // CHECK: %arrayinit.endOfInit = alloca ptr, align 8
-  S s{{Printy("a"), Printy("b")},
-      // CHECK: call void @_ZN6PrintyC1EPKc
-      // CHECK: call void @_ZN6PrintyC1EPKc
-      {Printy("a"),
-      // CHECK: [[ARRAYINIT_BEGIN:%.+]] = getelementptr inbounds [2 x %struct.Printy]
-      // CHECK: store ptr [[ARRAYINIT_BEGIN]], ptr %arrayinit.endOfInit, align 8
-      // CHECK: call void @_ZN6PrintyC1EPKc
-      // CHECK: [[ARRAYINIT_ELEMENT:%.+]] = getelementptr inbounds %struct.Printy
-      // CHECK: store ptr [[ARRAYINIT_ELEMENT]], ptr %arrayinit.endOfInit, align 8
-      ({
-         if (foo()) {
-           return;
-           // CHECK:      if.then:
-           // CHECK-NEXT:   [[V0:%.+]] = load ptr, ptr %arrayinit.endOfInit, align 8
-           // CHECK-NEXT:   %arraydestroy.isempty = icmp eq ptr [[ARRAYINIT_BEGIN]], [[V0]]
-           // CHECK-NEXT:   br i1 %arraydestroy.isempty, label %[[ARRAY_DESTROY_DONE:.+]], label %[[ARRAY_DESTROY_BODY:.+]]
-         }
-         Printy("b");
-       })
-      },
-      Printy("c")
-      // CHECK:       if.end:
-      // CHECK-NEXT:    call void @_ZN6PrintyC1EPKc
-      // CHECK:         call void @_ZN6PrintyC1EPKc
-      // CHECK-NEXT:    call void @_ZZ15ArraySubobjectsvEN1SD1Ev
-      // CHECK-NEXT:    br label %return
-    };
-    // CHECK:       return:
-    // CHECK-NEXT:    ret void
-
-    // CHECK:       [[ARRAY_DESTROY_BODY]]:
-    // CHECK-NEXT:    %arraydestroy.elementPast = phi ptr [ %0, %if.then ], [ %arraydestroy.element, %[[ARRAY_DESTROY_BODY]] ]
-    // CHECK-NEXT:    %arraydestroy.element = getelementptr inbounds %struct.Printy, ptr %arraydestroy.elementPast, i64 -1
-    // CHECK-NEXT:    call void @_ZN6PrintyD1Ev(ptr noundef nonnull align 8 dereferenceable(8) %arraydestroy.element)
-    // CHECK-NEXT:    %arraydestroy.done = icmp eq ptr %arraydestroy.element, [[ARRAYINIT_BEGIN]]
-    // CHECK-NEXT:    br i1 %arraydestroy.done, label %[[ARRAY_DESTROY_DONE]], label %[[ARRAY_DESTROY_BODY]]
-
-    // CHECK:       [[ARRAY_DESTROY_DONE]]
-    // CHECK-NEXT:    [[ARRAY_BEGIN:%.+]] = getelementptr inbounds [2 x %struct.Printy], ptr %arr1, i32 0, i32 0
-    // CHECK-NEXT:    [[V1:%.+]] = getelementptr inbounds %struct.Printy, ptr [[ARRAY_BEGIN]], i64 2
-    // CHECK-NEXT:    br label %[[ARRAY_DESTROY_BODY2:.+]]
-
-    // CHECK:       [[ARRAY_DESTROY_BODY2]]:
-    // CHECK-NEXT:    %arraydestroy.elementPast5 = phi ptr [ %1, %[[ARRAY_DESTROY_DONE]] ], [ %arraydestroy.element6, %[[ARRAY_DESTROY_BODY2]] ]
-    // CHECK-NEXT:    %arraydestroy.element6 = getelementptr inbounds %struct.Printy, ptr %arraydestroy.elementPast5, i64 -1
-    // CHECK-NEXT:    call void @_ZN6PrintyD1Ev(ptr noundef nonnull align 8 dereferenceable(8) %arraydestroy.element6)
-    // CHECK-NEXT:    %arraydestroy.done7 = icmp eq ptr %arraydestroy.element6, [[ARRAY_BEGIN]]
-    // CHECK-NEXT:    br i1 %arraydestroy.done7, label %[[ARRAY_DESTROY_DONE2:.+]], label %[[ARRAY_DESTROY_BODY2]]
-
-
-    // CHECK:     [[ARRAY_DESTROY_DONE2]]:
-    // CHECK-NEXT:  br label %return
-}
-
-void LambdaInit() {
-  // CHECK-LABEL: define dso_local void @_Z10LambdaInitv()
-  auto S = [a = Printy("a"), b = ({
-                               if (foo()) {
-                                 return;
-                                 // CHECK:       if.then:
-                                 // CHECK-NEXT:    call void @_ZN6PrintyD1Ev
-                                 // CHECK-NEXT:    br label %return
-                               }
-                               Printy("b");
-                             })]() { return a; };
-}
-
-void LifetimeExtended() {
-  // CHECK-LABEL: define dso_local void @_Z16LifetimeExtendedv
-  struct PrintyRefBind {
-    const Printy &a;
-    const Printy &b;
-  };
-  PrintyRefBind ps = {Printy("a"), ({
-                        if (foo()) {
-                          return;
-                          // CHECK: if.then:
-                          // CHECK-NEXT: call void @_ZN6PrintyD1Ev
-                          // CHECK-NEXT: br label %return
-                        }
-                        Printy("b");
-                      })};
-}
-
-void NewArrayInit() {
-  // CHECK-LABEL: define dso_local void @_Z12NewArrayInitv()
-  // CHECK: %array.init.end = alloca ptr, align 8
-  // CHECK: store ptr %0, ptr %array.init.end, align 8
-  Printy *array = new Printy[3]{
-    "a",
-    // CHECK: call void @_ZN6PrintyC1EPKc
-    // CHECK: store ptr %array.exp.next, ptr %array.init.end, align 8
-    "b", 
-    // CHECK: call void @_ZN6PrintyC1EPKc
-    // CHECK: store ptr %array.exp.next1, ptr %array.init.end, align 8
-    ({
-        if (foo()) {
-          return;
-          // CHECK: if.then:
-          // CHECK:   br i1 %arraydestroy.isempty, label %arraydestroy.done{{.*}}, label %arraydestroy.body
-        }
-        "b";
-        // CHECK: if.end:
-        // CHECK:   call void @_ZN6PrintyC1EPKc
-    })};
-  // CHECK:       arraydestroy.body:
-  // CHECK-NEXT:    %arraydestroy.elementPast = phi ptr [ %{{.*}}, %if.then ], [ %arraydestroy.element, %arraydestroy.body ]
-  // CHECK-NEXT:    %arraydestroy.element = getelementptr inbounds %struct.Printy, ptr %arraydestroy.elementPast, i64 -1
-  // CHECK-NEXT:    call void @_ZN6PrintyD1Ev(ptr noundef nonnull align 8 dereferenceable(8) %arraydestroy.element)
-  // CHECK-NEXT:    %arraydestroy.done = icmp eq ptr %arraydestroy.element, %0
-  // CHECK-NEXT:    br i1 %arraydestroy.done, label %arraydestroy.done{{.*}}, label %arraydestroy.body
-
-  // CHECK:       arraydestroy.done{{.*}}:                               ; preds = %arraydestroy.body, %if.then
-  // CHECK-NEXT:    br label %return
-}
-
-void DestroyInConditionalCleanup() {
-  // EH-LABEL: DestroyInConditionalCleanupv()
-  // NOEH-LABEL: DestroyInConditionalCleanupv()
-  struct A {
-    A() {}
-    ~A() {}
-  };
-
-  struct Value {
-    Value(A) {}
-    ~Value() {}
-  };
-
-  struct V2 {
-    Value K;
-    Value V;
-  };
-  // Verify we use conditional cleanups.
-  (void)(foo() ? V2{A(), A()} : V2{A(), A()});
-  // NOEH:   cond.true:
-  // NOEH:      call void @_ZZ27DestroyInConditionalCleanupvEN1AC1Ev
-  // NOEH:      store ptr %{{.*}}, ptr %cond-cleanup.save
-
-  // EH:   cond.true:
-  // EH:        invoke void @_ZZ27DestroyInConditionalCleanupvEN1AC1Ev
-  // EH:        store ptr %{{.*}}, ptr %cond-cleanup.save
-}
-
-void ArrayInitWithContinue() {
-  // CHECK-LABEL: @_Z21ArrayInitWithContinuev
-  // Verify that we start to emit the array destructor.
-  // CHECK: %arrayinit.endOfInit = alloca ptr, align 8
-  for (int i = 0; i < 1; ++i) {
-    Printy arr[2] = {"a", ({
-                       if (foo()) {
-                         continue;
-                       }
-                       "b";
-                     })};
-  }
-}
-
-struct [[clang::trivial_abi]] HasTrivialABI {
-  HasTrivialABI();
-  ~HasTrivialABI();
-};
-void AcceptTrivialABI(HasTrivialABI, int);
-void TrivialABI() {
-  // CHECK-LABEL: define dso_local void @_Z10TrivialABIv()
-  AcceptTrivialABI(HasTrivialABI(), ({
-                     if (foo()) return;
-                     // CHECK:      if.then:
-                     // CHECK-NEXT:   call void @_ZN13HasTrivialABID1Ev
-                     // CHECK-NEXT:   br label %return
-                     0;
-                   }));
-}
diff --git a/clang/test/CodeGenCoroutines/coro-suspend-cleanups.cpp b/clang/test/CodeGenCoroutines/coro-suspend-cleanups.cpp
deleted file mode 100644
index 06cc2069dbe9..000000000000
--- a/clang/test/CodeGenCoroutines/coro-suspend-cleanups.cpp
+++ /dev/null
@@ -1,93 +0,0 @@
-// RUN: %clang_cc1 --std=c++20 -triple x86_64-linux-gnu -emit-llvm %s -o - | FileCheck %s
-
-#include "Inputs/coroutine.h"
-
-struct Printy {
-  Printy(const char *name) : name(name) {}
-  ~Printy() {}
-  const char *name;
-};
-
-struct coroutine {
-  struct promise_type;
-  std::coroutine_handle<promise_type> handle;
-  ~coroutine() {
-    if (handle) handle.destroy();
-  }
-};
-
-struct coroutine::promise_type {
-  coroutine get_return_object() {
-    return {std::coroutine_handle<promise_type>::from_promise(*this)};
-  }
-  std::suspend_never initial_suspend() noexcept { return {}; }
-  std::suspend_always final_suspend() noexcept { return {}; }
-  void return_void() {}
-  void unhandled_exception() {}
-};
-
-struct Awaiter : std::suspend_always {
-  Printy await_resume() { return {"awaited"}; }
-};
-
-int foo() { return 2; }
-
-coroutine ArrayInitCoro() {
-  // Verify that:
-  //  - We do the necessary stores for array cleanups.
-  //  - Array cleanups are called by await.cleanup.
-  //  - We activate the cleanup after the first element and deactivate it in await.ready (see cleanup.isactive).
-
-  // CHECK-LABEL: define dso_local void @_Z13ArrayInitCorov
-  // CHECK: %arrayinit.endOfInit = alloca ptr, align 8
-  // CHECK: %cleanup.isactive = alloca i1, align 1
-  Printy arr[2] = {
-    Printy("a"),
-    // CHECK:       %arrayinit.begin = getelementptr inbounds [2 x %struct.Printy], ptr %arr.reload.addr, i64 0, i64 0
-    // CHECK-NEXT:  %arrayinit.begin.spill.addr = getelementptr inbounds %_Z13ArrayInitCorov.Frame, ptr %0, i32 0, i32 10
-    // CHECK-NEXT:  store ptr %arrayinit.begin, ptr %arrayinit.begin.spill.addr, align 8
-    // CHECK-NEXT:  store i1 true, ptr %cleanup.isactive.reload.addr, align 1
-    // CHECK-NEXT:  store ptr %arrayinit.begin, ptr %arrayinit.endOfInit.reload.addr, align 8
-    // CHECK-NEXT:  call void @_ZN6PrintyC1EPKc(ptr noundef nonnull align 8 dereferenceable(8) %arrayinit.begin, ptr noundef @.str)
-    // CHECK-NEXT:  %arrayinit.element = getelementptr inbounds %struct.Printy, ptr %arrayinit.begin, i64 1
-    // CHECK-NEXT:  %arrayinit.element.spill.addr = getelementptr inbounds %_Z13ArrayInitCorov.Frame, ptr %0, i32 0, i32 11
-    // CHECK-NEXT:  store ptr %arrayinit.element, ptr %arrayinit.element.spill.addr, align 8
-    // CHECK-NEXT:  store ptr %arrayinit.element, ptr %arrayinit.endOfInit.reload.addr, align 8
-    co_await Awaiter{}
-    // CHECK-NEXT:  @_ZNSt14suspend_always11await_readyEv
-    // CHECK-NEXT:  br i1 %{{.+}}, label %await.ready, label %CoroSave30
-  };
-  // CHECK:       await.cleanup:                                    ; preds = %AfterCoroSuspend{{.*}}
-  // CHECK-NEXT:    br label %cleanup{{.*}}.from.await.cleanup
-
-  // CHECK:       cleanup{{.*}}.from.await.cleanup:                      ; preds = %await.cleanup
-  // CHECK:         br label %cleanup{{.*}}
-
-  // CHECK:       await.ready:
-  // CHECK-NEXT:    %arrayinit.element.reload.addr = getelementptr inbounds %_Z13ArrayInitCorov.Frame, ptr %0, i32 0, i32 11
-  // CHECK-NEXT:    %arrayinit.element.reload = load ptr, ptr %arrayinit.element.reload.addr, align 8
-  // CHECK-NEXT:    call void @_ZN7Awaiter12await_resumeEv
-  // CHECK-NEXT:    store i1 false, ptr %cleanup.isactive.reload.addr, align 1
-  // CHECK-NEXT:    br label %cleanup{{.*}}.from.await.ready
-
-  // CHECK:       cleanup{{.*}}:                                         ; preds = %cleanup{{.*}}.from.await.ready, %cleanup{{.*}}.from.await.cleanup
-  // CHECK:         %cleanup.is_active = load i1, ptr %cleanup.isactive.reload.addr, align 1
-  // CHECK-NEXT:    br i1 %cleanup.is_active, label %cleanup.action, label %cleanup.done
-
-  // CHECK:       cleanup.action:
-  // CHECK:         %arraydestroy.isempty = icmp eq ptr %arrayinit.begin.reload{{.*}}, %{{.*}}
-  // CHECK-NEXT:    br i1 %arraydestroy.isempty, label %arraydestroy.done{{.*}}, label %arraydestroy.body.from.cleanup.action
-  // Ignore rest of the array cleanup.
-}
-
-coroutine ArrayInitWithCoReturn() {
-  // CHECK-LABEL: define dso_local void @_Z21ArrayInitWithCoReturnv
-  // Verify that we start to emit the array destructor.
-  // CHECK: %arrayinit.endOfInit = alloca ptr, align 8
-  Printy arr[2] = {"a", ({
-                      if (foo()) {
-                        co_return;
-                      }
-                      "b";
-                    })};
-}
-- 
GitLab


From 9ce74d6d470437cde82e91631395322ec1c36eeb Mon Sep 17 00:00:00 2001
From: Shilei Tian <i@tianshilei.me>
Date: Tue, 16 Apr 2024 09:34:18 -0400
Subject: [PATCH 400/464] [AMDGPU][CodeGen] Improve handling of memcpy for
 -Os/-Oz compilations (#87632)

We had some instances when LLVM would not inline fixed-count memcpy and
ended up
attempting to lower it a a libcall, which would not work on AMDGPU as
the
address space doesn't meet the requirement, causing compiler crash.

The patch relaxes the threshold used for -Os/-Oz compilation so we're
always allowed
to inline memory copy functions.

This patch basically does the same thing as
https://reviews.llvm.org/D158226 for
AMDGPU.

Fix #88497.
---
 llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp |    6 +
 llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll    | 2696 +++++++++++++++++
 2 files changed, 2702 insertions(+)
 create mode 100644 llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index f283af6fa07d..db69d50799e7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -59,6 +59,12 @@ unsigned AMDGPUTargetLowering::numBitsSigned(SDValue Op, SelectionDAG &DAG) {
 AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
                                            const AMDGPUSubtarget &STI)
     : TargetLowering(TM), Subtarget(&STI) {
+  // Always lower memset, memcpy, and memmove intrinsics to load/store
+  // instructions, rather then generating calls to memset, mempcy or memmove.
+  MaxStoresPerMemset = MaxStoresPerMemsetOptSize = ~0U;
+  MaxStoresPerMemcpy = MaxStoresPerMemcpyOptSize = ~0U;
+  MaxStoresPerMemmove = MaxStoresPerMemmoveOptSize = ~0U;
+
   // Lower floating point store/load to integer store/load to reduce the number
   // of patterns in tablegen.
   setOperationAction(ISD::LOAD, MVT::f32, Promote);
diff --git a/llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll b/llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll
new file mode 100644
index 000000000000..358f42dfe8dd
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll
@@ -0,0 +1,2696 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx908 %s -o - | FileCheck %s
+
+%struct.S = type { [32 x i32] }
+
+@shared = addrspace(3) global %struct.S undef, align 4
+
+define amdgpu_kernel void @memcpy_p0_p0_minsize(ptr %dest, ptr readonly %src) #0 {
+; CHECK-LABEL: memcpy_p0_p0_minsize:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v0, s2
+; CHECK-NEXT:    v_mov_b32_e32 v1, s3
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1]
+; CHECK-NEXT:    v_mov_b32_e32 v3, s1
+; CHECK-NEXT:    v_mov_b32_e32 v2, s0
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:1
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:1
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:2
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:2
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:3
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:3
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:4
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:4
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:5
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:5
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:6
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:6
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:7
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:7
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:8
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:8
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:9
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:9
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:10
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:10
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:11
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:11
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:12
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:12
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:13
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:13
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:14
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:14
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:15
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:15
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:16
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:17
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:17
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:18
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:18
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:19
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:19
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:20
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:20
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:21
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:21
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:22
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:22
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:23
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:23
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:24
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:24
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:25
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:25
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:26
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:26
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:27
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:27
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:28
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:28
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:29
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:29
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:30
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:30
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:31
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:31
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:32
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:32
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:33
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:33
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:34
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:34
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:35
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:35
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:36
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:36
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:37
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:37
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:38
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:38
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:39
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:39
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:40
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:40
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:41
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:41
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:42
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:42
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:43
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:43
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:44
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:44
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:45
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:45
+; CHECK-NEXT:    flat_load_ubyte v0, v[0:1] offset:46
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v0 offset:46
+; CHECK-NEXT:    s_endpgm
+entry:
+  tail call void @llvm.memcpy.p0.p0.i64(ptr %dest, ptr %src, i64 47, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_p1_p1_minsize(ptr addrspace(1) %dest, ptr addrspace(1) %src) #0 {
+; CHECK-LABEL: memcpy_p1_p1_minsize:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CHECK-NEXT:    v_mov_b32_e32 v4, 0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    global_load_dwordx2 v[0:1], v4, s[2:3] offset:32
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    global_store_dwordx2 v4, v[0:1], s[0:1] offset:32
+; CHECK-NEXT:    global_load_dwordx2 v[0:1], v4, s[2:3] offset:39
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    global_store_dwordx2 v4, v[0:1], s[0:1] offset:39
+; CHECK-NEXT:    global_load_dwordx4 v[0:3], v4, s[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
+; CHECK-NEXT:    global_load_dwordx4 v[0:3], v4, s[2:3] offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
+; CHECK-NEXT:    s_endpgm
+entry:
+  tail call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 47, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_p1_p4_minsize(ptr addrspace(1) %global, ptr addrspace(4) %0) #0 {
+; CHECK-LABEL: memcpy_p1_p4_minsize:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CHECK-NEXT:    v_mov_b32_e32 v4, 0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    global_load_dwordx4 v[0:3], v4, s[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
+; CHECK-NEXT:    global_load_dwordx4 v[0:3], v4, s[2:3] offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
+; CHECK-NEXT:    global_load_dwordx4 v[0:3], v4, s[2:3] offset:32
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:32
+; CHECK-NEXT:    global_load_dwordx4 v[0:3], v4, s[2:3] offset:48
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:48
+; CHECK-NEXT:    global_load_dwordx4 v[0:3], v4, s[2:3] offset:64
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:64
+; CHECK-NEXT:    global_load_dwordx4 v[0:3], v4, s[2:3] offset:80
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:80
+; CHECK-NEXT:    global_load_dwordx4 v[0:3], v4, s[2:3] offset:96
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:96
+; CHECK-NEXT:    global_load_dwordx4 v[0:3], v4, s[2:3] offset:112
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:112
+; CHECK-NEXT:    s_endpgm
+entry:
+  tail call void @llvm.memcpy.p1.p4.i64(ptr addrspace(1) %global, ptr addrspace(4) %0, i64 128, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_p5_p4_minsize(ptr addrspace(5) %local, ptr addrspace(4) %0) #0 {
+; CHECK-LABEL: memcpy_p5_p4_minsize:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_mov_b64 s[10:11], s[2:3]
+; CHECK-NEXT:    s_mov_b64 s[8:9], s[0:1]
+; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; CHECK-NEXT:    s_load_dword s2, s[4:5], 0x0
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    s_add_u32 s8, s8, s7
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    global_load_ubyte v2, v0, s[0:1]
+; CHECK-NEXT:    global_load_ubyte v3, v0, s[0:1] offset:1
+; CHECK-NEXT:    global_load_ubyte v4, v0, s[0:1] offset:2
+; CHECK-NEXT:    global_load_ubyte v5, v0, s[0:1] offset:3
+; CHECK-NEXT:    global_load_ubyte v6, v0, s[0:1] offset:4
+; CHECK-NEXT:    global_load_ubyte v7, v0, s[0:1] offset:5
+; CHECK-NEXT:    global_load_ubyte v8, v0, s[0:1] offset:6
+; CHECK-NEXT:    global_load_ubyte v9, v0, s[0:1] offset:7
+; CHECK-NEXT:    global_load_ubyte v10, v0, s[0:1] offset:8
+; CHECK-NEXT:    global_load_ubyte v11, v0, s[0:1] offset:9
+; CHECK-NEXT:    global_load_ubyte v12, v0, s[0:1] offset:10
+; CHECK-NEXT:    global_load_ubyte v13, v0, s[0:1] offset:11
+; CHECK-NEXT:    global_load_ubyte v14, v0, s[0:1] offset:12
+; CHECK-NEXT:    global_load_ubyte v15, v0, s[0:1] offset:13
+; CHECK-NEXT:    global_load_ubyte v16, v0, s[0:1] offset:14
+; CHECK-NEXT:    global_load_ubyte v17, v0, s[0:1] offset:15
+; CHECK-NEXT:    s_addc_u32 s9, s9, 0
+; CHECK-NEXT:    v_mov_b32_e32 v1, s2
+; CHECK-NEXT:    global_load_ubyte v18, v0, s[0:1] offset:16
+; CHECK-NEXT:    global_load_ubyte v19, v0, s[0:1] offset:17
+; CHECK-NEXT:    global_load_ubyte v20, v0, s[0:1] offset:18
+; CHECK-NEXT:    s_waitcnt vmcnt(18)
+; CHECK-NEXT:    buffer_store_byte v2, v1, s[8:11], 0 offen
+; CHECK-NEXT:    global_load_ubyte v2, v0, s[0:1] offset:19
+; CHECK-NEXT:    s_waitcnt vmcnt(19)
+; CHECK-NEXT:    buffer_store_byte v3, v1, s[8:11], 0 offen offset:1
+; CHECK-NEXT:    global_load_ubyte v3, v0, s[0:1] offset:20
+; CHECK-NEXT:    s_waitcnt vmcnt(20)
+; CHECK-NEXT:    buffer_store_byte v4, v1, s[8:11], 0 offen offset:2
+; CHECK-NEXT:    global_load_ubyte v4, v0, s[0:1] offset:21
+; CHECK-NEXT:    s_waitcnt vmcnt(21)
+; CHECK-NEXT:    buffer_store_byte v5, v1, s[8:11], 0 offen offset:3
+; CHECK-NEXT:    global_load_ubyte v5, v0, s[0:1] offset:22
+; CHECK-NEXT:    s_waitcnt vmcnt(22)
+; CHECK-NEXT:    buffer_store_byte v6, v1, s[8:11], 0 offen offset:4
+; CHECK-NEXT:    global_load_ubyte v6, v0, s[0:1] offset:23
+; CHECK-NEXT:    s_waitcnt vmcnt(23)
+; CHECK-NEXT:    buffer_store_byte v7, v1, s[8:11], 0 offen offset:5
+; CHECK-NEXT:    global_load_ubyte v7, v0, s[0:1] offset:24
+; CHECK-NEXT:    s_waitcnt vmcnt(24)
+; CHECK-NEXT:    buffer_store_byte v8, v1, s[8:11], 0 offen offset:6
+; CHECK-NEXT:    global_load_ubyte v8, v0, s[0:1] offset:25
+; CHECK-NEXT:    s_waitcnt vmcnt(25)
+; CHECK-NEXT:    buffer_store_byte v9, v1, s[8:11], 0 offen offset:7
+; CHECK-NEXT:    global_load_ubyte v9, v0, s[0:1] offset:26
+; CHECK-NEXT:    s_waitcnt vmcnt(26)
+; CHECK-NEXT:    buffer_store_byte v10, v1, s[8:11], 0 offen offset:8
+; CHECK-NEXT:    global_load_ubyte v10, v0, s[0:1] offset:27
+; CHECK-NEXT:    s_waitcnt vmcnt(27)
+; CHECK-NEXT:    buffer_store_byte v11, v1, s[8:11], 0 offen offset:9
+; CHECK-NEXT:    global_load_ubyte v11, v0, s[0:1] offset:28
+; CHECK-NEXT:    s_waitcnt vmcnt(28)
+; CHECK-NEXT:    buffer_store_byte v12, v1, s[8:11], 0 offen offset:10
+; CHECK-NEXT:    global_load_ubyte v12, v0, s[0:1] offset:29
+; CHECK-NEXT:    s_waitcnt vmcnt(29)
+; CHECK-NEXT:    buffer_store_byte v13, v1, s[8:11], 0 offen offset:11
+; CHECK-NEXT:    global_load_ubyte v13, v0, s[0:1] offset:30
+; CHECK-NEXT:    s_waitcnt vmcnt(30)
+; CHECK-NEXT:    buffer_store_byte v14, v1, s[8:11], 0 offen offset:12
+; CHECK-NEXT:    global_load_ubyte v14, v0, s[0:1] offset:31
+; CHECK-NEXT:    s_waitcnt vmcnt(31)
+; CHECK-NEXT:    buffer_store_byte v15, v1, s[8:11], 0 offen offset:13
+; CHECK-NEXT:    global_load_ubyte v15, v0, s[0:1] offset:32
+; CHECK-NEXT:    s_waitcnt vmcnt(32)
+; CHECK-NEXT:    buffer_store_byte v16, v1, s[8:11], 0 offen offset:14
+; CHECK-NEXT:    global_load_ubyte v16, v0, s[0:1] offset:33
+; CHECK-NEXT:    s_waitcnt vmcnt(33)
+; CHECK-NEXT:    buffer_store_byte v17, v1, s[8:11], 0 offen offset:15
+; CHECK-NEXT:    global_load_ubyte v17, v0, s[0:1] offset:34
+; CHECK-NEXT:    s_waitcnt vmcnt(34)
+; CHECK-NEXT:    buffer_store_byte v18, v1, s[8:11], 0 offen offset:16
+; CHECK-NEXT:    global_load_ubyte v18, v0, s[0:1] offset:35
+; CHECK-NEXT:    s_waitcnt vmcnt(35)
+; CHECK-NEXT:    buffer_store_byte v19, v1, s[8:11], 0 offen offset:17
+; CHECK-NEXT:    global_load_ubyte v19, v0, s[0:1] offset:36
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v20, v1, s[8:11], 0 offen offset:18
+; CHECK-NEXT:    global_load_ubyte v20, v0, s[0:1] offset:37
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v2, v1, s[8:11], 0 offen offset:19
+; CHECK-NEXT:    global_load_ubyte v2, v0, s[0:1] offset:38
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v3, v1, s[8:11], 0 offen offset:20
+; CHECK-NEXT:    global_load_ubyte v3, v0, s[0:1] offset:39
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v4, v1, s[8:11], 0 offen offset:21
+; CHECK-NEXT:    global_load_ubyte v4, v0, s[0:1] offset:40
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v5, v1, s[8:11], 0 offen offset:22
+; CHECK-NEXT:    global_load_ubyte v5, v0, s[0:1] offset:41
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v6, v1, s[8:11], 0 offen offset:23
+; CHECK-NEXT:    global_load_ubyte v6, v0, s[0:1] offset:42
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v7, v1, s[8:11], 0 offen offset:24
+; CHECK-NEXT:    global_load_ubyte v7, v0, s[0:1] offset:43
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v8, v1, s[8:11], 0 offen offset:25
+; CHECK-NEXT:    global_load_ubyte v8, v0, s[0:1] offset:44
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v9, v1, s[8:11], 0 offen offset:26
+; CHECK-NEXT:    global_load_ubyte v9, v0, s[0:1] offset:45
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v10, v1, s[8:11], 0 offen offset:27
+; CHECK-NEXT:    global_load_ubyte v10, v0, s[0:1] offset:46
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v11, v1, s[8:11], 0 offen offset:28
+; CHECK-NEXT:    global_load_ubyte v11, v0, s[0:1] offset:47
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v12, v1, s[8:11], 0 offen offset:29
+; CHECK-NEXT:    global_load_ubyte v12, v0, s[0:1] offset:48
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v13, v1, s[8:11], 0 offen offset:30
+; CHECK-NEXT:    global_load_ubyte v13, v0, s[0:1] offset:49
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v14, v1, s[8:11], 0 offen offset:31
+; CHECK-NEXT:    global_load_ubyte v14, v0, s[0:1] offset:50
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v15, v1, s[8:11], 0 offen offset:32
+; CHECK-NEXT:    global_load_ubyte v15, v0, s[0:1] offset:51
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v16, v1, s[8:11], 0 offen offset:33
+; CHECK-NEXT:    global_load_ubyte v16, v0, s[0:1] offset:52
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v17, v1, s[8:11], 0 offen offset:34
+; CHECK-NEXT:    global_load_ubyte v17, v0, s[0:1] offset:53
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v18, v1, s[8:11], 0 offen offset:35
+; CHECK-NEXT:    global_load_ubyte v18, v0, s[0:1] offset:54
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v19, v1, s[8:11], 0 offen offset:36
+; CHECK-NEXT:    global_load_ubyte v19, v0, s[0:1] offset:55
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v20, v1, s[8:11], 0 offen offset:37
+; CHECK-NEXT:    global_load_ubyte v20, v0, s[0:1] offset:56
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v2, v1, s[8:11], 0 offen offset:38
+; CHECK-NEXT:    global_load_ubyte v2, v0, s[0:1] offset:57
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v3, v1, s[8:11], 0 offen offset:39
+; CHECK-NEXT:    global_load_ubyte v3, v0, s[0:1] offset:58
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v4, v1, s[8:11], 0 offen offset:40
+; CHECK-NEXT:    global_load_ubyte v4, v0, s[0:1] offset:59
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v5, v1, s[8:11], 0 offen offset:41
+; CHECK-NEXT:    global_load_ubyte v5, v0, s[0:1] offset:60
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v6, v1, s[8:11], 0 offen offset:42
+; CHECK-NEXT:    global_load_ubyte v6, v0, s[0:1] offset:61
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v7, v1, s[8:11], 0 offen offset:43
+; CHECK-NEXT:    global_load_ubyte v7, v0, s[0:1] offset:62
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v8, v1, s[8:11], 0 offen offset:44
+; CHECK-NEXT:    global_load_ubyte v8, v0, s[0:1] offset:63
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v9, v1, s[8:11], 0 offen offset:45
+; CHECK-NEXT:    global_load_ubyte v9, v0, s[0:1] offset:64
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v10, v1, s[8:11], 0 offen offset:46
+; CHECK-NEXT:    global_load_ubyte v10, v0, s[0:1] offset:65
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v11, v1, s[8:11], 0 offen offset:47
+; CHECK-NEXT:    global_load_ubyte v11, v0, s[0:1] offset:66
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v12, v1, s[8:11], 0 offen offset:48
+; CHECK-NEXT:    global_load_ubyte v12, v0, s[0:1] offset:67
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v13, v1, s[8:11], 0 offen offset:49
+; CHECK-NEXT:    global_load_ubyte v13, v0, s[0:1] offset:68
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v14, v1, s[8:11], 0 offen offset:50
+; CHECK-NEXT:    global_load_ubyte v14, v0, s[0:1] offset:69
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v15, v1, s[8:11], 0 offen offset:51
+; CHECK-NEXT:    global_load_ubyte v15, v0, s[0:1] offset:70
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v16, v1, s[8:11], 0 offen offset:52
+; CHECK-NEXT:    global_load_ubyte v16, v0, s[0:1] offset:71
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v17, v1, s[8:11], 0 offen offset:53
+; CHECK-NEXT:    global_load_ubyte v17, v0, s[0:1] offset:72
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v18, v1, s[8:11], 0 offen offset:54
+; CHECK-NEXT:    global_load_ubyte v18, v0, s[0:1] offset:73
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v19, v1, s[8:11], 0 offen offset:55
+; CHECK-NEXT:    global_load_ubyte v19, v0, s[0:1] offset:74
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v20, v1, s[8:11], 0 offen offset:56
+; CHECK-NEXT:    global_load_ubyte v20, v0, s[0:1] offset:75
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v2, v1, s[8:11], 0 offen offset:57
+; CHECK-NEXT:    global_load_ubyte v2, v0, s[0:1] offset:76
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v3, v1, s[8:11], 0 offen offset:58
+; CHECK-NEXT:    global_load_ubyte v3, v0, s[0:1] offset:77
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v4, v1, s[8:11], 0 offen offset:59
+; CHECK-NEXT:    global_load_ubyte v4, v0, s[0:1] offset:78
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v5, v1, s[8:11], 0 offen offset:60
+; CHECK-NEXT:    global_load_ubyte v5, v0, s[0:1] offset:79
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v6, v1, s[8:11], 0 offen offset:61
+; CHECK-NEXT:    global_load_ubyte v6, v0, s[0:1] offset:80
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v7, v1, s[8:11], 0 offen offset:62
+; CHECK-NEXT:    global_load_ubyte v7, v0, s[0:1] offset:81
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v8, v1, s[8:11], 0 offen offset:63
+; CHECK-NEXT:    global_load_ubyte v8, v0, s[0:1] offset:82
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v9, v1, s[8:11], 0 offen offset:64
+; CHECK-NEXT:    global_load_ubyte v9, v0, s[0:1] offset:83
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v10, v1, s[8:11], 0 offen offset:65
+; CHECK-NEXT:    global_load_ubyte v10, v0, s[0:1] offset:84
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v11, v1, s[8:11], 0 offen offset:66
+; CHECK-NEXT:    global_load_ubyte v11, v0, s[0:1] offset:85
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v12, v1, s[8:11], 0 offen offset:67
+; CHECK-NEXT:    global_load_ubyte v12, v0, s[0:1] offset:86
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v13, v1, s[8:11], 0 offen offset:68
+; CHECK-NEXT:    global_load_ubyte v13, v0, s[0:1] offset:87
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v14, v1, s[8:11], 0 offen offset:69
+; CHECK-NEXT:    global_load_ubyte v14, v0, s[0:1] offset:88
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v15, v1, s[8:11], 0 offen offset:70
+; CHECK-NEXT:    s_waitcnt vmcnt(35)
+; CHECK-NEXT:    buffer_store_byte v16, v1, s[8:11], 0 offen offset:71
+; CHECK-NEXT:    global_load_ubyte v15, v0, s[0:1] offset:89
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    global_load_ubyte v16, v0, s[0:1] offset:90
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v17, v1, s[8:11], 0 offen offset:72
+; CHECK-NEXT:    global_load_ubyte v17, v0, s[0:1] offset:91
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v18, v1, s[8:11], 0 offen offset:73
+; CHECK-NEXT:    global_load_ubyte v18, v0, s[0:1] offset:92
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v19, v1, s[8:11], 0 offen offset:74
+; CHECK-NEXT:    global_load_ubyte v19, v0, s[0:1] offset:93
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v20, v1, s[8:11], 0 offen offset:75
+; CHECK-NEXT:    global_load_ubyte v20, v0, s[0:1] offset:94
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v2, v1, s[8:11], 0 offen offset:76
+; CHECK-NEXT:    global_load_ubyte v2, v0, s[0:1] offset:95
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v3, v1, s[8:11], 0 offen offset:77
+; CHECK-NEXT:    global_load_ubyte v3, v0, s[0:1] offset:96
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v4, v1, s[8:11], 0 offen offset:78
+; CHECK-NEXT:    global_load_ubyte v4, v0, s[0:1] offset:97
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v5, v1, s[8:11], 0 offen offset:79
+; CHECK-NEXT:    global_load_ubyte v5, v0, s[0:1] offset:98
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v6, v1, s[8:11], 0 offen offset:80
+; CHECK-NEXT:    global_load_ubyte v6, v0, s[0:1] offset:99
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v7, v1, s[8:11], 0 offen offset:81
+; CHECK-NEXT:    global_load_ubyte v7, v0, s[0:1] offset:100
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v8, v1, s[8:11], 0 offen offset:82
+; CHECK-NEXT:    global_load_ubyte v8, v0, s[0:1] offset:101
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v9, v1, s[8:11], 0 offen offset:83
+; CHECK-NEXT:    global_load_ubyte v9, v0, s[0:1] offset:102
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v10, v1, s[8:11], 0 offen offset:84
+; CHECK-NEXT:    global_load_ubyte v10, v0, s[0:1] offset:103
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v11, v1, s[8:11], 0 offen offset:85
+; CHECK-NEXT:    global_load_ubyte v11, v0, s[0:1] offset:104
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v12, v1, s[8:11], 0 offen offset:86
+; CHECK-NEXT:    global_load_ubyte v12, v0, s[0:1] offset:105
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v13, v1, s[8:11], 0 offen offset:87
+; CHECK-NEXT:    global_load_ubyte v13, v0, s[0:1] offset:106
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v14, v1, s[8:11], 0 offen offset:88
+; CHECK-NEXT:    global_load_ubyte v14, v0, s[0:1] offset:107
+; CHECK-NEXT:    s_waitcnt vmcnt(35)
+; CHECK-NEXT:    buffer_store_byte v15, v1, s[8:11], 0 offen offset:89
+; CHECK-NEXT:    s_waitcnt vmcnt(35)
+; CHECK-NEXT:    buffer_store_byte v16, v1, s[8:11], 0 offen offset:90
+; CHECK-NEXT:    s_waitcnt vmcnt(34)
+; CHECK-NEXT:    buffer_store_byte v17, v1, s[8:11], 0 offen offset:91
+; CHECK-NEXT:    s_waitcnt vmcnt(33)
+; CHECK-NEXT:    buffer_store_byte v18, v1, s[8:11], 0 offen offset:92
+; CHECK-NEXT:    s_waitcnt vmcnt(32)
+; CHECK-NEXT:    buffer_store_byte v19, v1, s[8:11], 0 offen offset:93
+; CHECK-NEXT:    s_waitcnt vmcnt(31)
+; CHECK-NEXT:    buffer_store_byte v20, v1, s[8:11], 0 offen offset:94
+; CHECK-NEXT:    s_waitcnt vmcnt(30)
+; CHECK-NEXT:    buffer_store_byte v2, v1, s[8:11], 0 offen offset:95
+; CHECK-NEXT:    s_waitcnt vmcnt(29)
+; CHECK-NEXT:    buffer_store_byte v3, v1, s[8:11], 0 offen offset:96
+; CHECK-NEXT:    s_waitcnt vmcnt(28)
+; CHECK-NEXT:    buffer_store_byte v4, v1, s[8:11], 0 offen offset:97
+; CHECK-NEXT:    s_waitcnt vmcnt(27)
+; CHECK-NEXT:    buffer_store_byte v5, v1, s[8:11], 0 offen offset:98
+; CHECK-NEXT:    s_waitcnt vmcnt(26)
+; CHECK-NEXT:    buffer_store_byte v6, v1, s[8:11], 0 offen offset:99
+; CHECK-NEXT:    s_waitcnt vmcnt(25)
+; CHECK-NEXT:    buffer_store_byte v7, v1, s[8:11], 0 offen offset:100
+; CHECK-NEXT:    global_load_ubyte v2, v0, s[0:1] offset:108
+; CHECK-NEXT:    global_load_ubyte v3, v0, s[0:1] offset:109
+; CHECK-NEXT:    global_load_ubyte v4, v0, s[0:1] offset:110
+; CHECK-NEXT:    global_load_ubyte v5, v0, s[0:1] offset:111
+; CHECK-NEXT:    global_load_ubyte v6, v0, s[0:1] offset:112
+; CHECK-NEXT:    global_load_ubyte v7, v0, s[0:1] offset:113
+; CHECK-NEXT:    global_load_ubyte v15, v0, s[0:1] offset:114
+; CHECK-NEXT:    global_load_ubyte v16, v0, s[0:1] offset:115
+; CHECK-NEXT:    global_load_ubyte v17, v0, s[0:1] offset:116
+; CHECK-NEXT:    global_load_ubyte v18, v0, s[0:1] offset:117
+; CHECK-NEXT:    global_load_ubyte v19, v0, s[0:1] offset:118
+; CHECK-NEXT:    global_load_ubyte v20, v0, s[0:1] offset:119
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v8, v1, s[8:11], 0 offen offset:101
+; CHECK-NEXT:    global_load_ubyte v8, v0, s[0:1] offset:120
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v9, v1, s[8:11], 0 offen offset:102
+; CHECK-NEXT:    global_load_ubyte v9, v0, s[0:1] offset:121
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v10, v1, s[8:11], 0 offen offset:103
+; CHECK-NEXT:    global_load_ubyte v10, v0, s[0:1] offset:122
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v11, v1, s[8:11], 0 offen offset:104
+; CHECK-NEXT:    global_load_ubyte v11, v0, s[0:1] offset:123
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v12, v1, s[8:11], 0 offen offset:105
+; CHECK-NEXT:    global_load_ubyte v12, v0, s[0:1] offset:124
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v13, v1, s[8:11], 0 offen offset:106
+; CHECK-NEXT:    global_load_ubyte v13, v0, s[0:1] offset:125
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v14, v1, s[8:11], 0 offen offset:107
+; CHECK-NEXT:    global_load_ubyte v14, v0, s[0:1] offset:126
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    global_load_ubyte v21, v0, s[0:1] offset:127
+; CHECK-NEXT:    s_waitcnt vmcnt(26)
+; CHECK-NEXT:    buffer_store_byte v2, v1, s[8:11], 0 offen offset:108
+; CHECK-NEXT:    s_waitcnt vmcnt(26)
+; CHECK-NEXT:    buffer_store_byte v3, v1, s[8:11], 0 offen offset:109
+; CHECK-NEXT:    s_waitcnt vmcnt(26)
+; CHECK-NEXT:    buffer_store_byte v4, v1, s[8:11], 0 offen offset:110
+; CHECK-NEXT:    s_waitcnt vmcnt(26)
+; CHECK-NEXT:    buffer_store_byte v5, v1, s[8:11], 0 offen offset:111
+; CHECK-NEXT:    s_waitcnt vmcnt(26)
+; CHECK-NEXT:    buffer_store_byte v6, v1, s[8:11], 0 offen offset:112
+; CHECK-NEXT:    s_waitcnt vmcnt(26)
+; CHECK-NEXT:    buffer_store_byte v7, v1, s[8:11], 0 offen offset:113
+; CHECK-NEXT:    s_waitcnt vmcnt(26)
+; CHECK-NEXT:    buffer_store_byte v15, v1, s[8:11], 0 offen offset:114
+; CHECK-NEXT:    s_waitcnt vmcnt(26)
+; CHECK-NEXT:    buffer_store_byte v16, v1, s[8:11], 0 offen offset:115
+; CHECK-NEXT:    s_waitcnt vmcnt(26)
+; CHECK-NEXT:    buffer_store_byte v17, v1, s[8:11], 0 offen offset:116
+; CHECK-NEXT:    s_waitcnt vmcnt(26)
+; CHECK-NEXT:    buffer_store_byte v18, v1, s[8:11], 0 offen offset:117
+; CHECK-NEXT:    s_waitcnt vmcnt(26)
+; CHECK-NEXT:    buffer_store_byte v19, v1, s[8:11], 0 offen offset:118
+; CHECK-NEXT:    s_waitcnt vmcnt(26)
+; CHECK-NEXT:    buffer_store_byte v20, v1, s[8:11], 0 offen offset:119
+; CHECK-NEXT:    s_waitcnt vmcnt(25)
+; CHECK-NEXT:    buffer_store_byte v8, v1, s[8:11], 0 offen offset:120
+; CHECK-NEXT:    s_waitcnt vmcnt(24)
+; CHECK-NEXT:    buffer_store_byte v9, v1, s[8:11], 0 offen offset:121
+; CHECK-NEXT:    s_waitcnt vmcnt(23)
+; CHECK-NEXT:    buffer_store_byte v10, v1, s[8:11], 0 offen offset:122
+; CHECK-NEXT:    s_waitcnt vmcnt(22)
+; CHECK-NEXT:    buffer_store_byte v11, v1, s[8:11], 0 offen offset:123
+; CHECK-NEXT:    s_waitcnt vmcnt(21)
+; CHECK-NEXT:    buffer_store_byte v12, v1, s[8:11], 0 offen offset:124
+; CHECK-NEXT:    s_waitcnt vmcnt(20)
+; CHECK-NEXT:    buffer_store_byte v13, v1, s[8:11], 0 offen offset:125
+; CHECK-NEXT:    s_waitcnt vmcnt(19)
+; CHECK-NEXT:    buffer_store_byte v14, v1, s[8:11], 0 offen offset:126
+; CHECK-NEXT:    s_waitcnt vmcnt(19)
+; CHECK-NEXT:    buffer_store_byte v21, v1, s[8:11], 0 offen offset:127
+; CHECK-NEXT:    s_endpgm
+entry:
+  tail call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) %local, ptr addrspace(4) %0, i64 128, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_p0_p5_minsize(ptr %generic, ptr addrspace(5) %src) #0 {
+; CHECK-LABEL: memcpy_p0_p5_minsize:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_mov_b64 s[10:11], s[2:3]
+; CHECK-NEXT:    s_mov_b64 s[8:9], s[0:1]
+; CHECK-NEXT:    s_load_dword s0, s[4:5], 0x8
+; CHECK-NEXT:    s_add_u32 s8, s8, s7
+; CHECK-NEXT:    s_addc_u32 s9, s9, 0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v2, s0
+; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[8:11], 0 offen
+; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[8:11], 0 offen offset:1
+; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[8:11], 0 offen offset:2
+; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[8:11], 0 offen offset:3
+; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[8:11], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[8:11], 0 offen offset:5
+; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[8:11], 0 offen offset:6
+; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[8:11], 0 offen offset:7
+; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[8:11], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[8:11], 0 offen offset:9
+; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[8:11], 0 offen offset:10
+; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[8:11], 0 offen offset:11
+; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[8:11], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[8:11], 0 offen offset:13
+; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[8:11], 0 offen offset:14
+; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[8:11], 0 offen offset:15
+; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[8:11], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[8:11], 0 offen offset:17
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v0, s0
+; CHECK-NEXT:    v_mov_b32_e32 v1, s1
+; CHECK-NEXT:    s_waitcnt vmcnt(17)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3
+; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[8:11], 0 offen offset:18
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:1
+; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[8:11], 0 offen offset:19
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:2
+; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[8:11], 0 offen offset:20
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:3
+; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[8:11], 0 offen offset:21
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:4
+; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[8:11], 0 offen offset:22
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:5
+; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[8:11], 0 offen offset:23
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:6
+; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[8:11], 0 offen offset:24
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:7
+; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[8:11], 0 offen offset:25
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:8
+; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[8:11], 0 offen offset:26
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:9
+; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[8:11], 0 offen offset:27
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:10
+; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[8:11], 0 offen offset:28
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:11
+; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[8:11], 0 offen offset:29
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:12
+; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[8:11], 0 offen offset:30
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:13
+; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[8:11], 0 offen offset:31
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:14
+; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[8:11], 0 offen offset:32
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:15
+; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[8:11], 0 offen offset:33
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:16
+; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[8:11], 0 offen offset:34
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:17
+; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[8:11], 0 offen offset:35
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:18
+; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[8:11], 0 offen offset:36
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:19
+; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[8:11], 0 offen offset:37
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:20
+; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[8:11], 0 offen offset:38
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:21
+; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[8:11], 0 offen offset:39
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:22
+; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[8:11], 0 offen offset:40
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:23
+; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[8:11], 0 offen offset:41
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:24
+; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[8:11], 0 offen offset:42
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:25
+; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[8:11], 0 offen offset:43
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:26
+; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[8:11], 0 offen offset:44
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:27
+; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[8:11], 0 offen offset:45
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:28
+; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[8:11], 0 offen offset:46
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:29
+; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[8:11], 0 offen offset:47
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:30
+; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[8:11], 0 offen offset:48
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:31
+; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[8:11], 0 offen offset:49
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:32
+; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[8:11], 0 offen offset:50
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:33
+; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[8:11], 0 offen offset:51
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:34
+; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[8:11], 0 offen offset:52
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:35
+; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[8:11], 0 offen offset:53
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:36
+; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[8:11], 0 offen offset:54
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:37
+; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[8:11], 0 offen offset:55
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:38
+; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[8:11], 0 offen offset:56
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:39
+; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[8:11], 0 offen offset:57
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:40
+; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[8:11], 0 offen offset:58
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:41
+; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[8:11], 0 offen offset:59
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:42
+; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[8:11], 0 offen offset:60
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:43
+; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[8:11], 0 offen offset:61
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:44
+; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[8:11], 0 offen offset:62
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:45
+; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[8:11], 0 offen offset:63
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:46
+; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[8:11], 0 offen offset:64
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:47
+; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[8:11], 0 offen offset:65
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:48
+; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[8:11], 0 offen offset:66
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:49
+; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[8:11], 0 offen offset:67
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:50
+; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[8:11], 0 offen offset:68
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:51
+; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[8:11], 0 offen offset:69
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:52
+; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[8:11], 0 offen offset:70
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:53
+; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[8:11], 0 offen offset:71
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:54
+; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[8:11], 0 offen offset:72
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:55
+; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[8:11], 0 offen offset:73
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:56
+; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[8:11], 0 offen offset:74
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:57
+; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[8:11], 0 offen offset:75
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:58
+; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[8:11], 0 offen offset:76
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:59
+; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[8:11], 0 offen offset:77
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:60
+; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[8:11], 0 offen offset:78
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:61
+; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[8:11], 0 offen offset:79
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:62
+; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[8:11], 0 offen offset:80
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:63
+; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[8:11], 0 offen offset:81
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:64
+; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[8:11], 0 offen offset:82
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:65
+; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[8:11], 0 offen offset:83
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:66
+; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[8:11], 0 offen offset:84
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:67
+; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[8:11], 0 offen offset:85
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:68
+; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[8:11], 0 offen offset:86
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:69
+; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[8:11], 0 offen offset:87
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:70
+; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[8:11], 0 offen offset:88
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:71
+; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[8:11], 0 offen offset:89
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:72
+; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[8:11], 0 offen offset:90
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:73
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:74
+; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[8:11], 0 offen offset:91
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[8:11], 0 offen offset:92
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:75
+; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[8:11], 0 offen offset:93
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:76
+; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[8:11], 0 offen offset:94
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:77
+; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[8:11], 0 offen offset:95
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:78
+; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[8:11], 0 offen offset:96
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:79
+; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[8:11], 0 offen offset:97
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:80
+; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[8:11], 0 offen offset:98
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:81
+; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[8:11], 0 offen offset:99
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:82
+; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[8:11], 0 offen offset:100
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:83
+; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[8:11], 0 offen offset:101
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:84
+; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[8:11], 0 offen offset:102
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:85
+; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[8:11], 0 offen offset:103
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:86
+; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[8:11], 0 offen offset:104
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:87
+; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[8:11], 0 offen offset:105
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:88
+; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[8:11], 0 offen offset:106
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:89
+; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[8:11], 0 offen offset:107
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:90
+; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[8:11], 0 offen offset:108
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:91
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:92
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:93
+; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:94
+; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:95
+; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:96
+; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:97
+; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:98
+; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:99
+; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:100
+; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:101
+; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[8:11], 0 offen offset:109
+; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[8:11], 0 offen offset:110
+; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[8:11], 0 offen offset:111
+; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[8:11], 0 offen offset:112
+; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[8:11], 0 offen offset:113
+; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[8:11], 0 offen offset:114
+; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[8:11], 0 offen offset:115
+; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[8:11], 0 offen offset:116
+; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[8:11], 0 offen offset:117
+; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[8:11], 0 offen offset:118
+; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[8:11], 0 offen offset:119
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:102
+; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[8:11], 0 offen offset:120
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:103
+; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[8:11], 0 offen offset:121
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:104
+; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[8:11], 0 offen offset:122
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:105
+; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[8:11], 0 offen offset:123
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:106
+; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[8:11], 0 offen offset:124
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:107
+; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[8:11], 0 offen offset:125
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:108
+; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[8:11], 0 offen offset:126
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    buffer_load_ubyte v21, v2, s[8:11], 0 offen offset:127
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:109
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:110
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:111
+; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:112
+; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:113
+; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:114
+; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:115
+; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:116
+; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:117
+; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:118
+; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:119
+; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:120
+; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:121
+; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:122
+; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:123
+; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:124
+; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:125
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:126
+; CHECK-NEXT:    flat_store_byte v[0:1], v21 offset:127
+; CHECK-NEXT:    s_endpgm
+entry:
+  tail call void @llvm.memcpy.p0.p5.i64(ptr %generic, ptr addrspace(5) %src, i64 128, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_p3_p4_minsize(ptr addrspace(4) %0) #0 {
+; CHECK-LABEL: memcpy_p3_p4_minsize:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; CHECK-NEXT:    v_mov_b32_e32 v24, 0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    global_load_dwordx4 v[0:3], v24, s[0:1] offset:112
+; CHECK-NEXT:    global_load_dwordx4 v[4:7], v24, s[0:1] offset:96
+; CHECK-NEXT:    global_load_dwordx4 v[8:11], v24, s[0:1] offset:80
+; CHECK-NEXT:    global_load_dwordx4 v[12:15], v24, s[0:1] offset:64
+; CHECK-NEXT:    global_load_dwordx4 v[16:19], v24, s[0:1] offset:48
+; CHECK-NEXT:    global_load_dwordx4 v[20:23], v24, s[0:1] offset:32
+; CHECK-NEXT:    s_waitcnt vmcnt(5)
+; CHECK-NEXT:    ds_write2_b64 v24, v[0:1], v[2:3] offset0:14 offset1:15
+; CHECK-NEXT:    s_waitcnt vmcnt(4)
+; CHECK-NEXT:    ds_write2_b64 v24, v[4:5], v[6:7] offset0:12 offset1:13
+; CHECK-NEXT:    global_load_dwordx4 v[0:3], v24, s[0:1] offset:16
+; CHECK-NEXT:    global_load_dwordx4 v[4:7], v24, s[0:1]
+; CHECK-NEXT:    s_waitcnt vmcnt(5)
+; CHECK-NEXT:    ds_write2_b64 v24, v[8:9], v[10:11] offset0:10 offset1:11
+; CHECK-NEXT:    s_waitcnt vmcnt(4)
+; CHECK-NEXT:    ds_write2_b64 v24, v[12:13], v[14:15] offset0:8 offset1:9
+; CHECK-NEXT:    s_waitcnt vmcnt(3)
+; CHECK-NEXT:    ds_write2_b64 v24, v[16:17], v[18:19] offset0:6 offset1:7
+; CHECK-NEXT:    s_waitcnt vmcnt(2)
+; CHECK-NEXT:    ds_write2_b64 v24, v[20:21], v[22:23] offset0:4 offset1:5
+; CHECK-NEXT:    s_waitcnt vmcnt(1)
+; CHECK-NEXT:    ds_write2_b64 v24, v[0:1], v[2:3] offset0:2 offset1:3
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    ds_write2_b64 v24, v[4:5], v[6:7] offset1:1
+; CHECK-NEXT:    s_endpgm
+entry:
+  tail call void @llvm.memcpy.p3.p4.i64(ptr addrspace(3) @shared, ptr addrspace(4) %0, i64 128, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_p0_p3_minsize(ptr %generic) #0 {
+; CHECK-LABEL: memcpy_p0_p3_minsize:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; CHECK-NEXT:    v_mov_b32_e32 v2, 0
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:127
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:126
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:125
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:124
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v0, s0
+; CHECK-NEXT:    v_mov_b32_e32 v1, s1
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:127
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:126
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:123
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:125
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:124
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:122
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:121
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:123
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:120
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:119
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:122
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:121
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:118
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:120
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:119
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:117
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:116
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:118
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:115
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:114
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:117
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:116
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:113
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:115
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:114
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:112
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:111
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:113
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:110
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:109
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:112
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:111
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:108
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:110
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:109
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:107
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:106
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:108
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:105
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:104
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:107
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:106
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:103
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:105
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:104
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:102
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:101
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:103
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:100
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:99
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:102
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:101
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:98
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:100
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:99
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:97
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:96
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:98
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:95
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:94
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:97
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:96
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:93
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:95
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:94
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:92
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:91
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:93
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:90
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:89
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:92
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:91
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:88
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:90
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:89
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:87
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:86
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:88
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:85
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:84
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:87
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:86
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:83
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:85
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:84
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:82
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:81
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:83
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:80
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:79
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:82
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:81
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:78
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:80
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:79
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:77
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:76
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:78
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:75
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:74
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:77
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:76
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:73
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:75
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:74
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:72
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:71
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:73
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:70
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:69
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:72
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:71
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:68
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:70
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:69
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:67
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:66
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:68
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:65
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:64
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:67
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:66
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:63
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:65
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:64
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:62
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:61
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:63
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:60
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:59
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:62
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:61
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:58
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:60
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:59
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:57
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:56
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:58
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:55
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:54
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:57
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:56
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:53
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:55
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:54
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:52
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:51
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:53
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:50
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:49
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:52
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:51
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:48
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:50
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:49
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:47
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:46
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:48
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:45
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:44
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:47
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:46
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:43
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:45
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:44
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:42
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:41
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:43
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:40
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:39
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:42
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:41
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:38
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:40
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:39
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:37
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:36
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:38
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:35
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:34
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:37
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:36
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:33
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:35
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:34
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:32
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:31
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:33
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:30
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:29
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:32
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:31
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:28
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:30
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:29
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:27
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:26
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:28
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:25
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:24
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:27
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:26
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:23
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:25
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:24
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:22
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:21
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:23
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:20
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:19
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:22
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:21
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:18
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:20
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:19
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:16
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:17
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:18
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:8
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:9
+; CHECK-NEXT:    ds_read_u8 v7, v2 offset:10
+; CHECK-NEXT:    ds_read_u8 v8, v2 offset:11
+; CHECK-NEXT:    ds_read_u8 v9, v2 offset:12
+; CHECK-NEXT:    ds_read_u8 v10, v2 offset:13
+; CHECK-NEXT:    ds_read_u8 v11, v2 offset:14
+; CHECK-NEXT:    ds_read_u8 v12, v2 offset:15
+; CHECK-NEXT:    ds_read_u8 v13, v2
+; CHECK-NEXT:    ds_read_u8 v14, v2 offset:1
+; CHECK-NEXT:    ds_read_u8 v15, v2 offset:2
+; CHECK-NEXT:    ds_read_u8 v16, v2 offset:3
+; CHECK-NEXT:    ds_read_u8 v17, v2 offset:4
+; CHECK-NEXT:    ds_read_u8 v18, v2 offset:5
+; CHECK-NEXT:    ds_read_u8 v19, v2 offset:6
+; CHECK-NEXT:    ds_read_u8 v2, v2 offset:7
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:17
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:16
+; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:15
+; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:14
+; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:13
+; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:12
+; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:11
+; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:10
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:9
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:8
+; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:7
+; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:6
+; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:5
+; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:4
+; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:3
+; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:2
+; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:1
+; CHECK-NEXT:    flat_store_byte v[0:1], v13
+; CHECK-NEXT:    s_endpgm
+entry:
+  tail call void @llvm.memcpy.p0.p3.i64(ptr %generic, ptr addrspace(3) @shared, i64 128, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_p0_p0_optsize(ptr %dest, ptr %src) #1 {
+; CHECK-LABEL: memcpy_p0_p0_optsize:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v0, s2
+; CHECK-NEXT:    v_mov_b32_e32 v1, s3
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1]
+; CHECK-NEXT:    v_mov_b32_e32 v3, s1
+; CHECK-NEXT:    v_mov_b32_e32 v2, s0
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:1
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:1
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:2
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:2
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:3
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:3
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:4
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:4
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:5
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:5
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:6
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:6
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:7
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:7
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:8
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:8
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:9
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:9
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:10
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:10
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:11
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:11
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:12
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:12
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:13
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:13
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:14
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:14
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:15
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:15
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:16
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:17
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:17
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:18
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:18
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:19
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:19
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:20
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:20
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:21
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:21
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:22
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:22
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:23
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:23
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:24
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:24
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:25
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:25
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:26
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:26
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:27
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:27
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:28
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:28
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:29
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:29
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:30
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:30
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:31
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:31
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:32
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:32
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:33
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:33
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:34
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:34
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:35
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:35
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:36
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:36
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:37
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:37
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:38
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:38
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:39
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:39
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:40
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:40
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:41
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:41
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:42
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:42
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:43
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:43
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:44
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:44
+; CHECK-NEXT:    flat_load_ubyte v4, v[0:1] offset:45
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v4 offset:45
+; CHECK-NEXT:    flat_load_ubyte v0, v[0:1] offset:46
+; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[2:3], v0 offset:46
+; CHECK-NEXT:    s_endpgm
+entry:
+  tail call void @llvm.memcpy.p0.p0.i64(ptr %dest, ptr %src, i64 47, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_p1_p1_optsize(ptr addrspace(1) %dest, ptr addrspace(1) %src) #1 {
+; CHECK-LABEL: memcpy_p1_p1_optsize:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CHECK-NEXT:    v_mov_b32_e32 v4, 0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    global_load_dwordx2 v[0:1], v4, s[2:3] offset:32
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    global_store_dwordx2 v4, v[0:1], s[0:1] offset:32
+; CHECK-NEXT:    global_load_dwordx2 v[0:1], v4, s[2:3] offset:39
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    global_store_dwordx2 v4, v[0:1], s[0:1] offset:39
+; CHECK-NEXT:    global_load_dwordx4 v[0:3], v4, s[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
+; CHECK-NEXT:    global_load_dwordx4 v[0:3], v4, s[2:3] offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
+; CHECK-NEXT:    s_endpgm
+entry:
+  tail call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) %dest, ptr addrspace(1) %src, i64 47, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_p1_p4_optsize(ptr addrspace(1) %global, ptr addrspace(4) %0) #1 {
+; CHECK-LABEL: memcpy_p1_p4_optsize:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CHECK-NEXT:    v_mov_b32_e32 v4, 0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    global_load_dwordx4 v[0:3], v4, s[2:3]
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
+; CHECK-NEXT:    global_load_dwordx4 v[0:3], v4, s[2:3] offset:16
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:16
+; CHECK-NEXT:    global_load_dwordx4 v[0:3], v4, s[2:3] offset:32
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:32
+; CHECK-NEXT:    global_load_dwordx4 v[0:3], v4, s[2:3] offset:48
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:48
+; CHECK-NEXT:    global_load_dwordx4 v[0:3], v4, s[2:3] offset:64
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:64
+; CHECK-NEXT:    global_load_dwordx4 v[0:3], v4, s[2:3] offset:80
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:80
+; CHECK-NEXT:    global_load_dwordx4 v[0:3], v4, s[2:3] offset:96
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:96
+; CHECK-NEXT:    global_load_dwordx4 v[0:3], v4, s[2:3] offset:112
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1] offset:112
+; CHECK-NEXT:    s_endpgm
+entry:
+  tail call void @llvm.memcpy.p1.p4.i64(ptr addrspace(1) %global, ptr addrspace(4) %0, i64 128, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_p5_p4_optsize(ptr addrspace(5) %local, ptr addrspace(4) %0) #1 {
+; CHECK-LABEL: memcpy_p5_p4_optsize:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_mov_b64 s[10:11], s[2:3]
+; CHECK-NEXT:    s_mov_b64 s[8:9], s[0:1]
+; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
+; CHECK-NEXT:    s_load_dword s2, s[4:5], 0x0
+; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    s_add_u32 s8, s8, s7
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    global_load_ubyte v2, v0, s[0:1]
+; CHECK-NEXT:    global_load_ubyte v3, v0, s[0:1] offset:1
+; CHECK-NEXT:    global_load_ubyte v4, v0, s[0:1] offset:2
+; CHECK-NEXT:    global_load_ubyte v5, v0, s[0:1] offset:3
+; CHECK-NEXT:    global_load_ubyte v6, v0, s[0:1] offset:4
+; CHECK-NEXT:    global_load_ubyte v7, v0, s[0:1] offset:5
+; CHECK-NEXT:    global_load_ubyte v8, v0, s[0:1] offset:6
+; CHECK-NEXT:    global_load_ubyte v9, v0, s[0:1] offset:7
+; CHECK-NEXT:    global_load_ubyte v10, v0, s[0:1] offset:8
+; CHECK-NEXT:    global_load_ubyte v11, v0, s[0:1] offset:9
+; CHECK-NEXT:    global_load_ubyte v12, v0, s[0:1] offset:10
+; CHECK-NEXT:    global_load_ubyte v13, v0, s[0:1] offset:11
+; CHECK-NEXT:    global_load_ubyte v14, v0, s[0:1] offset:12
+; CHECK-NEXT:    global_load_ubyte v15, v0, s[0:1] offset:13
+; CHECK-NEXT:    global_load_ubyte v16, v0, s[0:1] offset:14
+; CHECK-NEXT:    global_load_ubyte v17, v0, s[0:1] offset:15
+; CHECK-NEXT:    s_addc_u32 s9, s9, 0
+; CHECK-NEXT:    v_mov_b32_e32 v1, s2
+; CHECK-NEXT:    global_load_ubyte v18, v0, s[0:1] offset:16
+; CHECK-NEXT:    global_load_ubyte v19, v0, s[0:1] offset:17
+; CHECK-NEXT:    global_load_ubyte v20, v0, s[0:1] offset:18
+; CHECK-NEXT:    s_waitcnt vmcnt(18)
+; CHECK-NEXT:    buffer_store_byte v2, v1, s[8:11], 0 offen
+; CHECK-NEXT:    global_load_ubyte v2, v0, s[0:1] offset:19
+; CHECK-NEXT:    s_waitcnt vmcnt(19)
+; CHECK-NEXT:    buffer_store_byte v3, v1, s[8:11], 0 offen offset:1
+; CHECK-NEXT:    global_load_ubyte v3, v0, s[0:1] offset:20
+; CHECK-NEXT:    s_waitcnt vmcnt(20)
+; CHECK-NEXT:    buffer_store_byte v4, v1, s[8:11], 0 offen offset:2
+; CHECK-NEXT:    global_load_ubyte v4, v0, s[0:1] offset:21
+; CHECK-NEXT:    s_waitcnt vmcnt(21)
+; CHECK-NEXT:    buffer_store_byte v5, v1, s[8:11], 0 offen offset:3
+; CHECK-NEXT:    global_load_ubyte v5, v0, s[0:1] offset:22
+; CHECK-NEXT:    s_waitcnt vmcnt(22)
+; CHECK-NEXT:    buffer_store_byte v6, v1, s[8:11], 0 offen offset:4
+; CHECK-NEXT:    global_load_ubyte v6, v0, s[0:1] offset:23
+; CHECK-NEXT:    s_waitcnt vmcnt(23)
+; CHECK-NEXT:    buffer_store_byte v7, v1, s[8:11], 0 offen offset:5
+; CHECK-NEXT:    global_load_ubyte v7, v0, s[0:1] offset:24
+; CHECK-NEXT:    s_waitcnt vmcnt(24)
+; CHECK-NEXT:    buffer_store_byte v8, v1, s[8:11], 0 offen offset:6
+; CHECK-NEXT:    global_load_ubyte v8, v0, s[0:1] offset:25
+; CHECK-NEXT:    s_waitcnt vmcnt(25)
+; CHECK-NEXT:    buffer_store_byte v9, v1, s[8:11], 0 offen offset:7
+; CHECK-NEXT:    global_load_ubyte v9, v0, s[0:1] offset:26
+; CHECK-NEXT:    s_waitcnt vmcnt(26)
+; CHECK-NEXT:    buffer_store_byte v10, v1, s[8:11], 0 offen offset:8
+; CHECK-NEXT:    global_load_ubyte v10, v0, s[0:1] offset:27
+; CHECK-NEXT:    s_waitcnt vmcnt(27)
+; CHECK-NEXT:    buffer_store_byte v11, v1, s[8:11], 0 offen offset:9
+; CHECK-NEXT:    global_load_ubyte v11, v0, s[0:1] offset:28
+; CHECK-NEXT:    s_waitcnt vmcnt(28)
+; CHECK-NEXT:    buffer_store_byte v12, v1, s[8:11], 0 offen offset:10
+; CHECK-NEXT:    global_load_ubyte v12, v0, s[0:1] offset:29
+; CHECK-NEXT:    s_waitcnt vmcnt(29)
+; CHECK-NEXT:    buffer_store_byte v13, v1, s[8:11], 0 offen offset:11
+; CHECK-NEXT:    global_load_ubyte v13, v0, s[0:1] offset:30
+; CHECK-NEXT:    s_waitcnt vmcnt(30)
+; CHECK-NEXT:    buffer_store_byte v14, v1, s[8:11], 0 offen offset:12
+; CHECK-NEXT:    global_load_ubyte v14, v0, s[0:1] offset:31
+; CHECK-NEXT:    s_waitcnt vmcnt(31)
+; CHECK-NEXT:    buffer_store_byte v15, v1, s[8:11], 0 offen offset:13
+; CHECK-NEXT:    global_load_ubyte v15, v0, s[0:1] offset:32
+; CHECK-NEXT:    s_waitcnt vmcnt(32)
+; CHECK-NEXT:    buffer_store_byte v16, v1, s[8:11], 0 offen offset:14
+; CHECK-NEXT:    global_load_ubyte v16, v0, s[0:1] offset:33
+; CHECK-NEXT:    s_waitcnt vmcnt(33)
+; CHECK-NEXT:    buffer_store_byte v17, v1, s[8:11], 0 offen offset:15
+; CHECK-NEXT:    global_load_ubyte v17, v0, s[0:1] offset:34
+; CHECK-NEXT:    s_waitcnt vmcnt(34)
+; CHECK-NEXT:    buffer_store_byte v18, v1, s[8:11], 0 offen offset:16
+; CHECK-NEXT:    global_load_ubyte v18, v0, s[0:1] offset:35
+; CHECK-NEXT:    s_waitcnt vmcnt(35)
+; CHECK-NEXT:    buffer_store_byte v19, v1, s[8:11], 0 offen offset:17
+; CHECK-NEXT:    global_load_ubyte v19, v0, s[0:1] offset:36
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v20, v1, s[8:11], 0 offen offset:18
+; CHECK-NEXT:    global_load_ubyte v20, v0, s[0:1] offset:37
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v2, v1, s[8:11], 0 offen offset:19
+; CHECK-NEXT:    global_load_ubyte v2, v0, s[0:1] offset:38
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v3, v1, s[8:11], 0 offen offset:20
+; CHECK-NEXT:    global_load_ubyte v3, v0, s[0:1] offset:39
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v4, v1, s[8:11], 0 offen offset:21
+; CHECK-NEXT:    global_load_ubyte v4, v0, s[0:1] offset:40
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v5, v1, s[8:11], 0 offen offset:22
+; CHECK-NEXT:    global_load_ubyte v5, v0, s[0:1] offset:41
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v6, v1, s[8:11], 0 offen offset:23
+; CHECK-NEXT:    global_load_ubyte v6, v0, s[0:1] offset:42
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v7, v1, s[8:11], 0 offen offset:24
+; CHECK-NEXT:    global_load_ubyte v7, v0, s[0:1] offset:43
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v8, v1, s[8:11], 0 offen offset:25
+; CHECK-NEXT:    global_load_ubyte v8, v0, s[0:1] offset:44
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v9, v1, s[8:11], 0 offen offset:26
+; CHECK-NEXT:    global_load_ubyte v9, v0, s[0:1] offset:45
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v10, v1, s[8:11], 0 offen offset:27
+; CHECK-NEXT:    global_load_ubyte v10, v0, s[0:1] offset:46
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v11, v1, s[8:11], 0 offen offset:28
+; CHECK-NEXT:    global_load_ubyte v11, v0, s[0:1] offset:47
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v12, v1, s[8:11], 0 offen offset:29
+; CHECK-NEXT:    global_load_ubyte v12, v0, s[0:1] offset:48
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v13, v1, s[8:11], 0 offen offset:30
+; CHECK-NEXT:    global_load_ubyte v13, v0, s[0:1] offset:49
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v14, v1, s[8:11], 0 offen offset:31
+; CHECK-NEXT:    global_load_ubyte v14, v0, s[0:1] offset:50
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v15, v1, s[8:11], 0 offen offset:32
+; CHECK-NEXT:    global_load_ubyte v15, v0, s[0:1] offset:51
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v16, v1, s[8:11], 0 offen offset:33
+; CHECK-NEXT:    global_load_ubyte v16, v0, s[0:1] offset:52
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v17, v1, s[8:11], 0 offen offset:34
+; CHECK-NEXT:    global_load_ubyte v17, v0, s[0:1] offset:53
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v18, v1, s[8:11], 0 offen offset:35
+; CHECK-NEXT:    global_load_ubyte v18, v0, s[0:1] offset:54
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v19, v1, s[8:11], 0 offen offset:36
+; CHECK-NEXT:    global_load_ubyte v19, v0, s[0:1] offset:55
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v20, v1, s[8:11], 0 offen offset:37
+; CHECK-NEXT:    global_load_ubyte v20, v0, s[0:1] offset:56
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v2, v1, s[8:11], 0 offen offset:38
+; CHECK-NEXT:    global_load_ubyte v2, v0, s[0:1] offset:57
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v3, v1, s[8:11], 0 offen offset:39
+; CHECK-NEXT:    global_load_ubyte v3, v0, s[0:1] offset:58
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v4, v1, s[8:11], 0 offen offset:40
+; CHECK-NEXT:    global_load_ubyte v4, v0, s[0:1] offset:59
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v5, v1, s[8:11], 0 offen offset:41
+; CHECK-NEXT:    global_load_ubyte v5, v0, s[0:1] offset:60
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v6, v1, s[8:11], 0 offen offset:42
+; CHECK-NEXT:    global_load_ubyte v6, v0, s[0:1] offset:61
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v7, v1, s[8:11], 0 offen offset:43
+; CHECK-NEXT:    global_load_ubyte v7, v0, s[0:1] offset:62
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v8, v1, s[8:11], 0 offen offset:44
+; CHECK-NEXT:    global_load_ubyte v8, v0, s[0:1] offset:63
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v9, v1, s[8:11], 0 offen offset:45
+; CHECK-NEXT:    global_load_ubyte v9, v0, s[0:1] offset:64
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v10, v1, s[8:11], 0 offen offset:46
+; CHECK-NEXT:    global_load_ubyte v10, v0, s[0:1] offset:65
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v11, v1, s[8:11], 0 offen offset:47
+; CHECK-NEXT:    global_load_ubyte v11, v0, s[0:1] offset:66
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v12, v1, s[8:11], 0 offen offset:48
+; CHECK-NEXT:    global_load_ubyte v12, v0, s[0:1] offset:67
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v13, v1, s[8:11], 0 offen offset:49
+; CHECK-NEXT:    global_load_ubyte v13, v0, s[0:1] offset:68
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v14, v1, s[8:11], 0 offen offset:50
+; CHECK-NEXT:    global_load_ubyte v14, v0, s[0:1] offset:69
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v15, v1, s[8:11], 0 offen offset:51
+; CHECK-NEXT:    global_load_ubyte v15, v0, s[0:1] offset:70
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v16, v1, s[8:11], 0 offen offset:52
+; CHECK-NEXT:    global_load_ubyte v16, v0, s[0:1] offset:71
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v17, v1, s[8:11], 0 offen offset:53
+; CHECK-NEXT:    global_load_ubyte v17, v0, s[0:1] offset:72
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v18, v1, s[8:11], 0 offen offset:54
+; CHECK-NEXT:    global_load_ubyte v18, v0, s[0:1] offset:73
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v19, v1, s[8:11], 0 offen offset:55
+; CHECK-NEXT:    global_load_ubyte v19, v0, s[0:1] offset:74
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v20, v1, s[8:11], 0 offen offset:56
+; CHECK-NEXT:    global_load_ubyte v20, v0, s[0:1] offset:75
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v2, v1, s[8:11], 0 offen offset:57
+; CHECK-NEXT:    global_load_ubyte v2, v0, s[0:1] offset:76
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v3, v1, s[8:11], 0 offen offset:58
+; CHECK-NEXT:    global_load_ubyte v3, v0, s[0:1] offset:77
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v4, v1, s[8:11], 0 offen offset:59
+; CHECK-NEXT:    global_load_ubyte v4, v0, s[0:1] offset:78
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v5, v1, s[8:11], 0 offen offset:60
+; CHECK-NEXT:    global_load_ubyte v5, v0, s[0:1] offset:79
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v6, v1, s[8:11], 0 offen offset:61
+; CHECK-NEXT:    global_load_ubyte v6, v0, s[0:1] offset:80
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v7, v1, s[8:11], 0 offen offset:62
+; CHECK-NEXT:    global_load_ubyte v7, v0, s[0:1] offset:81
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v8, v1, s[8:11], 0 offen offset:63
+; CHECK-NEXT:    global_load_ubyte v8, v0, s[0:1] offset:82
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v9, v1, s[8:11], 0 offen offset:64
+; CHECK-NEXT:    global_load_ubyte v9, v0, s[0:1] offset:83
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v10, v1, s[8:11], 0 offen offset:65
+; CHECK-NEXT:    global_load_ubyte v10, v0, s[0:1] offset:84
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v11, v1, s[8:11], 0 offen offset:66
+; CHECK-NEXT:    global_load_ubyte v11, v0, s[0:1] offset:85
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v12, v1, s[8:11], 0 offen offset:67
+; CHECK-NEXT:    global_load_ubyte v12, v0, s[0:1] offset:86
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v13, v1, s[8:11], 0 offen offset:68
+; CHECK-NEXT:    global_load_ubyte v13, v0, s[0:1] offset:87
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v14, v1, s[8:11], 0 offen offset:69
+; CHECK-NEXT:    global_load_ubyte v14, v0, s[0:1] offset:88
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v15, v1, s[8:11], 0 offen offset:70
+; CHECK-NEXT:    s_waitcnt vmcnt(35)
+; CHECK-NEXT:    buffer_store_byte v16, v1, s[8:11], 0 offen offset:71
+; CHECK-NEXT:    global_load_ubyte v15, v0, s[0:1] offset:89
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    global_load_ubyte v16, v0, s[0:1] offset:90
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v17, v1, s[8:11], 0 offen offset:72
+; CHECK-NEXT:    global_load_ubyte v17, v0, s[0:1] offset:91
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v18, v1, s[8:11], 0 offen offset:73
+; CHECK-NEXT:    global_load_ubyte v18, v0, s[0:1] offset:92
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v19, v1, s[8:11], 0 offen offset:74
+; CHECK-NEXT:    global_load_ubyte v19, v0, s[0:1] offset:93
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v20, v1, s[8:11], 0 offen offset:75
+; CHECK-NEXT:    global_load_ubyte v20, v0, s[0:1] offset:94
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v2, v1, s[8:11], 0 offen offset:76
+; CHECK-NEXT:    global_load_ubyte v2, v0, s[0:1] offset:95
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v3, v1, s[8:11], 0 offen offset:77
+; CHECK-NEXT:    global_load_ubyte v3, v0, s[0:1] offset:96
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v4, v1, s[8:11], 0 offen offset:78
+; CHECK-NEXT:    global_load_ubyte v4, v0, s[0:1] offset:97
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v5, v1, s[8:11], 0 offen offset:79
+; CHECK-NEXT:    global_load_ubyte v5, v0, s[0:1] offset:98
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v6, v1, s[8:11], 0 offen offset:80
+; CHECK-NEXT:    global_load_ubyte v6, v0, s[0:1] offset:99
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v7, v1, s[8:11], 0 offen offset:81
+; CHECK-NEXT:    global_load_ubyte v7, v0, s[0:1] offset:100
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v8, v1, s[8:11], 0 offen offset:82
+; CHECK-NEXT:    global_load_ubyte v8, v0, s[0:1] offset:101
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v9, v1, s[8:11], 0 offen offset:83
+; CHECK-NEXT:    global_load_ubyte v9, v0, s[0:1] offset:102
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v10, v1, s[8:11], 0 offen offset:84
+; CHECK-NEXT:    global_load_ubyte v10, v0, s[0:1] offset:103
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v11, v1, s[8:11], 0 offen offset:85
+; CHECK-NEXT:    global_load_ubyte v11, v0, s[0:1] offset:104
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v12, v1, s[8:11], 0 offen offset:86
+; CHECK-NEXT:    global_load_ubyte v12, v0, s[0:1] offset:105
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v13, v1, s[8:11], 0 offen offset:87
+; CHECK-NEXT:    global_load_ubyte v13, v0, s[0:1] offset:106
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v14, v1, s[8:11], 0 offen offset:88
+; CHECK-NEXT:    global_load_ubyte v14, v0, s[0:1] offset:107
+; CHECK-NEXT:    s_waitcnt vmcnt(35)
+; CHECK-NEXT:    buffer_store_byte v15, v1, s[8:11], 0 offen offset:89
+; CHECK-NEXT:    s_waitcnt vmcnt(35)
+; CHECK-NEXT:    buffer_store_byte v16, v1, s[8:11], 0 offen offset:90
+; CHECK-NEXT:    s_waitcnt vmcnt(34)
+; CHECK-NEXT:    buffer_store_byte v17, v1, s[8:11], 0 offen offset:91
+; CHECK-NEXT:    s_waitcnt vmcnt(33)
+; CHECK-NEXT:    buffer_store_byte v18, v1, s[8:11], 0 offen offset:92
+; CHECK-NEXT:    s_waitcnt vmcnt(32)
+; CHECK-NEXT:    buffer_store_byte v19, v1, s[8:11], 0 offen offset:93
+; CHECK-NEXT:    s_waitcnt vmcnt(31)
+; CHECK-NEXT:    buffer_store_byte v20, v1, s[8:11], 0 offen offset:94
+; CHECK-NEXT:    s_waitcnt vmcnt(30)
+; CHECK-NEXT:    buffer_store_byte v2, v1, s[8:11], 0 offen offset:95
+; CHECK-NEXT:    s_waitcnt vmcnt(29)
+; CHECK-NEXT:    buffer_store_byte v3, v1, s[8:11], 0 offen offset:96
+; CHECK-NEXT:    s_waitcnt vmcnt(28)
+; CHECK-NEXT:    buffer_store_byte v4, v1, s[8:11], 0 offen offset:97
+; CHECK-NEXT:    s_waitcnt vmcnt(27)
+; CHECK-NEXT:    buffer_store_byte v5, v1, s[8:11], 0 offen offset:98
+; CHECK-NEXT:    s_waitcnt vmcnt(26)
+; CHECK-NEXT:    buffer_store_byte v6, v1, s[8:11], 0 offen offset:99
+; CHECK-NEXT:    s_waitcnt vmcnt(25)
+; CHECK-NEXT:    buffer_store_byte v7, v1, s[8:11], 0 offen offset:100
+; CHECK-NEXT:    global_load_ubyte v2, v0, s[0:1] offset:108
+; CHECK-NEXT:    global_load_ubyte v3, v0, s[0:1] offset:109
+; CHECK-NEXT:    global_load_ubyte v4, v0, s[0:1] offset:110
+; CHECK-NEXT:    global_load_ubyte v5, v0, s[0:1] offset:111
+; CHECK-NEXT:    global_load_ubyte v6, v0, s[0:1] offset:112
+; CHECK-NEXT:    global_load_ubyte v7, v0, s[0:1] offset:113
+; CHECK-NEXT:    global_load_ubyte v15, v0, s[0:1] offset:114
+; CHECK-NEXT:    global_load_ubyte v16, v0, s[0:1] offset:115
+; CHECK-NEXT:    global_load_ubyte v17, v0, s[0:1] offset:116
+; CHECK-NEXT:    global_load_ubyte v18, v0, s[0:1] offset:117
+; CHECK-NEXT:    global_load_ubyte v19, v0, s[0:1] offset:118
+; CHECK-NEXT:    global_load_ubyte v20, v0, s[0:1] offset:119
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v8, v1, s[8:11], 0 offen offset:101
+; CHECK-NEXT:    global_load_ubyte v8, v0, s[0:1] offset:120
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v9, v1, s[8:11], 0 offen offset:102
+; CHECK-NEXT:    global_load_ubyte v9, v0, s[0:1] offset:121
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v10, v1, s[8:11], 0 offen offset:103
+; CHECK-NEXT:    global_load_ubyte v10, v0, s[0:1] offset:122
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v11, v1, s[8:11], 0 offen offset:104
+; CHECK-NEXT:    global_load_ubyte v11, v0, s[0:1] offset:123
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v12, v1, s[8:11], 0 offen offset:105
+; CHECK-NEXT:    global_load_ubyte v12, v0, s[0:1] offset:124
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v13, v1, s[8:11], 0 offen offset:106
+; CHECK-NEXT:    global_load_ubyte v13, v0, s[0:1] offset:125
+; CHECK-NEXT:    s_waitcnt vmcnt(36)
+; CHECK-NEXT:    buffer_store_byte v14, v1, s[8:11], 0 offen offset:107
+; CHECK-NEXT:    global_load_ubyte v14, v0, s[0:1] offset:126
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    global_load_ubyte v21, v0, s[0:1] offset:127
+; CHECK-NEXT:    s_waitcnt vmcnt(26)
+; CHECK-NEXT:    buffer_store_byte v2, v1, s[8:11], 0 offen offset:108
+; CHECK-NEXT:    s_waitcnt vmcnt(26)
+; CHECK-NEXT:    buffer_store_byte v3, v1, s[8:11], 0 offen offset:109
+; CHECK-NEXT:    s_waitcnt vmcnt(26)
+; CHECK-NEXT:    buffer_store_byte v4, v1, s[8:11], 0 offen offset:110
+; CHECK-NEXT:    s_waitcnt vmcnt(26)
+; CHECK-NEXT:    buffer_store_byte v5, v1, s[8:11], 0 offen offset:111
+; CHECK-NEXT:    s_waitcnt vmcnt(26)
+; CHECK-NEXT:    buffer_store_byte v6, v1, s[8:11], 0 offen offset:112
+; CHECK-NEXT:    s_waitcnt vmcnt(26)
+; CHECK-NEXT:    buffer_store_byte v7, v1, s[8:11], 0 offen offset:113
+; CHECK-NEXT:    s_waitcnt vmcnt(26)
+; CHECK-NEXT:    buffer_store_byte v15, v1, s[8:11], 0 offen offset:114
+; CHECK-NEXT:    s_waitcnt vmcnt(26)
+; CHECK-NEXT:    buffer_store_byte v16, v1, s[8:11], 0 offen offset:115
+; CHECK-NEXT:    s_waitcnt vmcnt(26)
+; CHECK-NEXT:    buffer_store_byte v17, v1, s[8:11], 0 offen offset:116
+; CHECK-NEXT:    s_waitcnt vmcnt(26)
+; CHECK-NEXT:    buffer_store_byte v18, v1, s[8:11], 0 offen offset:117
+; CHECK-NEXT:    s_waitcnt vmcnt(26)
+; CHECK-NEXT:    buffer_store_byte v19, v1, s[8:11], 0 offen offset:118
+; CHECK-NEXT:    s_waitcnt vmcnt(26)
+; CHECK-NEXT:    buffer_store_byte v20, v1, s[8:11], 0 offen offset:119
+; CHECK-NEXT:    s_waitcnt vmcnt(25)
+; CHECK-NEXT:    buffer_store_byte v8, v1, s[8:11], 0 offen offset:120
+; CHECK-NEXT:    s_waitcnt vmcnt(24)
+; CHECK-NEXT:    buffer_store_byte v9, v1, s[8:11], 0 offen offset:121
+; CHECK-NEXT:    s_waitcnt vmcnt(23)
+; CHECK-NEXT:    buffer_store_byte v10, v1, s[8:11], 0 offen offset:122
+; CHECK-NEXT:    s_waitcnt vmcnt(22)
+; CHECK-NEXT:    buffer_store_byte v11, v1, s[8:11], 0 offen offset:123
+; CHECK-NEXT:    s_waitcnt vmcnt(21)
+; CHECK-NEXT:    buffer_store_byte v12, v1, s[8:11], 0 offen offset:124
+; CHECK-NEXT:    s_waitcnt vmcnt(20)
+; CHECK-NEXT:    buffer_store_byte v13, v1, s[8:11], 0 offen offset:125
+; CHECK-NEXT:    s_waitcnt vmcnt(19)
+; CHECK-NEXT:    buffer_store_byte v14, v1, s[8:11], 0 offen offset:126
+; CHECK-NEXT:    s_waitcnt vmcnt(19)
+; CHECK-NEXT:    buffer_store_byte v21, v1, s[8:11], 0 offen offset:127
+; CHECK-NEXT:    s_endpgm
+entry:
+  tail call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) %local, ptr addrspace(4) %0, i64 128, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_p0_p5_optsize(ptr %generic, ptr addrspace(5) %src) #1 {
+; CHECK-LABEL: memcpy_p0_p5_optsize:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_mov_b64 s[10:11], s[2:3]
+; CHECK-NEXT:    s_mov_b64 s[8:9], s[0:1]
+; CHECK-NEXT:    s_load_dword s0, s[4:5], 0x8
+; CHECK-NEXT:    s_add_u32 s8, s8, s7
+; CHECK-NEXT:    s_addc_u32 s9, s9, 0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v2, s0
+; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[8:11], 0 offen
+; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[8:11], 0 offen offset:1
+; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[8:11], 0 offen offset:2
+; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[8:11], 0 offen offset:3
+; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[8:11], 0 offen offset:4
+; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[8:11], 0 offen offset:5
+; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[8:11], 0 offen offset:6
+; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[8:11], 0 offen offset:7
+; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[8:11], 0 offen offset:8
+; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[8:11], 0 offen offset:9
+; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[8:11], 0 offen offset:10
+; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[8:11], 0 offen offset:11
+; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[8:11], 0 offen offset:12
+; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[8:11], 0 offen offset:13
+; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[8:11], 0 offen offset:14
+; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[8:11], 0 offen offset:15
+; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[8:11], 0 offen offset:16
+; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[8:11], 0 offen offset:17
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v0, s0
+; CHECK-NEXT:    v_mov_b32_e32 v1, s1
+; CHECK-NEXT:    s_waitcnt vmcnt(17)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3
+; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[8:11], 0 offen offset:18
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:1
+; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[8:11], 0 offen offset:19
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:2
+; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[8:11], 0 offen offset:20
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:3
+; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[8:11], 0 offen offset:21
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:4
+; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[8:11], 0 offen offset:22
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:5
+; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[8:11], 0 offen offset:23
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:6
+; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[8:11], 0 offen offset:24
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:7
+; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[8:11], 0 offen offset:25
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:8
+; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[8:11], 0 offen offset:26
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:9
+; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[8:11], 0 offen offset:27
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:10
+; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[8:11], 0 offen offset:28
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:11
+; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[8:11], 0 offen offset:29
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:12
+; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[8:11], 0 offen offset:30
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:13
+; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[8:11], 0 offen offset:31
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:14
+; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[8:11], 0 offen offset:32
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:15
+; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[8:11], 0 offen offset:33
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:16
+; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[8:11], 0 offen offset:34
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:17
+; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[8:11], 0 offen offset:35
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:18
+; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[8:11], 0 offen offset:36
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:19
+; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[8:11], 0 offen offset:37
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:20
+; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[8:11], 0 offen offset:38
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:21
+; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[8:11], 0 offen offset:39
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:22
+; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[8:11], 0 offen offset:40
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:23
+; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[8:11], 0 offen offset:41
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:24
+; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[8:11], 0 offen offset:42
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:25
+; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[8:11], 0 offen offset:43
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:26
+; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[8:11], 0 offen offset:44
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:27
+; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[8:11], 0 offen offset:45
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:28
+; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[8:11], 0 offen offset:46
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:29
+; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[8:11], 0 offen offset:47
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:30
+; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[8:11], 0 offen offset:48
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:31
+; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[8:11], 0 offen offset:49
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:32
+; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[8:11], 0 offen offset:50
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:33
+; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[8:11], 0 offen offset:51
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:34
+; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[8:11], 0 offen offset:52
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:35
+; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[8:11], 0 offen offset:53
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:36
+; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[8:11], 0 offen offset:54
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:37
+; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[8:11], 0 offen offset:55
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:38
+; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[8:11], 0 offen offset:56
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:39
+; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[8:11], 0 offen offset:57
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:40
+; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[8:11], 0 offen offset:58
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:41
+; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[8:11], 0 offen offset:59
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:42
+; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[8:11], 0 offen offset:60
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:43
+; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[8:11], 0 offen offset:61
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:44
+; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[8:11], 0 offen offset:62
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:45
+; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[8:11], 0 offen offset:63
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:46
+; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[8:11], 0 offen offset:64
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:47
+; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[8:11], 0 offen offset:65
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:48
+; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[8:11], 0 offen offset:66
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:49
+; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[8:11], 0 offen offset:67
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:50
+; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[8:11], 0 offen offset:68
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:51
+; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[8:11], 0 offen offset:69
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:52
+; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[8:11], 0 offen offset:70
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:53
+; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[8:11], 0 offen offset:71
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:54
+; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[8:11], 0 offen offset:72
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:55
+; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[8:11], 0 offen offset:73
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:56
+; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[8:11], 0 offen offset:74
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:57
+; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[8:11], 0 offen offset:75
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:58
+; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[8:11], 0 offen offset:76
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:59
+; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[8:11], 0 offen offset:77
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:60
+; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[8:11], 0 offen offset:78
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:61
+; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[8:11], 0 offen offset:79
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:62
+; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[8:11], 0 offen offset:80
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:63
+; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[8:11], 0 offen offset:81
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:64
+; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[8:11], 0 offen offset:82
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:65
+; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[8:11], 0 offen offset:83
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:66
+; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[8:11], 0 offen offset:84
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:67
+; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[8:11], 0 offen offset:85
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:68
+; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[8:11], 0 offen offset:86
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:69
+; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[8:11], 0 offen offset:87
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:70
+; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[8:11], 0 offen offset:88
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:71
+; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[8:11], 0 offen offset:89
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:72
+; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[8:11], 0 offen offset:90
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:73
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:74
+; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[8:11], 0 offen offset:91
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[8:11], 0 offen offset:92
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:75
+; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[8:11], 0 offen offset:93
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:76
+; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[8:11], 0 offen offset:94
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:77
+; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[8:11], 0 offen offset:95
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:78
+; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[8:11], 0 offen offset:96
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:79
+; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[8:11], 0 offen offset:97
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:80
+; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[8:11], 0 offen offset:98
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:81
+; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[8:11], 0 offen offset:99
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:82
+; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[8:11], 0 offen offset:100
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:83
+; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[8:11], 0 offen offset:101
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:84
+; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[8:11], 0 offen offset:102
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:85
+; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[8:11], 0 offen offset:103
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:86
+; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[8:11], 0 offen offset:104
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:87
+; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[8:11], 0 offen offset:105
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:88
+; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[8:11], 0 offen offset:106
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:89
+; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[8:11], 0 offen offset:107
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:90
+; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[8:11], 0 offen offset:108
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:91
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:92
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:93
+; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:94
+; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:95
+; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:96
+; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:97
+; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:98
+; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:99
+; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:100
+; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:101
+; CHECK-NEXT:    buffer_load_ubyte v4, v2, s[8:11], 0 offen offset:109
+; CHECK-NEXT:    buffer_load_ubyte v5, v2, s[8:11], 0 offen offset:110
+; CHECK-NEXT:    buffer_load_ubyte v6, v2, s[8:11], 0 offen offset:111
+; CHECK-NEXT:    buffer_load_ubyte v7, v2, s[8:11], 0 offen offset:112
+; CHECK-NEXT:    buffer_load_ubyte v8, v2, s[8:11], 0 offen offset:113
+; CHECK-NEXT:    buffer_load_ubyte v9, v2, s[8:11], 0 offen offset:114
+; CHECK-NEXT:    buffer_load_ubyte v10, v2, s[8:11], 0 offen offset:115
+; CHECK-NEXT:    buffer_load_ubyte v11, v2, s[8:11], 0 offen offset:116
+; CHECK-NEXT:    buffer_load_ubyte v12, v2, s[8:11], 0 offen offset:117
+; CHECK-NEXT:    buffer_load_ubyte v13, v2, s[8:11], 0 offen offset:118
+; CHECK-NEXT:    buffer_load_ubyte v14, v2, s[8:11], 0 offen offset:119
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:102
+; CHECK-NEXT:    buffer_load_ubyte v15, v2, s[8:11], 0 offen offset:120
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:103
+; CHECK-NEXT:    buffer_load_ubyte v16, v2, s[8:11], 0 offen offset:121
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:104
+; CHECK-NEXT:    buffer_load_ubyte v17, v2, s[8:11], 0 offen offset:122
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:105
+; CHECK-NEXT:    buffer_load_ubyte v18, v2, s[8:11], 0 offen offset:123
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:106
+; CHECK-NEXT:    buffer_load_ubyte v19, v2, s[8:11], 0 offen offset:124
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:107
+; CHECK-NEXT:    buffer_load_ubyte v20, v2, s[8:11], 0 offen offset:125
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:108
+; CHECK-NEXT:    buffer_load_ubyte v3, v2, s[8:11], 0 offen offset:126
+; CHECK-NEXT:    s_nop 0
+; CHECK-NEXT:    buffer_load_ubyte v21, v2, s[8:11], 0 offen offset:127
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:109
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:110
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:111
+; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:112
+; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:113
+; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:114
+; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:115
+; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:116
+; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:117
+; CHECK-NEXT:    flat_store_byte v[0:1], v13 offset:118
+; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:119
+; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:120
+; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:121
+; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:122
+; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:123
+; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:124
+; CHECK-NEXT:    flat_store_byte v[0:1], v20 offset:125
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:126
+; CHECK-NEXT:    flat_store_byte v[0:1], v21 offset:127
+; CHECK-NEXT:    s_endpgm
+entry:
+  tail call void @llvm.memcpy.p0.p5.i64(ptr %generic, ptr addrspace(5) %src, i64 128, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_p3_p4_optsize(ptr addrspace(4) %0) #1 {
+; CHECK-LABEL: memcpy_p3_p4_optsize:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; CHECK-NEXT:    v_mov_b32_e32 v24, 0
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    global_load_dwordx4 v[0:3], v24, s[0:1] offset:112
+; CHECK-NEXT:    global_load_dwordx4 v[4:7], v24, s[0:1] offset:96
+; CHECK-NEXT:    global_load_dwordx4 v[8:11], v24, s[0:1] offset:80
+; CHECK-NEXT:    global_load_dwordx4 v[12:15], v24, s[0:1] offset:64
+; CHECK-NEXT:    global_load_dwordx4 v[16:19], v24, s[0:1] offset:48
+; CHECK-NEXT:    global_load_dwordx4 v[20:23], v24, s[0:1] offset:32
+; CHECK-NEXT:    s_waitcnt vmcnt(5)
+; CHECK-NEXT:    ds_write2_b64 v24, v[0:1], v[2:3] offset0:14 offset1:15
+; CHECK-NEXT:    s_waitcnt vmcnt(4)
+; CHECK-NEXT:    ds_write2_b64 v24, v[4:5], v[6:7] offset0:12 offset1:13
+; CHECK-NEXT:    global_load_dwordx4 v[0:3], v24, s[0:1] offset:16
+; CHECK-NEXT:    global_load_dwordx4 v[4:7], v24, s[0:1]
+; CHECK-NEXT:    s_waitcnt vmcnt(5)
+; CHECK-NEXT:    ds_write2_b64 v24, v[8:9], v[10:11] offset0:10 offset1:11
+; CHECK-NEXT:    s_waitcnt vmcnt(4)
+; CHECK-NEXT:    ds_write2_b64 v24, v[12:13], v[14:15] offset0:8 offset1:9
+; CHECK-NEXT:    s_waitcnt vmcnt(3)
+; CHECK-NEXT:    ds_write2_b64 v24, v[16:17], v[18:19] offset0:6 offset1:7
+; CHECK-NEXT:    s_waitcnt vmcnt(2)
+; CHECK-NEXT:    ds_write2_b64 v24, v[20:21], v[22:23] offset0:4 offset1:5
+; CHECK-NEXT:    s_waitcnt vmcnt(1)
+; CHECK-NEXT:    ds_write2_b64 v24, v[0:1], v[2:3] offset0:2 offset1:3
+; CHECK-NEXT:    s_waitcnt vmcnt(0)
+; CHECK-NEXT:    ds_write2_b64 v24, v[4:5], v[6:7] offset1:1
+; CHECK-NEXT:    s_endpgm
+entry:
+  tail call void @llvm.memcpy.p3.p4.i64(ptr addrspace(3) @shared, ptr addrspace(4) %0, i64 128, i1 false)
+  ret void
+}
+
+define amdgpu_kernel void @memcpy_p0_p3_optsize(ptr %generic) #1 {
+; CHECK-LABEL: memcpy_p0_p3_optsize:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; CHECK-NEXT:    v_mov_b32_e32 v2, 0
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:127
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:126
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:125
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:124
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    v_mov_b32_e32 v0, s0
+; CHECK-NEXT:    v_mov_b32_e32 v1, s1
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:127
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:126
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:123
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:125
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:124
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:122
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:121
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:123
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:120
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:119
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:122
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:121
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:118
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:120
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:119
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:117
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:116
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:118
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:115
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:114
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:117
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:116
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:113
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:115
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:114
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:112
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:111
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:113
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:110
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:109
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:112
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:111
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:108
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:110
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:109
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:107
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:106
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:108
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:105
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:104
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:107
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:106
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:103
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:105
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:104
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:102
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:101
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:103
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:100
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:99
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:102
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:101
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:98
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:100
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:99
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:97
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:96
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:98
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:95
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:94
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:97
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:96
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:93
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:95
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:94
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:92
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:91
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:93
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:90
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:89
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:92
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:91
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:88
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:90
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:89
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:87
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:86
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:88
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:85
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:84
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:87
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:86
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:83
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:85
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:84
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:82
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:81
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:83
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:80
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:79
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:82
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:81
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:78
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:80
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:79
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:77
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:76
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:78
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:75
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:74
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:77
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:76
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:73
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:75
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:74
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:72
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:71
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:73
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:70
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:69
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:72
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:71
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:68
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:70
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:69
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:67
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:66
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:68
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:65
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:64
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:67
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:66
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:63
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:65
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:64
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:62
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:61
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:63
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:60
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:59
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:62
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:61
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:58
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:60
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:59
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:57
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:56
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:58
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:55
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:54
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:57
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:56
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:53
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:55
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:54
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:52
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:51
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:53
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:50
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:49
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:52
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:51
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:48
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:50
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:49
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:47
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:46
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:48
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:45
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:44
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:47
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:46
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:43
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:45
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:44
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:42
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:41
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:43
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:40
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:39
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:42
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:41
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:38
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:40
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:39
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:37
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:36
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:38
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:35
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:34
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:37
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:36
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:33
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:35
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:34
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:32
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:31
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:33
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:30
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:29
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:32
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:31
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:28
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:30
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:29
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:27
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:26
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:28
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:25
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:24
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:27
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:26
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:23
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:25
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:24
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:22
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:21
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:23
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:20
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:19
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:22
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:21
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:18
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:20
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:19
+; CHECK-NEXT:    ds_read_u8 v3, v2 offset:16
+; CHECK-NEXT:    ds_read_u8 v5, v2 offset:17
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:18
+; CHECK-NEXT:    ds_read_u8 v4, v2 offset:8
+; CHECK-NEXT:    ds_read_u8 v6, v2 offset:9
+; CHECK-NEXT:    ds_read_u8 v7, v2 offset:10
+; CHECK-NEXT:    ds_read_u8 v8, v2 offset:11
+; CHECK-NEXT:    ds_read_u8 v9, v2 offset:12
+; CHECK-NEXT:    ds_read_u8 v10, v2 offset:13
+; CHECK-NEXT:    ds_read_u8 v11, v2 offset:14
+; CHECK-NEXT:    ds_read_u8 v12, v2 offset:15
+; CHECK-NEXT:    ds_read_u8 v13, v2
+; CHECK-NEXT:    ds_read_u8 v14, v2 offset:1
+; CHECK-NEXT:    ds_read_u8 v15, v2 offset:2
+; CHECK-NEXT:    ds_read_u8 v16, v2 offset:3
+; CHECK-NEXT:    ds_read_u8 v17, v2 offset:4
+; CHECK-NEXT:    ds_read_u8 v18, v2 offset:5
+; CHECK-NEXT:    ds_read_u8 v19, v2 offset:6
+; CHECK-NEXT:    ds_read_u8 v2, v2 offset:7
+; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
+; CHECK-NEXT:    flat_store_byte v[0:1], v5 offset:17
+; CHECK-NEXT:    flat_store_byte v[0:1], v3 offset:16
+; CHECK-NEXT:    flat_store_byte v[0:1], v12 offset:15
+; CHECK-NEXT:    flat_store_byte v[0:1], v11 offset:14
+; CHECK-NEXT:    flat_store_byte v[0:1], v10 offset:13
+; CHECK-NEXT:    flat_store_byte v[0:1], v9 offset:12
+; CHECK-NEXT:    flat_store_byte v[0:1], v8 offset:11
+; CHECK-NEXT:    flat_store_byte v[0:1], v7 offset:10
+; CHECK-NEXT:    flat_store_byte v[0:1], v6 offset:9
+; CHECK-NEXT:    flat_store_byte v[0:1], v4 offset:8
+; CHECK-NEXT:    flat_store_byte v[0:1], v2 offset:7
+; CHECK-NEXT:    flat_store_byte v[0:1], v19 offset:6
+; CHECK-NEXT:    flat_store_byte v[0:1], v18 offset:5
+; CHECK-NEXT:    flat_store_byte v[0:1], v17 offset:4
+; CHECK-NEXT:    flat_store_byte v[0:1], v16 offset:3
+; CHECK-NEXT:    flat_store_byte v[0:1], v15 offset:2
+; CHECK-NEXT:    flat_store_byte v[0:1], v14 offset:1
+; CHECK-NEXT:    flat_store_byte v[0:1], v13
+; CHECK-NEXT:    s_endpgm
+entry:
+  tail call void @llvm.memcpy.p0.p3.i64(ptr %generic, ptr addrspace(3) @shared, i64 128, i1 false)
+  ret void
+}
+
+declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg) #2
+
+declare void @llvm.memcpy.p0.p5.i64(ptr noalias nocapture writeonly, ptr addrspace(5) noalias nocapture readonly, i64, i1 immarg) #2
+
+declare void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noalias nocapture writeonly, ptr addrspace(1) noalias nocapture readonly, i64, i1 immarg) #2
+
+declare void @llvm.memcpy.p1.p4.i64(ptr addrspace(1) noalias nocapture writeonly, ptr addrspace(4) noalias nocapture readonly, i64, i1 immarg) #2
+
+declare void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) noalias nocapture writeonly, ptr addrspace(4) noalias nocapture readonly, i64, i1 immarg) #2
+
+declare void @llvm.memcpy.p3.p4.i64(ptr addrspace(3) noalias nocapture writeonly, ptr addrspace(4) noalias nocapture readonly, i64, i1 immarg) #2
+
+declare void @llvm.memcpy.p0.p3.i64(ptr noalias nocapture writeonly, ptr addrspace(3) noalias nocapture readonly, i64, i1 immarg) #2
+
+attributes #0 = { minsize }
+attributes #1 = { optsize }
+attributes #2 = { nocallback nofree nounwind willreturn memory(argmem: readwrite) }
-- 
GitLab


From 6ab5927238395798c2e8b657bb59a0304046099d Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Tue, 16 Apr 2024 06:13:03 -0700
Subject: [PATCH 401/464] [SLP][NFC]Add a test with the incorrect vectorization
 of smax with unsigned arg.

---
 .../RISCV/smax-unsigned-operand.ll            | 31 +++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 llvm/test/Transforms/SLPVectorizer/RISCV/smax-unsigned-operand.ll

diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/smax-unsigned-operand.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/smax-unsigned-operand.ll
new file mode 100644
index 000000000000..577d995f6a76
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/smax-unsigned-operand.ll
@@ -0,0 +1,31 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -S --passes=slp-vectorizer -mtriple=riscv64-unknown-linux-gnu -mattr=+v < %s | FileCheck %s
+target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"
+target triple = "riscv64-unknown-linux-gnu"
+
+@e = global [2 x i8] zeroinitializer
+
+define void @main(ptr noalias %p) {
+; CHECK-LABEL: define void @main(
+; CHECK-SAME: ptr noalias [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[P]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> <i32 0, i32 poison>, i32 [[TMP0]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i32> @llvm.smax.v2i32(<2 x i32> [[TMP1]], <2 x i32> <i32 0, i32 1>)
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc <2 x i32> [[TMP2]] to <2 x i8>
+; CHECK-NEXT:    store <2 x i8> [[TMP3]], ptr getelementptr inbounds ([11 x i8], ptr @e, i64 0, i64 4), align 1
+; CHECK-NEXT:    ret void
+;
+bb:
+  %conv.4 = zext i32 0 to i64
+  %cond.4 = tail call i64 @llvm.smax.i64(i64 %conv.4, i64 0)
+  %conv5.4 = trunc i64 %cond.4 to i8
+  store i8 %conv5.4, ptr getelementptr inbounds ([11 x i8], ptr @e, i64 0, i64 4), align 1
+  %0 = load i32, ptr %p, align 4
+  %conv.5 = zext i32 %0 to i64
+  %cond.5 = tail call i64 @llvm.smax.i64(i64 %conv.5, i64 1)
+  %conv5.5 = trunc i64 %cond.5 to i8
+  store i8 %conv5.5, ptr getelementptr inbounds ([11 x i8], ptr @e, i64 0, i64 5), align 1
+  ret void
+}
+
-- 
GitLab


From 26ebe16d78b22329d602db0398ce163ad610b0dc Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Tue, 16 Apr 2024 06:42:15 -0700
Subject: [PATCH 402/464] [SLP]Fix PR88834: check if unsigned arg can be
 trunced, being used in smax/smin intrinsics.

Need to check that unsigned argument can be safely used in smax/smin
intrinsics by checking if at least single sign bit is cleared, otherwise
its value may be treated as negative instead of positive.
---
 llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp      |  8 +++++++-
 .../SLPVectorizer/RISCV/smax-unsigned-operand.ll     | 12 ++++++++----
 2 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index d0bcdceae392..0cd3ca32933c 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -14639,10 +14639,16 @@ bool BoUpSLP::collectValuesToDemote(
         assert((ID == Intrinsic::smin || ID == Intrinsic::smax) &&
                "Expected min/max intrinsics only.");
         unsigned SignBits = OrigBitWidth - BitWidth;
+        APInt Mask = APInt::getBitsSetFrom(OrigBitWidth, BitWidth - 1);
         return SignBits <= ComputeNumSignBits(I->getOperand(0), *DL, 0, AC,
                                               nullptr, DT) &&
+               (!isKnownNonNegative(I->getOperand(0), SimplifyQuery(*DL)) ||
+                MaskedValueIsZero(I->getOperand(0), Mask,
+                                  SimplifyQuery(*DL))) &&
                SignBits <= ComputeNumSignBits(I->getOperand(1), *DL, 0, AC,
-                                              nullptr, DT);
+                                              nullptr, DT) &&
+               (!isKnownNonNegative(I->getOperand(1), SimplifyQuery(*DL)) ||
+                MaskedValueIsZero(I->getOperand(1), Mask, SimplifyQuery(*DL)));
       });
     };
     if (ID != Intrinsic::abs) {
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/smax-unsigned-operand.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/smax-unsigned-operand.ll
index 577d995f6a76..5db148ac1b48 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/smax-unsigned-operand.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/smax-unsigned-operand.ll
@@ -9,11 +9,15 @@ define void @main(ptr noalias %p) {
 ; CHECK-LABEL: define void @main(
 ; CHECK-SAME: ptr noalias [[P:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[CONV_4:%.*]] = zext i32 0 to i64
+; CHECK-NEXT:    [[COND_4:%.*]] = tail call i64 @llvm.smax.i64(i64 [[CONV_4]], i64 0)
+; CHECK-NEXT:    [[CONV5_4:%.*]] = trunc i64 [[COND_4]] to i8
+; CHECK-NEXT:    store i8 [[CONV5_4]], ptr getelementptr inbounds ([11 x i8], ptr @e, i64 0, i64 4), align 1
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[P]], align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> <i32 0, i32 poison>, i32 [[TMP0]], i32 1
-; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i32> @llvm.smax.v2i32(<2 x i32> [[TMP1]], <2 x i32> <i32 0, i32 1>)
-; CHECK-NEXT:    [[TMP3:%.*]] = trunc <2 x i32> [[TMP2]] to <2 x i8>
-; CHECK-NEXT:    store <2 x i8> [[TMP3]], ptr getelementptr inbounds ([11 x i8], ptr @e, i64 0, i64 4), align 1
+; CHECK-NEXT:    [[CONV_5:%.*]] = zext i32 [[TMP0]] to i64
+; CHECK-NEXT:    [[COND_5:%.*]] = tail call i64 @llvm.smax.i64(i64 [[CONV_5]], i64 1)
+; CHECK-NEXT:    [[CONV5_5:%.*]] = trunc i64 [[COND_5]] to i8
+; CHECK-NEXT:    store i8 [[CONV5_5]], ptr getelementptr inbounds ([11 x i8], ptr @e, i64 0, i64 5), align 1
 ; CHECK-NEXT:    ret void
 ;
 bb:
-- 
GitLab


From e185978321db4500d72fb1186476ee4104c5928b Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 16 Apr 2024 14:23:50 +0100
Subject: [PATCH 403/464] [VectorCombine][X86] Regenerate shuffle.ll +
 shuffle-of-casts.ll

Use v4 of UTC to improve regex matching of argument names to fix a filecheck matching in a future patch
---
 .../VectorCombine/X86/shuffle-of-casts.ll     | 130 +++++++-----
 .../Transforms/VectorCombine/X86/shuffle.ll   | 188 ++++++++++--------
 2 files changed, 185 insertions(+), 133 deletions(-)

diff --git a/llvm/test/Transforms/VectorCombine/X86/shuffle-of-casts.ll b/llvm/test/Transforms/VectorCombine/X86/shuffle-of-casts.ll
index 7d9f7e390b9c..3a5d2095e2b9 100644
--- a/llvm/test/Transforms/VectorCombine/X86/shuffle-of-casts.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/shuffle-of-casts.ll
@@ -1,12 +1,13 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
 ; RUN: opt < %s -passes=vector-combine -S -mtriple=x86_64-- -mattr=sse2 | FileCheck %s --check-prefixes=CHECK,SSE
 ; RUN: opt < %s -passes=vector-combine -S -mtriple=x86_64-- -mattr=avx2 | FileCheck %s --check-prefixes=CHECK,AVX
 
 ; standard vector concatenations
 
 define <16 x i32> @concat_zext_v8i16_v16i32(<8 x i16> %a0, <8 x i16> %a1) {
-; CHECK-LABEL: @concat_zext_v8i16_v16i32(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A0:%.*]], <8 x i16> [[A1:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-LABEL: define <16 x i32> @concat_zext_v8i16_v16i32(
+; CHECK-SAME: <8 x i16> [[A0:%.*]], <8 x i16> [[A1:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A0]], <8 x i16> [[A1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    [[R:%.*]] = zext <16 x i16> [[TMP1]] to <16 x i32>
 ; CHECK-NEXT:    ret <16 x i32> [[R]]
 ;
@@ -17,8 +18,9 @@ define <16 x i32> @concat_zext_v8i16_v16i32(<8 x i16> %a0, <8 x i16> %a1) {
 }
 
 define <16 x i32> @concat_zext_nneg_v8i16_v16i32(<8 x i16> %a0, <8 x i16> %a1) {
-; CHECK-LABEL: @concat_zext_nneg_v8i16_v16i32(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A0:%.*]], <8 x i16> [[A1:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-LABEL: define <16 x i32> @concat_zext_nneg_v8i16_v16i32(
+; CHECK-SAME: <8 x i16> [[A0:%.*]], <8 x i16> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A0]], <8 x i16> [[A1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    [[R:%.*]] = zext nneg <16 x i16> [[TMP1]] to <16 x i32>
 ; CHECK-NEXT:    ret <16 x i32> [[R]]
 ;
@@ -29,14 +31,16 @@ define <16 x i32> @concat_zext_nneg_v8i16_v16i32(<8 x i16> %a0, <8 x i16> %a1) {
 }
 
 define <16 x i32> @concat_sext_zext_nneg_v8i16_v8i32(<8 x i16> %a0, <8 x i16> %a1) {
-; SSE-LABEL: @concat_sext_zext_nneg_v8i16_v8i32(
-; SSE-NEXT:    [[X0:%.*]] = sext <8 x i16> [[A0:%.*]] to <8 x i32>
-; SSE-NEXT:    [[X1:%.*]] = zext nneg <8 x i16> [[A1:%.*]] to <8 x i32>
+; SSE-LABEL: define <16 x i32> @concat_sext_zext_nneg_v8i16_v8i32(
+; SSE-SAME: <8 x i16> [[A0:%.*]], <8 x i16> [[A1:%.*]]) #[[ATTR0]] {
+; SSE-NEXT:    [[X0:%.*]] = sext <8 x i16> [[A0]] to <8 x i32>
+; SSE-NEXT:    [[X1:%.*]] = zext nneg <8 x i16> [[A1]] to <8 x i32>
 ; SSE-NEXT:    [[R:%.*]] = shufflevector <8 x i32> [[X0]], <8 x i32> [[X1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; SSE-NEXT:    ret <16 x i32> [[R]]
 ;
-; AVX-LABEL: @concat_sext_zext_nneg_v8i16_v8i32(
-; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A0:%.*]], <8 x i16> [[A1:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX-LABEL: define <16 x i32> @concat_sext_zext_nneg_v8i16_v8i32(
+; AVX-SAME: <8 x i16> [[A0:%.*]], <8 x i16> [[A1:%.*]]) #[[ATTR0]] {
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A0]], <8 x i16> [[A1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; AVX-NEXT:    [[R:%.*]] = sext <16 x i16> [[TMP1]] to <16 x i32>
 ; AVX-NEXT:    ret <16 x i32> [[R]]
 ;
@@ -47,8 +51,9 @@ define <16 x i32> @concat_sext_zext_nneg_v8i16_v8i32(<8 x i16> %a0, <8 x i16> %a
 }
 
 define <16 x i32> @concat_sext_v8i16_v16i32(<8 x i16> %a0, <8 x i16> %a1) {
-; CHECK-LABEL: @concat_sext_v8i16_v16i32(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A0:%.*]], <8 x i16> [[A1:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-LABEL: define <16 x i32> @concat_sext_v8i16_v16i32(
+; CHECK-SAME: <8 x i16> [[A0:%.*]], <8 x i16> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A0]], <8 x i16> [[A1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    [[R:%.*]] = sext <16 x i16> [[TMP1]] to <16 x i32>
 ; CHECK-NEXT:    ret <16 x i32> [[R]]
 ;
@@ -59,8 +64,9 @@ define <16 x i32> @concat_sext_v8i16_v16i32(<8 x i16> %a0, <8 x i16> %a1) {
 }
 
 define <8 x i32> @concat_sext_v4i1_v8i32(<4 x i1> %a0, <4 x i1> %a1) {
-; CHECK-LABEL: @concat_sext_v4i1_v8i32(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i1> [[A0:%.*]], <4 x i1> [[A1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-LABEL: define <8 x i32> @concat_sext_v4i1_v8i32(
+; CHECK-SAME: <4 x i1> [[A0:%.*]], <4 x i1> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i1> [[A0]], <4 x i1> [[A1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    [[R:%.*]] = sext <8 x i1> [[TMP1]] to <8 x i32>
 ; CHECK-NEXT:    ret <8 x i32> [[R]]
 ;
@@ -71,8 +77,9 @@ define <8 x i32> @concat_sext_v4i1_v8i32(<4 x i1> %a0, <4 x i1> %a1) {
 }
 
 define <8 x i16> @concat_trunc_v4i32_v8i16(<4 x i32> %a0, <4 x i32> %a1) {
-; CHECK-LABEL: @concat_trunc_v4i32_v8i16(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[A0:%.*]], <4 x i32> [[A1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-LABEL: define <8 x i16> @concat_trunc_v4i32_v8i16(
+; CHECK-SAME: <4 x i32> [[A0:%.*]], <4 x i32> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[A0]], <4 x i32> [[A1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    [[R:%.*]] = trunc <8 x i32> [[TMP1]] to <8 x i16>
 ; CHECK-NEXT:    ret <8 x i16> [[R]]
 ;
@@ -83,8 +90,9 @@ define <8 x i16> @concat_trunc_v4i32_v8i16(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <8 x ptr> @concat_inttoptr_v4i32_v8iptr(<4 x i32> %a0, <4 x i32> %a1) {
-; CHECK-LABEL: @concat_inttoptr_v4i32_v8iptr(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[A0:%.*]], <4 x i32> [[A1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-LABEL: define <8 x ptr> @concat_inttoptr_v4i32_v8iptr(
+; CHECK-SAME: <4 x i32> [[A0:%.*]], <4 x i32> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[A0]], <4 x i32> [[A1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    [[R:%.*]] = inttoptr <8 x i32> [[TMP1]] to <8 x ptr>
 ; CHECK-NEXT:    ret <8 x ptr> [[R]]
 ;
@@ -95,8 +103,9 @@ define <8 x ptr> @concat_inttoptr_v4i32_v8iptr(<4 x i32> %a0, <4 x i32> %a1) {
 }
 
 define <16 x i64> @concat_ptrtoint_v8i16_v16i32(<8 x ptr> %a0, <8 x ptr> %a1) {
-; CHECK-LABEL: @concat_ptrtoint_v8i16_v16i32(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x ptr> [[A0:%.*]], <8 x ptr> [[A1:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-LABEL: define <16 x i64> @concat_ptrtoint_v8i16_v16i32(
+; CHECK-SAME: <8 x ptr> [[A0:%.*]], <8 x ptr> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x ptr> [[A0]], <8 x ptr> [[A1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    [[R:%.*]] = ptrtoint <16 x ptr> [[TMP1]] to <16 x i64>
 ; CHECK-NEXT:    ret <16 x i64> [[R]]
 ;
@@ -107,14 +116,16 @@ define <16 x i64> @concat_ptrtoint_v8i16_v16i32(<8 x ptr> %a0, <8 x ptr> %a1) {
 }
 
 define <8 x double> @concat_fpext_v4f32_v8f64(<4 x float> %a0, <4 x float> %a1) {
-; SSE-LABEL: @concat_fpext_v4f32_v8f64(
-; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-LABEL: define <8 x double> @concat_fpext_v4f32_v8f64(
+; SSE-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]]) #[[ATTR0]] {
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[A0]], <4 x float> [[A1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; SSE-NEXT:    [[R:%.*]] = fpext <8 x float> [[TMP1]] to <8 x double>
 ; SSE-NEXT:    ret <8 x double> [[R]]
 ;
-; AVX-LABEL: @concat_fpext_v4f32_v8f64(
-; AVX-NEXT:    [[X0:%.*]] = fpext <4 x float> [[A0:%.*]] to <4 x double>
-; AVX-NEXT:    [[X1:%.*]] = fpext <4 x float> [[A1:%.*]] to <4 x double>
+; AVX-LABEL: define <8 x double> @concat_fpext_v4f32_v8f64(
+; AVX-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]]) #[[ATTR0]] {
+; AVX-NEXT:    [[X0:%.*]] = fpext <4 x float> [[A0]] to <4 x double>
+; AVX-NEXT:    [[X1:%.*]] = fpext <4 x float> [[A1]] to <4 x double>
 ; AVX-NEXT:    [[R:%.*]] = shufflevector <4 x double> [[X0]], <4 x double> [[X1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX-NEXT:    ret <8 x double> [[R]]
 ;
@@ -125,9 +136,10 @@ define <8 x double> @concat_fpext_v4f32_v8f64(<4 x float> %a0, <4 x float> %a1)
 }
 
 define <16 x float> @concat_fptrunc_v8f64_v16f32(<8 x double> %a0, <8 x double> %a1) {
-; CHECK-LABEL: @concat_fptrunc_v8f64_v16f32(
-; CHECK-NEXT:    [[X0:%.*]] = fptrunc <8 x double> [[A0:%.*]] to <8 x float>
-; CHECK-NEXT:    [[X1:%.*]] = fptrunc <8 x double> [[A1:%.*]] to <8 x float>
+; CHECK-LABEL: define <16 x float> @concat_fptrunc_v8f64_v16f32(
+; CHECK-SAME: <8 x double> [[A0:%.*]], <8 x double> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[X0:%.*]] = fptrunc <8 x double> [[A0]] to <8 x float>
+; CHECK-NEXT:    [[X1:%.*]] = fptrunc <8 x double> [[A1]] to <8 x float>
 ; CHECK-NEXT:    [[R:%.*]] = shufflevector <8 x float> [[X0]], <8 x float> [[X1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    ret <16 x float> [[R]]
 ;
@@ -140,8 +152,9 @@ define <16 x float> @concat_fptrunc_v8f64_v16f32(<8 x double> %a0, <8 x double>
 ; commuted vector concatenation
 
 define <16 x i32> @rconcat_sext_v8i16_v16i32(<8 x i16> %a0, <8 x i16> %a1) {
-; CHECK-LABEL: @rconcat_sext_v8i16_v16i32(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A0:%.*]], <8 x i16> [[A1:%.*]], <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-LABEL: define <16 x i32> @rconcat_sext_v8i16_v16i32(
+; CHECK-SAME: <8 x i16> [[A0:%.*]], <8 x i16> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A0]], <8 x i16> [[A1]], <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    [[R:%.*]] = sext <16 x i16> [[TMP1]] to <16 x i32>
 ; CHECK-NEXT:    ret <16 x i32> [[R]]
 ;
@@ -154,8 +167,9 @@ define <16 x i32> @rconcat_sext_v8i16_v16i32(<8 x i16> %a0, <8 x i16> %a1) {
 ; interleaved shuffle
 
 define <8 x double> @interleave_fpext_v4f32_v8f64(<4 x float> %a0, <4 x float> %a1) {
-; CHECK-LABEL: @interleave_fpext_v4f32_v8f64(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]], <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; CHECK-LABEL: define <8 x double> @interleave_fpext_v4f32_v8f64(
+; CHECK-SAME: <4 x float> [[A0:%.*]], <4 x float> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[A0]], <4 x float> [[A1]], <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
 ; CHECK-NEXT:    [[R:%.*]] = fpext <8 x float> [[TMP1]] to <8 x double>
 ; CHECK-NEXT:    ret <8 x double> [[R]]
 ;
@@ -168,8 +182,9 @@ define <8 x double> @interleave_fpext_v4f32_v8f64(<4 x float> %a0, <4 x float> %
 ; bitcasts (same element count)
 
 define <8 x float> @concat_bitcast_v4i32_v8f32(<4 x i32> %a0, <4 x i32> %a1) {
-; CHECK-LABEL: @concat_bitcast_v4i32_v8f32(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[A0:%.*]], <4 x i32> [[A1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-LABEL: define <8 x float> @concat_bitcast_v4i32_v8f32(
+; CHECK-SAME: <4 x i32> [[A0:%.*]], <4 x i32> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[A0]], <4 x i32> [[A1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    [[R:%.*]] = bitcast <8 x i32> [[TMP1]] to <8 x float>
 ; CHECK-NEXT:    ret <8 x float> [[R]]
 ;
@@ -182,8 +197,9 @@ define <8 x float> @concat_bitcast_v4i32_v8f32(<4 x i32> %a0, <4 x i32> %a1) {
 ; bitcasts (lower element count)
 
 define <4 x double> @concat_bitcast_v8i16_v4f64(<8 x i16> %a0, <8 x i16> %a1) {
-; CHECK-LABEL: @concat_bitcast_v8i16_v4f64(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A0:%.*]], <8 x i16> [[A1:%.*]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-LABEL: define <4 x double> @concat_bitcast_v8i16_v4f64(
+; CHECK-SAME: <8 x i16> [[A0:%.*]], <8 x i16> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i16> [[A0]], <8 x i16> [[A1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    [[R:%.*]] = bitcast <16 x i16> [[TMP1]] to <4 x double>
 ; CHECK-NEXT:    ret <4 x double> [[R]]
 ;
@@ -196,8 +212,9 @@ define <4 x double> @concat_bitcast_v8i16_v4f64(<8 x i16> %a0, <8 x i16> %a1) {
 ; bitcasts (higher element count)
 
 define <16 x i16> @concat_bitcast_v4i32_v16i16(<4 x i32> %a0, <4 x i32> %a1) {
-; CHECK-LABEL: @concat_bitcast_v4i32_v16i16(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[A0:%.*]], <4 x i32> [[A1:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-LABEL: define <16 x i16> @concat_bitcast_v4i32_v16i16(
+; CHECK-SAME: <4 x i32> [[A0:%.*]], <4 x i32> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[A0]], <4 x i32> [[A1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    [[R:%.*]] = bitcast <8 x i32> [[TMP1]] to <16 x i16>
 ; CHECK-NEXT:    ret <16 x i16> [[R]]
 ;
@@ -210,11 +227,12 @@ define <16 x i16> @concat_bitcast_v4i32_v16i16(<4 x i32> %a0, <4 x i32> %a1) {
 ; negative - multiuse
 
 define <8 x i16> @concat_trunc_v4i32_v8i16_multiuse(<4 x i32> %a0, <4 x i32> %a1, ptr %a2) {
-; CHECK-LABEL: @concat_trunc_v4i32_v8i16_multiuse(
-; CHECK-NEXT:    [[X0:%.*]] = trunc <4 x i32> [[A0:%.*]] to <4 x i16>
-; CHECK-NEXT:    [[X1:%.*]] = trunc <4 x i32> [[A1:%.*]] to <4 x i16>
+; CHECK-LABEL: define <8 x i16> @concat_trunc_v4i32_v8i16_multiuse(
+; CHECK-SAME: <4 x i32> [[A0:%.*]], <4 x i32> [[A1:%.*]], ptr [[A2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[X0:%.*]] = trunc <4 x i32> [[A0]] to <4 x i16>
+; CHECK-NEXT:    [[X1:%.*]] = trunc <4 x i32> [[A1]] to <4 x i16>
 ; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i16> [[X0]], <4 x i16> [[X1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT:    store <4 x i16> [[X0]], ptr [[A2:%.*]], align 8
+; CHECK-NEXT:    store <4 x i16> [[X0]], ptr [[A2]], align 8
 ; CHECK-NEXT:    ret <8 x i16> [[R]]
 ;
   %x0 = trunc <4 x i32> %a0 to <4 x i16>
@@ -227,9 +245,10 @@ define <8 x i16> @concat_trunc_v4i32_v8i16_multiuse(<4 x i32> %a0, <4 x i32> %a1
 ; negative - bitcasts (unscalable higher element count)
 
 define <16 x i16> @revpair_bitcast_v4i32_v16i16(<4 x i32> %a0, <4 x i32> %a1) {
-; CHECK-LABEL: @revpair_bitcast_v4i32_v16i16(
-; CHECK-NEXT:    [[X0:%.*]] = bitcast <4 x i32> [[A0:%.*]] to <8 x i16>
-; CHECK-NEXT:    [[X1:%.*]] = bitcast <4 x i32> [[A1:%.*]] to <8 x i16>
+; CHECK-LABEL: define <16 x i16> @revpair_bitcast_v4i32_v16i16(
+; CHECK-SAME: <4 x i32> [[A0:%.*]], <4 x i32> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[X0:%.*]] = bitcast <4 x i32> [[A0]] to <8 x i16>
+; CHECK-NEXT:    [[X1:%.*]] = bitcast <4 x i32> [[A1]] to <8 x i16>
 ; CHECK-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[X0]], <8 x i16> [[X1]], <16 x i32> <i32 1, i32 0, i32 3, i32 3, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
 ; CHECK-NEXT:    ret <16 x i16> [[R]]
 ;
@@ -242,9 +261,10 @@ define <16 x i16> @revpair_bitcast_v4i32_v16i16(<4 x i32> %a0, <4 x i32> %a1) {
 ; negative - bitcasts (unscalable element counts)
 
 define <4 x i32> @shuffle_bitcast_v32i40_v4i32(<32 x i40> %a0, <32 x i40> %a1) {
-; CHECK-LABEL: @shuffle_bitcast_v32i40_v4i32(
-; CHECK-NEXT:    [[X0:%.*]] = bitcast <32 x i40> [[A0:%.*]] to <40 x i32>
-; CHECK-NEXT:    [[X1:%.*]] = bitcast <32 x i40> [[A1:%.*]] to <40 x i32>
+; CHECK-LABEL: define <4 x i32> @shuffle_bitcast_v32i40_v4i32(
+; CHECK-SAME: <32 x i40> [[A0:%.*]], <32 x i40> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[X0:%.*]] = bitcast <32 x i40> [[A0]] to <40 x i32>
+; CHECK-NEXT:    [[X1:%.*]] = bitcast <32 x i40> [[A1]] to <40 x i32>
 ; CHECK-NEXT:    [[R:%.*]] = shufflevector <40 x i32> [[X0]], <40 x i32> [[X1]], <4 x i32> <i32 0, i32 42, i32 poison, i32 poison>
 ; CHECK-NEXT:    ret <4 x i32> [[R]]
 ;
@@ -257,9 +277,10 @@ define <4 x i32> @shuffle_bitcast_v32i40_v4i32(<32 x i40> %a0, <32 x i40> %a1) {
 ; negative - src type mismatch
 
 define <8 x i32> @concat_sext_v4i8_v4i16_v8i32(<4 x i8> %a0, <4 x i16> %a1) {
-; CHECK-LABEL: @concat_sext_v4i8_v4i16_v8i32(
-; CHECK-NEXT:    [[X0:%.*]] = sext <4 x i8> [[A0:%.*]] to <4 x i32>
-; CHECK-NEXT:    [[X1:%.*]] = sext <4 x i16> [[A1:%.*]] to <4 x i32>
+; CHECK-LABEL: define <8 x i32> @concat_sext_v4i8_v4i16_v8i32(
+; CHECK-SAME: <4 x i8> [[A0:%.*]], <4 x i16> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[X0:%.*]] = sext <4 x i8> [[A0]] to <4 x i32>
+; CHECK-NEXT:    [[X1:%.*]] = sext <4 x i16> [[A1]] to <4 x i32>
 ; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[X0]], <4 x i32> [[X1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    ret <8 x i32> [[R]]
 ;
@@ -272,9 +293,10 @@ define <8 x i32> @concat_sext_v4i8_v4i16_v8i32(<4 x i8> %a0, <4 x i16> %a1) {
 ; negative - castop mismatch
 
 define <16 x i32> @concat_sext_zext_v8i16_v16i32(<8 x i16> %a0, <8 x i16> %a1) {
-; CHECK-LABEL: @concat_sext_zext_v8i16_v16i32(
-; CHECK-NEXT:    [[X0:%.*]] = sext <8 x i16> [[A0:%.*]] to <8 x i32>
-; CHECK-NEXT:    [[X1:%.*]] = zext <8 x i16> [[A1:%.*]] to <8 x i32>
+; CHECK-LABEL: define <16 x i32> @concat_sext_zext_v8i16_v16i32(
+; CHECK-SAME: <8 x i16> [[A0:%.*]], <8 x i16> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[X0:%.*]] = sext <8 x i16> [[A0]] to <8 x i32>
+; CHECK-NEXT:    [[X1:%.*]] = zext <8 x i16> [[A1]] to <8 x i32>
 ; CHECK-NEXT:    [[R:%.*]] = shufflevector <8 x i32> [[X0]], <8 x i32> [[X1]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 ; CHECK-NEXT:    ret <16 x i32> [[R]]
 ;
diff --git a/llvm/test/Transforms/VectorCombine/X86/shuffle.ll b/llvm/test/Transforms/VectorCombine/X86/shuffle.ll
index 8337bb37bc54..bb6402347a9b 100644
--- a/llvm/test/Transforms/VectorCombine/X86/shuffle.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/shuffle.ll
@@ -1,17 +1,19 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
 ; RUN: opt < %s -passes=vector-combine -S -mtriple=x86_64-- -mattr=SSE2 | FileCheck %s --check-prefixes=CHECK,SSE
 ; RUN: opt < %s -passes=vector-combine -S -mtriple=x86_64-- -mattr=AVX2 | FileCheck %s --check-prefixes=CHECK,AVX
 
 ; x86 does not have a cheap v16i8 shuffle until SSSE3 (pshufb)
 
 define <16 x i8> @bitcast_shuf_narrow_element(<4 x i32> %v) {
-; SSE-LABEL: @bitcast_shuf_narrow_element(
-; SSE-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[V:%.*]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; SSE-LABEL: define <16 x i8> @bitcast_shuf_narrow_element(
+; SSE-SAME: <4 x i32> [[V:%.*]]) #[[ATTR0:[0-9]+]] {
+; SSE-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[V]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
 ; SSE-NEXT:    [[R:%.*]] = bitcast <4 x i32> [[SHUF]] to <16 x i8>
 ; SSE-NEXT:    ret <16 x i8> [[R]]
 ;
-; AVX-LABEL: @bitcast_shuf_narrow_element(
-; AVX-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[V:%.*]] to <16 x i8>
+; AVX-LABEL: define <16 x i8> @bitcast_shuf_narrow_element(
+; AVX-SAME: <4 x i32> [[V:%.*]]) #[[ATTR0:[0-9]+]] {
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[V]] to <16 x i8>
 ; AVX-NEXT:    [[R:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> <i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
 ; AVX-NEXT:    ret <16 x i8> [[R]]
 ;
@@ -23,8 +25,9 @@ define <16 x i8> @bitcast_shuf_narrow_element(<4 x i32> %v) {
 ; v4f32 is the same cost as v4i32, so this always works
 
 define <4 x float> @bitcast_shuf_same_size(<4 x i32> %v) {
-; CHECK-LABEL: @bitcast_shuf_same_size(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[V:%.*]] to <4 x float>
+; CHECK-LABEL: define <4 x float> @bitcast_shuf_same_size(
+; CHECK-SAME: <4 x i32> [[V:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <4 x i32> [[V]] to <4 x float>
 ; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
@@ -36,13 +39,15 @@ define <4 x float> @bitcast_shuf_same_size(<4 x i32> %v) {
 ; Length-changing shuffles
 
 define <16 x i8> @bitcast_shuf_narrow_element_subvector(<2 x i32> %v) {
-; SSE-LABEL: @bitcast_shuf_narrow_element_subvector(
-; SSE-NEXT:    [[SHUF:%.*]] = shufflevector <2 x i32> [[V:%.*]], <2 x i32> poison, <4 x i32> <i32 1, i32 0, i32 1, i32 0>
+; SSE-LABEL: define <16 x i8> @bitcast_shuf_narrow_element_subvector(
+; SSE-SAME: <2 x i32> [[V:%.*]]) #[[ATTR0]] {
+; SSE-NEXT:    [[SHUF:%.*]] = shufflevector <2 x i32> [[V]], <2 x i32> poison, <4 x i32> <i32 1, i32 0, i32 1, i32 0>
 ; SSE-NEXT:    [[R:%.*]] = bitcast <4 x i32> [[SHUF]] to <16 x i8>
 ; SSE-NEXT:    ret <16 x i8> [[R]]
 ;
-; AVX-LABEL: @bitcast_shuf_narrow_element_subvector(
-; AVX-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[V:%.*]] to <8 x i8>
+; AVX-LABEL: define <16 x i8> @bitcast_shuf_narrow_element_subvector(
+; AVX-SAME: <2 x i32> [[V:%.*]]) #[[ATTR0]] {
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast <2 x i32> [[V]] to <8 x i8>
 ; AVX-NEXT:    [[R:%.*]] = shufflevector <8 x i8> [[TMP1]], <8 x i8> poison, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
 ; AVX-NEXT:    ret <16 x i8> [[R]]
 ;
@@ -52,13 +57,15 @@ define <16 x i8> @bitcast_shuf_narrow_element_subvector(<2 x i32> %v) {
 }
 
 define <16 x i16> @bitcast_shuf_narrow_element_concat_subvectors(<2 x i64> %v) {
-; SSE-LABEL: @bitcast_shuf_narrow_element_concat_subvectors(
-; SSE-NEXT:    [[SHUF:%.*]] = shufflevector <2 x i64> [[V:%.*]], <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; SSE-LABEL: define <16 x i16> @bitcast_shuf_narrow_element_concat_subvectors(
+; SSE-SAME: <2 x i64> [[V:%.*]]) #[[ATTR0]] {
+; SSE-NEXT:    [[SHUF:%.*]] = shufflevector <2 x i64> [[V]], <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
 ; SSE-NEXT:    [[R:%.*]] = bitcast <4 x i64> [[SHUF]] to <16 x i16>
 ; SSE-NEXT:    ret <16 x i16> [[R]]
 ;
-; AVX-LABEL: @bitcast_shuf_narrow_element_concat_subvectors(
-; AVX-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[V:%.*]] to <8 x i16>
+; AVX-LABEL: define <16 x i16> @bitcast_shuf_narrow_element_concat_subvectors(
+; AVX-SAME: <2 x i64> [[V:%.*]]) #[[ATTR0]] {
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[V]] to <8 x i16>
 ; AVX-NEXT:    [[R:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; AVX-NEXT:    ret <16 x i16> [[R]]
 ;
@@ -68,8 +75,9 @@ define <16 x i16> @bitcast_shuf_narrow_element_concat_subvectors(<2 x i64> %v) {
 }
 
 define <16 x i8> @bitcast_shuf_extract_subvector(<8 x i32> %v) {
-; CHECK-LABEL: @bitcast_shuf_extract_subvector(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i32> [[V:%.*]] to <32 x i8>
+; CHECK-LABEL: define <16 x i8> @bitcast_shuf_extract_subvector(
+; CHECK-SAME: <8 x i32> [[V:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i32> [[V]] to <32 x i8>
 ; CHECK-NEXT:    [[R:%.*]] = shufflevector <32 x i8> [[TMP1]], <32 x i8> poison, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
 ; CHECK-NEXT:    ret <16 x i8> [[R]]
 ;
@@ -81,8 +89,9 @@ define <16 x i8> @bitcast_shuf_extract_subvector(<8 x i32> %v) {
 ; Negative test - must cast to vector type
 
 define i128 @bitcast_shuf_narrow_element_wrong_type(<4 x i32> %v) {
-; CHECK-LABEL: @bitcast_shuf_narrow_element_wrong_type(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[V:%.*]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-LABEL: define i128 @bitcast_shuf_narrow_element_wrong_type(
+; CHECK-SAME: <4 x i32> [[V:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[V]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    [[R:%.*]] = bitcast <4 x i32> [[SHUF]] to i128
 ; CHECK-NEXT:    ret i128 [[R]]
 ;
@@ -94,8 +103,9 @@ define i128 @bitcast_shuf_narrow_element_wrong_type(<4 x i32> %v) {
 ; Widen shuffle elements
 
 define <4 x i32> @bitcast_shuf_wide_element(<8 x i16> %v) {
-; CHECK-LABEL: @bitcast_shuf_wide_element(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i16> [[V:%.*]] to <4 x i32>
+; CHECK-LABEL: define <4 x i32> @bitcast_shuf_wide_element(
+; CHECK-SAME: <8 x i16> [[V:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <8 x i16> [[V]] to <4 x i32>
 ; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
 ; CHECK-NEXT:    ret <4 x i32> [[R]]
 ;
@@ -109,8 +119,9 @@ declare void @use(<4 x i32>)
 ; Negative test - don't create an extra shuffle
 
 define <16 x i8> @bitcast_shuf_uses(<4 x i32> %v) {
-; CHECK-LABEL: @bitcast_shuf_uses(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[V:%.*]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-LABEL: define <16 x i8> @bitcast_shuf_uses(
+; CHECK-SAME: <4 x i32> [[V:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <4 x i32> [[V]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
 ; CHECK-NEXT:    call void @use(<4 x i32> [[SHUF]])
 ; CHECK-NEXT:    [[R:%.*]] = bitcast <4 x i32> [[SHUF]] to <16 x i8>
 ; CHECK-NEXT:    ret <16 x i8> [[R]]
@@ -125,8 +136,9 @@ define <16 x i8> @bitcast_shuf_uses(<4 x i32> %v) {
 ; TODO - can we remove the empty bitcast(bitcast()) ?
 
 define <4 x i64> @bitcast_shuf_remove_bitcasts(<2 x i64> %a0, <2 x i64> %a1) {
-; CHECK-LABEL: @bitcast_shuf_remove_bitcasts(
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i64> [[A0:%.*]], <2 x i64> [[A1:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-LABEL: define <4 x i64> @bitcast_shuf_remove_bitcasts(
+; CHECK-SAME: <2 x i64> [[A0:%.*]], <2 x i64> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i64> [[A0]], <2 x i64> [[A1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[SHUF:%.*]] = bitcast <4 x i64> [[R]] to <8 x i32>
 ; CHECK-NEXT:    [[R1:%.*]] = bitcast <8 x i32> [[SHUF]] to <4 x i64>
 ; CHECK-NEXT:    ret <4 x i64> [[R1]]
@@ -141,9 +153,10 @@ define <4 x i64> @bitcast_shuf_remove_bitcasts(<2 x i64> %a0, <2 x i64> %a1) {
 ; shuffle of 2 operands must reduce bitcasts
 
 define <8 x i32> @bitcast_shuf_one_bitcast(<4 x i32> %a0, <2 x i64> %a1) {
-; CHECK-LABEL: @bitcast_shuf_one_bitcast(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[A1:%.*]] to <4 x i32>
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[A0:%.*]], <4 x i32> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-LABEL: define <8 x i32> @bitcast_shuf_one_bitcast(
+; CHECK-SAME: <4 x i32> [[A0:%.*]], <2 x i64> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[A1]] to <4 x i32>
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[A0]], <4 x i32> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
 ; CHECK-NEXT:    ret <8 x i32> [[R]]
 ;
   %bc0 = bitcast <4 x i32> %a0 to <2 x i64>
@@ -155,8 +168,9 @@ define <8 x i32> @bitcast_shuf_one_bitcast(<4 x i32> %a0, <2 x i64> %a1) {
 ; Negative test - shuffle of 2 operands must not increase bitcasts
 
 define <8 x i32> @bitcast_shuf_too_many_bitcasts(<2 x i64> %a0, <2 x i64> %a1) {
-; CHECK-LABEL: @bitcast_shuf_too_many_bitcasts(
-; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x i64> [[A0:%.*]], <2 x i64> [[A1:%.*]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-LABEL: define <8 x i32> @bitcast_shuf_too_many_bitcasts(
+; CHECK-SAME: <2 x i64> [[A0:%.*]], <2 x i64> [[A1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x i64> [[A0]], <2 x i64> [[A1]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK-NEXT:    [[R:%.*]] = bitcast <4 x i64> [[SHUF]] to <8 x i32>
 ; CHECK-NEXT:    ret <8 x i32> [[R]]
 ;
@@ -166,8 +180,9 @@ define <8 x i32> @bitcast_shuf_too_many_bitcasts(<2 x i64> %a0, <2 x i64> %a1) {
 }
 
 define <2 x i64> @PR35454_1(<2 x i64> %v) {
-; SSE-LABEL: @PR35454_1(
-; SSE-NEXT:    [[BC:%.*]] = bitcast <2 x i64> [[V:%.*]] to <4 x i32>
+; SSE-LABEL: define <2 x i64> @PR35454_1(
+; SSE-SAME: <2 x i64> [[V:%.*]]) #[[ATTR0]] {
+; SSE-NEXT:    [[BC:%.*]] = bitcast <2 x i64> [[V]] to <4 x i32>
 ; SSE-NEXT:    [[PERMIL:%.*]] = shufflevector <4 x i32> [[BC]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
 ; SSE-NEXT:    [[BC1:%.*]] = bitcast <4 x i32> [[PERMIL]] to <16 x i8>
 ; SSE-NEXT:    [[ADD:%.*]] = shl <16 x i8> [[BC1]], <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
@@ -176,8 +191,9 @@ define <2 x i64> @PR35454_1(<2 x i64> %v) {
 ; SSE-NEXT:    [[BC3:%.*]] = bitcast <4 x i32> [[PERMIL1]] to <2 x i64>
 ; SSE-NEXT:    ret <2 x i64> [[BC3]]
 ;
-; AVX-LABEL: @PR35454_1(
-; AVX-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[V:%.*]] to <16 x i8>
+; AVX-LABEL: define <2 x i64> @PR35454_1(
+; AVX-SAME: <2 x i64> [[V:%.*]]) #[[ATTR0]] {
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[V]] to <16 x i8>
 ; AVX-NEXT:    [[BC1:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <16 x i32> <i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
 ; AVX-NEXT:    [[ADD:%.*]] = shl <16 x i8> [[BC1]], <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
 ; AVX-NEXT:    [[BC2:%.*]] = bitcast <16 x i8> [[ADD]] to <4 x i32>
@@ -196,8 +212,9 @@ define <2 x i64> @PR35454_1(<2 x i64> %v) {
 }
 
 define <2 x i64> @PR35454_2(<2 x i64> %v) {
-; SSE-LABEL: @PR35454_2(
-; SSE-NEXT:    [[BC:%.*]] = bitcast <2 x i64> [[V:%.*]] to <4 x i32>
+; SSE-LABEL: define <2 x i64> @PR35454_2(
+; SSE-SAME: <2 x i64> [[V:%.*]]) #[[ATTR0]] {
+; SSE-NEXT:    [[BC:%.*]] = bitcast <2 x i64> [[V]] to <4 x i32>
 ; SSE-NEXT:    [[PERMIL:%.*]] = shufflevector <4 x i32> [[BC]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
 ; SSE-NEXT:    [[BC1:%.*]] = bitcast <4 x i32> [[PERMIL]] to <8 x i16>
 ; SSE-NEXT:    [[ADD:%.*]] = shl <8 x i16> [[BC1]], <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
@@ -206,8 +223,9 @@ define <2 x i64> @PR35454_2(<2 x i64> %v) {
 ; SSE-NEXT:    [[BC3:%.*]] = bitcast <4 x i32> [[PERMIL1]] to <2 x i64>
 ; SSE-NEXT:    ret <2 x i64> [[BC3]]
 ;
-; AVX-LABEL: @PR35454_2(
-; AVX-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[V:%.*]] to <8 x i16>
+; AVX-LABEL: define <2 x i64> @PR35454_2(
+; AVX-SAME: <2 x i64> [[V:%.*]]) #[[ATTR0]] {
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast <2 x i64> [[V]] to <8 x i16>
 ; AVX-NEXT:    [[BC1:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 6, i32 7, i32 4, i32 5, i32 2, i32 3, i32 0, i32 1>
 ; AVX-NEXT:    [[ADD:%.*]] = shl <8 x i16> [[BC1]], <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
 ; AVX-NEXT:    [[BC2:%.*]] = bitcast <8 x i16> [[ADD]] to <4 x i32>
@@ -228,10 +246,11 @@ define <2 x i64> @PR35454_2(<2 x i64> %v) {
 ; Shuffle is much cheaper than fdiv. FMF are intersected.
 
 define <4 x float> @shuf_fdiv_v4f32_yy(<4 x float> %x, <4 x float> %y, <4 x float> %z) {
-; CHECK-LABEL: @shuf_fdiv_v4f32_yy(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> [[Z:%.*]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[Y:%.*]], <4 x float> poison, <4 x i32> <i32 1, i32 3, i32 1, i32 3>
-; CHECK-NEXT:    [[R:%.*]] = fdiv arcp <4 x float> [[TMP1]], [[TMP2]]
+; CHECK-LABEL: define <4 x float> @shuf_fdiv_v4f32_yy(
+; CHECK-SAME: <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]], <4 x float> [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[X]], <4 x float> [[Z]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[Y]], <4 x float> poison, <4 x i32> <i32 1, i32 3, i32 1, i32 3>
+; CHECK-NEXT:    [[R:%.*]] = fdiv arcp <4 x float> [[TMP2]], [[TMP3]]
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
   %b0 = fdiv fast <4 x float> %x, %y
@@ -243,11 +262,12 @@ define <4 x float> @shuf_fdiv_v4f32_yy(<4 x float> %x, <4 x float> %y, <4 x floa
 ; Common operand is op0 of the binops.
 
 define <4 x i32> @shuf_add_v4i32_xx(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
-; CHECK-LABEL: @shuf_add_v4i32_xx(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 poison, i32 poison, i32 2, i32 0>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[Y:%.*]], <4 x i32> [[Z:%.*]], <4 x i32> <i32 poison, i32 poison, i32 6, i32 0>
-; CHECK-NEXT:    [[R:%.*]] = add <4 x i32> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    ret <4 x i32> [[R]]
+; CHECK-LABEL: define <4 x i32> @shuf_add_v4i32_xx(
+; CHECK-SAME: <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[X]], <4 x i32> poison, <4 x i32> <i32 poison, i32 poison, i32 2, i32 0>
+; CHECK-NEXT:    [[R1:%.*]] = shufflevector <4 x i32> [[Y]], <4 x i32> [[Z]], <4 x i32> <i32 poison, i32 poison, i32 6, i32 0>
+; CHECK-NEXT:    [[R2:%.*]] = add <4 x i32> [[TMP1]], [[R1]]
+; CHECK-NEXT:    ret <4 x i32> [[R2]]
 ;
   %b0 = add <4 x i32> %x, %y
   %b1 = add <4 x i32> %x, %z
@@ -258,9 +278,10 @@ define <4 x i32> @shuf_add_v4i32_xx(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
 ; For commutative instructions, common operand may be swapped.
 
 define <4 x float> @shuf_fmul_v4f32_xx_swap(<4 x float> %x, <4 x float> %y, <4 x float> %z) {
-; CHECK-LABEL: @shuf_fmul_v4f32_xx_swap(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[Y:%.*]], <4 x float> [[Z:%.*]], <4 x i32> <i32 0, i32 3, i32 4, i32 7>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> poison, <4 x i32> <i32 0, i32 3, i32 0, i32 3>
+; CHECK-LABEL: define <4 x float> @shuf_fmul_v4f32_xx_swap(
+; CHECK-SAME: <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]], <4 x float> [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[Y]], <4 x float> [[Z]], <4 x i32> <i32 0, i32 3, i32 4, i32 7>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[X]], <4 x float> poison, <4 x i32> <i32 0, i32 3, i32 0, i32 3>
 ; CHECK-NEXT:    [[R:%.*]] = fmul <4 x float> [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
@@ -273,9 +294,10 @@ define <4 x float> @shuf_fmul_v4f32_xx_swap(<4 x float> %x, <4 x float> %y, <4 x
 ; For commutative instructions, common operand may be swapped.
 
 define <2 x i64> @shuf_and_v2i64_yy_swap(<2 x i64> %x, <2 x i64> %y, <2 x i64> %z) {
-; CHECK-LABEL: @shuf_and_v2i64_yy_swap(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i64> [[Y:%.*]], <2 x i64> poison, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i64> [[X:%.*]], <2 x i64> [[Z:%.*]], <2 x i32> <i32 3, i32 0>
+; CHECK-LABEL: define <2 x i64> @shuf_and_v2i64_yy_swap(
+; CHECK-SAME: <2 x i64> [[X:%.*]], <2 x i64> [[Y:%.*]], <2 x i64> [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i64> [[Y]], <2 x i64> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i64> [[X]], <2 x i64> [[Z]], <2 x i32> <i32 3, i32 0>
 ; CHECK-NEXT:    [[R:%.*]] = and <2 x i64> [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    ret <2 x i64> [[R]]
 ;
@@ -288,9 +310,10 @@ define <2 x i64> @shuf_and_v2i64_yy_swap(<2 x i64> %x, <2 x i64> %y, <2 x i64> %
 ; non-commutative binop, but common op0
 
 define <4 x i32> @shuf_shl_v4i32_xx(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
-; CHECK-LABEL: @shuf_shl_v4i32_xx(
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> poison, <4 x i32> <i32 3, i32 1, i32 1, i32 2>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[Y:%.*]], <4 x i32> [[Z:%.*]], <4 x i32> <i32 3, i32 1, i32 1, i32 6>
+; CHECK-LABEL: define <4 x i32> @shuf_shl_v4i32_xx(
+; CHECK-SAME: <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[X]], <4 x i32> poison, <4 x i32> <i32 3, i32 1, i32 1, i32 2>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[Y]], <4 x i32> [[Z]], <4 x i32> <i32 3, i32 1, i32 1, i32 6>
 ; CHECK-NEXT:    [[R:%.*]] = shl <4 x i32> [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    ret <4 x i32> [[R]]
 ;
@@ -303,11 +326,12 @@ define <4 x i32> @shuf_shl_v4i32_xx(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
 ; negative test - common operand, but not commutable
 
 define <4 x i32> @shuf_shl_v4i32_xx_swap(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
-; CHECK-LABEL: @shuf_shl_v4i32_xx_swap(
-; CHECK-NEXT:    [[B0:%.*]] = shl <4 x i32> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[B1:%.*]] = shl <4 x i32> [[Z:%.*]], [[X]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[B0]], <4 x i32> [[B1]], <4 x i32> <i32 3, i32 2, i32 2, i32 5>
-; CHECK-NEXT:    ret <4 x i32> [[R]]
+; CHECK-LABEL: define <4 x i32> @shuf_shl_v4i32_xx_swap(
+; CHECK-SAME: <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[B0:%.*]] = shl <4 x i32> [[X]], [[Y]]
+; CHECK-NEXT:    [[B1:%.*]] = shl <4 x i32> [[Z]], [[X]]
+; CHECK-NEXT:    [[R1:%.*]] = shufflevector <4 x i32> [[B0]], <4 x i32> [[B1]], <4 x i32> <i32 3, i32 2, i32 2, i32 5>
+; CHECK-NEXT:    ret <4 x i32> [[R1]]
 ;
   %b0 = shl <4 x i32> %x, %y
   %b1 = shl <4 x i32> %z, %x
@@ -318,9 +342,10 @@ define <4 x i32> @shuf_shl_v4i32_xx_swap(<4 x i32> %x, <4 x i32> %y, <4 x i32> %
 ; negative test - mismatched opcodes
 
 define <2 x i64> @shuf_sub_add_v2i64_yy(<2 x i64> %x, <2 x i64> %y, <2 x i64> %z) {
-; CHECK-LABEL: @shuf_sub_add_v2i64_yy(
-; CHECK-NEXT:    [[B0:%.*]] = sub <2 x i64> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[B1:%.*]] = add <2 x i64> [[Z:%.*]], [[Y]]
+; CHECK-LABEL: define <2 x i64> @shuf_sub_add_v2i64_yy(
+; CHECK-SAME: <2 x i64> [[X:%.*]], <2 x i64> [[Y:%.*]], <2 x i64> [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[B0:%.*]] = sub <2 x i64> [[X]], [[Y]]
+; CHECK-NEXT:    [[B1:%.*]] = add <2 x i64> [[Z]], [[Y]]
 ; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i64> [[B0]], <2 x i64> [[B1]], <2 x i32> <i32 3, i32 0>
 ; CHECK-NEXT:    ret <2 x i64> [[R]]
 ;
@@ -333,9 +358,10 @@ define <2 x i64> @shuf_sub_add_v2i64_yy(<2 x i64> %x, <2 x i64> %y, <2 x i64> %z
 ; negative test - type change via shuffle
 
 define <8 x float> @shuf_fmul_v4f32_xx_type(<4 x float> %x, <4 x float> %y, <4 x float> %z) {
-; CHECK-LABEL: @shuf_fmul_v4f32_xx_type(
-; CHECK-NEXT:    [[B0:%.*]] = fmul <4 x float> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[B1:%.*]] = fmul <4 x float> [[Z:%.*]], [[X]]
+; CHECK-LABEL: define <8 x float> @shuf_fmul_v4f32_xx_type(
+; CHECK-SAME: <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]], <4 x float> [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[B0:%.*]] = fmul <4 x float> [[X]], [[Y]]
+; CHECK-NEXT:    [[B1:%.*]] = fmul <4 x float> [[Z]], [[X]]
 ; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[B0]], <4 x float> [[B1]], <8 x i32> <i32 0, i32 3, i32 4, i32 7, i32 0, i32 1, i32 1, i32 6>
 ; CHECK-NEXT:    ret <8 x float> [[R]]
 ;
@@ -348,10 +374,11 @@ define <8 x float> @shuf_fmul_v4f32_xx_type(<4 x float> %x, <4 x float> %y, <4 x
 ; negative test - uses
 
 define <4 x i32> @shuf_lshr_v4i32_yy_use1(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
-; CHECK-LABEL: @shuf_lshr_v4i32_yy_use1(
-; CHECK-NEXT:    [[B0:%.*]] = lshr <4 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-LABEL: define <4 x i32> @shuf_lshr_v4i32_yy_use1(
+; CHECK-SAME: <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[B0:%.*]] = lshr <4 x i32> [[X]], [[Y]]
 ; CHECK-NEXT:    call void @use(<4 x i32> [[B0]])
-; CHECK-NEXT:    [[B1:%.*]] = lshr <4 x i32> [[Z:%.*]], [[Y]]
+; CHECK-NEXT:    [[B1:%.*]] = lshr <4 x i32> [[Z]], [[Y]]
 ; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[B0]], <4 x i32> [[B1]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
 ; CHECK-NEXT:    ret <4 x i32> [[R]]
 ;
@@ -365,9 +392,10 @@ define <4 x i32> @shuf_lshr_v4i32_yy_use1(<4 x i32> %x, <4 x i32> %y, <4 x i32>
 ; negative test - uses
 
 define <4 x i32> @shuf_mul_v4i32_yy_use2(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
-; CHECK-LABEL: @shuf_mul_v4i32_yy_use2(
-; CHECK-NEXT:    [[B0:%.*]] = mul <4 x i32> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[B1:%.*]] = mul <4 x i32> [[Z:%.*]], [[Y]]
+; CHECK-LABEL: define <4 x i32> @shuf_mul_v4i32_yy_use2(
+; CHECK-SAME: <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[B0:%.*]] = mul <4 x i32> [[X]], [[Y]]
+; CHECK-NEXT:    [[B1:%.*]] = mul <4 x i32> [[Z]], [[Y]]
 ; CHECK-NEXT:    call void @use(<4 x i32> [[B1]])
 ; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[B0]], <4 x i32> [[B1]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
 ; CHECK-NEXT:    ret <4 x i32> [[R]]
@@ -382,9 +410,10 @@ define <4 x i32> @shuf_mul_v4i32_yy_use2(<4 x i32> %x, <4 x i32> %y, <4 x i32> %
 ; negative test - must have matching operand
 
 define <4 x float> @shuf_fadd_v4f32_no_common_op(<4 x float> %x, <4 x float> %y, <4 x float> %z, <4 x float> %w) {
-; CHECK-LABEL: @shuf_fadd_v4f32_no_common_op(
-; CHECK-NEXT:    [[B0:%.*]] = fadd <4 x float> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[B1:%.*]] = fadd <4 x float> [[Z:%.*]], [[W:%.*]]
+; CHECK-LABEL: define <4 x float> @shuf_fadd_v4f32_no_common_op(
+; CHECK-SAME: <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]], <4 x float> [[Z:%.*]], <4 x float> [[W:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[B0:%.*]] = fadd <4 x float> [[X]], [[Y]]
+; CHECK-NEXT:    [[B1:%.*]] = fadd <4 x float> [[Z]], [[W]]
 ; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[B0]], <4 x float> [[B1]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
 ; CHECK-NEXT:    ret <4 x float> [[R]]
 ;
@@ -397,9 +426,10 @@ define <4 x float> @shuf_fadd_v4f32_no_common_op(<4 x float> %x, <4 x float> %y,
 ; negative test - binops may be relatively cheap
 
 define <16 x i16> @shuf_and_v16i16_yy_expensive_shuf(<16 x i16> %x, <16 x i16> %y, <16 x i16> %z) {
-; CHECK-LABEL: @shuf_and_v16i16_yy_expensive_shuf(
-; CHECK-NEXT:    [[B0:%.*]] = and <16 x i16> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[B1:%.*]] = and <16 x i16> [[Y]], [[Z:%.*]]
+; CHECK-LABEL: define <16 x i16> @shuf_and_v16i16_yy_expensive_shuf(
+; CHECK-SAME: <16 x i16> [[X:%.*]], <16 x i16> [[Y:%.*]], <16 x i16> [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[B0:%.*]] = and <16 x i16> [[X]], [[Y]]
+; CHECK-NEXT:    [[B1:%.*]] = and <16 x i16> [[Y]], [[Z]]
 ; CHECK-NEXT:    [[R:%.*]] = shufflevector <16 x i16> [[B0]], <16 x i16> [[B1]], <16 x i32> <i32 15, i32 22, i32 25, i32 13, i32 28, i32 0, i32 poison, i32 3, i32 0, i32 30, i32 3, i32 7, i32 9, i32 19, i32 2, i32 22>
 ; CHECK-NEXT:    ret <16 x i16> [[R]]
 ;
-- 
GitLab


From e84b2fb48df882ee1f49bbbca29d44fba22192a8 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Tue, 16 Apr 2024 09:48:13 -0400
Subject: [PATCH 404/464] [LV][NFCI]Use integer for cost/trip count
 calculations instead of double, fix possible UB.

Using fp type in the compiler is not the best idea, here it used with
the comparison for equal to 0 and may cause undefined behavior in some
cases.

Reviewers: fhahn

Reviewed By: fhahn

Pull Request: https://github.com/llvm/llvm-project/pull/87241
---
 .../Transforms/Vectorize/LoopVectorize.cpp    | 14 ++---
 .../vectorize-force-tail-with-evl.ll          | 53 +++++++++++++++++++
 2 files changed, 60 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 5535cc55e932..2057cab46135 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -9722,7 +9722,7 @@ static bool areRuntimeChecksProfitable(GeneratedRTChecks &Checks,
   }
 
   // The scalar cost should only be 0 when vectorizing with a user specified VF/IC. In those cases, runtime checks should always be generated.
-  double ScalarC = *VF.ScalarCost.getValue();
+  uint64_t ScalarC = *VF.ScalarCost.getValue();
   if (ScalarC == 0)
     return true;
 
@@ -9749,7 +9749,7 @@ static bool areRuntimeChecksProfitable(GeneratedRTChecks &Checks,
   //   RtC + VecC * (TC / VF) + EpiC <  ScalarC * TC
   //
   // Now we can compute the minimum required trip count TC as
-  //   (RtC + EpiC) / (ScalarC - (VecC / VF)) < TC
+  //   VF * (RtC + EpiC) / (ScalarC * VF - VecC) < TC
   //
   // For now we assume the epilogue cost EpiC = 0 for simplicity. Note that
   // the computations are performed on doubles, not integers and the result
@@ -9761,9 +9761,9 @@ static bool areRuntimeChecksProfitable(GeneratedRTChecks &Checks,
       AssumedMinimumVscale = *VScale;
     IntVF *= AssumedMinimumVscale;
   }
-  double VecCOverVF = double(*VF.Cost.getValue()) / IntVF;
-  double RtC = *CheckCost.getValue();
-  double MinTC1 = RtC / (ScalarC - VecCOverVF);
+  uint64_t RtC = *CheckCost.getValue();
+  uint64_t Div = ScalarC * IntVF - *VF.Cost.getValue();
+  uint64_t MinTC1 = Div == 0 ? 0 : divideCeil(RtC * IntVF, Div);
 
   // Second, compute a minimum iteration count so that the cost of the
   // runtime checks is only a fraction of the total scalar loop cost. This
@@ -9772,12 +9772,12 @@ static bool areRuntimeChecksProfitable(GeneratedRTChecks &Checks,
   // * TC. To bound the runtime check to be a fraction 1/X of the scalar
   // cost, compute
   //   RtC < ScalarC * TC * (1 / X)  ==>  RtC * X / ScalarC < TC
-  double MinTC2 = RtC * 10 / ScalarC;
+  uint64_t MinTC2 = divideCeil(RtC * 10, ScalarC);
 
   // Now pick the larger minimum. If it is not a multiple of VF and a scalar
   // epilogue is allowed, choose the next closest multiple of VF. This should
   // partly compensate for ignoring the epilogue cost.
-  uint64_t MinTC = std::ceil(std::max(MinTC1, MinTC2));
+  uint64_t MinTC = std::max(MinTC1, MinTC2);
   if (SEL == CM_ScalarEpilogueAllowed)
     MinTC = alignTo(MinTC, IntVF);
   VF.MinProfitableTripCount = ElementCount::getFixed(MinTC);
diff --git a/llvm/test/Transforms/LoopVectorize/vectorize-force-tail-with-evl.ll b/llvm/test/Transforms/LoopVectorize/vectorize-force-tail-with-evl.ll
index a90b38c6a960..fe98ca167a08 100644
--- a/llvm/test/Transforms/LoopVectorize/vectorize-force-tail-with-evl.ll
+++ b/llvm/test/Transforms/LoopVectorize/vectorize-force-tail-with-evl.ll
@@ -9,6 +9,11 @@
 ; RUN: -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -force-vector-width=4 \
 ; RUN: -force-target-supports-scalable-vectors -scalable-vectorization=on -S < %s | FileCheck --check-prefix=NO-VP %s
 
+; RUN: opt -passes=loop-vectorize \
+; RUN: -force-tail-folding-style=none \
+; RUN: -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue \
+; RUN: -force-target-supports-scalable-vectors -scalable-vectorization=on -S < %s | FileCheck --check-prefix=NO-VP-DEF %s
+
 ; The target does not support predicated vectorization.
 define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
 ; IF-EVL-LABEL: @foo(
@@ -80,6 +85,54 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
 ; NO-VP:       for.cond.cleanup:
 ; NO-VP-NEXT:    ret void
 ;
+; NO-VP-DEF-LABEL: @foo(
+; NO-VP-DEF-NEXT:  entry:
+; NO-VP-DEF-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-DEF-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP0]]
+; NO-VP-DEF-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; NO-VP-DEF:       vector.ph:
+; NO-VP-DEF-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-DEF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP1]]
+; NO-VP-DEF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; NO-VP-DEF-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; NO-VP-DEF-NEXT:    br label [[VECTOR_BODY:%.*]]
+; NO-VP-DEF:       vector.body:
+; NO-VP-DEF-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; NO-VP-DEF-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 0
+; NO-VP-DEF-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[TMP3]]
+; NO-VP-DEF-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[TMP4]], i32 0
+; NO-VP-DEF-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 1 x i32>, ptr [[TMP5]], align 4
+; NO-VP-DEF-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[C:%.*]], i64 [[TMP3]]
+; NO-VP-DEF-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i32 0
+; NO-VP-DEF-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 1 x i32>, ptr [[TMP7]], align 4
+; NO-VP-DEF-NEXT:    [[TMP8:%.*]] = add nsw <vscale x 1 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD]]
+; NO-VP-DEF-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[TMP3]]
+; NO-VP-DEF-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP9]], i32 0
+; NO-VP-DEF-NEXT:    store <vscale x 1 x i32> [[TMP8]], ptr [[TMP10]], align 4
+; NO-VP-DEF-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; NO-VP-DEF-NEXT:    [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; NO-VP-DEF-NEXT:    br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; NO-VP-DEF:       middle.block:
+; NO-VP-DEF-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; NO-VP-DEF-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; NO-VP-DEF:       scalar.ph:
+; NO-VP-DEF-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; NO-VP-DEF-NEXT:    br label [[FOR_BODY:%.*]]
+; NO-VP-DEF:       for.body:
+; NO-VP-DEF-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
+; NO-VP-DEF-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]]
+; NO-VP-DEF-NEXT:    [[TMP12:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; NO-VP-DEF-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[IV]]
+; NO-VP-DEF-NEXT:    [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4
+; NO-VP-DEF-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP13]], [[TMP12]]
+; NO-VP-DEF-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]]
+; NO-VP-DEF-NEXT:    store i32 [[ADD]], ptr [[ARRAYIDX4]], align 4
+; NO-VP-DEF-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
+; NO-VP-DEF-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; NO-VP-DEF-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; NO-VP-DEF:       for.cond.cleanup:
+; NO-VP-DEF-NEXT:    ret void
+;
 entry:
   br label %for.body
 
-- 
GitLab


From 254df2e35c14414564d233902b25305ed17d251c Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 16 Apr 2024 14:52:16 +0100
Subject: [PATCH 405/464] [VectorCombine][X86] shuffle-of-binops.ll - split off
 foldShuffleOfBinops tests from shuffle.ll

---
 .../VectorCombine/X86/shuffle-of-binops.ll    | 204 ++++++++++++++++++
 .../Transforms/VectorCombine/X86/shuffle.ll   | 200 +----------------
 2 files changed, 206 insertions(+), 198 deletions(-)
 create mode 100644 llvm/test/Transforms/VectorCombine/X86/shuffle-of-binops.ll

diff --git a/llvm/test/Transforms/VectorCombine/X86/shuffle-of-binops.ll b/llvm/test/Transforms/VectorCombine/X86/shuffle-of-binops.ll
new file mode 100644
index 000000000000..e2ff343944cf
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/X86/shuffle-of-binops.ll
@@ -0,0 +1,204 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt < %s -passes=vector-combine -S -mtriple=x86_64-- -mattr=SSE2 | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -passes=vector-combine -S -mtriple=x86_64-- -mattr=AVX2 | FileCheck %s --check-prefixes=CHECK,AVX
+
+declare void @use(<4 x i32>)
+
+; Shuffle is much cheaper than fdiv. FMF are intersected.
+
+define <4 x float> @shuf_fdiv_v4f32_yy(<4 x float> %x, <4 x float> %y, <4 x float> %z) {
+; CHECK-LABEL: define <4 x float> @shuf_fdiv_v4f32_yy(
+; CHECK-SAME: <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]], <4 x float> [[Z:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[X]], <4 x float> [[Z]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[Y]], <4 x float> poison, <4 x i32> <i32 1, i32 3, i32 1, i32 3>
+; CHECK-NEXT:    [[R:%.*]] = fdiv arcp <4 x float> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    ret <4 x float> [[R]]
+;
+  %b0 = fdiv fast <4 x float> %x, %y
+  %b1 = fdiv arcp <4 x float> %z, %y
+  %r = shufflevector <4 x float> %b0, <4 x float> %b1, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  ret <4 x float> %r
+}
+
+; Common operand is op0 of the binops.
+
+define <4 x i32> @shuf_add_v4i32_xx(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
+; CHECK-LABEL: define <4 x i32> @shuf_add_v4i32_xx(
+; CHECK-SAME: <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[X]], <4 x i32> poison, <4 x i32> <i32 poison, i32 poison, i32 2, i32 0>
+; CHECK-NEXT:    [[R1:%.*]] = shufflevector <4 x i32> [[Y]], <4 x i32> [[Z]], <4 x i32> <i32 poison, i32 poison, i32 6, i32 0>
+; CHECK-NEXT:    [[R2:%.*]] = add <4 x i32> [[TMP1]], [[R1]]
+; CHECK-NEXT:    ret <4 x i32> [[R2]]
+;
+  %b0 = add <4 x i32> %x, %y
+  %b1 = add <4 x i32> %x, %z
+  %r = shufflevector <4 x i32> %b0, <4 x i32> %b1, <4 x i32> <i32 poison, i32 poison, i32 6, i32 0>
+  ret <4 x i32> %r
+}
+
+; For commutative instructions, common operand may be swapped.
+
+define <4 x float> @shuf_fmul_v4f32_xx_swap(<4 x float> %x, <4 x float> %y, <4 x float> %z) {
+; CHECK-LABEL: define <4 x float> @shuf_fmul_v4f32_xx_swap(
+; CHECK-SAME: <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]], <4 x float> [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[Y]], <4 x float> [[Z]], <4 x i32> <i32 0, i32 3, i32 4, i32 7>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[X]], <4 x float> poison, <4 x i32> <i32 0, i32 3, i32 0, i32 3>
+; CHECK-NEXT:    [[R:%.*]] = fmul <4 x float> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <4 x float> [[R]]
+;
+  %b0 = fmul <4 x float> %x, %y
+  %b1 = fmul <4 x float> %z, %x
+  %r = shufflevector <4 x float> %b0, <4 x float> %b1, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
+  ret <4 x float> %r
+}
+
+; For commutative instructions, common operand may be swapped.
+
+define <2 x i64> @shuf_and_v2i64_yy_swap(<2 x i64> %x, <2 x i64> %y, <2 x i64> %z) {
+; CHECK-LABEL: define <2 x i64> @shuf_and_v2i64_yy_swap(
+; CHECK-SAME: <2 x i64> [[X:%.*]], <2 x i64> [[Y:%.*]], <2 x i64> [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i64> [[Y]], <2 x i64> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i64> [[X]], <2 x i64> [[Z]], <2 x i32> <i32 3, i32 0>
+; CHECK-NEXT:    [[R:%.*]] = and <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <2 x i64> [[R]]
+;
+  %b0 = and <2 x i64> %x, %y
+  %b1 = and <2 x i64> %y, %z
+  %r = shufflevector <2 x i64> %b0, <2 x i64> %b1, <2 x i32> <i32 3, i32 0>
+  ret <2 x i64> %r
+}
+
+; non-commutative binop, but common op0
+
+define <4 x i32> @shuf_shl_v4i32_xx(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
+; CHECK-LABEL: define <4 x i32> @shuf_shl_v4i32_xx(
+; CHECK-SAME: <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[X]], <4 x i32> poison, <4 x i32> <i32 3, i32 1, i32 1, i32 2>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[Y]], <4 x i32> [[Z]], <4 x i32> <i32 3, i32 1, i32 1, i32 6>
+; CHECK-NEXT:    [[R:%.*]] = shl <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %b0 = shl <4 x i32> %x, %y
+  %b1 = shl <4 x i32> %x, %z
+  %r = shufflevector <4 x i32> %b0, <4 x i32> %b1, <4 x i32> <i32 3, i32 1, i32 1, i32 6>
+  ret <4 x i32> %r
+}
+
+; negative test - common operand, but not commutable
+
+define <4 x i32> @shuf_shl_v4i32_xx_swap(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
+; CHECK-LABEL: define <4 x i32> @shuf_shl_v4i32_xx_swap(
+; CHECK-SAME: <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[B0:%.*]] = shl <4 x i32> [[X]], [[Y]]
+; CHECK-NEXT:    [[B1:%.*]] = shl <4 x i32> [[Z]], [[X]]
+; CHECK-NEXT:    [[R1:%.*]] = shufflevector <4 x i32> [[B0]], <4 x i32> [[B1]], <4 x i32> <i32 3, i32 2, i32 2, i32 5>
+; CHECK-NEXT:    ret <4 x i32> [[R1]]
+;
+  %b0 = shl <4 x i32> %x, %y
+  %b1 = shl <4 x i32> %z, %x
+  %r = shufflevector <4 x i32> %b0, <4 x i32> %b1, <4 x i32> <i32 3, i32 2, i32 2, i32 5>
+  ret <4 x i32> %r
+}
+
+; negative test - mismatched opcodes
+
+define <2 x i64> @shuf_sub_add_v2i64_yy(<2 x i64> %x, <2 x i64> %y, <2 x i64> %z) {
+; CHECK-LABEL: define <2 x i64> @shuf_sub_add_v2i64_yy(
+; CHECK-SAME: <2 x i64> [[X:%.*]], <2 x i64> [[Y:%.*]], <2 x i64> [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[B0:%.*]] = sub <2 x i64> [[X]], [[Y]]
+; CHECK-NEXT:    [[B1:%.*]] = add <2 x i64> [[Z]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i64> [[B0]], <2 x i64> [[B1]], <2 x i32> <i32 3, i32 0>
+; CHECK-NEXT:    ret <2 x i64> [[R]]
+;
+  %b0 = sub <2 x i64> %x, %y
+  %b1 = add <2 x i64> %z, %y
+  %r = shufflevector <2 x i64> %b0, <2 x i64> %b1, <2 x i32> <i32 3, i32 0>
+  ret <2 x i64> %r
+}
+
+; negative test - type change via shuffle
+
+define <8 x float> @shuf_fmul_v4f32_xx_type(<4 x float> %x, <4 x float> %y, <4 x float> %z) {
+; CHECK-LABEL: define <8 x float> @shuf_fmul_v4f32_xx_type(
+; CHECK-SAME: <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]], <4 x float> [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[B0:%.*]] = fmul <4 x float> [[X]], [[Y]]
+; CHECK-NEXT:    [[B1:%.*]] = fmul <4 x float> [[Z]], [[X]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[B0]], <4 x float> [[B1]], <8 x i32> <i32 0, i32 3, i32 4, i32 7, i32 0, i32 1, i32 1, i32 6>
+; CHECK-NEXT:    ret <8 x float> [[R]]
+;
+  %b0 = fmul <4 x float> %x, %y
+  %b1 = fmul <4 x float> %z, %x
+  %r = shufflevector <4 x float> %b0, <4 x float> %b1, <8 x i32> <i32 0, i32 3, i32 4, i32 7, i32 0, i32 1, i32 1, i32 6>
+  ret <8 x float> %r
+}
+
+; negative test - uses
+
+define <4 x i32> @shuf_lshr_v4i32_yy_use1(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
+; CHECK-LABEL: define <4 x i32> @shuf_lshr_v4i32_yy_use1(
+; CHECK-SAME: <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[B0:%.*]] = lshr <4 x i32> [[X]], [[Y]]
+; CHECK-NEXT:    call void @use(<4 x i32> [[B0]])
+; CHECK-NEXT:    [[B1:%.*]] = lshr <4 x i32> [[Z]], [[Y]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[B0]], <4 x i32> [[B1]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %b0 = lshr <4 x i32> %x, %y
+  call void @use(<4 x i32> %b0)
+  %b1 = lshr <4 x i32> %z, %y
+  %r = shufflevector <4 x i32> %b0, <4 x i32> %b1, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  ret <4 x i32> %r
+}
+
+; negative test - uses
+
+define <4 x i32> @shuf_mul_v4i32_yy_use2(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
+; CHECK-LABEL: define <4 x i32> @shuf_mul_v4i32_yy_use2(
+; CHECK-SAME: <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[B0:%.*]] = mul <4 x i32> [[X]], [[Y]]
+; CHECK-NEXT:    [[B1:%.*]] = mul <4 x i32> [[Z]], [[Y]]
+; CHECK-NEXT:    call void @use(<4 x i32> [[B1]])
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[B0]], <4 x i32> [[B1]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %b0 = mul <4 x i32> %x, %y
+  %b1 = mul <4 x i32> %z, %y
+  call void @use(<4 x i32> %b1)
+  %r = shufflevector <4 x i32> %b0, <4 x i32> %b1, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  ret <4 x i32> %r
+}
+
+; negative test - must have matching operand
+
+define <4 x float> @shuf_fadd_v4f32_no_common_op(<4 x float> %x, <4 x float> %y, <4 x float> %z, <4 x float> %w) {
+; CHECK-LABEL: define <4 x float> @shuf_fadd_v4f32_no_common_op(
+; CHECK-SAME: <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]], <4 x float> [[Z:%.*]], <4 x float> [[W:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[B0:%.*]] = fadd <4 x float> [[X]], [[Y]]
+; CHECK-NEXT:    [[B1:%.*]] = fadd <4 x float> [[Z]], [[W]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[B0]], <4 x float> [[B1]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT:    ret <4 x float> [[R]]
+;
+  %b0 = fadd <4 x float> %x, %y
+  %b1 = fadd <4 x float> %z, %w
+  %r = shufflevector <4 x float> %b0, <4 x float> %b1, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  ret <4 x float> %r
+}
+
+; negative test - binops may be relatively cheap
+
+define <16 x i16> @shuf_and_v16i16_yy_expensive_shuf(<16 x i16> %x, <16 x i16> %y, <16 x i16> %z) {
+; CHECK-LABEL: define <16 x i16> @shuf_and_v16i16_yy_expensive_shuf(
+; CHECK-SAME: <16 x i16> [[X:%.*]], <16 x i16> [[Y:%.*]], <16 x i16> [[Z:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[B0:%.*]] = and <16 x i16> [[X]], [[Y]]
+; CHECK-NEXT:    [[B1:%.*]] = and <16 x i16> [[Y]], [[Z]]
+; CHECK-NEXT:    [[R:%.*]] = shufflevector <16 x i16> [[B0]], <16 x i16> [[B1]], <16 x i32> <i32 15, i32 22, i32 25, i32 13, i32 28, i32 0, i32 poison, i32 3, i32 0, i32 30, i32 3, i32 7, i32 9, i32 19, i32 2, i32 22>
+; CHECK-NEXT:    ret <16 x i16> [[R]]
+;
+  %b0 = and <16 x i16> %x, %y
+  %b1 = and <16 x i16> %y, %z
+  %r = shufflevector <16 x i16> %b0, <16 x i16> %b1, <16 x i32> <i32 15, i32 22, i32 25, i32 13, i32 28, i32 0, i32 poison, i32 3, i32 0, i32 30, i32 3, i32 7, i32 9, i32 19, i32 2, i32 22>
+  ret <16 x i16> %r
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; AVX: {{.*}}
+; SSE: {{.*}}
diff --git a/llvm/test/Transforms/VectorCombine/X86/shuffle.ll b/llvm/test/Transforms/VectorCombine/X86/shuffle.ll
index bb6402347a9b..c8c9aa161ae2 100644
--- a/llvm/test/Transforms/VectorCombine/X86/shuffle.ll
+++ b/llvm/test/Transforms/VectorCombine/X86/shuffle.ll
@@ -2,6 +2,8 @@
 ; RUN: opt < %s -passes=vector-combine -S -mtriple=x86_64-- -mattr=SSE2 | FileCheck %s --check-prefixes=CHECK,SSE
 ; RUN: opt < %s -passes=vector-combine -S -mtriple=x86_64-- -mattr=AVX2 | FileCheck %s --check-prefixes=CHECK,AVX
 
+declare void @use(<4 x i32>)
+
 ; x86 does not have a cheap v16i8 shuffle until SSSE3 (pshufb)
 
 define <16 x i8> @bitcast_shuf_narrow_element(<4 x i32> %v) {
@@ -114,8 +116,6 @@ define <4 x i32> @bitcast_shuf_wide_element(<8 x i16> %v) {
   ret <4 x i32> %r
 }
 
-declare void @use(<4 x i32>)
-
 ; Negative test - don't create an extra shuffle
 
 define <16 x i8> @bitcast_shuf_uses(<4 x i32> %v) {
@@ -242,199 +242,3 @@ define <2 x i64> @PR35454_2(<2 x i64> %v) {
   %bc3 = bitcast <4 x i32> %permil1 to <2 x i64>
   ret <2 x i64> %bc3
 }
-
-; Shuffle is much cheaper than fdiv. FMF are intersected.
-
-define <4 x float> @shuf_fdiv_v4f32_yy(<4 x float> %x, <4 x float> %y, <4 x float> %z) {
-; CHECK-LABEL: define <4 x float> @shuf_fdiv_v4f32_yy(
-; CHECK-SAME: <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]], <4 x float> [[Z:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[X]], <4 x float> [[Z]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[Y]], <4 x float> poison, <4 x i32> <i32 1, i32 3, i32 1, i32 3>
-; CHECK-NEXT:    [[R:%.*]] = fdiv arcp <4 x float> [[TMP2]], [[TMP3]]
-; CHECK-NEXT:    ret <4 x float> [[R]]
-;
-  %b0 = fdiv fast <4 x float> %x, %y
-  %b1 = fdiv arcp <4 x float> %z, %y
-  %r = shufflevector <4 x float> %b0, <4 x float> %b1, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-  ret <4 x float> %r
-}
-
-; Common operand is op0 of the binops.
-
-define <4 x i32> @shuf_add_v4i32_xx(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
-; CHECK-LABEL: define <4 x i32> @shuf_add_v4i32_xx(
-; CHECK-SAME: <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> [[Z:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[X]], <4 x i32> poison, <4 x i32> <i32 poison, i32 poison, i32 2, i32 0>
-; CHECK-NEXT:    [[R1:%.*]] = shufflevector <4 x i32> [[Y]], <4 x i32> [[Z]], <4 x i32> <i32 poison, i32 poison, i32 6, i32 0>
-; CHECK-NEXT:    [[R2:%.*]] = add <4 x i32> [[TMP1]], [[R1]]
-; CHECK-NEXT:    ret <4 x i32> [[R2]]
-;
-  %b0 = add <4 x i32> %x, %y
-  %b1 = add <4 x i32> %x, %z
-  %r = shufflevector <4 x i32> %b0, <4 x i32> %b1, <4 x i32> <i32 poison, i32 poison, i32 6, i32 0>
-  ret <4 x i32> %r
-}
-
-; For commutative instructions, common operand may be swapped.
-
-define <4 x float> @shuf_fmul_v4f32_xx_swap(<4 x float> %x, <4 x float> %y, <4 x float> %z) {
-; CHECK-LABEL: define <4 x float> @shuf_fmul_v4f32_xx_swap(
-; CHECK-SAME: <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]], <4 x float> [[Z:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[Y]], <4 x float> [[Z]], <4 x i32> <i32 0, i32 3, i32 4, i32 7>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[X]], <4 x float> poison, <4 x i32> <i32 0, i32 3, i32 0, i32 3>
-; CHECK-NEXT:    [[R:%.*]] = fmul <4 x float> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    ret <4 x float> [[R]]
-;
-  %b0 = fmul <4 x float> %x, %y
-  %b1 = fmul <4 x float> %z, %x
-  %r = shufflevector <4 x float> %b0, <4 x float> %b1, <4 x i32> <i32 0, i32 3, i32 4, i32 7>
-  ret <4 x float> %r
-}
-
-; For commutative instructions, common operand may be swapped.
-
-define <2 x i64> @shuf_and_v2i64_yy_swap(<2 x i64> %x, <2 x i64> %y, <2 x i64> %z) {
-; CHECK-LABEL: define <2 x i64> @shuf_and_v2i64_yy_swap(
-; CHECK-SAME: <2 x i64> [[X:%.*]], <2 x i64> [[Y:%.*]], <2 x i64> [[Z:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i64> [[Y]], <2 x i64> poison, <2 x i32> <i32 1, i32 0>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i64> [[X]], <2 x i64> [[Z]], <2 x i32> <i32 3, i32 0>
-; CHECK-NEXT:    [[R:%.*]] = and <2 x i64> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    ret <2 x i64> [[R]]
-;
-  %b0 = and <2 x i64> %x, %y
-  %b1 = and <2 x i64> %y, %z
-  %r = shufflevector <2 x i64> %b0, <2 x i64> %b1, <2 x i32> <i32 3, i32 0>
-  ret <2 x i64> %r
-}
-
-; non-commutative binop, but common op0
-
-define <4 x i32> @shuf_shl_v4i32_xx(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
-; CHECK-LABEL: define <4 x i32> @shuf_shl_v4i32_xx(
-; CHECK-SAME: <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> [[Z:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <4 x i32> [[X]], <4 x i32> poison, <4 x i32> <i32 3, i32 1, i32 1, i32 2>
-; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x i32> [[Y]], <4 x i32> [[Z]], <4 x i32> <i32 3, i32 1, i32 1, i32 6>
-; CHECK-NEXT:    [[R:%.*]] = shl <4 x i32> [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    ret <4 x i32> [[R]]
-;
-  %b0 = shl <4 x i32> %x, %y
-  %b1 = shl <4 x i32> %x, %z
-  %r = shufflevector <4 x i32> %b0, <4 x i32> %b1, <4 x i32> <i32 3, i32 1, i32 1, i32 6>
-  ret <4 x i32> %r
-}
-
-; negative test - common operand, but not commutable
-
-define <4 x i32> @shuf_shl_v4i32_xx_swap(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
-; CHECK-LABEL: define <4 x i32> @shuf_shl_v4i32_xx_swap(
-; CHECK-SAME: <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> [[Z:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[B0:%.*]] = shl <4 x i32> [[X]], [[Y]]
-; CHECK-NEXT:    [[B1:%.*]] = shl <4 x i32> [[Z]], [[X]]
-; CHECK-NEXT:    [[R1:%.*]] = shufflevector <4 x i32> [[B0]], <4 x i32> [[B1]], <4 x i32> <i32 3, i32 2, i32 2, i32 5>
-; CHECK-NEXT:    ret <4 x i32> [[R1]]
-;
-  %b0 = shl <4 x i32> %x, %y
-  %b1 = shl <4 x i32> %z, %x
-  %r = shufflevector <4 x i32> %b0, <4 x i32> %b1, <4 x i32> <i32 3, i32 2, i32 2, i32 5>
-  ret <4 x i32> %r
-}
-
-; negative test - mismatched opcodes
-
-define <2 x i64> @shuf_sub_add_v2i64_yy(<2 x i64> %x, <2 x i64> %y, <2 x i64> %z) {
-; CHECK-LABEL: define <2 x i64> @shuf_sub_add_v2i64_yy(
-; CHECK-SAME: <2 x i64> [[X:%.*]], <2 x i64> [[Y:%.*]], <2 x i64> [[Z:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[B0:%.*]] = sub <2 x i64> [[X]], [[Y]]
-; CHECK-NEXT:    [[B1:%.*]] = add <2 x i64> [[Z]], [[Y]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <2 x i64> [[B0]], <2 x i64> [[B1]], <2 x i32> <i32 3, i32 0>
-; CHECK-NEXT:    ret <2 x i64> [[R]]
-;
-  %b0 = sub <2 x i64> %x, %y
-  %b1 = add <2 x i64> %z, %y
-  %r = shufflevector <2 x i64> %b0, <2 x i64> %b1, <2 x i32> <i32 3, i32 0>
-  ret <2 x i64> %r
-}
-
-; negative test - type change via shuffle
-
-define <8 x float> @shuf_fmul_v4f32_xx_type(<4 x float> %x, <4 x float> %y, <4 x float> %z) {
-; CHECK-LABEL: define <8 x float> @shuf_fmul_v4f32_xx_type(
-; CHECK-SAME: <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]], <4 x float> [[Z:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[B0:%.*]] = fmul <4 x float> [[X]], [[Y]]
-; CHECK-NEXT:    [[B1:%.*]] = fmul <4 x float> [[Z]], [[X]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[B0]], <4 x float> [[B1]], <8 x i32> <i32 0, i32 3, i32 4, i32 7, i32 0, i32 1, i32 1, i32 6>
-; CHECK-NEXT:    ret <8 x float> [[R]]
-;
-  %b0 = fmul <4 x float> %x, %y
-  %b1 = fmul <4 x float> %z, %x
-  %r = shufflevector <4 x float> %b0, <4 x float> %b1, <8 x i32> <i32 0, i32 3, i32 4, i32 7, i32 0, i32 1, i32 1, i32 6>
-  ret <8 x float> %r
-}
-
-; negative test - uses
-
-define <4 x i32> @shuf_lshr_v4i32_yy_use1(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
-; CHECK-LABEL: define <4 x i32> @shuf_lshr_v4i32_yy_use1(
-; CHECK-SAME: <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> [[Z:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[B0:%.*]] = lshr <4 x i32> [[X]], [[Y]]
-; CHECK-NEXT:    call void @use(<4 x i32> [[B0]])
-; CHECK-NEXT:    [[B1:%.*]] = lshr <4 x i32> [[Z]], [[Y]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[B0]], <4 x i32> [[B1]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEXT:    ret <4 x i32> [[R]]
-;
-  %b0 = lshr <4 x i32> %x, %y
-  call void @use(<4 x i32> %b0)
-  %b1 = lshr <4 x i32> %z, %y
-  %r = shufflevector <4 x i32> %b0, <4 x i32> %b1, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-  ret <4 x i32> %r
-}
-
-; negative test - uses
-
-define <4 x i32> @shuf_mul_v4i32_yy_use2(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
-; CHECK-LABEL: define <4 x i32> @shuf_mul_v4i32_yy_use2(
-; CHECK-SAME: <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]], <4 x i32> [[Z:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[B0:%.*]] = mul <4 x i32> [[X]], [[Y]]
-; CHECK-NEXT:    [[B1:%.*]] = mul <4 x i32> [[Z]], [[Y]]
-; CHECK-NEXT:    call void @use(<4 x i32> [[B1]])
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x i32> [[B0]], <4 x i32> [[B1]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEXT:    ret <4 x i32> [[R]]
-;
-  %b0 = mul <4 x i32> %x, %y
-  %b1 = mul <4 x i32> %z, %y
-  call void @use(<4 x i32> %b1)
-  %r = shufflevector <4 x i32> %b0, <4 x i32> %b1, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-  ret <4 x i32> %r
-}
-
-; negative test - must have matching operand
-
-define <4 x float> @shuf_fadd_v4f32_no_common_op(<4 x float> %x, <4 x float> %y, <4 x float> %z, <4 x float> %w) {
-; CHECK-LABEL: define <4 x float> @shuf_fadd_v4f32_no_common_op(
-; CHECK-SAME: <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]], <4 x float> [[Z:%.*]], <4 x float> [[W:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[B0:%.*]] = fadd <4 x float> [[X]], [[Y]]
-; CHECK-NEXT:    [[B1:%.*]] = fadd <4 x float> [[Z]], [[W]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <4 x float> [[B0]], <4 x float> [[B1]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; CHECK-NEXT:    ret <4 x float> [[R]]
-;
-  %b0 = fadd <4 x float> %x, %y
-  %b1 = fadd <4 x float> %z, %w
-  %r = shufflevector <4 x float> %b0, <4 x float> %b1, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-  ret <4 x float> %r
-}
-
-; negative test - binops may be relatively cheap
-
-define <16 x i16> @shuf_and_v16i16_yy_expensive_shuf(<16 x i16> %x, <16 x i16> %y, <16 x i16> %z) {
-; CHECK-LABEL: define <16 x i16> @shuf_and_v16i16_yy_expensive_shuf(
-; CHECK-SAME: <16 x i16> [[X:%.*]], <16 x i16> [[Y:%.*]], <16 x i16> [[Z:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT:    [[B0:%.*]] = and <16 x i16> [[X]], [[Y]]
-; CHECK-NEXT:    [[B1:%.*]] = and <16 x i16> [[Y]], [[Z]]
-; CHECK-NEXT:    [[R:%.*]] = shufflevector <16 x i16> [[B0]], <16 x i16> [[B1]], <16 x i32> <i32 15, i32 22, i32 25, i32 13, i32 28, i32 0, i32 poison, i32 3, i32 0, i32 30, i32 3, i32 7, i32 9, i32 19, i32 2, i32 22>
-; CHECK-NEXT:    ret <16 x i16> [[R]]
-;
-  %b0 = and <16 x i16> %x, %y
-  %b1 = and <16 x i16> %y, %z
-  %r = shufflevector <16 x i16> %b0, <16 x i16> %b1, <16 x i32> <i32 15, i32 22, i32 25, i32 13, i32 28, i32 0, i32 poison, i32 3, i32 0, i32 30, i32 3, i32 7, i32 9, i32 19, i32 2, i32 22>
-  ret <16 x i16> %r
-}
-- 
GitLab


From 6133878227efc30355c02c2f089e06ce58231a3d Mon Sep 17 00:00:00 2001
From: Erich Keane <ekeane@nvidia.com>
Date: Tue, 16 Apr 2024 06:57:36 -0700
Subject: [PATCH 406/464] [OpenACC] Implement `self` clause for compute
 constructs (#88760)

`self` clauses on compute constructs take an optional condition
expression. We again limit the implementation to ONLY compute constructs
to ensure we get all the rules correct for others. However, this one
will be particularly complicated, as it takes a `var-list` for `update`,
so when we get to that construct/clause combination, we need to do that
as well.

This patch also furthers uses of the `OpenACCClauses.def` as it became
useful while implementing this (as well as some other minor refactors as
I went through).

Finally, `self` and `if` clauses have an interaction with each other, if
an `if` clause evaluates to `true`, the `self` clause has no effect.
While this is intended and can be used 'meaningfully', we are warning on
this with a very granular warning, so that this edge case will be
noticed by newer users, but can be disabled trivially.
---
 clang/include/clang/AST/OpenACCClause.h       |  65 ++++--------
 clang/include/clang/AST/StmtOpenACC.h         |   4 +-
 .../clang/Basic/DiagnosticSemaKinds.td        |   4 +
 clang/include/clang/Basic/OpenACCClauses.def  |   1 +
 clang/include/clang/Basic/OpenACCKinds.h      |   6 ++
 clang/include/clang/Sema/SemaOpenACC.h        |  18 +++-
 clang/lib/AST/OpenACCClause.cpp               |  26 +++++
 clang/lib/AST/StmtProfile.cpp                 |   5 +
 clang/lib/AST/TextNodeDumper.cpp              |   1 +
 clang/lib/Parse/ParseOpenACC.cpp              |  16 ++-
 clang/lib/Sema/SemaOpenACC.cpp                |  68 ++++++++++--
 clang/lib/Sema/TreeTransform.h                | 100 +++++++++++++-----
 clang/lib/Serialization/ASTReader.cpp         |   7 +-
 clang/lib/Serialization/ASTWriter.cpp         |   9 +-
 clang/test/ParserOpenACC/parse-clauses.c      |   5 +-
 .../compute-construct-clause-ast.cpp          |  91 ++++++++++++++++
 .../compute-construct-self-clause.c           |  82 ++++++++++++++
 .../compute-construct-self-clause.cpp         |  99 +++++++++++++++++
 clang/tools/libclang/CIndex.cpp               |   4 +
 19 files changed, 513 insertions(+), 98 deletions(-)
 create mode 100644 clang/test/SemaOpenACC/compute-construct-self-clause.c
 create mode 100644 clang/test/SemaOpenACC/compute-construct-self-clause.cpp

diff --git a/clang/include/clang/AST/OpenACCClause.h b/clang/include/clang/AST/OpenACCClause.h
index 401b8e904a1b..07587849eb12 100644
--- a/clang/include/clang/AST/OpenACCClause.h
+++ b/clang/include/clang/AST/OpenACCClause.h
@@ -145,6 +145,17 @@ public:
                                  SourceLocation EndLoc);
 };
 
+/// A 'self' clause, which has an optional condition expression.
+class OpenACCSelfClause : public OpenACCClauseWithCondition {
+  OpenACCSelfClause(SourceLocation BeginLoc, SourceLocation LParenLoc,
+                    Expr *ConditionExpr, SourceLocation EndLoc);
+
+public:
+  static OpenACCSelfClause *Create(const ASTContext &C, SourceLocation BeginLoc,
+                                   SourceLocation LParenLoc,
+                                   Expr *ConditionExpr, SourceLocation EndLoc);
+};
+
 template <class Impl> class OpenACCClauseVisitor {
   Impl &getDerived() { return static_cast<Impl &>(*this); }
 
@@ -159,53 +170,13 @@ public:
       return;
 
     switch (C->getClauseKind()) {
-    case OpenACCClauseKind::Default:
-      VisitDefaultClause(*cast<OpenACCDefaultClause>(C));
-      return;
-    case OpenACCClauseKind::If:
-      VisitIfClause(*cast<OpenACCIfClause>(C));
-      return;
-    case OpenACCClauseKind::Finalize:
-    case OpenACCClauseKind::IfPresent:
-    case OpenACCClauseKind::Seq:
-    case OpenACCClauseKind::Independent:
-    case OpenACCClauseKind::Auto:
-    case OpenACCClauseKind::Worker:
-    case OpenACCClauseKind::Vector:
-    case OpenACCClauseKind::NoHost:
-    case OpenACCClauseKind::Self:
-    case OpenACCClauseKind::Copy:
-    case OpenACCClauseKind::UseDevice:
-    case OpenACCClauseKind::Attach:
-    case OpenACCClauseKind::Delete:
-    case OpenACCClauseKind::Detach:
-    case OpenACCClauseKind::Device:
-    case OpenACCClauseKind::DevicePtr:
-    case OpenACCClauseKind::DeviceResident:
-    case OpenACCClauseKind::FirstPrivate:
-    case OpenACCClauseKind::Host:
-    case OpenACCClauseKind::Link:
-    case OpenACCClauseKind::NoCreate:
-    case OpenACCClauseKind::Present:
-    case OpenACCClauseKind::Private:
-    case OpenACCClauseKind::CopyOut:
-    case OpenACCClauseKind::CopyIn:
-    case OpenACCClauseKind::Create:
-    case OpenACCClauseKind::Reduction:
-    case OpenACCClauseKind::Collapse:
-    case OpenACCClauseKind::Bind:
-    case OpenACCClauseKind::VectorLength:
-    case OpenACCClauseKind::NumGangs:
-    case OpenACCClauseKind::NumWorkers:
-    case OpenACCClauseKind::DeviceNum:
-    case OpenACCClauseKind::DefaultAsync:
-    case OpenACCClauseKind::DeviceType:
-    case OpenACCClauseKind::DType:
-    case OpenACCClauseKind::Async:
-    case OpenACCClauseKind::Tile:
-    case OpenACCClauseKind::Gang:
-    case OpenACCClauseKind::Wait:
-    case OpenACCClauseKind::Invalid:
+#define VISIT_CLAUSE(CLAUSE_NAME)                                              \
+  case OpenACCClauseKind::CLAUSE_NAME:                                         \
+    Visit##CLAUSE_NAME##Clause(*cast<OpenACC##CLAUSE_NAME##Clause>(C));        \
+    return;
+#include "clang/Basic/OpenACCClauses.def"
+
+    default:
       llvm_unreachable("Clause visitor not yet implemented");
     }
     llvm_unreachable("Invalid Clause kind");
diff --git a/clang/include/clang/AST/StmtOpenACC.h b/clang/include/clang/AST/StmtOpenACC.h
index 419cb6cada0b..66f8f844e0b2 100644
--- a/clang/include/clang/AST/StmtOpenACC.h
+++ b/clang/include/clang/AST/StmtOpenACC.h
@@ -142,9 +142,7 @@ class OpenACCComputeConstruct final
                           Stmt *StructuredBlock)
       : OpenACCAssociatedStmtConstruct(OpenACCComputeConstructClass, K, Start,
                                        End, StructuredBlock) {
-    assert((K == OpenACCDirectiveKind::Parallel ||
-            K == OpenACCDirectiveKind::Serial ||
-            K == OpenACCDirectiveKind::Kernels) &&
+    assert(isOpenACCComputeDirectiveKind(K) &&
            "Only parallel, serial, and kernels constructs should be "
            "represented by this type");
 
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 5ec0218aedfe..44f802c0c28e 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12274,4 +12274,8 @@ def note_acc_branch_into_compute_construct
     : Note<"invalid branch into OpenACC Compute Construct">;
 def note_acc_branch_out_of_compute_construct
     : Note<"invalid branch out of OpenACC Compute Construct">;
+def warn_acc_if_self_conflict
+    : Warning<"OpenACC construct 'self' has no effect when an 'if' clause "
+              "evaluates to true">,
+      InGroup<DiagGroup<"openacc-self-if-potential-conflict">>;
 } // end of sema component.
diff --git a/clang/include/clang/Basic/OpenACCClauses.def b/clang/include/clang/Basic/OpenACCClauses.def
index 7fd2720e02ce..378495d2c090 100644
--- a/clang/include/clang/Basic/OpenACCClauses.def
+++ b/clang/include/clang/Basic/OpenACCClauses.def
@@ -17,5 +17,6 @@
 
 VISIT_CLAUSE(Default)
 VISIT_CLAUSE(If)
+VISIT_CLAUSE(Self)
 
 #undef VISIT_CLAUSE
diff --git a/clang/include/clang/Basic/OpenACCKinds.h b/clang/include/clang/Basic/OpenACCKinds.h
index 3414df999917..e3f741784332 100644
--- a/clang/include/clang/Basic/OpenACCKinds.h
+++ b/clang/include/clang/Basic/OpenACCKinds.h
@@ -146,6 +146,12 @@ inline llvm::raw_ostream &operator<<(llvm::raw_ostream &Out,
   return printOpenACCDirectiveKind(Out, K);
 }
 
+inline bool isOpenACCComputeDirectiveKind(OpenACCDirectiveKind K) {
+  return K == OpenACCDirectiveKind::Parallel ||
+         K == OpenACCDirectiveKind::Serial ||
+         K == OpenACCDirectiveKind::Kernels;
+}
+
 enum class OpenACCAtomicKind {
   Read,
   Write,
diff --git a/clang/include/clang/Sema/SemaOpenACC.h b/clang/include/clang/Sema/SemaOpenACC.h
index c1fe0f5b9c0f..329dc3945fa2 100644
--- a/clang/include/clang/Sema/SemaOpenACC.h
+++ b/clang/include/clang/Sema/SemaOpenACC.h
@@ -44,7 +44,8 @@ public:
       Expr *ConditionExpr;
     };
 
-    std::variant<DefaultDetails, ConditionDetails> Details;
+    std::variant<std::monostate, DefaultDetails, ConditionDetails> Details =
+        std::monostate{};
 
   public:
     OpenACCParsedClause(OpenACCDirectiveKind DirKind,
@@ -72,8 +73,17 @@ public:
     }
 
     Expr *getConditionExpr() {
-      assert(ClauseKind == OpenACCClauseKind::If &&
+      assert((ClauseKind == OpenACCClauseKind::If ||
+              (ClauseKind == OpenACCClauseKind::Self &&
+               DirKind != OpenACCDirectiveKind::Update)) &&
              "Parsed clause kind does not have a condition expr");
+
+      // 'self' has an optional ConditionExpr, so be tolerant of that. This will
+      // assert in variant otherwise.
+      if (ClauseKind == OpenACCClauseKind::Self &&
+          std::holds_alternative<std::monostate>(Details))
+        return nullptr;
+
       return std::get<ConditionDetails>(Details).ConditionExpr;
     }
 
@@ -87,7 +97,9 @@ public:
     }
 
     void setConditionDetails(Expr *ConditionExpr) {
-      assert(ClauseKind == OpenACCClauseKind::If &&
+      assert((ClauseKind == OpenACCClauseKind::If ||
+              (ClauseKind == OpenACCClauseKind::Self &&
+               DirKind != OpenACCDirectiveKind::Update)) &&
              "Parsed clause kind does not have a condition expr");
       // In C++ we can count on this being a 'bool', but in C this gets left as
       // some sort of scalar that codegen will have to take care of converting.
diff --git a/clang/lib/AST/OpenACCClause.cpp b/clang/lib/AST/OpenACCClause.cpp
index dcb512cb5141..9c259c8f9bd0 100644
--- a/clang/lib/AST/OpenACCClause.cpp
+++ b/clang/lib/AST/OpenACCClause.cpp
@@ -48,6 +48,26 @@ OpenACCIfClause::OpenACCIfClause(SourceLocation BeginLoc,
          "Condition expression type not scalar/dependent");
 }
 
+OpenACCSelfClause *OpenACCSelfClause::Create(const ASTContext &C,
+                                             SourceLocation BeginLoc,
+                                             SourceLocation LParenLoc,
+                                             Expr *ConditionExpr,
+                                             SourceLocation EndLoc) {
+  void *Mem = C.Allocate(sizeof(OpenACCIfClause), alignof(OpenACCIfClause));
+  return new (Mem)
+      OpenACCSelfClause(BeginLoc, LParenLoc, ConditionExpr, EndLoc);
+}
+
+OpenACCSelfClause::OpenACCSelfClause(SourceLocation BeginLoc,
+                                     SourceLocation LParenLoc,
+                                     Expr *ConditionExpr, SourceLocation EndLoc)
+    : OpenACCClauseWithCondition(OpenACCClauseKind::Self, BeginLoc, LParenLoc,
+                                 ConditionExpr, EndLoc) {
+  assert((!ConditionExpr || ConditionExpr->isInstantiationDependent() ||
+          ConditionExpr->getType()->isScalarType()) &&
+         "Condition expression type not scalar/dependent");
+}
+
 OpenACCClause::child_range OpenACCClause::children() {
   switch (getClauseKind()) {
   default:
@@ -72,3 +92,9 @@ void OpenACCClausePrinter::VisitDefaultClause(const OpenACCDefaultClause &C) {
 void OpenACCClausePrinter::VisitIfClause(const OpenACCIfClause &C) {
   OS << "if(" << C.getConditionExpr() << ")";
 }
+
+void OpenACCClausePrinter::VisitSelfClause(const OpenACCSelfClause &C) {
+  OS << "self";
+  if (const Expr *CondExpr = C.getConditionExpr())
+    OS << "(" << CondExpr << ")";
+}
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index 789e4634bd29..b26d804c6f07 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -2491,6 +2491,11 @@ void OpenACCClauseProfiler::VisitIfClause(const OpenACCIfClause &Clause) {
          "if clause requires a valid condition expr");
   Profiler.VisitStmt(Clause.getConditionExpr());
 }
+
+void OpenACCClauseProfiler::VisitSelfClause(const OpenACCSelfClause &Clause) {
+  if (Clause.hasConditionExpr())
+    Profiler.VisitStmt(Clause.getConditionExpr());
+}
 } // namespace
 
 void StmtProfiler::VisitOpenACCComputeConstruct(
diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp
index 688daa64d619..ff5b3df2d6df 100644
--- a/clang/lib/AST/TextNodeDumper.cpp
+++ b/clang/lib/AST/TextNodeDumper.cpp
@@ -398,6 +398,7 @@ void TextNodeDumper::Visit(const OpenACCClause *C) {
       OS << '(' << cast<OpenACCDefaultClause>(C)->getDefaultClauseKind() << ')';
       break;
     case OpenACCClauseKind::If:
+    case OpenACCClauseKind::Self:
       // The condition expression will be printed as a part of the 'children',
       // but print 'clause' here so it is clear what is happening from the dump.
       OS << " clause";
diff --git a/clang/lib/Parse/ParseOpenACC.cpp b/clang/lib/Parse/ParseOpenACC.cpp
index 91f2b8afcf0c..123be476e928 100644
--- a/clang/lib/Parse/ParseOpenACC.cpp
+++ b/clang/lib/Parse/ParseOpenACC.cpp
@@ -835,19 +835,23 @@ Parser::OpenACCClauseParseResult Parser::ParseOpenACCClauseParams(
     case OpenACCClauseKind::Default: {
       Token DefKindTok = getCurToken();
 
-      if (expectIdentifierOrKeyword(*this))
-        break;
+      if (expectIdentifierOrKeyword(*this)) {
+        Parens.skipToEnd();
+        return OpenACCCanContinue();
+      }
 
       ConsumeToken();
 
       OpenACCDefaultClauseKind DefKind =
           getOpenACCDefaultClauseKind(DefKindTok);
 
-      if (DefKind == OpenACCDefaultClauseKind::Invalid)
+      if (DefKind == OpenACCDefaultClauseKind::Invalid) {
         Diag(DefKindTok, diag::err_acc_invalid_default_clause_kind);
-      else
-        ParsedClause.setDefaultDetails(DefKind);
+        Parens.skipToEnd();
+        return OpenACCCanContinue();
+      }
 
+      ParsedClause.setDefaultDetails(DefKind);
       break;
     }
     case OpenACCClauseKind::If: {
@@ -977,6 +981,8 @@ Parser::OpenACCClauseParseResult Parser::ParseOpenACCClauseParams(
       case OpenACCClauseKind::Self: {
         assert(DirKind != OpenACCDirectiveKind::Update);
         ExprResult CondExpr = ParseOpenACCConditionExpr();
+        ParsedClause.setConditionDetails(CondExpr.isUsable() ? CondExpr.get()
+                                                             : nullptr);
 
         if (CondExpr.isInvalid()) {
           Parens.skipToEnd();
diff --git a/clang/lib/Sema/SemaOpenACC.cpp b/clang/lib/Sema/SemaOpenACC.cpp
index 1249136c8765..59f65eaf47a6 100644
--- a/clang/lib/Sema/SemaOpenACC.cpp
+++ b/clang/lib/Sema/SemaOpenACC.cpp
@@ -15,6 +15,7 @@
 #include "clang/AST/StmtOpenACC.h"
 #include "clang/Basic/DiagnosticSema.h"
 #include "clang/Sema/Sema.h"
+#include "llvm/Support/Casting.h"
 
 using namespace clang;
 
@@ -76,6 +77,19 @@ bool doesClauseApplyToDirective(OpenACCDirectiveKind DirectiveKind,
     default:
       return false;
     }
+  case OpenACCClauseKind::Self:
+    switch (DirectiveKind) {
+    case OpenACCDirectiveKind::Parallel:
+    case OpenACCDirectiveKind::Serial:
+    case OpenACCDirectiveKind::Kernels:
+    case OpenACCDirectiveKind::Update:
+    case OpenACCDirectiveKind::ParallelLoop:
+    case OpenACCDirectiveKind::SerialLoop:
+    case OpenACCDirectiveKind::KernelsLoop:
+      return true;
+    default:
+      return false;
+    }
   default:
     // Do nothing so we can go to the 'unimplemented' diagnostic instead.
     return true;
@@ -121,9 +135,7 @@ SemaOpenACC::ActOnClause(ArrayRef<const OpenACCClause *> ExistingClauses,
     // Restrictions only properly implemented on 'compute' constructs, and
     // 'compute' constructs are the only construct that can do anything with
     // this yet, so skip/treat as unimplemented in this case.
-    if (Clause.getDirectiveKind() != OpenACCDirectiveKind::Parallel &&
-        Clause.getDirectiveKind() != OpenACCDirectiveKind::Serial &&
-        Clause.getDirectiveKind() != OpenACCDirectiveKind::Kernels)
+    if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
       break;
 
     // Don't add an invalid clause to the AST.
@@ -146,9 +158,7 @@ SemaOpenACC::ActOnClause(ArrayRef<const OpenACCClause *> ExistingClauses,
     // Restrictions only properly implemented on 'compute' constructs, and
     // 'compute' constructs are the only construct that can do anything with
     // this yet, so skip/treat as unimplemented in this case.
-    if (Clause.getDirectiveKind() != OpenACCDirectiveKind::Parallel &&
-        Clause.getDirectiveKind() != OpenACCDirectiveKind::Serial &&
-        Clause.getDirectiveKind() != OpenACCDirectiveKind::Kernels)
+    if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
       break;
 
     // There is no prose in the standard that says duplicates aren't allowed,
@@ -160,12 +170,54 @@ SemaOpenACC::ActOnClause(ArrayRef<const OpenACCClause *> ExistingClauses,
     // The parser has ensured that we have a proper condition expr, so there
     // isn't really much to do here.
 
-    // TODO OpenACC: When we implement 'self', this clauses causes us to
-    // 'ignore' the self clause, so we should implement a warning here.
+    // If the 'if' clause is true, it makes the 'self' clause have no effect,
+    // diagnose that here.
+    // TODO OpenACC: When we add these two to other constructs, we might not
+    // want to warn on this (for example, 'update').
+    const auto *Itr =
+        llvm::find_if(ExistingClauses, llvm::IsaPred<OpenACCSelfClause>);
+    if (Itr != ExistingClauses.end()) {
+      Diag(Clause.getBeginLoc(), diag::warn_acc_if_self_conflict);
+      Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here);
+    }
+
     return OpenACCIfClause::Create(
         getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(),
         Clause.getConditionExpr(), Clause.getEndLoc());
   }
+
+  case OpenACCClauseKind::Self: {
+    // Restrictions only properly implemented on 'compute' constructs, and
+    // 'compute' constructs are the only construct that can do anything with
+    // this yet, so skip/treat as unimplemented in this case.
+    if (!isOpenACCComputeDirectiveKind(Clause.getDirectiveKind()))
+      break;
+
+    // TODO OpenACC: When we implement this for 'update', this takes a
+    // 'var-list' instead of a condition expression, so semantics/handling has
+    // to happen differently here.
+
+    // There is no prose in the standard that says duplicates aren't allowed,
+    // but this diagnostic is present in other compilers, as well as makes
+    // sense.
+    if (checkAlreadyHasClauseOfKind(*this, ExistingClauses, Clause))
+      return nullptr;
+
+    // If the 'if' clause is true, it makes the 'self' clause have no effect,
+    // diagnose that here.
+    // TODO OpenACC: When we add these two to other constructs, we might not
+    // want to warn on this (for example, 'update').
+    const auto *Itr =
+        llvm::find_if(ExistingClauses, llvm::IsaPred<OpenACCIfClause>);
+    if (Itr != ExistingClauses.end()) {
+      Diag(Clause.getBeginLoc(), diag::warn_acc_if_self_conflict);
+      Diag((*Itr)->getBeginLoc(), diag::note_acc_previous_clause_here);
+    }
+
+    return OpenACCSelfClause::Create(
+        getASTContext(), Clause.getBeginLoc(), Clause.getLParenLoc(),
+        Clause.getConditionExpr(), Clause.getEndLoc());
+  }
   default:
     break;
   }
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index acc2d7ff9d42..0c7fdb357235 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -11088,6 +11088,77 @@ OMPClause *TreeTransform<Derived>::TransformOMPXBareClause(OMPXBareClause *C) {
 //===----------------------------------------------------------------------===//
 // OpenACC transformation
 //===----------------------------------------------------------------------===//
+namespace {
+template <typename Derived>
+class OpenACCClauseTransform final
+    : public OpenACCClauseVisitor<OpenACCClauseTransform<Derived>> {
+  TreeTransform<Derived> &Self;
+  SemaOpenACC::OpenACCParsedClause &ParsedClause;
+  OpenACCClause *NewClause = nullptr;
+
+public:
+  OpenACCClauseTransform(TreeTransform<Derived> &Self,
+                         SemaOpenACC::OpenACCParsedClause &PC)
+      : Self(Self), ParsedClause(PC) {}
+
+  OpenACCClause *CreatedClause() const { return NewClause; }
+
+#define VISIT_CLAUSE(CLAUSE_NAME)                                              \
+  void Visit##CLAUSE_NAME##Clause(const OpenACC##CLAUSE_NAME##Clause &Clause);
+#include "clang/Basic/OpenACCClauses.def"
+};
+
+template <typename Derived>
+void OpenACCClauseTransform<Derived>::VisitDefaultClause(
+    const OpenACCDefaultClause &C) {
+  ParsedClause.setDefaultDetails(C.getDefaultClauseKind());
+
+  NewClause = OpenACCDefaultClause::Create(
+      Self.getSema().getASTContext(), ParsedClause.getDefaultClauseKind(),
+      ParsedClause.getBeginLoc(), ParsedClause.getLParenLoc(),
+      ParsedClause.getEndLoc());
+}
+
+template <typename Derived>
+void OpenACCClauseTransform<Derived>::VisitIfClause(const OpenACCIfClause &C) {
+  Expr *Cond = const_cast<Expr *>(C.getConditionExpr());
+  assert(Cond && "If constructed with invalid Condition");
+  Sema::ConditionResult Res = Self.TransformCondition(
+      Cond->getExprLoc(), /*Var=*/nullptr, Cond, Sema::ConditionKind::Boolean);
+
+  if (Res.isInvalid() || !Res.get().second)
+    return;
+
+  ParsedClause.setConditionDetails(Res.get().second);
+
+  NewClause = OpenACCIfClause::Create(
+      Self.getSema().getASTContext(), ParsedClause.getBeginLoc(),
+      ParsedClause.getLParenLoc(), ParsedClause.getConditionExpr(),
+      ParsedClause.getEndLoc());
+}
+
+template <typename Derived>
+void OpenACCClauseTransform<Derived>::VisitSelfClause(
+    const OpenACCSelfClause &C) {
+
+  if (C.hasConditionExpr()) {
+    Expr *Cond = const_cast<Expr *>(C.getConditionExpr());
+    Sema::ConditionResult Res =
+        Self.TransformCondition(Cond->getExprLoc(), /*Var=*/nullptr, Cond,
+                                Sema::ConditionKind::Boolean);
+
+    if (Res.isInvalid() || !Res.get().second)
+      return;
+
+    ParsedClause.setConditionDetails(Res.get().second);
+  }
+
+  NewClause = OpenACCSelfClause::Create(
+      Self.getSema().getASTContext(), ParsedClause.getBeginLoc(),
+      ParsedClause.getLParenLoc(), ParsedClause.getConditionExpr(),
+      ParsedClause.getEndLoc());
+}
+} // namespace
 template <typename Derived>
 OpenACCClause *TreeTransform<Derived>::TransformOpenACCClause(
     ArrayRef<const OpenACCClause *> ExistingClauses,
@@ -11100,33 +11171,10 @@ OpenACCClause *TreeTransform<Derived>::TransformOpenACCClause(
   if (const auto *WithParms = dyn_cast<OpenACCClauseWithParams>(OldClause))
     ParsedClause.setLParenLoc(WithParms->getLParenLoc());
 
-  switch (OldClause->getClauseKind()) {
-  case OpenACCClauseKind::Default:
-    // There is nothing to do here as nothing dependent can appear in this
-    // clause. So just set the values so Sema can set the right value.
-    ParsedClause.setDefaultDetails(
-        cast<OpenACCDefaultClause>(OldClause)->getDefaultClauseKind());
-    break;
-  case OpenACCClauseKind::If: {
-    Expr *Cond = const_cast<Expr *>(
-        cast<OpenACCIfClause>(OldClause)->getConditionExpr());
-    assert(Cond && "If constructed with invalid Condition");
-    Sema::ConditionResult Res =
-        TransformCondition(Cond->getExprLoc(), /*Var=*/nullptr, Cond,
-                           Sema::ConditionKind::Boolean);
-
-    if (Res.isInvalid() || !Res.get().second)
-      return nullptr;
-
-    ParsedClause.setConditionDetails(Res.get().second);
-    break;
-  }
-  default:
-    assert(false && "Unhandled OpenACC clause in TreeTransform");
-    return nullptr;
-  }
+  OpenACCClauseTransform<Derived> Transform{*this, ParsedClause};
+  Transform.Visit(OldClause);
 
-  return getSema().OpenACC().ActOnClause(ExistingClauses, ParsedClause);
+  return Transform.CreatedClause();
 }
 
 template <typename Derived>
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index f47d540ea4b8..cf0726460bfc 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -11781,6 +11781,12 @@ OpenACCClause *ASTRecordReader::readOpenACCClause() {
     return OpenACCIfClause::Create(getContext(), BeginLoc, LParenLoc, CondExpr,
                                    EndLoc);
   }
+  case OpenACCClauseKind::Self: {
+    SourceLocation LParenLoc = readSourceLocation();
+    Expr *CondExpr = readBool() ? readSubExpr() : nullptr;
+    return OpenACCSelfClause::Create(getContext(), BeginLoc, LParenLoc,
+                                     CondExpr, EndLoc);
+  }
   case OpenACCClauseKind::Finalize:
   case OpenACCClauseKind::IfPresent:
   case OpenACCClauseKind::Seq:
@@ -11789,7 +11795,6 @@ OpenACCClause *ASTRecordReader::readOpenACCClause() {
   case OpenACCClauseKind::Worker:
   case OpenACCClauseKind::Vector:
   case OpenACCClauseKind::NoHost:
-  case OpenACCClauseKind::Self:
   case OpenACCClauseKind::Copy:
   case OpenACCClauseKind::UseDevice:
   case OpenACCClauseKind::Attach:
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index ce6fa1feb1ee..b2a078b6d80f 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -7524,6 +7524,14 @@ void ASTRecordWriter::writeOpenACCClause(const OpenACCClause *C) {
     AddStmt(const_cast<Expr *>(IC->getConditionExpr()));
     return;
   }
+  case OpenACCClauseKind::Self: {
+    const auto *SC = cast<OpenACCIfClause>(C);
+    writeSourceLocation(SC->getLParenLoc());
+    writeBool(SC->hasConditionExpr());
+    if (SC->hasConditionExpr())
+      AddStmt(const_cast<Expr *>(SC->getConditionExpr()));
+    return;
+  }
   case OpenACCClauseKind::Finalize:
   case OpenACCClauseKind::IfPresent:
   case OpenACCClauseKind::Seq:
@@ -7532,7 +7540,6 @@ void ASTRecordWriter::writeOpenACCClause(const OpenACCClause *C) {
   case OpenACCClauseKind::Worker:
   case OpenACCClauseKind::Vector:
   case OpenACCClauseKind::NoHost:
-  case OpenACCClauseKind::Self:
   case OpenACCClauseKind::Copy:
   case OpenACCClauseKind::UseDevice:
   case OpenACCClauseKind::Attach:
diff --git a/clang/test/ParserOpenACC/parse-clauses.c b/clang/test/ParserOpenACC/parse-clauses.c
index 2369df58308a..4462f0df540f 100644
--- a/clang/test/ParserOpenACC/parse-clauses.c
+++ b/clang/test/ParserOpenACC/parse-clauses.c
@@ -376,16 +376,13 @@ void SelfClause() {
 #pragma acc serial self(i > j, seq
   for(;;){}
 
-  // expected-warning@+2{{left operand of comma operator has no effect}}
-  // expected-warning@+1{{OpenACC clause 'self' not yet implemented, clause ignored}}
+  // expected-warning@+1{{left operand of comma operator has no effect}}
 #pragma acc serial self(i, j)
   for(;;){}
 
-  // expected-warning@+1{{OpenACC clause 'self' not yet implemented, clause ignored}}
 #pragma acc serial self(i > j)
   for(;;){}
 
-  // expected-warning@+2{{OpenACC clause 'self' not yet implemented, clause ignored}}
   // expected-warning@+1{{OpenACC clause 'seq' not yet implemented, clause ignored}}
 #pragma acc serial self(1+5>3), seq
   for(;;){}
diff --git a/clang/test/SemaOpenACC/compute-construct-clause-ast.cpp b/clang/test/SemaOpenACC/compute-construct-clause-ast.cpp
index 018f0b68c781..6d2efcf81eb6 100644
--- a/clang/test/SemaOpenACC/compute-construct-clause-ast.cpp
+++ b/clang/test/SemaOpenACC/compute-construct-clause-ast.cpp
@@ -110,6 +110,50 @@ void TemplFunc() {
   // CHECK-NEXT: CXXBoolLiteralExpr
   // CHECK-NEXT: NullStmt
 
+#pragma acc serial self
+  while(true);
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}serial
+  // CHECK-NEXT: self clause
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: NullStmt
+
+#pragma acc kernels self(T::SomeFloat)
+  while(true);
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}kernels
+  // CHECK-NEXT: self clause
+  // CHECK-NEXT: DependentScopeDeclRefExpr{{.*}} '<dependent type>' lvalue
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'T'
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: NullStmt
+
+#pragma acc parallel self(T::SomeFloat) if (T::SomeFloat)
+  while(true);
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: self clause
+  // CHECK-NEXT: DependentScopeDeclRefExpr{{.*}} '<dependent type>' lvalue
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'T'
+  // CHECK-NEXT: if clause
+  // CHECK-NEXT: DependentScopeDeclRefExpr{{.*}} '<dependent type>' lvalue
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'T'
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: NullStmt
+
+#pragma acc serial if(T::SomeFloat) self(T::SomeFloat)
+  while(true);
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}serial
+  // CHECK-NEXT: if clause
+  // CHECK-NEXT: DependentScopeDeclRefExpr{{.*}} '<dependent type>' lvalue
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'T'
+  // CHECK-NEXT: self clause
+  // CHECK-NEXT: DependentScopeDeclRefExpr{{.*}} '<dependent type>' lvalue
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'T'
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: NullStmt
+
   // Match the instantiation:
   // CHECK: FunctionDecl{{.*}}TemplFunc{{.*}}implicit_instantiation
   // CHECK-NEXT: TemplateArgument type 'InstTy'
@@ -171,6 +215,53 @@ void TemplFunc() {
   // CHECK-NEXT: WhileStmt
   // CHECK-NEXT: CXXBoolLiteralExpr
   // CHECK-NEXT: NullStmt
+
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}serial
+  // CHECK-NEXT: self clause
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: NullStmt
+
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}kernels
+  // CHECK-NEXT: self clause
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'bool' <FloatingToBoolean>
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'float' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'const float' lvalue Var{{.*}} 'SomeFloat' 'const float'
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'InstTy'
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: NullStmt
+
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}parallel
+  // CHECK-NEXT: self clause
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'bool' <FloatingToBoolean>
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'float' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'const float' lvalue Var{{.*}} 'SomeFloat' 'const float'
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'InstTy'
+  // CHECK-NEXT: if clause
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'bool' <FloatingToBoolean>
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'float' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'const float' lvalue Var{{.*}} 'SomeFloat' 'const float'
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'InstTy'
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: NullStmt
+
+  // CHECK-NEXT: OpenACCComputeConstruct{{.*}}serial
+  // CHECK-NEXT: if clause
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'bool' <FloatingToBoolean>
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'float' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'const float' lvalue Var{{.*}} 'SomeFloat' 'const float'
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'InstTy'
+  // CHECK-NEXT: self clause
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'bool' <FloatingToBoolean>
+  // CHECK-NEXT: ImplicitCastExpr{{.*}}'float' <LValueToRValue>
+  // CHECK-NEXT: DeclRefExpr{{.*}} 'const float' lvalue Var{{.*}} 'SomeFloat' 'const float'
+  // CHECK-NEXT: NestedNameSpecifier TypeSpec 'InstTy'
+  // CHECK-NEXT: WhileStmt
+  // CHECK-NEXT: CXXBoolLiteralExpr
+  // CHECK-NEXT: NullStmt
+
 }
 
 struct BoolConversion{ operator bool() const;};
diff --git a/clang/test/SemaOpenACC/compute-construct-self-clause.c b/clang/test/SemaOpenACC/compute-construct-self-clause.c
new file mode 100644
index 000000000000..fbed2953419a
--- /dev/null
+++ b/clang/test/SemaOpenACC/compute-construct-self-clause.c
@@ -0,0 +1,82 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+void BoolExpr(int *I, float *F) {
+  typedef struct {} SomeStruct;
+  struct C{};
+  // expected-error@+1{{expected expression}}
+#pragma acc parallel self (struct C f())
+  while(0);
+
+  // expected-error@+1{{unexpected type name 'SomeStruct': expected expression}}
+#pragma acc serial self (SomeStruct)
+  while(0);
+
+  // expected-error@+1{{unexpected type name 'SomeStruct': expected expression}}
+#pragma acc serial self (SomeStruct())
+  while(0);
+
+  SomeStruct S;
+  // expected-error@+1{{statement requires expression of scalar type ('SomeStruct' invalid)}}
+#pragma acc serial self (S)
+  while(0);
+
+#pragma acc parallel self (I)
+  while(0);
+
+#pragma acc serial self (F)
+  while(0);
+
+#pragma acc kernels self (*I < *F)
+  while(0);
+}
+
+void WarnMaybeNotUsed(int val1, int val2) {
+
+  // expected-warning@+2{{OpenACC construct 'self' has no effect when an 'if' clause evaluates to true}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel self if(val1)
+  while(0);
+
+  // expected-warning@+2{{OpenACC construct 'self' has no effect when an 'if' clause evaluates to true}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel self(val1) if(val1)
+  while(0);
+
+  // expected-warning@+2{{OpenACC construct 'self' has no effect when an 'if' clause evaluates to true}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel if(val1) self
+  while(0);
+
+  // expected-warning@+2{{OpenACC construct 'self' has no effect when an 'if' clause evaluates to true}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel if(val1) self(val2)
+  while(0);
+
+  // The below don't warn because one side or the other has an error, thus is
+  // not added to the AST.
+
+  // expected-error@+1{{use of undeclared identifier 'invalid'}}
+#pragma acc parallel self if(invalid)
+  while(0);
+
+  // expected-error@+1{{use of undeclared identifier 'invalid'}}
+#pragma acc parallel self(invalid) if(val1)
+  while(0);
+
+  // expected-error@+2{{expected expression}}
+  // expected-error@+1{{use of undeclared identifier 'invalid'}}
+#pragma acc parallel self() if(invalid)
+  while(0);
+
+  // expected-error@+1{{use of undeclared identifier 'invalid'}}
+#pragma acc parallel if(invalid) self
+  while(0);
+
+  // expected-error@+1{{use of undeclared identifier 'invalid'}}
+#pragma acc parallel if(val2) self(invalid)
+  while(0);
+
+  // expected-error@+1{{use of undeclared identifier 'invalid'}}
+#pragma acc parallel if(invalid) self(val1)
+  while(0);
+}
diff --git a/clang/test/SemaOpenACC/compute-construct-self-clause.cpp b/clang/test/SemaOpenACC/compute-construct-self-clause.cpp
new file mode 100644
index 000000000000..60edbdc2b119
--- /dev/null
+++ b/clang/test/SemaOpenACC/compute-construct-self-clause.cpp
@@ -0,0 +1,99 @@
+// RUN: %clang_cc1 %s -fopenacc -verify
+
+struct NoBoolConversion{};
+struct BoolConversion{
+  operator bool();
+};
+
+template <typename T, typename U>
+void BoolExpr() {
+  // expected-error@+1{{value of type 'NoBoolConversion' is not contextually convertible to 'bool'}}
+#pragma acc parallel self (NoBoolConversion{})
+  while(0);
+  // expected-error@+2{{no member named 'NotValid' in 'NoBoolConversion'}}
+  // expected-note@#INST{{in instantiation of function template specialization}}
+#pragma acc parallel self (T::NotValid)
+  while(0);
+
+#pragma acc parallel self (BoolConversion{})
+  while(0);
+
+  // expected-error@+1{{value of type 'NoBoolConversion' is not contextually convertible to 'bool'}}
+#pragma acc parallel self (T{})
+  while(0);
+
+#pragma acc parallel self (U{})
+  while(0);
+}
+
+struct HasBool {
+  static constexpr bool B = true;
+};
+
+template<typename T>
+void WarnMaybeNotUsed() {
+  // expected-warning@+2{{OpenACC construct 'self' has no effect when an 'if' clause evaluates to true}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel self if(T::B)
+  while(0);
+
+  // expected-warning@+2{{OpenACC construct 'self' has no effect when an 'if' clause evaluates to true}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel self(T::B) if(T::B)
+  while(0);
+
+  // expected-warning@+2{{OpenACC construct 'self' has no effect when an 'if' clause evaluates to true}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel if(T::B) self
+  while(0);
+
+  // expected-warning@+2{{OpenACC construct 'self' has no effect when an 'if' clause evaluates to true}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel if(T::B) self(T::B)
+  while(0);
+
+  // We still warn in the cases of dependent failures, since the diagnostic
+  // happens immediately rather than during instantiation.
+
+  // expected-error@+4{{no member named 'Invalid' in 'HasBool'}}
+  // expected-note@#NOT_USED_INST{{in instantiation of function template specialization 'WarnMaybeNotUsed<HasBool>' requested here}}
+  // expected-warning@+2{{OpenACC construct 'self' has no effect when an 'if' clause evaluates to true}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel self if(T::Invalid)
+  while(0);
+
+  // expected-error@+3{{no member named 'Invalid' in 'HasBool'}}
+  // expected-warning@+2{{OpenACC construct 'self' has no effect when an 'if' clause evaluates to true}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel self(T::Invalid) if(T::B)
+  while(0);
+
+  // expected-error@+3{{no member named 'Invalid' in 'HasBool'}}
+  // expected-warning@+2{{OpenACC construct 'self' has no effect when an 'if' clause evaluates to true}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel self(T::B) if(T::Invalid)
+  while(0);
+
+  // expected-error@+3{{no member named 'Invalid' in 'HasBool'}}
+  // expected-warning@+2{{OpenACC construct 'self' has no effect when an 'if' clause evaluates to true}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel if(T::Invalid) self
+  while(0);
+
+  // expected-error@+3{{no member named 'Invalid' in 'HasBool'}}
+  // expected-warning@+2{{OpenACC construct 'self' has no effect when an 'if' clause evaluates to true}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel if(T::Invalid) self(T::B)
+  while(0);
+
+  // expected-error@+3{{no member named 'Invalid' in 'HasBool'}}
+  // expected-warning@+2{{OpenACC construct 'self' has no effect when an 'if' clause evaluates to true}}
+  // expected-note@+1{{previous clause is here}}
+#pragma acc parallel if(T::B) self(T::Invalid)
+  while(0);
+}
+
+void Instantiate() {
+  BoolExpr<NoBoolConversion, BoolConversion>(); // #INST
+  WarnMaybeNotUsed<HasBool>(); // #NOT_USED_INST
+}
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index f304786ff9df..2ef599d2cd26 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2791,6 +2791,10 @@ void OpenACCClauseEnqueue::VisitDefaultClause(const OpenACCDefaultClause &C) {}
 void OpenACCClauseEnqueue::VisitIfClause(const OpenACCIfClause &C) {
   Visitor.AddStmt(C.getConditionExpr());
 }
+void OpenACCClauseEnqueue::VisitSelfClause(const OpenACCSelfClause &C) {
+  if (C.hasConditionExpr())
+    Visitor.AddStmt(C.getConditionExpr());
+}
 } // namespace
 
 void EnqueueVisitor::EnqueueChildren(const OpenACCClause *C) {
-- 
GitLab


From ac791888bbbe58651e597cf7a4b2276424b77a92 Mon Sep 17 00:00:00 2001
From: Benjamin Kramer <benny.kra@googlemail.com>
Date: Tue, 16 Apr 2024 15:58:16 +0200
Subject: [PATCH 407/464] [bazel] Add missing dependency for
 1c076b43c294c7d29d99dd50f6853b33a5b99789

---
 utils/bazel/llvm-project-overlay/mlir/BUILD.bazel | 1 +
 1 file changed, 1 insertion(+)

diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 653c4bd30600..03386549a011 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -11631,6 +11631,7 @@ cc_library(
         ":DialectUtils",
         ":FuncDialect",
         ":IR",
+        ":InferTypeOpInterface",
         ":LinalgDialect",
         ":MathDialect",
         ":Pass",
-- 
GitLab


From a0f8191af3945482f0f7a7c8f030e8c519a795b7 Mon Sep 17 00:00:00 2001
From: Fraser Cormack <fraser@codeplay.com>
Date: Tue, 16 Apr 2024 13:12:52 +0100
Subject: [PATCH 408/464] [libclc] Give built bytecode objects a .bc extension.
 NFC

---
 libclc/CMakeLists.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libclc/CMakeLists.txt b/libclc/CMakeLists.txt
index 7528228b3b7f..302f559dba7a 100644
--- a/libclc/CMakeLists.txt
+++ b/libclc/CMakeLists.txt
@@ -326,7 +326,7 @@ foreach( t ${LIBCLC_TARGETS_TO_BUILD} )
         # Generated files are given just as file names, which we must make
         # absolute to the binary directory.
         set( input_file ${CMAKE_CURRENT_BINARY_DIR}/${file} )
-        set( output_file "${LIBCLC_ARCH_OBJFILE_DIR}/${file}.o" )
+        set( output_file "${LIBCLC_ARCH_OBJFILE_DIR}/${file}.bc" )
       else()
         # Other files are originally relative to each SOURCE file, which are
         # then make relative to the libclc root directory. We must normalize
@@ -336,7 +336,7 @@ foreach( t ${LIBCLC_TARGETS_TO_BUILD} )
         get_filename_component( abs_path ${file} ABSOLUTE BASE_DIR ${PROJECT_SOURCE_DIR} )
         file( RELATIVE_PATH root_rel_path ${PROJECT_SOURCE_DIR} ${abs_path} )
         set( input_file ${PROJECT_SOURCE_DIR}/${file} )
-        set( output_file "${LIBCLC_ARCH_OBJFILE_DIR}/${root_rel_path}.o" )
+        set( output_file "${LIBCLC_ARCH_OBJFILE_DIR}/${root_rel_path}.bc" )
       endif()
 
       get_filename_component( file_dir ${file} DIRECTORY )
-- 
GitLab


From 3d118f92081ea0c7048749dc5d08c8e8217be4eb Mon Sep 17 00:00:00 2001
From: Fraser Cormack <fraser@codeplay.com>
Date: Tue, 16 Apr 2024 13:59:59 +0100
Subject: [PATCH 409/464] [libclc] Fix dependencies between targets

We need file-level - not target-level - dependencies for these custom
commands to re-trigger when their dependencies change.
---
 libclc/CMakeLists.txt | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/libclc/CMakeLists.txt b/libclc/CMakeLists.txt
index 302f559dba7a..ed2764847e70 100644
--- a/libclc/CMakeLists.txt
+++ b/libclc/CMakeLists.txt
@@ -364,7 +364,7 @@ foreach( t ${LIBCLC_TARGETS_TO_BUILD} )
       set( spv_suffix ${arch_suffix}.spv )
       add_custom_command( OUTPUT ${spv_suffix}
         COMMAND ${LLVM_SPIRV} ${spvflags} -o ${spv_suffix} ${builtins_link_lib}
-        DEPENDS ${builtins_link_lib_tgt}
+        DEPENDS ${builtins_link_lib}
       )
       add_custom_target( "prepare-${spv_suffix}" ALL DEPENDS "${spv_suffix}" )
       install( FILES ${CMAKE_CURRENT_BINARY_DIR}/${spv_suffix}
@@ -376,7 +376,7 @@ foreach( t ${LIBCLC_TARGETS_TO_BUILD} )
       add_custom_command( OUTPUT ${builtins_opt_lib_tgt}.bc
         COMMAND libclc::opt ${opt_flags} -o ${builtins_opt_lib_tgt}.bc
           ${builtins_link_lib}
-        DEPENDS libclc::opt ${builtins_link_lib_tgt}
+        DEPENDS libclc::opt ${builtins_link_lib}
       )
       add_custom_target( ${builtins_opt_lib_tgt}
         ALL DEPENDS ${builtins_opt_lib_tgt}.bc
@@ -385,12 +385,13 @@ foreach( t ${LIBCLC_TARGETS_TO_BUILD} )
         PROPERTIES TARGET_FILE ${builtins_opt_lib_tgt}.bc
       )
 
+      set( builtins_opt_lib $<TARGET_PROPERTY:${builtins_opt_lib_tgt},TARGET_FILE> )
+
       # Add prepare target
       set( obj_suffix ${arch_suffix}.bc )
       add_custom_command( OUTPUT ${obj_suffix}
-        COMMAND prepare_builtins -o ${obj_suffix}
-          $<TARGET_PROPERTY:${builtins_opt_lib_tgt},TARGET_FILE>
-        DEPENDS ${builtins_opt_lib_tgt} prepare_builtins )
+        COMMAND prepare_builtins -o ${obj_suffix} ${builtins_opt_lib}
+        DEPENDS ${builtins_opt_lib} prepare_builtins )
       add_custom_target( prepare-${obj_suffix} ALL DEPENDS ${obj_suffix} )
 
       # nvptx-- targets don't include workitem builtins
-- 
GitLab


From 60de56c743c414240b293a8b8ee10bc2129d7e10 Mon Sep 17 00:00:00 2001
From: Harald van Dijk <harald.vandijk@codeplay.com>
Date: Tue, 16 Apr 2024 15:21:09 +0100
Subject: [PATCH 410/464] [ValueTracking] Restore isKnownNonZero parameter
 order. (#88873)

Prior to #85863, the required parameters of llvm::isKnownNonZero were
Value and DataLayout. After, they are Value, Depth, and SimplifyQuery,
where SimplifyQuery is implicitly constructible from DataLayout. The
change to move Depth before SimplifyQuery needed callers to be updated
unnecessarily, and as commented in #85863, we actually want Depth to be
after SimplifyQuery anyway so that it can be defaulted and the caller
does not need to specify it.
---
 clang/lib/CodeGen/CGCall.cpp                  |   3 +-
 llvm/include/llvm/Analysis/ValueTracking.h    |   2 +-
 llvm/lib/Analysis/BasicAliasAnalysis.cpp      |   3 +-
 llvm/lib/Analysis/InstructionSimplify.cpp     |  25 ++---
 llvm/lib/Analysis/LazyValueInfo.cpp           |   5 +-
 llvm/lib/Analysis/Loads.cpp                   |   4 +-
 llvm/lib/Analysis/ScalarEvolution.cpp         |   2 +-
 llvm/lib/Analysis/ValueTracking.cpp           | 106 +++++++++---------
 llvm/lib/CodeGen/CodeGenPrepare.cpp           |   2 +-
 .../Transforms/IPO/AttributorAttributes.cpp   |   2 +-
 llvm/lib/Transforms/IPO/FunctionAttrs.cpp     |   2 +-
 .../InstCombine/InstCombineAddSub.cpp         |   2 +-
 .../InstCombine/InstCombineAndOrXor.cpp       |   4 +-
 .../InstCombine/InstCombineCalls.cpp          |  11 +-
 .../InstCombine/InstCombineCompares.cpp       |  18 +--
 .../Transforms/InstCombine/InstCombinePHI.cpp |   3 +-
 .../InstCombine/InstructionCombining.cpp      |   2 +-
 .../Instrumentation/MemorySanitizer.cpp       |   4 +-
 .../Utils/PromoteMemoryToRegister.cpp         |   2 +-
 .../lib/Transforms/Utils/SimplifyLibCalls.cpp |  20 ++--
 .../Transforms/Vectorize/VectorCombine.cpp    |   2 +-
 llvm/unittests/Analysis/ValueTrackingTest.cpp |  13 +--
 22 files changed, 113 insertions(+), 124 deletions(-)

diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index 7a0bc6fa77b8..3f5463a9a70e 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -4124,8 +4124,7 @@ static bool isProvablyNull(llvm::Value *addr) {
 }
 
 static bool isProvablyNonNull(Address Addr, CodeGenFunction &CGF) {
-  return llvm::isKnownNonZero(Addr.getBasePointer(), /*Depth=*/0,
-                              CGF.CGM.getDataLayout());
+  return llvm::isKnownNonZero(Addr.getBasePointer(), CGF.CGM.getDataLayout());
 }
 
 /// Emit the actual writing-back of a writeback.
diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h
index 9db0894162af..e1c41b3b55cc 100644
--- a/llvm/include/llvm/Analysis/ValueTracking.h
+++ b/llvm/include/llvm/Analysis/ValueTracking.h
@@ -124,7 +124,7 @@ bool isOnlyUsedInZeroEqualityComparison(const Instruction *CxtI);
 /// specified, perform context-sensitive analysis and return true if the
 /// pointer couldn't possibly be null at the specified instruction.
 /// Supports values with integer or pointer type and vectors of integers.
-bool isKnownNonZero(const Value *V, unsigned Depth, const SimplifyQuery &Q);
+bool isKnownNonZero(const Value *V, const SimplifyQuery &Q, unsigned Depth = 0);
 
 /// Return true if the two given values are negation.
 /// Currently can recoginze Value pair:
diff --git a/llvm/lib/Analysis/BasicAliasAnalysis.cpp b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
index b082dfe8fbd2..16ee2ca49d0e 100644
--- a/llvm/lib/Analysis/BasicAliasAnalysis.cpp
+++ b/llvm/lib/Analysis/BasicAliasAnalysis.cpp
@@ -1283,8 +1283,7 @@ AliasResult BasicAAResult::aliasGEP(
     // VarIndex = Scale*V.
     const VariableGEPIndex &Var = DecompGEP1.VarIndices[0];
     if (Var.Val.TruncBits == 0 &&
-        isKnownNonZero(Var.Val.V, /*Depth=*/0,
-                       SimplifyQuery(DL, DT, &AC, Var.CxtI))) {
+        isKnownNonZero(Var.Val.V, SimplifyQuery(DL, DT, &AC, Var.CxtI))) {
       // Check if abs(V*Scale) >= abs(Scale) holds in the presence of
       // potentially wrapping math.
       auto MultiplyByScaleNoWrap = [](const VariableGEPIndex &Var) {
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index 4e6e66692267..8955de6375de 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -1586,10 +1586,10 @@ static Value *simplifyUnsignedRangeCheck(ICmpInst *ZeroICmp,
     if (match(UnsignedICmp,
               m_c_ICmp(UnsignedPred, m_Specific(Y), m_Specific(A)))) {
       if (UnsignedPred == ICmpInst::ICMP_UGE && IsAnd &&
-          EqPred == ICmpInst::ICMP_NE && isKnownNonZero(B, /*Depth=*/0, Q))
+          EqPred == ICmpInst::ICMP_NE && isKnownNonZero(B, Q))
         return UnsignedICmp;
       if (UnsignedPred == ICmpInst::ICMP_ULT && !IsAnd &&
-          EqPred == ICmpInst::ICMP_EQ && isKnownNonZero(B, /*Depth=*/0, Q))
+          EqPred == ICmpInst::ICMP_EQ && isKnownNonZero(B, Q))
         return UnsignedICmp;
     }
   }
@@ -1607,13 +1607,13 @@ static Value *simplifyUnsignedRangeCheck(ICmpInst *ZeroICmp,
   // X > Y && Y == 0  -->  Y == 0  iff X != 0
   // X > Y || Y == 0  -->  X > Y   iff X != 0
   if (UnsignedPred == ICmpInst::ICMP_UGT && EqPred == ICmpInst::ICMP_EQ &&
-      isKnownNonZero(X, /*Depth=*/0, Q))
+      isKnownNonZero(X, Q))
     return IsAnd ? ZeroICmp : UnsignedICmp;
 
   // X <= Y && Y != 0  -->  X <= Y  iff X != 0
   // X <= Y || Y != 0  -->  Y != 0  iff X != 0
   if (UnsignedPred == ICmpInst::ICMP_ULE && EqPred == ICmpInst::ICMP_NE &&
-      isKnownNonZero(X, /*Depth=*/0, Q))
+      isKnownNonZero(X, Q))
     return IsAnd ? UnsignedICmp : ZeroICmp;
 
   // The transforms below here are expected to be handled more generally with
@@ -2817,10 +2817,9 @@ static Constant *computePointerICmp(CmpInst::Predicate Pred, Value *LHS,
     // the other operand can not be based on the alloc - if it were, then
     // the cmp itself would be a capture.
     Value *MI = nullptr;
-    if (isAllocLikeFn(LHS, TLI) && llvm::isKnownNonZero(RHS, /*Depth=*/0, Q))
+    if (isAllocLikeFn(LHS, TLI) && llvm::isKnownNonZero(RHS, Q))
       MI = LHS;
-    else if (isAllocLikeFn(RHS, TLI) &&
-             llvm::isKnownNonZero(LHS, /*Depth=*/0, Q))
+    else if (isAllocLikeFn(RHS, TLI) && llvm::isKnownNonZero(LHS, Q))
       MI = RHS;
     if (MI) {
       // FIXME: This is incorrect, see PR54002. While we can assume that the
@@ -2976,12 +2975,12 @@ static Value *simplifyICmpWithZero(CmpInst::Predicate Pred, Value *LHS,
     return getTrue(ITy);
   case ICmpInst::ICMP_EQ:
   case ICmpInst::ICMP_ULE:
-    if (isKnownNonZero(LHS, /*Depth=*/0, Q))
+    if (isKnownNonZero(LHS, Q))
       return getFalse(ITy);
     break;
   case ICmpInst::ICMP_NE:
   case ICmpInst::ICMP_UGT:
-    if (isKnownNonZero(LHS, /*Depth=*/0, Q))
+    if (isKnownNonZero(LHS, Q))
       return getTrue(ITy);
     break;
   case ICmpInst::ICMP_SLT: {
@@ -2996,7 +2995,7 @@ static Value *simplifyICmpWithZero(CmpInst::Predicate Pred, Value *LHS,
     KnownBits LHSKnown = computeKnownBits(LHS, /* Depth */ 0, Q);
     if (LHSKnown.isNegative())
       return getTrue(ITy);
-    if (LHSKnown.isNonNegative() && isKnownNonZero(LHS, /*Depth=*/0, Q))
+    if (LHSKnown.isNonNegative() && isKnownNonZero(LHS, Q))
       return getFalse(ITy);
     break;
   }
@@ -3012,7 +3011,7 @@ static Value *simplifyICmpWithZero(CmpInst::Predicate Pred, Value *LHS,
     KnownBits LHSKnown = computeKnownBits(LHS, /* Depth */ 0, Q);
     if (LHSKnown.isNegative())
       return getFalse(ITy);
-    if (LHSKnown.isNonNegative() && isKnownNonZero(LHS, /*Depth=*/0, Q))
+    if (LHSKnown.isNonNegative() && isKnownNonZero(LHS, Q))
       return getTrue(ITy);
     break;
   }
@@ -3165,7 +3164,7 @@ static Value *simplifyICmpWithBinOpOnLHS(CmpInst::Predicate Pred,
   const APInt *C;
   if ((match(LBO, m_LShr(m_Specific(RHS), m_APInt(C))) && *C != 0) ||
       (match(LBO, m_UDiv(m_Specific(RHS), m_APInt(C))) && *C != 1)) {
-    if (isKnownNonZero(RHS, /*Depth=*/0, Q)) {
+    if (isKnownNonZero(RHS, Q)) {
       switch (Pred) {
       default:
         break;
@@ -3398,7 +3397,7 @@ static Value *simplifyICmpWithBinOp(CmpInst::Predicate Pred, Value *LHS,
       bool NUW = Q.IIQ.hasNoUnsignedWrap(LBO) && Q.IIQ.hasNoUnsignedWrap(RBO);
       bool NSW = Q.IIQ.hasNoSignedWrap(LBO) && Q.IIQ.hasNoSignedWrap(RBO);
       if (!NUW || (ICmpInst::isSigned(Pred) && !NSW) ||
-          !isKnownNonZero(LBO->getOperand(0), /*Depth=*/0, Q))
+          !isKnownNonZero(LBO->getOperand(0), Q))
         break;
       if (Value *V = simplifyICmpInst(Pred, LBO->getOperand(1),
                                       RBO->getOperand(1), Q, MaxRecurse - 1))
diff --git a/llvm/lib/Analysis/LazyValueInfo.cpp b/llvm/lib/Analysis/LazyValueInfo.cpp
index 3223b0564e6c..6cded828c25f 100644
--- a/llvm/lib/Analysis/LazyValueInfo.cpp
+++ b/llvm/lib/Analysis/LazyValueInfo.cpp
@@ -645,7 +645,7 @@ LazyValueInfoImpl::solveBlockValueImpl(Value *Val, BasicBlock *BB) {
   // instruction is placed, even if it could legally be hoisted much higher.
   // That is unfortunate.
   PointerType *PT = dyn_cast<PointerType>(BBI->getType());
-  if (PT && isKnownNonZero(BBI, /*Depth=*/0, DL))
+  if (PT && isKnownNonZero(BBI, DL))
     return ValueLatticeElement::getNot(ConstantPointerNull::get(PT));
 
   if (BBI->getType()->isIntegerTy()) {
@@ -1863,8 +1863,7 @@ LazyValueInfo::getPredicateAt(unsigned Pred, Value *V, Constant *C,
   Module *M = CxtI->getModule();
   const DataLayout &DL = M->getDataLayout();
   if (V->getType()->isPointerTy() && C->isNullValue() &&
-      isKnownNonZero(V->stripPointerCastsSameRepresentation(), /*Depth=*/0,
-                     DL)) {
+      isKnownNonZero(V->stripPointerCastsSameRepresentation(), DL)) {
     if (Pred == ICmpInst::ICMP_EQ)
       return LazyValueInfo::False;
     else if (Pred == ICmpInst::ICMP_NE)
diff --git a/llvm/lib/Analysis/Loads.cpp b/llvm/lib/Analysis/Loads.cpp
index b5403408cf2a..ac508e19c9e0 100644
--- a/llvm/lib/Analysis/Loads.cpp
+++ b/llvm/lib/Analysis/Loads.cpp
@@ -100,7 +100,7 @@ static bool isDereferenceableAndAlignedPointer(
   if (KnownDerefBytes.getBoolValue() && KnownDerefBytes.uge(Size) &&
       !CheckForFreed)
     if (!CheckForNonNull ||
-        isKnownNonZero(V, /*Depth=*/0, SimplifyQuery(DL, DT, AC, CtxI))) {
+        isKnownNonZero(V, SimplifyQuery(DL, DT, AC, CtxI))) {
       // As we recursed through GEPs to get here, we've incrementally checked
       // that each step advanced by a multiple of the alignment. If our base is
       // properly aligned, then the original offset accessed must also be.
@@ -134,7 +134,7 @@ static bool isDereferenceableAndAlignedPointer(
     if (getObjectSize(V, ObjSize, DL, TLI, Opts)) {
       APInt KnownDerefBytes(Size.getBitWidth(), ObjSize);
       if (KnownDerefBytes.getBoolValue() && KnownDerefBytes.uge(Size) &&
-          isKnownNonZero(V, /*Depth=*/0, SimplifyQuery(DL, DT, AC, CtxI)) &&
+          isKnownNonZero(V, SimplifyQuery(DL, DT, AC, CtxI)) &&
           !V->canBeFreed()) {
         // As we recursed through GEPs to get here, we've incrementally
         // checked that each step advanced by a multiple of the alignment. If
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 1c98b0295e52..95440dda3b4c 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -6900,7 +6900,7 @@ const ConstantRange &ScalarEvolution::getRangeRef(
         uint64_t Rem = MaxVal.urem(Align);
         MaxVal -= APInt(BitWidth, Rem);
         APInt MinVal = APInt::getZero(BitWidth);
-        if (llvm::isKnownNonZero(V, /*Depth=*/0, DL))
+        if (llvm::isKnownNonZero(V, DL))
           MinVal = Align;
         ConservativeResult = ConservativeResult.intersectWith(
             ConstantRange::getNonEmpty(MinVal, MaxVal + 1), RangeType);
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index b3abf016cfb9..e91dc07f3164 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -272,7 +272,7 @@ bool llvm::isKnownToBeAPowerOfTwo(const Value *V, const DataLayout &DL,
 }
 
 static bool isKnownNonZero(const Value *V, const APInt &DemandedElts,
-                           unsigned Depth, const SimplifyQuery &Q);
+                           const SimplifyQuery &Q, unsigned Depth);
 
 bool llvm::isKnownNonNegative(const Value *V, const SimplifyQuery &SQ,
                               unsigned Depth) {
@@ -288,7 +288,7 @@ bool llvm::isKnownPositive(const Value *V, const SimplifyQuery &SQ,
   // this updated.
   KnownBits Known = computeKnownBits(V, Depth, SQ);
   return Known.isNonNegative() &&
-         (Known.isNonZero() || isKnownNonZero(V, Depth, SQ));
+         (Known.isNonZero() || isKnownNonZero(V, SQ, Depth));
 }
 
 bool llvm::isKnownNegative(const Value *V, const SimplifyQuery &SQ,
@@ -868,7 +868,7 @@ static void computeKnownBitsFromShiftOperator(
   bool ShAmtNonZero =
       Known.isNonZero() ||
       (Known.getMaxValue().ult(Known.getBitWidth()) &&
-       isKnownNonZero(I->getOperand(1), DemandedElts, Depth + 1, Q));
+       isKnownNonZero(I->getOperand(1), DemandedElts, Q, Depth + 1));
   Known = KF(Known2, Known, ShAmtNonZero);
 }
 
@@ -2124,7 +2124,7 @@ bool isKnownToBeAPowerOfTwo(const Value *V, bool OrZero, unsigned Depth,
   case Instruction::Mul:
     return isKnownToBeAPowerOfTwo(I->getOperand(1), OrZero, Depth, Q) &&
            isKnownToBeAPowerOfTwo(I->getOperand(0), OrZero, Depth, Q) &&
-           (OrZero || isKnownNonZero(I, Depth, Q));
+           (OrZero || isKnownNonZero(I, Q, Depth));
   case Instruction::And:
     // A power of two and'd with anything is a power of two or zero.
     if (OrZero &&
@@ -2134,7 +2134,7 @@ bool isKnownToBeAPowerOfTwo(const Value *V, bool OrZero, unsigned Depth,
     // X & (-X) is always a power of two or zero.
     if (match(I->getOperand(0), m_Neg(m_Specific(I->getOperand(1)))) ||
         match(I->getOperand(1), m_Neg(m_Specific(I->getOperand(0)))))
-      return OrZero || isKnownNonZero(I->getOperand(0), Depth, Q);
+      return OrZero || isKnownNonZero(I->getOperand(0), Q, Depth);
     return false;
   case Instruction::Add: {
     // Adding a power-of-two or zero to the same power-of-two or zero yields
@@ -2249,7 +2249,7 @@ static bool isGEPKnownNonNull(const GEPOperator *GEP, unsigned Depth,
 
   // If the base pointer is non-null, we cannot walk to a null address with an
   // inbounds GEP in address space zero.
-  if (isKnownNonZero(GEP->getPointerOperand(), Depth, Q))
+  if (isKnownNonZero(GEP->getPointerOperand(), Q, Depth))
     return true;
 
   // Walk the GEP operands and see if any operand introduces a non-zero offset.
@@ -2288,7 +2288,7 @@ static bool isGEPKnownNonNull(const GEPOperator *GEP, unsigned Depth,
     if (Depth++ >= MaxAnalysisRecursionDepth)
       continue;
 
-    if (isKnownNonZero(GTI.getOperand(), Depth, Q))
+    if (isKnownNonZero(GTI.getOperand(), Q, Depth))
       return true;
   }
 
@@ -2441,8 +2441,8 @@ static bool isNonZeroAdd(const APInt &DemandedElts, unsigned Depth,
                          const SimplifyQuery &Q, unsigned BitWidth, Value *X,
                          Value *Y, bool NSW, bool NUW) {
   if (NUW)
-    return isKnownNonZero(Y, DemandedElts, Depth, Q) ||
-           isKnownNonZero(X, DemandedElts, Depth, Q);
+    return isKnownNonZero(Y, DemandedElts, Q, Depth) ||
+           isKnownNonZero(X, DemandedElts, Q, Depth);
 
   KnownBits XKnown = computeKnownBits(X, DemandedElts, Depth, Q);
   KnownBits YKnown = computeKnownBits(Y, DemandedElts, Depth, Q);
@@ -2450,8 +2450,8 @@ static bool isNonZeroAdd(const APInt &DemandedElts, unsigned Depth,
   // If X and Y are both non-negative (as signed values) then their sum is not
   // zero unless both X and Y are zero.
   if (XKnown.isNonNegative() && YKnown.isNonNegative())
-    if (isKnownNonZero(Y, DemandedElts, Depth, Q) ||
-        isKnownNonZero(X, DemandedElts, Depth, Q))
+    if (isKnownNonZero(Y, DemandedElts, Q, Depth) ||
+        isKnownNonZero(X, DemandedElts, Q, Depth))
       return true;
 
   // If X and Y are both negative (as signed values) then their sum is not
@@ -2485,7 +2485,7 @@ static bool isNonZeroSub(const APInt &DemandedElts, unsigned Depth,
                          Value *Y) {
   // TODO: Move this case into isKnownNonEqual().
   if (auto *C = dyn_cast<Constant>(X))
-    if (C->isNullValue() && isKnownNonZero(Y, DemandedElts, Depth, Q))
+    if (C->isNullValue() && isKnownNonZero(Y, DemandedElts, Q, Depth))
       return true;
 
   return ::isKnownNonEqual(X, Y, Depth, Q);
@@ -2497,18 +2497,18 @@ static bool isNonZeroMul(const APInt &DemandedElts, unsigned Depth,
   // If X and Y are non-zero then so is X * Y as long as the multiplication
   // does not overflow.
   if (NSW || NUW)
-    return isKnownNonZero(X, DemandedElts, Depth, Q) &&
-           isKnownNonZero(Y, DemandedElts, Depth, Q);
+    return isKnownNonZero(X, DemandedElts, Q, Depth) &&
+           isKnownNonZero(Y, DemandedElts, Q, Depth);
 
   // If either X or Y is odd, then if the other is non-zero the result can't
   // be zero.
   KnownBits XKnown = computeKnownBits(X, DemandedElts, Depth, Q);
   if (XKnown.One[0])
-    return isKnownNonZero(Y, DemandedElts, Depth, Q);
+    return isKnownNonZero(Y, DemandedElts, Q, Depth);
 
   KnownBits YKnown = computeKnownBits(Y, DemandedElts, Depth, Q);
   if (YKnown.One[0])
-    return XKnown.isNonZero() || isKnownNonZero(X, DemandedElts, Depth, Q);
+    return XKnown.isNonZero() || isKnownNonZero(X, DemandedElts, Q, Depth);
 
   // If there exists any subset of X (sX) and subset of Y (sY) s.t sX * sY is
   // non-zero, then X * Y is non-zero. We can find sX and sY by just taking
@@ -2564,7 +2564,7 @@ static bool isNonZeroShift(const Operator *I, const APInt &DemandedElts,
   // non-zero then at least one non-zero bit must remain.
   if (InvShiftOp(KnownVal.Zero, NumBits - MaxShift)
           .eq(InvShiftOp(APInt::getAllOnes(NumBits), NumBits - MaxShift)) &&
-      isKnownNonZero(I->getOperand(0), DemandedElts, Depth, Q))
+      isKnownNonZero(I->getOperand(0), DemandedElts, Q, Depth))
     return true;
 
   return false;
@@ -2613,7 +2613,7 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
     Type *FromTy = I->getOperand(0)->getType();
     if ((FromTy->isIntOrIntVectorTy() || FromTy->isPtrOrPtrVectorTy()) &&
         (BitWidth % getBitWidth(FromTy->getScalarType(), Q.DL)) == 0)
-      return isKnownNonZero(I->getOperand(0), Depth, Q);
+      return isKnownNonZero(I->getOperand(0), Q, Depth);
   } break;
   case Instruction::IntToPtr:
     // Note that we have to take special care to avoid looking through
@@ -2622,7 +2622,7 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
     if (!isa<ScalableVectorType>(I->getType()) &&
         Q.DL.getTypeSizeInBits(I->getOperand(0)->getType()).getFixedValue() <=
             Q.DL.getTypeSizeInBits(I->getType()).getFixedValue())
-      return isKnownNonZero(I->getOperand(0), Depth, Q);
+      return isKnownNonZero(I->getOperand(0), Q, Depth);
     break;
   case Instruction::PtrToInt:
     // Similar to int2ptr above, we can look through ptr2int here if the cast
@@ -2630,25 +2630,25 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
     if (!isa<ScalableVectorType>(I->getType()) &&
         Q.DL.getTypeSizeInBits(I->getOperand(0)->getType()).getFixedValue() <=
             Q.DL.getTypeSizeInBits(I->getType()).getFixedValue())
-      return isKnownNonZero(I->getOperand(0), Depth, Q);
+      return isKnownNonZero(I->getOperand(0), Q, Depth);
     break;
   case Instruction::Sub:
     return isNonZeroSub(DemandedElts, Depth, Q, BitWidth, I->getOperand(0),
                         I->getOperand(1));
   case Instruction::Or:
     // X | Y != 0 if X != 0 or Y != 0.
-    return isKnownNonZero(I->getOperand(1), DemandedElts, Depth, Q) ||
-           isKnownNonZero(I->getOperand(0), DemandedElts, Depth, Q);
+    return isKnownNonZero(I->getOperand(1), DemandedElts, Q, Depth) ||
+           isKnownNonZero(I->getOperand(0), DemandedElts, Q, Depth);
   case Instruction::SExt:
   case Instruction::ZExt:
     // ext X != 0 if X != 0.
-    return isKnownNonZero(I->getOperand(0), Depth, Q);
+    return isKnownNonZero(I->getOperand(0), Q, Depth);
 
   case Instruction::Shl: {
     // shl nsw/nuw can't remove any non-zero bits.
     const OverflowingBinaryOperator *BO = cast<OverflowingBinaryOperator>(I);
     if (Q.IIQ.hasNoUnsignedWrap(BO) || Q.IIQ.hasNoSignedWrap(BO))
-      return isKnownNonZero(I->getOperand(0), Depth, Q);
+      return isKnownNonZero(I->getOperand(0), Q, Depth);
 
     // shl X, Y != 0 if X is odd.  Note that the value of the shift is undefined
     // if the lowest bit is shifted off the end.
@@ -2664,7 +2664,7 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
     // shr exact can only shift out zero bits.
     const PossiblyExactOperator *BO = cast<PossiblyExactOperator>(I);
     if (BO->isExact())
-      return isKnownNonZero(I->getOperand(0), Depth, Q);
+      return isKnownNonZero(I->getOperand(0), Q, Depth);
 
     // shr X, Y != 0 if X is negative.  Note that the value of the shift is not
     // defined if the sign bit is shifted off the end.
@@ -2680,7 +2680,7 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
     // X / Y
     // div exact can only produce a zero if the dividend is zero.
     if (cast<PossiblyExactOperator>(I)->isExact())
-      return isKnownNonZero(I->getOperand(0), DemandedElts, Depth, Q);
+      return isKnownNonZero(I->getOperand(0), DemandedElts, Q, Depth);
 
     std::optional<bool> XUgeY;
     KnownBits XKnown =
@@ -2730,7 +2730,7 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
       Value *Op;
       Op = IsTrueArm ? I->getOperand(1) : I->getOperand(2);
       // Op is trivially non-zero.
-      if (isKnownNonZero(Op, DemandedElts, Depth, Q))
+      if (isKnownNonZero(Op, DemandedElts, Q, Depth))
         return true;
 
       // The condition of the select dominates the true/false arm. Check if the
@@ -2780,7 +2780,7 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
         }
       }
       // Finally recurse on the edge and check it directly.
-      return isKnownNonZero(U.get(), DemandedElts, NewDepth, RecQ);
+      return isKnownNonZero(U.get(), DemandedElts, RecQ, NewDepth);
     });
   }
   case Instruction::InsertElement: {
@@ -2802,9 +2802,9 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
 
     // Result is zero if Elt is non-zero and rest of the demanded elts in Vec
     // are non-zero.
-    return (SkipElt || isKnownNonZero(Elt, Depth, Q)) &&
+    return (SkipElt || isKnownNonZero(Elt, Q, Depth)) &&
            (DemandedVecElts.isZero() ||
-            isKnownNonZero(Vec, DemandedVecElts, Depth, Q));
+            isKnownNonZero(Vec, DemandedVecElts, Q, Depth));
   }
   case Instruction::ExtractElement:
     if (const auto *EEI = dyn_cast<ExtractElementInst>(I)) {
@@ -2816,7 +2816,7 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
         APInt DemandedVecElts = APInt::getAllOnes(NumElts);
         if (CIdx && CIdx->getValue().ult(NumElts))
           DemandedVecElts = APInt::getOneBitSet(NumElts, CIdx->getZExtValue());
-        return isKnownNonZero(Vec, DemandedVecElts, Depth, Q);
+        return isKnownNonZero(Vec, DemandedVecElts, Q, Depth);
       }
     }
     break;
@@ -2831,12 +2831,12 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
       break;
     // If demanded elements for both vecs are non-zero, the shuffle is non-zero.
     return (DemandedRHS.isZero() ||
-            isKnownNonZero(Shuf->getOperand(1), DemandedRHS, Depth, Q)) &&
+            isKnownNonZero(Shuf->getOperand(1), DemandedRHS, Q, Depth)) &&
            (DemandedLHS.isZero() ||
-            isKnownNonZero(Shuf->getOperand(0), DemandedLHS, Depth, Q));
+            isKnownNonZero(Shuf->getOperand(0), DemandedLHS, Q, Depth));
   }
   case Instruction::Freeze:
-    return isKnownNonZero(I->getOperand(0), Depth, Q) &&
+    return isKnownNonZero(I->getOperand(0), Q, Depth) &&
            isGuaranteedNotToBePoison(I->getOperand(0), Q.AC, Q.CxtI, Q.DT,
                                      Depth);
   case Instruction::Load: {
@@ -2886,7 +2886,7 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
       if (Call->isReturnNonNull())
         return true;
       if (const auto *RP = getArgumentAliasingToReturnedPointer(Call, true))
-        return isKnownNonZero(RP, Depth, Q);
+        return isKnownNonZero(RP, Q, Depth);
     } else {
       if (MDNode *Ranges = Q.IIQ.getMetadata(Call, LLVMContext::MD_range))
         return rangeMetadataExcludesValue(Ranges, APInt::getZero(BitWidth));
@@ -2896,7 +2896,7 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
           return true;
       }
       if (const Value *RV = Call->getReturnedArgOperand())
-        if (RV->getType() == I->getType() && isKnownNonZero(RV, Depth, Q))
+        if (RV->getType() == I->getType() && isKnownNonZero(RV, Q, Depth))
           return true;
     }
 
@@ -2908,7 +2908,7 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
       case Intrinsic::bitreverse:
       case Intrinsic::bswap:
       case Intrinsic::ctpop:
-        return isKnownNonZero(II->getArgOperand(0), DemandedElts, Depth, Q);
+        return isKnownNonZero(II->getArgOperand(0), DemandedElts, Q, Depth);
         // NB: We don't do usub_sat here as in any case we can prove its
         // non-zero, we will fold it to `sub nuw` in InstCombine.
       case Intrinsic::ssub_sat:
@@ -2924,11 +2924,11 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
       case Intrinsic::vector_reduce_umin:
       case Intrinsic::vector_reduce_smax:
       case Intrinsic::vector_reduce_smin:
-        return isKnownNonZero(II->getArgOperand(0), Depth, Q);
+        return isKnownNonZero(II->getArgOperand(0), Q, Depth);
       case Intrinsic::umax:
       case Intrinsic::uadd_sat:
-        return isKnownNonZero(II->getArgOperand(1), DemandedElts, Depth, Q) ||
-               isKnownNonZero(II->getArgOperand(0), DemandedElts, Depth, Q);
+        return isKnownNonZero(II->getArgOperand(1), DemandedElts, Q, Depth) ||
+               isKnownNonZero(II->getArgOperand(0), DemandedElts, Q, Depth);
       case Intrinsic::smax: {
         // If either arg is strictly positive the result is non-zero. Otherwise
         // the result is non-zero if both ops are non-zero.
@@ -2936,7 +2936,7 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
                              const KnownBits &OpKnown) {
           if (!OpNonZero.has_value())
             OpNonZero = OpKnown.isNonZero() ||
-                        isKnownNonZero(Op, DemandedElts, Depth, Q);
+                        isKnownNonZero(Op, DemandedElts, Q, Depth);
           return *OpNonZero;
         };
         // Avoid re-computing isKnownNonZero.
@@ -2971,8 +2971,8 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
       }
         [[fallthrough]];
       case Intrinsic::umin:
-        return isKnownNonZero(II->getArgOperand(0), DemandedElts, Depth, Q) &&
-               isKnownNonZero(II->getArgOperand(1), DemandedElts, Depth, Q);
+        return isKnownNonZero(II->getArgOperand(0), DemandedElts, Q, Depth) &&
+               isKnownNonZero(II->getArgOperand(1), DemandedElts, Q, Depth);
       case Intrinsic::cttz:
         return computeKnownBits(II->getArgOperand(0), DemandedElts, Depth, Q)
             .Zero[0];
@@ -2983,12 +2983,12 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
       case Intrinsic::fshl:
         // If Op0 == Op1, this is a rotate. rotate(x, y) != 0 iff x != 0.
         if (II->getArgOperand(0) == II->getArgOperand(1))
-          return isKnownNonZero(II->getArgOperand(0), DemandedElts, Depth, Q);
+          return isKnownNonZero(II->getArgOperand(0), DemandedElts, Q, Depth);
         break;
       case Intrinsic::vscale:
         return true;
       case Intrinsic::experimental_get_vector_length:
-        return isKnownNonZero(I->getOperand(0), Depth, Q);
+        return isKnownNonZero(I->getOperand(0), Q, Depth);
       default:
         break;
       }
@@ -3010,8 +3010,8 @@ static bool isKnownNonZeroFromOperator(const Operator *I,
 /// specified, perform context-sensitive analysis and return true if the
 /// pointer couldn't possibly be null at the specified instruction.
 /// Supports values with integer or pointer type and vectors of integers.
-bool isKnownNonZero(const Value *V, const APInt &DemandedElts, unsigned Depth,
-                    const SimplifyQuery &Q) {
+bool isKnownNonZero(const Value *V, const APInt &DemandedElts,
+                    const SimplifyQuery &Q, unsigned Depth) {
   Type *Ty = V->getType();
 
 #ifndef NDEBUG
@@ -3101,12 +3101,12 @@ bool isKnownNonZero(const Value *V, const APInt &DemandedElts, unsigned Depth,
   return false;
 }
 
-bool llvm::isKnownNonZero(const Value *V, unsigned Depth,
-                          const SimplifyQuery &Q) {
+bool llvm::isKnownNonZero(const Value *V, const SimplifyQuery &Q,
+                          unsigned Depth) {
   auto *FVTy = dyn_cast<FixedVectorType>(V->getType());
   APInt DemandedElts =
       FVTy ? APInt::getAllOnes(FVTy->getNumElements()) : APInt(1, 1);
-  return ::isKnownNonZero(V, DemandedElts, Depth, Q);
+  return ::isKnownNonZero(V, DemandedElts, Q, Depth);
 }
 
 /// If the pair of operators are the same invertible function, return the
@@ -3253,7 +3253,7 @@ static bool isModifyingBinopOfNonZero(const Value *V1, const Value *V2,
       Op = BO->getOperand(0);
     else
       return false;
-    return isKnownNonZero(Op, Depth + 1, Q);
+    return isKnownNonZero(Op, Q, Depth + 1);
   }
   return false;
 }
@@ -3266,7 +3266,7 @@ static bool isNonEqualMul(const Value *V1, const Value *V2, unsigned Depth,
     const APInt *C;
     return match(OBO, m_Mul(m_Specific(V1), m_APInt(C))) &&
            (OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap()) &&
-           !C->isZero() && !C->isOne() && isKnownNonZero(V1, Depth + 1, Q);
+           !C->isZero() && !C->isOne() && isKnownNonZero(V1, Q, Depth + 1);
   }
   return false;
 }
@@ -3279,7 +3279,7 @@ static bool isNonEqualShl(const Value *V1, const Value *V2, unsigned Depth,
     const APInt *C;
     return match(OBO, m_Shl(m_Specific(V1), m_APInt(C))) &&
            (OBO->hasNoUnsignedWrap() || OBO->hasNoSignedWrap()) &&
-           !C->isZero() && isKnownNonZero(V1, Depth + 1, Q);
+           !C->isZero() && isKnownNonZero(V1, Q, Depth + 1);
   }
   return false;
 }
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 22dbb3198a9f..e657872c3828 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -2314,7 +2314,7 @@ static bool despeculateCountZeros(IntrinsicInst *CountZeros,
 
   // Bail if the value is never zero.
   Use &Op = CountZeros->getOperandUse(0);
-  if (isKnownNonZero(Op, /*Depth=*/0, *DL))
+  if (isKnownNonZero(Op, *DL))
     return false;
 
   // The intrinsic will be sunk behind a compare against zero and branch.
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index f27d8d64a104..41b66aafe7d3 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -2453,7 +2453,7 @@ bool AANonNull::isImpliedByIR(Attributor &A, const IRPosition &IRP,
 
   if (llvm::any_of(Worklist, [&](AA::ValueAndContext VAC) {
         return !isKnownNonZero(
-            VAC.getValue(), /*Depth=*/0,
+            VAC.getValue(),
             SimplifyQuery(A.getDataLayout(), DT, AC, VAC.getCtxI()));
       }))
     return false;
diff --git a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
index 14612b251d1a..7ebf265e17ba 100644
--- a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
+++ b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
@@ -1175,7 +1175,7 @@ static bool isReturnNonNull(Function *F, const SCCNodeSet &SCCNodes,
     Value *RetVal = FlowsToReturn[i];
 
     // If this value is locally known to be non-null, we're good
-    if (isKnownNonZero(RetVal, /*Depth=*/0, DL))
+    if (isKnownNonZero(RetVal, DL))
       continue;
 
     // Otherwise, we need to look upwards since we can't make any local
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
index 07c50d866544..c59b867b10e7 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
@@ -988,7 +988,7 @@ Instruction *InstCombinerImpl::foldAddWithConstant(BinaryOperator &Add) {
   if (C->isOne()) {
     if (match(Op0, m_ZExt(m_Add(m_Value(X), m_AllOnes())))) {
       const SimplifyQuery Q = SQ.getWithInstruction(&Add);
-      if (llvm::isKnownNonZero(X, /*Depth=*/0, Q))
+      if (llvm::isKnownNonZero(X, Q))
         return new ZExtInst(X, Ty);
     }
   }
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index 2c0c4ee46e80..d311690be64f 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -1039,9 +1039,9 @@ static Value *foldUnsignedUnderflowCheck(ICmpInst *ZeroICmp,
       match(ZeroCmpOp, m_c_Add(m_Specific(A), m_Value(B))) &&
       (ZeroICmp->hasOneUse() || UnsignedICmp->hasOneUse())) {
     auto GetKnownNonZeroAndOther = [&](Value *&NonZero, Value *&Other) {
-      if (!isKnownNonZero(NonZero, /*Depth=*/0, Q))
+      if (!isKnownNonZero(NonZero, Q))
         std::swap(NonZero, Other);
-      return isKnownNonZero(NonZero, /*Depth=*/0, Q);
+      return isKnownNonZero(NonZero, Q);
     };
 
     // Given  ZeroCmpOp = (A + B)
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index ba5db854647a..60e4be883f51 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -601,8 +601,7 @@ static Instruction *foldCttzCtlz(IntrinsicInst &II, InstCombinerImpl &IC) {
   // then change the 'ZeroIsPoison' parameter to 'true'
   // because we know the zero behavior can't affect the result.
   if (!Known.One.isZero() ||
-      isKnownNonZero(Op0, /*Depth=*/0,
-                     IC.getSimplifyQuery().getWithInstruction(&II))) {
+      isKnownNonZero(Op0, IC.getSimplifyQuery().getWithInstruction(&II))) {
     if (!match(II.getArgOperand(1), m_One()))
       return IC.replaceOperand(II, 1, IC.Builder.getTrue());
   }
@@ -2067,8 +2066,7 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
     // See if we can deduce non-null.
     if (!CI.hasRetAttr(Attribute::NonNull) &&
         (Known.isNonZero() ||
-         isKnownNonZero(II, /*Depth=*/0,
-                        getSimplifyQuery().getWithInstruction(II)))) {
+         isKnownNonZero(II, getSimplifyQuery().getWithInstruction(II)))) {
       CI.addRetAttr(Attribute::NonNull);
       Changed = true;
     }
@@ -3664,8 +3662,7 @@ Instruction *InstCombinerImpl::visitCallBase(CallBase &Call) {
   for (Value *V : Call.args()) {
     if (V->getType()->isPointerTy() &&
         !Call.paramHasAttr(ArgNo, Attribute::NonNull) &&
-        isKnownNonZero(V, /*Depth=*/0,
-                       getSimplifyQuery().getWithInstruction(&Call)))
+        isKnownNonZero(V, getSimplifyQuery().getWithInstruction(&Call)))
       ArgNos.push_back(ArgNo);
     ArgNo++;
   }
@@ -3845,7 +3842,7 @@ Instruction *InstCombinerImpl::visitCallBase(CallBase &Call) {
 
         // isKnownNonNull -> nonnull attribute
         if (!GCR.hasRetAttr(Attribute::NonNull) &&
-            isKnownNonZero(DerivedPtr, /*Depth=*/0,
+            isKnownNonZero(DerivedPtr,
                            getSimplifyQuery().getWithInstruction(&Call))) {
           GCR.addRetAttr(Attribute::NonNull);
           // We discovered new fact, re-check users.
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index ee783eed190a..de9090770174 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -1273,12 +1273,12 @@ Instruction *InstCombinerImpl::foldICmpWithZero(ICmpInst &Cmp) {
 
       // if X non-zero and NoOverflow(X * Y)
       //    (icmp eq/ne Y)
-      if (!XKnown.One.isZero() || isKnownNonZero(X, /*Depth=*/0, Q))
+      if (!XKnown.One.isZero() || isKnownNonZero(X, Q))
         return new ICmpInst(Pred, Y, Cmp.getOperand(1));
 
       // if Y non-zero and NoOverflow(X * Y)
       //    (icmp eq/ne X)
-      if (!YKnown.One.isZero() || isKnownNonZero(Y, /*Depth=*/0, Q))
+      if (!YKnown.One.isZero() || isKnownNonZero(Y, Q))
         return new ICmpInst(Pred, X, Cmp.getOperand(1));
     }
     // Note, we are skipping cases:
@@ -3087,7 +3087,7 @@ Instruction *InstCombinerImpl::foldICmpAddConstant(ICmpInst &Cmp,
   // (X + -1) <u C --> X <=u C (if X is never null)
   if (Pred == CmpInst::ICMP_ULT && C2->isAllOnes()) {
     const SimplifyQuery Q = SQ.getWithInstruction(&Cmp);
-    if (llvm::isKnownNonZero(X, /*Depth=*/0, Q))
+    if (llvm::isKnownNonZero(X, Q))
       return new ICmpInst(ICmpInst::ICMP_ULE, X, ConstantInt::get(Ty, C));
   }
 
@@ -4275,7 +4275,7 @@ static Value *foldICmpWithLowBitMaskedVal(ICmpInst::Predicate Pred, Value *Op0,
 
       // Look for: x & ~Mask pred ~Mask
       if (isMaskOrZero(X, /*Not=*/true, Q)) {
-        return !ICmpInst::isSigned(Pred) || isKnownNonZero(X, /*Depth=*/0, Q);
+        return !ICmpInst::isSigned(Pred) || isKnownNonZero(X, Q);
       }
       return false;
     }
@@ -4779,7 +4779,7 @@ static Instruction *foldICmpXorXX(ICmpInst &I, const SimplifyQuery &Q,
   // icmp (X ^ Y_NonZero) s>= X --> icmp (X ^ Y_NonZero) s> X
   // icmp (X ^ Y_NonZero) s<= X --> icmp (X ^ Y_NonZero) s< X
   CmpInst::Predicate PredOut = CmpInst::getStrictPredicate(Pred);
-  if (PredOut != Pred && isKnownNonZero(A, /*Depth=*/0, Q))
+  if (PredOut != Pred && isKnownNonZero(A, Q))
     return new ICmpInst(PredOut, Op0, Op1);
 
   return nullptr;
@@ -5062,11 +5062,11 @@ Instruction *InstCombinerImpl::foldICmpBinOp(ICmpInst &I,
     return new ICmpInst(Pred, C, D);
   // (A - B) u>=/u< A --> B u>/u<= A  iff B != 0
   if (A == Op1 && (Pred == ICmpInst::ICMP_UGE || Pred == ICmpInst::ICMP_ULT) &&
-      isKnownNonZero(B, /*Depth=*/0, Q))
+      isKnownNonZero(B, Q))
     return new ICmpInst(CmpInst::getFlippedStrictnessPredicate(Pred), B, A);
   // C u<=/u> (C - D) --> C u</u>= D  iff B != 0
   if (C == Op0 && (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_UGT) &&
-      isKnownNonZero(D, /*Depth=*/0, Q))
+      isKnownNonZero(D, Q))
     return new ICmpInst(CmpInst::getFlippedStrictnessPredicate(Pred), C, D);
 
   // icmp (A-B), (C-B) -> icmp A, C for equalities or if there is no overflow.
@@ -5108,13 +5108,13 @@ Instruction *InstCombinerImpl::foldICmpBinOp(ICmpInst &I,
         //    X * Z eq/ne Y * Z -> X eq/ne Y
         if (ZKnown.countMaxTrailingZeros() == 0)
           return new ICmpInst(Pred, X, Y);
-        NonZero = !ZKnown.One.isZero() || isKnownNonZero(Z, /*Depth=*/0, Q);
+        NonZero = !ZKnown.One.isZero() || isKnownNonZero(Z, Q);
         // if Z != 0 and nsw(X * Z) and nsw(Y * Z)
         //    X * Z eq/ne Y * Z -> X eq/ne Y
         if (NonZero && BO0 && BO1 && Op0HasNSW && Op1HasNSW)
           return new ICmpInst(Pred, X, Y);
       } else
-        NonZero = isKnownNonZero(Z, /*Depth=*/0, Q);
+        NonZero = isKnownNonZero(Z, Q);
 
       // If Z != 0 and nuw(X * Z) and nuw(Y * Z)
       //    X * Z u{lt/le/gt/ge}/eq/ne Y * Z -> X u{lt/le/gt/ge}/eq/ne Y
diff --git a/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp b/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp
index 9838e2aa9f3a..52803e9bea45 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombinePHI.cpp
@@ -1537,8 +1537,7 @@ Instruction *InstCombinerImpl::visitPHINode(PHINode &PN) {
       for (unsigned I = 0, E = PN.getNumIncomingValues(); I != E; ++I) {
         Instruction *CtxI = PN.getIncomingBlock(I)->getTerminator();
         Value *VA = PN.getIncomingValue(I);
-        if (isKnownNonZero(VA, 0,
-                           getSimplifyQuery().getWithInstruction(CtxI))) {
+        if (isKnownNonZero(VA, getSimplifyQuery().getWithInstruction(CtxI))) {
           if (!NonZeroConst)
             NonZeroConst = getAnyNonZeroConstInt(PN);
           if (NonZeroConst != VA) {
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 4c00f2a0ea17..5a144cc73789 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -1431,7 +1431,7 @@ Instruction *InstCombinerImpl::foldFBinOpOfIntCastsFromSign(
     if (OpsKnown[OpNo].hasKnownBits() &&
         OpsKnown[OpNo].getKnownBits(SQ).isNonZero())
       return true;
-    return isKnownNonZero(IntOps[OpNo], /*Depth=*/0, SQ);
+    return isKnownNonZero(IntOps[OpNo], SQ);
   };
 
   auto IsNonNeg = [&](unsigned OpNo) -> bool {
diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index a72b0ee9a08e..ee3531bbd68d 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -1281,7 +1281,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
         // ignored.
         return;
       }
-      if (llvm::isKnownNonZero(ConvertedShadow, /*Depth=*/0, DL)) {
+      if (llvm::isKnownNonZero(ConvertedShadow, DL)) {
         // Copy origin as the value is definitely uninitialized.
         paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
                     OriginAlignment);
@@ -1427,7 +1427,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
           // Skip, value is initialized or const shadow is ignored.
           continue;
         }
-        if (llvm::isKnownNonZero(ConvertedShadow, /*Depth=*/0, DL)) {
+        if (llvm::isKnownNonZero(ConvertedShadow, DL)) {
           // Report as the value is definitely uninitialized.
           insertWarningFn(IRB, ShadowData.Origin);
           if (!MS.Recover)
diff --git a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
index f376b5f7d68d..40d0f6b75d69 100644
--- a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
+++ b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
@@ -459,7 +459,7 @@ static void convertMetadataToAssumes(LoadInst *LI, Value *Val,
   // we can only do this if the value is known non-poison.
   if (AC && LI->getMetadata(LLVMContext::MD_nonnull) &&
       LI->getMetadata(LLVMContext::MD_noundef) &&
-      !isKnownNonZero(Val, /*Depth=*/0, SimplifyQuery(DL, DT, AC, LI)))
+      !isKnownNonZero(Val, SimplifyQuery(DL, DT, AC, LI)))
     addAssumeNonNull(AC, LI);
 }
 
diff --git a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
index 7e9e91606fe2..2e68a9c01898 100644
--- a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
@@ -305,7 +305,7 @@ static void annotateNonNullAndDereferenceable(CallInst *CI, ArrayRef<unsigned> A
   if (ConstantInt *LenC = dyn_cast<ConstantInt>(Size)) {
     annotateNonNullNoUndefBasedOnAccess(CI, ArgNos);
     annotateDereferenceableBytes(CI, ArgNos, LenC->getZExtValue());
-  } else if (isKnownNonZero(Size, /*Depth=*/0, DL)) {
+  } else if (isKnownNonZero(Size, DL)) {
     annotateNonNullNoUndefBasedOnAccess(CI, ArgNos);
     const APInt *X, *Y;
     uint64_t DerefMin = 1;
@@ -394,7 +394,7 @@ Value *LibCallSimplifier::optimizeStrNCat(CallInst *CI, IRBuilderBase &B) {
   Value *Size = CI->getArgOperand(2);
   uint64_t Len;
   annotateNonNullNoUndefBasedOnAccess(CI, 0);
-  if (isKnownNonZero(Size, /*Depth=*/0, DL))
+  if (isKnownNonZero(Size, DL))
     annotateNonNullNoUndefBasedOnAccess(CI, 1);
 
   // We don't do anything if length is not constant.
@@ -613,7 +613,7 @@ Value *LibCallSimplifier::optimizeStrNCmp(CallInst *CI, IRBuilderBase &B) {
   if (Str1P == Str2P) // strncmp(x,x,n)  -> 0
     return ConstantInt::get(CI->getType(), 0);
 
-  if (isKnownNonZero(Size, /*Depth=*/0, DL))
+  if (isKnownNonZero(Size, DL))
     annotateNonNullNoUndefBasedOnAccess(CI, {0, 1});
   // Get the length argument if it is constant.
   uint64_t Length;
@@ -749,7 +749,7 @@ Value *LibCallSimplifier::optimizeStpCpy(CallInst *CI, IRBuilderBase &B) {
 
 Value *LibCallSimplifier::optimizeStrLCpy(CallInst *CI, IRBuilderBase &B) {
   Value *Size = CI->getArgOperand(2);
-  if (isKnownNonZero(Size, /*Depth=*/0, DL))
+  if (isKnownNonZero(Size, DL))
     // Like snprintf, the function stores into the destination only when
     // the size argument is nonzero.
     annotateNonNullNoUndefBasedOnAccess(CI, 0);
@@ -833,7 +833,7 @@ Value *LibCallSimplifier::optimizeStringNCpy(CallInst *CI, bool RetEnd,
   Value *Src = CI->getArgOperand(1);
   Value *Size = CI->getArgOperand(2);
 
-  if (isKnownNonZero(Size, /*Depth=*/0, DL)) {
+  if (isKnownNonZero(Size, DL)) {
     // Both st{p,r}ncpy(D, S, N) access the source and destination arrays
     // only when N is nonzero.
     annotateNonNullNoUndefBasedOnAccess(CI, 0);
@@ -926,7 +926,7 @@ Value *LibCallSimplifier::optimizeStringLength(CallInst *CI, IRBuilderBase &B,
   Type *CharTy = B.getIntNTy(CharSize);
 
   if (isOnlyUsedInZeroEqualityComparison(CI) &&
-      (!Bound || isKnownNonZero(Bound, /*Depth=*/0, DL))) {
+      (!Bound || isKnownNonZero(Bound, DL))) {
     // Fold strlen:
     //   strlen(x) != 0 --> *x != 0
     //   strlen(x) == 0 --> *x == 0
@@ -1047,7 +1047,7 @@ Value *LibCallSimplifier::optimizeStrNLen(CallInst *CI, IRBuilderBase &B) {
   if (Value *V = optimizeStringLength(CI, B, 8, Bound))
     return V;
 
-  if (isKnownNonZero(Bound, /*Depth=*/0, DL))
+  if (isKnownNonZero(Bound, DL))
     annotateNonNullNoUndefBasedOnAccess(CI, 0);
   return nullptr;
 }
@@ -1291,7 +1291,7 @@ Value *LibCallSimplifier::optimizeMemChr(CallInst *CI, IRBuilderBase &B) {
   Value *SrcStr = CI->getArgOperand(0);
   Value *Size = CI->getArgOperand(2);
 
-  if (isKnownNonZero(Size, /*Depth=*/0, DL)) {
+  if (isKnownNonZero(Size, DL)) {
     annotateNonNullNoUndefBasedOnAccess(CI, 0);
     if (isOnlyUsedInEqualityComparison(CI, SrcStr))
       return memChrToCharCompare(CI, Size, B, DL);
@@ -2976,7 +2976,7 @@ Value *LibCallSimplifier::optimizeStrToInt(CallInst *CI, IRBuilderBase &B,
     // It would be readonly too, except that it still may write to errno.
     CI->addParamAttr(0, Attribute::NoCapture);
     EndPtr = nullptr;
-  } else if (!isKnownNonZero(EndPtr, /*Depth=*/0, DL))
+  } else if (!isKnownNonZero(EndPtr, DL))
     return nullptr;
 
   StringRef Str;
@@ -3402,7 +3402,7 @@ Value *LibCallSimplifier::optimizeSnPrintF(CallInst *CI, IRBuilderBase &B) {
     return V;
   }
 
-  if (isKnownNonZero(CI->getOperand(1), /*Depth=*/0, DL))
+  if (isKnownNonZero(CI->getOperand(1), DL))
     annotateNonNullNoUndefBasedOnAccess(CI, 0);
   return nullptr;
 }
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index e0e2f50c89ad..4918cee1fa82 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -886,7 +886,7 @@ bool VectorCombine::scalarizeVPIntrinsic(Instruction &I) {
     SafeToSpeculate = isSafeToSpeculativelyExecuteWithOpcode(
         *FunctionalOpcode, &VPI, nullptr, &AC, &DT);
   if (!SafeToSpeculate &&
-      !isKnownNonZero(EVL, /*Depth=*/0, SimplifyQuery(*DL, &DT, &AC, &VPI)))
+      !isKnownNonZero(EVL, SimplifyQuery(*DL, &DT, &AC, &VPI)))
     return false;
 
   Value *ScalarVal =
diff --git a/llvm/unittests/Analysis/ValueTrackingTest.cpp b/llvm/unittests/Analysis/ValueTrackingTest.cpp
index 8ebd9b511f39..8738af91b652 100644
--- a/llvm/unittests/Analysis/ValueTrackingTest.cpp
+++ b/llvm/unittests/Analysis/ValueTrackingTest.cpp
@@ -2110,8 +2110,7 @@ TEST_F(ValueTrackingTest, isNonZeroRecurrence) {
   )");
   const DataLayout &DL = M->getDataLayout();
   AssumptionCache AC(*F);
-  EXPECT_TRUE(isKnownNonZero(A, /*Depth=*/0,
-                             SimplifyQuery(DL, /*DT=*/nullptr, &AC, CxtI)));
+  EXPECT_TRUE(isKnownNonZero(A, SimplifyQuery(DL, /*DT=*/nullptr, &AC, CxtI)));
 }
 
 TEST_F(ValueTrackingTest, KnownNonZeroFromDomCond) {
@@ -2135,9 +2134,8 @@ TEST_F(ValueTrackingTest, KnownNonZeroFromDomCond) {
   DominatorTree DT(*F);
   const DataLayout &DL = M->getDataLayout();
   const SimplifyQuery SQ(DL, &DT, &AC);
-  EXPECT_EQ(isKnownNonZero(A, /*Depth=*/0, SQ.getWithInstruction(CxtI)), true);
-  EXPECT_EQ(isKnownNonZero(A, /*Depth=*/0, SQ.getWithInstruction(CxtI2)),
-            false);
+  EXPECT_EQ(isKnownNonZero(A, SQ.getWithInstruction(CxtI)), true);
+  EXPECT_EQ(isKnownNonZero(A, SQ.getWithInstruction(CxtI2)), false);
 }
 
 TEST_F(ValueTrackingTest, KnownNonZeroFromDomCond2) {
@@ -2161,9 +2159,8 @@ TEST_F(ValueTrackingTest, KnownNonZeroFromDomCond2) {
   DominatorTree DT(*F);
   const DataLayout &DL = M->getDataLayout();
   const SimplifyQuery SQ(DL, &DT, &AC);
-  EXPECT_EQ(isKnownNonZero(A, /*Depth=*/0, SQ.getWithInstruction(CxtI)), true);
-  EXPECT_EQ(isKnownNonZero(A, /*Depth=*/0, SQ.getWithInstruction(CxtI2)),
-            false);
+  EXPECT_EQ(isKnownNonZero(A, SQ.getWithInstruction(CxtI)), true);
+  EXPECT_EQ(isKnownNonZero(A, SQ.getWithInstruction(CxtI2)), false);
 }
 
 TEST_F(ValueTrackingTest, IsImpliedConditionAnd) {
-- 
GitLab


From 5a34ff12b8f4a73f5dcd4be1b2575dc38cf13bee Mon Sep 17 00:00:00 2001
From: Jeremy Kun <jkun@google.com>
Date: Tue, 16 Apr 2024 07:35:36 -0700
Subject: [PATCH 411/464] fix Polynomial.td doc filename (#88900)

Not sure how best to test this, but I think it fixes the error
https://github.com/llvm/mlir-www/actions/runs/8699908058/job/23859264085#step:7:1111

Co-authored-by: Jeremy Kun <j2kun@users.noreply.github.com>
Co-authored-by: Jacques Pienaar <jpienaar@google.com>
---
 mlir/include/mlir/Dialect/Polynomial/IR/CMakeLists.txt | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Polynomial/IR/CMakeLists.txt b/mlir/include/mlir/Dialect/Polynomial/IR/CMakeLists.txt
index d8039deb5ee2..dd0384d8b79d 100644
--- a/mlir/include/mlir/Dialect/Polynomial/IR/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/Polynomial/IR/CMakeLists.txt
@@ -1,8 +1,8 @@
 add_mlir_dialect(Polynomial polynomial)
-add_mlir_doc(PolynomialDialect PolynomialDialect Polynomial/ -gen-dialect-doc)
-add_mlir_doc(PolynomialOps PolynomialOps Polynomial/ -gen-op-doc)
-add_mlir_doc(PolynomialAttributes PolynomialAttributes Dialects/ -gen-attrdef-doc)
-add_mlir_doc(PolynomialTypes PolynomialTypes Dialects/ -gen-typedef-doc)
+add_mlir_doc(Polynomial PolynomialDialect Polynomial/ -gen-dialect-doc)
+add_mlir_doc(Polynomial PolynomialOps Polynomial/ -gen-op-doc)
+add_mlir_doc(Polynomial PolynomialAttributes Dialects/ -gen-attrdef-doc)
+add_mlir_doc(Polynomial PolynomialTypes Dialects/ -gen-typedef-doc)
 
 set(LLVM_TARGET_DEFINITIONS Polynomial.td)
 mlir_tablegen(PolynomialAttributes.cpp.inc -gen-attrdef-defs -attrdefs-dialect=polynomial)
-- 
GitLab


From b63247627c9e87e898dec5bf0bea255b3f0eec5c Mon Sep 17 00:00:00 2001
From: Ding Fei <fding@feysh.com>
Date: Tue, 16 Apr 2024 22:38:27 +0800
Subject: [PATCH 412/464] [AST][RecoveryExpr] Fix a crash on c89/c90 invalid
 InitListExpr (#88008) (#88014)

Use refactored `CheckForConstantInitializer()` to skip checking expr
with error.

---------

Co-authored-by: Aaron Ballman <aaron@aaronballman.com>
---
 clang/docs/ReleaseNotes.rst                   |  2 ++
 clang/include/clang/Sema/Sema.h               |  4 ++-
 clang/lib/Sema/SemaDecl.cpp                   | 28 ++++++++-----------
 clang/lib/Sema/SemaExpr.cpp                   |  2 +-
 .../test/Sema/recover-expr-gh88008-nocrash.c  | 11 ++++++++
 5 files changed, 28 insertions(+), 19 deletions(-)
 create mode 100644 clang/test/Sema/recover-expr-gh88008-nocrash.c

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index db90db6fa4ab..d8ec8bcb8df5 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -417,6 +417,8 @@ Bug Fixes in This Version
 - Fixed a regression in CTAD that a friend declaration that befriends itself may cause
   incorrect constraint substitution. (#GH86769).
 
+- Fixed an assertion failure on invalid InitListExpr in C89 mode (#GH88008).
+
 Bug Fixes to Compiler Builtins
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index a5fe83a539aa..77150a318ee4 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -55,6 +55,7 @@
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/SemaBase.h"
 #include "clang/Sema/SemaConcept.h"
+#include "clang/Sema/SemaDiagnostic.h"
 #include "clang/Sema/TypoCorrection.h"
 #include "clang/Sema/Weak.h"
 #include "llvm/ADT/ArrayRef.h"
@@ -3427,7 +3428,8 @@ public:
       bool ConstexprSupported, bool CLinkageMayDiffer);
 
   /// type checking declaration initializers (C99 6.7.8)
-  bool CheckForConstantInitializer(Expr *e, QualType t);
+  bool CheckForConstantInitializer(
+      Expr *Init, unsigned DiagID = diag::err_init_element_not_constant);
 
   QualType deduceVarTypeFromInitializer(VarDecl *VDecl, DeclarationName Name,
                                         QualType Type, TypeSourceInfo *TSI,
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 390da508518e..745cf41e204e 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -12671,7 +12671,7 @@ void Sema::CheckMSVCRTEntryPoint(FunctionDecl *FD) {
   }
 }
 
-bool Sema::CheckForConstantInitializer(Expr *Init, QualType DclT) {
+bool Sema::CheckForConstantInitializer(Expr *Init, unsigned DiagID) {
   // FIXME: Need strict checking.  In C89, we need to check for
   // any assignment, increment, decrement, function-calls, or
   // commas outside of a sizeof.  In C99, it's the same list,
@@ -12689,8 +12689,7 @@ bool Sema::CheckForConstantInitializer(Expr *Init, QualType DclT) {
   const Expr *Culprit;
   if (Init->isConstantInitializer(Context, false, &Culprit))
     return false;
-  Diag(Culprit->getExprLoc(), diag::err_init_element_not_constant)
-    << Culprit->getSourceRange();
+  Diag(Culprit->getExprLoc(), DiagID) << Culprit->getSourceRange();
   return true;
 }
 
@@ -13808,29 +13807,24 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
     // OpenCL v1.2 s6.5.3: __constant locals must be constant-initialized.
     // This is true even in C++ for OpenCL.
     } else if (VDecl->getType().getAddressSpace() == LangAS::opencl_constant) {
-      CheckForConstantInitializer(Init, DclT);
+      CheckForConstantInitializer(Init);
 
-    // Otherwise, C++ does not restrict the initializer.
+      // Otherwise, C++ does not restrict the initializer.
     } else if (getLangOpts().CPlusPlus) {
       // do nothing
 
     // C99 6.7.8p4: All the expressions in an initializer for an object that has
     // static storage duration shall be constant expressions or string literals.
     } else if (VDecl->getStorageClass() == SC_Static) {
-      CheckForConstantInitializer(Init, DclT);
+      CheckForConstantInitializer(Init);
 
-    // C89 is stricter than C99 for aggregate initializers.
-    // C89 6.5.7p3: All the expressions [...] in an initializer list
-    // for an object that has aggregate or union type shall be
-    // constant expressions.
+      // C89 is stricter than C99 for aggregate initializers.
+      // C89 6.5.7p3: All the expressions [...] in an initializer list
+      // for an object that has aggregate or union type shall be
+      // constant expressions.
     } else if (!getLangOpts().C99 && VDecl->getType()->isAggregateType() &&
                isa<InitListExpr>(Init)) {
-      const Expr *Culprit;
-      if (!Init->isConstantInitializer(Context, false, &Culprit)) {
-        Diag(Culprit->getExprLoc(),
-             diag::ext_aggregate_init_not_constant)
-          << Culprit->getSourceRange();
-      }
+      CheckForConstantInitializer(Init, diag::ext_aggregate_init_not_constant);
     }
 
     if (auto *E = dyn_cast<ExprWithCleanups>(Init))
@@ -13963,7 +13957,7 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
     // Avoid duplicate diagnostics for constexpr variables.
     if (!getLangOpts().CPlusPlus && !VDecl->isInvalidDecl() &&
         !VDecl->isConstexpr())
-      CheckForConstantInitializer(Init, DclT);
+      CheckForConstantInitializer(Init);
   }
 
   QualType InitType = Init->getType();
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index cabffa47c931..7c3faba0f788 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -7331,7 +7331,7 @@ Sema::BuildCompoundLiteralExpr(SourceLocation LParenLoc, TypeSourceInfo *TInfo,
     if (!LiteralExpr->isTypeDependent() &&
         !LiteralExpr->isValueDependent() &&
         !literalType->isDependentType()) // C99 6.5.2.5p3
-      if (CheckForConstantInitializer(LiteralExpr, literalType))
+      if (CheckForConstantInitializer(LiteralExpr))
         return ExprError();
   } else if (literalType.getAddressSpace() != LangAS::opencl_private &&
              literalType.getAddressSpace() != LangAS::Default) {
diff --git a/clang/test/Sema/recover-expr-gh88008-nocrash.c b/clang/test/Sema/recover-expr-gh88008-nocrash.c
new file mode 100644
index 000000000000..5500b33dd0e8
--- /dev/null
+++ b/clang/test/Sema/recover-expr-gh88008-nocrash.c
@@ -0,0 +1,11 @@
+// RUN: %clang_cc1 %s -verify -fsyntax-only -std=c90
+
+struct S {
+  int v;
+};
+
+struct T; // expected-note {{forward declaration of 'struct T'}}
+
+void gh88008_nocrash(struct T *t) {
+  struct S s = { .v = t->y }; // expected-error {{incomplete definition of type 'struct T'}}
+}
-- 
GitLab


From d2d4a1bbdc455a30d600743eb59fb1c69205967a Mon Sep 17 00:00:00 2001
From: XChy <xxs_chy@outlook.com>
Date: Tue, 16 Apr 2024 22:52:19 +0800
Subject: [PATCH 413/464] Revert "[JumpThreading] Thread over BB with only an
 unconditional branch" (#88907)

Reverts llvm/llvm-project#86312
---
 llvm/lib/Transforms/Utils/Local.cpp           |  10 +-
 llvm/test/CodeGen/AArch64/and-sink.ll         |   9 +-
 .../AArch64/combine-comparisons-by-cse.ll     | 122 ++++++++++-------
 llvm/test/CodeGen/Hexagon/vect/zext-v4i1.ll   |  18 ++-
 llvm/test/Transforms/JumpThreading/pr79175.ll |   8 +-
 llvm/test/Transforms/JumpThreading/select.ll  |  50 ++++---
 .../Transforms/JumpThreading/thread-prob-7.ll |   8 +-
 .../Transforms/JumpThreading/uncond-no-phi.ll | 123 ------------------
 .../PhaseOrdering/thread-uncond-bb.ll         |  62 ---------
 9 files changed, 126 insertions(+), 284 deletions(-)
 delete mode 100644 llvm/test/Transforms/JumpThreading/uncond-no-phi.ll
 delete mode 100644 llvm/test/Transforms/PhaseOrdering/thread-uncond-bb.ll

diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp
index baec51a07fcb..a42ef0c4e6ae 100644
--- a/llvm/lib/Transforms/Utils/Local.cpp
+++ b/llvm/lib/Transforms/Utils/Local.cpp
@@ -1019,14 +1019,12 @@ CanRedirectPredsOfEmptyBBToSucc(BasicBlock *BB, BasicBlock *Succ,
                                 const SmallPtrSetImpl<BasicBlock *> &SuccPreds,
                                 BasicBlock *&CommonPred) {
 
-  // When Succ has no phis, BB may be merged into Succ directly. We don't need
-  // to redirect the predecessors of BB in this case.
-  if (Succ->phis().empty())
+  // There must be phis in BB, otherwise BB will be merged into Succ directly
+  if (BB->phis().empty() || Succ->phis().empty())
     return false;
 
-  // BB must have multiple different predecessors, so that at least one of
-  // predecessors can be redirected to Succ, except the common predecessor.
-  if (BB->getUniquePredecessor() || pred_empty(BB))
+  // BB must have predecessors not shared that can be redirected to Succ
+  if (!BB->hasNPredecessorsOrMore(2))
     return false;
 
   // Get single common predecessors of both BB and Succ
diff --git a/llvm/test/CodeGen/AArch64/and-sink.ll b/llvm/test/CodeGen/AArch64/and-sink.ll
index a57e9d54f307..f298a55dab72 100644
--- a/llvm/test/CodeGen/AArch64/and-sink.ll
+++ b/llvm/test/CodeGen/AArch64/and-sink.ll
@@ -11,14 +11,15 @@
 define dso_local i32 @and_sink1(i32 %a, i1 %c) {
 ; CHECK-LABEL: and_sink1:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    tbz w1, #0, .LBB0_2
+; CHECK-NEXT:    tbz w1, #0, .LBB0_3
 ; CHECK-NEXT:  // %bb.1: // %bb0
-; CHECK-NEXT:    tst w0, #0x4
 ; CHECK-NEXT:    adrp x8, A
-; CHECK-NEXT:    cset w0, eq
 ; CHECK-NEXT:    str wzr, [x8, :lo12:A]
+; CHECK-NEXT:    tbnz w0, #2, .LBB0_3
+; CHECK-NEXT:  // %bb.2:
+; CHECK-NEXT:    mov w0, #1 // =0x1
 ; CHECK-NEXT:    ret
-; CHECK-NEXT:  .LBB0_2:
+; CHECK-NEXT:  .LBB0_3: // %bb2
 ; CHECK-NEXT:    mov w0, wzr
 ; CHECK-NEXT:    ret
 
diff --git a/llvm/test/CodeGen/AArch64/combine-comparisons-by-cse.ll b/llvm/test/CodeGen/AArch64/combine-comparisons-by-cse.ll
index dde3e81833a6..6449c3e11d66 100644
--- a/llvm/test/CodeGen/AArch64/combine-comparisons-by-cse.ll
+++ b/llvm/test/CodeGen/AArch64/combine-comparisons-by-cse.ll
@@ -13,10 +13,10 @@ define i32 @combine_gt_ge_10() #0 {
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    adrp x8, :got:a
 ; CHECK-NEXT:    ldr x8, [x8, :got_lo12:a]
-; CHECK-NEXT:    ldr w9, [x8]
+; CHECK-NEXT:    ldr w8, [x8]
+; CHECK-NEXT:    cmp w8, #10
 ; CHECK-NEXT:    adrp x8, :got:b
 ; CHECK-NEXT:    ldr x8, [x8, :got_lo12:b]
-; CHECK-NEXT:    cmp w9, #10
 ; CHECK-NEXT:    b.le .LBB0_3
 ; CHECK-NEXT:  // %bb.1: // %land.lhs.true
 ; CHECK-NEXT:    adrp x9, :got:c
@@ -29,17 +29,18 @@ define i32 @combine_gt_ge_10() #0 {
 ; CHECK-NEXT:    mov w0, #1 // =0x1
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB0_3: // %lor.lhs.false
-; CHECK-NEXT:    cmp w9, #10
-; CHECK-NEXT:    b.lt .LBB0_5
+; CHECK-NEXT:    b.lt .LBB0_6
 ; CHECK-NEXT:  .LBB0_4: // %land.lhs.true3
 ; CHECK-NEXT:    adrp x9, :got:d
 ; CHECK-NEXT:    ldr x9, [x9, :got_lo12:d]
 ; CHECK-NEXT:    ldr w8, [x8]
 ; CHECK-NEXT:    ldr w9, [x9]
 ; CHECK-NEXT:    cmp w8, w9
-; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    b.ne .LBB0_6
+; CHECK-NEXT:  // %bb.5:
+; CHECK-NEXT:    mov w0, #1 // =0x1
 ; CHECK-NEXT:    ret
-; CHECK-NEXT:  .LBB0_5:
+; CHECK-NEXT:  .LBB0_6: // %if.end
 ; CHECK-NEXT:    mov w0, wzr
 ; CHECK-NEXT:    ret
 entry:
@@ -144,10 +145,10 @@ define i32 @combine_lt_ge_5() #0 {
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    adrp x8, :got:a
 ; CHECK-NEXT:    ldr x8, [x8, :got_lo12:a]
-; CHECK-NEXT:    ldr w9, [x8]
+; CHECK-NEXT:    ldr w8, [x8]
+; CHECK-NEXT:    cmp w8, #5
 ; CHECK-NEXT:    adrp x8, :got:b
 ; CHECK-NEXT:    ldr x8, [x8, :got_lo12:b]
-; CHECK-NEXT:    cmp w9, #5
 ; CHECK-NEXT:    b.ge .LBB2_3
 ; CHECK-NEXT:  // %bb.1: // %land.lhs.true
 ; CHECK-NEXT:    adrp x9, :got:c
@@ -160,17 +161,18 @@ define i32 @combine_lt_ge_5() #0 {
 ; CHECK-NEXT:    mov w0, #1 // =0x1
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB2_3: // %lor.lhs.false
-; CHECK-NEXT:    cmp w9, #5
-; CHECK-NEXT:    b.gt .LBB2_5
+; CHECK-NEXT:    b.gt .LBB2_6
 ; CHECK-NEXT:  .LBB2_4: // %land.lhs.true3
 ; CHECK-NEXT:    adrp x9, :got:d
 ; CHECK-NEXT:    ldr x9, [x9, :got_lo12:d]
 ; CHECK-NEXT:    ldr w8, [x8]
 ; CHECK-NEXT:    ldr w9, [x9]
 ; CHECK-NEXT:    cmp w8, w9
-; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    b.ne .LBB2_6
+; CHECK-NEXT:  // %bb.5:
+; CHECK-NEXT:    mov w0, #1 // =0x1
 ; CHECK-NEXT:    ret
-; CHECK-NEXT:  .LBB2_5:
+; CHECK-NEXT:  .LBB2_6: // %if.end
 ; CHECK-NEXT:    mov w0, wzr
 ; CHECK-NEXT:    ret
 entry:
@@ -497,17 +499,24 @@ define i32 @do_nothing_if_resultant_opcodes_would_differ() #0 {
 ; CHECK-NEXT:  // %bb.3: // %while.cond.while.end_crit_edge
 ; CHECK-NEXT:    ldr w8, [x19]
 ; CHECK-NEXT:  .LBB7_4: // %while.end
-; CHECK-NEXT:    adrp x9, :got:b
-; CHECK-NEXT:    adrp x10, :got:d
-; CHECK-NEXT:    ldr x9, [x9, :got_lo12:b]
-; CHECK-NEXT:    ldr x10, [x10, :got_lo12:d]
-; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    cmp w8, #1
+; CHECK-NEXT:    b.gt .LBB7_7
+; CHECK-NEXT:  // %bb.5: // %land.lhs.true
+; CHECK-NEXT:    adrp x8, :got:b
+; CHECK-NEXT:    adrp x9, :got:d
+; CHECK-NEXT:    ldr x8, [x8, :got_lo12:b]
+; CHECK-NEXT:    ldr x9, [x9, :got_lo12:d]
+; CHECK-NEXT:    ldr w8, [x8]
 ; CHECK-NEXT:    ldr w9, [x9]
-; CHECK-NEXT:    ldr w10, [x10]
-; CHECK-NEXT:    cmp w9, w10
-; CHECK-NEXT:    ccmp w8, #2, #0, eq
-; CHECK-NEXT:    mov w8, #123 // =0x7b
-; CHECK-NEXT:    csel w0, w8, wzr, lt
+; CHECK-NEXT:    cmp w8, w9
+; CHECK-NEXT:    b.ne .LBB7_7
+; CHECK-NEXT:  // %bb.6:
+; CHECK-NEXT:    mov w0, #123 // =0x7b
+; CHECK-NEXT:    b .LBB7_8
+; CHECK-NEXT:  .LBB7_7: // %if.end
+; CHECK-NEXT:    mov w0, wzr
+; CHECK-NEXT:  .LBB7_8: // %return
+; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
 ; CHECK-NEXT:    ldr x30, [sp], #32 // 8-byte Folded Reload
 ; CHECK-NEXT:    .cfi_def_cfa_offset 0
 ; CHECK-NEXT:    .cfi_restore w19
@@ -555,42 +564,52 @@ return:                                           ; preds = %if.end, %land.lhs.t
 define i32 @do_nothing_if_compares_can_not_be_adjusted_to_each_other() #0 {
 ; CHECK-LABEL: do_nothing_if_compares_can_not_be_adjusted_to_each_other:
 ; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w19, -8
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    .cfi_remember_state
 ; CHECK-NEXT:    adrp x8, :got:a
 ; CHECK-NEXT:    ldr x8, [x8, :got_lo12:a]
 ; CHECK-NEXT:    ldr w8, [x8]
 ; CHECK-NEXT:    cmp w8, #0
-; CHECK-NEXT:    b.gt .LBB8_4
+; CHECK-NEXT:    b.gt .LBB8_3
 ; CHECK-NEXT:  // %bb.1: // %while.body.preheader
-; CHECK-NEXT:    stp x30, x19, [sp, #-16]! // 16-byte Folded Spill
-; CHECK-NEXT:    .cfi_def_cfa_offset 16
-; CHECK-NEXT:    .cfi_offset w19, -8
-; CHECK-NEXT:    .cfi_offset w30, -16
 ; CHECK-NEXT:    sub w19, w8, #1
 ; CHECK-NEXT:  .LBB8_2: // %while.body
 ; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    bl do_something
 ; CHECK-NEXT:    adds w19, w19, #1
 ; CHECK-NEXT:    b.mi .LBB8_2
-; CHECK-NEXT:  // %bb.3:
-; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
-; CHECK-NEXT:    .cfi_def_cfa_offset 0
-; CHECK-NEXT:    .cfi_restore w19
-; CHECK-NEXT:    .cfi_restore w30
-; CHECK-NEXT:  .LBB8_4: // %while.end
+; CHECK-NEXT:  .LBB8_3: // %while.end
+; CHECK-NEXT:    adrp x8, :got:c
+; CHECK-NEXT:    ldr x8, [x8, :got_lo12:c]
+; CHECK-NEXT:    ldr w8, [x8]
+; CHECK-NEXT:    cmn w8, #2
+; CHECK-NEXT:    b.lt .LBB8_6
+; CHECK-NEXT:  // %bb.4: // %land.lhs.true
 ; CHECK-NEXT:    adrp x8, :got:b
 ; CHECK-NEXT:    adrp x9, :got:d
-; CHECK-NEXT:    adrp x10, :got:c
 ; CHECK-NEXT:    ldr x8, [x8, :got_lo12:b]
 ; CHECK-NEXT:    ldr x9, [x9, :got_lo12:d]
-; CHECK-NEXT:    ldr x10, [x10, :got_lo12:c]
 ; CHECK-NEXT:    ldr w8, [x8]
 ; CHECK-NEXT:    ldr w9, [x9]
-; CHECK-NEXT:    ldr w10, [x10]
 ; CHECK-NEXT:    cmp w8, w9
-; CHECK-NEXT:    mov w8, #-3 // =0xfffffffd
-; CHECK-NEXT:    ccmp w10, w8, #4, eq
-; CHECK-NEXT:    mov w8, #123 // =0x7b
-; CHECK-NEXT:    csel w0, w8, wzr, gt
+; CHECK-NEXT:    b.ne .LBB8_6
+; CHECK-NEXT:  // %bb.5:
+; CHECK-NEXT:    mov w0, #123 // =0x7b
+; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB8_6: // %if.end
+; CHECK-NEXT:    .cfi_restore_state
+; CHECK-NEXT:    mov w0, wzr
+; CHECK-NEXT:    ldp x30, x19, [sp], #16 // 16-byte Folded Reload
+; CHECK-NEXT:    .cfi_def_cfa_offset 0
+; CHECK-NEXT:    .cfi_restore w19
+; CHECK-NEXT:    .cfi_restore w30
 ; CHECK-NEXT:    ret
 entry:
   %0 = load i32, ptr @a, align 4
@@ -763,14 +782,12 @@ define i32 @combine_gt_ge_sel(i64 %v, ptr %p) #0 {
 ; CHECK-NEXT:    cmp w8, #0
 ; CHECK-NEXT:    csel x9, x0, xzr, gt
 ; CHECK-NEXT:    str x9, [x1]
-; CHECK-NEXT:    b.le .LBB11_3
+; CHECK-NEXT:    b.le .LBB11_2
 ; CHECK-NEXT:  // %bb.1: // %lor.lhs.false
 ; CHECK-NEXT:    cmp w8, #2
-; CHECK-NEXT:    b.ge .LBB11_5
-; CHECK-NEXT:  // %bb.2:
-; CHECK-NEXT:    mov w0, wzr
-; CHECK-NEXT:    ret
-; CHECK-NEXT:  .LBB11_3: // %land.lhs.true
+; CHECK-NEXT:    b.ge .LBB11_4
+; CHECK-NEXT:    b .LBB11_6
+; CHECK-NEXT:  .LBB11_2: // %land.lhs.true
 ; CHECK-NEXT:    adrp x8, :got:b
 ; CHECK-NEXT:    adrp x9, :got:c
 ; CHECK-NEXT:    ldr x8, [x8, :got_lo12:b]
@@ -778,11 +795,11 @@ define i32 @combine_gt_ge_sel(i64 %v, ptr %p) #0 {
 ; CHECK-NEXT:    ldr w8, [x8]
 ; CHECK-NEXT:    ldr w9, [x9]
 ; CHECK-NEXT:    cmp w8, w9
-; CHECK-NEXT:    b.ne .LBB11_5
-; CHECK-NEXT:  // %bb.4:
+; CHECK-NEXT:    b.ne .LBB11_4
+; CHECK-NEXT:  // %bb.3:
 ; CHECK-NEXT:    mov w0, #1 // =0x1
 ; CHECK-NEXT:    ret
-; CHECK-NEXT:  .LBB11_5: // %land.lhs.true3
+; CHECK-NEXT:  .LBB11_4: // %land.lhs.true3
 ; CHECK-NEXT:    adrp x8, :got:b
 ; CHECK-NEXT:    adrp x9, :got:d
 ; CHECK-NEXT:    ldr x8, [x8, :got_lo12:b]
@@ -790,7 +807,12 @@ define i32 @combine_gt_ge_sel(i64 %v, ptr %p) #0 {
 ; CHECK-NEXT:    ldr w8, [x8]
 ; CHECK-NEXT:    ldr w9, [x9]
 ; CHECK-NEXT:    cmp w8, w9
-; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    b.ne .LBB11_6
+; CHECK-NEXT:  // %bb.5:
+; CHECK-NEXT:    mov w0, #1 // =0x1
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB11_6: // %if.end
+; CHECK-NEXT:    mov w0, wzr
 ; CHECK-NEXT:    ret
 entry:
   %0 = load i32, ptr @a, align 4
diff --git a/llvm/test/CodeGen/Hexagon/vect/zext-v4i1.ll b/llvm/test/CodeGen/Hexagon/vect/zext-v4i1.ll
index c33c81841be6..dddc4bd953d7 100644
--- a/llvm/test/CodeGen/Hexagon/vect/zext-v4i1.ll
+++ b/llvm/test/CodeGen/Hexagon/vect/zext-v4i1.ll
@@ -10,13 +10,12 @@ define i32 @fred(ptr %a0) #0 {
 ; CHECK-LABEL: fred:
 ; CHECK:       // %bb.0: // %b0
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     r1:0 = combine(r0,#0)
-; CHECK-NEXT:     if (p0) jumpr r31
+; CHECK-NEXT:     if (p0) jump:nt .LBB0_2
 ; CHECK-NEXT:    }
-; CHECK-NEXT:  .LBB0_1: // %b2
+; CHECK-NEXT:  // %bb.1: // %b2
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     r3:2 = combine(#0,#0)
-; CHECK-NEXT:     r1:0 = memd(r1+#0)
+; CHECK-NEXT:     r1:0 = memd(r0+#0)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
 ; CHECK-NEXT:     p0 = vcmph.eq(r1:0,r3:2)
@@ -28,7 +27,16 @@ define i32 @fred(ptr %a0) #0 {
 ; CHECK-NEXT:     r0 = and(r0,#1)
 ; CHECK-NEXT:    }
 ; CHECK-NEXT:    {
-; CHECK-NEXT:     r0 = !cmp.eq(r0,#11)
+; CHECK-NEXT:     p0 = cmp.eq(r0,#11)
+; CHECK-NEXT:     r0 = #1
+; CHECK-NEXT:    }
+; CHECK-NEXT:    {
+; CHECK-NEXT:     if (p0) r0 = #0
+; CHECK-NEXT:     jumpr r31
+; CHECK-NEXT:    }
+; CHECK-NEXT:  .LBB0_2: // %b14
+; CHECK-NEXT:    {
+; CHECK-NEXT:     r0 = #0
 ; CHECK-NEXT:     jumpr r31
 ; CHECK-NEXT:    }
 b0:
diff --git a/llvm/test/Transforms/JumpThreading/pr79175.ll b/llvm/test/Transforms/JumpThreading/pr79175.ll
index cce30ce07999..2c7ee0770cdc 100644
--- a/llvm/test/Transforms/JumpThreading/pr79175.ll
+++ b/llvm/test/Transforms/JumpThreading/pr79175.ll
@@ -17,11 +17,11 @@ define i32 @test(i64 %idx, i32 %val) {
 ; CHECK:       cond.end:
 ; CHECK-NEXT:    [[CMP_I:%.*]] = icmp sgt i32 [[VAL]], 0
 ; CHECK-NEXT:    [[COND_FR:%.*]] = freeze i1 [[CMP_I]]
-; CHECK-NEXT:    br i1 [[COND_FR]], label [[TMP0:%.*]], label [[COND_END_THREAD]]
-; CHECK:       0:
-; CHECK-NEXT:    br label [[COND_END_THREAD]]
+; CHECK-NEXT:    br i1 [[COND_FR]], label [[COND_END_THREAD]], label [[TMP0:%.*]]
 ; CHECK:       cond.end.thread:
-; CHECK-NEXT:    [[TMP1:%.*]] = phi i32 [ [[VAL]], [[COND_END]] ], [ 0, [[TMP0]] ], [ 0, [[FOR_BODY]] ]
+; CHECK-NEXT:    br label [[TMP0]]
+; CHECK:       0:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i32 [ 0, [[COND_END_THREAD]] ], [ [[VAL]], [[COND_END]] ]
 ; CHECK-NEXT:    [[F_IDX:%.*]] = getelementptr inbounds i32, ptr @f, i64 [[IDX]]
 ; CHECK-NEXT:    store i32 [[TMP1]], ptr [[F_IDX]], align 4
 ; CHECK-NEXT:    [[F_RELOAD:%.*]] = load i32, ptr @f, align 4
diff --git a/llvm/test/Transforms/JumpThreading/select.ll b/llvm/test/Transforms/JumpThreading/select.ll
index 27ebf4c25da5..4ec55a66bb8a 100644
--- a/llvm/test/Transforms/JumpThreading/select.ll
+++ b/llvm/test/Transforms/JumpThreading/select.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals
 ; RUN: opt -S -passes="jump-threading" -debug-only=branch-prob < %s 2>&1 | FileCheck %s
-; RUN: opt -S -passes="require<branch-prob>,jump-threading" -debug-only=branch-prob  -disable-output < %s 2>&1 | FileCheck -check-prefix=CHECK-BPI %s
+; RUN: opt -S -passes="require<branch-prob>,jump-threading" -debug-only=branch-prob < %s 2>&1 | FileCheck -check-prefixes=CHECK,CHECK-BPI %s
 ; REQUIRES: asserts
 
 ; CHECK-BPI-LABEL:  ---- Branch Probability Info : unfold1 ----
@@ -21,7 +21,7 @@ declare void @quux()
 ; booleans where at least one operand is true/false/undef.
 
 ;.
-; CHECK: @anchor = constant [3 x ptr] [ptr blockaddress(@test_indirectbr, %L1), ptr inttoptr (i32 1 to ptr), ptr blockaddress(@test_indirectbr, %L3)]
+; CHECK: @[[ANCHOR:[a-zA-Z0-9_$"\\.-]+]] = constant [3 x ptr] [ptr blockaddress(@test_indirectbr, [[L1:%.*]]), ptr inttoptr (i32 1 to ptr), ptr blockaddress(@test_indirectbr, [[L3:%.*]])]
 ;.
 define void @test_br(i1 %cond, i1 %value) nounwind {
 ; CHECK-LABEL: @test_br(
@@ -66,8 +66,8 @@ define void @test_switch(i1 %cond, i8 %value) nounwind {
 ; CHECK-NEXT:    call void @quux()
 ; CHECK-NEXT:    [[EXPR:%.*]] = select i1 [[COND]], i8 1, i8 [[VALUE:%.*]]
 ; CHECK-NEXT:    switch i8 [[EXPR]], label [[L3:%.*]] [
-; CHECK-NEXT:      i8 1, label [[L1]]
-; CHECK-NEXT:      i8 2, label [[L2:%.*]]
+; CHECK-NEXT:    i8 1, label [[L1]]
+; CHECK-NEXT:    i8 2, label [[L2:%.*]]
 ; CHECK-NEXT:    ]
 ; CHECK:       L1:
 ; CHECK-NEXT:    call void @foo()
@@ -192,8 +192,8 @@ define void @test_switch_cmp(i1 %cond, i32 %val, i8 %value) nounwind {
 ; CHECK:       0:
 ; CHECK-NEXT:    [[TMP1:%.*]] = phi i8 [ [[VALUE:%.*]], [[L0]] ]
 ; CHECK-NEXT:    switch i8 [[TMP1]], label [[L3:%.*]] [
-; CHECK-NEXT:      i8 1, label [[L1]]
-; CHECK-NEXT:      i8 2, label [[L2:%.*]]
+; CHECK-NEXT:    i8 1, label [[L1]]
+; CHECK-NEXT:    i8 2, label [[L2:%.*]]
 ; CHECK-NEXT:    ]
 ; CHECK:       L1:
 ; CHECK-NEXT:    call void @foo()
@@ -237,8 +237,8 @@ define void @test_switch_default(ptr nocapture %status) nounwind {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr [[STATUS:%.*]], align 4
 ; CHECK-NEXT:    switch i32 [[TMP0]], label [[L2:%.*]] [
-; CHECK-NEXT:      i32 5061, label [[L2_THREAD:%.*]]
-; CHECK-NEXT:      i32 0, label [[L2]]
+; CHECK-NEXT:    i32 5061, label [[L2_THREAD:%.*]]
+; CHECK-NEXT:    i32 0, label [[L2]]
 ; CHECK-NEXT:    ]
 ; CHECK:       L2.thread:
 ; CHECK-NEXT:    store i32 10025, ptr [[STATUS]], align 4
@@ -377,21 +377,21 @@ define i32 @unfold3(i32 %u, i32 %v, i32 %w, i32 %x, i32 %y, i32 %z, i32 %j) noun
 ; CHECK-NEXT:    br i1 [[CMP_I]], label [[DOTEXIT_THREAD4:%.*]], label [[COND_FALSE_I:%.*]]
 ; CHECK:       cond.false.i:
 ; CHECK-NEXT:    [[CMP4_I:%.*]] = icmp sgt i32 [[U]], [[V]]
-; CHECK-NEXT:    br i1 [[CMP4_I]], label [[DOTEXIT_THREAD4]], label [[COND_FALSE_6_I:%.*]]
+; CHECK-NEXT:    br i1 [[CMP4_I]], label [[DOTEXIT_THREAD:%.*]], label [[COND_FALSE_6_I:%.*]]
 ; CHECK:       cond.false.6.i:
 ; CHECK-NEXT:    [[CMP8_I:%.*]] = icmp slt i32 [[W:%.*]], [[X:%.*]]
 ; CHECK-NEXT:    br i1 [[CMP8_I]], label [[DOTEXIT_THREAD4]], label [[COND_FALSE_10_I:%.*]]
 ; CHECK:       cond.false.10.i:
 ; CHECK-NEXT:    [[CMP13_I:%.*]] = icmp sgt i32 [[W]], [[X]]
-; CHECK-NEXT:    br i1 [[CMP13_I]], label [[DOTEXIT_THREAD4]], label [[DOTEXIT:%.*]]
+; CHECK-NEXT:    br i1 [[CMP13_I]], label [[DOTEXIT_THREAD]], label [[DOTEXIT:%.*]]
 ; CHECK:       .exit:
 ; CHECK-NEXT:    [[PHITMP:%.*]] = icmp sge i32 [[Y:%.*]], [[Z:%.*]]
 ; CHECK-NEXT:    [[COND_FR:%.*]] = freeze i1 [[PHITMP]]
-; CHECK-NEXT:    br i1 [[COND_FR]], label [[DOTEXIT_THREAD:%.*]], label [[DOTEXIT_THREAD4]]
-; CHECK:       0:
-; CHECK-NEXT:    br label [[DOTEXIT_THREAD4]]
+; CHECK-NEXT:    br i1 [[COND_FR]], label [[DOTEXIT_THREAD]], label [[DOTEXIT_THREAD4]]
 ; CHECK:       .exit.thread:
-; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ [[ADD3]], [[DOTEXIT]] ], [ [[J]], [[DOTEXIT_THREAD]] ], [ [[J]], [[COND_FALSE_I]] ], [ [[J]], [[COND_FALSE_10_I]] ], [ [[ADD3]], [[ENTRY:%.*]] ], [ [[ADD3]], [[COND_FALSE_6_I]] ]
+; CHECK-NEXT:    br label [[DOTEXIT_THREAD4]]
+; CHECK:       .exit.thread4:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ [[J]], [[DOTEXIT_THREAD]] ], [ [[ADD3]], [[DOTEXIT]] ], [ [[ADD3]], [[ENTRY:%.*]] ], [ [[ADD3]], [[COND_FALSE_6_I]] ]
 ; CHECK-NEXT:    ret i32 [[TMP0]]
 ;
 entry:
@@ -430,23 +430,23 @@ define i32 @unfold4(i32 %u, i32 %v, i32 %w, i32 %x, i32 %y, i32 %z, i32 %j) noun
 ; CHECK-NEXT:    br i1 [[CMP_I]], label [[DOTEXIT_THREAD:%.*]], label [[COND_FALSE_I:%.*]]
 ; CHECK:       cond.false.i:
 ; CHECK-NEXT:    [[CMP4_I:%.*]] = icmp sgt i32 [[U]], [[V]]
-; CHECK-NEXT:    br i1 [[CMP4_I]], label [[DOTEXIT_THREAD]], label [[COND_FALSE_6_I:%.*]]
+; CHECK-NEXT:    br i1 [[CMP4_I]], label [[DOTEXIT_THREAD5:%.*]], label [[COND_FALSE_6_I:%.*]]
 ; CHECK:       cond.false.6.i:
 ; CHECK-NEXT:    [[CMP8_I:%.*]] = icmp slt i32 [[W:%.*]], [[X:%.*]]
 ; CHECK-NEXT:    br i1 [[CMP8_I]], label [[DOTEXIT_THREAD]], label [[COND_FALSE_10_I:%.*]]
 ; CHECK:       cond.false.10.i:
 ; CHECK-NEXT:    [[CMP13_I:%.*]] = icmp sgt i32 [[W]], [[X]]
-; CHECK-NEXT:    br i1 [[CMP13_I]], label [[DOTEXIT_THREAD]], label [[DOTEXIT:%.*]]
+; CHECK-NEXT:    br i1 [[CMP13_I]], label [[DOTEXIT_THREAD5]], label [[DOTEXIT:%.*]]
 ; CHECK:       .exit:
 ; CHECK-NEXT:    [[CMP19_I:%.*]] = icmp sge i32 [[Y:%.*]], [[Z:%.*]]
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP19_I]] to i32
 ; CHECK-NEXT:    [[LNOT_I18:%.*]] = icmp eq i32 [[CONV]], 1
 ; CHECK-NEXT:    [[COND_FR:%.*]] = freeze i1 [[LNOT_I18]]
-; CHECK-NEXT:    br i1 [[COND_FR]], label [[TMP1:%.*]], label [[DOTEXIT_THREAD]]
-; CHECK:       0:
-; CHECK-NEXT:    br label [[DOTEXIT_THREAD]]
+; CHECK-NEXT:    br i1 [[COND_FR]], label [[DOTEXIT_THREAD]], label [[DOTEXIT_THREAD5]]
 ; CHECK:       .exit.thread:
-; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ [[ADD3]], [[DOTEXIT]] ], [ [[J]], [[TMP1]] ], [ [[J]], [[ENTRY:%.*]] ], [ [[J]], [[COND_FALSE_6_I]] ], [ [[ADD3]], [[COND_FALSE_I]] ], [ [[ADD3]], [[COND_FALSE_10_I]] ]
+; CHECK-NEXT:    br label [[DOTEXIT_THREAD5]]
+; CHECK:       .exit.thread5:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ [[J]], [[DOTEXIT_THREAD]] ], [ [[ADD3]], [[DOTEXIT]] ], [ [[ADD3]], [[COND_FALSE_I]] ], [ [[ADD3]], [[COND_FALSE_10_I]] ]
 ; CHECK-NEXT:    ret i32 [[TMP0]]
 ;
 entry:
@@ -560,10 +560,10 @@ define void @test_func(ptr nocapture readonly %a, ptr nocapture readonly %b, ptr
 ; CHECK:       if.end:
 ; CHECK-NEXT:    [[LOCAL_VAR_0:%.*]] = phi i32 [ [[TMP1]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    switch i32 [[LOCAL_VAR_0]], label [[SW_DEFAULT]] [
-; CHECK-NEXT:      i32 2, label [[SW_BB]]
-; CHECK-NEXT:      i32 4, label [[SW_BB7]]
-; CHECK-NEXT:      i32 5, label [[SW_BB8:%.*]]
-; CHECK-NEXT:      i32 7, label [[SW_BB9:%.*]]
+; CHECK-NEXT:    i32 2, label [[SW_BB]]
+; CHECK-NEXT:    i32 4, label [[SW_BB7]]
+; CHECK-NEXT:    i32 5, label [[SW_BB8:%.*]]
+; CHECK-NEXT:    i32 7, label [[SW_BB9:%.*]]
 ; CHECK-NEXT:    ]
 ; CHECK:       sw.bb:
 ; CHECK-NEXT:    call void @foo()
@@ -674,5 +674,3 @@ if.end:
 ; CHECK: [[META0:![0-9]+]] = !{!"function_entry_count", i64 1984}
 ; CHECK: [[PROF1]] = !{!"branch_weights", i64 1073741824, i64 3221225472}
 ;.
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK-BPI: {{.*}}
diff --git a/llvm/test/Transforms/JumpThreading/thread-prob-7.ll b/llvm/test/Transforms/JumpThreading/thread-prob-7.ll
index 4623a579be48..8c9d89871d00 100644
--- a/llvm/test/Transforms/JumpThreading/thread-prob-7.ll
+++ b/llvm/test/Transforms/JumpThreading/thread-prob-7.ll
@@ -14,15 +14,15 @@ define i32 @func0(i32 %a0, i32 %a1) !prof !0 {
 ; CHECK-NEXT:    br i1 [[CMP1]], label [[BB_JOIN_THREAD:%.*]], label [[TEST2_FALSE:%.*]], !prof [[PROF2:![0-9]+]]
 ; CHECK:       test2_false:
 ; CHECK-NEXT:    call void @foobar()
-; CHECK-NEXT:    br label [[BB_JOIN_THREAD]]
+; CHECK-NEXT:    br label [[TMP0:%.*]]
 ; CHECK:       bb_join:
 ; CHECK-NEXT:    [[C:%.*]] = phi i1 [ [[CX]], [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[COND_FR:%.*]] = freeze i1 [[C]]
-; CHECK-NEXT:    br i1 [[COND_FR]], label [[BB_JOIN_THREAD1:%.*]], label [[BB_JOIN_THREAD]], !prof [[PROF3:![0-9]+]]
+; CHECK-NEXT:    br i1 [[COND_FR]], label [[BB_JOIN_THREAD]], label [[TMP0]], !prof [[PROF3:![0-9]+]]
 ; CHECK:       bb_join.thread:
-; CHECK-NEXT:    br label [[BB_JOIN_THREAD]]
+; CHECK-NEXT:    br label [[TMP0]]
 ; CHECK:       0:
-; CHECK-NEXT:    [[TMP1:%.*]] = phi i32 [ 7, [[BB_JOIN]] ], [ 7, [[TEST2_FALSE]] ], [ 42, [[TEST2]] ], [ 42, [[BB_JOIN_THREAD1]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i32 [ 42, [[BB_JOIN_THREAD]] ], [ 7, [[BB_JOIN]] ], [ 7, [[TEST2_FALSE]] ]
 ; CHECK-NEXT:    ret i32 [[TMP1]]
 ;
 entry:
diff --git a/llvm/test/Transforms/JumpThreading/uncond-no-phi.ll b/llvm/test/Transforms/JumpThreading/uncond-no-phi.ll
deleted file mode 100644
index 6104e8f8778b..000000000000
--- a/llvm/test/Transforms/JumpThreading/uncond-no-phi.ll
+++ /dev/null
@@ -1,123 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt -passes=jump-threading -S < %s | FileCheck %s
-
-define i1 @if_else(i1 %c, i1 %c1) {
-; CHECK-LABEL: define i1 @if_else(
-; CHECK-SAME: i1 [[C:%.*]], i1 [[C1:%.*]]) {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[C]], label [[THEN:%.*]], label [[RETURN:%.*]]
-; CHECK:       then:
-; CHECK-NEXT:    call void @dummy()
-; CHECK-NEXT:    br i1 [[C1]], label [[ELSE:%.*]], label [[RETURN]]
-; CHECK:       else:
-; CHECK-NEXT:    br label [[RETURN]]
-; CHECK:       return:
-; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i1 [ false, [[THEN]] ], [ true, [[ENTRY:%.*]] ], [ true, [[ELSE]] ]
-; CHECK-NEXT:    ret i1 [[RETVAL_0]]
-;
-entry:
-  br i1 %c, label %then, label %else
-
-then:
-  call void @dummy()
-  br i1 %c1, label %else, label %return
-
-else:
-  br label %return
-
-return:
-  %retval.0 = phi i1 [ true, %else ], [ false, %then ]
-  ret i1 %retval.0
-}
-
-define i8 @switch_uncond(i8 %arg) {
-; CHECK-LABEL: define i8 @switch_uncond(
-; CHECK-SAME: i8 [[ARG:%.*]]) {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    switch i8 [[ARG]], label [[DEFAULT:%.*]] [
-; CHECK-NEXT:      i8 0, label [[BB1:%.*]]
-; CHECK-NEXT:      i8 1, label [[BB3:%.*]]
-; CHECK-NEXT:      i8 2, label [[BB2:%.*]]
-; CHECK-NEXT:      i8 3, label [[END:%.*]]
-; CHECK-NEXT:    ]
-; CHECK:       default:
-; CHECK-NEXT:    unreachable
-; CHECK:       bb:
-; CHECK-NEXT:    call void @dummy()
-; CHECK-NEXT:    br label [[END]]
-; CHECK:       bb1:
-; CHECK-NEXT:    call void @dummy()
-; CHECK-NEXT:    br label [[END]]
-; CHECK:       bb2:
-; CHECK-NEXT:    br label [[END]]
-; CHECK:       end:
-; CHECK-NEXT:    [[PHI:%.*]] = phi i8 [ 1, [[ENTRY:%.*]] ], [ 0, [[BB3]] ], [ 0, [[BB1]] ], [ 0, [[BB2]] ]
-; CHECK-NEXT:    ret i8 [[PHI]]
-;
-entry:
-  switch i8 %arg, label %default [
-  i8 0, label %bb
-  i8 1, label %bb1
-  i8 2, label %bb2
-  i8 3, label %end
-  ]
-
-default:
-  unreachable
-
-bb:
-  call void @dummy()
-  br label %bb2
-
-bb1:
-  call void @dummy()
-  br label %bb2
-
-; Predecessors of %bb2 are %bb and %bb1, they are not identical.
-; So we can thread %bb2.
-bb2:
-  br label %end
-
-end:
-  %phi = phi i8 [ 0, %bb2 ], [ 1, %entry ]
-  ret i8 %phi
-}
-
-define i8 @switch_uncond_fail(i8 %arg) {
-; CHECK-LABEL: define i8 @switch_uncond_fail(
-; CHECK-SAME: i8 [[ARG:%.*]]) {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    switch i8 [[ARG]], label [[DEFAULT:%.*]] [
-; CHECK-NEXT:      i8 0, label [[BB:%.*]]
-; CHECK-NEXT:      i8 1, label [[BB]]
-; CHECK-NEXT:      i8 2, label [[END:%.*]]
-; CHECK-NEXT:    ]
-; CHECK:       default:
-; CHECK-NEXT:    br label [[END]]
-; CHECK:       bb:
-; CHECK-NEXT:    br label [[END]]
-; CHECK:       end:
-; CHECK-NEXT:    [[PHI:%.*]] = phi i8 [ 0, [[BB]] ], [ 1, [[ENTRY:%.*]] ], [ 2, [[DEFAULT]] ]
-; CHECK-NEXT:    ret i8 [[PHI]]
-;
-entry:
-  switch i8 %arg, label %default [
-  i8 0, label %bb
-  i8 1, label %bb
-  i8 2, label %end
-  ]
-
-default:
-  br label %end
-
-; Predecessor of %bb is only %entry (though there are two in predecessor list),
-; thus it's unthreadable.
-bb:
-  br label %end
-
-end:
-  %phi = phi i8 [ 0, %bb ], [ 1, %entry ], [ 2, %default ]
-  ret i8 %phi
-}
-
-declare void @dummy()
diff --git a/llvm/test/Transforms/PhaseOrdering/thread-uncond-bb.ll b/llvm/test/Transforms/PhaseOrdering/thread-uncond-bb.ll
deleted file mode 100644
index 17146d7d5987..000000000000
--- a/llvm/test/Transforms/PhaseOrdering/thread-uncond-bb.ll
+++ /dev/null
@@ -1,62 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
-; RUN: opt < %s -O3 -S | FileCheck %s
-
-define i32 @thread_uncond_bb_cmp(i1 %c, i32 %v) {
-; CHECK-LABEL: define i32 @thread_uncond_bb_cmp(
-; CHECK-SAME: i1 [[C:%.*]], i32 [[V:%.*]]) local_unnamed_addr {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[C]], label [[DO_END:%.*]], label [[IF_THEN:%.*]]
-; CHECK:       if.then:
-; CHECK-NEXT:    tail call void @dummy()
-; CHECK-NEXT:    br label [[DO_END]]
-; CHECK:       return:
-; CHECK-NEXT:    [[RETVAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[V]], [[IF_THEN]] ]
-; CHECK-NEXT:    ret i32 [[RETVAL]]
-;
-entry:
-  br i1 %c, label %do.end, label %if.then
-
-if.then:                                          ; preds = %entry
-  call void @dummy()
-  %tobool = icmp eq i32 %v, 0
-  br i1 %tobool, label %do.end, label %return
-
-do.end:                                           ; preds = %entry, %if.then
-  br label %return
-
-return:                                           ; preds = %if.then, %do.end
-  %retval = phi i32 [ 0, %do.end ], [ %v, %if.then ]
-  ret i32 %retval
-}
-
-define i32 @thread_uncond_bb_cmp_zext(i1 %c, i32 %v) {
-; CHECK-LABEL: define i32 @thread_uncond_bb_cmp_zext(
-; CHECK-SAME: i1 [[C:%.*]], i32 [[V:%.*]]) local_unnamed_addr {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[C]], label [[DO_END:%.*]], label [[IF_THEN:%.*]]
-; CHECK:       if.then:
-; CHECK-NEXT:    tail call void @dummy()
-; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[V]], 0
-; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = zext i1 [[TOBOOL]] to i32
-; CHECK-NEXT:    br label [[DO_END]]
-; CHECK:       return:
-; CHECK-NEXT:    [[RETVAL:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[SPEC_SELECT]], [[IF_THEN]] ]
-; CHECK-NEXT:    ret i32 [[RETVAL]]
-;
-entry:
-  br i1 %c, label %do.end, label %if.then
-
-if.then:                                          ; preds = %entry
-  call void @dummy()
-  %tobool = icmp eq i32 %v, 0
-  br i1 %tobool, label %do.end, label %return
-
-do.end:                                           ; preds = %entry, %if.then
-  br label %return
-
-return:                                           ; preds = %if.then, %do.end
-  %retval = phi i32 [ 0, %do.end ], [ 1, %if.then ]
-  ret i32 %retval
-}
-
-declare void @dummy()
-- 
GitLab


From 22629bb22a1bea95eebfc9b3171005de107c38f1 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Tue, 16 Apr 2024 10:54:28 -0400
Subject: [PATCH 414/464] [libc++] Use availability to rely on key functions
 for bad_expected_access and bad_function_call (#87390)

This patch uses our availability machinery to allow defining a key
function for bad_function_call and bad_expected_access at all times but
only rely on it when we can. This prevents compilers from complaining
about weak vtables and reduces code bloat and the amount of work done by
the dynamic linker.

rdar://111917845
---
 libcxx/include/__availability                 | 16 ++++++++++++
 libcxx/include/__config                       | 26 ++++---------------
 .../include/__expected/bad_expected_access.h  | 13 ++++++----
 libcxx/include/__functional/function.h        |  5 +++-
 ...bcxxabi.v1.stable.exceptions.nonew.abilist |  4 +++
 ...bcxxabi.v1.stable.exceptions.nonew.abilist |  4 +++
 ...bcxxabi.v1.stable.exceptions.nonew.abilist |  4 +++
 ...bcxxabi.v1.stable.exceptions.nonew.abilist |  4 +++
 ...bcxxabi.v1.stable.exceptions.nonew.abilist |  4 +++
 ...bcxxabi.v1.stable.exceptions.nonew.abilist |  4 +++
 ...bcxxabi.v1.stable.exceptions.nonew.abilist |  4 +++
 ...bcxxabi.v1.stable.exceptions.nonew.abilist |  4 +++
 ...xxabi.v1.stable.noexceptions.nonew.abilist |  4 +++
 libcxx/src/CMakeLists.txt                     |  1 +
 libcxx/src/expected.cpp                       | 13 ++++++++++
 libcxx/src/functional.cpp                     |  2 --
 16 files changed, 83 insertions(+), 29 deletions(-)
 create mode 100644 libcxx/src/expected.cpp

diff --git a/libcxx/include/__availability b/libcxx/include/__availability
index bb3ed0a8da52..aa761eb5bfe5 100644
--- a/libcxx/include/__availability
+++ b/libcxx/include/__availability
@@ -160,6 +160,15 @@
 #  define _LIBCPP_AVAILABILITY_HAS_TZDB 1
 #  define _LIBCPP_AVAILABILITY_TZDB
 
+// These macros determine whether we assume that std::bad_function_call and
+// std::bad_expected_access provide a key function in the dylib. This allows
+// centralizing their vtable and typeinfo instead of having all TUs provide
+// a weak definition that then gets deduplicated.
+#  define _LIBCPP_AVAILABILITY_HAS_BAD_FUNCTION_CALL_KEY_FUNCTION 1
+#  define _LIBCPP_AVAILABILITY_BAD_FUNCTION_CALL_KEY_FUNCTION
+#  define _LIBCPP_AVAILABILITY_HAS_BAD_EXPECTED_ACCESS_KEY_FUNCTION 1
+#  define _LIBCPP_AVAILABILITY_BAD_EXPECTED_ACCESS_KEY_FUNCTION
+
 #elif defined(__APPLE__)
 
 #  define _LIBCPP_AVAILABILITY_HAS_BAD_OPTIONAL_ACCESS                                                                 \
@@ -290,6 +299,13 @@
 #  else
 #    define _LIBCPP_AVAILABILITY_HAS_ADDITIONAL_IOSTREAM_EXPLICIT_INSTANTIATIONS_1 1
 #  endif
+
+#  define _LIBCPP_AVAILABILITY_HAS_BAD_FUNCTION_CALL_KEY_FUNCTION 0
+#  define _LIBCPP_AVAILABILITY_BAD_FUNCTION_CALL_KEY_FUNCTION __attribute__((unavailable))
+
+#  define _LIBCPP_AVAILABILITY_HAS_BAD_EXPECTED_ACCESS_KEY_FUNCTION 0
+#  define _LIBCPP_AVAILABILITY_BAD_EXPECTED_ACCESS_KEY_FUNCTION __attribute__((unavailable))
+
 #else
 
 // ...New vendors can add availability markup here...
diff --git a/libcxx/include/__config b/libcxx/include/__config
index 82782b31c557..e9fda9cd24eb 100644
--- a/libcxx/include/__config
+++ b/libcxx/include/__config
@@ -120,14 +120,11 @@
 #    define _LIBCPP_ABI_FIX_UNORDERED_NODE_POINTER_UB
 #    define _LIBCPP_ABI_FORWARD_LIST_REMOVE_NODE_POINTER_UB
 #    define _LIBCPP_ABI_FIX_UNORDERED_CONTAINER_SIZE_TYPE
-// Define a key function for `bad_function_call` in the library, to centralize
-// its vtable and typeinfo to libc++ rather than having all other libraries
-// using that class define their own copies.
-#    define _LIBCPP_ABI_BAD_FUNCTION_CALL_KEY_FUNCTION
-// Override the default return value of exception::what() for
-// bad_function_call::what() with a string that is specific to
-// bad_function_call (see http://wg21.link/LWG2233). This is an ABI break
-// because it changes the vtable layout of bad_function_call.
+// Override the default return value of exception::what() for bad_function_call::what()
+// with a string that is specific to bad_function_call (see http://wg21.link/LWG2233).
+// This is an ABI break on platforms that sign and authenticate vtable function pointers
+// because it changes the mangling of the virtual function located in the vtable, which
+// changes how it gets signed.
 #    define _LIBCPP_ABI_BAD_FUNCTION_CALL_GOOD_WHAT_MESSAGE
 // Enable optimized version of __do_get_(un)signed which avoids redundant copies.
 #    define _LIBCPP_ABI_OPTIMIZED_LOCALE_NUM_GET
@@ -197,19 +194,6 @@
 #    if defined(__FreeBSD__) && __FreeBSD__ < 14
 #      define _LIBCPP_DEPRECATED_ABI_DISABLE_PAIR_TRIVIAL_COPY_CTOR
 #    endif
-// For XCOFF linkers, we have problems if we see a weak hidden version of a symbol
-// in user code (like you get with -fvisibility-inlines-hidden) and then a strong def
-// in the library, so we need to always rely on the library version.
-#    if defined(_AIX)
-#      define _LIBCPP_ABI_BAD_FUNCTION_CALL_KEY_FUNCTION
-#    endif
-#  endif
-
-#  if defined(_LIBCPP_BUILDING_LIBRARY) || _LIBCPP_ABI_VERSION >= 2
-// Define a key function for `bad_function_call` in the library, to centralize
-// its vtable and typeinfo to libc++ rather than having all other libraries
-// using that class define their own copies.
-#    define _LIBCPP_ABI_BAD_FUNCTION_CALL_KEY_FUNCTION
 #  endif
 
 // We had some bugs where we use [[no_unique_address]] together with construct_at,
diff --git a/libcxx/include/__expected/bad_expected_access.h b/libcxx/include/__expected/bad_expected_access.h
index 9d490307b680..ef29fa508831 100644
--- a/libcxx/include/__expected/bad_expected_access.h
+++ b/libcxx/include/__expected/bad_expected_access.h
@@ -9,6 +9,7 @@
 #ifndef _LIBCPP___EXPECTED_BAD_EXPECTED_ACCESS_H
 #define _LIBCPP___EXPECTED_BAD_EXPECTED_ACCESS_H
 
+#include <__availability>
 #include <__config>
 #include <__exception/exception.h>
 #include <__utility/move.h>
@@ -28,9 +29,11 @@ template <class _Err>
 class bad_expected_access;
 
 _LIBCPP_DIAGNOSTIC_PUSH
+#  if !_LIBCPP_AVAILABILITY_HAS_BAD_EXPECTED_ACCESS_KEY_FUNCTION
 _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wweak-vtables")
+#  endif
 template <>
-class bad_expected_access<void> : public exception {
+class _LIBCPP_EXPORTED_FROM_ABI bad_expected_access<void> : public exception {
 protected:
   _LIBCPP_HIDE_FROM_ABI bad_expected_access() noexcept                                      = default;
   _LIBCPP_HIDE_FROM_ABI bad_expected_access(const bad_expected_access&) noexcept            = default;
@@ -40,11 +43,11 @@ protected:
   _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~bad_expected_access() override                             = default;
 
 public:
-  // The way this has been designed (by using a class template below) means that we'll already
-  // have a profusion of these vtables in TUs, and the dynamic linker will already have a bunch
-  // of work to do. So it is not worth hiding the <void> specialization in the dylib, given that
-  // it adds deployment target restrictions.
+#  if _LIBCPP_AVAILABILITY_HAS_BAD_EXPECTED_ACCESS_KEY_FUNCTION
+  const char* what() const noexcept override;
+#  else
   _LIBCPP_HIDE_FROM_ABI_VIRTUAL const char* what() const noexcept override { return "bad access to std::expected"; }
+#  endif
 };
 _LIBCPP_DIAGNOSTIC_POP
 
diff --git a/libcxx/include/__functional/function.h b/libcxx/include/__functional/function.h
index 1faa9e92ebd6..36057706933d 100644
--- a/libcxx/include/__functional/function.h
+++ b/libcxx/include/__functional/function.h
@@ -11,6 +11,7 @@
 #define _LIBCPP___FUNCTIONAL_FUNCTION_H
 
 #include <__assert>
+#include <__availability>
 #include <__config>
 #include <__exception/exception.h>
 #include <__functional/binary_function.h>
@@ -55,7 +56,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 // bad_function_call
 
 _LIBCPP_DIAGNOSTIC_PUSH
+#  if !_LIBCPP_AVAILABILITY_HAS_BAD_FUNCTION_CALL_KEY_FUNCTION
 _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wweak-vtables")
+#  endif
 class _LIBCPP_EXPORTED_FROM_ABI bad_function_call : public exception {
 public:
   _LIBCPP_HIDE_FROM_ABI bad_function_call() _NOEXCEPT                                    = default;
@@ -64,7 +67,7 @@ public:
 // Note that when a key function is not used, every translation unit that uses
 // bad_function_call will end up containing a weak definition of the vtable and
 // typeinfo.
-#  ifdef _LIBCPP_ABI_BAD_FUNCTION_CALL_KEY_FUNCTION
+#  if _LIBCPP_AVAILABILITY_HAS_BAD_FUNCTION_CALL_KEY_FUNCTION
   ~bad_function_call() _NOEXCEPT override;
 #  else
   _LIBCPP_HIDE_FROM_ABI_VIRTUAL ~bad_function_call() _NOEXCEPT override {}
diff --git a/libcxx/lib/abi/arm64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/arm64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist
index 46353986f5d7..64cf368e6e68 100644
--- a/libcxx/lib/abi/arm64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist
+++ b/libcxx/lib/abi/arm64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist
@@ -575,6 +575,7 @@
 {'is_defined': True, 'name': '__ZNKSt3__118__time_get_storageIcE15__do_date_orderEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNKSt3__118__time_get_storageIwE15__do_date_orderEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNKSt3__119__shared_weak_count13__get_deleterERKSt9type_info', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNKSt3__119bad_expected_accessIvE4whatEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNKSt3__120__codecvt_utf8_utf16IDiE10do_unshiftER11__mbstate_tPcS4_RS4_', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNKSt3__120__codecvt_utf8_utf16IDiE11do_encodingEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNKSt3__120__codecvt_utf8_utf16IDiE13do_max_lengthEv', 'type': 'FUNC'}
@@ -2073,6 +2074,7 @@
 {'is_defined': True, 'name': '__ZTINSt3__117moneypunct_bynameIwLb1EEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTINSt3__118basic_stringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTINSt3__119__shared_weak_countE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTINSt3__119bad_expected_accessIvEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTINSt3__119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTINSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTINSt3__120__codecvt_utf8_utf16IDiEE', 'size': 0, 'type': 'OBJECT'}
@@ -2264,6 +2266,7 @@
 {'is_defined': True, 'name': '__ZTSNSt3__117moneypunct_bynameIwLb0EEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__117moneypunct_bynameIwLb1EEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__118basic_stringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__119bad_expected_accessIvEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__13pmr15memory_resourceE', 'size': 0, 'type': 'OBJECT'}
@@ -2482,6 +2485,7 @@
 {'is_defined': True, 'name': '__ZTVNSt3__117moneypunct_bynameIwLb1EEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTVNSt3__118basic_stringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTVNSt3__119__shared_weak_countE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTVNSt3__119bad_expected_accessIvEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTVNSt3__119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTVNSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTVNSt3__120__codecvt_utf8_utf16IDiEE', 'size': 0, 'type': 'OBJECT'}
diff --git a/libcxx/lib/abi/i686-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/i686-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist
index fec3a4505a0c..8751dffe2302 100644
--- a/libcxx/lib/abi/i686-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist
+++ b/libcxx/lib/abi/i686-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist
@@ -209,6 +209,7 @@
 {'is_defined': True, 'name': '_ZNKSt6__ndk118__time_get_storageIcE15__do_date_orderEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt6__ndk118__time_get_storageIwE15__do_date_orderEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt6__ndk119__shared_weak_count13__get_deleterERKSt9type_info', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNKSt6__ndk119bad_expected_accessIvE4whatEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt6__ndk120__codecvt_utf8_utf16IDiE10do_unshiftER9mbstate_tPcS4_RS4_', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt6__ndk120__codecvt_utf8_utf16IDiE11do_encodingEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt6__ndk120__codecvt_utf8_utf16IDiE13do_max_lengthEv', 'type': 'FUNC'}
@@ -1722,6 +1723,7 @@
 {'is_defined': True, 'name': '_ZTINSt6__ndk118__time_get_storageIwEE', 'size': 12, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt6__ndk118basic_stringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 12, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt6__ndk119__shared_weak_countE', 'size': 24, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZTINSt6__ndk119bad_expected_accessIvEE', 'size': 12, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt6__ndk119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 12, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt6__ndk119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 12, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt6__ndk120__codecvt_utf8_utf16IDiEE', 'size': 12, 'type': 'OBJECT'}
@@ -1958,6 +1960,7 @@
 {'is_defined': True, 'name': '_ZTSNSt6__ndk118__time_get_storageIwEE', 'size': 35, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt6__ndk118basic_stringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 72, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt6__ndk119__shared_weak_countE', 'size': 33, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZTSNSt6__ndk119bad_expected_accessIvEE', 'size': 36, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt6__ndk119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 73, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt6__ndk119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 73, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt6__ndk120__codecvt_utf8_utf16IDiEE', 'size': 38, 'type': 'OBJECT'}
@@ -2188,6 +2191,7 @@
 {'is_defined': True, 'name': '_ZTVNSt6__ndk117moneypunct_bynameIwLb1EEE', 'size': 56, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt6__ndk118basic_stringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 60, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt6__ndk119__shared_weak_countE', 'size': 28, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZTVNSt6__ndk119bad_expected_accessIvEE', 'size': 20, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt6__ndk119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 40, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt6__ndk119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 40, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt6__ndk120__codecvt_utf8_utf16IDiEE', 'size': 48, 'type': 'OBJECT'}
diff --git a/libcxx/lib/abi/powerpc-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/powerpc-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist
index e52cf98dd4c4..7e223e665288 100644
--- a/libcxx/lib/abi/powerpc-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist
+++ b/libcxx/lib/abi/powerpc-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist
@@ -99,6 +99,7 @@
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNKSt3__118__time_get_storageIcE15__do_date_orderEv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNKSt3__118__time_get_storageIwE15__do_date_orderEv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNKSt3__119__shared_weak_count13__get_deleterERKSt9type_info', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
+{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNKSt3__119bad_expected_accessIvE4whatEv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNKSt3__120__codecvt_utf8_utf16IDiE10do_unshiftERPcS2_S2_S3_', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNKSt3__120__codecvt_utf8_utf16IDiE11do_encodingEv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNKSt3__120__codecvt_utf8_utf16IDiE13do_max_lengthEv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
@@ -910,6 +911,7 @@
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTINSt3__117__widen_from_utf8ILm32EEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTINSt3__117bad_function_callE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTINSt3__119__shared_weak_countE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
+{'import_export': 'EXP', 'is_defined': True, 'name': '_ZTINSt3__119bad_expected_accessIvEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTINSt3__120__codecvt_utf8_utf16IDiEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTINSt3__120__codecvt_utf8_utf16IDsEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTINSt3__120__codecvt_utf8_utf16IwEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
@@ -969,6 +971,7 @@
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTSNSt3__117__widen_from_utf8ILm32EEE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTSNSt3__117bad_function_callE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTSNSt3__119__shared_weak_countE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
+{'import_export': 'EXP', 'is_defined': True, 'name': '_ZTSNSt3__119bad_expected_accessIvEE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTSNSt3__120__codecvt_utf8_utf16IDiEE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTSNSt3__120__codecvt_utf8_utf16IDsEE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTSNSt3__120__codecvt_utf8_utf16IwEE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
@@ -1031,6 +1034,7 @@
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTVNSt3__117__widen_from_utf8ILm32EEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTVNSt3__117bad_function_callE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTVNSt3__119__shared_weak_countE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
+{'import_export': 'EXP', 'is_defined': True, 'name': '_ZTVNSt3__119bad_expected_accessIvEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTVNSt3__120__codecvt_utf8_utf16IDiEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTVNSt3__120__codecvt_utf8_utf16IDsEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTVNSt3__120__codecvt_utf8_utf16IwEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
diff --git a/libcxx/lib/abi/powerpc64-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/powerpc64-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist
index 52a04706ddf2..407d0456757a 100644
--- a/libcxx/lib/abi/powerpc64-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist
+++ b/libcxx/lib/abi/powerpc64-ibm-aix.libcxxabi.v1.stable.exceptions.nonew.abilist
@@ -99,6 +99,7 @@
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNKSt3__118__time_get_storageIcE15__do_date_orderEv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNKSt3__118__time_get_storageIwE15__do_date_orderEv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNKSt3__119__shared_weak_count13__get_deleterERKSt9type_info', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
+{'import_export': 'EXP', 'is_defined': True, 'name': '_ZNKSt3__119bad_expected_accessIvE4whatEv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNKSt3__120__codecvt_utf8_utf16IDiE10do_unshiftERPcS2_S2_S3_', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNKSt3__120__codecvt_utf8_utf16IDiE11do_encodingEv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZNKSt3__120__codecvt_utf8_utf16IDiE13do_max_lengthEv', 'storage_mapping_class': 'DS', 'type': 'FUNC'}
@@ -910,6 +911,7 @@
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTINSt3__117__widen_from_utf8ILm32EEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTINSt3__117bad_function_callE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTINSt3__119__shared_weak_countE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
+{'import_export': 'EXP', 'is_defined': True, 'name': '_ZTINSt3__119bad_expected_accessIvEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTINSt3__120__codecvt_utf8_utf16IDiEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTINSt3__120__codecvt_utf8_utf16IDsEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTINSt3__120__codecvt_utf8_utf16IwEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
@@ -969,6 +971,7 @@
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTSNSt3__117__widen_from_utf8ILm32EEE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTSNSt3__117bad_function_callE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTSNSt3__119__shared_weak_countE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
+{'import_export': 'EXP', 'is_defined': True, 'name': '_ZTSNSt3__119bad_expected_accessIvEE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTSNSt3__120__codecvt_utf8_utf16IDiEE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTSNSt3__120__codecvt_utf8_utf16IDsEE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTSNSt3__120__codecvt_utf8_utf16IwEE', 'storage_mapping_class': 'RO', 'type': 'OBJECT'}
@@ -1031,6 +1034,7 @@
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTVNSt3__117__widen_from_utf8ILm32EEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTVNSt3__117bad_function_callE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTVNSt3__119__shared_weak_countE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
+{'import_export': 'EXP', 'is_defined': True, 'name': '_ZTVNSt3__119bad_expected_accessIvEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTVNSt3__120__codecvt_utf8_utf16IDiEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTVNSt3__120__codecvt_utf8_utf16IDsEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
 {'import_export': 'EXP', 'is_defined': True, 'name': '_ZTVNSt3__120__codecvt_utf8_utf16IwEE', 'storage_mapping_class': 'RW', 'type': 'OBJECT'}
diff --git a/libcxx/lib/abi/x86_64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/x86_64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist
index c169b4a99252..d578b41383c0 100644
--- a/libcxx/lib/abi/x86_64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist
+++ b/libcxx/lib/abi/x86_64-apple-darwin.libcxxabi.v1.stable.exceptions.nonew.abilist
@@ -575,6 +575,7 @@
 {'is_defined': True, 'name': '__ZNKSt3__118__time_get_storageIcE15__do_date_orderEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNKSt3__118__time_get_storageIwE15__do_date_orderEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNKSt3__119__shared_weak_count13__get_deleterERKSt9type_info', 'type': 'FUNC'}
+{'is_defined': True, 'name': '__ZNKSt3__119bad_expected_accessIvE4whatEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNKSt3__120__codecvt_utf8_utf16IDiE10do_unshiftER11__mbstate_tPcS4_RS4_', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNKSt3__120__codecvt_utf8_utf16IDiE11do_encodingEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '__ZNKSt3__120__codecvt_utf8_utf16IDiE13do_max_lengthEv', 'type': 'FUNC'}
@@ -2087,6 +2088,7 @@
 {'is_defined': True, 'name': '__ZTINSt3__118__time_get_storageIwEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTINSt3__118basic_stringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTINSt3__119__shared_weak_countE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTINSt3__119bad_expected_accessIvEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTINSt3__119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTINSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTINSt3__120__codecvt_utf8_utf16IDiEE', 'size': 0, 'type': 'OBJECT'}
@@ -2291,6 +2293,7 @@
 {'is_defined': True, 'name': '__ZTSNSt3__117moneypunct_bynameIwLb0EEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__117moneypunct_bynameIwLb1EEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__118basic_stringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTSNSt3__119bad_expected_accessIvEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTSNSt3__13pmr15memory_resourceE', 'size': 0, 'type': 'OBJECT'}
@@ -2516,6 +2519,7 @@
 {'is_defined': True, 'name': '__ZTVNSt3__117moneypunct_bynameIwLb1EEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTVNSt3__118basic_stringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTVNSt3__119__shared_weak_countE', 'size': 0, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '__ZTVNSt3__119bad_expected_accessIvEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTVNSt3__119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTVNSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 0, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '__ZTVNSt3__120__codecvt_utf8_utf16IDiEE', 'size': 0, 'type': 'OBJECT'}
diff --git a/libcxx/lib/abi/x86_64-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/x86_64-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist
index efa2189e9c92..fc0f4fcf415e 100644
--- a/libcxx/lib/abi/x86_64-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist
+++ b/libcxx/lib/abi/x86_64-linux-android21.libcxxabi.v1.stable.exceptions.nonew.abilist
@@ -209,6 +209,7 @@
 {'is_defined': True, 'name': '_ZNKSt6__ndk118__time_get_storageIcE15__do_date_orderEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt6__ndk118__time_get_storageIwE15__do_date_orderEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt6__ndk119__shared_weak_count13__get_deleterERKSt9type_info', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNKSt6__ndk119bad_expected_accessIvE4whatEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt6__ndk120__codecvt_utf8_utf16IDiE10do_unshiftER9mbstate_tPcS4_RS4_', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt6__ndk120__codecvt_utf8_utf16IDiE11do_encodingEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt6__ndk120__codecvt_utf8_utf16IDiE13do_max_lengthEv', 'type': 'FUNC'}
@@ -1722,6 +1723,7 @@
 {'is_defined': True, 'name': '_ZTINSt6__ndk118__time_get_storageIwEE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt6__ndk118basic_stringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt6__ndk119__shared_weak_countE', 'size': 40, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZTINSt6__ndk119bad_expected_accessIvEE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt6__ndk119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt6__ndk119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt6__ndk120__codecvt_utf8_utf16IDiEE', 'size': 24, 'type': 'OBJECT'}
@@ -1955,6 +1957,7 @@
 {'is_defined': True, 'name': '_ZTSNSt6__ndk118__time_get_storageIwEE', 'size': 35, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt6__ndk118basic_stringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 72, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt6__ndk119__shared_weak_countE', 'size': 33, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZTSNSt6__ndk119bad_expected_accessIvEE', 'size': 36, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt6__ndk119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 73, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt6__ndk119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 73, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt6__ndk120__codecvt_utf8_utf16IDiEE', 'size': 38, 'type': 'OBJECT'}
@@ -2182,6 +2185,7 @@
 {'is_defined': True, 'name': '_ZTVNSt6__ndk117moneypunct_bynameIwLb1EEE', 'size': 112, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt6__ndk118basic_stringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 120, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt6__ndk119__shared_weak_countE', 'size': 56, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZTVNSt6__ndk119bad_expected_accessIvEE', 'size': 40, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt6__ndk119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 80, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt6__ndk119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 80, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt6__ndk120__codecvt_utf8_utf16IDiEE', 'size': 96, 'type': 'OBJECT'}
diff --git a/libcxx/lib/abi/x86_64-unknown-freebsd.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/x86_64-unknown-freebsd.libcxxabi.v1.stable.exceptions.nonew.abilist
index ebda5b0dfba5..4022339562b3 100644
--- a/libcxx/lib/abi/x86_64-unknown-freebsd.libcxxabi.v1.stable.exceptions.nonew.abilist
+++ b/libcxx/lib/abi/x86_64-unknown-freebsd.libcxxabi.v1.stable.exceptions.nonew.abilist
@@ -266,6 +266,7 @@
 {'is_defined': True, 'name': '_ZNKSt3__118__time_get_storageIcE15__do_date_orderEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt3__118__time_get_storageIwE15__do_date_orderEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt3__119__shared_weak_count13__get_deleterERKSt9type_info', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNKSt3__119bad_expected_accessIvE4whatEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt3__120__codecvt_utf8_utf16IDiE10do_unshiftER11__mbstate_tPcS4_RS4_', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt3__120__codecvt_utf8_utf16IDiE11do_encodingEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt3__120__codecvt_utf8_utf16IDiE13do_max_lengthEv', 'type': 'FUNC'}
@@ -1695,6 +1696,7 @@
 {'is_defined': True, 'name': '_ZTINSt3__118__time_get_storageIwEE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__118basic_stringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__119__shared_weak_countE', 'size': 40, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZTINSt3__119bad_expected_accessIvEE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__120__codecvt_utf8_utf16IDiEE', 'size': 24, 'type': 'OBJECT'}
@@ -1829,6 +1831,7 @@
 {'is_defined': True, 'name': '_ZTSNSt3__118__time_get_storageIwEE', 'size': 32, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__118basic_stringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 69, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__119__shared_weak_countE', 'size': 30, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZTSNSt3__119bad_expected_accessIvEE', 'size': 33, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 70, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 70, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__120__codecvt_utf8_utf16IDiEE', 'size': 35, 'type': 'OBJECT'}
@@ -1962,6 +1965,7 @@
 {'is_defined': True, 'name': '_ZTVNSt3__117moneypunct_bynameIwLb1EEE', 'size': 112, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__118basic_stringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 120, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__119__shared_weak_countE', 'size': 56, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZTVNSt3__119bad_expected_accessIvEE', 'size': 40, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 80, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 80, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__120__codecvt_utf8_utf16IDiEE', 'size': 96, 'type': 'OBJECT'}
diff --git a/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.exceptions.nonew.abilist b/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.exceptions.nonew.abilist
index 6432ad3be358..574c4504c59b 100644
--- a/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.exceptions.nonew.abilist
+++ b/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.exceptions.nonew.abilist
@@ -264,6 +264,7 @@
 {'is_defined': True, 'name': '_ZNKSt3__118__time_get_storageIcE15__do_date_orderEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt3__118__time_get_storageIwE15__do_date_orderEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt3__119__shared_weak_count13__get_deleterERKSt9type_info', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNKSt3__119bad_expected_accessIvE4whatEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt3__120__codecvt_utf8_utf16IDiE10do_unshiftER11__mbstate_tPcS4_RS4_', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt3__120__codecvt_utf8_utf16IDiE11do_encodingEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt3__120__codecvt_utf8_utf16IDiE13do_max_lengthEv', 'type': 'FUNC'}
@@ -1696,6 +1697,7 @@
 {'is_defined': True, 'name': '_ZTINSt3__118__time_get_storageIwEE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__118basic_stringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__119__shared_weak_countE', 'size': 40, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZTINSt3__119bad_expected_accessIvEE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__120__codecvt_utf8_utf16IDiEE', 'size': 24, 'type': 'OBJECT'}
@@ -1830,6 +1832,7 @@
 {'is_defined': True, 'name': '_ZTSNSt3__118__time_get_storageIwEE', 'size': 32, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__118basic_stringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 69, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__119__shared_weak_countE', 'size': 30, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZTSNSt3__119bad_expected_accessIvEE', 'size': 33, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 70, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 70, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__120__codecvt_utf8_utf16IDiEE', 'size': 35, 'type': 'OBJECT'}
@@ -1963,6 +1966,7 @@
 {'is_defined': True, 'name': '_ZTVNSt3__117moneypunct_bynameIwLb1EEE', 'size': 112, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__118basic_stringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 120, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__119__shared_weak_countE', 'size': 56, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZTVNSt3__119bad_expected_accessIvEE', 'size': 40, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 80, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 80, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__120__codecvt_utf8_utf16IDiEE', 'size': 96, 'type': 'OBJECT'}
diff --git a/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.noexceptions.nonew.abilist b/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.noexceptions.nonew.abilist
index 1fe84e17b3f7..665546699e8d 100644
--- a/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.noexceptions.nonew.abilist
+++ b/libcxx/lib/abi/x86_64-unknown-linux-gnu.libcxxabi.v1.stable.noexceptions.nonew.abilist
@@ -235,6 +235,7 @@
 {'is_defined': True, 'name': '_ZNKSt3__118__time_get_storageIcE15__do_date_orderEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt3__118__time_get_storageIwE15__do_date_orderEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt3__119__shared_weak_count13__get_deleterERKSt9type_info', 'type': 'FUNC'}
+{'is_defined': True, 'name': '_ZNKSt3__119bad_expected_accessIvE4whatEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt3__120__codecvt_utf8_utf16IDiE10do_unshiftER11__mbstate_tPcS4_RS4_', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt3__120__codecvt_utf8_utf16IDiE11do_encodingEv', 'type': 'FUNC'}
 {'is_defined': True, 'name': '_ZNKSt3__120__codecvt_utf8_utf16IDiE13do_max_lengthEv', 'type': 'FUNC'}
@@ -1667,6 +1668,7 @@
 {'is_defined': True, 'name': '_ZTINSt3__118__time_get_storageIwEE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__118basic_stringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__119__shared_weak_countE', 'size': 40, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZTINSt3__119bad_expected_accessIvEE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 24, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTINSt3__120__codecvt_utf8_utf16IDiEE', 'size': 24, 'type': 'OBJECT'}
@@ -1801,6 +1803,7 @@
 {'is_defined': True, 'name': '_ZTSNSt3__118__time_get_storageIwEE', 'size': 32, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__118basic_stringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 69, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__119__shared_weak_countE', 'size': 30, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZTSNSt3__119bad_expected_accessIvEE', 'size': 33, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 70, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 70, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTSNSt3__120__codecvt_utf8_utf16IDiEE', 'size': 35, 'type': 'OBJECT'}
@@ -1934,6 +1937,7 @@
 {'is_defined': True, 'name': '_ZTVNSt3__117moneypunct_bynameIwLb1EEE', 'size': 112, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__118basic_stringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 120, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__119__shared_weak_countE', 'size': 56, 'type': 'OBJECT'}
+{'is_defined': True, 'name': '_ZTVNSt3__119bad_expected_accessIvEE', 'size': 40, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__119basic_istringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 80, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__119basic_ostringstreamIcNS_11char_traitsIcEENS_9allocatorIcEEEE', 'size': 80, 'type': 'OBJECT'}
 {'is_defined': True, 'name': '_ZTVNSt3__120__codecvt_utf8_utf16IDiEE', 'size': 96, 'type': 'OBJECT'}
diff --git a/libcxx/src/CMakeLists.txt b/libcxx/src/CMakeLists.txt
index 208500ec14fc..a4a3fee86457 100644
--- a/libcxx/src/CMakeLists.txt
+++ b/libcxx/src/CMakeLists.txt
@@ -10,6 +10,7 @@ set(LIBCXX_SOURCES
   chrono.cpp
   error_category.cpp
   exception.cpp
+  expected.cpp
   filesystem/filesystem_clock.cpp
   filesystem/filesystem_error.cpp
   filesystem/path_parser.h
diff --git a/libcxx/src/expected.cpp b/libcxx/src/expected.cpp
new file mode 100644
index 000000000000..f30efb516479
--- /dev/null
+++ b/libcxx/src/expected.cpp
@@ -0,0 +1,13 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <expected>
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+const char* bad_expected_access<void>::what() const noexcept { return "bad access to std::expected"; }
+_LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/src/functional.cpp b/libcxx/src/functional.cpp
index 570bb78e150b..ef53e3e84da0 100644
--- a/libcxx/src/functional.cpp
+++ b/libcxx/src/functional.cpp
@@ -10,9 +10,7 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#ifdef _LIBCPP_ABI_BAD_FUNCTION_CALL_KEY_FUNCTION
 bad_function_call::~bad_function_call() noexcept {}
-#endif
 
 #ifdef _LIBCPP_ABI_BAD_FUNCTION_CALL_GOOD_WHAT_MESSAGE
 const char* bad_function_call::what() const noexcept { return "std::bad_function_call"; }
-- 
GitLab


From 9ddedf07ed80076e0e419940753aeaaf719a09ec Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Tue, 16 Apr 2024 10:57:48 -0400
Subject: [PATCH 415/464] [libc++] Deprecate the C++20 synchronization library
 before C++20 (#86410)

When we initially implemented the C++20 synchronization library, we
reluctantly accepted for the implementation to be backported to C++03
upon request from the person who provided the patch. This was when we
were only starting to have experience with the issues this can create,
so we flinched. Nowadays, we have a much stricter stance about not
backporting features to previous standards.

We have recently started fixing several bugs (and near bugs) in our
implementation of the synchronization library. A recurring theme during
these reviews has been how difficult to understand the current code is,
and upon inspection it becomes clear that being able to use a few recent
C++ features (in particular lambdas) would help a great deal. The code
would still be pretty intricate, but it would be a lot easier to reason
about the flow of callbacks through things like
__thread_poll_with_backoff.

As a result, this patch deprecates support for the synchronization
library before C++20. In the next release, we can remove that support
entirely.
---
 libcxx/.clang-format                          |  1 +
 libcxx/docs/ReleaseNotes/19.rst               |  4 +++
 libcxx/include/__atomic/atomic.h              | 12 ++++---
 libcxx/include/__atomic/atomic_flag.h         | 34 +++++++++++--------
 libcxx/include/__config                       |  8 +++++
 libcxx/include/barrier                        |  2 +-
 libcxx/include/latch                          |  2 +-
 libcxx/include/semaphore                      |  6 ++--
 .../atomic_notify_all.pass.cpp                |  5 ++-
 .../atomic_notify_one.pass.cpp                |  5 ++-
 .../atomic_wait.pass.cpp                      |  5 ++-
 .../atomic_wait_explicit.pass.cpp             |  5 ++-
 .../std/thread/thread.barrier/arrive.pass.cpp |  3 ++
 .../thread.barrier/arrive_and_drop.pass.cpp   |  3 ++
 .../thread.barrier/arrive_and_wait.pass.cpp   |  3 ++
 .../thread/thread.barrier/completion.pass.cpp |  3 ++
 .../thread.barrier/ctor.compile.pass.cpp      |  3 ++
 .../std/thread/thread.barrier/max.pass.cpp    |  3 ++
 .../thread.latch/arrive_and_wait.pass.cpp     |  3 ++
 .../thread/thread.latch/count_down.pass.cpp   |  3 ++
 .../std/thread/thread.latch/ctor.pass.cpp     |  3 ++
 .../test/std/thread/thread.latch/max.pass.cpp |  3 ++
 .../std/thread/thread.latch/try_wait.pass.cpp |  3 ++
 .../thread/thread.semaphore/acquire.pass.cpp  |  3 ++
 .../thread/thread.semaphore/binary.pass.cpp   |  3 ++
 .../thread.semaphore/ctor.compile.pass.cpp    |  3 ++
 .../std/thread/thread.semaphore/max.pass.cpp  |  3 ++
 .../thread/thread.semaphore/release.pass.cpp  |  3 ++
 .../thread/thread.semaphore/timed.pass.cpp    |  3 ++
 .../thread.semaphore/try_acquire.pass.cpp     |  3 ++
 30 files changed, 117 insertions(+), 26 deletions(-)

diff --git a/libcxx/.clang-format b/libcxx/.clang-format
index 39ae1322ffa8..c37ab817bca9 100644
--- a/libcxx/.clang-format
+++ b/libcxx/.clang-format
@@ -24,6 +24,7 @@ AttributeMacros: [
                   '_LIBCPP_CONSTEXPR_SINCE_CXX23',
                   '_LIBCPP_CONSTEXPR',
                   '_LIBCPP_CONSTINIT',
+                  '_LIBCPP_DEPRECATED_ATOMIC_SYNC',
                   '_LIBCPP_DEPRECATED_IN_CXX11',
                   '_LIBCPP_DEPRECATED_IN_CXX14',
                   '_LIBCPP_DEPRECATED_IN_CXX17',
diff --git a/libcxx/docs/ReleaseNotes/19.rst b/libcxx/docs/ReleaseNotes/19.rst
index e5db17daa482..45aac88e4550 100644
--- a/libcxx/docs/ReleaseNotes/19.rst
+++ b/libcxx/docs/ReleaseNotes/19.rst
@@ -75,6 +75,10 @@ Improvements and New Features
 Deprecations and Removals
 -------------------------
 
+- The C++20 synchronization library (``<barrier>``, ``<latch>``, ``atomic::wait``, etc.) has been deprecated
+  in language modes prior to C++20. If you are using these features prior to C++20, please update to ``-std=c++20``.
+  In LLVM 20, the C++20 synchronization library will be removed entirely in language modes prior to C++20.
+
 - TODO: The ``LIBCXX_ENABLE_ASSERTIONS`` CMake variable that was used to enable the safe mode has been deprecated and setting
   it triggers an error; use the ``LIBCXX_HARDENING_MODE`` CMake variable with the value ``extensive`` instead. Similarly,
   the ``_LIBCPP_ENABLE_ASSERTIONS`` macro has been deprecated (setting it to ``1`` still enables the extensive mode in
diff --git a/libcxx/include/__atomic/atomic.h b/libcxx/include/__atomic/atomic.h
index 3dfb6937d032..bd3f659c22df 100644
--- a/libcxx/include/__atomic/atomic.h
+++ b/libcxx/include/__atomic/atomic.h
@@ -462,22 +462,26 @@ atomic_wait_explicit(const atomic<_Tp>* __o, typename atomic<_Tp>::value_type __
 // atomic_notify_one
 
 template <class _Tp>
-_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void atomic_notify_one(volatile atomic<_Tp>* __o) _NOEXCEPT {
+_LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
+atomic_notify_one(volatile atomic<_Tp>* __o) _NOEXCEPT {
   __o->notify_one();
 }
 template <class _Tp>
-_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void atomic_notify_one(atomic<_Tp>* __o) _NOEXCEPT {
+_LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
+atomic_notify_one(atomic<_Tp>* __o) _NOEXCEPT {
   __o->notify_one();
 }
 
 // atomic_notify_all
 
 template <class _Tp>
-_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void atomic_notify_all(volatile atomic<_Tp>* __o) _NOEXCEPT {
+_LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
+atomic_notify_all(volatile atomic<_Tp>* __o) _NOEXCEPT {
   __o->notify_all();
 }
 template <class _Tp>
-_LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void atomic_notify_all(atomic<_Tp>* __o) _NOEXCEPT {
+_LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
+atomic_notify_all(atomic<_Tp>* __o) _NOEXCEPT {
   __o->notify_all();
 }
 
diff --git a/libcxx/include/__atomic/atomic_flag.h b/libcxx/include/__atomic/atomic_flag.h
index 084366237c16..3ec3366ecaaf 100644
--- a/libcxx/include/__atomic/atomic_flag.h
+++ b/libcxx/include/__atomic/atomic_flag.h
@@ -49,22 +49,26 @@ struct atomic_flag {
     __cxx_atomic_store(&__a_, _LIBCPP_ATOMIC_FLAG_TYPE(false), __m);
   }
 
-  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void wait(bool __v, memory_order __m = memory_order_seq_cst) const
-      volatile _NOEXCEPT {
+  _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
+  wait(bool __v, memory_order __m = memory_order_seq_cst) const volatile _NOEXCEPT {
     std::__atomic_wait(*this, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);
   }
-  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
+  _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void
   wait(bool __v, memory_order __m = memory_order_seq_cst) const _NOEXCEPT {
     std::__atomic_wait(*this, _LIBCPP_ATOMIC_FLAG_TYPE(__v), __m);
   }
-  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() volatile _NOEXCEPT {
+  _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() volatile _NOEXCEPT {
+    std::__atomic_notify_one(*this);
+  }
+  _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT {
     std::__atomic_notify_one(*this);
   }
-  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_one() _NOEXCEPT { std::__atomic_notify_one(*this); }
   _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() volatile _NOEXCEPT {
     std::__atomic_notify_all(*this);
   }
-  _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT { std::__atomic_notify_all(*this); }
+  _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_AVAILABILITY_SYNC _LIBCPP_HIDE_FROM_ABI void notify_all() _NOEXCEPT {
+    std::__atomic_notify_all(*this);
+  }
 
 #if _LIBCPP_STD_VER >= 20
   _LIBCPP_HIDE_FROM_ABI constexpr atomic_flag() _NOEXCEPT : __a_(false) {}
@@ -141,41 +145,43 @@ inline _LIBCPP_HIDE_FROM_ABI void atomic_flag_clear_explicit(atomic_flag* __o, m
   __o->clear(__m);
 }
 
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void
+inline _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void
 atomic_flag_wait(const volatile atomic_flag* __o, bool __v) _NOEXCEPT {
   __o->wait(__v);
 }
 
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void
+inline _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void
 atomic_flag_wait(const atomic_flag* __o, bool __v) _NOEXCEPT {
   __o->wait(__v);
 }
 
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void
+inline _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void
 atomic_flag_wait_explicit(const volatile atomic_flag* __o, bool __v, memory_order __m) _NOEXCEPT {
   __o->wait(__v, __m);
 }
 
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void
+inline _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void
 atomic_flag_wait_explicit(const atomic_flag* __o, bool __v, memory_order __m) _NOEXCEPT {
   __o->wait(__v, __m);
 }
 
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void
+inline _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void
 atomic_flag_notify_one(volatile atomic_flag* __o) _NOEXCEPT {
   __o->notify_one();
 }
 
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void atomic_flag_notify_one(atomic_flag* __o) _NOEXCEPT {
+inline _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void
+atomic_flag_notify_one(atomic_flag* __o) _NOEXCEPT {
   __o->notify_one();
 }
 
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void
+inline _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void
 atomic_flag_notify_all(volatile atomic_flag* __o) _NOEXCEPT {
   __o->notify_all();
 }
 
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void atomic_flag_notify_all(atomic_flag* __o) _NOEXCEPT {
+inline _LIBCPP_DEPRECATED_ATOMIC_SYNC _LIBCPP_HIDE_FROM_ABI _LIBCPP_AVAILABILITY_SYNC void
+atomic_flag_notify_all(atomic_flag* __o) _NOEXCEPT {
   __o->notify_all();
 }
 
diff --git a/libcxx/include/__config b/libcxx/include/__config
index e9fda9cd24eb..9b4155af1e3c 100644
--- a/libcxx/include/__config
+++ b/libcxx/include/__config
@@ -956,6 +956,14 @@ typedef __char32_t char32_t;
 #    define _LIBCPP_DEPRECATED_(m)
 #  endif
 
+#  if _LIBCPP_STD_VER < 20
+#    define _LIBCPP_DEPRECATED_ATOMIC_SYNC                                                                             \
+      _LIBCPP_DEPRECATED_("The C++20 synchronization library has been deprecated prior to C++20. Please update to "    \
+                          "using -std=c++20 if you need to use these facilities.")
+#  else
+#    define _LIBCPP_DEPRECATED_ATOMIC_SYNC /* nothing */
+#  endif
+
 #  if !defined(_LIBCPP_CXX03_LANG)
 #    define _LIBCPP_DEPRECATED_IN_CXX11 _LIBCPP_DEPRECATED
 #  else
diff --git a/libcxx/include/barrier b/libcxx/include/barrier
index c5fd84b91925..d77607826762 100644
--- a/libcxx/include/barrier
+++ b/libcxx/include/barrier
@@ -257,7 +257,7 @@ public:
 #  endif // !_LIBCPP_HAS_NO_TREE_BARRIER
 
 template <class _CompletionF = __empty_completion>
-class barrier {
+class _LIBCPP_DEPRECATED_ATOMIC_SYNC barrier {
   __barrier_base<_CompletionF> __b_;
 
 public:
diff --git a/libcxx/include/latch b/libcxx/include/latch
index 3cc725838114..1937617f7dcc 100644
--- a/libcxx/include/latch
+++ b/libcxx/include/latch
@@ -66,7 +66,7 @@ _LIBCPP_PUSH_MACROS
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-class latch {
+class _LIBCPP_DEPRECATED_ATOMIC_SYNC latch {
   __atomic_base<ptrdiff_t> __a_;
 
 public:
diff --git a/libcxx/include/semaphore b/libcxx/include/semaphore
index 1375ec3f7c04..cb2f42c106ca 100644
--- a/libcxx/include/semaphore
+++ b/libcxx/include/semaphore
@@ -127,7 +127,7 @@ private:
 };
 
 template <ptrdiff_t __least_max_value = _LIBCPP_SEMAPHORE_MAX>
-class counting_semaphore {
+class _LIBCPP_DEPRECATED_ATOMIC_SYNC counting_semaphore {
   __atomic_semaphore_base __semaphore_;
 
 public:
@@ -172,7 +172,9 @@ public:
   }
 };
 
-using binary_semaphore = counting_semaphore<1>;
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
+using binary_semaphore _LIBCPP_DEPRECATED_ATOMIC_SYNC = counting_semaphore<1>;
+_LIBCPP_SUPPRESS_DEPRECATED_POP
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/test/std/atomics/atomics.types.operations/atomics.types.operations.wait/atomic_notify_all.pass.cpp b/libcxx/test/std/atomics/atomics.types.operations/atomics.types.operations.wait/atomic_notify_all.pass.cpp
index 2b9f34b731f8..0ec530c922e7 100644
--- a/libcxx/test/std/atomics/atomics.types.operations/atomics.types.operations.wait/atomic_notify_all.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.types.operations/atomics.types.operations.wait/atomic_notify_all.pass.cpp
@@ -7,9 +7,12 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// XFAIL: c++03
+// UNSUPPORTED: c++03
 // XFAIL: !has-1024-bit-atomics
 
+// Until we drop support for the synchronization library in C++11/14/17
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // XFAIL: availability-synchronization_library-missing
 
 // <atomic>
diff --git a/libcxx/test/std/atomics/atomics.types.operations/atomics.types.operations.wait/atomic_notify_one.pass.cpp b/libcxx/test/std/atomics/atomics.types.operations/atomics.types.operations.wait/atomic_notify_one.pass.cpp
index dfa781c56600..c21b67d479ae 100644
--- a/libcxx/test/std/atomics/atomics.types.operations/atomics.types.operations.wait/atomic_notify_one.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.types.operations/atomics.types.operations.wait/atomic_notify_one.pass.cpp
@@ -7,9 +7,12 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// XFAIL: c++03
+// UNSUPPORTED: c++03
 // XFAIL: !has-1024-bit-atomics
 
+// Until we drop support for the synchronization library in C++11/14/17
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // XFAIL: availability-synchronization_library-missing
 
 // <atomic>
diff --git a/libcxx/test/std/atomics/atomics.types.operations/atomics.types.operations.wait/atomic_wait.pass.cpp b/libcxx/test/std/atomics/atomics.types.operations/atomics.types.operations.wait/atomic_wait.pass.cpp
index 38142b336e72..af99113f1349 100644
--- a/libcxx/test/std/atomics/atomics.types.operations/atomics.types.operations.wait/atomic_wait.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.types.operations/atomics.types.operations.wait/atomic_wait.pass.cpp
@@ -7,9 +7,12 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// XFAIL: c++03
+// UNSUPPORTED: c++03
 // XFAIL: !has-1024-bit-atomics
 
+// Until we drop support for the synchronization library in C++11/14/17
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // XFAIL: availability-synchronization_library-missing
 
 // <atomic>
diff --git a/libcxx/test/std/atomics/atomics.types.operations/atomics.types.operations.wait/atomic_wait_explicit.pass.cpp b/libcxx/test/std/atomics/atomics.types.operations/atomics.types.operations.wait/atomic_wait_explicit.pass.cpp
index 2db95a0b67a7..bb8c64593b54 100644
--- a/libcxx/test/std/atomics/atomics.types.operations/atomics.types.operations.wait/atomic_wait_explicit.pass.cpp
+++ b/libcxx/test/std/atomics/atomics.types.operations/atomics.types.operations.wait/atomic_wait_explicit.pass.cpp
@@ -7,9 +7,12 @@
 //===----------------------------------------------------------------------===//
 //
 // UNSUPPORTED: no-threads
-// XFAIL: c++03
+// UNSUPPORTED: c++03
 // XFAIL: !has-1024-bit-atomics
 
+// Until we drop support for the synchronization library in C++11/14/17
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // XFAIL: availability-synchronization_library-missing
 
 // <atomic>
diff --git a/libcxx/test/std/thread/thread.barrier/arrive.pass.cpp b/libcxx/test/std/thread/thread.barrier/arrive.pass.cpp
index 18cdc6d654ac..d9d9c1dba6bb 100644
--- a/libcxx/test/std/thread/thread.barrier/arrive.pass.cpp
+++ b/libcxx/test/std/thread/thread.barrier/arrive.pass.cpp
@@ -9,6 +9,9 @@
 // UNSUPPORTED: no-threads
 // UNSUPPORTED: c++03, c++11
 
+// Until we drop support for the synchronization library in C++11/14/17
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // XFAIL: availability-synchronization_library-missing
 
 // <barrier>
diff --git a/libcxx/test/std/thread/thread.barrier/arrive_and_drop.pass.cpp b/libcxx/test/std/thread/thread.barrier/arrive_and_drop.pass.cpp
index 3fc48261de1b..aff7b26e16f7 100644
--- a/libcxx/test/std/thread/thread.barrier/arrive_and_drop.pass.cpp
+++ b/libcxx/test/std/thread/thread.barrier/arrive_and_drop.pass.cpp
@@ -9,6 +9,9 @@
 // UNSUPPORTED: no-threads
 // UNSUPPORTED: c++03, c++11
 
+// Until we drop support for the synchronization library in C++11/14/17
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // XFAIL: availability-synchronization_library-missing
 
 // <barrier>
diff --git a/libcxx/test/std/thread/thread.barrier/arrive_and_wait.pass.cpp b/libcxx/test/std/thread/thread.barrier/arrive_and_wait.pass.cpp
index 2aee8624ae3d..8c45ba9278f2 100644
--- a/libcxx/test/std/thread/thread.barrier/arrive_and_wait.pass.cpp
+++ b/libcxx/test/std/thread/thread.barrier/arrive_and_wait.pass.cpp
@@ -9,6 +9,9 @@
 // UNSUPPORTED: no-threads
 // UNSUPPORTED: c++03, c++11
 
+// Until we drop support for the synchronization library in C++11/14/17
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // XFAIL: availability-synchronization_library-missing
 
 // <barrier>
diff --git a/libcxx/test/std/thread/thread.barrier/completion.pass.cpp b/libcxx/test/std/thread/thread.barrier/completion.pass.cpp
index 7354dbe6ffe8..633a0c8bf236 100644
--- a/libcxx/test/std/thread/thread.barrier/completion.pass.cpp
+++ b/libcxx/test/std/thread/thread.barrier/completion.pass.cpp
@@ -9,6 +9,9 @@
 // UNSUPPORTED: no-threads
 // UNSUPPORTED: c++03, c++11
 
+// Until we drop support for the synchronization library in C++11/14/17
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // XFAIL: availability-synchronization_library-missing
 
 // <barrier>
diff --git a/libcxx/test/std/thread/thread.barrier/ctor.compile.pass.cpp b/libcxx/test/std/thread/thread.barrier/ctor.compile.pass.cpp
index d47127a18613..fe7068d2a574 100644
--- a/libcxx/test/std/thread/thread.barrier/ctor.compile.pass.cpp
+++ b/libcxx/test/std/thread/thread.barrier/ctor.compile.pass.cpp
@@ -9,6 +9,9 @@
 // UNSUPPORTED: no-threads
 // UNSUPPORTED: c++03, c++11
 
+// Until we drop support for the synchronization library in C++11/14/17
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // <barrier>
 
 // explicit barrier(ptrdiff_t __count, _CompletionF __completion = _CompletionF());
diff --git a/libcxx/test/std/thread/thread.barrier/max.pass.cpp b/libcxx/test/std/thread/thread.barrier/max.pass.cpp
index ec03c5c87a09..b09a02e1bdef 100644
--- a/libcxx/test/std/thread/thread.barrier/max.pass.cpp
+++ b/libcxx/test/std/thread/thread.barrier/max.pass.cpp
@@ -9,6 +9,9 @@
 // UNSUPPORTED: no-threads
 // UNSUPPORTED: c++03, c++11
 
+// Until we drop support for the synchronization library in C++11/14/17
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // <barrier>
 
 #include <barrier>
diff --git a/libcxx/test/std/thread/thread.latch/arrive_and_wait.pass.cpp b/libcxx/test/std/thread/thread.latch/arrive_and_wait.pass.cpp
index ddc06d2038cc..8ca4f37b73b9 100644
--- a/libcxx/test/std/thread/thread.latch/arrive_and_wait.pass.cpp
+++ b/libcxx/test/std/thread/thread.latch/arrive_and_wait.pass.cpp
@@ -9,6 +9,9 @@
 // UNSUPPORTED: no-threads
 // UNSUPPORTED: c++03, c++11
 
+// Until we drop support for the synchronization library in C++11/14/17
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // XFAIL: availability-synchronization_library-missing
 
 // <latch>
diff --git a/libcxx/test/std/thread/thread.latch/count_down.pass.cpp b/libcxx/test/std/thread/thread.latch/count_down.pass.cpp
index 1503c09509a6..eb524abd24b9 100644
--- a/libcxx/test/std/thread/thread.latch/count_down.pass.cpp
+++ b/libcxx/test/std/thread/thread.latch/count_down.pass.cpp
@@ -9,6 +9,9 @@
 // UNSUPPORTED: no-threads
 // UNSUPPORTED: c++03, c++11
 
+// Until we drop support for the synchronization library in C++11/14/17
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // XFAIL: availability-synchronization_library-missing
 
 // <latch>
diff --git a/libcxx/test/std/thread/thread.latch/ctor.pass.cpp b/libcxx/test/std/thread/thread.latch/ctor.pass.cpp
index 1983f6409cb5..bca4561bd2f7 100644
--- a/libcxx/test/std/thread/thread.latch/ctor.pass.cpp
+++ b/libcxx/test/std/thread/thread.latch/ctor.pass.cpp
@@ -9,6 +9,9 @@
 // UNSUPPORTED: no-threads
 // UNSUPPORTED: c++03, c++11
 
+// Until we drop support for the synchronization library in C++11/14/17
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // <latch>
 
 // inline constexpr explicit latch(ptrdiff_t __expected);
diff --git a/libcxx/test/std/thread/thread.latch/max.pass.cpp b/libcxx/test/std/thread/thread.latch/max.pass.cpp
index 8b9176c8cac5..bcf353ed9712 100644
--- a/libcxx/test/std/thread/thread.latch/max.pass.cpp
+++ b/libcxx/test/std/thread/thread.latch/max.pass.cpp
@@ -9,6 +9,9 @@
 // UNSUPPORTED: no-threads
 // UNSUPPORTED: c++03, c++11
 
+// Until we drop support for the synchronization library in C++11/14/17
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // <latch>
 
 #include <latch>
diff --git a/libcxx/test/std/thread/thread.latch/try_wait.pass.cpp b/libcxx/test/std/thread/thread.latch/try_wait.pass.cpp
index 70ef2cdf7125..8f354463a869 100644
--- a/libcxx/test/std/thread/thread.latch/try_wait.pass.cpp
+++ b/libcxx/test/std/thread/thread.latch/try_wait.pass.cpp
@@ -9,6 +9,9 @@
 // UNSUPPORTED: no-threads
 // UNSUPPORTED: c++03, c++11
 
+// Until we drop support for the synchronization library in C++11/14/17
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // XFAIL: availability-synchronization_library-missing
 
 // <latch>
diff --git a/libcxx/test/std/thread/thread.semaphore/acquire.pass.cpp b/libcxx/test/std/thread/thread.semaphore/acquire.pass.cpp
index 3f6e3107e8bc..22eed736c6b7 100644
--- a/libcxx/test/std/thread/thread.semaphore/acquire.pass.cpp
+++ b/libcxx/test/std/thread/thread.semaphore/acquire.pass.cpp
@@ -9,6 +9,9 @@
 // UNSUPPORTED: no-threads
 // UNSUPPORTED: c++03, c++11
 
+// Until we drop support for the synchronization library in C++11/14/17
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // XFAIL: availability-synchronization_library-missing
 
 // <semaphore>
diff --git a/libcxx/test/std/thread/thread.semaphore/binary.pass.cpp b/libcxx/test/std/thread/thread.semaphore/binary.pass.cpp
index 111a650b5ea3..c01c78506587 100644
--- a/libcxx/test/std/thread/thread.semaphore/binary.pass.cpp
+++ b/libcxx/test/std/thread/thread.semaphore/binary.pass.cpp
@@ -9,6 +9,9 @@
 // UNSUPPORTED: no-threads
 // UNSUPPORTED: c++03, c++11
 
+// Until we drop support for the synchronization library in C++11/14/17
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // XFAIL: availability-synchronization_library-missing
 
 // <semaphore>
diff --git a/libcxx/test/std/thread/thread.semaphore/ctor.compile.pass.cpp b/libcxx/test/std/thread/thread.semaphore/ctor.compile.pass.cpp
index 28ccc0124d48..dcc298ce11ce 100644
--- a/libcxx/test/std/thread/thread.semaphore/ctor.compile.pass.cpp
+++ b/libcxx/test/std/thread/thread.semaphore/ctor.compile.pass.cpp
@@ -9,6 +9,9 @@
 // UNSUPPORTED: no-threads
 // UNSUPPORTED: c++03, c++11
 
+// Until we drop support for the synchronization library in C++11/14/17
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // <semaphore>
 
 // constexpr explicit counting_semaphore(ptrdiff_t desired);
diff --git a/libcxx/test/std/thread/thread.semaphore/max.pass.cpp b/libcxx/test/std/thread/thread.semaphore/max.pass.cpp
index ca7ad0c92e60..6f3ed5e345e0 100644
--- a/libcxx/test/std/thread/thread.semaphore/max.pass.cpp
+++ b/libcxx/test/std/thread/thread.semaphore/max.pass.cpp
@@ -9,6 +9,9 @@
 // UNSUPPORTED: no-threads
 // UNSUPPORTED: c++03, c++11
 
+// Until we drop support for the synchronization library in C++11/14/17
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // <semaphore>
 
 #include <semaphore>
diff --git a/libcxx/test/std/thread/thread.semaphore/release.pass.cpp b/libcxx/test/std/thread/thread.semaphore/release.pass.cpp
index bf3dd7f7d814..3c4d179e5043 100644
--- a/libcxx/test/std/thread/thread.semaphore/release.pass.cpp
+++ b/libcxx/test/std/thread/thread.semaphore/release.pass.cpp
@@ -9,6 +9,9 @@
 // UNSUPPORTED: no-threads
 // UNSUPPORTED: c++03, c++11
 
+// Until we drop support for the synchronization library in C++11/14/17
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // XFAIL: availability-synchronization_library-missing
 
 // <semaphore>
diff --git a/libcxx/test/std/thread/thread.semaphore/timed.pass.cpp b/libcxx/test/std/thread/thread.semaphore/timed.pass.cpp
index 9fa01fc03590..77f15ece221d 100644
--- a/libcxx/test/std/thread/thread.semaphore/timed.pass.cpp
+++ b/libcxx/test/std/thread/thread.semaphore/timed.pass.cpp
@@ -9,6 +9,9 @@
 // UNSUPPORTED: no-threads
 // UNSUPPORTED: c++03, c++11
 
+// Until we drop support for the synchronization library in C++11/14/17
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // XFAIL: availability-synchronization_library-missing
 
 // <semaphore>
diff --git a/libcxx/test/std/thread/thread.semaphore/try_acquire.pass.cpp b/libcxx/test/std/thread/thread.semaphore/try_acquire.pass.cpp
index 0d0f7792592f..ec159daf87a3 100644
--- a/libcxx/test/std/thread/thread.semaphore/try_acquire.pass.cpp
+++ b/libcxx/test/std/thread/thread.semaphore/try_acquire.pass.cpp
@@ -9,6 +9,9 @@
 // UNSUPPORTED: no-threads
 // UNSUPPORTED: c++03, c++11
 
+// Until we drop support for the synchronization library in C++11/14/17
+// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
+
 // XFAIL: availability-synchronization_library-missing
 
 // <semaphore>
-- 
GitLab


From bd28889732e14ac6baca686c3ec99a82fc9cd89d Mon Sep 17 00:00:00 2001
From: Philip Reames <preames@rivosinc.com>
Date: Tue, 16 Apr 2024 07:54:51 -0700
Subject: [PATCH 416/464] [RISCV] Add coverage for strength reduction of mul
 2^N +/- 3/5/9

---
 llvm/test/CodeGen/RISCV/rv64zba.ll | 60 ++++++++++++++++++++++++++++++
 1 file changed, 60 insertions(+)

diff --git a/llvm/test/CodeGen/RISCV/rv64zba.ll b/llvm/test/CodeGen/RISCV/rv64zba.ll
index 0d1d4838c611..a84b9e5e7962 100644
--- a/llvm/test/CodeGen/RISCV/rv64zba.ll
+++ b/llvm/test/CodeGen/RISCV/rv64zba.ll
@@ -567,6 +567,66 @@ define i64 @mul96(i64 %a) {
   ret i64 %c
 }
 
+define i64 @mul119(i64 %a) {
+; CHECK-LABEL: mul119:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a1, 119
+; CHECK-NEXT:    mul a0, a0, a1
+; CHECK-NEXT:    ret
+  %c = mul i64 %a, 119
+  ret i64 %c
+}
+
+define i64 @mul123(i64 %a) {
+; CHECK-LABEL: mul123:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a1, 123
+; CHECK-NEXT:    mul a0, a0, a1
+; CHECK-NEXT:    ret
+  %c = mul i64 %a, 123
+  ret i64 %c
+}
+
+define i64 @mul125(i64 %a) {
+; CHECK-LABEL: mul125:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a1, 125
+; CHECK-NEXT:    mul a0, a0, a1
+; CHECK-NEXT:    ret
+  %c = mul i64 %a, 125
+  ret i64 %c
+}
+
+define i64 @mul131(i64 %a) {
+; CHECK-LABEL: mul131:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a1, 131
+; CHECK-NEXT:    mul a0, a0, a1
+; CHECK-NEXT:    ret
+  %c = mul i64 %a, 131
+  ret i64 %c
+}
+
+define i64 @mul133(i64 %a) {
+; CHECK-LABEL: mul133:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a1, 133
+; CHECK-NEXT:    mul a0, a0, a1
+; CHECK-NEXT:    ret
+  %c = mul i64 %a, 133
+  ret i64 %c
+}
+
+define i64 @mul137(i64 %a) {
+; CHECK-LABEL: mul137:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a1, 137
+; CHECK-NEXT:    mul a0, a0, a1
+; CHECK-NEXT:    ret
+  %c = mul i64 %a, 137
+  ret i64 %c
+}
+
 define i64 @mul160(i64 %a) {
 ; RV64I-LABEL: mul160:
 ; RV64I:       # %bb.0:
-- 
GitLab


From 1334c034a73b7bf8a7af08be1c33d24a58127c47 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Tue, 16 Apr 2024 10:01:34 -0500
Subject: [PATCH 417/464] [flang] Fix test after
 4078afc6d23e25df6baedad61b224ef86a94d42f

This tests requires the OpenMP runtime to be present, but the way that
the lit config detects it fails when "openmp" is added to RUNTIMES
instead of PROJECTS. This caused the tests to be skipped as unsupported
in local and upstream tests.

The actual bug was a missing word in the message, and putting the check
at the wrong line.
---
 flang/test/Semantics/OpenMP/clause-validity01.f90 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/test/Semantics/OpenMP/clause-validity01.f90 b/flang/test/Semantics/OpenMP/clause-validity01.f90
index 74f154bb0ad6..21b99cb82549 100644
--- a/flang/test/Semantics/OpenMP/clause-validity01.f90
+++ b/flang/test/Semantics/OpenMP/clause-validity01.f90
@@ -342,8 +342,8 @@ use omp_lib
   a = 1.0
   !ERROR: COPYPRIVATE clause is not allowed on the END WORKSHARE directive
   !$omp end workshare nowait copyprivate(a)
+  !ERROR: NOWAIT clause is not allowed on the OMP WORKSHARE directive, use it on OMP END WORKSHARE directive 
   !$omp workshare nowait
-  !ERROR: NOWAIT clause is not allowed on the WORKSHARE directive, use it on OMP END WORKSHARE directive
   !$omp end workshare
   !$omp end parallel
 
-- 
GitLab


From bf1ad1d267b1f911cb9846403d2c3d3250a40870 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 16 Apr 2024 16:01:57 +0100
Subject: [PATCH 418/464] [VectorCombine][X86] Add initial
 shuffle-of-shuffles.ll test cover for #88743

---
 .../VectorCombine/X86/shuffle-of-shuffles.ll  | 51 +++++++++++++++++++
 1 file changed, 51 insertions(+)
 create mode 100644 llvm/test/Transforms/VectorCombine/X86/shuffle-of-shuffles.ll

diff --git a/llvm/test/Transforms/VectorCombine/X86/shuffle-of-shuffles.ll b/llvm/test/Transforms/VectorCombine/X86/shuffle-of-shuffles.ll
new file mode 100644
index 000000000000..b5b5bb997c6c
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/X86/shuffle-of-shuffles.ll
@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt < %s -passes=vector-combine -S -mtriple=x86_64-- -mattr=sse2 | FileCheck %s
+; RUN: opt < %s -passes=vector-combine -S -mtriple=x86_64-- -mattr=avx2 | FileCheck %s
+
+; TODO: fold to identity
+
+define <8 x i32> @concat_extract_subvectors(<8 x i32> %x) {
+; CHECK-LABEL: define <8 x i32> @concat_extract_subvectors(
+; CHECK-SAME: <8 x i32> [[X:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[LO:%.*]] = shufflevector <8 x i32> [[X]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[HI:%.*]] = shufflevector <8 x i32> [[X]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[CONCAT:%.*]] = shufflevector <4 x i32> [[LO]], <4 x i32> [[HI]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <8 x i32> [[CONCAT]]
+;
+  %lo = shufflevector <8 x i32> %x, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %hi = shufflevector <8 x i32> %x, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %concat = shufflevector <4 x i32> %lo, <4 x i32> %hi, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i32> %concat
+}
+
+; negative test - shuffle contains undef
+
+define <8 x i32> @concat_extract_subvectors_undef(<8 x i32> %x) {
+; CHECK-LABEL: define <8 x i32> @concat_extract_subvectors_undef(
+; CHECK-SAME: <8 x i32> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[LO:%.*]] = shufflevector <8 x i32> [[X]], <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 8>
+; CHECK-NEXT:    [[HI:%.*]] = shufflevector <8 x i32> [[X]], <8 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 8>
+; CHECK-NEXT:    [[CONCAT:%.*]] = shufflevector <4 x i32> [[LO]], <4 x i32> [[HI]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <8 x i32> [[CONCAT]]
+;
+  %lo = shufflevector <8 x i32> %x, <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 8>
+  %hi = shufflevector <8 x i32> %x, <8 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 8>
+  %concat = shufflevector <4 x i32> %lo, <4 x i32> %hi, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i32> %concat
+}
+
+; negative test - shuffle contains poision
+
+define <8 x i32> @concat_extract_subvectors_poison(<8 x i32> %x) {
+; CHECK-LABEL: define <8 x i32> @concat_extract_subvectors_poison(
+; CHECK-SAME: <8 x i32> [[X:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT:    [[LO:%.*]] = shufflevector <8 x i32> [[X]], <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 8>
+; CHECK-NEXT:    [[HI:%.*]] = shufflevector <8 x i32> [[X]], <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 8>
+; CHECK-NEXT:    [[CONCAT:%.*]] = shufflevector <4 x i32> [[LO]], <4 x i32> [[HI]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    ret <8 x i32> [[CONCAT]]
+;
+  %lo = shufflevector <8 x i32> %x, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 8>
+  %hi = shufflevector <8 x i32> %x, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 8>
+  %concat = shufflevector <4 x i32> %lo, <4 x i32> %hi, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  ret <8 x i32> %concat
+}
-- 
GitLab


From f8e2ec13a8c6d33cb7b4f37869b4429ddcf43f01 Mon Sep 17 00:00:00 2001
From: Adrian Prantl <aprantl@apple.com>
Date: Tue, 16 Apr 2024 08:29:09 -0700
Subject: [PATCH 419/464] Revert "Add asan tests for libsanitizers. (#88349)"

This reverts commit 82f479ba315a417b6cd01a8c2efdc15c26689f2e due to bot breakage.
---
 lldb/test/API/functionalities/asan/Makefile   |  6 +-
 .../functionalities/asan/TestMemoryHistory.py | 73 +------------------
 .../functionalities/asan/TestReportData.py    | 20 +----
 .../API/functionalities/libsanitizers/util.py |  3 -
 4 files changed, 5 insertions(+), 97 deletions(-)
 delete mode 100644 lldb/test/API/functionalities/libsanitizers/util.py

diff --git a/lldb/test/API/functionalities/asan/Makefile b/lldb/test/API/functionalities/asan/Makefile
index d66696fed707..4913a18d8cc6 100644
--- a/lldb/test/API/functionalities/asan/Makefile
+++ b/lldb/test/API/functionalities/asan/Makefile
@@ -1,8 +1,4 @@
 C_SOURCES := main.c
-asan: CFLAGS_EXTRAS := -fsanitize=address -g -gcolumn-info
-asan: all
-
-libsanitizers: CFLAGS_EXTRAS := -fsanitize=address -fsanitize-stable-abi -g -gcolumn-info
-libsanitizers: all
+CFLAGS_EXTRAS := -fsanitize=address -g -gcolumn-info
 
 include Makefile.rules
diff --git a/lldb/test/API/functionalities/asan/TestMemoryHistory.py b/lldb/test/API/functionalities/asan/TestMemoryHistory.py
index ee7939203ead..00162ae8822c 100644
--- a/lldb/test/API/functionalities/asan/TestMemoryHistory.py
+++ b/lldb/test/API/functionalities/asan/TestMemoryHistory.py
@@ -9,21 +9,15 @@ from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbplatform
 from lldbsuite.test import lldbutil
 
-from functionalities.libsanitizers.util import no_libsanitizers
 
 class AsanTestCase(TestBase):
     @skipIfFreeBSD  # llvm.org/pr21136 runtimes not yet available by default
     @expectedFailureNetBSD
     @skipUnlessAddressSanitizer
     def test(self):
-        self.build(make_targets=["asan"])
+        self.build()
         self.asan_tests()
 
-    @skipIf(oslist=no_match(["macosx"]))
-    def test_libsanitizers_asan(self):
-        self.build(make_targets=["libsanitizers"])
-        self.libsanitizer_tests()
-
     def setUp(self):
         # Call super's setUp().
         TestBase.setUp(self)
@@ -32,71 +26,6 @@ class AsanTestCase(TestBase):
         self.line_free = line_number("main.c", "// free line")
         self.line_breakpoint = line_number("main.c", "// break line")
 
-    # Test line numbers: rdar://126237493
-    def libsanitizer_tests(self):
-        target = self.createTestTarget()
-
-        if no_libsanitizers(self):
-            self.skipTest("libsanitizers not found")
-
-        self.runCmd(
-            "env SanitizersAddress=1 MallocSanitizerZone=1 MallocSecureAllocator=0"
-        )
-
-        self.runCmd("run")
-
-        # In libsanitizers, memory history is not supported until a report has been generated
-        self.expect(
-            "thread list",
-            "Process should be stopped due to ASan report",
-            substrs=["stopped", "stop reason = Use of deallocated memory"],
-        )
-
-        # test the 'memory history' command
-        self.expect(
-            "memory history 'pointer'",
-            substrs=[
-                "Memory deallocated by Thread",
-                "a.out`f2",
-                "main.c",
-                "Memory allocated by Thread",
-                "a.out`f1",
-                "main.c",
-            ],
-        )
-
-        # do the same using SB API
-        process = self.dbg.GetSelectedTarget().process
-        val = (
-            process.GetSelectedThread().GetSelectedFrame().EvaluateExpression("pointer")
-        )
-        addr = val.GetValueAsUnsigned()
-        threads = process.GetHistoryThreads(addr)
-        self.assertEqual(threads.GetSize(), 2)
-
-        history_thread = threads.GetThreadAtIndex(0)
-        self.assertTrue(history_thread.num_frames >= 2)
-        self.assertEqual(
-            history_thread.frames[1].GetLineEntry().GetFileSpec().GetFilename(),
-            "main.c",
-        )
-
-        history_thread = threads.GetThreadAtIndex(1)
-        self.assertTrue(history_thread.num_frames >= 2)
-        self.assertEqual(
-            history_thread.frames[1].GetLineEntry().GetFileSpec().GetFilename(),
-            "main.c",
-        )
-
-        # let's free the container (SBThreadCollection) and see if the
-        # SBThreads still live
-        threads = None
-        self.assertTrue(history_thread.num_frames >= 2)
-        self.assertEqual(
-            history_thread.frames[1].GetLineEntry().GetFileSpec().GetFilename(),
-            "main.c",
-        )
-
     def asan_tests(self):
         target = self.createTestTarget()
 
diff --git a/lldb/test/API/functionalities/asan/TestReportData.py b/lldb/test/API/functionalities/asan/TestReportData.py
index de0c1206a57a..543c5fe66a20 100644
--- a/lldb/test/API/functionalities/asan/TestReportData.py
+++ b/lldb/test/API/functionalities/asan/TestReportData.py
@@ -9,7 +9,6 @@ from lldbsuite.test.decorators import *
 from lldbsuite.test.lldbtest import *
 from lldbsuite.test import lldbutil
 
-from functionalities.libsanitizers.util import no_libsanitizers
 
 class AsanTestReportDataCase(TestBase):
     @skipIfFreeBSD  # llvm.org/pr21136 runtimes not yet available by default
@@ -17,14 +16,9 @@ class AsanTestReportDataCase(TestBase):
     @skipUnlessAddressSanitizer
     @skipIf(archs=["i386"], bugnumber="llvm.org/PR36710")
     def test(self):
-        self.build(make_targets=["asan"])
+        self.build()
         self.asan_tests()
 
-    @skipIf(oslist=no_match(["macosx"]))
-    def test_libsanitizers_asan(self):
-        self.build(make_targets=["libsanitizers"])
-        self.asan_tests(libsanitizers=True)
-
     def setUp(self):
         # Call super's setUp().
         TestBase.setUp(self)
@@ -35,18 +29,10 @@ class AsanTestReportDataCase(TestBase):
         self.line_crash = line_number("main.c", "// BOOM line")
         self.col_crash = 16
 
-    def asan_tests(self, libsanitizers=False):
+    def asan_tests(self):
         target = self.createTestTarget()
 
-        if libsanitizers and no_libsanitizers(self):
-            self.skipTest("libsanitizers not found")
-
-        if libsanitizers:
-            self.runCmd(
-                "env SanitizersAddress=1 MallocSanitizerZone=1 MallocSecureAllocator=0"
-            )
-        else:
-            self.registerSanitizerLibrariesWithTarget(target)
+        self.registerSanitizerLibrariesWithTarget(target)
 
         self.runCmd("run")
 
diff --git a/lldb/test/API/functionalities/libsanitizers/util.py b/lldb/test/API/functionalities/libsanitizers/util.py
deleted file mode 100644
index ad68541aba8d..000000000000
--- a/lldb/test/API/functionalities/libsanitizers/util.py
+++ /dev/null
@@ -1,3 +0,0 @@
-def no_libsanitizers(testbase):
-    testbase.runCmd("image list libsystem_sanitizers.dylib", check=False)
-    return not "libsystem_sanitizers.dylib" in testbase.res.GetOutput()
-- 
GitLab


From 8cee94e989b5bf6fb6455087d48eb6c6e0e23c54 Mon Sep 17 00:00:00 2001
From: Harald van Dijk <harald.vandijk@codeplay.com>
Date: Tue, 16 Apr 2024 16:32:57 +0100
Subject: [PATCH 420/464] [RISCV] Fix obvious copy paste error.

CASE_VFMA_OPCODE_VV and CASE_VFMA_CHANGE_OPCODE_VV need to match up if we are
are to avoid "Unexpected opcode" errors, but in CASE_VFMA_CHANGE_OPCODE_VV,
CASE_VFMA_CHANGE_OPCODE_LMULS_MF2 had mistakenly been used instead of
CASE_VFMA_CHANGE_OPCODE_LMULS_MF4.
---
 llvm/lib/Target/RISCV/RISCVInstrInfo.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
index 668062c8d33f..14b5cbea7172 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -3016,7 +3016,7 @@ bool RISCVInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
   CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE, SEW)
 
 #define CASE_VFMA_CHANGE_OPCODE_VV(OLDOP, NEWOP)                               \
-  CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VV, E16)                     \
+  CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VV, E16)                     \
   CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VV, E32)                     \
   CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VV, E64)
 
-- 
GitLab


From 51b42b762112f2e77d032efd16fa0d9d31cde494 Mon Sep 17 00:00:00 2001
From: Mehdi Amini <joker.eph@gmail.com>
Date: Tue, 16 Apr 2024 17:45:02 +0200
Subject: [PATCH 421/464] Revert "[Sema] Mark alias/ifunc targets used and
 consider mangled names" (#88919)

Reverts llvm/llvm-project#87130

Bot is broken with clang crash:
https://lab.llvm.org/buildbot/#/builders/272/builds/14063/steps/6/logs/stdio
---
 clang/lib/Sema/CMakeLists.txt                 |  1 -
 clang/lib/Sema/SemaDeclAttr.cpp               | 44 +++++--------------
 clang/test/AST/ast-dump-attr-json.cpp         |  1 -
 clang/test/Sema/alias-unused-win.cpp          |  2 +-
 clang/test/Sema/alias-unused.cpp              | 16 +++----
 .../llvm-project-overlay/clang/BUILD.bazel    |  1 -
 6 files changed, 19 insertions(+), 46 deletions(-)

diff --git a/clang/lib/Sema/CMakeLists.txt b/clang/lib/Sema/CMakeLists.txt
index a96439df6642..ab3b813a9ccd 100644
--- a/clang/lib/Sema/CMakeLists.txt
+++ b/clang/lib/Sema/CMakeLists.txt
@@ -1,6 +1,5 @@
 set(LLVM_LINK_COMPONENTS
   Core
-  Demangle
   FrontendHLSL
   FrontendOpenMP
   MC
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index d26f130b5774..b7b1fbc625a1 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -45,7 +45,6 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/STLForwardCompat.h"
 #include "llvm/ADT/StringExtras.h"
-#include "llvm/Demangle/Demangle.h"
 #include "llvm/IR/Assumptions.h"
 #include "llvm/MC/MCSectionMachO.h"
 #include "llvm/Support/Error.h"
@@ -1984,36 +1983,6 @@ static void handleWeakRefAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   D->addAttr(::new (S.Context) WeakRefAttr(S.Context, AL));
 }
 
-// Mark alias/ifunc target as used. Due to name mangling, we look up the
-// demangled name ignoring parameters (not supported by microsoftDemangle
-// https://github.com/llvm/llvm-project/issues/88825). This should handle the
-// majority of use cases while leaving namespace scope names unmarked.
-static void markUsedForAliasOrIfunc(Sema &S, Decl *D, const ParsedAttr &AL,
-                                    StringRef Str) {
-  std::unique_ptr<char, llvm::FreeDeleter> Demangled;
-  if (S.getASTContext().getCXXABIKind() != TargetCXXABI::Microsoft)
-    Demangled.reset(llvm::itaniumDemangle(Str, /*ParseParams=*/false));
-  std::unique_ptr<MangleContext> MC(S.Context.createMangleContext());
-  SmallString<256> Name;
-
-  const DeclarationNameInfo Target(
-      &S.Context.Idents.get(Demangled ? Demangled.get() : Str), AL.getLoc());
-  LookupResult LR(S, Target, Sema::LookupOrdinaryName);
-  if (S.LookupName(LR, S.TUScope)) {
-    for (NamedDecl *ND : LR) {
-      if (MC->shouldMangleDeclName(ND)) {
-        llvm::raw_svector_ostream Out(Name);
-        Name.clear();
-        MC->mangleName(GlobalDecl(ND), Out);
-      } else {
-        Name = ND->getIdentifier()->getName();
-      }
-      if (Name == Str)
-        ND->markUsed(S.Context);
-    }
-  }
-}
-
 static void handleIFuncAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   StringRef Str;
   if (!S.checkStringLiteralArgumentAttr(AL, 0, Str))
@@ -2026,7 +1995,6 @@ static void handleIFuncAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
     return;
   }
 
-  markUsedForAliasOrIfunc(S, D, AL, Str);
   D->addAttr(::new (S.Context) IFuncAttr(S.Context, AL, Str));
 }
 
@@ -2061,7 +2029,17 @@ static void handleAliasAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
     }
   }
 
-  markUsedForAliasOrIfunc(S, D, AL, Str);
+  // Mark target used to prevent unneeded-internal-declaration warnings.
+  if (!S.LangOpts.CPlusPlus) {
+    // FIXME: demangle Str for C++, as the attribute refers to the mangled
+    // linkage name, not the pre-mangled identifier.
+    const DeclarationNameInfo target(&S.Context.Idents.get(Str), AL.getLoc());
+    LookupResult LR(S, target, Sema::LookupOrdinaryName);
+    if (S.LookupQualifiedName(LR, S.getCurLexicalContext()))
+      for (NamedDecl *ND : LR)
+        ND->markUsed(S.Context);
+  }
+
   D->addAttr(::new (S.Context) AliasAttr(S.Context, AL, Str));
 }
 
diff --git a/clang/test/AST/ast-dump-attr-json.cpp b/clang/test/AST/ast-dump-attr-json.cpp
index 883e584bfedf..051c2956abfd 100644
--- a/clang/test/AST/ast-dump-attr-json.cpp
+++ b/clang/test/AST/ast-dump-attr-json.cpp
@@ -46,7 +46,6 @@ __thread __attribute__ ((tls_model ("local-exec"))) int tls_model_var;
 // CHECK-NEXT:    "tokLen": 11
 // CHECK-NEXT:   }
 // CHECK-NEXT:  },
-// CHECK-NEXT:  "isUsed": true,
 // CHECK-NEXT:  "name": "global_decl",
 // CHECK-NEXT:  "mangledName": "global_decl",
 // CHECK-NEXT:  "type": {
diff --git a/clang/test/Sema/alias-unused-win.cpp b/clang/test/Sema/alias-unused-win.cpp
index 97d57a3bbd1e..47c96d411751 100644
--- a/clang/test/Sema/alias-unused-win.cpp
+++ b/clang/test/Sema/alias-unused-win.cpp
@@ -7,7 +7,7 @@ extern "C" {
 static int f(void) { return 42; } // cxx-warning{{unused function 'f'}}
 int g(void) __attribute__((alias("f")));
 
-static int foo [] = { 42, 0xDEAD };
+static int foo [] = { 42, 0xDEAD }; // cxx-warning{{variable 'foo' is not needed and will not be emitted}}
 extern typeof(foo) bar __attribute__((unused, alias("foo")));
 
 static int __attribute__((overloadable)) f0(int x) { return x; } // expected-warning{{unused function 'f0'}}
diff --git a/clang/test/Sema/alias-unused.cpp b/clang/test/Sema/alias-unused.cpp
index c0b541c880e5..dc8e46f072d7 100644
--- a/clang/test/Sema/alias-unused.cpp
+++ b/clang/test/Sema/alias-unused.cpp
@@ -14,26 +14,24 @@ extern typeof(foo) bar __attribute__((unused, alias("foo")));
 /// We report a warning in C++ mode because the internal linkage `resolver` gets
 /// mangled as it does not have a language linkage. GCC does not mangle
 /// `resolver` or report a warning.
-static int (*resolver(void))(void) { return f; } // cxx-warning{{unused function 'resolver'}}
+static int (*resolver(void))(void) { return f; } // expected-warning{{unused function 'resolver'}}
 int ifunc(void) __attribute__((ifunc("resolver")));
 
-static int __attribute__((overloadable)) f0(int x) { return x; }
+static int __attribute__((overloadable)) f0(int x) { return x; } // expected-warning{{unused function 'f0'}}
 static float __attribute__((overloadable)) f0(float x) { return x; } // expected-warning{{unused function 'f0'}}
 int g0(void) __attribute__((alias("_ZL2f0i")));
 
 #ifdef __cplusplus
-static int f1() { return 42; }
+static int f1() { return 42; } // expected-warning{{unused function 'f1'}}
 int g1(void) __attribute__((alias("_ZL2f1v")));
 }
 
-/// We demangle alias/ifunc target and mark all found functions as used.
-
-static int f2(int) { return 42; } // cxx-warning{{unused function 'f2'}}
-static int f2() { return 42; }
+static int f2(int) { return 42; } // expected-warning{{unused function 'f2'}}
+static int f2() { return 42; } // expected-warning{{unused function 'f2'}}
 int g2() __attribute__((alias("_ZL2f2v")));
 
-static int (*resolver1())() { return f; } // cxx-warning{{unused function 'resolver1'}}
-static int (*resolver1(int))() { return f; }
+static int (*resolver1())() { return f; } // expected-warning{{unused function 'resolver1'}}
+static int (*resolver1(int))() { return f; } // expected-warning{{unused function 'resolver1'}}
 int ifunc1() __attribute__((ifunc("_ZL9resolver1i")));
 
 /// TODO: We should report "unused function" for f3(int).
diff --git a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
index 725ac6bb3812..c2f77e3abca0 100644
--- a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
@@ -1136,7 +1136,6 @@ cc_library(
         "//llvm:AllTargetsAsmParsers",
         "//llvm:AllTargetsCodeGens",
         "//llvm:Core",
-        "//llvm:Demangle",
         "//llvm:FrontendHLSL",
         "//llvm:FrontendOpenMP",
         "//llvm:MC",
-- 
GitLab


From 9d111286322ec99b32467eef3aeec6b588c49f18 Mon Sep 17 00:00:00 2001
From: Fraser Cormack <fraser@codeplay.com>
Date: Tue, 16 Apr 2024 16:48:59 +0100
Subject: [PATCH 422/464] [libclc] Improve clarity of CMake foreach. NFC.

Should be a bit easier to read.
---
 libclc/CMakeLists.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/libclc/CMakeLists.txt b/libclc/CMakeLists.txt
index ed2764847e70..f605c3bbbe9d 100644
--- a/libclc/CMakeLists.txt
+++ b/libclc/CMakeLists.txt
@@ -50,7 +50,7 @@ if( LIBCLC_STANDALONE_BUILD OR CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DI
   endif()
 
   # Import required tools as targets
-  foreach( tool clang llvm-as llvm-link opt )
+  foreach( tool IN ITEMS clang llvm-as llvm-link opt )
     find_program( LLVM_TOOL_${tool} ${tool} PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH )
     add_executable( libclc::${tool} IMPORTED GLOBAL )
     set_target_properties( libclc::${tool} PROPERTIES IMPORTED_LOCATION ${LLVM_TOOL_${tool}} )
@@ -68,7 +68,7 @@ else()
     message(FATAL_ERROR "Clang is not enabled, but is required to build libclc in-tree")
   endif()
 
-  foreach( tool clang llvm-as llvm-link opt )
+  foreach( tool IN ITEMS clang llvm-as llvm-link opt )
     add_executable(libclc::${tool} ALIAS ${tool})
   endforeach()
 endif()
@@ -181,7 +181,7 @@ install( FILES ${CMAKE_CURRENT_BINARY_DIR}/libclc.pc DESTINATION "${CMAKE_INSTAL
 install( DIRECTORY generic/include/clc DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" )
 
 if( ENABLE_RUNTIME_SUBNORMAL )
-  foreach( file subnormal_use_default subnormal_disable )
+  foreach( file IN ITEMS subnormal_use_default subnormal_disable )
     link_bc(
        TARGET ${file}
        INPUTS ${PROJECT_SOURCE_DIR}/generic/lib/${file}.ll
-- 
GitLab


From 2704ebaf2885a16155ab7144f8dd0dd459d77089 Mon Sep 17 00:00:00 2001
From: Slava Zakharin <szakharin@nvidia.com>
Date: Tue, 16 Apr 2024 08:53:12 -0700
Subject: [PATCH 423/464] [flang][runtime] Create CUDA PTX OBJECT library
 target for F18 runtime CUDA build. (#88821)

This is to experiment with distributing F18 runtime CUDA library
in the form of a pure PTX library. The change is under
FLANG_EXPERIMENTAL_CUDA_RUNTIME CMake control.
---
 flang/cmake/modules/AddFlangOffloadRuntime.cmake | 6 +++++-
 flang/lib/Decimal/CMakeLists.txt                 | 2 +-
 flang/runtime/CMakeLists.txt                     | 2 +-
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/flang/cmake/modules/AddFlangOffloadRuntime.cmake b/flang/cmake/modules/AddFlangOffloadRuntime.cmake
index 6fb6213e90fc..e34d3851187a 100644
--- a/flang/cmake/modules/AddFlangOffloadRuntime.cmake
+++ b/flang/cmake/modules/AddFlangOffloadRuntime.cmake
@@ -10,7 +10,7 @@ set(FLANG_EXPERIMENTAL_OMP_OFFLOAD_BUILD "off" CACHE STRING
 set(FLANG_OMP_DEVICE_ARCHITECTURES "all" CACHE STRING
   "List of OpenMP device architectures to be used to compile the Fortran runtime (e.g. 'gfx1103;sm_90')")
 
-macro(enable_cuda_compilation files)
+macro(enable_cuda_compilation name files)
   if (FLANG_EXPERIMENTAL_CUDA_RUNTIME)
     if (BUILD_SHARED_LIBS)
       message(FATAL_ERROR
@@ -52,6 +52,10 @@ macro(enable_cuda_compilation files)
       include_directories(AFTER ${FLANG_LIBCUDACXX_PATH}/include)
       add_compile_definitions(RT_USE_LIBCUDACXX=1)
     endif()
+
+    # Add an OBJECT library consisting of CUDA PTX.
+    llvm_add_library(${name}PTX OBJECT PARTIAL_SOURCES_INTENDED ${files})
+    set_property(TARGET obj.${name}PTX PROPERTY CUDA_PTX_COMPILATION ON)
   endif()
 endmacro()
 
diff --git a/flang/lib/Decimal/CMakeLists.txt b/flang/lib/Decimal/CMakeLists.txt
index 3d562b8e3ce1..880b190f1c58 100644
--- a/flang/lib/Decimal/CMakeLists.txt
+++ b/flang/lib/Decimal/CMakeLists.txt
@@ -55,7 +55,7 @@ set(sources
 )
 
 include(AddFlangOffloadRuntime)
-enable_cuda_compilation("${sources}")
+enable_cuda_compilation(FortranDecimal "${sources}")
 enable_omp_offload_compilation("${sources}")
 
 add_flang_library(FortranDecimal INSTALL_WITH_TOOLCHAIN ${sources})
diff --git a/flang/runtime/CMakeLists.txt b/flang/runtime/CMakeLists.txt
index 2a65a22ab674..bdd0e07bbfd4 100644
--- a/flang/runtime/CMakeLists.txt
+++ b/flang/runtime/CMakeLists.txt
@@ -224,7 +224,7 @@ set(supported_files
   utf.cpp
   )
 
-enable_cuda_compilation("${supported_files}")
+enable_cuda_compilation(FortranRuntime "${supported_files}")
 enable_omp_offload_compilation("${supported_files}")
 
 if (NOT TARGET FortranFloat128Math)
-- 
GitLab


From a79783d7ad00c4c10c30f637f4bf13551e47f3dd Mon Sep 17 00:00:00 2001
From: Robin Caloudis <robin.caloudis@gmx.de>
Date: Tue, 16 Apr 2024 17:58:11 +0200
Subject: [PATCH 424/464] [libc][fenv] Use proxy header (#88787)

Include types `fexcept_t` and `fenv_t ` from corresponding proxy
headers, as they are available since
https://github.com/llvm/llvm-project/pull/88467.
---
 libc/src/fenv/fegetexceptflag.h | 2 +-
 libc/src/fenv/fesetexceptflag.h | 2 +-
 libc/src/fenv/feupdateenv.cpp   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/libc/src/fenv/fegetexceptflag.h b/libc/src/fenv/fegetexceptflag.h
index ad72161e536f..fcb9598658d4 100644
--- a/libc/src/fenv/fegetexceptflag.h
+++ b/libc/src/fenv/fegetexceptflag.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_LIBC_SRC_FENV_FEGETEXCEPTFLAG_H
 #define LLVM_LIBC_SRC_FENV_FEGETEXCEPTFLAG_H
 
-#include <fenv.h>
+#include "hdr/types/fexcept_t.h"
 
 namespace LIBC_NAMESPACE {
 
diff --git a/libc/src/fenv/fesetexceptflag.h b/libc/src/fenv/fesetexceptflag.h
index 15e62eda1b84..a018358dc9df 100644
--- a/libc/src/fenv/fesetexceptflag.h
+++ b/libc/src/fenv/fesetexceptflag.h
@@ -9,7 +9,7 @@
 #ifndef LLVM_LIBC_SRC_FENV_FESETEXCEPTFLAG_H
 #define LLVM_LIBC_SRC_FENV_FESETEXCEPTFLAG_H
 
-#include <fenv.h>
+#include "hdr/types/fexcept_t.h"
 
 namespace LIBC_NAMESPACE {
 
diff --git a/libc/src/fenv/feupdateenv.cpp b/libc/src/fenv/feupdateenv.cpp
index 7e81b9476da9..066486353815 100644
--- a/libc/src/fenv/feupdateenv.cpp
+++ b/libc/src/fenv/feupdateenv.cpp
@@ -10,7 +10,7 @@
 #include "src/__support/FPUtil/FEnvImpl.h"
 #include "src/__support/common.h"
 
-#include <fenv.h>
+#include "hdr/types/fenv_t.h"
 
 namespace LIBC_NAMESPACE {
 
-- 
GitLab


From 38895e6578f7728cfb3d41d0880a0e9b358d9afd Mon Sep 17 00:00:00 2001
From: Robin Caloudis <robin.caloudis@gmx.de>
Date: Tue, 16 Apr 2024 17:58:56 +0200
Subject: [PATCH 425/464] [libc][fenv] Remove unnecessary dependencies (#88788)

Remove the fenv macro dependency from the CMake files as the underlying targets
do not make use of it. Note that we do not have to worry about
[corresponding Bazel targets](https://github.com/llvm/llvm-project/blob/main/utils/bazel/llvm-project-overlay/libc/BUILD.bazel#L1138-L1288),
as they look good.
---
 libc/src/fenv/CMakeLists.txt | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/libc/src/fenv/CMakeLists.txt b/libc/src/fenv/CMakeLists.txt
index a28a7ca4c2d8..17e994741206 100644
--- a/libc/src/fenv/CMakeLists.txt
+++ b/libc/src/fenv/CMakeLists.txt
@@ -17,7 +17,6 @@ add_entrypoint_object(
   HDRS
     fesetround.h
   DEPENDS
-    libc.hdr.fenv_macros
     libc.src.__support.FPUtil.fenv_impl
   COMPILE_OPTIONS
     -O2
@@ -30,7 +29,6 @@ add_entrypoint_object(
   HDRS
     feclearexcept.h
   DEPENDS
-    libc.hdr.fenv_macros
     libc.src.__support.FPUtil.fenv_impl
   COMPILE_OPTIONS
     -O2
@@ -43,7 +41,6 @@ add_entrypoint_object(
   HDRS
     feraiseexcept.h
   DEPENDS
-    libc.hdr.fenv_macros
     libc.src.__support.FPUtil.fenv_impl
   COMPILE_OPTIONS
     -O2
@@ -56,7 +53,6 @@ add_entrypoint_object(
   HDRS
     fetestexcept.h
   DEPENDS
-    libc.hdr.fenv_macros
     libc.src.__support.FPUtil.fenv_impl
   COMPILE_OPTIONS
     -O2
@@ -69,7 +65,6 @@ add_entrypoint_object(
   HDRS
     fegetenv.h
   DEPENDS
-    libc.hdr.fenv_macros
     libc.hdr.types.fenv_t
     libc.src.__support.FPUtil.fenv_impl
   COMPILE_OPTIONS
@@ -83,7 +78,6 @@ add_entrypoint_object(
   HDRS
     fesetenv.h
   DEPENDS
-    libc.hdr.fenv_macros
     libc.hdr.types.fenv_t
     libc.src.__support.FPUtil.fenv_impl
   COMPILE_OPTIONS
@@ -111,7 +105,6 @@ add_entrypoint_object(
   HDRS
     fesetexcept.h
   DEPENDS
-    libc.hdr.fenv_macros
     libc.src.__support.FPUtil.fenv_impl
   COMPILE_OPTIONS
     -O2
@@ -166,7 +159,6 @@ add_entrypoint_object(
   HDRS
     feenableexcept.h
   DEPENDS
-    libc.hdr.fenv_macros
     libc.src.__support.FPUtil.fenv_impl
   COMPILE_OPTIONS
     -O2
@@ -179,7 +171,6 @@ add_entrypoint_object(
   HDRS
     fedisableexcept.h
   DEPENDS
-    libc.hdr.fenv_macros
     libc.src.__support.FPUtil.fenv_impl
   COMPILE_OPTIONS
     -O2
@@ -192,7 +183,6 @@ add_entrypoint_object(
   HDRS
     fegetexcept.h
   DEPENDS
-    libc.hdr.fenv_macros
     libc.src.__support.FPUtil.fenv_impl
   COMPILE_OPTIONS
     -O2
-- 
GitLab


From 1c6b0f779f66494cb597884c1a52e377bde4bc54 Mon Sep 17 00:00:00 2001
From: Stephen Tozer <stephen.tozer@sony.com>
Date: Tue, 16 Apr 2024 17:07:46 +0100
Subject: [PATCH 426/464] [RemoveDI] Add support for debug records to debugify
 (#87383)

This patch changes debugify to support debug variable records, and
subsequently to no longer convert modules automatically to intrinsics
when entering debugify.
---
 llvm/lib/CodeGen/MachineDebugify.cpp          |  18 ++-
 llvm/lib/Transforms/Utils/Debugify.cpp        | 111 +++++++-----------
 .../check-line-and-variables-x.mir            |   1 +
 .../MIRDebugify/check-line-and-variables.ll   |   1 +
 .../MIRDebugify/check-line-and-variables.mir  |   2 +
 .../MIRDebugify/locations-and-values.mir      |   4 +
 .../MIRDebugify/multifunction-module.mir      |   3 +-
 .../DebugInfo/debugify-bogus-dbg-value.ll     |   1 +
 llvm/test/DebugInfo/debugify-each.ll          |  34 ++++++
 llvm/test/DebugInfo/debugify-export.ll        |   3 +
 llvm/test/DebugInfo/debugify-ignore-phi.ll    |   1 +
 .../debugify-original-no-dbg-info.ll          |   1 +
 .../debugify-report-missing-locs-only.ll      |   1 +
 llvm/test/DebugInfo/debugify.ll               |  27 +++++
 llvm/test/DebugInfo/pr37964.ll                |   1 +
 .../test/DebugInfo/salvage-cast-debug-info.ll |   2 +-
 llvm/test/DebugInfo/verify-di-preserve.ll     |   4 +-
 17 files changed, 142 insertions(+), 73 deletions(-)

diff --git a/llvm/lib/CodeGen/MachineDebugify.cpp b/llvm/lib/CodeGen/MachineDebugify.cpp
index c264e199cf47..bffdd51bfbca 100644
--- a/llvm/lib/CodeGen/MachineDebugify.cpp
+++ b/llvm/lib/CodeGen/MachineDebugify.cpp
@@ -65,6 +65,7 @@ bool applyDebugifyMetadataToMachineFunction(MachineModuleInfo &MMI,
   // all the others.
   Function *DbgValF = M.getFunction("llvm.dbg.value");
   DbgValueInst *EarliestDVI = nullptr;
+  DbgVariableRecord *EarliestDVR = nullptr;
   DenseMap<unsigned, DILocalVariable *> Line2Var;
   DIExpression *Expr = nullptr;
   if (DbgValF) {
@@ -80,6 +81,20 @@ bool applyDebugifyMetadataToMachineFunction(MachineModuleInfo &MMI,
       Expr = DVI->getExpression();
     }
   }
+  for (BasicBlock &BB : F) {
+    for (Instruction &I : BB) {
+      for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange())) {
+        if (!DVR.isDbgValue())
+          continue;
+        unsigned Line = DVR.getDebugLoc().getLine();
+        assert(Line != 0 && "debugify should not insert line 0 locations");
+        Line2Var[Line] = DVR.getVariable();
+        if (!EarliestDVR || Line < EarliestDVR->getDebugLoc().getLine())
+          EarliestDVR = &DVR;
+        Expr = DVR.getExpression();
+      }
+    }
+  }
   if (Line2Var.empty())
     return true;
 
@@ -109,7 +124,8 @@ bool applyDebugifyMetadataToMachineFunction(MachineModuleInfo &MMI,
       // Find a suitable local variable for the DBG_VALUE.
       unsigned Line = MI.getDebugLoc().getLine();
       if (!Line2Var.count(Line))
-        Line = EarliestDVI->getDebugLoc().getLine();
+        Line = EarliestDVI ? EarliestDVI->getDebugLoc().getLine()
+                           : EarliestDVR->getDebugLoc().getLine();
       DILocalVariable *LocalVar = Line2Var[Line];
       assert(LocalVar && "No variable for current line?");
       VarSet.insert(LocalVar);
diff --git a/llvm/lib/Transforms/Utils/Debugify.cpp b/llvm/lib/Transforms/Utils/Debugify.cpp
index 200bad22148f..fcc82eadac36 100644
--- a/llvm/lib/Transforms/Utils/Debugify.cpp
+++ b/llvm/lib/Transforms/Utils/Debugify.cpp
@@ -87,10 +87,6 @@ bool llvm::applyDebugifyMetadata(
     return false;
   }
 
-  bool NewDebugMode = M.IsNewDbgInfoFormat;
-  if (NewDebugMode)
-    M.convertFromNewDbgValues();
-
   DIBuilder DIB(M);
   LLVMContext &Ctx = M.getContext();
   auto *Int32Ty = Type::getInt32Ty(Ctx);
@@ -214,9 +210,6 @@ bool llvm::applyDebugifyMetadata(
   if (!M.getModuleFlag(DIVersionKey))
     M.addModuleFlag(Module::Warning, DIVersionKey, DEBUG_METADATA_VERSION);
 
-  if (NewDebugMode)
-    M.convertToNewDbgValues();
-
   return true;
 }
 
@@ -311,10 +304,6 @@ bool llvm::collectDebugInfoMetadata(Module &M,
     return false;
   }
 
-  bool NewDebugMode = M.IsNewDbgInfoFormat;
-  if (NewDebugMode)
-    M.convertFromNewDbgValues();
-
   uint64_t FunctionsCnt = DebugInfoBeforePass.DIFunctions.size();
   // Visit each instruction.
   for (Function &F : Functions) {
@@ -349,20 +338,23 @@ bool llvm::collectDebugInfoMetadata(Module &M,
 
         // Cllect dbg.values and dbg.declare.
         if (DebugifyLevel > Level::Locations) {
-          if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&I)) {
+          auto HandleDbgVariable = [&](auto *DbgVar) {
             if (!SP)
-              continue;
+              return;
             // Skip inlined variables.
-            if (I.getDebugLoc().getInlinedAt())
-              continue;
+            if (DbgVar->getDebugLoc().getInlinedAt())
+              return;
             // Skip undef values.
-            if (DVI->isKillLocation())
-              continue;
+            if (DbgVar->isKillLocation())
+              return;
 
-            auto *Var = DVI->getVariable();
+            auto *Var = DbgVar->getVariable();
             DebugInfoBeforePass.DIVariables[Var]++;
-            continue;
-          }
+          };
+          for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange()))
+            HandleDbgVariable(&DVR);
+          if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&I))
+            HandleDbgVariable(DVI);
         }
 
         // Skip debug instructions other than dbg.value and dbg.declare.
@@ -379,9 +371,6 @@ bool llvm::collectDebugInfoMetadata(Module &M,
     }
   }
 
-  if (NewDebugMode)
-    M.convertToNewDbgValues();
-
   return true;
 }
 
@@ -561,10 +550,6 @@ bool llvm::checkDebugInfoMetadata(Module &M,
     return false;
   }
 
-  bool NewDebugMode = M.IsNewDbgInfoFormat;
-  if (NewDebugMode)
-    M.convertFromNewDbgValues();
-
   // Map the debug info holding DIs after a pass.
   DebugInfoPerPass DebugInfoAfterPass;
 
@@ -599,20 +584,23 @@ bool llvm::checkDebugInfoMetadata(Module &M,
 
         // Collect dbg.values and dbg.declares.
         if (DebugifyLevel > Level::Locations) {
-          if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&I)) {
+          auto HandleDbgVariable = [&](auto *DbgVar) {
             if (!SP)
-              continue;
+              return;
             // Skip inlined variables.
-            if (I.getDebugLoc().getInlinedAt())
-              continue;
+            if (DbgVar->getDebugLoc().getInlinedAt())
+              return;
             // Skip undef values.
-            if (DVI->isKillLocation())
-              continue;
+            if (DbgVar->isKillLocation())
+              return;
 
-            auto *Var = DVI->getVariable();
+            auto *Var = DbgVar->getVariable();
             DebugInfoAfterPass.DIVariables[Var]++;
-            continue;
-          }
+          };
+          for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange()))
+            HandleDbgVariable(&DVR);
+          if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&I))
+            HandleDbgVariable(DVI);
         }
 
         // Skip debug instructions other than dbg.value and dbg.declare.
@@ -675,16 +663,14 @@ bool llvm::checkDebugInfoMetadata(Module &M,
   // the debugging information from the previous pass.
   DebugInfoBeforePass = DebugInfoAfterPass;
 
-  if (NewDebugMode)
-    M.convertToNewDbgValues();
-
   LLVM_DEBUG(dbgs() << "\n\n");
   return Result;
 }
 
 namespace {
-/// Return true if a mis-sized diagnostic is issued for \p DVI.
-bool diagnoseMisSizedDbgValue(Module &M, DbgValueInst *DVI) {
+/// Return true if a mis-sized diagnostic is issued for \p DbgVal.
+template <typename DbgValTy>
+bool diagnoseMisSizedDbgValue(Module &M, DbgValTy *DbgVal) {
   // The size of a dbg.value's value operand should match the size of the
   // variable it corresponds to.
   //
@@ -693,22 +679,22 @@ bool diagnoseMisSizedDbgValue(Module &M, DbgValueInst *DVI) {
 
   // For now, don't try to interpret anything more complicated than an empty
   // DIExpression. Eventually we should try to handle OP_deref and fragments.
-  if (DVI->getExpression()->getNumElements())
+  if (DbgVal->getExpression()->getNumElements())
     return false;
 
-  Value *V = DVI->getVariableLocationOp(0);
+  Value *V = DbgVal->getVariableLocationOp(0);
   if (!V)
     return false;
 
   Type *Ty = V->getType();
   uint64_t ValueOperandSize = getAllocSizeInBits(M, Ty);
-  std::optional<uint64_t> DbgVarSize = DVI->getFragmentSizeInBits();
+  std::optional<uint64_t> DbgVarSize = DbgVal->getFragmentSizeInBits();
   if (!ValueOperandSize || !DbgVarSize)
     return false;
 
   bool HasBadSize = false;
   if (Ty->isIntegerTy()) {
-    auto Signedness = DVI->getVariable()->getSignedness();
+    auto Signedness = DbgVal->getVariable()->getSignedness();
     if (Signedness && *Signedness == DIBasicType::Signedness::Signed)
       HasBadSize = ValueOperandSize < *DbgVarSize;
   } else {
@@ -718,7 +704,7 @@ bool diagnoseMisSizedDbgValue(Module &M, DbgValueInst *DVI) {
   if (HasBadSize) {
     dbg() << "ERROR: dbg.value operand has size " << ValueOperandSize
           << ", but its variable has size " << *DbgVarSize << ": ";
-    DVI->print(dbg());
+    DbgVal->print(dbg());
     dbg() << "\n";
   }
   return HasBadSize;
@@ -735,10 +721,6 @@ bool checkDebugifyMetadata(Module &M,
     return false;
   }
 
-  bool NewDebugMode = M.IsNewDbgInfoFormat;
-  if (NewDebugMode)
-    M.convertFromNewDbgValues();
-
   auto getDebugifyOperand = [&](unsigned Idx) -> unsigned {
     return mdconst::extract<ConstantInt>(NMD->getOperand(Idx)->getOperand(0))
         ->getZExtValue();
@@ -780,18 +762,23 @@ bool checkDebugifyMetadata(Module &M,
     }
 
     // Find missing variables and mis-sized debug values.
-    for (Instruction &I : instructions(F)) {
-      auto *DVI = dyn_cast<DbgValueInst>(&I);
-      if (!DVI)
-        continue;
-
+    auto CheckForMisSized = [&](auto *DbgVal) {
       unsigned Var = ~0U;
-      (void)to_integer(DVI->getVariable()->getName(), Var, 10);
+      (void)to_integer(DbgVal->getVariable()->getName(), Var, 10);
       assert(Var <= OriginalNumVars && "Unexpected name for DILocalVariable");
-      bool HasBadSize = diagnoseMisSizedDbgValue(M, DVI);
+      bool HasBadSize = diagnoseMisSizedDbgValue(M, DbgVal);
       if (!HasBadSize)
         MissingVars.reset(Var - 1);
       HasErrors |= HasBadSize;
+    };
+    for (Instruction &I : instructions(F)) {
+      for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange()))
+        if (DVR.isDbgValue() || DVR.isDbgAssign())
+          CheckForMisSized(&DVR);
+      auto *DVI = dyn_cast<DbgValueInst>(&I);
+      if (!DVI)
+        continue;
+      CheckForMisSized(DVI);
     }
   }
 
@@ -820,9 +807,6 @@ bool checkDebugifyMetadata(Module &M,
   if (Strip)
     Ret = stripDebugifyMetadata(M);
 
-  if (NewDebugMode)
-    M.convertToNewDbgValues();
-
   return Ret;
 }
 
@@ -1052,10 +1036,6 @@ FunctionPass *createCheckDebugifyFunctionPass(
 
 PreservedAnalyses NewPMCheckDebugifyPass::run(Module &M,
                                               ModuleAnalysisManager &) {
-  bool NewDebugMode = M.IsNewDbgInfoFormat;
-  if (NewDebugMode)
-    M.convertFromNewDbgValues();
-
   if (Mode == DebugifyMode::SyntheticDebugInfo)
     checkDebugifyMetadata(M, M.functions(), NameOfWrappedPass,
                                    "CheckModuleDebugify", Strip, StatsMap);
@@ -1065,9 +1045,6 @@ PreservedAnalyses NewPMCheckDebugifyPass::run(Module &M,
       "CheckModuleDebugify (original debuginfo)", NameOfWrappedPass,
       OrigDIVerifyBugsReportFilePath);
 
-  if (NewDebugMode)
-    M.convertToNewDbgValues();
-
   return PreservedAnalyses::all();
 }
 
diff --git a/llvm/test/CodeGen/Generic/MIRDebugify/check-line-and-variables-x.mir b/llvm/test/CodeGen/Generic/MIRDebugify/check-line-and-variables-x.mir
index eaa627966347..40ea01189f2c 100644
--- a/llvm/test/CodeGen/Generic/MIRDebugify/check-line-and-variables-x.mir
+++ b/llvm/test/CodeGen/Generic/MIRDebugify/check-line-and-variables-x.mir
@@ -1,5 +1,6 @@
 # REQUIRES: x86-registered-target
 # RUN: llc -mtriple=x86_64-unknown-linux-gnu -run-pass=mir-check-debugify -o - %s 2>&1 | FileCheck %s
+# RUN: llc --experimental-debuginfo-iterators=false -mtriple=x86_64-unknown-linux-gnu -run-pass=mir-check-debugify -o - %s 2>&1 | FileCheck %s
 --- |
   ; ModuleID = 'check-line-and-variables.mir'
   source_filename = "check-line-and-variables.c"
diff --git a/llvm/test/CodeGen/Generic/MIRDebugify/check-line-and-variables.ll b/llvm/test/CodeGen/Generic/MIRDebugify/check-line-and-variables.ll
index 9033fd2f147c..56c7cf45705a 100644
--- a/llvm/test/CodeGen/Generic/MIRDebugify/check-line-and-variables.ll
+++ b/llvm/test/CodeGen/Generic/MIRDebugify/check-line-and-variables.ll
@@ -1,4 +1,5 @@
 ; RUN: llc -debugify-check-and-strip-all-safe -o - %s 2>&1 | FileCheck %s
+; RUN: llc --experimental-debuginfo-iterators=false -debugify-check-and-strip-all-safe -o - %s 2>&1 | FileCheck %s
 
 ; ModuleID = 'main.c'
 source_filename = "main.c"
diff --git a/llvm/test/CodeGen/Generic/MIRDebugify/check-line-and-variables.mir b/llvm/test/CodeGen/Generic/MIRDebugify/check-line-and-variables.mir
index 9eb722258b70..0805a7f4cfc6 100644
--- a/llvm/test/CodeGen/Generic/MIRDebugify/check-line-and-variables.mir
+++ b/llvm/test/CodeGen/Generic/MIRDebugify/check-line-and-variables.mir
@@ -1,6 +1,8 @@
 # REQUIRES: x86-registered-target
 # RUN: llc -mtriple=x86_64-unknown-linux-gnu -run-pass=mir-debugify,dead-mi-elimination,mir-check-debugify -o - %s 2>&1 | FileCheck %s
 # RUN: llc -mtriple=x86_64-unknown-linux-gnu -run-pass=mir-debugify,mir-check-debugify -o - %s 2>&1 | FileCheck %s --check-prefix=CHECK-PASS
+# RUN: llc --experimental-debuginfo-iterators=false -mtriple=x86_64-unknown-linux-gnu -run-pass=mir-debugify,dead-mi-elimination,mir-check-debugify -o - %s 2>&1 | FileCheck %s
+# RUN: llc --experimental-debuginfo-iterators=false -mtriple=x86_64-unknown-linux-gnu -run-pass=mir-debugify,mir-check-debugify -o - %s 2>&1 | FileCheck %s --check-prefix=CHECK-PASS
 --- |
   ; ModuleID = 'check-line-and-variables.mir'
   source_filename = "check-line-and-variables.ll"
diff --git a/llvm/test/CodeGen/Generic/MIRDebugify/locations-and-values.mir b/llvm/test/CodeGen/Generic/MIRDebugify/locations-and-values.mir
index 59dcff9efd4d..3035fb8eab3f 100644
--- a/llvm/test/CodeGen/Generic/MIRDebugify/locations-and-values.mir
+++ b/llvm/test/CodeGen/Generic/MIRDebugify/locations-and-values.mir
@@ -2,6 +2,10 @@
 # RUN: llc -run-pass=mir-debugify -debugify-level=locations -o - %s | FileCheck --check-prefixes=ALL --implicit-check-not=dbg.value %s
 # RUN: llc -run-pass=mir-debugify,mir-strip-debug,mir-debugify -o - %s | FileCheck --check-prefixes=ALL,VALUE %s
 # RUN: llc -run-pass=mir-debugify,mir-strip-debug -o - %s | FileCheck --check-prefix=STRIP %s
+# RUN: llc --experimental-debuginfo-iterators=false -run-pass=mir-debugify -o - %s | FileCheck --check-prefixes=ALL,VALUE %s
+# RUN: llc --experimental-debuginfo-iterators=false -run-pass=mir-debugify -debugify-level=locations -o - %s | FileCheck --check-prefixes=ALL --implicit-check-not=dbg.value %s
+# RUN: llc --experimental-debuginfo-iterators=false -run-pass=mir-debugify,mir-strip-debug,mir-debugify -o - %s | FileCheck --check-prefixes=ALL,VALUE %s
+# RUN: llc --experimental-debuginfo-iterators=false -run-pass=mir-debugify,mir-strip-debug -o - %s | FileCheck --check-prefix=STRIP %s
 
 --- |
   ; ModuleID = 'loc-only.ll'
diff --git a/llvm/test/CodeGen/Generic/MIRDebugify/multifunction-module.mir b/llvm/test/CodeGen/Generic/MIRDebugify/multifunction-module.mir
index fe4fcc1a15bb..8079db926e1b 100644
--- a/llvm/test/CodeGen/Generic/MIRDebugify/multifunction-module.mir
+++ b/llvm/test/CodeGen/Generic/MIRDebugify/multifunction-module.mir
@@ -1,6 +1,5 @@
-# FIXME: Remove rm after a few weeks.
-# RUN: rm -f %S/multifunction-module.s
 # RUN: llc -run-pass=mir-debugify,mir-check-debugify -o - %s 2>&1 | FileCheck %s
+# RUN: llc --experimental-debuginfo-iterators=false -run-pass=mir-debugify,mir-check-debugify -o - %s 2>&1 | FileCheck %s
 
 # CHECK: Machine IR debug info check: PASS
 # CHECK-NOT: Assertion `Var <= NumVars && "Unexpected name for DILocalVariable"'
diff --git a/llvm/test/DebugInfo/debugify-bogus-dbg-value.ll b/llvm/test/DebugInfo/debugify-bogus-dbg-value.ll
index 4990979f10c5..55e436b1a93b 100644
--- a/llvm/test/DebugInfo/debugify-bogus-dbg-value.ll
+++ b/llvm/test/DebugInfo/debugify-bogus-dbg-value.ll
@@ -1,4 +1,5 @@
 ; RUN: opt -passes=check-debugify < %s 2>&1 | FileCheck %s
+; RUN: opt --experimental-debuginfo-iterators=false -passes=check-debugify < %s 2>&1 | FileCheck %s
 
 define <2 x i64> @test-fun(<2 x i64> %A) !dbg !6 {
   %and = and <2 x i64> %A, <i64 23, i64 42>, !dbg !14
diff --git a/llvm/test/DebugInfo/debugify-each.ll b/llvm/test/DebugInfo/debugify-each.ll
index e9241dedb696..7685b57b5dd1 100644
--- a/llvm/test/DebugInfo/debugify-each.ll
+++ b/llvm/test/DebugInfo/debugify-each.ll
@@ -40,6 +40,40 @@
 ; RUN: opt -debugify-each -passes=globalopt -S -o /dev/null < %s 2> %t
 ; RUN: FileCheck %s -input-file=%t -check-prefix=MODULE-PASS-ONE
 
+; Repeat the same checks with debug intrinsics enabled.
+; RUN: opt --experimental-debuginfo-iterators=false -debugify-each -O3 -S -o /dev/null < %s 2> %t
+; RUN: FileCheck %s -input-file=%t -check-prefix=MODULE-PASS
+; RUN: FileCheck %s -input-file=%t -check-prefix=FUNCTION-PASS
+; RUN: opt --experimental-debuginfo-iterators=false -disable-output -debugify-each -passes='default<O3>' %s 2> %t
+; RUN: FileCheck %s -input-file=%t -check-prefix=MODULE-PASS
+; RUN: FileCheck %s -input-file=%t -check-prefix=FUNCTION-PASS
+
+; RUN: opt --experimental-debuginfo-iterators=false -enable-debugify -debugify-each -O3 -S -o /dev/null < %s 2> %t
+; RUN: FileCheck %s -input-file=%t -check-prefix=MODULE-PASS
+; RUN: FileCheck %s -input-file=%t -check-prefix=FUNCTION-PASS
+
+; RUN: opt --experimental-debuginfo-iterators=false -debugify-each -passes='instrprof,instrprof,sroa,sccp' -S -o /dev/null < %s 2> %t
+; RUN: FileCheck %s -input-file=%t -check-prefix=MODULE-PASS
+; RUN: FileCheck %s -input-file=%t -check-prefix=FUNCTION-PASS
+
+; RUN: opt --experimental-debuginfo-iterators=false -debugify-each -O1 < %s | opt -O2 -o /dev/null
+
+; RUN: opt --experimental-debuginfo-iterators=false -disable-output -debugify-quiet -debugify-each -O1 < %s 2>&1 | count 0
+
+; RUN: opt --experimental-debuginfo-iterators=false -O1 < %s -S -o %t.before
+; RUN: opt --experimental-debuginfo-iterators=false -O1 -debugify-each < %s -S -o %t.after
+; RUN: diff %t.before %t.after
+
+; RUN: opt --experimental-debuginfo-iterators=false -O1 < %s | llvm-dis -o %t.before
+; RUN: opt --experimental-debuginfo-iterators=false -O1 -debugify-each < %s | llvm-dis -o %t.after
+; RUN: diff %t.before %t.after
+
+; RUN: opt --experimental-debuginfo-iterators=false -debugify-each -passes=instsimplify -S -o /dev/null < %s 2> %t
+; RUN: FileCheck %s -input-file=%t -check-prefix=FUNCTION-PASS-ONE
+
+; RUN: opt --experimental-debuginfo-iterators=false -debugify-each -passes=globalopt -S -o /dev/null < %s 2> %t
+; RUN: FileCheck %s -input-file=%t -check-prefix=MODULE-PASS-ONE
+
 define void @foo(i32 %arg) {
   call i32 asm "bswap $0", "=r,r"(i32 %arg)
   ret void
diff --git a/llvm/test/DebugInfo/debugify-export.ll b/llvm/test/DebugInfo/debugify-export.ll
index 6e5952d433da..30333ca908b0 100644
--- a/llvm/test/DebugInfo/debugify-export.ll
+++ b/llvm/test/DebugInfo/debugify-export.ll
@@ -1,6 +1,9 @@
 ; RUN: opt %s -disable-output -debugify-each -debugify-quiet -debugify-export - -passes=globalopt | FileCheck %s
 ; RUN: opt %s -disable-output -debugify-each -debugify-quiet -debugify-export - -passes=globalopt | FileCheck %s
 
+; RUN: opt --experimental-debuginfo-iterators=false %s -disable-output -debugify-each -debugify-quiet -debugify-export - -passes=globalopt | FileCheck %s
+; RUN: opt --experimental-debuginfo-iterators=false %s -disable-output -debugify-each -debugify-quiet -debugify-export - -passes=globalopt | FileCheck %s
+
 ; CHECK: Pass Name
 ; CHECK-SAME: # of missing debug values
 ; CHECK-SAME: # of missing locations
diff --git a/llvm/test/DebugInfo/debugify-ignore-phi.ll b/llvm/test/DebugInfo/debugify-ignore-phi.ll
index 322ccafa22ac..643df1d96048 100644
--- a/llvm/test/DebugInfo/debugify-ignore-phi.ll
+++ b/llvm/test/DebugInfo/debugify-ignore-phi.ll
@@ -1,4 +1,5 @@
 ; RUN: opt -passes=check-debugify < %s -S 2>&1 | FileCheck %s
+; RUN: opt --experimental-debuginfo-iterators=false -passes=check-debugify < %s -S 2>&1 | FileCheck %s
 
 define void @test_phi(i1 %cond) !dbg !6 {
   br i1 %cond, label %1, label %2, !dbg !11
diff --git a/llvm/test/DebugInfo/debugify-original-no-dbg-info.ll b/llvm/test/DebugInfo/debugify-original-no-dbg-info.ll
index 941b294fb855..4cbbfc5c215e 100644
--- a/llvm/test/DebugInfo/debugify-original-no-dbg-info.ll
+++ b/llvm/test/DebugInfo/debugify-original-no-dbg-info.ll
@@ -1,4 +1,5 @@
 ; RUN: opt -verify-debuginfo-preserve -passes=instcombine -S -o - < %s 2>&1 | FileCheck %s
+; RUN: opt --experimental-debuginfo-iterators=false -verify-debuginfo-preserve -passes=instcombine -S -o - < %s 2>&1 | FileCheck %s
 
 ; CHECK: ModuleDebugify (original debuginfo): Skipping module without debug info
 ; CHECK-NEXT: CheckModuleDebugify (original debuginfo): Skipping module without debug info
diff --git a/llvm/test/DebugInfo/debugify-report-missing-locs-only.ll b/llvm/test/DebugInfo/debugify-report-missing-locs-only.ll
index 1c5daa19c648..04b7636f025a 100644
--- a/llvm/test/DebugInfo/debugify-report-missing-locs-only.ll
+++ b/llvm/test/DebugInfo/debugify-report-missing-locs-only.ll
@@ -1,4 +1,5 @@
 ; RUN: opt -passes=check-debugify < %s -S -o - 2>&1 | FileCheck %s -implicit-check-not "WARNING: Instruction with empty DebugLoc in function bar"
+; RUN: opt --experimental-debuginfo-iterators=false -passes=check-debugify < %s -S -o - 2>&1 | FileCheck %s -implicit-check-not "WARNING: Instruction with empty DebugLoc in function bar"
 
 ; CHECK: WARNING: Instruction with empty DebugLoc in function foo --   ret void
 define void @foo() !dbg !6 {
diff --git a/llvm/test/DebugInfo/debugify.ll b/llvm/test/DebugInfo/debugify.ll
index 5ce6795d41b6..191015f82593 100644
--- a/llvm/test/DebugInfo/debugify.ll
+++ b/llvm/test/DebugInfo/debugify.ll
@@ -25,6 +25,33 @@
 ; RUN: opt -enable-debugify -O1 < %s | opt -O2 -o /dev/null
 ; RUN: opt -passes=debugify,mem2reg,check-debugify < %s | opt -O2 -o /dev/null
 
+;; Perform the same checks again for intrinsic debug info
+; RUN: opt --experimental-debuginfo-iterators=false -passes=debugify -S -o - < %s | FileCheck %s
+; RUN: opt --experimental-debuginfo-iterators=false -passes=debugify -S -o - < %s | FileCheck %s
+
+; RUN: opt --experimental-debuginfo-iterators=false -passes=debugify,debugify -S -o - < %s 2>&1 | \
+; RUN:   FileCheck %s -check-prefix=CHECK-REPEAT
+; RUN: opt --experimental-debuginfo-iterators=false -passes=debugify,debugify -S -o - < %s 2>&1 | \
+; RUN:   FileCheck %s -check-prefix=CHECK-REPEAT
+
+; RUN: opt --experimental-debuginfo-iterators=false -passes=debugify,check-debugify -S -o - < %s | \
+; RUN:   FileCheck %s -implicit-check-not="CheckModuleDebugify: FAIL"
+; RUN: opt --experimental-debuginfo-iterators=false -passes=debugify,check-debugify -S -o - < %s | \
+; RUN:   FileCheck %s -implicit-check-not="CheckModuleDebugify: FAIL"
+; RUN: opt --experimental-debuginfo-iterators=false -enable-debugify -passes=verify -S -o - < %s | \
+; RUN:   FileCheck %s -implicit-check-not="CheckModuleDebugify: FAIL"
+
+; RUN: opt --experimental-debuginfo-iterators=false -passes=debugify,strip,check-debugify -S -o - < %s 2>&1 | \
+; RUN:   FileCheck %s -check-prefix=CHECK-WARN
+
+; RUN: opt --experimental-debuginfo-iterators=false -enable-debugify -passes=strip -S -o - < %s 2>&1 | \
+; RUN:   FileCheck %s -check-prefix=CHECK-WARN
+
+; RUN: opt --experimental-debuginfo-iterators=false -enable-debugify -S -o - < %s 2>&1 | FileCheck %s -check-prefix=PASS
+
+; RUN: opt --experimental-debuginfo-iterators=false -enable-debugify -O1 < %s | opt -O2 -o /dev/null
+; RUN: opt --experimental-debuginfo-iterators=false -passes=debugify,mem2reg,check-debugify < %s | opt -O2 -o /dev/null
+
 ; CHECK-LABEL: define void @foo
 define void @foo() {
 ; CHECK: ret void, !dbg ![[RET1:.*]]
diff --git a/llvm/test/DebugInfo/pr37964.ll b/llvm/test/DebugInfo/pr37964.ll
index 9581f1a6b35d..63db67d2bd37 100644
--- a/llvm/test/DebugInfo/pr37964.ll
+++ b/llvm/test/DebugInfo/pr37964.ll
@@ -1,4 +1,5 @@
 ; RUN: opt -disable-output -debugify-each -passes=gvn < %s 2>&1 | FileCheck %s
+; RUN: opt --experimental-debuginfo-iterators=false -disable-output -debugify-each -passes=gvn < %s 2>&1 | FileCheck %s
 
 ; CHECK-NOT: ERROR: Instruction with empty DebugLoc in function _Z3bazv --  {{%.*}} = phi
 ; CHECK: CheckFunctionDebugify [GVNPass]: PASS
diff --git a/llvm/test/DebugInfo/salvage-cast-debug-info.ll b/llvm/test/DebugInfo/salvage-cast-debug-info.ll
index 4676aee3d4e4..b72f717a4f2d 100644
--- a/llvm/test/DebugInfo/salvage-cast-debug-info.ll
+++ b/llvm/test/DebugInfo/salvage-cast-debug-info.ll
@@ -1,5 +1,5 @@
 ; RUN: opt %s -passes=debugify,early-cse -earlycse-debug-hash -S | FileCheck %s
-; RUN: opt %s -passes=debugify,early-cse -earlycse-debug-hash -S --try-experimental-debuginfo-iterators | FileCheck %s
+; RUN: opt --experimental-debuginfo-iterators=false %s -passes=debugify,early-cse -earlycse-debug-hash -S | FileCheck %s
 define i32 @foo(i64 %nose, i32 %more) {
 ; CHECK-LABEL: @foo(
 ; CHECK: call void @llvm.dbg.value(metadata i64 %nose, metadata [[V1:![0-9]+]], metadata !DIExpression(DW_OP_LLVM_convert, 64, DW_ATE_unsigned, DW_OP_LLVM_convert, 32, DW_ATE_unsigned
diff --git a/llvm/test/DebugInfo/verify-di-preserve.ll b/llvm/test/DebugInfo/verify-di-preserve.ll
index a2f1b1dd78dc..92fc62a0b34c 100644
--- a/llvm/test/DebugInfo/verify-di-preserve.ll
+++ b/llvm/test/DebugInfo/verify-di-preserve.ll
@@ -1,10 +1,10 @@
 ; RUN: opt %s -verify-debuginfo-preserve -passes=instcombine -disable-output 2>&1 | FileCheck --check-prefix=VERIFY %s
-; RUN: opt --try-experimental-debuginfo-iterators %s -verify-debuginfo-preserve -passes=instcombine -disable-output 2>&1 | FileCheck --check-prefix=VERIFY %s
+; RUN: opt --experimental-debuginfo-iterators=false %s -verify-debuginfo-preserve -passes=instcombine -disable-output 2>&1 | FileCheck --check-prefix=VERIFY %s
 
 ; VERIFY: CheckModuleDebugify (original debuginfo):
 
 ; RUN: opt %s -verify-each-debuginfo-preserve -O2 -disable-output 2>&1 | FileCheck --check-prefix=VERIFY-EACH %s
-; RUN: opt %s  --try-experimental-debuginfo-iterators -verify-each-debuginfo-preserve -O2 -disable-output 2>&1 | FileCheck --check-prefix=VERIFY-EACH %s
+; RUN: opt %s --experimental-debuginfo-iterators=false -verify-each-debuginfo-preserve -O2 -disable-output 2>&1 | FileCheck --check-prefix=VERIFY-EACH %s
 
 ; VERIFY-EACH: DeadArgumentEliminationPass
 ; VERIFY-EACH: GlobalDCEPass
-- 
GitLab


From 45eabd1362527d1b3a27a90f7479865785b763ee Mon Sep 17 00:00:00 2001
From: YunQiang Su <syq@debian.org>
Date: Wed, 17 Apr 2024 00:52:15 +0800
Subject: [PATCH 427/464] [flang,test] Add -resource-dir option to
 msvc-dependent-lib-flags.f90 (#88894)

For aarch64-windows-msvc, clang_rt.builtins is placed in windows subdir
instead of triple subdir, and the name of clang_rt.builtins is
clang_rt.builtins-aarch64.lib.

So let's use `-resource-dir` option to fix test failure.

Please see talk for PR#87866.
---
 flang/test/Driver/msvc-dependent-lib-flags.f90 | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/flang/test/Driver/msvc-dependent-lib-flags.f90 b/flang/test/Driver/msvc-dependent-lib-flags.f90
index 643dbe9e949c..6cfc969e92b2 100644
--- a/flang/test/Driver/msvc-dependent-lib-flags.f90
+++ b/flang/test/Driver/msvc-dependent-lib-flags.f90
@@ -1,7 +1,7 @@
-! RUN: %flang -### --target=aarch64-windows-msvc %S/Inputs/hello.f90 -v 2>&1 | FileCheck %s --check-prefixes=MSVC
-! RUN: %flang -### --target=aarch64-windows-msvc -fms-runtime-lib=static_dbg %S/Inputs/hello.f90 -v 2>&1 | FileCheck %s --check-prefixes=MSVC-DEBUG
-! RUN: %flang -### --target=aarch64-windows-msvc -fms-runtime-lib=dll %S/Inputs/hello.f90 -v 2>&1 | FileCheck %s --check-prefixes=MSVC-DLL
-! RUN: %flang -### --target=aarch64-windows-msvc -fms-runtime-lib=dll_dbg %S/Inputs/hello.f90 -v 2>&1 | FileCheck %s --check-prefixes=MSVC-DLL-DEBUG
+! RUN: %flang -### --target=aarch64-windows-msvc -resource-dir=%S/Inputs/resource_dir %S/Inputs/hello.f90 -v 2>&1 | FileCheck %s --check-prefixes=MSVC
+! RUN: %flang -### --target=aarch64-windows-msvc -resource-dir=%S/Inputs/resource_dir -fms-runtime-lib=static_dbg %S/Inputs/hello.f90 -v 2>&1 | FileCheck %s --check-prefixes=MSVC-DEBUG
+! RUN: %flang -### --target=aarch64-windows-msvc -resource-dir=%S/Inputs/resource_dir -fms-runtime-lib=dll %S/Inputs/hello.f90 -v 2>&1 | FileCheck %s --check-prefixes=MSVC-DLL
+! RUN: %flang -### --target=aarch64-windows-msvc -resource-dir=%S/Inputs/resource_dir -fms-runtime-lib=dll_dbg %S/Inputs/hello.f90 -v 2>&1 | FileCheck %s --check-prefixes=MSVC-DLL-DEBUG
 
 ! MSVC: -fc1
 ! MSVC-SAME: --dependent-lib=clang_rt.builtins.lib
-- 
GitLab


From 22e6bf77ad8781810fc81fff4c447c03cdf6f419 Mon Sep 17 00:00:00 2001
From: Volodymyr Sapsai <vsapsai@apple.com>
Date: Tue, 16 Apr 2024 10:12:26 -0700
Subject: [PATCH 428/464] [unused-includes][Serialization] Remove unused
 includes. NFC. (#88790)

---
 clang/include/clang/Serialization/ModuleFileExtension.h | 1 -
 clang/lib/Serialization/ASTReader.cpp                   | 1 -
 clang/lib/Serialization/ASTWriterDecl.cpp               | 1 -
 clang/lib/Serialization/ASTWriterStmt.cpp               | 1 -
 clang/lib/Serialization/GeneratePCH.cpp                 | 1 -
 clang/lib/Serialization/GlobalModuleIndex.cpp           | 1 -
 clang/lib/Serialization/ModuleFileExtension.cpp         | 2 +-
 clang/lib/Serialization/PCHContainerOperations.cpp      | 2 --
 8 files changed, 1 insertion(+), 9 deletions(-)

diff --git a/clang/include/clang/Serialization/ModuleFileExtension.h b/clang/include/clang/Serialization/ModuleFileExtension.h
index d7d456c8b5db..50ce40151627 100644
--- a/clang/include/clang/Serialization/ModuleFileExtension.h
+++ b/clang/include/clang/Serialization/ModuleFileExtension.h
@@ -9,7 +9,6 @@
 #ifndef LLVM_CLANG_SERIALIZATION_MODULEFILEEXTENSION_H
 #define LLVM_CLANG_SERIALIZATION_MODULEFILEEXTENSION_H
 
-#include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/Support/ExtensibleRTTI.h"
 #include "llvm/Support/HashBuilder.h"
 #include "llvm/Support/MD5.h"
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index cf0726460bfc..b28df03b4a95 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -31,7 +31,6 @@
 #include "clang/AST/ExternalASTSource.h"
 #include "clang/AST/NestedNameSpecifier.h"
 #include "clang/AST/ODRDiagsEmitter.h"
-#include "clang/AST/ODRHash.h"
 #include "clang/AST/OpenACCClause.h"
 #include "clang/AST/OpenMPClause.h"
 #include "clang/AST/RawCommentList.h"
diff --git a/clang/lib/Serialization/ASTWriterDecl.cpp b/clang/lib/Serialization/ASTWriterDecl.cpp
index d0d49bcdf991..c6db107e0ca4 100644
--- a/clang/lib/Serialization/ASTWriterDecl.cpp
+++ b/clang/lib/Serialization/ASTWriterDecl.cpp
@@ -16,7 +16,6 @@
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/DeclVisitor.h"
 #include "clang/AST/Expr.h"
-#include "clang/AST/ODRHash.h"
 #include "clang/AST/OpenMPClause.h"
 #include "clang/AST/PrettyDeclStackTrace.h"
 #include "clang/Basic/SourceManager.h"
diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp
index e3816181e2b2..a736a7b0ef72 100644
--- a/clang/lib/Serialization/ASTWriterStmt.cpp
+++ b/clang/lib/Serialization/ASTWriterStmt.cpp
@@ -19,7 +19,6 @@
 #include "clang/AST/ExprOpenMP.h"
 #include "clang/AST/StmtVisitor.h"
 #include "clang/Lex/Token.h"
-#include "clang/Sema/DeclSpec.h"
 #include "clang/Serialization/ASTRecordWriter.h"
 #include "llvm/Bitstream/BitstreamWriter.h"
 using namespace clang;
diff --git a/clang/lib/Serialization/GeneratePCH.cpp b/clang/lib/Serialization/GeneratePCH.cpp
index 2fece29f3448..bed74399098d 100644
--- a/clang/lib/Serialization/GeneratePCH.cpp
+++ b/clang/lib/Serialization/GeneratePCH.cpp
@@ -17,7 +17,6 @@
 #include "clang/Lex/HeaderSearchOptions.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Sema/SemaConsumer.h"
-#include "clang/Serialization/ASTReader.h"
 #include "clang/Serialization/ASTWriter.h"
 #include "llvm/Bitstream/BitstreamWriter.h"
 
diff --git a/clang/lib/Serialization/GlobalModuleIndex.cpp b/clang/lib/Serialization/GlobalModuleIndex.cpp
index 8ff10f6a8621..f09ceb8d3162 100644
--- a/clang/lib/Serialization/GlobalModuleIndex.cpp
+++ b/clang/lib/Serialization/GlobalModuleIndex.cpp
@@ -13,7 +13,6 @@
 #include "clang/Serialization/GlobalModuleIndex.h"
 #include "ASTReaderInternals.h"
 #include "clang/Basic/FileManager.h"
-#include "clang/Lex/HeaderSearch.h"
 #include "clang/Serialization/ASTBitCodes.h"
 #include "clang/Serialization/ModuleFile.h"
 #include "clang/Serialization/PCHContainerOperations.h"
diff --git a/clang/lib/Serialization/ModuleFileExtension.cpp b/clang/lib/Serialization/ModuleFileExtension.cpp
index 95fff41e0d7a..729529b5fca1 100644
--- a/clang/lib/Serialization/ModuleFileExtension.cpp
+++ b/clang/lib/Serialization/ModuleFileExtension.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 #include "clang/Serialization/ModuleFileExtension.h"
-#include "llvm/ADT/Hashing.h"
+
 using namespace clang;
 
 char ModuleFileExtension::ID = 0;
diff --git a/clang/lib/Serialization/PCHContainerOperations.cpp b/clang/lib/Serialization/PCHContainerOperations.cpp
index 56ca3394385b..4aedb7debcff 100644
--- a/clang/lib/Serialization/PCHContainerOperations.cpp
+++ b/clang/lib/Serialization/PCHContainerOperations.cpp
@@ -12,8 +12,6 @@
 
 #include "clang/Serialization/PCHContainerOperations.h"
 #include "clang/AST/ASTConsumer.h"
-#include "clang/Lex/ModuleLoader.h"
-#include "llvm/Bitstream/BitstreamReader.h"
 #include "llvm/Support/raw_ostream.h"
 #include <utility>
 
-- 
GitLab


From b566810add5b7c5695bdd2c39710b78af9dc83ba Mon Sep 17 00:00:00 2001
From: Volodymyr Sapsai <vsapsai@apple.com>
Date: Tue, 16 Apr 2024 10:13:15 -0700
Subject: [PATCH 429/464] [unused-includes] PCHContainerOperations uses
 MemoryBufferRef, not MemoryBuffer. NFC. (#88794)

---
 clang/include/clang/Serialization/PCHContainerOperations.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/include/clang/Serialization/PCHContainerOperations.h b/clang/include/clang/Serialization/PCHContainerOperations.h
index ddfddf2dafad..c9a7e334ce6e 100644
--- a/clang/include/clang/Serialization/PCHContainerOperations.h
+++ b/clang/include/clang/Serialization/PCHContainerOperations.h
@@ -12,7 +12,7 @@
 #include "clang/Basic/Module.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringMap.h"
-#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/MemoryBufferRef.h"
 #include <memory>
 
 namespace llvm {
-- 
GitLab


From 454d4496970f665200b5b300578894d78405b6ca Mon Sep 17 00:00:00 2001
From: Xing Xue <xingxue@outlook.com>
Date: Tue, 16 Apr 2024 13:13:49 -0400
Subject: [PATCH 430/464] [OpenMP] Use a memory fence before incrementing the
 dispatch buffer index (#87995)

This patch uses a memory fence in function `__kmp_dispatch_next()` to
flush pending memory write invalidates before incrementing the
`volatile` variable `buffer_index` to fix intermittent time-outs of
OpenMP runtime LIT test cases `env/kmp_set_dispatch_buf.c` and
`worksharing/for/kmp_set_dispatch_buf.c`, noting that the same is needed
for incrementing `buffer_index` in function `__kmpc_next_section()`
(line 2600 of `kmp_dispatch.cpp`).
---
 openmp/runtime/src/kmp_dispatch.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/openmp/runtime/src/kmp_dispatch.cpp b/openmp/runtime/src/kmp_dispatch.cpp
index ac85b2b3f2fc..fc3337651181 100644
--- a/openmp/runtime/src/kmp_dispatch.cpp
+++ b/openmp/runtime/src/kmp_dispatch.cpp
@@ -2397,6 +2397,8 @@ static int __kmp_dispatch_next(ident_t *loc, int gtid, kmp_int32 *p_last,
           sh->u.s.ordered_iteration = 0;
         }
 
+        KMP_MB(); /* Flush all pending memory write invalidates.  */
+
         sh->buffer_index += __kmp_dispatch_num_buffers;
         KD_TRACE(100, ("__kmp_dispatch_next: T#%d change buffer_index:%d\n",
                        gtid, sh->buffer_index));
-- 
GitLab


From 22bba85d82637d0446928ff6ff41f98583f3d3b2 Mon Sep 17 00:00:00 2001
From: Xing Xue <xingxue@outlook.com>
Date: Tue, 16 Apr 2024 13:14:29 -0400
Subject: [PATCH 431/464] [OpenMP][test][AIX] Make 64 the max number of threads
 for capacity tests in AIX 32-bit (#88739)

This patch makes 64 the max number of threads for 2 capacity tests in
AIX 32-bit mode rather than `XFAIL`ing them.
---
 .../capacity_mix_threads.cpp                  | 20 +++++++++++++------
 .../hidden_helper_task/capacity_nthreads.cpp  | 20 +++++++++++++------
 2 files changed, 28 insertions(+), 12 deletions(-)

diff --git a/openmp/runtime/test/tasking/hidden_helper_task/capacity_mix_threads.cpp b/openmp/runtime/test/tasking/hidden_helper_task/capacity_mix_threads.cpp
index 3f2ceef0c4ad..36825dbebafb 100644
--- a/openmp/runtime/test/tasking/hidden_helper_task/capacity_mix_threads.cpp
+++ b/openmp/runtime/test/tasking/hidden_helper_task/capacity_mix_threads.cpp
@@ -1,7 +1,4 @@
 // RUN: %libomp-cxx-compile-and-run
-//
-// AIX runs out of resource in 32-bit with 4*omp_get_max_threads() threads.
-// XFAIL: aix && ppc
 
 #include <omp.h>
 
@@ -11,6 +8,12 @@
 #include <thread>
 #include <vector>
 
+// AIX runs out of resource in 32-bit if 4*omp_get_max_threads() is more
+// than 64 threads with the default stack size.
+#if defined(_AIX) && !__LP64__
+#define MAX_THREADS 64
+#endif
+
 void dummy_root() {
   // omp_get_max_threads() will do middle initialization
   int nthreads = omp_get_max_threads();
@@ -18,9 +21,14 @@ void dummy_root() {
 }
 
 int main(int argc, char *argv[]) {
-  const int N = std::min(std::max(std::max(32, 4 * omp_get_max_threads()),
-                                  4 * omp_get_num_procs()),
-                         std::numeric_limits<int>::max());
+  int N = std::min(std::max(std::max(32, 4 * omp_get_max_threads()),
+                            4 * omp_get_num_procs()),
+                   std::numeric_limits<int>::max());
+
+#if defined(_AIX) && !__LP64__
+  if (N > MAX_THREADS)
+    N = MAX_THREADS;
+#endif
 
   std::vector<int> data(N);
 
diff --git a/openmp/runtime/test/tasking/hidden_helper_task/capacity_nthreads.cpp b/openmp/runtime/test/tasking/hidden_helper_task/capacity_nthreads.cpp
index f7405d00255c..1cceee95e704 100644
--- a/openmp/runtime/test/tasking/hidden_helper_task/capacity_nthreads.cpp
+++ b/openmp/runtime/test/tasking/hidden_helper_task/capacity_nthreads.cpp
@@ -1,7 +1,4 @@
 // RUN: %libomp-cxx-compile-and-run
-//
-// AIX runs out of resource in 32-bit with 4*omp_get_max_threads() threads.
-// XFAIL: aix && ppc
 
 #include <omp.h>
 
@@ -10,10 +7,21 @@
 #include <limits>
 #include <vector>
 
+// AIX runs out of resource in 32-bit if 4*omp_get_max_threads() is more
+// than 64 threads with the default stacksize.
+#if defined(_AIX) && !__LP64__
+#define MAX_THREADS 64
+#endif
+
 int main(int argc, char *argv[]) {
-  const int N = std::min(std::max(std::max(32, 4 * omp_get_max_threads()),
-                                  4 * omp_get_num_procs()),
-                         std::numeric_limits<int>::max());
+  int N = std::min(std::max(std::max(32, 4 * omp_get_max_threads()),
+                            4 * omp_get_num_procs()),
+                   std::numeric_limits<int>::max());
+
+#if defined(_AIX) && !__LP64__
+  if (N > MAX_THREADS)
+    N = MAX_THREADS;
+#endif
 
   std::vector<int> data(N);
 
-- 
GitLab


From 8137bd9e03d636a27701a85b6efe899f9571cac5 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Tue, 16 Apr 2024 10:16:48 -0700
Subject: [PATCH 432/464] [memprof] Use CSId to construct MemProfRecord
 (#88362)

We are in the process of referring to call stacks with CallStackId in
IndexedMemProfRecord and IndexedAllocationInfo instead of holding call
stacks inline (both in memory and the serialized format).  Doing so
deduplicates call stacks and reduces the MemProf profile file size.

Before we can eliminate the two fields holding call stacks inline:

- IndexedAllocationInfo::CallStack
- IndexedMemProfRecord::CallSites

we need to eliminate all the read operations on them.

This patch is a step toward that direction.  Specifically, we
eliminate the read operations in the context of MemProfReader and
RawMemProfReader.  A subsequent patch will eliminate the read
operations during the serialization.
---
 llvm/include/llvm/ProfileData/MemProf.h       |  8 ++
 llvm/include/llvm/ProfileData/MemProfReader.h | 20 ++++-
 llvm/lib/ProfileData/MemProf.cpp              | 18 ++++
 llvm/lib/ProfileData/MemProfReader.cpp        | 26 ++++++
 llvm/unittests/ProfileData/MemProfTest.cpp    | 85 +++++++++++++++++++
 5 files changed, 153 insertions(+), 4 deletions(-)

diff --git a/llvm/include/llvm/ProfileData/MemProf.h b/llvm/include/llvm/ProfileData/MemProf.h
index 0431c182276e..3520034fb1c9 100644
--- a/llvm/include/llvm/ProfileData/MemProf.h
+++ b/llvm/include/llvm/ProfileData/MemProf.h
@@ -16,6 +16,8 @@
 namespace llvm {
 namespace memprof {
 
+struct MemProfRecord;
+
 // The versions of the indexed MemProf format
 enum IndexedVersion : uint64_t {
   // Version 0: This version didn't have a version field.
@@ -392,6 +394,12 @@ struct IndexedMemProfRecord {
                                           const unsigned char *Buffer,
                                           IndexedVersion Version);
 
+  // Convert IndexedMemProfRecord to MemProfRecord.  Callback is used to
+  // translate CallStackId to call stacks with frames inline.
+  MemProfRecord toMemProfRecord(
+      std::function<const llvm::SmallVector<Frame>(const CallStackId)> Callback)
+      const;
+
   // Returns the GUID for the function name after canonicalization. For
   // memprof, we remove any .llvm suffix added by LTO. MemProfRecords are
   // mapped to functions using this GUID.
diff --git a/llvm/include/llvm/ProfileData/MemProfReader.h b/llvm/include/llvm/ProfileData/MemProfReader.h
index 89f49a20a608..1f84fefad03e 100644
--- a/llvm/include/llvm/ProfileData/MemProfReader.h
+++ b/llvm/include/llvm/ProfileData/MemProfReader.h
@@ -70,8 +70,20 @@ public:
       Callback =
           std::bind(&MemProfReader::idToFrame, this, std::placeholders::_1);
 
+    auto CallStackCallback = [&](CallStackId CSId) {
+      llvm::SmallVector<Frame> CallStack;
+      auto Iter = CSIdToCallStack.find(CSId);
+      assert(Iter != CSIdToCallStack.end());
+      for (FrameId Id : Iter->second)
+        CallStack.push_back(Callback(Id));
+      return CallStack;
+    };
+
     const IndexedMemProfRecord &IndexedRecord = Iter->second;
-    GuidRecord = {Iter->first, MemProfRecord(IndexedRecord, Callback)};
+    GuidRecord = {
+        Iter->first,
+        IndexedRecord.toMemProfRecord(CallStackCallback),
+    };
     Iter++;
     return Error::success();
   }
@@ -84,9 +96,7 @@ public:
   // Initialize the MemProfReader with the frame mappings and profile contents.
   MemProfReader(
       llvm::DenseMap<FrameId, Frame> FrameIdMap,
-      llvm::MapVector<GlobalValue::GUID, IndexedMemProfRecord> ProfData)
-      : IdToFrame(std::move(FrameIdMap)),
-        FunctionProfileData(std::move(ProfData)) {}
+      llvm::MapVector<GlobalValue::GUID, IndexedMemProfRecord> ProfData);
 
 protected:
   // A helper method to extract the frame from the IdToFrame map.
@@ -97,6 +107,8 @@ protected:
   }
   // A mapping from FrameId (a hash of the contents) to the frame.
   llvm::DenseMap<FrameId, Frame> IdToFrame;
+  // A mapping from CallStackId to the call stack.
+  llvm::DenseMap<CallStackId, llvm::SmallVector<FrameId>> CSIdToCallStack;
   // A mapping from function GUID, hash of the canonical function symbol to the
   // memprof profile data for that function, i.e allocation and callsite info.
   llvm::MapVector<GlobalValue::GUID, IndexedMemProfRecord> FunctionProfileData;
diff --git a/llvm/lib/ProfileData/MemProf.cpp b/llvm/lib/ProfileData/MemProf.cpp
index 97414505f1c1..1ca0a02d3cbd 100644
--- a/llvm/lib/ProfileData/MemProf.cpp
+++ b/llvm/lib/ProfileData/MemProf.cpp
@@ -224,6 +224,24 @@ IndexedMemProfRecord::deserialize(const MemProfSchema &Schema,
   llvm_unreachable("unsupported MemProf version");
 }
 
+MemProfRecord IndexedMemProfRecord::toMemProfRecord(
+    std::function<const llvm::SmallVector<Frame>(const CallStackId)> Callback)
+    const {
+  MemProfRecord Record;
+
+  for (const memprof::IndexedAllocationInfo &IndexedAI : AllocSites) {
+    memprof::AllocationInfo AI;
+    AI.Info = IndexedAI.Info;
+    AI.CallStack = Callback(IndexedAI.CSId);
+    Record.AllocSites.push_back(AI);
+  }
+
+  for (memprof::CallStackId CSId : CallSiteIds)
+    Record.CallSites.push_back(Callback(CSId));
+
+  return Record;
+}
+
 GlobalValue::GUID IndexedMemProfRecord::getGUID(const StringRef FunctionName) {
   // Canonicalize the function name to drop suffixes such as ".llvm.". Note
   // we do not drop any ".__uniq." suffixes, as getCanonicalFnName does not drop
diff --git a/llvm/lib/ProfileData/MemProfReader.cpp b/llvm/lib/ProfileData/MemProfReader.cpp
index 580867a9083f..91556f036c77 100644
--- a/llvm/lib/ProfileData/MemProfReader.cpp
+++ b/llvm/lib/ProfileData/MemProfReader.cpp
@@ -183,6 +183,28 @@ std::string getBuildIdString(const SegmentEntry &Entry) {
 }
 } // namespace
 
+MemProfReader::MemProfReader(
+    llvm::DenseMap<FrameId, Frame> FrameIdMap,
+    llvm::MapVector<GlobalValue::GUID, IndexedMemProfRecord> ProfData)
+    : IdToFrame(std::move(FrameIdMap)),
+      FunctionProfileData(std::move(ProfData)) {
+  // Populate CSId in each IndexedAllocationInfo and IndexedMemProfRecord
+  // while storing CallStack in CSIdToCallStack.
+  for (auto &KV : FunctionProfileData) {
+    IndexedMemProfRecord &Record = KV.second;
+    for (auto &AS : Record.AllocSites) {
+      CallStackId CSId = hashCallStack(AS.CallStack);
+      AS.CSId = CSId;
+      CSIdToCallStack.insert({CSId, AS.CallStack});
+    }
+    for (auto &CS : Record.CallSites) {
+      CallStackId CSId = hashCallStack(CS);
+      Record.CallSiteIds.push_back(CSId);
+      CSIdToCallStack.insert({CSId, CS});
+    }
+  }
+}
+
 Expected<std::unique_ptr<RawMemProfReader>>
 RawMemProfReader::create(const Twine &Path, const StringRef ProfiledBinary,
                          bool KeepName) {
@@ -445,6 +467,7 @@ Error RawMemProfReader::mapRawProfileToRecords() {
     }
 
     CallStackId CSId = hashCallStack(Callstack);
+    CSIdToCallStack.insert({CSId, Callstack});
 
     // We attach the memprof record to each function bottom-up including the
     // first non-inline frame.
@@ -467,7 +490,10 @@ Error RawMemProfReader::mapRawProfileToRecords() {
     auto Result = FunctionProfileData.insert({Id, IndexedMemProfRecord()});
     IndexedMemProfRecord &Record = Result.first->second;
     for (LocationPtr Loc : Locs) {
+      CallStackId CSId = hashCallStack(*Loc);
+      CSIdToCallStack.insert({CSId, *Loc});
       Record.CallSites.push_back(*Loc);
+      Record.CallSiteIds.push_back(CSId);
     }
   }
 
diff --git a/llvm/unittests/ProfileData/MemProfTest.cpp b/llvm/unittests/ProfileData/MemProfTest.cpp
index 9cf307472d65..ab9227e9df88 100644
--- a/llvm/unittests/ProfileData/MemProfTest.cpp
+++ b/llvm/unittests/ProfileData/MemProfTest.cpp
@@ -21,9 +21,11 @@ using ::llvm::DILineInfo;
 using ::llvm::DILineInfoSpecifier;
 using ::llvm::DILocal;
 using ::llvm::StringRef;
+using ::llvm::memprof::CallStackId;
 using ::llvm::memprof::CallStackMap;
 using ::llvm::memprof::Frame;
 using ::llvm::memprof::FrameId;
+using ::llvm::memprof::IndexedAllocationInfo;
 using ::llvm::memprof::IndexedMemProfRecord;
 using ::llvm::memprof::MemInfoBlock;
 using ::llvm::memprof::MemProfReader;
@@ -36,6 +38,7 @@ using ::llvm::memprof::SegmentEntry;
 using ::llvm::object::SectionedAddress;
 using ::llvm::symbolize::SymbolizableModule;
 using ::testing::Return;
+using ::testing::SizeIs;
 
 class MockSymbolizer : public SymbolizableModule {
 public:
@@ -432,4 +435,86 @@ TEST(MemProf, BaseMemProfReader) {
   EXPECT_THAT(Records[0].AllocSites[0].CallStack[1],
               FrameContains("bar", 10U, 2U, false));
 }
+
+TEST(MemProf, IndexedMemProfRecordToMemProfRecord) {
+  // Verify that MemProfRecord can be constructed from IndexedMemProfRecord with
+  // CallStackIds only.
+
+  llvm::DenseMap<FrameId, Frame> FrameIdMap;
+  Frame F1(1, 0, 0, false);
+  Frame F2(2, 0, 0, false);
+  Frame F3(3, 0, 0, false);
+  Frame F4(4, 0, 0, false);
+  FrameIdMap.insert({F1.hash(), F1});
+  FrameIdMap.insert({F2.hash(), F2});
+  FrameIdMap.insert({F3.hash(), F3});
+  FrameIdMap.insert({F4.hash(), F4});
+
+  llvm::DenseMap<CallStackId, llvm::SmallVector<FrameId>> CallStackIdMap;
+  llvm::SmallVector<FrameId> CS1 = {F1.hash(), F2.hash()};
+  llvm::SmallVector<FrameId> CS2 = {F1.hash(), F3.hash()};
+  llvm::SmallVector<FrameId> CS3 = {F2.hash(), F3.hash()};
+  llvm::SmallVector<FrameId> CS4 = {F2.hash(), F4.hash()};
+  CallStackIdMap.insert({llvm::memprof::hashCallStack(CS1), CS1});
+  CallStackIdMap.insert({llvm::memprof::hashCallStack(CS2), CS2});
+  CallStackIdMap.insert({llvm::memprof::hashCallStack(CS3), CS3});
+  CallStackIdMap.insert({llvm::memprof::hashCallStack(CS4), CS4});
+
+  IndexedMemProfRecord IndexedRecord;
+  IndexedAllocationInfo AI;
+  AI.CSId = llvm::memprof::hashCallStack(CS1);
+  IndexedRecord.AllocSites.push_back(AI);
+  AI.CSId = llvm::memprof::hashCallStack(CS2);
+  IndexedRecord.AllocSites.push_back(AI);
+  IndexedRecord.CallSiteIds.push_back(llvm::memprof::hashCallStack(CS3));
+  IndexedRecord.CallSiteIds.push_back(llvm::memprof::hashCallStack(CS4));
+
+  bool CSIdMissing = false;
+  bool FrameIdMissing = false;
+
+  auto Callback = [&](CallStackId CSId) -> llvm::SmallVector<Frame> {
+    llvm::SmallVector<Frame> CallStack;
+    llvm::SmallVector<FrameId> FrameIds;
+
+    auto Iter = CallStackIdMap.find(CSId);
+    if (Iter == CallStackIdMap.end())
+      CSIdMissing = true;
+    else
+      FrameIds = Iter->second;
+
+    for (FrameId Id : FrameIds) {
+      Frame F(0, 0, 0, false);
+      auto Iter = FrameIdMap.find(Id);
+      if (Iter == FrameIdMap.end())
+        FrameIdMissing = true;
+      else
+        F = Iter->second;
+      CallStack.push_back(F);
+    }
+
+    return CallStack;
+  };
+
+  MemProfRecord Record = IndexedRecord.toMemProfRecord(Callback);
+
+  // Make sure that all lookups are successful.
+  ASSERT_FALSE(CSIdMissing);
+  ASSERT_FALSE(FrameIdMissing);
+
+  // Verify the contents of Record.
+  ASSERT_THAT(Record.AllocSites, SizeIs(2));
+  ASSERT_THAT(Record.AllocSites[0].CallStack, SizeIs(2));
+  EXPECT_EQ(Record.AllocSites[0].CallStack[0].hash(), F1.hash());
+  EXPECT_EQ(Record.AllocSites[0].CallStack[1].hash(), F2.hash());
+  ASSERT_THAT(Record.AllocSites[1].CallStack, SizeIs(2));
+  EXPECT_EQ(Record.AllocSites[1].CallStack[0].hash(), F1.hash());
+  EXPECT_EQ(Record.AllocSites[1].CallStack[1].hash(), F3.hash());
+  ASSERT_THAT(Record.CallSites, SizeIs(2));
+  ASSERT_THAT(Record.CallSites[0], SizeIs(2));
+  EXPECT_EQ(Record.CallSites[0][0].hash(), F2.hash());
+  EXPECT_EQ(Record.CallSites[0][1].hash(), F3.hash());
+  ASSERT_THAT(Record.CallSites[1], SizeIs(2));
+  EXPECT_EQ(Record.CallSites[1][0].hash(), F2.hash());
+  EXPECT_EQ(Record.CallSites[1][1].hash(), F4.hash());
+}
 } // namespace
-- 
GitLab


From 8cd8ebe153391993a3668d2ac8d2994d9491f3ef Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman@google.com>
Date: Tue, 16 Apr 2024 10:25:02 -0700
Subject: [PATCH 433/464] [MLGO] Use double comparison facilities for reg alloc
 scoring tests (#88862)

This patch switches from using direct equality (ASSERT_EQ) to the
floating point comparison facilities (ASSERT_DOUBLE_EQ) within google
test to avoid weird floating point problems. There is at least one
downstream that maintains a patch for issues cropping up from the direct
equality.


https://gitlab.alpinelinux.org/alpine/aports/-/blob/master/main/llvm17/allocscore.patch
---
 llvm/unittests/CodeGen/RegAllocScoreTest.cpp | 27 ++++++++++----------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/llvm/unittests/CodeGen/RegAllocScoreTest.cpp b/llvm/unittests/CodeGen/RegAllocScoreTest.cpp
index ff7146eaf943..eae517f9d01c 100644
--- a/llvm/unittests/CodeGen/RegAllocScoreTest.cpp
+++ b/llvm/unittests/CodeGen/RegAllocScoreTest.cpp
@@ -166,19 +166,20 @@ TEST(RegAllocScoreTest, Counts) {
   ASSERT_EQ(MF->size(), 2U);
   const auto TotalScore =
       llvm::calculateRegAllocScore(*MF, MBBFreqMock, IsRemat);
-  ASSERT_EQ(Freq1, TotalScore.copyCounts());
-  ASSERT_EQ(2.0 * Freq1 + Freq2, TotalScore.loadCounts());
-  ASSERT_EQ(Freq1 + Freq2, TotalScore.storeCounts());
-  ASSERT_EQ(Freq2, TotalScore.loadStoreCounts());
-  ASSERT_EQ(Freq1, TotalScore.cheapRematCounts());
-  ASSERT_EQ(Freq2, TotalScore.expensiveRematCounts());
-  ASSERT_EQ(TotalScore.getScore(),
-            TotalScore.copyCounts() * CopyWeight +
-                TotalScore.loadCounts() * LoadWeight +
-                TotalScore.storeCounts() * StoreWeight +
-                TotalScore.loadStoreCounts() * (LoadWeight + StoreWeight) +
-                TotalScore.cheapRematCounts() * CheapRematWeight +
-                TotalScore.expensiveRematCounts() * ExpensiveRematWeight
+  ASSERT_DOUBLE_EQ(Freq1, TotalScore.copyCounts());
+  ASSERT_DOUBLE_EQ(2.0 * Freq1 + Freq2, TotalScore.loadCounts());
+  ASSERT_DOUBLE_EQ(Freq1 + Freq2, TotalScore.storeCounts());
+  ASSERT_DOUBLE_EQ(Freq2, TotalScore.loadStoreCounts());
+  ASSERT_DOUBLE_EQ(Freq1, TotalScore.cheapRematCounts());
+  ASSERT_DOUBLE_EQ(Freq2, TotalScore.expensiveRematCounts());
+  ASSERT_DOUBLE_EQ(
+      TotalScore.getScore(),
+      TotalScore.copyCounts() * CopyWeight +
+          TotalScore.loadCounts() * LoadWeight +
+          TotalScore.storeCounts() * StoreWeight +
+          TotalScore.loadStoreCounts() * (LoadWeight + StoreWeight) +
+          TotalScore.cheapRematCounts() * CheapRematWeight +
+          TotalScore.expensiveRematCounts() * ExpensiveRematWeight
 
   );
 }
-- 
GitLab


From 75054525ae58f26c86e418382164540760871186 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Tue, 16 Apr 2024 18:42:23 +0200
Subject: [PATCH 434/464] [clang][Interp] Implement VectorSplat casts

---
 clang/lib/AST/Interp/ByteCodeExprGen.cpp | 29 ++++++++++++++++++++++++
 clang/lib/AST/Interp/ByteCodeExprGen.h   |  9 +++++++-
 clang/test/AST/Interp/vectors.cpp        | 17 ++++++++++++--
 3 files changed, 52 insertions(+), 3 deletions(-)

diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
index 00c4a9f16130..6b4b51aac41e 100644
--- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp
+++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
@@ -398,6 +398,35 @@ bool ByteCodeExprGen<Emitter>::VisitCastExpr(const CastExpr *CE) {
     return true;
   }
 
+  case CK_VectorSplat: {
+    assert(!classify(CE->getType()));
+    assert(classify(SubExpr->getType()));
+    assert(CE->getType()->isVectorType());
+
+    if (DiscardResult)
+      return this->discard(SubExpr);
+
+    assert(Initializing); // FIXME: Not always correct.
+    const auto *VT = CE->getType()->getAs<VectorType>();
+    PrimType ElemT = classifyPrim(SubExpr);
+    unsigned ElemOffset = allocateLocalPrimitive(
+        SubExpr, ElemT, /*IsConst=*/true, /*IsExtended=*/false);
+
+    if (!this->visit(SubExpr))
+      return false;
+    if (!this->emitSetLocal(ElemT, ElemOffset, CE))
+      return false;
+
+    for (unsigned I = 0; I != VT->getNumElements(); ++I) {
+      if (!this->emitGetLocal(ElemT, ElemOffset, CE))
+        return false;
+      if (!this->emitInitElem(ElemT, I, CE))
+        return false;
+    }
+
+    return true;
+  }
+
   case CK_ToVoid:
     return discard(SubExpr);
 
diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.h b/clang/lib/AST/Interp/ByteCodeExprGen.h
index db0d73ce23f7..7e9dc8631fc0 100644
--- a/clang/lib/AST/Interp/ByteCodeExprGen.h
+++ b/clang/lib/AST/Interp/ByteCodeExprGen.h
@@ -148,13 +148,20 @@ protected:
     return Ctx.classify(Ty);
   }
 
-  /// Classifies a known primitive type
+  /// Classifies a known primitive type.
   PrimType classifyPrim(QualType Ty) const {
     if (auto T = classify(Ty)) {
       return *T;
     }
     llvm_unreachable("not a primitive type");
   }
+  /// Classifies a known primitive expression.
+  PrimType classifyPrim(const Expr *E) const {
+    if (auto T = classify(E))
+      return *T;
+    llvm_unreachable("not a primitive type");
+  }
+
   /// Evaluates an expression and places the result on the stack. If the
   /// expression is of composite type, a local variable will be created
   /// and a pointer to said variable will be placed on the stack.
diff --git a/clang/test/AST/Interp/vectors.cpp b/clang/test/AST/Interp/vectors.cpp
index 6c5d916f51f5..5c4694f122d8 100644
--- a/clang/test/AST/Interp/vectors.cpp
+++ b/clang/test/AST/Interp/vectors.cpp
@@ -1,10 +1,23 @@
 // RUN: %clang_cc1 -fexperimental-new-constant-interpreter -verify=expected,both %s
 // RUN: %clang_cc1 -verify=ref,both %s
 
-// ref-no-diagnostics
-
 typedef int __attribute__((vector_size(16))) VI4;
 constexpr VI4 A = {1,2,3,4};
+static_assert(A[0] == 1, ""); // ref-error {{not an integral constant expression}}
+static_assert(A[1] == 2, ""); // ref-error {{not an integral constant expression}}
+static_assert(A[2] == 3, ""); // ref-error {{not an integral constant expression}}
+static_assert(A[3] == 4, ""); // ref-error {{not an integral constant expression}}
+
+/// VectorSplat casts
+typedef __attribute__(( ext_vector_type(4) )) float float4;
+constexpr float4 vec4_0 = (float4)0.5f;
+static_assert(vec4_0[0] == 0.5, ""); // ref-error {{not an integral constant expression}}
+static_assert(vec4_0[1] == 0.5, ""); // ref-error {{not an integral constant expression}}
+static_assert(vec4_0[2] == 0.5, ""); // ref-error {{not an integral constant expression}}
+static_assert(vec4_0[3] == 0.5, ""); // ref-error {{not an integral constant expression}}
+constexpr int vec4_0_discarded = ((float4)12.0f, 0);
+
+
 
 /// From constant-expression-cxx11.cpp
 namespace Vector {
-- 
GitLab


From 184ba038ac1d444980b3e554b0057f3f30c516ab Mon Sep 17 00:00:00 2001
From: Philip Reames <preames@rivosinc.com>
Date: Tue, 16 Apr 2024 10:46:27 -0700
Subject: [PATCH 435/464] [RISCV] Avoid matching 3/5/9 * 2^N as 2^N + 2/4/8
 (e.g. 24) (#88937)

The former is better as a zero extend can be folded into the sll,
whereas the later currently produces a seperate zext.w due to bad
interactions with other combines.
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp |  6 ++++++
 llvm/test/CodeGen/RISCV/addimm-mulimm.ll    |  9 ++++-----
 llvm/test/CodeGen/RISCV/rv64zba.ll          | 22 +++++++++++++++++++++
 3 files changed, 32 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 765838aafb58..de2ad639f0d6 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -13416,6 +13416,12 @@ static SDValue expandMul(SDNode *N, SelectionDAG &DAG,
     return SDValue();
   uint64_t MulAmt = CNode->getZExtValue();
 
+  // 3/5/9 * 2^N -> shXadd (sll X, C), (sll X, C)
+  // Matched in tablegen, avoid perturbing patterns.
+  for (uint64_t Divisor : {3, 5, 9})
+    if (MulAmt % Divisor == 0 && isPowerOf2_64(MulAmt / Divisor))
+      return SDValue();
+
   // If this is a power 2 + 2/4/8, we can use a shift followed by a single
   // shXadd. First check if this a sum of two power of 2s because that's
   // easy. Then count how many zeros are up to the first bit.
diff --git a/llvm/test/CodeGen/RISCV/addimm-mulimm.ll b/llvm/test/CodeGen/RISCV/addimm-mulimm.ll
index 10103f071462..48fa69e10456 100644
--- a/llvm/test/CodeGen/RISCV/addimm-mulimm.ll
+++ b/llvm/test/CodeGen/RISCV/addimm-mulimm.ll
@@ -551,9 +551,8 @@ define i64 @add_mul_combine_infinite_loop(i64 %x) {
 ; RV32IMB-NEXT:    sh3add a1, a1, a2
 ; RV32IMB-NEXT:    sh1add a0, a0, a0
 ; RV32IMB-NEXT:    slli a2, a0, 3
-; RV32IMB-NEXT:    li a3, 1
-; RV32IMB-NEXT:    slli a3, a3, 11
-; RV32IMB-NEXT:    sh3add a0, a0, a3
+; RV32IMB-NEXT:    addi a0, a2, 2047
+; RV32IMB-NEXT:    addi a0, a0, 1
 ; RV32IMB-NEXT:    sltu a2, a0, a2
 ; RV32IMB-NEXT:    add a1, a1, a2
 ; RV32IMB-NEXT:    ret
@@ -562,8 +561,8 @@ define i64 @add_mul_combine_infinite_loop(i64 %x) {
 ; RV64IMB:       # %bb.0:
 ; RV64IMB-NEXT:    addi a0, a0, 86
 ; RV64IMB-NEXT:    sh1add a0, a0, a0
-; RV64IMB-NEXT:    slli a0, a0, 3
-; RV64IMB-NEXT:    addi a0, a0, -16
+; RV64IMB-NEXT:    li a1, -16
+; RV64IMB-NEXT:    sh3add a0, a0, a1
 ; RV64IMB-NEXT:    ret
   %tmp0 = mul i64 %x, 24
   %tmp1 = add i64 %tmp0, 2048
diff --git a/llvm/test/CodeGen/RISCV/rv64zba.ll b/llvm/test/CodeGen/RISCV/rv64zba.ll
index a84b9e5e7962..c3c757656be9 100644
--- a/llvm/test/CodeGen/RISCV/rv64zba.ll
+++ b/llvm/test/CodeGen/RISCV/rv64zba.ll
@@ -2490,3 +2490,25 @@ define ptr @test_gep_gep_dont_crash(ptr %p, i64 %a1, i64 %a2) {
   %gep2 = getelementptr i64, ptr %gep1, i64 %a1
   ret ptr %gep2
 }
+
+define i64 @regression(i32 signext %x, i32 signext %y) {
+; RV64I-LABEL: regression:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    subw a0, a0, a1
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    li a1, 3
+; RV64I-NEXT:    slli a1, a1, 35
+; RV64I-NEXT:    mulhu a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64ZBA-LABEL: regression:
+; RV64ZBA:       # %bb.0:
+; RV64ZBA-NEXT:    subw a0, a0, a1
+; RV64ZBA-NEXT:    slli.uw a0, a0, 3
+; RV64ZBA-NEXT:    sh1add a0, a0, a0
+; RV64ZBA-NEXT:    ret
+  %sub = sub i32 %x, %y
+  %ext = zext i32 %sub to i64
+  %res = mul nuw nsw i64 %ext, 24
+  ret i64 %res
+}
-- 
GitLab


From 4082a7554521572a65a5a0008c4661a534df659d Mon Sep 17 00:00:00 2001
From: Aaron Ballman <aaron@aaronballman.com>
Date: Tue, 16 Apr 2024 13:48:13 -0400
Subject: [PATCH 436/464] Improve stack usage to increase recursive
 initialization depth (#88546)

We were crashing due to stack exhaustion on rather reasonable C++
template code. After some investigation, I found that we have a
stack-allocated object that was huge: `InitializationSequence` was 7016
bytes. This caused an overflow with deep call stacks in initialization
code.

With these change, `InitializationSequence` is now 248 bytes.

With the original code, testing RelWithDebInfo on Windows 10, all the
tests in SemaCXX took about 6s 800ms. The max template depth I could
reach on my machine using the code in the issue was 708. After that, I
would get `-Wstack-exhausted` warnings until crashing at 976
instantiations.

With these changes on the same machine, all the tests in SemaCXX took
about 6s 500ms. The max template depth I could reach was 1492. After
that, I would get `-Wstack-exhausted` warnings until crashing at 2898
instantiations.

This improves the behavior of #88330 but there's still an outstanding
question of why we run out of stack space and crash in some
circumstances before we're able to issue a diagnostic about stack space
exhaustion.
---
 clang/docs/ReleaseNotes.rst               |  6 ++
 clang/include/clang/Sema/Initialization.h |  6 +-
 clang/include/clang/Sema/Overload.h       | 70 ++++++-----------------
 clang/lib/Sema/SemaInit.cpp               | 26 +++++----
 clang/lib/Sema/SemaOverload.cpp           | 21 +++----
 5 files changed, 54 insertions(+), 75 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index d8ec8bcb8df5..e6c345a2f5c0 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -203,6 +203,12 @@ Non-comprehensive list of changes in this release
 - ``__typeof_unqual__`` is available in all C modes as an extension, which behaves
   like ``typeof_unqual`` from C23, similar to ``__typeof__`` and ``typeof``.
 
+- Improved stack usage with C++ initialization code. This allows significantly
+  more levels of recursive initialization before reaching stack exhaustion
+  limits. This will positively impact recursive template instantiation code,
+  but should also reduce memory overhead for initializations in general.
+  Fixes #GH88330
+
 New Compiler Flags
 ------------------
 - ``-fsanitize=implicit-bitfield-conversion`` checks implicit truncation and
diff --git a/clang/include/clang/Sema/Initialization.h b/clang/include/clang/Sema/Initialization.h
index 2072cd8d1c3e..1ceacf0f49f5 100644
--- a/clang/include/clang/Sema/Initialization.h
+++ b/clang/include/clang/Sema/Initialization.h
@@ -1134,7 +1134,7 @@ private:
   OverloadingResult FailedOverloadResult;
 
   /// The candidate set created when initialization failed.
-  OverloadCandidateSet FailedCandidateSet;
+  std::unique_ptr<OverloadCandidateSet> FailedCandidateSet;
 
   /// The incomplete type that caused a failure.
   QualType FailedIncompleteType;
@@ -1403,7 +1403,9 @@ public:
   /// Retrieve a reference to the candidate set when overload
   /// resolution fails.
   OverloadCandidateSet &getFailedCandidateSet() {
-    return FailedCandidateSet;
+    assert(FailedCandidateSet &&
+           "this should have been allocated in the constructor!");
+    return *FailedCandidateSet;
   }
 
   /// Get the overloading result, for when the initialization
diff --git a/clang/include/clang/Sema/Overload.h b/clang/include/clang/Sema/Overload.h
index 76311b00d2fc..e6f88bbf7c4f 100644
--- a/clang/include/clang/Sema/Overload.h
+++ b/clang/include/clang/Sema/Overload.h
@@ -37,6 +37,7 @@
 #include <cassert>
 #include <cstddef>
 #include <cstdint>
+#include <memory>
 #include <utility>
 
 namespace clang {
@@ -874,7 +875,8 @@ class Sema;
     ConversionFixItGenerator Fix;
 
     /// Viable - True to indicate that this overload candidate is viable.
-    bool Viable : 1;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned Viable : 1;
 
     /// Whether this candidate is the best viable function, or tied for being
     /// the best viable function.
@@ -883,12 +885,14 @@ class Sema;
     /// was part of the ambiguity kernel: the minimal non-empty set of viable
     /// candidates such that all elements of the ambiguity kernel are better
     /// than all viable candidates not in the ambiguity kernel.
-    bool Best : 1;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned Best : 1;
 
     /// IsSurrogate - True to indicate that this candidate is a
     /// surrogate for a conversion to a function pointer or reference
     /// (C++ [over.call.object]).
-    bool IsSurrogate : 1;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned IsSurrogate : 1;
 
     /// IgnoreObjectArgument - True to indicate that the first
     /// argument's conversion, which for this function represents the
@@ -897,18 +901,20 @@ class Sema;
     /// implicit object argument is just a placeholder) or a
     /// non-static member function when the call doesn't have an
     /// object argument.
-    bool IgnoreObjectArgument : 1;
+    LLVM_PREFERRED_TYPE(bool)
+    unsigned IgnoreObjectArgument : 1;
 
     /// True if the candidate was found using ADL.
-    CallExpr::ADLCallKind IsADLCandidate : 1;
+    LLVM_PREFERRED_TYPE(CallExpr::ADLCallKind)
+    unsigned IsADLCandidate : 1;
 
     /// Whether this is a rewritten candidate, and if so, of what kind?
     LLVM_PREFERRED_TYPE(OverloadCandidateRewriteKind)
     unsigned RewriteKind : 2;
 
     /// FailureKind - The reason why this candidate is not viable.
-    /// Actually an OverloadFailureKind.
-    unsigned char FailureKind;
+    LLVM_PREFERRED_TYPE(OverloadFailureKind)
+    unsigned FailureKind : 5;
 
     /// The number of call arguments that were explicitly provided,
     /// to be used while performing partial ordering of function templates.
@@ -972,7 +978,9 @@ class Sema;
   private:
     friend class OverloadCandidateSet;
     OverloadCandidate()
-        : IsSurrogate(false), IsADLCandidate(CallExpr::NotADL), RewriteKind(CRK_None) {}
+        : IsSurrogate(false),
+          IsADLCandidate(static_cast<unsigned>(CallExpr::NotADL)),
+          RewriteKind(CRK_None) {}
   };
 
   /// OverloadCandidateSet - A set of overload candidates, used in C++
@@ -1070,51 +1078,16 @@ class Sema;
     };
 
   private:
-    SmallVector<OverloadCandidate, 16> Candidates;
-    llvm::SmallPtrSet<uintptr_t, 16> Functions;
-
-    // Allocator for ConversionSequenceLists. We store the first few of these
-    // inline to avoid allocation for small sets.
-    llvm::BumpPtrAllocator SlabAllocator;
+    SmallVector<OverloadCandidate, 4> Candidates;
+    llvm::SmallPtrSet<uintptr_t, 4> Functions;
 
     SourceLocation Loc;
     CandidateSetKind Kind;
     OperatorRewriteInfo RewriteInfo;
 
-    constexpr static unsigned NumInlineBytes =
-        24 * sizeof(ImplicitConversionSequence);
-    unsigned NumInlineBytesUsed = 0;
-    alignas(void *) char InlineSpace[NumInlineBytes];
-
     // Address space of the object being constructed.
     LangAS DestAS = LangAS::Default;
 
-    /// If we have space, allocates from inline storage. Otherwise, allocates
-    /// from the slab allocator.
-    /// FIXME: It would probably be nice to have a SmallBumpPtrAllocator
-    /// instead.
-    /// FIXME: Now that this only allocates ImplicitConversionSequences, do we
-    /// want to un-generalize this?
-    template <typename T>
-    T *slabAllocate(unsigned N) {
-      // It's simpler if this doesn't need to consider alignment.
-      static_assert(alignof(T) == alignof(void *),
-                    "Only works for pointer-aligned types.");
-      static_assert(std::is_trivial<T>::value ||
-                        std::is_same<ImplicitConversionSequence, T>::value,
-                    "Add destruction logic to OverloadCandidateSet::clear().");
-
-      unsigned NBytes = sizeof(T) * N;
-      if (NBytes > NumInlineBytes - NumInlineBytesUsed)
-        return SlabAllocator.Allocate<T>(N);
-      char *FreeSpaceStart = InlineSpace + NumInlineBytesUsed;
-      assert(uintptr_t(FreeSpaceStart) % alignof(void *) == 0 &&
-             "Misaligned storage!");
-
-      NumInlineBytesUsed += NBytes;
-      return reinterpret_cast<T *>(FreeSpaceStart);
-    }
-
     void destroyCandidates();
 
   public:
@@ -1163,12 +1136,7 @@ class Sema;
     ConversionSequenceList
     allocateConversionSequences(unsigned NumConversions) {
       ImplicitConversionSequence *Conversions =
-          slabAllocate<ImplicitConversionSequence>(NumConversions);
-
-      // Construct the new objects.
-      for (unsigned I = 0; I != NumConversions; ++I)
-        new (&Conversions[I]) ImplicitConversionSequence();
-
+          new ImplicitConversionSequence[NumConversions];
       return ConversionSequenceList(Conversions, NumConversions);
     }
 
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index fb7a80ab0284..791c0b6e6df2 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -6114,7 +6114,8 @@ InitializationSequence::InitializationSequence(
     Sema &S, const InitializedEntity &Entity, const InitializationKind &Kind,
     MultiExprArg Args, bool TopLevelOfInitList, bool TreatUnavailableAsInvalid)
     : FailedOverloadResult(OR_Success),
-      FailedCandidateSet(Kind.getLocation(), OverloadCandidateSet::CSK_Normal) {
+      FailedCandidateSet(new OverloadCandidateSet(
+          Kind.getLocation(), OverloadCandidateSet::CSK_Normal)) {
   InitializeFrom(S, Entity, Kind, Args, TopLevelOfInitList,
                  TreatUnavailableAsInvalid);
 }
@@ -9735,7 +9736,7 @@ bool InitializationSequence::Diagnose(Sema &S,
     switch (FailedOverloadResult) {
     case OR_Ambiguous:
 
-      FailedCandidateSet.NoteCandidates(
+      FailedCandidateSet->NoteCandidates(
           PartialDiagnosticAt(
               Kind.getLocation(),
               Failure == FK_UserConversionOverloadFailed
@@ -9749,7 +9750,8 @@ bool InitializationSequence::Diagnose(Sema &S,
       break;
 
     case OR_No_Viable_Function: {
-      auto Cands = FailedCandidateSet.CompleteCandidates(S, OCD_AllCandidates, Args);
+      auto Cands =
+          FailedCandidateSet->CompleteCandidates(S, OCD_AllCandidates, Args);
       if (!S.RequireCompleteType(Kind.getLocation(),
                                  DestType.getNonReferenceType(),
                           diag::err_typecheck_nonviable_condition_incomplete,
@@ -9759,13 +9761,13 @@ bool InitializationSequence::Diagnose(Sema &S,
           << OnlyArg->getType() << Args[0]->getSourceRange()
           << DestType.getNonReferenceType();
 
-      FailedCandidateSet.NoteCandidates(S, Args, Cands);
+      FailedCandidateSet->NoteCandidates(S, Args, Cands);
       break;
     }
     case OR_Deleted: {
       OverloadCandidateSet::iterator Best;
-      OverloadingResult Ovl
-        = FailedCandidateSet.BestViableFunction(S, Kind.getLocation(), Best);
+      OverloadingResult Ovl =
+          FailedCandidateSet->BestViableFunction(S, Kind.getLocation(), Best);
 
       StringLiteral *Msg = Best->Function->getDeletedMessage();
       S.Diag(Kind.getLocation(), diag::err_typecheck_deleted_function)
@@ -9949,7 +9951,7 @@ bool InitializationSequence::Diagnose(Sema &S,
     // bad.
     switch (FailedOverloadResult) {
       case OR_Ambiguous:
-        FailedCandidateSet.NoteCandidates(
+        FailedCandidateSet->NoteCandidates(
             PartialDiagnosticAt(Kind.getLocation(),
                                 S.PDiag(diag::err_ovl_ambiguous_init)
                                     << DestType << ArgsRange),
@@ -10003,7 +10005,7 @@ bool InitializationSequence::Diagnose(Sema &S,
           break;
         }
 
-        FailedCandidateSet.NoteCandidates(
+        FailedCandidateSet->NoteCandidates(
             PartialDiagnosticAt(
                 Kind.getLocation(),
                 S.PDiag(diag::err_ovl_no_viable_function_in_init)
@@ -10013,8 +10015,8 @@ bool InitializationSequence::Diagnose(Sema &S,
 
       case OR_Deleted: {
         OverloadCandidateSet::iterator Best;
-        OverloadingResult Ovl
-          = FailedCandidateSet.BestViableFunction(S, Kind.getLocation(), Best);
+        OverloadingResult Ovl =
+            FailedCandidateSet->BestViableFunction(S, Kind.getLocation(), Best);
         if (Ovl != OR_Deleted) {
           S.Diag(Kind.getLocation(), diag::err_ovl_deleted_init)
               << DestType << ArgsRange;
@@ -10093,8 +10095,8 @@ bool InitializationSequence::Diagnose(Sema &S,
     S.Diag(Kind.getLocation(), diag::err_selected_explicit_constructor)
       << Args[0]->getSourceRange();
     OverloadCandidateSet::iterator Best;
-    OverloadingResult Ovl
-      = FailedCandidateSet.BestViableFunction(S, Kind.getLocation(), Best);
+    OverloadingResult Ovl =
+        FailedCandidateSet->BestViableFunction(S, Kind.getLocation(), Best);
     (void)Ovl;
     assert(Ovl == OR_Success && "Inconsistent overload resolution");
     CXXConstructorDecl *CtorDecl = cast<CXXConstructorDecl>(Best->Function);
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index 227ef564ba3e..bcde0d86cf10 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -1057,8 +1057,7 @@ bool OverloadCandidateSet::OperatorRewriteInfo::shouldAddReversed(
 
 void OverloadCandidateSet::destroyCandidates() {
   for (iterator i = begin(), e = end(); i != e; ++i) {
-    for (auto &C : i->Conversions)
-      C.~ImplicitConversionSequence();
+    delete[] i->Conversions.data();
     if (!i->Viable && i->FailureKind == ovl_fail_bad_deduction)
       i->DeductionFailure.Destroy();
   }
@@ -1066,8 +1065,6 @@ void OverloadCandidateSet::destroyCandidates() {
 
 void OverloadCandidateSet::clear(CandidateSetKind CSK) {
   destroyCandidates();
-  SlabAllocator.Reset();
-  NumInlineBytesUsed = 0;
   Candidates.clear();
   Functions.clear();
   Kind = CSK;
@@ -6983,7 +6980,7 @@ void Sema::AddOverloadCandidate(
   Candidate.RewriteKind =
       CandidateSet.getRewriteInfo().getRewriteKind(Function, PO);
   Candidate.IsSurrogate = false;
-  Candidate.IsADLCandidate = IsADLCandidate;
+  Candidate.IsADLCandidate = static_cast<unsigned>(IsADLCandidate);
   Candidate.IgnoreObjectArgument = false;
   Candidate.ExplicitCallArguments = Args.size();
 
@@ -7815,7 +7812,7 @@ void Sema::AddTemplateOverloadCandidate(
     Candidate.RewriteKind =
       CandidateSet.getRewriteInfo().getRewriteKind(Candidate.Function, PO);
     Candidate.IsSurrogate = false;
-    Candidate.IsADLCandidate = IsADLCandidate;
+    Candidate.IsADLCandidate = static_cast<unsigned>(IsADLCandidate);
     // Ignore the object argument if there is one, since we don't have an object
     // type.
     Candidate.IgnoreObjectArgument =
@@ -14125,7 +14122,8 @@ static ExprResult FinishOverloadedCallExpr(Sema &SemaRef, Scope *S, Expr *Fn,
       return ExprError();
     return SemaRef.BuildResolvedCallExpr(
         Res.get(), FDecl, LParenLoc, Args, RParenLoc, ExecConfig,
-        /*IsExecConfig=*/false, (*Best)->IsADLCandidate);
+        /*IsExecConfig=*/false,
+        static_cast<CallExpr::ADLCallKind>((*Best)->IsADLCandidate));
   }
 
   case OR_No_Viable_Function: {
@@ -14184,7 +14182,8 @@ static ExprResult FinishOverloadedCallExpr(Sema &SemaRef, Scope *S, Expr *Fn,
       return ExprError();
     return SemaRef.BuildResolvedCallExpr(
         Res.get(), FDecl, LParenLoc, Args, RParenLoc, ExecConfig,
-        /*IsExecConfig=*/false, (*Best)->IsADLCandidate);
+        /*IsExecConfig=*/false,
+        static_cast<CallExpr::ADLCallKind>((*Best)->IsADLCandidate));
   }
   }
 
@@ -14491,7 +14490,8 @@ Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, UnaryOperatorKind Opc,
       Args[0] = Input;
       CallExpr *TheCall = CXXOperatorCallExpr::Create(
           Context, Op, FnExpr.get(), ArgsArray, ResultTy, VK, OpLoc,
-          CurFPFeatureOverrides(), Best->IsADLCandidate);
+          CurFPFeatureOverrides(),
+          static_cast<CallExpr::ADLCallKind>(Best->IsADLCandidate));
 
       if (CheckCallReturnType(FnDecl->getReturnType(), OpLoc, TheCall, FnDecl))
         return ExprError();
@@ -14909,7 +14909,8 @@ ExprResult Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
         // members; CodeGen should take care not to emit the this pointer.
         TheCall = CXXOperatorCallExpr::Create(
             Context, ChosenOp, FnExpr.get(), Args, ResultTy, VK, OpLoc,
-            CurFPFeatureOverrides(), Best->IsADLCandidate);
+            CurFPFeatureOverrides(),
+            static_cast<CallExpr::ADLCallKind>(Best->IsADLCandidate));
 
         if (const auto *Method = dyn_cast<CXXMethodDecl>(FnDecl);
             Method && Method->isImplicitObjectMemberFunction()) {
-- 
GitLab


From aefff774a0d6f75565243263555f2513ac3c9fdf Mon Sep 17 00:00:00 2001
From: Vlad Serebrennikov <serebrennikov.vladislav@gmail.com>
Date: Tue, 16 Apr 2024 21:50:22 +0400
Subject: [PATCH 437/464] [clang] Migrate DR tests to `static_assert` (#88611)

This patch touches a number of tests that run in C++98 mode that have
been using array size as a context that requires a constant expression,
replacing it with a `static_assert` backported via a macro. This reduces
noise in expected directives that comes from diagnostics around VLAs.

This patch also showcases that DR tests would benefit from folding in
constant expressions in C++98 mode, but I'm not sure it's even on the
table. If it is, I'd be happy to prepare a PR for that, and rebase this
PR on top of it.

CC @AaronBallman
---
 clang/test/CXX/drs/dr0xx.cpp  | 14 +++++---
 clang/test/CXX/drs/dr16xx.cpp |  5 ++-
 clang/test/CXX/drs/dr1xx.cpp  | 61 +++++++++++++++++++----------------
 clang/test/CXX/drs/dr2xx.cpp  | 15 ++++++---
 clang/test/CXX/drs/dr3xx.cpp  | 38 ++++++++++++----------
 clang/test/CXX/drs/dr4xx.cpp  | 60 +++++++++++++++++-----------------
 clang/test/CXX/drs/dr5xx.cpp  | 13 +++++---
 clang/test/CXX/drs/dr6xx.cpp  |  6 ++--
 8 files changed, 117 insertions(+), 95 deletions(-)

diff --git a/clang/test/CXX/drs/dr0xx.cpp b/clang/test/CXX/drs/dr0xx.cpp
index a304862885c6..6c600bbc7c3f 100644
--- a/clang/test/CXX/drs/dr0xx.cpp
+++ b/clang/test/CXX/drs/dr0xx.cpp
@@ -5,6 +5,11 @@
 // RUN: %clang_cc1 -std=c++20 %s -verify=expected,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors -triple %itanium_abi_triple
 // RUN: %clang_cc1 -std=c++23 %s -verify=expected,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors -triple %itanium_abi_triple
 
+#if __cplusplus == 199711L
+#define static_assert(...) __extension__ _Static_assert(__VA_ARGS__)
+// cxx98-error@-1 {{variadic macros are a C99 feature}}
+#endif
+
 namespace cwg1 { // cwg1: no
   namespace X { extern "C" void cwg1_f(int a = 1); }
   namespace Y { extern "C" void cwg1_f(int a = 1); }
@@ -897,7 +902,7 @@ namespace cwg54 { // cwg54: 2.8
 
 namespace cwg55 { // cwg55: yes
   enum E { e = 5 };
-  int test[(e + 1 == 6) ? 1 : -1];
+  static_assert(e + 1 == 6, "");
 }
 
 namespace cwg56 { // cwg56: yes
@@ -1163,10 +1168,9 @@ namespace cwg75 { // cwg75: yes
 
 namespace cwg76 { // cwg76: yes
   const volatile int n = 1;
-  int arr[n]; // #cwg76-vla
-  // expected-error@#cwg76-vla {{variable length arrays in C++ are a Clang extension}}
-  //   expected-note@#cwg76-vla {{read of volatile-qualified type 'const volatile int' is not allowed in a constant expression}}
-  // expected-error@#cwg76-vla {{variable length array declaration not allowed at file scope}}
+  static_assert(n, "");
+  // expected-error@-1 {{static assertion expression is not an integral constant expression}}
+  //   expected-note@-2 {{read of volatile-qualified type 'const volatile int' is not allowed in a constant expression}}
 }
 
 namespace cwg77 { // cwg77: yes
diff --git a/clang/test/CXX/drs/dr16xx.cpp b/clang/test/CXX/drs/dr16xx.cpp
index 6d7bb7619f8b..cf6b45ceabf2 100644
--- a/clang/test/CXX/drs/dr16xx.cpp
+++ b/clang/test/CXX/drs/dr16xx.cpp
@@ -153,10 +153,9 @@ namespace cwg1645 { // cwg1645: 3.9
 
 namespace cwg1652 { // cwg1652: 3.6
   int a, b;
-  int arr[&a + 1 == &b ? 1 : 2];
-  // expected-error@-1 {{variable length arrays in C++ are a Clang extension}}
+  static_assert(&a + 1 == &b, "");
+  // expected-error@-1 {{static assertion expression is not an integral constant expression}}
   //   expected-note@-2 {{comparison against pointer '&a + 1' that points past the end of a complete object has unspecified value}}
-  // expected-error@-3 {{variable length array declaration not allowed at file scope}}
 }
 
 namespace cwg1653 { // cwg1653: 4 c++17
diff --git a/clang/test/CXX/drs/dr1xx.cpp b/clang/test/CXX/drs/dr1xx.cpp
index 5b497dda047d..a8f9b705a986 100644
--- a/clang/test/CXX/drs/dr1xx.cpp
+++ b/clang/test/CXX/drs/dr1xx.cpp
@@ -5,6 +5,17 @@
 // RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors
 
+#if __cplusplus == 199711L
+#define static_assert(...) __extension__ _Static_assert(__VA_ARGS__)
+// cxx98-error@-1 {{variadic macros are a C99 feature}}
+#endif
+
+#if __cplusplus == 199711L
+#define __enable_constant_folding(x) (__builtin_constant_p(x) ? (x) : (x))
+#else
+#define __enable_constant_folding
+#endif
+
 namespace cwg100 { // cwg100: yes
   template<const char (*)[4]> struct A {}; // #cwg100-A
   template<const char (&)[4]> struct B {}; // #cwg100-B
@@ -736,8 +747,8 @@ namespace cwg147 { // cwg147: yes
 
 namespace cwg148 { // cwg148: yes
   struct A { int A::*p; };
-  int check1[__is_pod(int(A::*)) ? 1 : -1];
-  int check2[__is_pod(A) ? 1 : -1];
+  static_assert(__is_pod(int(A::*)), "");
+  static_assert(__is_pod(A), "");
 }
 
 // cwg149: na
@@ -745,13 +756,7 @@ namespace cwg148 { // cwg148: yes
 namespace cwg151 { // cwg151: 3.1
   struct X {};
   typedef int X::*p;
-#if __cplusplus < 201103L
-#define fold(x) (__builtin_constant_p(0) ? (x) : (x))
-#else
-#define fold
-#endif
-  int check[fold(p() == 0) ? 1 : -1];
-#undef fold
+  static_assert(__enable_constant_folding(p() == 0), "");
 }
 
 namespace cwg152 { // cwg152: yes
@@ -956,42 +961,42 @@ namespace cwg171 {
 
 namespace cwg172 { // cwg172: yes
   enum { zero };
-  int check1[-1 < zero ? 1 : -1];
+  static_assert(-1 < zero, "");
 
   enum { x = -1, y = (unsigned int)-1 };
-  int check2[sizeof(x) > sizeof(int) ? 1 : -1];
+  static_assert(sizeof(x) > sizeof(int), "");
 
   enum { a = (unsigned int)-1 / 2 };
-  int check3a[sizeof(a) == sizeof(int) ? 1 : -1];
-  int check3b[-a < 0 ? 1 : -1];
+  static_assert(sizeof(a) == sizeof(int), "");
+  static_assert(-a < 0, "");
 
   enum { b = (unsigned int)-1 / 2 + 1 };
-  int check4a[sizeof(b) == sizeof(unsigned int) ? 1 : -1];
-  int check4b[-b > 0 ? 1 : -1];
+  static_assert(sizeof(b) == sizeof(unsigned int), "");
+  static_assert(-b > 0, "");
 
   enum { c = (unsigned long)-1 / 2 };
-  int check5a[sizeof(c) == sizeof(long) ? 1 : -1];
-  int check5b[-c < 0 ? 1 : -1];
+  static_assert(sizeof(c) == sizeof(long), "");
+  static_assert(-c < 0, "");
 
   enum { d = (unsigned long)-1 / 2 + 1 };
-  int check6a[sizeof(d) == sizeof(unsigned long) ? 1 : -1];
-  int check6b[-d > 0 ? 1 : -1];
+  static_assert(sizeof(d) == sizeof(unsigned long), "");
+  static_assert(-d > 0, "");
 
   enum { e = (unsigned long long)-1 / 2 };
   // cxx98-error@-1 {{'long long' is a C++11 extension}}
-  int check7a[sizeof(e) == sizeof(long) ? 1 : -1];
-  int check7b[-e < 0 ? 1 : -1];
+  static_assert(sizeof(e) == sizeof(long), "");
+  static_assert(-e < 0, "");
 
   enum { f = (unsigned long long)-1 / 2 + 1 };
   // cxx98-error@-1 {{'long long' is a C++11 extension}}
-  int check8a[sizeof(f) == sizeof(unsigned long) ? 1 : -1];
-  int check8b[-f > 0 ? 1 : -1];
+  static_assert(sizeof(f) == sizeof(unsigned long), "");
+  static_assert(-f > 0, "");
 }
 
 namespace cwg173 { // cwg173: yes
-  int check[('0' + 1 == '1' && '0' + 2 == '2' && '0' + 3 == '3' &&
-             '0' + 4 == '4' && '0' + 5 == '5' && '0' + 6 == '6' &&
-             '0' + 7 == '7' && '0' + 8 == '8' && '0' + 9 == '9') ? 1 : -1];
+  static_assert('0' + 1 == '1' && '0' + 2 == '2' && '0' + 3 == '3' &&
+                '0' + 4 == '4' && '0' + 5 == '5' && '0' + 6 == '6' &&
+                '0' + 7 == '7' && '0' + 8 == '8' && '0' + 9 == '9', "");
 }
 
 // cwg174: sup 1012
@@ -1070,7 +1075,7 @@ namespace cwg177 { // cwg177: yes
 }
 
 namespace cwg178 { // cwg178: yes
-  int check[int() == 0 ? 1 : -1];
+  static_assert(int() == 0, "");
 #if __cplusplus >= 201103L
   static_assert(int{} == 0, "");
   struct S { int a, b; };
@@ -1180,7 +1185,7 @@ namespace cwg187 { // cwg187: sup 481
 
 namespace cwg188 { // cwg188: yes
   char c[10];
-  int check[sizeof(0, c) == 10 ? 1 : -1];
+  static_assert(sizeof(0, c) == 10, "");
 }
 
 // cwg190 FIXME: add codegen test for tbaa
diff --git a/clang/test/CXX/drs/dr2xx.cpp b/clang/test/CXX/drs/dr2xx.cpp
index e655e7226d51..5d3e8ce4bea3 100644
--- a/clang/test/CXX/drs/dr2xx.cpp
+++ b/clang/test/CXX/drs/dr2xx.cpp
@@ -10,10 +10,15 @@
 typedef __SIZE_TYPE__ size_t;
 // cxx98-error@-1 0-1 {{'long long' is a C++11 extension}}
 
-#if __cplusplus < 201103L
-#define fold(x) (__builtin_constant_p(x) ? (x) : (x))
+#if __cplusplus == 199711L
+#define static_assert(...) __extension__ _Static_assert(__VA_ARGS__)
+// cxx98-error@-1 {{variadic macros are a C99 feature}}
+#endif
+
+#if __cplusplus == 199711L
+#define __enable_constant_folding(x) (__builtin_constant_p(x) ? (x) : (x))
 #else
-#define fold
+#define __enable_constant_folding
 #endif
 
 namespace cwg200 { // cwg200: dup 214
@@ -31,7 +36,7 @@ namespace cwg200 { // cwg200: dup 214
 namespace cwg202 { // cwg202: 3.1
   template<typename T> T f();
   template<int (*g)()> struct X {
-    int arr[fold(g == &f<int>) ? 1 : -1];
+    static_assert(__enable_constant_folding(g == &f<int>), "");
   };
   template struct X<f>;
 }
@@ -1024,7 +1029,7 @@ namespace cwg275 { // cwg275: no
 namespace cwg277 { // cwg277: 3.1
   typedef int *intp;
   int *p = intp();
-  int a[fold(intp() ? -1 : 1)];
+  static_assert(__enable_constant_folding(!intp()), "");
 }
 
 namespace cwg280 { // cwg280: 2.9
diff --git a/clang/test/CXX/drs/dr3xx.cpp b/clang/test/CXX/drs/dr3xx.cpp
index 6d1c6958ac8e..3e9228fe21fb 100644
--- a/clang/test/CXX/drs/dr3xx.cpp
+++ b/clang/test/CXX/drs/dr3xx.cpp
@@ -5,6 +5,17 @@
 // RUN: %clang_cc1 -std=c++11 -verify=expected,cxx98-14,cxx98-17,cxx98-20,cxx11-14,since-cxx11 -triple %itanium_abi_triple %s -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++98 -verify=expected,cxx98-14,cxx98-17,cxx98-20,cxx98 -triple %itanium_abi_triple %s -fexceptions -fcxx-exceptions -pedantic-errors
 
+#if __cplusplus == 199711L
+#define static_assert(...) __extension__ _Static_assert(__VA_ARGS__)
+// cxx98-error@-1 {{variadic macros are a C99 feature}}
+#endif
+
+#if __cplusplus == 199711L
+#define __enable_constant_folding(x) (__builtin_constant_p(x) ? (x) : (x))
+#else
+#define __enable_constant_folding
+#endif
+
 namespace cwg300 { // cwg300: yes
   template<typename R, typename A> void f(R (&)(A)) {}
   int g(int);
@@ -396,7 +407,7 @@ namespace cwg324 { // cwg324: 3.6
 
 namespace cwg326 { // cwg326: 3.1
   struct S {};
-  int test[__is_trivially_constructible(S, const S&) ? 1 : -1];
+  static_assert(__is_trivially_constructible(S, const S&), "");
 }
 
 namespace cwg327 { // cwg327: dup 538
@@ -653,7 +664,7 @@ namespace cwg339 { // cwg339: 2.8
 
   template<typename T> A<sizeof(f(T()))> make_A();
 
-  int a[conv_int<char>::value ? 1 : -1];
+  static_assert(conv_int<char>::value, "");
   bool b = conv_int2<char>(A<1>());
   A<1> c = make_A<char>();
 }
@@ -1099,21 +1110,14 @@ namespace cwg364 { // cwg364: yes
 #endif
 
 namespace cwg367 { // cwg367: yes
-  // FIXME: These diagnostics are terrible. Don't diagnose an ill-formed global
-  // array as being a VLA!
-  int a[true ? throw 0 : 4];
-  // expected-error@-1 {{variable length arrays in C++ are a Clang extension}}
-  // expected-error@-2 {{variable length array declaration not allowed at file scope}}
-  int b[true ? 4 : throw 0];
-  // cxx98-error@-1 {{variable length arrays in C++ are a Clang extension}}
-  // cxx98-error@-2 {{variable length array folded to constant array as an extension}}
-  int c[true ? *new int : 4];
-  // expected-error@-1 {{variable length arrays in C++ are a Clang extension}}
+  static_assert(__enable_constant_folding(true ? throw 0 : 4), "");
+  // expected-error@-1 {{expression is not an integral constant expression}}
+  static_assert(__enable_constant_folding(true ? 4 : throw 0), "");
+  static_assert(__enable_constant_folding(true ? *new int : 4), "");
+  // expected-error@-1 {{expression is not an integral constant expression}}
   //   expected-note@-2 {{read of uninitialized object is not allowed in a constant expression}}
-  // expected-error@-3 {{variable length array declaration not allowed at file scope}}
-  int d[true ? 4 : *new int];
-  // cxx98-error@-1 {{variable length arrays in C++ are a Clang extension}}
-  // cxx98-error@-2 {{variable length array folded to constant array as an extension}}
+  static_assert(__enable_constant_folding(true ? 4 : *new int), "");
+
 }
 
 namespace cwg368 { // cwg368: 3.6
@@ -1325,7 +1329,7 @@ namespace cwg383 { // cwg383: yes
   struct B { ~B(); };
   union C { C &operator=(const C&); };
   union D { ~D(); };
-  int check[(__is_pod(A) || __is_pod(B) || __is_pod(C) || __is_pod(D)) ? -1 : 1];
+  static_assert(!__is_pod(A) && !__is_pod(B) && !__is_pod(C) && !__is_pod(D), "");
 }
 
 namespace cwg384 { // cwg384: yes
diff --git a/clang/test/CXX/drs/dr4xx.cpp b/clang/test/CXX/drs/dr4xx.cpp
index 611b79147078..07162cc28f6b 100644
--- a/clang/test/CXX/drs/dr4xx.cpp
+++ b/clang/test/CXX/drs/dr4xx.cpp
@@ -6,6 +6,11 @@
 // RUN: env ASAN_OPTIONS=detect_stack_use_after_return=0 %clang_cc1 -std=c++23 %s -verify=expected,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: env ASAN_OPTIONS=detect_stack_use_after_return=0 %clang_cc1 -std=c++2c %s -verify=expected,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors
 
+#if __cplusplus == 199711L
+#define static_assert(...) __extension__ _Static_assert(__VA_ARGS__)
+// cxx98-error@-1 {{variadic macros are a C99 feature}}
+#endif
+
 // FIXME: __SIZE_TYPE__ expands to 'long long' on some targets.
 __extension__ typedef __SIZE_TYPE__ size_t;
 
@@ -217,7 +222,7 @@ namespace cwg407 { // cwg407: 3.8
 }
 
 namespace cwg408 { // cwg408: 3.4
-  template<int N> void g() { int arr[N != 1 ? 1 : -1]; }
+  template<int N> void g() { static_assert(N != 1, ""); }
   template<> void g<2>() { }
 
   template<typename T> struct S {
@@ -239,7 +244,7 @@ namespace cwg408 { // cwg408: 3.4
   };
   template<typename T> int R<T>::arr[1];
   template<typename T> void R<T>::f() {
-    int arr[sizeof(arr) != sizeof(int) ? 1 : -1];
+    static_assert(sizeof(arr) != sizeof(int), "");
   }
   template<> int R<int>::arr[2];
   template void R<int>::f();
@@ -842,11 +847,10 @@ namespace cwg451 { // cwg451: yes
   // expected-warning@-1 {{division by zero is undefined}}
   const int b = 1 / 0; // #cwg451-b
   // expected-warning@-1 {{division by zero is undefined}}
-  int arr[b]; // #cwg451-arr
-  // expected-error@-1 {{variable length arrays in C++ are a Clang extension}}
+  static_assert(b, "");
+  // expected-error@-1 {{expression is not an integral constant expression}}
   //   expected-note@-2 {{initializer of 'b' is not a constant expression}}
   //   expected-note@#cwg451-b {{declared here}}
-  // expected-error@#cwg451-arr {{variable length array declaration not allowed at file scope}}
 }
 
 namespace cwg452 { // cwg452: yes
@@ -876,11 +880,10 @@ namespace cwg456 { // cwg456: yes
 namespace cwg457 { // cwg457: yes
   const int a = 1;
   const volatile int b = 1;
-  int ax[a];
-  int bx[b];
-  // expected-error@-1 {{variable length arrays in C++ are a Clang extension}}
+  static_assert(a, "");
+  static_assert(b, "");
+  // expected-error@-1 {{expression is not an integral constant expression}}
   //   expected-note@-2 {{read of volatile-qualified type 'const volatile int' is not allowed in a constant expression}}
-  // expected-error@-3 {{variable length array declaration not allowed at file scope}}
 
   enum E {
     ea = a,
@@ -1276,20 +1279,18 @@ namespace cwg482 { // cwg482: 3.5
 
 namespace cwg483 { // cwg483: yes
   namespace climits {
-    int check1[__SCHAR_MAX__ >= 127 ? 1 : -1];
-    int check2[__SHRT_MAX__ >= 32767 ? 1 : -1];
-    int check3[__INT_MAX__ >= 32767 ? 1 : -1];
-    int check4[__LONG_MAX__ >= 2147483647 ? 1 : -1];
-    int check5[__LONG_LONG_MAX__ >= 9223372036854775807 ? 1 : -1];
-    // cxx98-error@-1 {{'long long' is a C++11 extension}}
-    // cxx98-error@-2 0-1{{'long long' is a C++11 extension}}
+    static_assert(__SCHAR_MAX__ >= 127, "");
+    static_assert(__SHRT_MAX__ >= 32767, "");
+    static_assert(__INT_MAX__ >= 32767, "");
+    static_assert(__LONG_MAX__ >= 2147483647, "");
+    static_assert(__LONG_LONG_MAX__ >= 9223372036854775807, "");
   }
   namespace cstdint {
-    int check1[__PTRDIFF_WIDTH__ >= 16 ? 1 : -1];
-    int check2[__SIG_ATOMIC_WIDTH__ >= 8 ? 1 : -1];
-    int check3[__SIZE_WIDTH__ >= 16 ? 1 : -1];
-    int check4[__WCHAR_WIDTH__ >= 8 ? 1 : -1];
-    int check5[__WINT_WIDTH__ >= 16 ? 1 : -1];
+    static_assert(__PTRDIFF_WIDTH__ >= 16, "");
+    static_assert(__SIG_ATOMIC_WIDTH__ >= 8, "");
+    static_assert(__SIZE_WIDTH__ >= 16, "");
+    static_assert(__WCHAR_WIDTH__ >= 8, "");
+    static_assert(__WINT_WIDTH__ >= 16, "");
   }
 }
 
@@ -1366,11 +1367,10 @@ namespace cwg486 { // cwg486: yes
 namespace cwg487 { // cwg487: yes
   enum E { e };
   int operator+(int, E); // #cwg487-operator-plus
-  int i[4 + e]; // #cwg487-i
-  // expected-error@-1 {{variable length arrays in C++ are a Clang extension}}
+  static_assert(4 + e, "");
+  // expected-error@-1 {{expression is not an integral constant expression}}
   //   since-cxx11-note@-2 {{non-constexpr function 'operator+' cannot be used in a constant expression}}
   //   since-cxx11-note@#cwg487-operator-plus {{declared here}}
-  // expected-error@#cwg487-i {{variable length array declaration not allowed at file scope}}
 }
 
 namespace cwg488 { // cwg488: yes c++11
@@ -1485,13 +1485,13 @@ namespace cwg495 { // cwg495: 3.5
 namespace cwg496 { // cwg496: sup 2094
   struct A { int n; };
   struct B { volatile int n; };
-  int check1[ __is_trivially_copyable(const int) ? 1 : -1];
+  static_assert(__is_trivially_copyable(const int), "");
   // This checks the cwg2094 behavior, not cwg496
-  int check2[ __is_trivially_copyable(volatile int) ? 1 : -1];
-  int check3[ __is_trivially_constructible(A, const A&) ? 1 : -1];
-  int check4[ __is_trivially_constructible(B, const B&) ? 1 : -1];
-  int check5[ __is_trivially_assignable(A, const A&) ? 1 : -1];
-  int check6[ __is_trivially_assignable(B, const B&) ? 1 : -1];
+  static_assert(__is_trivially_copyable(volatile int), "");
+  static_assert(__is_trivially_constructible(A, const A&), "");
+  static_assert(__is_trivially_constructible(B, const B&), "");
+  static_assert(__is_trivially_assignable(A, const A&), "");
+  static_assert(__is_trivially_assignable(B, const B&), "");
 }
 
 namespace cwg497 { // cwg497: sup 253
diff --git a/clang/test/CXX/drs/dr5xx.cpp b/clang/test/CXX/drs/dr5xx.cpp
index 0fe64102d70b..9d890f981348 100644
--- a/clang/test/CXX/drs/dr5xx.cpp
+++ b/clang/test/CXX/drs/dr5xx.cpp
@@ -5,6 +5,11 @@
 // RUN: %clang_cc1 -std=c++20 %s -verify=expected,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++23 %s -verify=expected,since-cxx23,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors
 
+#if __cplusplus == 199711L
+#define static_assert(...) __extension__ _Static_assert(__VA_ARGS__)
+// cxx98-error@-1 {{variadic macros are a C99 feature}}
+#endif
+
 // FIXME: This is included to avoid a diagnostic with no source location
 // pointing at the implicit operator new. We can't match such a diagnostic
 // with -verify.
@@ -819,7 +824,7 @@ namespace cwg565 { // cwg565: yes
 
 namespace cwg566 { // cwg566: yes
 #if __cplusplus >= 201103L
-  int check[int(-3.99) == -3 ? 1 : -1];
+  static_assert(int(-3.99) == -3, "");
 #endif
 }
 
@@ -834,7 +839,7 @@ namespace cwg568 { // cwg568: 3.0 c++11
   public:
     int n;
   };
-  int check_trivial[__is_trivial(trivial) ? 1 : -1];
+  static_assert(__is_trivial(trivial), "");
 
   struct std_layout {
     std_layout();
@@ -843,7 +848,7 @@ namespace cwg568 { // cwg568: 3.0 c++11
   private:
     int n;
   };
-  int check_std_layout[__is_standard_layout(std_layout) ? 1 : -1];
+  static_assert(__is_standard_layout(std_layout), "");
 
   struct aggregate {
     int x;
@@ -885,7 +890,7 @@ namespace cwg570 { // cwg570: dup 633
 
 namespace cwg572 { // cwg572: yes
   enum E { a = 1, b = 2 };
-  int check[a + b == 3 ? 1 : -1];
+  static_assert(a + b == 3, "");
 }
 
 namespace cwg573 { // cwg573: no
diff --git a/clang/test/CXX/drs/dr6xx.cpp b/clang/test/CXX/drs/dr6xx.cpp
index 9d3613ae8589..069102d9c597 100644
--- a/clang/test/CXX/drs/dr6xx.cpp
+++ b/clang/test/CXX/drs/dr6xx.cpp
@@ -144,7 +144,7 @@ namespace cwg608 { // cwg608: yes
   struct D : B, C {};
 }
 
-int cwg610[-0u == 0u ? 1 : -1]; // cwg610: yes
+static_assert(-0u == 0u, ""); // cwg610: yes
 
 namespace cwg611 { // cwg611: yes
   int k;
@@ -190,8 +190,8 @@ namespace cwg613 { // cwg613: yes c++11
   }
 }
 
-int cwg614_a[(-1) / 2 == 0 ? 1 : -1]; // cwg614: yes
-int cwg614_b[(-1) % 2 == -1 ? 1 : -1];
+static_assert((-1) / 2 == 0, ""); // cwg614: yes
+static_assert((-1) % 2 == -1, "");
 
 namespace cwg615 { // cwg615: yes
   int f();
-- 
GitLab


From 6b83fe552990966fdad0e5693a79b02b87d9526e Mon Sep 17 00:00:00 2001
From: Philip Reames <preames@rivosinc.com>
Date: Tue, 16 Apr 2024 11:03:53 -0700
Subject: [PATCH 438/464] [RISCV] Strength reduce mul by 2^n + 2/4/8 + 1
 (#88911)

With zba, we can expand this to (add (shl X, C1), (shXadd X, X)).

Note that this is our first expansion to a three instruction sequence. I
believe this to general be a reasonable tradeoff for most architectures,
but we may want to (someday) consider a tuning flag here.

I plan to support 2^n + (2/4/8 + 1) eventually as well, but that comes
behind 2^N - 2^M. Both are also three instruction sequences.

---------

Co-authored-by: Min-Yih Hsu <min@myhsu.dev>
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 37 +++++++++++++++
 llvm/test/CodeGen/RISCV/rv64zba.ll          | 51 +++++++++++++++------
 2 files changed, 73 insertions(+), 15 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index de2ad639f0d6..dc7c6f83b985 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -13437,6 +13437,43 @@ static SDValue expandMul(SDNode *N, SelectionDAG &DAG,
       return DAG.getNode(ISD::ADD, DL, VT, Shift1, Shift2);
     }
   }
+
+  // 2^(1,2,3) * 3,5,9 + 1 -> (shXadd (shYadd x, x), x)
+  // Matched in tablegen, avoid perturbing patterns.
+  switch (MulAmt) {
+  case 11:
+  case 13:
+  case 19:
+  case 21:
+  case 25:
+  case 27:
+  case 29:
+  case 37:
+  case 41:
+  case 45:
+  case 73:
+  case 91:
+    return SDValue();
+  default:
+    break;
+  }
+
+  // 2^n + 2/4/8 + 1 -> (add (shl X, C1), (shXadd X, X))
+  if (MulAmt > 2 && isPowerOf2_64((MulAmt - 1) & (MulAmt - 2))) {
+    unsigned ScaleShift = llvm::countr_zero(MulAmt - 1);
+    if (ScaleShift >= 1 && ScaleShift < 4) {
+      unsigned ShiftAmt = Log2_64(((MulAmt - 1) & (MulAmt - 2)));
+      SDLoc DL(N);
+      SDValue Shift1 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
+                                   DAG.getConstant(ShiftAmt, DL, VT));
+      SDValue Shift2 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
+                                   DAG.getConstant(ScaleShift, DL, VT));
+      return DAG.getNode(
+          ISD::ADD, DL, VT, Shift1,
+          DAG.getNode(ISD::ADD, DL, VT, Shift2, N->getOperand(0)));
+    }
+  }
+
   return SDValue();
 }
 
diff --git a/llvm/test/CodeGen/RISCV/rv64zba.ll b/llvm/test/CodeGen/RISCV/rv64zba.ll
index c3c757656be9..b4c80b60e0ba 100644
--- a/llvm/test/CodeGen/RISCV/rv64zba.ll
+++ b/llvm/test/CodeGen/RISCV/rv64zba.ll
@@ -598,31 +598,52 @@ define i64 @mul125(i64 %a) {
 }
 
 define i64 @mul131(i64 %a) {
-; CHECK-LABEL: mul131:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 131
-; CHECK-NEXT:    mul a0, a0, a1
-; CHECK-NEXT:    ret
+; RV64I-LABEL: mul131:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    li a1, 131
+; RV64I-NEXT:    mul a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64ZBA-LABEL: mul131:
+; RV64ZBA:       # %bb.0:
+; RV64ZBA-NEXT:    sh1add a1, a0, a0
+; RV64ZBA-NEXT:    slli a0, a0, 7
+; RV64ZBA-NEXT:    add a0, a0, a1
+; RV64ZBA-NEXT:    ret
   %c = mul i64 %a, 131
   ret i64 %c
 }
 
 define i64 @mul133(i64 %a) {
-; CHECK-LABEL: mul133:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 133
-; CHECK-NEXT:    mul a0, a0, a1
-; CHECK-NEXT:    ret
+; RV64I-LABEL: mul133:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    li a1, 133
+; RV64I-NEXT:    mul a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64ZBA-LABEL: mul133:
+; RV64ZBA:       # %bb.0:
+; RV64ZBA-NEXT:    sh2add a1, a0, a0
+; RV64ZBA-NEXT:    slli a0, a0, 7
+; RV64ZBA-NEXT:    add a0, a0, a1
+; RV64ZBA-NEXT:    ret
   %c = mul i64 %a, 133
   ret i64 %c
 }
 
 define i64 @mul137(i64 %a) {
-; CHECK-LABEL: mul137:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 137
-; CHECK-NEXT:    mul a0, a0, a1
-; CHECK-NEXT:    ret
+; RV64I-LABEL: mul137:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    li a1, 137
+; RV64I-NEXT:    mul a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64ZBA-LABEL: mul137:
+; RV64ZBA:       # %bb.0:
+; RV64ZBA-NEXT:    sh3add a1, a0, a0
+; RV64ZBA-NEXT:    slli a0, a0, 7
+; RV64ZBA-NEXT:    add a0, a0, a1
+; RV64ZBA-NEXT:    ret
   %c = mul i64 %a, 137
   ret i64 %c
 }
-- 
GitLab


From 1c2afbae9af22b58190c10e3517242d01d89d612 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Tue, 16 Apr 2024 11:05:45 -0700
Subject: [PATCH 439/464] [CodeGen,test] Test llvm-libc style alias attribute
 with UsingShadowDecl

The pattern is quite involved and deserves a specific codegen test.
This test would catch the bug in the first attempt of #87130
---
 clang/test/CodeGen/alias.cpp | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/clang/test/CodeGen/alias.cpp b/clang/test/CodeGen/alias.cpp
index 17c1e1ae32f0..a468c31d369e 100644
--- a/clang/test/CodeGen/alias.cpp
+++ b/clang/test/CodeGen/alias.cpp
@@ -1,27 +1,42 @@
-// RUN: %clang_cc1 -triple x86_64-linux -verify -emit-llvm-only %s
-// RUN: not %clang_cc1 -triple x86_64-linux -emit-llvm-only -fdiagnostics-parseable-fixits %s 2>&1 | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-linux -verify -emit-llvm-only -DERR %s
+// RUN: not %clang_cc1 -triple x86_64-linux -emit-llvm-only -fdiagnostics-parseable-fixits -DERR %s 2>&1 | FileCheck %s --check-prefix=FIXIT
+// RUN: %clang_cc1 -triple x86_64-linux -emit-llvm %s -o - | FileCheck %s
 
+#ifdef ERR
 void *f1_ifunc(void) { return nullptr; }
 void f1(void) __attribute__((alias("f1_ifunc")));
 // expected-error@-1 {{alias must point to a defined variable or function}}
 // expected-note@-2 {{must refer to its mangled name}}
 // expected-note@-3 {{function by that name is mangled as}}
-// CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:30-[[@LINE-4]]:47}:"alias(\"_Z8f1_ifuncv\")"
+// FIXIT: fix-it:"{{.*}}":{[[@LINE-4]]:30-[[@LINE-4]]:47}:"alias(\"_Z8f1_ifuncv\")"
 
 void *f6_resolver_resolver(void) { return 0; }
 void *f6_resolver(void) __attribute__((alias("f6_resolver_resolver")));
 // expected-error@-1 {{alias must point to a defined variable or function}}
 // expected-note@-2 {{must refer to its mangled name}}
 // expected-note@-3 {{function by that name is mangled as}}
-// CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:40-[[@LINE-4]]:69}:"alias(\"_Z20f6_resolver_resolverv\")"
+// FIXIT: fix-it:"{{.*}}":{[[@LINE-4]]:40-[[@LINE-4]]:69}:"alias(\"_Z20f6_resolver_resolverv\")"
 void f6(void) __attribute__((alias("f6_resolver")));
 // expected-error@-1 {{alias must point to a defined variable or function}}
 // expected-note@-2 {{must refer to its mangled name}}
 // expected-note@-3 {{function by that name is mangled as}}
-// CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:30-[[@LINE-4]]:50}:"alias(\"_Z11f6_resolverv\")"
+// FIXIT: fix-it:"{{.*}}":{[[@LINE-4]]:30-[[@LINE-4]]:50}:"alias(\"_Z11f6_resolverv\")"
 
 __attribute__((unused, alias("resolver"), deprecated("hahahaha, isn't C great?")))
 void func();
 // expected-error@-2 {{alias must point to a defined variable or function}}
 // expected-note@-3 {{must refer to its mangled name}}
+#endif
 
+// CHECK: @_ZN4libc4log2Ed ={{.*}} alias double (double), ptr @log2
+// CHECK: define{{.*}} @log2(
+namespace libc { double log2(double x); }
+extern "C" double log2(double);
+namespace std { using ::log2; }
+using std::log2;
+
+namespace libc {
+decltype(libc::log2) __log2_impl__ __asm__("log2");
+decltype(libc::log2) log2 [[gnu::alias("log2")]];
+double __log2_impl__(double x) { return x; }
+}
-- 
GitLab


From 5462b27026dee886fb896980d6ad9487200a6cbe Mon Sep 17 00:00:00 2001
From: Mark de Wever <koraq@xs4all.nl>
Date: Tue, 16 Apr 2024 20:07:03 +0200
Subject: [PATCH 440/464] [NFC][libc++][TZDB] Refactors argument order.
 (#85781)

Putting the output reference argument first looks more sensible.
---
 libcxx/include/__chrono/formatter.h | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/libcxx/include/__chrono/formatter.h b/libcxx/include/__chrono/formatter.h
index 4ad59382a414..b64cae529a29 100644
--- a/libcxx/include/__chrono/formatter.h
+++ b/libcxx/include/__chrono/formatter.h
@@ -79,7 +79,7 @@ namespace __formatter {
 // small). Therefore a duration uses its own conversion.
 template <class _CharT, class _Rep, class _Period>
 _LIBCPP_HIDE_FROM_ABI void
-__format_sub_seconds(const chrono::duration<_Rep, _Period>& __value, basic_stringstream<_CharT>& __sstr) {
+__format_sub_seconds(basic_stringstream<_CharT>& __sstr, const chrono::duration<_Rep, _Period>& __value) {
   __sstr << std::use_facet<numpunct<_CharT>>(__sstr.getloc()).decimal_point();
 
   using __duration = chrono::duration<_Rep, _Period>;
@@ -110,13 +110,13 @@ __format_sub_seconds(const chrono::duration<_Rep, _Period>& __value, basic_strin
 }
 
 template <class _CharT, __is_time_point _Tp>
-_LIBCPP_HIDE_FROM_ABI void __format_sub_seconds(const _Tp& __value, basic_stringstream<_CharT>& __sstr) {
-  __formatter::__format_sub_seconds(__value.time_since_epoch(), __sstr);
+_LIBCPP_HIDE_FROM_ABI void __format_sub_seconds(basic_stringstream<_CharT>& __sstr, const _Tp& __value) {
+  __formatter::__format_sub_seconds(__sstr, __value.time_since_epoch());
 }
 
 template <class _CharT, class _Duration>
 _LIBCPP_HIDE_FROM_ABI void
-__format_sub_seconds(const chrono::hh_mm_ss<_Duration>& __value, basic_stringstream<_CharT>& __sstr) {
+__format_sub_seconds(basic_stringstream<_CharT>& __sstr, const chrono::hh_mm_ss<_Duration>& __value) {
   __sstr << std::use_facet<numpunct<_CharT>>(__sstr.getloc()).decimal_point();
   if constexpr (chrono::treat_as_floating_point_v<typename _Duration::rep>)
     std::format_to(std::ostreambuf_iterator<_CharT>{__sstr},
@@ -143,7 +143,7 @@ consteval bool __use_fraction() {
 }
 
 template <class _CharT>
-_LIBCPP_HIDE_FROM_ABI void __format_year(int __year, basic_stringstream<_CharT>& __sstr) {
+_LIBCPP_HIDE_FROM_ABI void __format_year(basic_stringstream<_CharT>& __sstr, int __year) {
   if (__year < 0) {
     __sstr << _CharT('-');
     __year = -__year;
@@ -159,7 +159,7 @@ _LIBCPP_HIDE_FROM_ABI void __format_year(int __year, basic_stringstream<_CharT>&
 }
 
 template <class _CharT>
-_LIBCPP_HIDE_FROM_ABI void __format_century(int __year, basic_stringstream<_CharT>& __sstr) {
+_LIBCPP_HIDE_FROM_ABI void __format_century(basic_stringstream<_CharT>& __sstr, int __year) {
   // TODO FMT Write an issue
   // [tab:time.format.spec]
   //   %C The year divided by 100 using floored division. If the result is a
@@ -172,7 +172,7 @@ _LIBCPP_HIDE_FROM_ABI void __format_century(int __year, basic_stringstream<_Char
 
 template <class _CharT, class _Tp>
 _LIBCPP_HIDE_FROM_ABI void __format_chrono_using_chrono_specs(
-    const _Tp& __value, basic_stringstream<_CharT>& __sstr, basic_string_view<_CharT> __chrono_specs) {
+    basic_stringstream<_CharT>& __sstr, const _Tp& __value, basic_string_view<_CharT> __chrono_specs) {
   tm __t              = std::__convert_to_tm<tm>(__value);
   const auto& __facet = std::use_facet<time_put<_CharT>>(__sstr.getloc());
   for (auto __it = __chrono_specs.begin(); __it != __chrono_specs.end(); ++__it) {
@@ -196,7 +196,7 @@ _LIBCPP_HIDE_FROM_ABI void __format_chrono_using_chrono_specs(
         // strftime's output is only defined in the range [00, 99].
         int __year = __t.tm_year + 1900;
         if (__year < 1000 || __year > 9999)
-          __formatter::__format_century(__year, __sstr);
+          __formatter::__format_century(__sstr, __year);
         else
           __facet.put(
               {__sstr}, __sstr, _CharT(' '), std::addressof(__t), std::to_address(__s), std::to_address(__it + 1));
@@ -242,7 +242,7 @@ _LIBCPP_HIDE_FROM_ABI void __format_chrono_using_chrono_specs(
         __facet.put(
             {__sstr}, __sstr, _CharT(' '), std::addressof(__t), std::to_address(__s), std::to_address(__it + 1));
         if constexpr (__use_fraction<_Tp>())
-          __formatter::__format_sub_seconds(__value, __sstr);
+          __formatter::__format_sub_seconds(__sstr, __value);
         break;
 
         // Unlike time_put and strftime the formatting library requires %Y
@@ -283,13 +283,13 @@ _LIBCPP_HIDE_FROM_ABI void __format_chrono_using_chrono_specs(
         // Depending on the platform's libc the range of supported years is
         // limited. Intead of of testing all conditions use the internal
         // implementation unconditionally.
-        __formatter::__format_year(__t.tm_year + 1900, __sstr);
+        __formatter::__format_year(__sstr, __t.tm_year + 1900);
         break;
 
       case _CharT('F'): {
         int __year = __t.tm_year + 1900;
         if (__year < 1000) {
-          __formatter::__format_year(__year, __sstr);
+          __formatter::__format_year(__sstr, __year);
           __sstr << std::format(_LIBCPP_STATICALLY_WIDEN(_CharT, "-{:02}-{:02}"), __t.tm_mon + 1, __t.tm_mday);
         } else
           __facet.put(
@@ -310,7 +310,7 @@ _LIBCPP_HIDE_FROM_ABI void __format_chrono_using_chrono_specs(
             ++__it;
             __facet.put(
                 {__sstr}, __sstr, _CharT(' '), std::addressof(__t), std::to_address(__s), std::to_address(__it + 1));
-            __formatter::__format_sub_seconds(__value, __sstr);
+            __formatter::__format_sub_seconds(__sstr, __value);
             break;
           }
         }
@@ -512,7 +512,7 @@ __format_chrono(const _Tp& __value,
     if constexpr (chrono::__is_duration<_Tp>::value) {
       if (__value < __value.zero())
         __sstr << _CharT('-');
-      __formatter::__format_chrono_using_chrono_specs(chrono::abs(__value), __sstr, __chrono_specs);
+      __formatter::__format_chrono_using_chrono_specs(__sstr, chrono::abs(__value), __chrono_specs);
       // TODO FMT When keeping the precision it will truncate the string.
       // Note that the behaviour what the precision does isn't specified.
       __specs.__precision_ = -1;
@@ -556,7 +556,7 @@ __format_chrono(const _Tp& __value,
           __sstr << _CharT('-');
       }
 
-      __formatter::__format_chrono_using_chrono_specs(__value, __sstr, __chrono_specs);
+      __formatter::__format_chrono_using_chrono_specs(__sstr, __value, __chrono_specs);
     }
   }
 
-- 
GitLab


From a75c9d059791f5d175f6c263d114d59e51b46120 Mon Sep 17 00:00:00 2001
From: Mark de Wever <koraq@xs4all.nl>
Date: Tue, 16 Apr 2024 20:18:34 +0200
Subject: [PATCH 441/464] [NFC][libc++] Moves ios_base's forward declaration.
 (#88027)

According to our synopsis it belonged to ios_fwd. This is not true in
the C++11 version of the Standard, I did not validate against C++98.

Moving this to ios's forward where it's declared in the standard allows
removing a module quirk. An earlier removal of std::vectors forward
declaration allows to remove all quirks for the iosfwd module part.

Since iosfwd includes __fwd/ios.h this does not change the required
includes.
---
 libcxx/include/__fwd/ios.h          | 2 ++
 libcxx/include/iosfwd               | 3 ---
 libcxx/utils/libcxx/test/modules.py | 2 --
 3 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/libcxx/include/__fwd/ios.h b/libcxx/include/__fwd/ios.h
index 82c865d58cc7..48350709d4ce 100644
--- a/libcxx/include/__fwd/ios.h
+++ b/libcxx/include/__fwd/ios.h
@@ -18,6 +18,8 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
+class _LIBCPP_EXPORTED_FROM_ABI ios_base;
+
 template <class _CharT, class _Traits = char_traits<_CharT> >
 class _LIBCPP_TEMPLATE_VIS basic_ios;
 
diff --git a/libcxx/include/iosfwd b/libcxx/include/iosfwd
index 9af5e0503185..2481667dd972 100644
--- a/libcxx/include/iosfwd
+++ b/libcxx/include/iosfwd
@@ -25,7 +25,6 @@ template<>            struct char_traits<wchar_t>;
 
 template<class T>     class allocator;
 
-class ios_base;
 template <class charT, class traits = char_traits<charT> > class basic_ios;
 
 template <class charT, class traits = char_traits<charT> > class basic_streambuf;
@@ -124,8 +123,6 @@ using wosyncstream = basic_osyncstream<wchar_t>;  // C++20
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-class _LIBCPP_EXPORTED_FROM_ABI ios_base;
-
 template <class _CharT, class _Traits = char_traits<_CharT> >
 class _LIBCPP_TEMPLATE_VIS istreambuf_iterator;
 template <class _CharT, class _Traits = char_traits<_CharT> >
diff --git a/libcxx/utils/libcxx/test/modules.py b/libcxx/utils/libcxx/test/modules.py
index 3f3c7999a1a2..44c6292ff114 100644
--- a/libcxx/utils/libcxx/test/modules.py
+++ b/libcxx/utils/libcxx/test/modules.py
@@ -26,8 +26,6 @@ SkipDeclarations["cwchar"] = ["std::FILE"]
 # The operators are added for private types like __iom_t10.
 SkipDeclarations["iomanip"] = ["std::operator<<", "std::operator>>"]
 
-SkipDeclarations["iosfwd"] = ["std::ios_base", "std::vector"]
-
 # This header also provides declarations in the namespace that might be
 # an error.
 SkipDeclarations["filesystem"] = [
-- 
GitLab


From 9cd3e92f05fcc2c9168a7abc56d08f0d33bfdfdf Mon Sep 17 00:00:00 2001
From: Mark de Wever <koraq@xs4all.nl>
Date: Tue, 16 Apr 2024 20:19:53 +0200
Subject: [PATCH 442/464] [libc++][modules] Removes some validation quirks.
 (#88031)

Recent unrelated header cleanups caused these quirks to become obsolete.
---
 libcxx/utils/libcxx/test/modules.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/libcxx/utils/libcxx/test/modules.py b/libcxx/utils/libcxx/test/modules.py
index 44c6292ff114..aab7651c7bb0 100644
--- a/libcxx/utils/libcxx/test/modules.py
+++ b/libcxx/utils/libcxx/test/modules.py
@@ -52,8 +52,6 @@ SkipDeclarations["random"] = [
     "std::operator==",
 ]
 
-# Declared in the forward header since std::string uses std::allocator
-SkipDeclarations["string"] = ["std::allocator"]
 # TODO MODULES remove zombie names
 # https://libcxx.llvm.org/Status/Cxx20.html#note-p0619
 SkipDeclarations["memory"] = [
@@ -61,9 +59,6 @@ SkipDeclarations["memory"] = [
     "std::get_temporary_buffer",
 ]
 
-# TODO MODULES this should be part of ios instead
-SkipDeclarations["streambuf"] = ["std::basic_ios"]
-
 # include/__type_traits/is_swappable.h
 SkipDeclarations["type_traits"] = [
     "std::swap",
-- 
GitLab


From 41a830500aa5556a65198607ec751d8e3254c949 Mon Sep 17 00:00:00 2001
From: Mark de Wever <koraq@xs4all.nl>
Date: Tue, 16 Apr 2024 20:20:37 +0200
Subject: [PATCH 443/464] [libc++] Removes deprecated
 _LIBCPP_ENABLE_<VERSION>_REMOVED_FEATURES macros (#88548)

We marked those macros as deprecated in the last release with the intent
of
removing them in LLVM 19. This commit performs the removal.
---
 libcxx/docs/ReleaseNotes/19.rst               |  2 +-
 libcxx/docs/UsingLibcxx.rst                   | 12 -------
 libcxx/include/__config                       | 26 --------------
 ...le_removed_cpp17_features.compile.pass.cpp | 36 -------------------
 ...moved_cpp17_features.deprecated.verify.cpp | 20 -----------
 ...moved_cpp20_features.deprecated.verify.cpp | 20 -----------
 6 files changed, 1 insertion(+), 115 deletions(-)
 delete mode 100644 libcxx/test/libcxx/depr/enable_removed_cpp17_features.compile.pass.cpp
 delete mode 100644 libcxx/test/libcxx/depr/enable_removed_cpp17_features.deprecated.verify.cpp
 delete mode 100644 libcxx/test/libcxx/depr/enable_removed_cpp20_features.deprecated.verify.cpp

diff --git a/libcxx/docs/ReleaseNotes/19.rst b/libcxx/docs/ReleaseNotes/19.rst
index 45aac88e4550..53cc7a77d1af 100644
--- a/libcxx/docs/ReleaseNotes/19.rst
+++ b/libcxx/docs/ReleaseNotes/19.rst
@@ -97,7 +97,7 @@ Deprecations and Removals
 - The ``_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_MEMBERS`` and ``_LIBCPP_ENABLE_CXX20_REMOVED_ALLOCATOR_VOID_SPECIALIZATION``
   macros have been removed in LLVM 19.
 
-- TODO: The ``_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES`` and ``_LIBCPP_ENABLE_CXX20_REMOVED_FEATURES`` macros have
+- The ``_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES`` and ``_LIBCPP_ENABLE_CXX20_REMOVED_FEATURES`` macros have
   been removed in LLVM 19. C++17 and C++20 removed features can still be re-enabled individually.
 
 - The ``_LIBCPP_INLINE_VISIBILITY`` and ``_VSTD`` macros have been removed in LLVM 19.
diff --git a/libcxx/docs/UsingLibcxx.rst b/libcxx/docs/UsingLibcxx.rst
index c0e85ad4d5e2..8f945656de1c 100644
--- a/libcxx/docs/UsingLibcxx.rst
+++ b/libcxx/docs/UsingLibcxx.rst
@@ -208,12 +208,6 @@ safety annotations.
 
 C++17 Specific Configuration Macros
 -----------------------------------
-**_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES**:
-  This macro is used to re-enable all the features removed in C++17. The effect
-  is equivalent to manually defining each macro listed below.
-  This macro is deprecated and will be removed in LLVM-19. Use the
-  individual macros listed below.
-
 **_LIBCPP_ENABLE_CXX17_REMOVED_AUTO_PTR**:
   This macro is used to re-enable `auto_ptr`.
 
@@ -238,12 +232,6 @@ C++20 Specific Configuration Macros
   This macro is used to re-enable the function
   ``std::shared_ptr<...>::unique()``.
 
-**_LIBCPP_ENABLE_CXX20_REMOVED_FEATURES**:
-  This macro is used to re-enable all the features removed in C++20. The effect
-  is equivalent to manually defining each macro listed below.
-  This macro is deprecated and will be removed in LLVM-19. Use the
-  individual macros listed below.
-
 **_LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS**:
   This macro is used to re-enable the `argument_type`, `result_type`,
   `first_argument_type`, and `second_argument_type` members of class
diff --git a/libcxx/include/__config b/libcxx/include/__config
index 9b4155af1e3c..4ccef2ca0d73 100644
--- a/libcxx/include/__config
+++ b/libcxx/include/__config
@@ -16,17 +16,6 @@
 #  pragma GCC system_header
 #endif
 
-#if defined(_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES) && !defined(_LIBCPP_DISABLE_DEPRECATION_WARNINGS)
-#  pragma clang deprecated(                                                                                            \
-      _LIBCPP_ENABLE_CXX17_REMOVED_FEATURES,                                                                           \
-      "_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES is deprecated in LLVM 18 and will be removed in LLVM 19")
-#endif
-#if defined(_LIBCPP_ENABLE_CXX20_REMOVED_FEATURES) && !defined(_LIBCPP_DISABLE_DEPRECATION_WARNINGS)
-#  pragma clang deprecated(                                                                                            \
-      _LIBCPP_ENABLE_CXX20_REMOVED_FEATURES,                                                                           \
-      "_LIBCPP_ENABLE_CXX20_REMOVED_FEATURES is deprecated in LLVM 18 and will be removed in LLVM 19")
-#endif
-
 #if defined(__apple_build_version__)
 // Given AppleClang XX.Y.Z, _LIBCPP_APPLE_CLANG_VER is XXYZ (e.g. AppleClang 14.0.3 => 1403)
 #  define _LIBCPP_COMPILER_CLANG_BASED
@@ -1230,21 +1219,6 @@ typedef __char32_t char32_t;
 #    define _LIBCPP_IF_WIDE_CHARACTERS(...) __VA_ARGS__
 #  endif
 
-#  if defined(_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES)
-#    define _LIBCPP_ENABLE_CXX17_REMOVED_AUTO_PTR
-#    define _LIBCPP_ENABLE_CXX17_REMOVED_BINDERS
-#    define _LIBCPP_ENABLE_CXX17_REMOVED_RANDOM_SHUFFLE
-#    define _LIBCPP_ENABLE_CXX17_REMOVED_UNEXPECTED_FUNCTIONS
-#    define _LIBCPP_ENABLE_CXX17_REMOVED_UNARY_BINARY_FUNCTION
-#  endif // _LIBCPP_ENABLE_CXX17_REMOVED_FEATURES
-
-#  if defined(_LIBCPP_ENABLE_CXX20_REMOVED_FEATURES)
-#    define _LIBCPP_ENABLE_CXX20_REMOVED_BINDER_TYPEDEFS
-#    define _LIBCPP_ENABLE_CXX20_REMOVED_NEGATORS
-#    define _LIBCPP_ENABLE_CXX20_REMOVED_RAW_STORAGE_ITERATOR
-#    define _LIBCPP_ENABLE_CXX20_REMOVED_TYPE_TRAITS
-#  endif // _LIBCPP_ENABLE_CXX20_REMOVED_FEATURES
-
 // clang-format off
 #  define _LIBCPP_PUSH_MACROS _Pragma("push_macro(\"min\")") _Pragma("push_macro(\"max\")") _Pragma("push_macro(\"refresh\")") _Pragma("push_macro(\"move\")") _Pragma("push_macro(\"erase\")")
 #  define _LIBCPP_POP_MACROS _Pragma("pop_macro(\"min\")") _Pragma("pop_macro(\"max\")") _Pragma("pop_macro(\"refresh\")") _Pragma("pop_macro(\"move\")") _Pragma("pop_macro(\"erase\")")
diff --git a/libcxx/test/libcxx/depr/enable_removed_cpp17_features.compile.pass.cpp b/libcxx/test/libcxx/depr/enable_removed_cpp17_features.compile.pass.cpp
deleted file mode 100644
index 1b7acad3cfa4..000000000000
--- a/libcxx/test/libcxx/depr/enable_removed_cpp17_features.compile.pass.cpp
+++ /dev/null
@@ -1,36 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// Test that defining _LIBCPP_ENABLE_CXX17_REMOVED_FEATURES correctly defines
-// _LIBCPP_ENABLE_CXX17_REMOVED_FOO for each individual component macro.
-
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES -Wno-deprecated-pragma
-
-#include <__config>
-
-#include "test_macros.h"
-
-#ifndef _LIBCPP_ENABLE_CXX17_REMOVED_AUTO_PTR
-#  error _LIBCPP_ENABLE_CXX17_REMOVED_AUTO_PTR must be defined
-#endif
-
-#ifndef _LIBCPP_ENABLE_CXX17_REMOVED_BINDERS
-#  error _LIBCPP_ENABLE_CXX17_REMOVED_BINDERS must be defined
-#endif
-
-#ifndef _LIBCPP_ENABLE_CXX17_REMOVED_RANDOM_SHUFFLE
-#  error _LIBCPP_ENABLE_CXX17_REMOVED_RANDOM_SHUFFLE must be defined
-#endif
-
-#ifndef _LIBCPP_ENABLE_CXX17_REMOVED_UNEXPECTED_FUNCTIONS
-#error _LIBCPP_ENABLE_CXX17_REMOVED_UNEXPECTED_FUNCTIONS must be defined
-#endif
-
-#ifndef _LIBCPP_ENABLE_CXX17_REMOVED_AUTO_PTR
-#error _LIBCPP_ENABLE_CXX17_REMOVED_AUTO_PTR must be defined
-#endif
diff --git a/libcxx/test/libcxx/depr/enable_removed_cpp17_features.deprecated.verify.cpp b/libcxx/test/libcxx/depr/enable_removed_cpp17_features.deprecated.verify.cpp
deleted file mode 100644
index 059c1b3ead4f..000000000000
--- a/libcxx/test/libcxx/depr/enable_removed_cpp17_features.deprecated.verify.cpp
+++ /dev/null
@@ -1,20 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// <__config>
-
-// Ensure that defining _LIBCPP_ENABLE_CXX17_REMOVED_FEATURES yields a
-// deprecation warning. We intend to issue a deprecation warning in LLVM 18
-// and remove the macro entirely in LLVM 19. As such, this test will be quite
-// short lived.
-
-// UNSUPPORTED: clang-modules-build
-
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES
-
-#include <__config> // expected-warning@* 1+ {{macro '_LIBCPP_ENABLE_CXX17_REMOVED_FEATURES' has been marked as deprecated}}
diff --git a/libcxx/test/libcxx/depr/enable_removed_cpp20_features.deprecated.verify.cpp b/libcxx/test/libcxx/depr/enable_removed_cpp20_features.deprecated.verify.cpp
deleted file mode 100644
index 163ff7d8fbda..000000000000
--- a/libcxx/test/libcxx/depr/enable_removed_cpp20_features.deprecated.verify.cpp
+++ /dev/null
@@ -1,20 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// <__config>
-
-// Ensure that defining _LIBCPP_ENABLE_CXX20_REMOVED_FEATURES yields a
-// deprecation warning. We intend to issue a deprecation warning in LLVM 18
-// and remove the macro entirely in LLVM 19. As such, this test will be quite
-// short lived.
-
-// UNSUPPORTED: clang-modules-build
-
-// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_ENABLE_CXX20_REMOVED_FEATURES
-
-#include <version> // expected-warning@* 1+ {{macro '_LIBCPP_ENABLE_CXX20_REMOVED_FEATURES' has been marked as deprecated}}
-- 
GitLab


From 388da6a31b7ba3062f9306b894656e265b9b33eb Mon Sep 17 00:00:00 2001
From: Mark de Wever <koraq@xs4all.nl>
Date: Tue, 16 Apr 2024 20:21:39 +0200
Subject: [PATCH 444/464] [libc++][test] Removes Clang 16 validation. (#88558)

---
 libcxx/test/libcxx/containers/sequences/vector/asan.pass.cpp | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/libcxx/test/libcxx/containers/sequences/vector/asan.pass.cpp b/libcxx/test/libcxx/containers/sequences/vector/asan.pass.cpp
index 588ce2a3d17e..614323b1ffd7 100644
--- a/libcxx/test/libcxx/containers/sequences/vector/asan.pass.cpp
+++ b/libcxx/test/libcxx/containers/sequences/vector/asan.pass.cpp
@@ -29,8 +29,7 @@ void do_exit() {
 
 int main(int, char**)
 {
-#if TEST_STD_VER >= 11 && TEST_CLANG_VER >= 1600
-  // TODO(LLVM-18): Remove the special-casing
+#if TEST_STD_VER >= 11
   {
     typedef int T;
     typedef cpp17_input_iterator<T*> MyInputIter;
@@ -52,7 +51,7 @@ int main(int, char**)
     assert(v[1] == 'b');
     assert(is_contiguous_container_asan_correct(v));
   }
-#endif
+#endif // TEST_STD_VER >= 11
   {
     typedef cpp17_input_iterator<int*> MyInputIter;
     // Sould not trigger ASan.
-- 
GitLab


From 8e0a4a89f940d17b520bbca040981f54195d3ea4 Mon Sep 17 00:00:00 2001
From: Mark de Wever <koraq@xs4all.nl>
Date: Tue, 16 Apr 2024 20:22:48 +0200
Subject: [PATCH 445/464] [libc++][doc] Documents -DLIBCXX_INSTALL_MODULES=ON.
 (#88547)

Co-authored-by: Louis Dionne <ldionne.2@gmail.com>
---
 libcxx/docs/BuildingLibcxx.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/libcxx/docs/BuildingLibcxx.rst b/libcxx/docs/BuildingLibcxx.rst
index 28145ed1049e..a0a0cdb43397 100644
--- a/libcxx/docs/BuildingLibcxx.rst
+++ b/libcxx/docs/BuildingLibcxx.rst
@@ -206,6 +206,12 @@ libc++ specific options
 
   Toggle the installation of the libc++ headers.
 
+.. option:: LIBCXX_INSTALL_MODULES:BOOL
+
+  **Default**: ``OFF``
+
+  Toggle the installation of the experimental libc++ module sources.
+
 .. option:: LIBCXX_ENABLE_SHARED:BOOL
 
   **Default**: ``ON``
-- 
GitLab


From 002297bdaa63f3f9f56b0051110ccf48f31c6825 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Tue, 16 Apr 2024 14:28:58 -0400
Subject: [PATCH 446/464] [gn] port 22629bb22a1b

---
 llvm/utils/gn/secondary/libcxx/src/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/libcxx/src/BUILD.gn b/llvm/utils/gn/secondary/libcxx/src/BUILD.gn
index 1f6879358f22..955854c7a134 100644
--- a/llvm/utils/gn/secondary/libcxx/src/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/src/BUILD.gn
@@ -125,6 +125,7 @@ cxx_sources = [
   "condition_variable_destructor.cpp",
   "error_category.cpp",
   "exception.cpp",
+  "expected.cpp",
   "fstream.cpp",
   "functional.cpp",
   "future.cpp",
-- 
GitLab


From 8debcf03c535e14ee47b14fddfcaeae3f32d1317 Mon Sep 17 00:00:00 2001
From: Peiming Liu <peiming@google.com>
Date: Tue, 16 Apr 2024 11:31:09 -0700
Subject: [PATCH 447/464] [mlir][sparse] introduce sparse_tensor.iterate
 operation (#88807)

A `sparse_tensor.iterate` iterates over a sparse iteration space
extracted from `sparse_tensor.extract_iteration_space` operation
introduced in https://github.com/llvm/llvm-project/pull/88554.

*DO NOT MERGE* before https://github.com/llvm/llvm-project/pull/88554
---
 .../Dialect/SparseTensor/IR/SparseTensor.h    |  38 ++
 .../SparseTensor/IR/SparseTensorAttrDefs.td   |  15 +
 .../SparseTensor/IR/SparseTensorOps.td        | 152 +++++++-
 .../SparseTensor/IR/SparseTensorTypes.td      |  95 +++++
 .../SparseTensor/IR/SparseTensorDialect.cpp   | 365 ++++++++++++++++++
 mlir/test/Dialect/SparseTensor/invalid.mlir   | 139 +++++++
 mlir/test/Dialect/SparseTensor/roundtrip.mlir |  53 +++
 .../SparseTensor/sparse_itertion_licm.mlir    |  26 ++
 8 files changed, 882 insertions(+), 1 deletion(-)
 create mode 100644 mlir/test/Dialect/SparseTensor/sparse_itertion_licm.mlir

diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensor.h b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensor.h
index 5e523ec428ae..081a9b8cad8d 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensor.h
+++ b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensor.h
@@ -17,9 +17,13 @@
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/TensorEncoding.h"
+#include "mlir/Interfaces/ControlFlowInterfaces.h"
 #include "mlir/Interfaces/InferTypeOpInterface.h"
+#include "mlir/Interfaces/LoopLikeInterface.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 
+#include "llvm/ADT/bit.h"
+
 //===----------------------------------------------------------------------===//
 //
 // Type aliases to help code be more self-documenting. Unfortunately
@@ -41,6 +45,40 @@ using Level = uint64_t;
 /// including the value `ShapedType::kDynamic` (for shapes).
 using Size = int64_t;
 
+/// A simple wrapper to encode a bitset of defined  (at most 64) levels.
+class LevelSet {
+  uint64_t bits = 0;
+
+public:
+  LevelSet() = default;
+  explicit LevelSet(uint64_t bits) : bits(bits) {}
+  operator uint64_t() const { return bits; }
+
+  LevelSet &set(unsigned i) {
+    assert(i < 64);
+    bits |= 1 << i;
+    return *this;
+  }
+
+  LevelSet &operator|=(LevelSet lhs) {
+    bits |= static_cast<uint64_t>(lhs);
+    return *this;
+  }
+
+  LevelSet &lshift(unsigned offset) {
+    bits = bits << offset;
+    return *this;
+  }
+
+  bool operator[](unsigned i) const {
+    assert(i < 64);
+    return (bits & (1 << i)) != 0;
+  }
+
+  unsigned count() const { return llvm::popcount(bits); }
+  bool empty() const { return bits == 0; }
+};
+
 } // namespace sparse_tensor
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td
index 4a9b9169ae4b..d5398a98f5b1 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td
+++ b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td
@@ -19,6 +19,21 @@ class SparseTensor_Attr<string name,
                         list<Trait> traits = []>
     : AttrDef<SparseTensor_Dialect, name, traits>;
 
+//===----------------------------------------------------------------------===//
+// A simple bitset attribute wrapped over a single int64_t to encode a set of
+// sparse tensor levels.
+//===----------------------------------------------------------------------===//
+
+def LevelSetAttr :
+    TypedAttrBase<
+      I64, "IntegerAttr",
+      And<[CPred<"::llvm::isa<::mlir::IntegerAttr>($_self)">,
+           CPred<"::llvm::cast<::mlir::IntegerAttr>($_self).getType().isInteger(64)">]>,
+      "LevelSet attribute"> {
+  let returnType = [{::mlir::sparse_tensor::LevelSet}];
+  let convertFromStorage = [{::mlir::sparse_tensor::LevelSet($_self.getValue().getZExtValue())}];
+}
+
 //===----------------------------------------------------------------------===//
 // These attributes are just like `IndexAttr` except that they clarify whether
 // the index refers to a dimension (an axis of the semantic tensor) or a level
diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td
index 0cfc64f9988a..b43d716d5e86 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td
+++ b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td
@@ -15,6 +15,8 @@ include "mlir/Dialect/SparseTensor/IR/SparseTensorTypes.td"
 include "mlir/Dialect/SparseTensor/IR/SparseTensorInterfaces.td"
 include "mlir/Interfaces/InferTypeOpInterface.td"
 include "mlir/Interfaces/SideEffectInterfaces.td"
+include "mlir/Interfaces/ControlFlowInterfaces.td"
+include "mlir/Interfaces/LoopLikeInterface.td"
 
 //===----------------------------------------------------------------------===//
 // Base class.
@@ -1277,7 +1279,7 @@ def SparseTensor_SelectOp : SparseTensor_Op<"select", [Pure, SameOperandsAndResu
 
 def SparseTensor_YieldOp : SparseTensor_Op<"yield", [Pure, Terminator,
     ParentOneOf<["BinaryOp", "UnaryOp", "ReduceOp", "SelectOp",
-                 "ForeachOp"]>]>,
+                 "ForeachOp", "IterateOp"]>]>,
     Arguments<(ins Variadic<AnyType>:$results)> {
   let summary = "Yield from sparse_tensor set-like operations";
   let description = [{
@@ -1430,6 +1432,154 @@ def SparseTensor_ForeachOp : SparseTensor_Op<"foreach",
   let hasVerifier = 1;
 }
 
+//===----------------------------------------------------------------------===//
+// Sparse Tensor Iteration Operations.
+//===----------------------------------------------------------------------===//
+
+def ExtractIterSpaceOp : SparseTensor_Op<"extract_iteration_space",
+    [Pure, DeclareOpInterfaceMethods<InferTypeOpInterface>]> {
+
+  let arguments = (ins AnySparseTensor:$tensor,
+                       Optional<AnySparseIterator>:$parentIter,
+                       LevelAttr:$loLvl, LevelAttr:$hiLvl);
+
+  let results = (outs AnySparseIterSpace:$resultSpace);
+
+  let summary = "Extract an iteration space from a sparse tensor between certain levels";
+  let description = [{
+      Extracts a `!sparse_tensor.iter_space` from a sparse tensor between
+      certian (consecutive) levels.
+
+      `tensor`: the input sparse tensor that defines the iteration space.
+      `parentIter`: the iterator for the previous level, at which the iteration space
+      at the current levels will be extracted.
+      `loLvl`, `hiLvl`: the level range between [loLvl, hiLvl) in the input tensor that
+      the returned iteration space covers. `hiLvl - loLvl` defines the dimension of the
+      iteration space.
+
+      Example:
+      ```mlir
+      // Extracts a 1-D iteration space from a COO tensor at level 1.
+      %space = sparse_tensor.iteration.extract_space %sp at %it1 lvls = 1
+        : tensor<4x8xf32, #COO>, !sparse_tensor.iterator<#COO, lvls = 0>
+      ```
+  }];
+
+
+  let extraClassDeclaration = [{
+    std::pair<Level, Level> getLvlRange() {
+      return std::make_pair(getLoLvl(), getHiLvl());
+    }
+    unsigned getSpaceDim() {
+      return getHiLvl() - getLoLvl();
+    }
+    ArrayRef<::mlir::sparse_tensor::LevelType> getSpaceLvlTypes() {
+      return getResultSpace().getType().getLvlTypes();
+    }
+  }];
+
+  let hasVerifier = 1;
+  let assemblyFormat = "$tensor (`at` $parentIter^)? `lvls` `=` custom<LevelRange>($loLvl, $hiLvl) "
+                       " attr-dict `:` type($tensor) (`,` type($parentIter)^)?";
+}
+
+def IterateOp : SparseTensor_Op<"iterate",
+    [RecursiveMemoryEffects, RecursivelySpeculatable,
+     DeclareOpInterfaceMethods<LoopLikeOpInterface,
+      ["getInitsMutable", "getLoopResults", "getRegionIterArgs",
+       "getYieldedValuesMutable"]>,
+     DeclareOpInterfaceMethods<RegionBranchOpInterface,
+      ["getEntrySuccessorOperands"]>,
+     SingleBlockImplicitTerminator<"sparse_tensor::YieldOp">]> {
+
+  let arguments = (ins AnySparseIterSpace:$iterSpace,
+                       Variadic<AnyType>:$initArgs,
+                       LevelSetAttr:$crdUsedLvls);
+  let results = (outs Variadic<AnyType>:$results);
+  let regions = (region SizedRegion<1>:$region);
+
+  let summary = "Iterate over a sparse iteration space";
+  let description = [{
+      The `sparse_tensor.iterate` operations represents a loop over the
+      provided iteration space extracted from a specific sparse tensor.
+      The operation defines an SSA value for a sparse iterator that points
+      to the current stored element in the sparse tensor and SSA values
+      for coordinates of the stored element. The coordinates are always
+      converted to `index` type despite of the underlying sparse tensor
+      storage. When coordinates are not used, the SSA values can be skipped
+      by `_` symbols, which usually leads to simpler generated code after
+      sparsification. For example:
+
+      ```mlir
+      // The coordinate for level 0 is not used when iterating over a 2-D
+      // iteration space.
+      %sparse_tensor.iterate %iterator in %space at(_, %crd_1)
+        : !sparse_tensor.iter_space<#CSR, lvls = 0 to 2>
+      ```
+
+      `sparse_tensor.iterate` can also operate on loop-carried variables
+      and returns the final values after loop termination.
+      The initial values of the variables are passed as additional SSA operands
+      to the iterator SSA value and used coordinate SSA values mentioned
+      above. The operation region has an argument for the iterator, variadic
+      arguments for specified (used) coordiates and followed by one argument
+      for each loop-carried variable, representing the value of the variable
+      at the current iteration.
+      The body region must contain exactly one block that terminates with
+      `sparse_tensor.yield`.
+
+      `sparse_tensor.iterate` results hold the final values after the last
+      iteration. If the `sparse_tensor.iterate` defines any values, a yield
+      must be explicitly present.
+      The number and types of the `sparse_tensor.iterate` results must match
+      the initial values in the iter_args binding and the yield operands.
+
+
+      A nested `sparse_tensor.iterate` example that prints all the coordinates
+      stored in the sparse input:
+
+      ```mlir
+      func.func @nested_iterate(%sp : tensor<4x8xf32, #COO>) {
+        // Iterates over the first level of %sp
+        %l1 = sparse_tensor.extract_iteration_space %sp lvls = 0 : tensor<4x8xf32, #COO>
+        %r1 = sparse_tensor.iterate %it1 in %l1 at (%crd0)
+            : !sparse_tensor.iter_space<#COO, lvls = 0 to 1>  {
+          // Iterates over the second level of %sp
+          %l2 = sparse_tensor.extract_iteration_space %sp at %it1 lvls = 1
+              : tensor<4x8xf32, #COO>, !sparse_tensor.iterator<#COO, lvls = 0 to 1>
+          %r2 = sparse_tensor.iterate %it2 in %l2 at (crd1)
+              : !sparse_tensor.iter_space<#COO, lvls = 1 to 2>  {
+             vector.print %crd0 : index
+             vector.print %crd1 : index
+          }
+        }
+      }
+
+      ```
+  }];
+
+  let extraClassDeclaration = [{
+    unsigned getSpaceDim() {
+      return getIterSpace().getType().getSpaceDim();
+    }
+    BlockArgument getIterator() {
+      return getRegion().getArguments().front();
+    }
+    Block::BlockArgListType getCrds() {
+      // The first block argument is iterator, the remaining arguments are
+      // referenced coordinates.
+      return getRegion().getArguments().slice(1, getCrdUsedLvls().count());
+    }
+    unsigned getNumRegionIterArgs() {
+      return getRegion().getArguments().size() - 1 - getCrdUsedLvls().count();
+    }
+  }];
+
+  let hasVerifier = 1;
+  let hasRegionVerifier = 1;
+  let hasCustomAssemblyFormat = 1;
+}
+
 //===----------------------------------------------------------------------===//
 // Sparse Tensor Debugging and Test-Only Operations.
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorTypes.td b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorTypes.td
index 185cff46ae25..264a0a5b3bee 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorTypes.td
+++ b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorTypes.td
@@ -72,4 +72,99 @@ def SparseTensorStorageSpecifier
     : Type<CPred<"::llvm::isa<::mlir::sparse_tensor::StorageSpecifierType>($_self)">, "metadata",
           "::mlir::sparse_tensor::StorageSpecifierType">;
 
+//===----------------------------------------------------------------------===//
+// Sparse Tensor Iteration Types.
+//===----------------------------------------------------------------------===//
+
+def SparseTensor_IterSpace : SparseTensor_Type<"IterSpace"> {
+  let mnemonic = "iter_space";
+
+  let description = [{
+    A sparse iteration space that represents an abstract N-D (sparse) iteration space
+    extracted from a sparse tensor.
+
+    Examples:
+
+    ```mlir
+    // An iteration space extracted from a CSR tensor between levels [0, 2).
+    !iter_space<#CSR, lvls = 0 to 2>
+    ```
+  }];
+
+  let parameters = (ins
+     SparseTensorEncodingAttr : $encoding,
+     "Level" : $loLvl,
+     "Level" : $hiLvl
+  );
+
+  let extraClassDeclaration = [{
+     /// The the dimension of the iteration space.
+     unsigned getSpaceDim() const {
+       return getHiLvl() - getLoLvl();
+     }
+
+     /// Get the level types for the iteration space.
+     ArrayRef<LevelType> getLvlTypes() const {
+       return getEncoding().getLvlTypes().slice(getLoLvl(), getSpaceDim());
+     }
+
+     /// Whether the iteration space is unique (i.e., no duplicated coordinate).
+     bool isUnique() {
+       return !getLvlTypes().back().isa<LevelPropNonDefault::Nonunique>();
+     }
+
+     /// Get the corresponding iterator type.
+     ::mlir::sparse_tensor::IteratorType getIteratorType() const;
+  }];
+
+  let assemblyFormat="`<` $encoding `,` `lvls` `=` custom<LevelRange>($loLvl, $hiLvl) `>`";
+}
+
+def SparseTensor_Iterator : SparseTensor_Type<"Iterator"> {
+  let mnemonic = "iterator";
+
+  let description = [{
+    An iterator that points to the current element in the corresponding iteration space.
+
+    Examples:
+
+    ```mlir
+    // An iterator that iterates over a iteration space of type `!iter_space<#CSR, lvls = 0 to 2>`
+    !iterator<#CSR, lvls = 0 to 2>
+    ```
+  }];
+
+  let parameters = (ins
+     SparseTensorEncodingAttr : $encoding,
+     "Level" : $loLvl,
+     "Level" : $hiLvl
+  );
+
+  let extraClassDeclaration = [{
+     /// Get the corresponding iteration space type.
+     ::mlir::sparse_tensor::IterSpaceType getIterSpaceType() const;
+
+     unsigned getSpaceDim() const { return getIterSpaceType().getSpaceDim(); }
+     ArrayRef<LevelType> getLvlTypes() const { return getIterSpaceType().getLvlTypes(); }
+     bool isUnique() { return getIterSpaceType().isUnique(); }
+  }];
+
+  let assemblyFormat="`<` $encoding `,` `lvls` `=` custom<LevelRange>($loLvl, $hiLvl) `>`";
+}
+
+def IsSparseSparseIterSpaceTypePred
+    : CPred<"::llvm::isa<::mlir::sparse_tensor::IterSpaceType>($_self)">;
+
+def IsSparseSparseIteratorTypePred
+    : CPred<"::llvm::isa<::mlir::sparse_tensor::IteratorType>($_self)">;
+
+def AnySparseIterSpace
+    : Type<IsSparseSparseIterSpaceTypePred, "sparse iteration space",
+          "::mlir::sparse_tensor::IterSpaceType">;
+
+def AnySparseIterator
+    : Type<IsSparseSparseIteratorTypePred, "sparse iterator",
+          "::mlir::sparse_tensor::IteratorType">;
+
+
 #endif // SPARSETENSOR_TYPES
diff --git a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
index e9058394d33d..36908def09f4 100644
--- a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
+++ b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
@@ -30,6 +30,14 @@
 #include "mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.cpp.inc"
 #include "mlir/Dialect/SparseTensor/IR/SparseTensorAttrEnums.cpp.inc"
 
+// Forward declarations, following custom print/parsing methods are referenced
+// by the generated code for SparseTensorTypes.td.
+static mlir::ParseResult parseLevelRange(mlir::AsmParser &,
+                                         mlir::sparse_tensor::Level &,
+                                         mlir::sparse_tensor::Level &);
+static void printLevelRange(mlir::AsmPrinter &, mlir::sparse_tensor::Level,
+                            mlir::sparse_tensor::Level);
+
 #define GET_TYPEDEF_CLASSES
 #include "mlir/Dialect/SparseTensor/IR/SparseTensorTypes.cpp.inc"
 
@@ -1953,6 +1961,363 @@ LogicalResult SortOp::verify() {
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// Sparse Tensor Iteration Operations.
+//===----------------------------------------------------------------------===//
+
+IterSpaceType IteratorType::getIterSpaceType() const {
+  return IterSpaceType::get(getContext(), getEncoding(), getLoLvl(),
+                            getHiLvl());
+}
+
+IteratorType IterSpaceType::getIteratorType() const {
+  return IteratorType::get(getContext(), getEncoding(), getLoLvl(), getHiLvl());
+}
+
+/// Parses a level range in the form "$lo `to` $hi"
+/// or simply "$lo" if $hi - $lo = 1
+static ParseResult parseLevelRange(AsmParser &parser, Level &lvlLo,
+                                   Level &lvlHi) {
+  if (parser.parseInteger(lvlLo))
+    return failure();
+
+  if (succeeded(parser.parseOptionalKeyword("to"))) {
+    if (parser.parseInteger(lvlHi))
+      return failure();
+  } else {
+    lvlHi = lvlLo + 1;
+  }
+
+  if (lvlHi <= lvlLo)
+    parser.emitError(parser.getNameLoc(),
+                     "expect larger level upper bound than lower bound");
+
+  return success();
+}
+
+/// Parses a level range in the form "$lo `to` $hi"
+/// or simply "$lo" if $hi - $lo = 1
+static ParseResult parseLevelRange(OpAsmParser &parser, IntegerAttr &lvlLoAttr,
+                                   IntegerAttr &lvlHiAttr) {
+  Level lvlLo, lvlHi;
+  if (parseLevelRange(parser, lvlLo, lvlHi))
+    return failure();
+
+  lvlLoAttr = IntegerAttr::get(parser.getBuilder().getIndexType(), lvlLo);
+  lvlHiAttr = IntegerAttr::get(parser.getBuilder().getIndexType(), lvlHi);
+  return success();
+}
+
+/// Prints a level range in the form "$lo `to` $hi"
+/// or simply "$lo" if $hi - $lo = 1
+static void printLevelRange(AsmPrinter &p, Level lo, Level hi) {
+
+  if (lo + 1 == hi)
+    p << lo;
+  else
+    p << lo << " to " << hi;
+}
+
+/// Prints a level range in the form "$lo `to` $hi"
+/// or simply "$lo" if $hi - $lo = 1
+static void printLevelRange(OpAsmPrinter &p, Operation *, IntegerAttr lvlLo,
+                            IntegerAttr lvlHi) {
+  unsigned lo = lvlLo.getValue().getZExtValue();
+  unsigned hi = lvlHi.getValue().getZExtValue();
+  printLevelRange(p, lo, hi);
+}
+
+static ParseResult
+parseSparseSpaceLoop(OpAsmParser &parser, OperationState &state,
+                     SmallVectorImpl<OpAsmParser::Argument> &iterators,
+                     SmallVectorImpl<OpAsmParser::Argument> &iterArgs) {
+  SmallVector<OpAsmParser::UnresolvedOperand> spaces;
+  SmallVector<OpAsmParser::UnresolvedOperand> initArgs;
+
+  // Parses "%iters, ... in %spaces, ..."
+  if (parser.parseArgumentList(iterators) || parser.parseKeyword("in") ||
+      parser.parseOperandList(spaces))
+    return failure();
+
+  if (iterators.size() != spaces.size())
+    return parser.emitError(
+        parser.getNameLoc(),
+        "mismatch in number of sparse iterators and sparse spaces");
+
+  // Parse "at(%crd0, _, ...)"
+  LevelSet crdUsedLvlSet;
+  bool hasUsedCrds = succeeded(parser.parseOptionalKeyword("at"));
+  unsigned lvlCrdCnt = 0;
+  if (hasUsedCrds) {
+    ParseResult crdList = parser.parseCommaSeparatedList(
+        OpAsmParser::Delimiter::Paren, [&]() -> ParseResult {
+          if (parser.parseOptionalKeyword("_")) {
+            if (parser.parseArgument(iterArgs.emplace_back()))
+              return failure();
+            // Always use IndexType for the coordinate.
+            crdUsedLvlSet.set(lvlCrdCnt);
+            iterArgs.back().type = parser.getBuilder().getIndexType();
+          }
+          lvlCrdCnt += 1;
+          return success();
+        });
+    if (failed(crdList)) {
+      return parser.emitError(
+          parser.getNameLoc(),
+          "expecting SSA value or \"_\" for level coordinates");
+    }
+  }
+  // Set the CrdUsedLvl bitset.
+  state.addAttribute("crdUsedLvls",
+                     parser.getBuilder().getI64IntegerAttr(crdUsedLvlSet));
+
+  // Parse "iter_args(%arg = %init, ...)"
+  bool hasIterArgs = succeeded(parser.parseOptionalKeyword("iter_args"));
+  if (hasIterArgs)
+    if (parser.parseAssignmentList(iterArgs, initArgs))
+      return failure();
+
+  SmallVector<Type> iterSpaceTps;
+  // parse ": sparse_tensor.iter_space -> ret"
+  if (parser.parseColon() || parser.parseTypeList(iterSpaceTps))
+    return failure();
+  if (iterSpaceTps.size() != spaces.size())
+    return parser.emitError(parser.getNameLoc(),
+                            "mismatch in number of iteration space operands "
+                            "and iteration space types");
+
+  for (auto [it, tp] : llvm::zip_equal(iterators, iterSpaceTps)) {
+    IterSpaceType spaceTp = llvm::dyn_cast<IterSpaceType>(tp);
+    if (!spaceTp)
+      return parser.emitError(parser.getNameLoc(),
+                              "expected sparse_tensor.iter_space type for "
+                              "iteration space operands");
+    if (hasUsedCrds && spaceTp.getSpaceDim() != lvlCrdCnt)
+      return parser.emitError(parser.getNameLoc(),
+                              "mismatch in number of iteration space dimension "
+                              "and specified coordinates");
+    it.type = spaceTp.getIteratorType();
+  }
+
+  if (hasIterArgs)
+    if (parser.parseArrowTypeList(state.types))
+      return failure();
+
+  // Resolves input operands.
+  if (parser.resolveOperands(spaces, iterSpaceTps, parser.getNameLoc(),
+                             state.operands))
+    return failure();
+
+  if (hasIterArgs) {
+    unsigned numCrds = crdUsedLvlSet.count();
+    // Strip off leading args that used for coordinates.
+    MutableArrayRef args = MutableArrayRef(iterArgs).drop_front(numCrds);
+    if (args.size() != initArgs.size() || args.size() != state.types.size()) {
+      return parser.emitError(
+          parser.getNameLoc(),
+          "mismatch in number of iteration arguments and return values");
+    }
+
+    for (auto [it, init, tp] : llvm::zip_equal(args, initArgs, state.types)) {
+      it.type = tp;
+      if (parser.resolveOperand(init, tp, state.operands))
+        return failure();
+    }
+  }
+  return success();
+}
+
+LogicalResult ExtractIterSpaceOp::inferReturnTypes(
+    MLIRContext *ctx, std::optional<Location> loc, ValueRange ops,
+    DictionaryAttr attr, OpaqueProperties prop, RegionRange region,
+    SmallVectorImpl<mlir::Type> &ret) {
+
+  ExtractIterSpaceOp::Adaptor adaptor(ops, attr, prop, region);
+  SparseTensorType stt = getSparseTensorType(adaptor.getTensor());
+  ret.push_back(IterSpaceType::get(ctx, stt.getEncoding(), adaptor.getLoLvl(),
+                                   adaptor.getHiLvl()));
+  return success();
+}
+
+LogicalResult ExtractIterSpaceOp::verify() {
+  if (getLoLvl() >= getHiLvl())
+    return emitOpError("expected smaller level low than level high");
+
+  TypedValue<IteratorType> pIter = getParentIter();
+  if ((pIter && getLoLvl() == 0) || (!pIter && getLoLvl() != 0)) {
+    return emitOpError(
+        "parent iterator should be specified iff level lower bound equals 0");
+  }
+
+  if (pIter) {
+    IterSpaceType spaceTp = getResultSpace().getType();
+    if (pIter.getType().getEncoding() != spaceTp.getEncoding())
+      return emitOpError(
+          "mismatch in parent iterator encoding and iteration space encoding.");
+
+    if (spaceTp.getLoLvl() != pIter.getType().getHiLvl())
+      return emitOpError("parent iterator should be used to extract an "
+                         "iteration space from a consecutive level.");
+  }
+
+  return success();
+}
+
+ParseResult IterateOp::parse(OpAsmParser &parser, OperationState &result) {
+  OpAsmParser::Argument iterator;
+  OpAsmParser::UnresolvedOperand iterSpace;
+
+  SmallVector<OpAsmParser::Argument> iters, iterArgs;
+  if (parseSparseSpaceLoop(parser, result, iters, iterArgs))
+    return failure();
+  if (iters.size() != 1)
+    return parser.emitError(parser.getNameLoc(),
+                            "expected only one iterator/iteration space");
+
+  iters.append(iterArgs);
+  Region *body = result.addRegion();
+  if (parser.parseRegion(*body, iters))
+    return failure();
+
+  IterateOp::ensureTerminator(*body, parser.getBuilder(), result.location);
+
+  // Parse the optional attribute list.
+  if (parser.parseOptionalAttrDict(result.attributes))
+    return failure();
+
+  return success();
+}
+
+/// Prints the initialization list in the form of
+///   <prefix>(%inner = %outer, %inner2 = %outer2, <...>)
+/// where 'inner' values are assumed to be region arguments and 'outer' values
+/// are regular SSA values.
+static void printInitializationList(OpAsmPrinter &p,
+                                    Block::BlockArgListType blocksArgs,
+                                    ValueRange initializers,
+                                    StringRef prefix = "") {
+  assert(blocksArgs.size() == initializers.size() &&
+         "expected same length of arguments and initializers");
+  if (initializers.empty())
+    return;
+
+  p << prefix << '(';
+  llvm::interleaveComma(llvm::zip(blocksArgs, initializers), p, [&](auto it) {
+    p << std::get<0>(it) << " = " << std::get<1>(it);
+  });
+  p << ")";
+}
+
+static void printUsedCrdsList(OpAsmPrinter &p, unsigned spaceDim,
+                              Block::BlockArgListType blocksArgs,
+                              LevelSet crdUsedLvls) {
+  if (crdUsedLvls.empty())
+    return;
+
+  p << " at(";
+  for (unsigned i = 0; i < spaceDim; i++) {
+    if (crdUsedLvls[i]) {
+      p << blocksArgs.front();
+      blocksArgs = blocksArgs.drop_front();
+    } else {
+      p << "_";
+    }
+    if (i != spaceDim - 1)
+      p << ", ";
+  }
+  assert(blocksArgs.empty());
+  p << ")";
+}
+
+void IterateOp::print(OpAsmPrinter &p) {
+  p << " " << getIterator() << " in " << getIterSpace();
+  printUsedCrdsList(p, getSpaceDim(), getCrds(), getCrdUsedLvls());
+  printInitializationList(p, getRegionIterArgs(), getInitArgs(), " iter_args");
+
+  p << " : " << getIterSpace().getType() << " ";
+  if (!getInitArgs().empty())
+    p << "-> (" << getInitArgs().getTypes() << ") ";
+
+  p.printRegion(getRegion(), /*printEntryBlockArgs=*/false,
+                /*printBlockTerminators=*/!getInitArgs().empty());
+}
+
+LogicalResult IterateOp::verify() {
+  if (getInitArgs().size() != getNumResults()) {
+    return emitOpError(
+        "mismatch in number of loop-carried values and defined values");
+  }
+  return success();
+}
+
+LogicalResult IterateOp::verifyRegions() {
+  if (getIterator().getType() != getIterSpace().getType().getIteratorType())
+    return emitOpError("mismatch in iterator and iteration space type");
+  if (getNumRegionIterArgs() != getNumResults())
+    return emitOpError(
+        "mismatch in number of basic block args and defined values");
+
+  auto initArgs = getInitArgs();
+  auto iterArgs = getRegionIterArgs();
+  auto yieldVals = getYieldedValues();
+  auto opResults = getResults();
+  if (!llvm::all_equal({initArgs.size(), iterArgs.size(), yieldVals.size(),
+                        opResults.size()})) {
+    return emitOpError() << "number mismatch between iter args and results.";
+  }
+
+  for (auto [i, init, iter, yield, ret] :
+       llvm::enumerate(initArgs, iterArgs, yieldVals, opResults)) {
+    if (init.getType() != ret.getType())
+      return emitOpError() << "types mismatch between " << i
+                           << "th iter operand and defined value";
+    if (iter.getType() != ret.getType())
+      return emitOpError() << "types mismatch between " << i
+                           << "th iter region arg and defined value";
+    if (yield.getType() != ret.getType())
+      return emitOpError() << "types mismatch between " << i
+                           << "th yield value and defined value";
+  }
+
+  return success();
+}
+
+/// IterateOp implemented OpInterfaces' methods.
+SmallVector<Region *> IterateOp::getLoopRegions() { return {&getRegion()}; }
+
+MutableArrayRef<OpOperand> IterateOp::getInitsMutable() {
+  return getInitArgsMutable();
+}
+
+Block::BlockArgListType IterateOp::getRegionIterArgs() {
+  return getRegion().getArguments().take_back(getNumRegionIterArgs());
+}
+
+std::optional<MutableArrayRef<OpOperand>> IterateOp::getYieldedValuesMutable() {
+  return cast<sparse_tensor::YieldOp>(
+             getRegion().getBlocks().front().getTerminator())
+      .getResultsMutable();
+}
+
+std::optional<ResultRange> IterateOp::getLoopResults() { return getResults(); }
+
+OperandRange IterateOp::getEntrySuccessorOperands(RegionBranchPoint point) {
+  return getInitArgs();
+}
+
+void IterateOp::getSuccessorRegions(RegionBranchPoint point,
+                                    SmallVectorImpl<RegionSuccessor> &regions) {
+  // Both the operation itself and the region may be branching into the body or
+  // back into the operation itself.
+  regions.push_back(RegionSuccessor(&getRegion(), getRegionIterArgs()));
+  // It is possible for loop not to enter the body.
+  regions.push_back(RegionSuccessor(getResults()));
+}
+
+//===----------------------------------------------------------------------===//
+// Sparse Tensor Dialect Setups.
+//===----------------------------------------------------------------------===//
+
 /// Materialize a single constant operation from a given attribute value with
 /// the desired resultant type.
 Operation *SparseTensorDialect::materializeConstant(OpBuilder &builder,
diff --git a/mlir/test/Dialect/SparseTensor/invalid.mlir b/mlir/test/Dialect/SparseTensor/invalid.mlir
index 7f5c05190fc9..b13024cd4ed9 100644
--- a/mlir/test/Dialect/SparseTensor/invalid.mlir
+++ b/mlir/test/Dialect/SparseTensor/invalid.mlir
@@ -1012,3 +1012,142 @@ func.func @sparse_print(%arg0: tensor<10x10xf64>) {
   sparse_tensor.print %arg0 : tensor<10x10xf64>
   return
 }
+
+// -----
+
+#COO = #sparse_tensor.encoding<{
+  map = (i, j) -> (
+    i : compressed(nonunique),
+    j : singleton(soa)
+  )
+}>
+
+func.func @sparse_extract_iter_space(%sp : tensor<4x8xf32, #COO>, %it1 : !sparse_tensor.iterator<#COO, lvls = 2>) {
+  // expected-error@+1 {{'sparse_tensor.extract_iteration_space' expect larger level upper bound than lower bound}}
+  %l1 = sparse_tensor.extract_iteration_space %sp at %it1 lvls = 2 to 0 : tensor<4x8xf32, #COO>, !sparse_tensor.iterator<#COO, lvls = 2>
+  return
+}
+
+// -----
+
+#COO = #sparse_tensor.encoding<{
+  map = (i, j) -> (
+    i : compressed(nonunique),
+    j : singleton(soa)
+  )
+}>
+
+func.func @sparse_extract_iter_space(%sp : tensor<4x8xf32, #COO>, %it1 : !sparse_tensor.iterator<#COO, lvls = 0>) {
+  // expected-error@+1 {{'sparse_tensor.extract_iteration_space' op parent iterator should be specified iff level lower bound equals 0}}
+  %l1 = sparse_tensor.extract_iteration_space %sp at %it1 lvls = 0 : tensor<4x8xf32, #COO>, !sparse_tensor.iterator<#COO, lvls = 0>
+  return
+}
+
+// -----
+
+#COO = #sparse_tensor.encoding<{
+  map = (i, j) -> (
+    i : compressed(nonunique),
+    j : singleton(soa)
+  )
+}>
+
+func.func @sparse_extract_iter_space(%sp : tensor<4x8xf32, #COO>) {
+  // expected-error@+1 {{'sparse_tensor.extract_iteration_space' op parent iterator should be specified iff level lower bound equals 0}}
+  %l1 = sparse_tensor.extract_iteration_space %sp lvls = 1 : tensor<4x8xf32, #COO>
+  return
+}
+
+// -----
+
+#COO = #sparse_tensor.encoding<{
+  map = (i, j) -> (
+    i : compressed(nonunique),
+    j : singleton(soa)
+  )
+}>
+
+#CSR = #sparse_tensor.encoding<{
+  map = (i, j) -> (
+    i : dense,
+    j : compressed
+  )
+}>
+
+func.func @sparse_extract_iter_space(%sp : tensor<4x8xf32, #COO>, %it1 : !sparse_tensor.iterator<#CSR, lvls = 0>) {
+  // expected-error@+1 {{'sparse_tensor.extract_iteration_space' op mismatch in parent iterator encoding and iteration space encoding.}}
+  %l1 = sparse_tensor.extract_iteration_space %sp at %it1 lvls = 1 : tensor<4x8xf32, #COO>, !sparse_tensor.iterator<#CSR, lvls = 0>
+  return
+}
+
+// -----
+
+#COO = #sparse_tensor.encoding<{
+  map = (i, j) -> (
+    i : compressed(nonunique),
+    j : singleton(soa)
+  )
+}>
+
+func.func @sparse_extract_iter_space(%sp : tensor<4x8xf32, #COO>, %it1 : !sparse_tensor.iterator<#COO, lvls = 0>) {
+  // expected-error@+1 {{'sparse_tensor.extract_iteration_space' op parent iterator should be used to extract an iteration space from a consecutive level.}}
+  %l1 = sparse_tensor.extract_iteration_space %sp at %it1 lvls = 2 : tensor<4x8xf32, #COO>, !sparse_tensor.iterator<#COO, lvls = 0>
+  return
+}
+
+
+// -----
+
+#COO = #sparse_tensor.encoding<{
+  map = (i, j) -> (
+    i : compressed(nonunique),
+    j : singleton(soa)
+  )
+}>
+
+func.func @sparse_iterate(%sp : tensor<4x8xf32, #COO>, %i : index, %j : index) -> index {
+  %l1 = sparse_tensor.extract_iteration_space %sp lvls = 0 : tensor<4x8xf32, #COO>
+  // expected-error @+1 {{'sparse_tensor.iterate' op different number of region iter_args and yielded values: 2 != 1}}
+  %r1, %r2 = sparse_tensor.iterate %it1 in %l1 at (%crd) iter_args(%si = %i, %sj = %j): !sparse_tensor.iter_space<#COO, lvls = 0> -> (index, index) {
+    sparse_tensor.yield %si : index
+  }
+  return %r1 : index
+}
+
+// -----
+
+#COO = #sparse_tensor.encoding<{
+  map = (i, j) -> (
+    i : compressed(nonunique),
+    j : singleton(soa)
+  )
+}>
+
+// expected-note@+1 {{prior use here}}
+func.func @sparse_iterate(%sp : tensor<4x8xf32, #COO>, %i : index) -> f32 {
+  %l1 = sparse_tensor.extract_iteration_space %sp lvls = 0 : tensor<4x8xf32, #COO>
+  // expected-error @+1 {{use of value '%i' expects different type than prior uses: 'f32' vs 'index'}}
+  %r1 = sparse_tensor.iterate %it1 in %l1 at (%crd) iter_args(%outer = %i): !sparse_tensor.iter_space<#COO, lvls = 0> -> f32 {
+    sparse_tensor.yield %outer : f32
+  }
+  return %r1 : f32
+}
+
+// -----
+
+#COO = #sparse_tensor.encoding<{
+  map = (i, j) -> (
+    i : compressed(nonunique),
+    j : singleton(soa)
+  )
+}>
+
+func.func @sparse_iterate(%sp : tensor<4x8xf32, #COO>, %i : index, %j : index) -> index {
+  %l1 = sparse_tensor.extract_iteration_space %sp lvls = 0 : tensor<4x8xf32, #COO>
+  // expected-error @+1 {{'sparse_tensor.iterate' op 0-th region iter_arg and 0-th yielded value have different type: 'index' != 'f32'}}
+  %r1 = sparse_tensor.iterate %it1 in %l1 at (%crd) iter_args(%si = %i): !sparse_tensor.iter_space<#COO, lvls = 0> -> index {
+    %y = arith.constant 1.0 :  f32
+    sparse_tensor.yield %y : f32
+  }
+  return %r1 : index
+}
diff --git a/mlir/test/Dialect/SparseTensor/roundtrip.mlir b/mlir/test/Dialect/SparseTensor/roundtrip.mlir
index 12f69c1d37b9..e9a898f16b41 100644
--- a/mlir/test/Dialect/SparseTensor/roundtrip.mlir
+++ b/mlir/test/Dialect/SparseTensor/roundtrip.mlir
@@ -738,3 +738,56 @@ func.func @sparse_has_runtime() -> i1 {
   %has_runtime = sparse_tensor.has_runtime_library
   return %has_runtime : i1
 }
+
+// -----
+
+#COO = #sparse_tensor.encoding<{
+  map = (i, j) -> (
+    i : compressed(nonunique),
+    j : singleton(soa)
+  )
+}>
+
+// CHECK-LABEL:   func.func @sparse_extract_iter_space(
+// CHECK-SAME:      %[[VAL_0:.*]]: tensor<4x8xf32, #sparse{{[0-9]*}}>,
+// CHECK-SAME:      %[[VAL_1:.*]]: !sparse_tensor.iterator<#sparse{{[0-9]*}}, lvls = 0>)
+// CHECK:           %[[VAL_2:.*]] = sparse_tensor.extract_iteration_space %[[VAL_0]] lvls = 0
+// CHECK:           %[[VAL_3:.*]] = sparse_tensor.extract_iteration_space %[[VAL_0]] at %[[VAL_1]] lvls = 1
+// CHECK:           return %[[VAL_2]], %[[VAL_3]] : !sparse_tensor.iter_space<#sparse{{[0-9]*}}, lvls = 0>, !sparse_tensor.iter_space<#sparse{{[0-9]*}}, lvls = 1>
+// CHECK:         }
+func.func @sparse_extract_iter_space(%sp : tensor<4x8xf32, #COO>, %it1 : !sparse_tensor.iterator<#COO, lvls = 0>)
+  -> (!sparse_tensor.iter_space<#COO, lvls = 0>, !sparse_tensor.iter_space<#COO, lvls = 1>) {
+  // Extracting the iteration space for the first level needs no parent iterator.
+  %l1 = sparse_tensor.extract_iteration_space %sp lvls = 0 : tensor<4x8xf32, #COO>
+  // Extracting the iteration space for the second level needs a parent iterator.
+  %l2 = sparse_tensor.extract_iteration_space %sp at %it1 lvls = 1 : tensor<4x8xf32, #COO>, !sparse_tensor.iterator<#COO, lvls = 0>
+  return %l1, %l2 : !sparse_tensor.iter_space<#COO, lvls = 0>, !sparse_tensor.iter_space<#COO, lvls = 1>
+}
+
+
+// -----
+
+#COO = #sparse_tensor.encoding<{
+  map = (i, j) -> (
+    i : compressed(nonunique),
+    j : singleton(soa)
+  )
+}>
+
+// CHECK-LABEL:   func.func @sparse_iterate(
+// CHECK-SAME:      %[[VAL_0:.*]]: tensor<4x8xf32, #sparse{{[0-9]*}}>,
+// CHECK-SAME:      %[[VAL_1:.*]]: index,
+// CHECK-SAME:      %[[VAL_2:.*]]: index) -> index {
+// CHECK:           %[[VAL_3:.*]] = sparse_tensor.extract_iteration_space %[[VAL_0]] lvls = 0 : tensor<4x8xf32, #sparse{{[0-9]*}}>
+// CHECK:           %[[VAL_4:.*]] = sparse_tensor.iterate %[[VAL_5:.*]] in %[[VAL_3]] at(%[[VAL_6:.*]]) iter_args(%[[VAL_7:.*]] = %[[VAL_1]]) : !sparse_tensor.iter_space<#sparse{{[0-9]*}}, lvls = 0> -> (index) {
+// CHECK:             sparse_tensor.yield %[[VAL_7]] : index
+// CHECK:           }
+// CHECK:           return %[[VAL_4]] : index
+// CHECK:         }
+func.func @sparse_iterate(%sp : tensor<4x8xf32, #COO>, %i : index, %j : index) -> index {
+  %l1 = sparse_tensor.extract_iteration_space %sp lvls = 0 : tensor<4x8xf32, #COO>
+  %r1 = sparse_tensor.iterate %it1 in %l1 at (%crd) iter_args(%outer = %i): !sparse_tensor.iter_space<#COO, lvls = 0 to 1> -> index {
+    sparse_tensor.yield %outer : index
+  }
+  return %r1 : index
+}
diff --git a/mlir/test/Dialect/SparseTensor/sparse_itertion_licm.mlir b/mlir/test/Dialect/SparseTensor/sparse_itertion_licm.mlir
new file mode 100644
index 000000000000..e7158d04b37f
--- /dev/null
+++ b/mlir/test/Dialect/SparseTensor/sparse_itertion_licm.mlir
@@ -0,0 +1,26 @@
+// RUN: mlir-opt %s --loop-invariant-code-motion | FileCheck %s
+
+#CSR = #sparse_tensor.encoding<{
+  map = (i, j) -> (
+    i : dense,
+    j : compressed
+  )
+}>
+
+// Make sure that pure instructions are hoisted outside the loop.
+//
+// CHECK: sparse_tensor.values
+// CHECK: sparse_tensor.positions
+// CHECK: sparse_tensor.coordinate
+// CHECK: sparse_tensor.iterate
+func.func @sparse_iterate(%sp : tensor<?x?xf64, #CSR>) {
+  %l1 = sparse_tensor.extract_iteration_space %sp lvls = 0 : tensor<?x?xf64, #CSR>
+  sparse_tensor.iterate %it1 in %l1 at (%crd) : !sparse_tensor.iter_space<#CSR, lvls = 0> {
+    %0 = sparse_tensor.values %sp : tensor<?x?xf64, #CSR> to memref<?xf64>
+    %1 = sparse_tensor.positions %sp { level = 1 : index } : tensor<?x?xf64, #CSR> to memref<?xindex>
+    %2 = sparse_tensor.coordinates  %sp { level = 1 : index } : tensor<?x?xf64, #CSR> to memref<?xindex>
+    "test.op"(%0, %1, %2) : (memref<?xf64>, memref<?xindex>, memref<?xindex>) -> ()
+  }
+
+  return
+}
-- 
GitLab


From b9556532c7391a2acb77ab0f7d7b36e1dc382b24 Mon Sep 17 00:00:00 2001
From: Peiming Liu <peiming@google.com>
Date: Tue, 16 Apr 2024 11:31:33 -0700
Subject: [PATCH 448/464] Revert "[mlir][sparse] introduce
 sparse_tensor.iterate operation" (#88953)

Reverts llvm/llvm-project#88807 (merged by mistake)
---
 .../Dialect/SparseTensor/IR/SparseTensor.h    |  38 --
 .../SparseTensor/IR/SparseTensorAttrDefs.td   |  15 -
 .../SparseTensor/IR/SparseTensorOps.td        | 152 +-------
 .../SparseTensor/IR/SparseTensorTypes.td      |  95 -----
 .../SparseTensor/IR/SparseTensorDialect.cpp   | 365 ------------------
 mlir/test/Dialect/SparseTensor/invalid.mlir   | 139 -------
 mlir/test/Dialect/SparseTensor/roundtrip.mlir |  53 ---
 .../SparseTensor/sparse_itertion_licm.mlir    |  26 --
 8 files changed, 1 insertion(+), 882 deletions(-)
 delete mode 100644 mlir/test/Dialect/SparseTensor/sparse_itertion_licm.mlir

diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensor.h b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensor.h
index 081a9b8cad8d..5e523ec428ae 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensor.h
+++ b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensor.h
@@ -17,13 +17,9 @@
 #include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/OpImplementation.h"
 #include "mlir/IR/TensorEncoding.h"
-#include "mlir/Interfaces/ControlFlowInterfaces.h"
 #include "mlir/Interfaces/InferTypeOpInterface.h"
-#include "mlir/Interfaces/LoopLikeInterface.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 
-#include "llvm/ADT/bit.h"
-
 //===----------------------------------------------------------------------===//
 //
 // Type aliases to help code be more self-documenting. Unfortunately
@@ -45,40 +41,6 @@ using Level = uint64_t;
 /// including the value `ShapedType::kDynamic` (for shapes).
 using Size = int64_t;
 
-/// A simple wrapper to encode a bitset of defined  (at most 64) levels.
-class LevelSet {
-  uint64_t bits = 0;
-
-public:
-  LevelSet() = default;
-  explicit LevelSet(uint64_t bits) : bits(bits) {}
-  operator uint64_t() const { return bits; }
-
-  LevelSet &set(unsigned i) {
-    assert(i < 64);
-    bits |= 1 << i;
-    return *this;
-  }
-
-  LevelSet &operator|=(LevelSet lhs) {
-    bits |= static_cast<uint64_t>(lhs);
-    return *this;
-  }
-
-  LevelSet &lshift(unsigned offset) {
-    bits = bits << offset;
-    return *this;
-  }
-
-  bool operator[](unsigned i) const {
-    assert(i < 64);
-    return (bits & (1 << i)) != 0;
-  }
-
-  unsigned count() const { return llvm::popcount(bits); }
-  bool empty() const { return bits == 0; }
-};
-
 } // namespace sparse_tensor
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td
index d5398a98f5b1..4a9b9169ae4b 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td
+++ b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td
@@ -19,21 +19,6 @@ class SparseTensor_Attr<string name,
                         list<Trait> traits = []>
     : AttrDef<SparseTensor_Dialect, name, traits>;
 
-//===----------------------------------------------------------------------===//
-// A simple bitset attribute wrapped over a single int64_t to encode a set of
-// sparse tensor levels.
-//===----------------------------------------------------------------------===//
-
-def LevelSetAttr :
-    TypedAttrBase<
-      I64, "IntegerAttr",
-      And<[CPred<"::llvm::isa<::mlir::IntegerAttr>($_self)">,
-           CPred<"::llvm::cast<::mlir::IntegerAttr>($_self).getType().isInteger(64)">]>,
-      "LevelSet attribute"> {
-  let returnType = [{::mlir::sparse_tensor::LevelSet}];
-  let convertFromStorage = [{::mlir::sparse_tensor::LevelSet($_self.getValue().getZExtValue())}];
-}
-
 //===----------------------------------------------------------------------===//
 // These attributes are just like `IndexAttr` except that they clarify whether
 // the index refers to a dimension (an axis of the semantic tensor) or a level
diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td
index b43d716d5e86..0cfc64f9988a 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td
+++ b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td
@@ -15,8 +15,6 @@ include "mlir/Dialect/SparseTensor/IR/SparseTensorTypes.td"
 include "mlir/Dialect/SparseTensor/IR/SparseTensorInterfaces.td"
 include "mlir/Interfaces/InferTypeOpInterface.td"
 include "mlir/Interfaces/SideEffectInterfaces.td"
-include "mlir/Interfaces/ControlFlowInterfaces.td"
-include "mlir/Interfaces/LoopLikeInterface.td"
 
 //===----------------------------------------------------------------------===//
 // Base class.
@@ -1279,7 +1277,7 @@ def SparseTensor_SelectOp : SparseTensor_Op<"select", [Pure, SameOperandsAndResu
 
 def SparseTensor_YieldOp : SparseTensor_Op<"yield", [Pure, Terminator,
     ParentOneOf<["BinaryOp", "UnaryOp", "ReduceOp", "SelectOp",
-                 "ForeachOp", "IterateOp"]>]>,
+                 "ForeachOp"]>]>,
     Arguments<(ins Variadic<AnyType>:$results)> {
   let summary = "Yield from sparse_tensor set-like operations";
   let description = [{
@@ -1432,154 +1430,6 @@ def SparseTensor_ForeachOp : SparseTensor_Op<"foreach",
   let hasVerifier = 1;
 }
 
-//===----------------------------------------------------------------------===//
-// Sparse Tensor Iteration Operations.
-//===----------------------------------------------------------------------===//
-
-def ExtractIterSpaceOp : SparseTensor_Op<"extract_iteration_space",
-    [Pure, DeclareOpInterfaceMethods<InferTypeOpInterface>]> {
-
-  let arguments = (ins AnySparseTensor:$tensor,
-                       Optional<AnySparseIterator>:$parentIter,
-                       LevelAttr:$loLvl, LevelAttr:$hiLvl);
-
-  let results = (outs AnySparseIterSpace:$resultSpace);
-
-  let summary = "Extract an iteration space from a sparse tensor between certain levels";
-  let description = [{
-      Extracts a `!sparse_tensor.iter_space` from a sparse tensor between
-      certian (consecutive) levels.
-
-      `tensor`: the input sparse tensor that defines the iteration space.
-      `parentIter`: the iterator for the previous level, at which the iteration space
-      at the current levels will be extracted.
-      `loLvl`, `hiLvl`: the level range between [loLvl, hiLvl) in the input tensor that
-      the returned iteration space covers. `hiLvl - loLvl` defines the dimension of the
-      iteration space.
-
-      Example:
-      ```mlir
-      // Extracts a 1-D iteration space from a COO tensor at level 1.
-      %space = sparse_tensor.iteration.extract_space %sp at %it1 lvls = 1
-        : tensor<4x8xf32, #COO>, !sparse_tensor.iterator<#COO, lvls = 0>
-      ```
-  }];
-
-
-  let extraClassDeclaration = [{
-    std::pair<Level, Level> getLvlRange() {
-      return std::make_pair(getLoLvl(), getHiLvl());
-    }
-    unsigned getSpaceDim() {
-      return getHiLvl() - getLoLvl();
-    }
-    ArrayRef<::mlir::sparse_tensor::LevelType> getSpaceLvlTypes() {
-      return getResultSpace().getType().getLvlTypes();
-    }
-  }];
-
-  let hasVerifier = 1;
-  let assemblyFormat = "$tensor (`at` $parentIter^)? `lvls` `=` custom<LevelRange>($loLvl, $hiLvl) "
-                       " attr-dict `:` type($tensor) (`,` type($parentIter)^)?";
-}
-
-def IterateOp : SparseTensor_Op<"iterate",
-    [RecursiveMemoryEffects, RecursivelySpeculatable,
-     DeclareOpInterfaceMethods<LoopLikeOpInterface,
-      ["getInitsMutable", "getLoopResults", "getRegionIterArgs",
-       "getYieldedValuesMutable"]>,
-     DeclareOpInterfaceMethods<RegionBranchOpInterface,
-      ["getEntrySuccessorOperands"]>,
-     SingleBlockImplicitTerminator<"sparse_tensor::YieldOp">]> {
-
-  let arguments = (ins AnySparseIterSpace:$iterSpace,
-                       Variadic<AnyType>:$initArgs,
-                       LevelSetAttr:$crdUsedLvls);
-  let results = (outs Variadic<AnyType>:$results);
-  let regions = (region SizedRegion<1>:$region);
-
-  let summary = "Iterate over a sparse iteration space";
-  let description = [{
-      The `sparse_tensor.iterate` operations represents a loop over the
-      provided iteration space extracted from a specific sparse tensor.
-      The operation defines an SSA value for a sparse iterator that points
-      to the current stored element in the sparse tensor and SSA values
-      for coordinates of the stored element. The coordinates are always
-      converted to `index` type despite of the underlying sparse tensor
-      storage. When coordinates are not used, the SSA values can be skipped
-      by `_` symbols, which usually leads to simpler generated code after
-      sparsification. For example:
-
-      ```mlir
-      // The coordinate for level 0 is not used when iterating over a 2-D
-      // iteration space.
-      %sparse_tensor.iterate %iterator in %space at(_, %crd_1)
-        : !sparse_tensor.iter_space<#CSR, lvls = 0 to 2>
-      ```
-
-      `sparse_tensor.iterate` can also operate on loop-carried variables
-      and returns the final values after loop termination.
-      The initial values of the variables are passed as additional SSA operands
-      to the iterator SSA value and used coordinate SSA values mentioned
-      above. The operation region has an argument for the iterator, variadic
-      arguments for specified (used) coordiates and followed by one argument
-      for each loop-carried variable, representing the value of the variable
-      at the current iteration.
-      The body region must contain exactly one block that terminates with
-      `sparse_tensor.yield`.
-
-      `sparse_tensor.iterate` results hold the final values after the last
-      iteration. If the `sparse_tensor.iterate` defines any values, a yield
-      must be explicitly present.
-      The number and types of the `sparse_tensor.iterate` results must match
-      the initial values in the iter_args binding and the yield operands.
-
-
-      A nested `sparse_tensor.iterate` example that prints all the coordinates
-      stored in the sparse input:
-
-      ```mlir
-      func.func @nested_iterate(%sp : tensor<4x8xf32, #COO>) {
-        // Iterates over the first level of %sp
-        %l1 = sparse_tensor.extract_iteration_space %sp lvls = 0 : tensor<4x8xf32, #COO>
-        %r1 = sparse_tensor.iterate %it1 in %l1 at (%crd0)
-            : !sparse_tensor.iter_space<#COO, lvls = 0 to 1>  {
-          // Iterates over the second level of %sp
-          %l2 = sparse_tensor.extract_iteration_space %sp at %it1 lvls = 1
-              : tensor<4x8xf32, #COO>, !sparse_tensor.iterator<#COO, lvls = 0 to 1>
-          %r2 = sparse_tensor.iterate %it2 in %l2 at (crd1)
-              : !sparse_tensor.iter_space<#COO, lvls = 1 to 2>  {
-             vector.print %crd0 : index
-             vector.print %crd1 : index
-          }
-        }
-      }
-
-      ```
-  }];
-
-  let extraClassDeclaration = [{
-    unsigned getSpaceDim() {
-      return getIterSpace().getType().getSpaceDim();
-    }
-    BlockArgument getIterator() {
-      return getRegion().getArguments().front();
-    }
-    Block::BlockArgListType getCrds() {
-      // The first block argument is iterator, the remaining arguments are
-      // referenced coordinates.
-      return getRegion().getArguments().slice(1, getCrdUsedLvls().count());
-    }
-    unsigned getNumRegionIterArgs() {
-      return getRegion().getArguments().size() - 1 - getCrdUsedLvls().count();
-    }
-  }];
-
-  let hasVerifier = 1;
-  let hasRegionVerifier = 1;
-  let hasCustomAssemblyFormat = 1;
-}
-
 //===----------------------------------------------------------------------===//
 // Sparse Tensor Debugging and Test-Only Operations.
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorTypes.td b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorTypes.td
index 264a0a5b3bee..185cff46ae25 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorTypes.td
+++ b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorTypes.td
@@ -72,99 +72,4 @@ def SparseTensorStorageSpecifier
     : Type<CPred<"::llvm::isa<::mlir::sparse_tensor::StorageSpecifierType>($_self)">, "metadata",
           "::mlir::sparse_tensor::StorageSpecifierType">;
 
-//===----------------------------------------------------------------------===//
-// Sparse Tensor Iteration Types.
-//===----------------------------------------------------------------------===//
-
-def SparseTensor_IterSpace : SparseTensor_Type<"IterSpace"> {
-  let mnemonic = "iter_space";
-
-  let description = [{
-    A sparse iteration space that represents an abstract N-D (sparse) iteration space
-    extracted from a sparse tensor.
-
-    Examples:
-
-    ```mlir
-    // An iteration space extracted from a CSR tensor between levels [0, 2).
-    !iter_space<#CSR, lvls = 0 to 2>
-    ```
-  }];
-
-  let parameters = (ins
-     SparseTensorEncodingAttr : $encoding,
-     "Level" : $loLvl,
-     "Level" : $hiLvl
-  );
-
-  let extraClassDeclaration = [{
-     /// The the dimension of the iteration space.
-     unsigned getSpaceDim() const {
-       return getHiLvl() - getLoLvl();
-     }
-
-     /// Get the level types for the iteration space.
-     ArrayRef<LevelType> getLvlTypes() const {
-       return getEncoding().getLvlTypes().slice(getLoLvl(), getSpaceDim());
-     }
-
-     /// Whether the iteration space is unique (i.e., no duplicated coordinate).
-     bool isUnique() {
-       return !getLvlTypes().back().isa<LevelPropNonDefault::Nonunique>();
-     }
-
-     /// Get the corresponding iterator type.
-     ::mlir::sparse_tensor::IteratorType getIteratorType() const;
-  }];
-
-  let assemblyFormat="`<` $encoding `,` `lvls` `=` custom<LevelRange>($loLvl, $hiLvl) `>`";
-}
-
-def SparseTensor_Iterator : SparseTensor_Type<"Iterator"> {
-  let mnemonic = "iterator";
-
-  let description = [{
-    An iterator that points to the current element in the corresponding iteration space.
-
-    Examples:
-
-    ```mlir
-    // An iterator that iterates over a iteration space of type `!iter_space<#CSR, lvls = 0 to 2>`
-    !iterator<#CSR, lvls = 0 to 2>
-    ```
-  }];
-
-  let parameters = (ins
-     SparseTensorEncodingAttr : $encoding,
-     "Level" : $loLvl,
-     "Level" : $hiLvl
-  );
-
-  let extraClassDeclaration = [{
-     /// Get the corresponding iteration space type.
-     ::mlir::sparse_tensor::IterSpaceType getIterSpaceType() const;
-
-     unsigned getSpaceDim() const { return getIterSpaceType().getSpaceDim(); }
-     ArrayRef<LevelType> getLvlTypes() const { return getIterSpaceType().getLvlTypes(); }
-     bool isUnique() { return getIterSpaceType().isUnique(); }
-  }];
-
-  let assemblyFormat="`<` $encoding `,` `lvls` `=` custom<LevelRange>($loLvl, $hiLvl) `>`";
-}
-
-def IsSparseSparseIterSpaceTypePred
-    : CPred<"::llvm::isa<::mlir::sparse_tensor::IterSpaceType>($_self)">;
-
-def IsSparseSparseIteratorTypePred
-    : CPred<"::llvm::isa<::mlir::sparse_tensor::IteratorType>($_self)">;
-
-def AnySparseIterSpace
-    : Type<IsSparseSparseIterSpaceTypePred, "sparse iteration space",
-          "::mlir::sparse_tensor::IterSpaceType">;
-
-def AnySparseIterator
-    : Type<IsSparseSparseIteratorTypePred, "sparse iterator",
-          "::mlir::sparse_tensor::IteratorType">;
-
-
 #endif // SPARSETENSOR_TYPES
diff --git a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
index 36908def09f4..e9058394d33d 100644
--- a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
+++ b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
@@ -30,14 +30,6 @@
 #include "mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.cpp.inc"
 #include "mlir/Dialect/SparseTensor/IR/SparseTensorAttrEnums.cpp.inc"
 
-// Forward declarations, following custom print/parsing methods are referenced
-// by the generated code for SparseTensorTypes.td.
-static mlir::ParseResult parseLevelRange(mlir::AsmParser &,
-                                         mlir::sparse_tensor::Level &,
-                                         mlir::sparse_tensor::Level &);
-static void printLevelRange(mlir::AsmPrinter &, mlir::sparse_tensor::Level,
-                            mlir::sparse_tensor::Level);
-
 #define GET_TYPEDEF_CLASSES
 #include "mlir/Dialect/SparseTensor/IR/SparseTensorTypes.cpp.inc"
 
@@ -1961,363 +1953,6 @@ LogicalResult SortOp::verify() {
   return success();
 }
 
-//===----------------------------------------------------------------------===//
-// Sparse Tensor Iteration Operations.
-//===----------------------------------------------------------------------===//
-
-IterSpaceType IteratorType::getIterSpaceType() const {
-  return IterSpaceType::get(getContext(), getEncoding(), getLoLvl(),
-                            getHiLvl());
-}
-
-IteratorType IterSpaceType::getIteratorType() const {
-  return IteratorType::get(getContext(), getEncoding(), getLoLvl(), getHiLvl());
-}
-
-/// Parses a level range in the form "$lo `to` $hi"
-/// or simply "$lo" if $hi - $lo = 1
-static ParseResult parseLevelRange(AsmParser &parser, Level &lvlLo,
-                                   Level &lvlHi) {
-  if (parser.parseInteger(lvlLo))
-    return failure();
-
-  if (succeeded(parser.parseOptionalKeyword("to"))) {
-    if (parser.parseInteger(lvlHi))
-      return failure();
-  } else {
-    lvlHi = lvlLo + 1;
-  }
-
-  if (lvlHi <= lvlLo)
-    parser.emitError(parser.getNameLoc(),
-                     "expect larger level upper bound than lower bound");
-
-  return success();
-}
-
-/// Parses a level range in the form "$lo `to` $hi"
-/// or simply "$lo" if $hi - $lo = 1
-static ParseResult parseLevelRange(OpAsmParser &parser, IntegerAttr &lvlLoAttr,
-                                   IntegerAttr &lvlHiAttr) {
-  Level lvlLo, lvlHi;
-  if (parseLevelRange(parser, lvlLo, lvlHi))
-    return failure();
-
-  lvlLoAttr = IntegerAttr::get(parser.getBuilder().getIndexType(), lvlLo);
-  lvlHiAttr = IntegerAttr::get(parser.getBuilder().getIndexType(), lvlHi);
-  return success();
-}
-
-/// Prints a level range in the form "$lo `to` $hi"
-/// or simply "$lo" if $hi - $lo = 1
-static void printLevelRange(AsmPrinter &p, Level lo, Level hi) {
-
-  if (lo + 1 == hi)
-    p << lo;
-  else
-    p << lo << " to " << hi;
-}
-
-/// Prints a level range in the form "$lo `to` $hi"
-/// or simply "$lo" if $hi - $lo = 1
-static void printLevelRange(OpAsmPrinter &p, Operation *, IntegerAttr lvlLo,
-                            IntegerAttr lvlHi) {
-  unsigned lo = lvlLo.getValue().getZExtValue();
-  unsigned hi = lvlHi.getValue().getZExtValue();
-  printLevelRange(p, lo, hi);
-}
-
-static ParseResult
-parseSparseSpaceLoop(OpAsmParser &parser, OperationState &state,
-                     SmallVectorImpl<OpAsmParser::Argument> &iterators,
-                     SmallVectorImpl<OpAsmParser::Argument> &iterArgs) {
-  SmallVector<OpAsmParser::UnresolvedOperand> spaces;
-  SmallVector<OpAsmParser::UnresolvedOperand> initArgs;
-
-  // Parses "%iters, ... in %spaces, ..."
-  if (parser.parseArgumentList(iterators) || parser.parseKeyword("in") ||
-      parser.parseOperandList(spaces))
-    return failure();
-
-  if (iterators.size() != spaces.size())
-    return parser.emitError(
-        parser.getNameLoc(),
-        "mismatch in number of sparse iterators and sparse spaces");
-
-  // Parse "at(%crd0, _, ...)"
-  LevelSet crdUsedLvlSet;
-  bool hasUsedCrds = succeeded(parser.parseOptionalKeyword("at"));
-  unsigned lvlCrdCnt = 0;
-  if (hasUsedCrds) {
-    ParseResult crdList = parser.parseCommaSeparatedList(
-        OpAsmParser::Delimiter::Paren, [&]() -> ParseResult {
-          if (parser.parseOptionalKeyword("_")) {
-            if (parser.parseArgument(iterArgs.emplace_back()))
-              return failure();
-            // Always use IndexType for the coordinate.
-            crdUsedLvlSet.set(lvlCrdCnt);
-            iterArgs.back().type = parser.getBuilder().getIndexType();
-          }
-          lvlCrdCnt += 1;
-          return success();
-        });
-    if (failed(crdList)) {
-      return parser.emitError(
-          parser.getNameLoc(),
-          "expecting SSA value or \"_\" for level coordinates");
-    }
-  }
-  // Set the CrdUsedLvl bitset.
-  state.addAttribute("crdUsedLvls",
-                     parser.getBuilder().getI64IntegerAttr(crdUsedLvlSet));
-
-  // Parse "iter_args(%arg = %init, ...)"
-  bool hasIterArgs = succeeded(parser.parseOptionalKeyword("iter_args"));
-  if (hasIterArgs)
-    if (parser.parseAssignmentList(iterArgs, initArgs))
-      return failure();
-
-  SmallVector<Type> iterSpaceTps;
-  // parse ": sparse_tensor.iter_space -> ret"
-  if (parser.parseColon() || parser.parseTypeList(iterSpaceTps))
-    return failure();
-  if (iterSpaceTps.size() != spaces.size())
-    return parser.emitError(parser.getNameLoc(),
-                            "mismatch in number of iteration space operands "
-                            "and iteration space types");
-
-  for (auto [it, tp] : llvm::zip_equal(iterators, iterSpaceTps)) {
-    IterSpaceType spaceTp = llvm::dyn_cast<IterSpaceType>(tp);
-    if (!spaceTp)
-      return parser.emitError(parser.getNameLoc(),
-                              "expected sparse_tensor.iter_space type for "
-                              "iteration space operands");
-    if (hasUsedCrds && spaceTp.getSpaceDim() != lvlCrdCnt)
-      return parser.emitError(parser.getNameLoc(),
-                              "mismatch in number of iteration space dimension "
-                              "and specified coordinates");
-    it.type = spaceTp.getIteratorType();
-  }
-
-  if (hasIterArgs)
-    if (parser.parseArrowTypeList(state.types))
-      return failure();
-
-  // Resolves input operands.
-  if (parser.resolveOperands(spaces, iterSpaceTps, parser.getNameLoc(),
-                             state.operands))
-    return failure();
-
-  if (hasIterArgs) {
-    unsigned numCrds = crdUsedLvlSet.count();
-    // Strip off leading args that used for coordinates.
-    MutableArrayRef args = MutableArrayRef(iterArgs).drop_front(numCrds);
-    if (args.size() != initArgs.size() || args.size() != state.types.size()) {
-      return parser.emitError(
-          parser.getNameLoc(),
-          "mismatch in number of iteration arguments and return values");
-    }
-
-    for (auto [it, init, tp] : llvm::zip_equal(args, initArgs, state.types)) {
-      it.type = tp;
-      if (parser.resolveOperand(init, tp, state.operands))
-        return failure();
-    }
-  }
-  return success();
-}
-
-LogicalResult ExtractIterSpaceOp::inferReturnTypes(
-    MLIRContext *ctx, std::optional<Location> loc, ValueRange ops,
-    DictionaryAttr attr, OpaqueProperties prop, RegionRange region,
-    SmallVectorImpl<mlir::Type> &ret) {
-
-  ExtractIterSpaceOp::Adaptor adaptor(ops, attr, prop, region);
-  SparseTensorType stt = getSparseTensorType(adaptor.getTensor());
-  ret.push_back(IterSpaceType::get(ctx, stt.getEncoding(), adaptor.getLoLvl(),
-                                   adaptor.getHiLvl()));
-  return success();
-}
-
-LogicalResult ExtractIterSpaceOp::verify() {
-  if (getLoLvl() >= getHiLvl())
-    return emitOpError("expected smaller level low than level high");
-
-  TypedValue<IteratorType> pIter = getParentIter();
-  if ((pIter && getLoLvl() == 0) || (!pIter && getLoLvl() != 0)) {
-    return emitOpError(
-        "parent iterator should be specified iff level lower bound equals 0");
-  }
-
-  if (pIter) {
-    IterSpaceType spaceTp = getResultSpace().getType();
-    if (pIter.getType().getEncoding() != spaceTp.getEncoding())
-      return emitOpError(
-          "mismatch in parent iterator encoding and iteration space encoding.");
-
-    if (spaceTp.getLoLvl() != pIter.getType().getHiLvl())
-      return emitOpError("parent iterator should be used to extract an "
-                         "iteration space from a consecutive level.");
-  }
-
-  return success();
-}
-
-ParseResult IterateOp::parse(OpAsmParser &parser, OperationState &result) {
-  OpAsmParser::Argument iterator;
-  OpAsmParser::UnresolvedOperand iterSpace;
-
-  SmallVector<OpAsmParser::Argument> iters, iterArgs;
-  if (parseSparseSpaceLoop(parser, result, iters, iterArgs))
-    return failure();
-  if (iters.size() != 1)
-    return parser.emitError(parser.getNameLoc(),
-                            "expected only one iterator/iteration space");
-
-  iters.append(iterArgs);
-  Region *body = result.addRegion();
-  if (parser.parseRegion(*body, iters))
-    return failure();
-
-  IterateOp::ensureTerminator(*body, parser.getBuilder(), result.location);
-
-  // Parse the optional attribute list.
-  if (parser.parseOptionalAttrDict(result.attributes))
-    return failure();
-
-  return success();
-}
-
-/// Prints the initialization list in the form of
-///   <prefix>(%inner = %outer, %inner2 = %outer2, <...>)
-/// where 'inner' values are assumed to be region arguments and 'outer' values
-/// are regular SSA values.
-static void printInitializationList(OpAsmPrinter &p,
-                                    Block::BlockArgListType blocksArgs,
-                                    ValueRange initializers,
-                                    StringRef prefix = "") {
-  assert(blocksArgs.size() == initializers.size() &&
-         "expected same length of arguments and initializers");
-  if (initializers.empty())
-    return;
-
-  p << prefix << '(';
-  llvm::interleaveComma(llvm::zip(blocksArgs, initializers), p, [&](auto it) {
-    p << std::get<0>(it) << " = " << std::get<1>(it);
-  });
-  p << ")";
-}
-
-static void printUsedCrdsList(OpAsmPrinter &p, unsigned spaceDim,
-                              Block::BlockArgListType blocksArgs,
-                              LevelSet crdUsedLvls) {
-  if (crdUsedLvls.empty())
-    return;
-
-  p << " at(";
-  for (unsigned i = 0; i < spaceDim; i++) {
-    if (crdUsedLvls[i]) {
-      p << blocksArgs.front();
-      blocksArgs = blocksArgs.drop_front();
-    } else {
-      p << "_";
-    }
-    if (i != spaceDim - 1)
-      p << ", ";
-  }
-  assert(blocksArgs.empty());
-  p << ")";
-}
-
-void IterateOp::print(OpAsmPrinter &p) {
-  p << " " << getIterator() << " in " << getIterSpace();
-  printUsedCrdsList(p, getSpaceDim(), getCrds(), getCrdUsedLvls());
-  printInitializationList(p, getRegionIterArgs(), getInitArgs(), " iter_args");
-
-  p << " : " << getIterSpace().getType() << " ";
-  if (!getInitArgs().empty())
-    p << "-> (" << getInitArgs().getTypes() << ") ";
-
-  p.printRegion(getRegion(), /*printEntryBlockArgs=*/false,
-                /*printBlockTerminators=*/!getInitArgs().empty());
-}
-
-LogicalResult IterateOp::verify() {
-  if (getInitArgs().size() != getNumResults()) {
-    return emitOpError(
-        "mismatch in number of loop-carried values and defined values");
-  }
-  return success();
-}
-
-LogicalResult IterateOp::verifyRegions() {
-  if (getIterator().getType() != getIterSpace().getType().getIteratorType())
-    return emitOpError("mismatch in iterator and iteration space type");
-  if (getNumRegionIterArgs() != getNumResults())
-    return emitOpError(
-        "mismatch in number of basic block args and defined values");
-
-  auto initArgs = getInitArgs();
-  auto iterArgs = getRegionIterArgs();
-  auto yieldVals = getYieldedValues();
-  auto opResults = getResults();
-  if (!llvm::all_equal({initArgs.size(), iterArgs.size(), yieldVals.size(),
-                        opResults.size()})) {
-    return emitOpError() << "number mismatch between iter args and results.";
-  }
-
-  for (auto [i, init, iter, yield, ret] :
-       llvm::enumerate(initArgs, iterArgs, yieldVals, opResults)) {
-    if (init.getType() != ret.getType())
-      return emitOpError() << "types mismatch between " << i
-                           << "th iter operand and defined value";
-    if (iter.getType() != ret.getType())
-      return emitOpError() << "types mismatch between " << i
-                           << "th iter region arg and defined value";
-    if (yield.getType() != ret.getType())
-      return emitOpError() << "types mismatch between " << i
-                           << "th yield value and defined value";
-  }
-
-  return success();
-}
-
-/// IterateOp implemented OpInterfaces' methods.
-SmallVector<Region *> IterateOp::getLoopRegions() { return {&getRegion()}; }
-
-MutableArrayRef<OpOperand> IterateOp::getInitsMutable() {
-  return getInitArgsMutable();
-}
-
-Block::BlockArgListType IterateOp::getRegionIterArgs() {
-  return getRegion().getArguments().take_back(getNumRegionIterArgs());
-}
-
-std::optional<MutableArrayRef<OpOperand>> IterateOp::getYieldedValuesMutable() {
-  return cast<sparse_tensor::YieldOp>(
-             getRegion().getBlocks().front().getTerminator())
-      .getResultsMutable();
-}
-
-std::optional<ResultRange> IterateOp::getLoopResults() { return getResults(); }
-
-OperandRange IterateOp::getEntrySuccessorOperands(RegionBranchPoint point) {
-  return getInitArgs();
-}
-
-void IterateOp::getSuccessorRegions(RegionBranchPoint point,
-                                    SmallVectorImpl<RegionSuccessor> &regions) {
-  // Both the operation itself and the region may be branching into the body or
-  // back into the operation itself.
-  regions.push_back(RegionSuccessor(&getRegion(), getRegionIterArgs()));
-  // It is possible for loop not to enter the body.
-  regions.push_back(RegionSuccessor(getResults()));
-}
-
-//===----------------------------------------------------------------------===//
-// Sparse Tensor Dialect Setups.
-//===----------------------------------------------------------------------===//
-
 /// Materialize a single constant operation from a given attribute value with
 /// the desired resultant type.
 Operation *SparseTensorDialect::materializeConstant(OpBuilder &builder,
diff --git a/mlir/test/Dialect/SparseTensor/invalid.mlir b/mlir/test/Dialect/SparseTensor/invalid.mlir
index b13024cd4ed9..7f5c05190fc9 100644
--- a/mlir/test/Dialect/SparseTensor/invalid.mlir
+++ b/mlir/test/Dialect/SparseTensor/invalid.mlir
@@ -1012,142 +1012,3 @@ func.func @sparse_print(%arg0: tensor<10x10xf64>) {
   sparse_tensor.print %arg0 : tensor<10x10xf64>
   return
 }
-
-// -----
-
-#COO = #sparse_tensor.encoding<{
-  map = (i, j) -> (
-    i : compressed(nonunique),
-    j : singleton(soa)
-  )
-}>
-
-func.func @sparse_extract_iter_space(%sp : tensor<4x8xf32, #COO>, %it1 : !sparse_tensor.iterator<#COO, lvls = 2>) {
-  // expected-error@+1 {{'sparse_tensor.extract_iteration_space' expect larger level upper bound than lower bound}}
-  %l1 = sparse_tensor.extract_iteration_space %sp at %it1 lvls = 2 to 0 : tensor<4x8xf32, #COO>, !sparse_tensor.iterator<#COO, lvls = 2>
-  return
-}
-
-// -----
-
-#COO = #sparse_tensor.encoding<{
-  map = (i, j) -> (
-    i : compressed(nonunique),
-    j : singleton(soa)
-  )
-}>
-
-func.func @sparse_extract_iter_space(%sp : tensor<4x8xf32, #COO>, %it1 : !sparse_tensor.iterator<#COO, lvls = 0>) {
-  // expected-error@+1 {{'sparse_tensor.extract_iteration_space' op parent iterator should be specified iff level lower bound equals 0}}
-  %l1 = sparse_tensor.extract_iteration_space %sp at %it1 lvls = 0 : tensor<4x8xf32, #COO>, !sparse_tensor.iterator<#COO, lvls = 0>
-  return
-}
-
-// -----
-
-#COO = #sparse_tensor.encoding<{
-  map = (i, j) -> (
-    i : compressed(nonunique),
-    j : singleton(soa)
-  )
-}>
-
-func.func @sparse_extract_iter_space(%sp : tensor<4x8xf32, #COO>) {
-  // expected-error@+1 {{'sparse_tensor.extract_iteration_space' op parent iterator should be specified iff level lower bound equals 0}}
-  %l1 = sparse_tensor.extract_iteration_space %sp lvls = 1 : tensor<4x8xf32, #COO>
-  return
-}
-
-// -----
-
-#COO = #sparse_tensor.encoding<{
-  map = (i, j) -> (
-    i : compressed(nonunique),
-    j : singleton(soa)
-  )
-}>
-
-#CSR = #sparse_tensor.encoding<{
-  map = (i, j) -> (
-    i : dense,
-    j : compressed
-  )
-}>
-
-func.func @sparse_extract_iter_space(%sp : tensor<4x8xf32, #COO>, %it1 : !sparse_tensor.iterator<#CSR, lvls = 0>) {
-  // expected-error@+1 {{'sparse_tensor.extract_iteration_space' op mismatch in parent iterator encoding and iteration space encoding.}}
-  %l1 = sparse_tensor.extract_iteration_space %sp at %it1 lvls = 1 : tensor<4x8xf32, #COO>, !sparse_tensor.iterator<#CSR, lvls = 0>
-  return
-}
-
-// -----
-
-#COO = #sparse_tensor.encoding<{
-  map = (i, j) -> (
-    i : compressed(nonunique),
-    j : singleton(soa)
-  )
-}>
-
-func.func @sparse_extract_iter_space(%sp : tensor<4x8xf32, #COO>, %it1 : !sparse_tensor.iterator<#COO, lvls = 0>) {
-  // expected-error@+1 {{'sparse_tensor.extract_iteration_space' op parent iterator should be used to extract an iteration space from a consecutive level.}}
-  %l1 = sparse_tensor.extract_iteration_space %sp at %it1 lvls = 2 : tensor<4x8xf32, #COO>, !sparse_tensor.iterator<#COO, lvls = 0>
-  return
-}
-
-
-// -----
-
-#COO = #sparse_tensor.encoding<{
-  map = (i, j) -> (
-    i : compressed(nonunique),
-    j : singleton(soa)
-  )
-}>
-
-func.func @sparse_iterate(%sp : tensor<4x8xf32, #COO>, %i : index, %j : index) -> index {
-  %l1 = sparse_tensor.extract_iteration_space %sp lvls = 0 : tensor<4x8xf32, #COO>
-  // expected-error @+1 {{'sparse_tensor.iterate' op different number of region iter_args and yielded values: 2 != 1}}
-  %r1, %r2 = sparse_tensor.iterate %it1 in %l1 at (%crd) iter_args(%si = %i, %sj = %j): !sparse_tensor.iter_space<#COO, lvls = 0> -> (index, index) {
-    sparse_tensor.yield %si : index
-  }
-  return %r1 : index
-}
-
-// -----
-
-#COO = #sparse_tensor.encoding<{
-  map = (i, j) -> (
-    i : compressed(nonunique),
-    j : singleton(soa)
-  )
-}>
-
-// expected-note@+1 {{prior use here}}
-func.func @sparse_iterate(%sp : tensor<4x8xf32, #COO>, %i : index) -> f32 {
-  %l1 = sparse_tensor.extract_iteration_space %sp lvls = 0 : tensor<4x8xf32, #COO>
-  // expected-error @+1 {{use of value '%i' expects different type than prior uses: 'f32' vs 'index'}}
-  %r1 = sparse_tensor.iterate %it1 in %l1 at (%crd) iter_args(%outer = %i): !sparse_tensor.iter_space<#COO, lvls = 0> -> f32 {
-    sparse_tensor.yield %outer : f32
-  }
-  return %r1 : f32
-}
-
-// -----
-
-#COO = #sparse_tensor.encoding<{
-  map = (i, j) -> (
-    i : compressed(nonunique),
-    j : singleton(soa)
-  )
-}>
-
-func.func @sparse_iterate(%sp : tensor<4x8xf32, #COO>, %i : index, %j : index) -> index {
-  %l1 = sparse_tensor.extract_iteration_space %sp lvls = 0 : tensor<4x8xf32, #COO>
-  // expected-error @+1 {{'sparse_tensor.iterate' op 0-th region iter_arg and 0-th yielded value have different type: 'index' != 'f32'}}
-  %r1 = sparse_tensor.iterate %it1 in %l1 at (%crd) iter_args(%si = %i): !sparse_tensor.iter_space<#COO, lvls = 0> -> index {
-    %y = arith.constant 1.0 :  f32
-    sparse_tensor.yield %y : f32
-  }
-  return %r1 : index
-}
diff --git a/mlir/test/Dialect/SparseTensor/roundtrip.mlir b/mlir/test/Dialect/SparseTensor/roundtrip.mlir
index e9a898f16b41..12f69c1d37b9 100644
--- a/mlir/test/Dialect/SparseTensor/roundtrip.mlir
+++ b/mlir/test/Dialect/SparseTensor/roundtrip.mlir
@@ -738,56 +738,3 @@ func.func @sparse_has_runtime() -> i1 {
   %has_runtime = sparse_tensor.has_runtime_library
   return %has_runtime : i1
 }
-
-// -----
-
-#COO = #sparse_tensor.encoding<{
-  map = (i, j) -> (
-    i : compressed(nonunique),
-    j : singleton(soa)
-  )
-}>
-
-// CHECK-LABEL:   func.func @sparse_extract_iter_space(
-// CHECK-SAME:      %[[VAL_0:.*]]: tensor<4x8xf32, #sparse{{[0-9]*}}>,
-// CHECK-SAME:      %[[VAL_1:.*]]: !sparse_tensor.iterator<#sparse{{[0-9]*}}, lvls = 0>)
-// CHECK:           %[[VAL_2:.*]] = sparse_tensor.extract_iteration_space %[[VAL_0]] lvls = 0
-// CHECK:           %[[VAL_3:.*]] = sparse_tensor.extract_iteration_space %[[VAL_0]] at %[[VAL_1]] lvls = 1
-// CHECK:           return %[[VAL_2]], %[[VAL_3]] : !sparse_tensor.iter_space<#sparse{{[0-9]*}}, lvls = 0>, !sparse_tensor.iter_space<#sparse{{[0-9]*}}, lvls = 1>
-// CHECK:         }
-func.func @sparse_extract_iter_space(%sp : tensor<4x8xf32, #COO>, %it1 : !sparse_tensor.iterator<#COO, lvls = 0>)
-  -> (!sparse_tensor.iter_space<#COO, lvls = 0>, !sparse_tensor.iter_space<#COO, lvls = 1>) {
-  // Extracting the iteration space for the first level needs no parent iterator.
-  %l1 = sparse_tensor.extract_iteration_space %sp lvls = 0 : tensor<4x8xf32, #COO>
-  // Extracting the iteration space for the second level needs a parent iterator.
-  %l2 = sparse_tensor.extract_iteration_space %sp at %it1 lvls = 1 : tensor<4x8xf32, #COO>, !sparse_tensor.iterator<#COO, lvls = 0>
-  return %l1, %l2 : !sparse_tensor.iter_space<#COO, lvls = 0>, !sparse_tensor.iter_space<#COO, lvls = 1>
-}
-
-
-// -----
-
-#COO = #sparse_tensor.encoding<{
-  map = (i, j) -> (
-    i : compressed(nonunique),
-    j : singleton(soa)
-  )
-}>
-
-// CHECK-LABEL:   func.func @sparse_iterate(
-// CHECK-SAME:      %[[VAL_0:.*]]: tensor<4x8xf32, #sparse{{[0-9]*}}>,
-// CHECK-SAME:      %[[VAL_1:.*]]: index,
-// CHECK-SAME:      %[[VAL_2:.*]]: index) -> index {
-// CHECK:           %[[VAL_3:.*]] = sparse_tensor.extract_iteration_space %[[VAL_0]] lvls = 0 : tensor<4x8xf32, #sparse{{[0-9]*}}>
-// CHECK:           %[[VAL_4:.*]] = sparse_tensor.iterate %[[VAL_5:.*]] in %[[VAL_3]] at(%[[VAL_6:.*]]) iter_args(%[[VAL_7:.*]] = %[[VAL_1]]) : !sparse_tensor.iter_space<#sparse{{[0-9]*}}, lvls = 0> -> (index) {
-// CHECK:             sparse_tensor.yield %[[VAL_7]] : index
-// CHECK:           }
-// CHECK:           return %[[VAL_4]] : index
-// CHECK:         }
-func.func @sparse_iterate(%sp : tensor<4x8xf32, #COO>, %i : index, %j : index) -> index {
-  %l1 = sparse_tensor.extract_iteration_space %sp lvls = 0 : tensor<4x8xf32, #COO>
-  %r1 = sparse_tensor.iterate %it1 in %l1 at (%crd) iter_args(%outer = %i): !sparse_tensor.iter_space<#COO, lvls = 0 to 1> -> index {
-    sparse_tensor.yield %outer : index
-  }
-  return %r1 : index
-}
diff --git a/mlir/test/Dialect/SparseTensor/sparse_itertion_licm.mlir b/mlir/test/Dialect/SparseTensor/sparse_itertion_licm.mlir
deleted file mode 100644
index e7158d04b37f..000000000000
--- a/mlir/test/Dialect/SparseTensor/sparse_itertion_licm.mlir
+++ /dev/null
@@ -1,26 +0,0 @@
-// RUN: mlir-opt %s --loop-invariant-code-motion | FileCheck %s
-
-#CSR = #sparse_tensor.encoding<{
-  map = (i, j) -> (
-    i : dense,
-    j : compressed
-  )
-}>
-
-// Make sure that pure instructions are hoisted outside the loop.
-//
-// CHECK: sparse_tensor.values
-// CHECK: sparse_tensor.positions
-// CHECK: sparse_tensor.coordinate
-// CHECK: sparse_tensor.iterate
-func.func @sparse_iterate(%sp : tensor<?x?xf64, #CSR>) {
-  %l1 = sparse_tensor.extract_iteration_space %sp lvls = 0 : tensor<?x?xf64, #CSR>
-  sparse_tensor.iterate %it1 in %l1 at (%crd) : !sparse_tensor.iter_space<#CSR, lvls = 0> {
-    %0 = sparse_tensor.values %sp : tensor<?x?xf64, #CSR> to memref<?xf64>
-    %1 = sparse_tensor.positions %sp { level = 1 : index } : tensor<?x?xf64, #CSR> to memref<?xindex>
-    %2 = sparse_tensor.coordinates  %sp { level = 1 : index } : tensor<?x?xf64, #CSR> to memref<?xindex>
-    "test.op"(%0, %1, %2) : (memref<?xf64>, memref<?xindex>, memref<?xindex>) -> ()
-  }
-
-  return
-}
-- 
GitLab


From 481bd5d416df7a1d24e18cc81ae782e8701de965 Mon Sep 17 00:00:00 2001
From: Peiming Liu <peiming@google.com>
Date: Tue, 16 Apr 2024 11:32:30 -0700
Subject: [PATCH 449/464] [mlir][sparse] introduce
 `sparse_tensor.extract_iteration_space` operation. (#88554)

A `sparse_tensor.extract_space %tensor at %iterator` extracts a *sparse*
iteration space defined `%tensor`, the operation to traverse the
iteration space will be introduced in following PRs.
---
 .../SparseTensor/IR/SparseTensorOps.td        |  60 ++++++++++
 .../SparseTensor/IR/SparseTensorTypes.td      |  97 +++++++++++++++
 .../SparseTensor/IR/SparseTensorDialect.cpp   | 110 ++++++++++++++++++
 mlir/test/Dialect/SparseTensor/invalid.mlir   |  82 +++++++++++++
 mlir/test/Dialect/SparseTensor/roundtrip.mlir |  25 ++++
 5 files changed, 374 insertions(+)

diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td
index 0cfc64f9988a..d7121e8320a4 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td
+++ b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td
@@ -1430,6 +1430,66 @@ def SparseTensor_ForeachOp : SparseTensor_Op<"foreach",
   let hasVerifier = 1;
 }
 
+//===----------------------------------------------------------------------===//
+// Sparse Tensor Iteration Operations.
+//===----------------------------------------------------------------------===//
+
+def ExtractIterSpaceOp : SparseTensor_Op<"extract_iteration_space",
+    [Pure, DeclareOpInterfaceMethods<InferTypeOpInterface>]> {
+
+  let arguments = (ins AnySparseTensor:$tensor,
+                       Optional<AnySparseIterator>:$parentIter,
+                       LevelAttr:$loLvl, LevelAttr:$hiLvl);
+
+  let results = (outs AnySparseIterSpace:$resultSpace);
+
+  let summary = "Extracts an iteration space from a sparse tensor between certain levels";
+  let description = [{
+      Extracts a `!sparse_tensor.iter_space` from a sparse tensor between
+      certain (consecutive) levels. For sparse levels, it is usually done by
+      loading a postion range from the underlying sparse tensor storage.
+      E.g., for a compressed level, the iteration space is extracted by
+      [pos[i], pos[i+1]) supposing the the parent iterator points at `i`.
+
+      `tensor`: the input sparse tensor that defines the iteration space.
+      `parentIter`: the iterator for the previous level, at which the iteration space
+      at the current levels will be extracted.
+      `loLvl`, `hiLvl`: the level range between [loLvl, hiLvl) in the input tensor that
+      the returned iteration space covers. `hiLvl - loLvl` defines the dimension of the
+      iteration space.
+
+      The type of returned the value is automatically inferred to
+      `!sparse_tensor.iter_space<#INPUT_ENCODING, lvls = $loLvl to $hiLvl>`.
+      The returned iteration space can then be iterated over by
+      `sparse_tensor.iterate` operations to visit every stored element
+      (usually nonzeros) in the input sparse tensor.
+
+      Example:
+      ```mlir
+      // Extracts a 1-D iteration space from a COO tensor at level 1.
+      %space = sparse_tensor.iteration.extract_space %sp at %it1 lvls = 1
+        : tensor<4x8xf32, #COO>, !sparse_tensor.iterator<#COO, lvls = 0>
+      ```
+  }];
+
+
+  let extraClassDeclaration = [{
+    std::pair<Level, Level> getLvlRange() {
+      return std::make_pair(getLoLvl(), getHiLvl());
+    }
+    unsigned getSpaceDim() {
+      return getHiLvl() - getLoLvl();
+    }
+    ArrayRef<::mlir::sparse_tensor::LevelType> getSpaceLvlTypes() {
+      return getResultSpace().getType().getLvlTypes();
+    }
+  }];
+
+  let hasVerifier = 1;
+  let assemblyFormat = "$tensor (`at` $parentIter^)? `lvls` `=` custom<LevelRange>($loLvl, $hiLvl) "
+                       " attr-dict `:` type($tensor) (`,` type($parentIter)^)?";
+}
+
 //===----------------------------------------------------------------------===//
 // Sparse Tensor Debugging and Test-Only Operations.
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorTypes.td b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorTypes.td
index 185cff46ae25..79113d877874 100644
--- a/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorTypes.td
+++ b/mlir/include/mlir/Dialect/SparseTensor/IR/SparseTensorTypes.td
@@ -72,4 +72,101 @@ def SparseTensorStorageSpecifier
     : Type<CPred<"::llvm::isa<::mlir::sparse_tensor::StorageSpecifierType>($_self)">, "metadata",
           "::mlir::sparse_tensor::StorageSpecifierType">;
 
+//===----------------------------------------------------------------------===//
+// Sparse Tensor Iteration Types.
+//===----------------------------------------------------------------------===//
+
+def SparseTensor_IterSpace : SparseTensor_Type<"IterSpace"> {
+  let mnemonic = "iter_space";
+
+  let description = [{
+    A sparse iteration space that represents an abstract N-D (sparse) iteration space
+    extracted from a sparse tensor, i.e., a set of (crd_0, crd_1, ..., crd_N) for
+    every stored element (usually nonzeros) in a sparse tensor between the specified
+    [$loLvl, $hiLvl) levels.
+
+    Examples:
+
+    ```mlir
+    // An iteration space extracted from a CSR tensor between levels [0, 2).
+    !iter_space<#CSR, lvls = 0 to 2>
+    ```
+  }];
+
+  let parameters = (ins
+     SparseTensorEncodingAttr : $encoding,
+     "Level" : $loLvl,
+     "Level" : $hiLvl
+  );
+
+  let extraClassDeclaration = [{
+     /// The the dimension of the iteration space.
+     unsigned getSpaceDim() const {
+       return getHiLvl() - getLoLvl();
+     }
+
+     /// Get the level types for the iteration space.
+     ArrayRef<LevelType> getLvlTypes() const {
+       return getEncoding().getLvlTypes().slice(getLoLvl(), getSpaceDim());
+     }
+
+     /// Whether the iteration space is unique (i.e., no duplicated coordinate).
+     bool isUnique() {
+       return !getLvlTypes().back().isa<LevelPropNonDefault::Nonunique>();
+     }
+
+     /// Get the corresponding iterator type.
+     ::mlir::sparse_tensor::IteratorType getIteratorType() const;
+  }];
+
+  let assemblyFormat="`<` $encoding `,` `lvls` `=` custom<LevelRange>($loLvl, $hiLvl) `>`";
+}
+
+def SparseTensor_Iterator : SparseTensor_Type<"Iterator"> {
+  let mnemonic = "iterator";
+
+  let description = [{
+    An iterator that points to the current element in the corresponding iteration space.
+
+    Examples:
+
+    ```mlir
+    // An iterator that iterates over a iteration space of type `!iter_space<#CSR, lvls = 0 to 2>`
+    !iterator<#CSR, lvls = 0 to 2>
+    ```
+  }];
+
+  let parameters = (ins
+     SparseTensorEncodingAttr : $encoding,
+     "Level" : $loLvl,
+     "Level" : $hiLvl
+  );
+
+  let extraClassDeclaration = [{
+     /// Get the corresponding iteration space type.
+     ::mlir::sparse_tensor::IterSpaceType getIterSpaceType() const;
+
+     unsigned getSpaceDim() const { return getIterSpaceType().getSpaceDim(); }
+     ArrayRef<LevelType> getLvlTypes() const { return getIterSpaceType().getLvlTypes(); }
+     bool isUnique() { return getIterSpaceType().isUnique(); }
+  }];
+
+  let assemblyFormat="`<` $encoding `,` `lvls` `=` custom<LevelRange>($loLvl, $hiLvl) `>`";
+}
+
+def IsSparseSparseIterSpaceTypePred
+    : CPred<"::llvm::isa<::mlir::sparse_tensor::IterSpaceType>($_self)">;
+
+def IsSparseSparseIteratorTypePred
+    : CPred<"::llvm::isa<::mlir::sparse_tensor::IteratorType>($_self)">;
+
+def AnySparseIterSpace
+    : Type<IsSparseSparseIterSpaceTypePred, "sparse iteration space",
+          "::mlir::sparse_tensor::IterSpaceType">;
+
+def AnySparseIterator
+    : Type<IsSparseSparseIteratorTypePred, "sparse iterator",
+          "::mlir::sparse_tensor::IteratorType">;
+
+
 #endif // SPARSETENSOR_TYPES
diff --git a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
index e9058394d33d..516b0943bdcf 100644
--- a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
+++ b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp
@@ -30,6 +30,14 @@
 #include "mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.cpp.inc"
 #include "mlir/Dialect/SparseTensor/IR/SparseTensorAttrEnums.cpp.inc"
 
+// Forward declarations, following custom print/parsing methods are referenced
+// by the generated code for SparseTensorTypes.td.
+static mlir::ParseResult parseLevelRange(mlir::AsmParser &,
+                                         mlir::sparse_tensor::Level &,
+                                         mlir::sparse_tensor::Level &);
+static void printLevelRange(mlir::AsmPrinter &, mlir::sparse_tensor::Level,
+                            mlir::sparse_tensor::Level);
+
 #define GET_TYPEDEF_CLASSES
 #include "mlir/Dialect/SparseTensor/IR/SparseTensorTypes.cpp.inc"
 
@@ -1953,6 +1961,108 @@ LogicalResult SortOp::verify() {
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// Sparse Tensor Iteration Operations.
+//===----------------------------------------------------------------------===//
+
+IterSpaceType IteratorType::getIterSpaceType() const {
+  return IterSpaceType::get(getContext(), getEncoding(), getLoLvl(),
+                            getHiLvl());
+}
+
+IteratorType IterSpaceType::getIteratorType() const {
+  return IteratorType::get(getContext(), getEncoding(), getLoLvl(), getHiLvl());
+}
+
+/// Parses a level range in the form "$lo `to` $hi"
+/// or simply "$lo" if $hi - $lo = 1
+static ParseResult parseLevelRange(AsmParser &parser, Level &lvlLo,
+                                   Level &lvlHi) {
+  if (parser.parseInteger(lvlLo))
+    return failure();
+
+  if (succeeded(parser.parseOptionalKeyword("to"))) {
+    if (parser.parseInteger(lvlHi))
+      return failure();
+  } else {
+    lvlHi = lvlLo + 1;
+  }
+
+  if (lvlHi <= lvlLo)
+    parser.emitError(parser.getNameLoc(),
+                     "expect larger level upper bound than lower bound");
+
+  return success();
+}
+
+/// Parses a level range in the form "$lo `to` $hi"
+/// or simply "$lo" if $hi - $lo = 1
+static ParseResult parseLevelRange(OpAsmParser &parser, IntegerAttr &lvlLoAttr,
+                                   IntegerAttr &lvlHiAttr) {
+  Level lvlLo, lvlHi;
+  if (parseLevelRange(parser, lvlLo, lvlHi))
+    return failure();
+
+  lvlLoAttr = IntegerAttr::get(parser.getBuilder().getIndexType(), lvlLo);
+  lvlHiAttr = IntegerAttr::get(parser.getBuilder().getIndexType(), lvlHi);
+  return success();
+}
+
+/// Prints a level range in the form "$lo `to` $hi"
+/// or simply "$lo" if $hi - $lo = 1
+static void printLevelRange(AsmPrinter &p, Level lo, Level hi) {
+
+  if (lo + 1 == hi)
+    p << lo;
+  else
+    p << lo << " to " << hi;
+}
+
+/// Prints a level range in the form "$lo `to` $hi"
+/// or simply "$lo" if $hi - $lo = 1
+static void printLevelRange(OpAsmPrinter &p, Operation *, IntegerAttr lvlLo,
+                            IntegerAttr lvlHi) {
+  unsigned lo = lvlLo.getValue().getZExtValue();
+  unsigned hi = lvlHi.getValue().getZExtValue();
+  printLevelRange(p, lo, hi);
+}
+
+LogicalResult ExtractIterSpaceOp::inferReturnTypes(
+    MLIRContext *ctx, std::optional<Location> loc, ValueRange ops,
+    DictionaryAttr attr, OpaqueProperties prop, RegionRange region,
+    SmallVectorImpl<mlir::Type> &ret) {
+
+  ExtractIterSpaceOp::Adaptor adaptor(ops, attr, prop, region);
+  SparseTensorType stt = getSparseTensorType(adaptor.getTensor());
+  ret.push_back(IterSpaceType::get(ctx, stt.getEncoding(), adaptor.getLoLvl(),
+                                   adaptor.getHiLvl()));
+  return success();
+}
+
+LogicalResult ExtractIterSpaceOp::verify() {
+  if (getLoLvl() >= getHiLvl())
+    return emitOpError("expected smaller level low than level high");
+
+  TypedValue<IteratorType> pIter = getParentIter();
+  if ((pIter && getLoLvl() == 0) || (!pIter && getLoLvl() != 0)) {
+    return emitOpError(
+        "parent iterator should be specified iff level lower bound equals 0");
+  }
+
+  if (pIter) {
+    IterSpaceType spaceTp = getResultSpace().getType();
+    if (pIter.getType().getEncoding() != spaceTp.getEncoding())
+      return emitOpError(
+          "mismatch in parent iterator encoding and iteration space encoding.");
+
+    if (spaceTp.getLoLvl() != pIter.getType().getHiLvl())
+      return emitOpError("parent iterator should be used to extract an "
+                         "iteration space from a consecutive level.");
+  }
+
+  return success();
+}
+
 /// Materialize a single constant operation from a given attribute value with
 /// the desired resultant type.
 Operation *SparseTensorDialect::materializeConstant(OpBuilder &builder,
diff --git a/mlir/test/Dialect/SparseTensor/invalid.mlir b/mlir/test/Dialect/SparseTensor/invalid.mlir
index 7f5c05190fc9..3fa696e1600a 100644
--- a/mlir/test/Dialect/SparseTensor/invalid.mlir
+++ b/mlir/test/Dialect/SparseTensor/invalid.mlir
@@ -1012,3 +1012,85 @@ func.func @sparse_print(%arg0: tensor<10x10xf64>) {
   sparse_tensor.print %arg0 : tensor<10x10xf64>
   return
 }
+
+// -----
+
+#COO = #sparse_tensor.encoding<{
+  map = (i, j) -> (
+    i : compressed(nonunique),
+    j : singleton(soa)
+  )
+}>
+
+func.func @sparse_extract_iter_space(%sp : tensor<4x8xf32, #COO>, %it1 : !sparse_tensor.iterator<#COO, lvls = 2>) {
+  // expected-error@+1 {{'sparse_tensor.extract_iteration_space' expect larger level upper bound than lower bound}}
+  %l1 = sparse_tensor.extract_iteration_space %sp at %it1 lvls = 2 to 0 : tensor<4x8xf32, #COO>, !sparse_tensor.iterator<#COO, lvls = 2>
+  return
+}
+
+// -----
+
+#COO = #sparse_tensor.encoding<{
+  map = (i, j) -> (
+    i : compressed(nonunique),
+    j : singleton(soa)
+  )
+}>
+
+func.func @sparse_extract_iter_space(%sp : tensor<4x8xf32, #COO>, %it1 : !sparse_tensor.iterator<#COO, lvls = 0>) {
+  // expected-error@+1 {{'sparse_tensor.extract_iteration_space' op parent iterator should be specified iff level lower bound equals 0}}
+  %l1 = sparse_tensor.extract_iteration_space %sp at %it1 lvls = 0 : tensor<4x8xf32, #COO>, !sparse_tensor.iterator<#COO, lvls = 0>
+  return
+}
+
+// -----
+
+#COO = #sparse_tensor.encoding<{
+  map = (i, j) -> (
+    i : compressed(nonunique),
+    j : singleton(soa)
+  )
+}>
+
+func.func @sparse_extract_iter_space(%sp : tensor<4x8xf32, #COO>) {
+  // expected-error@+1 {{'sparse_tensor.extract_iteration_space' op parent iterator should be specified iff level lower bound equals 0}}
+  %l1 = sparse_tensor.extract_iteration_space %sp lvls = 1 : tensor<4x8xf32, #COO>
+  return
+}
+
+// -----
+
+#COO = #sparse_tensor.encoding<{
+  map = (i, j) -> (
+    i : compressed(nonunique),
+    j : singleton(soa)
+  )
+}>
+
+#CSR = #sparse_tensor.encoding<{
+  map = (i, j) -> (
+    i : dense,
+    j : compressed
+  )
+}>
+
+func.func @sparse_extract_iter_space(%sp : tensor<4x8xf32, #COO>, %it1 : !sparse_tensor.iterator<#CSR, lvls = 0>) {
+  // expected-error@+1 {{'sparse_tensor.extract_iteration_space' op mismatch in parent iterator encoding and iteration space encoding.}}
+  %l1 = sparse_tensor.extract_iteration_space %sp at %it1 lvls = 1 : tensor<4x8xf32, #COO>, !sparse_tensor.iterator<#CSR, lvls = 0>
+  return
+}
+
+// -----
+
+#COO = #sparse_tensor.encoding<{
+  map = (i, j) -> (
+    i : compressed(nonunique),
+    j : singleton(soa)
+  )
+}>
+
+func.func @sparse_extract_iter_space(%sp : tensor<4x8xf32, #COO>, %it1 : !sparse_tensor.iterator<#COO, lvls = 0>) {
+  // expected-error@+1 {{'sparse_tensor.extract_iteration_space' op parent iterator should be used to extract an iteration space from a consecutive level.}}
+  %l1 = sparse_tensor.extract_iteration_space %sp at %it1 lvls = 2 : tensor<4x8xf32, #COO>, !sparse_tensor.iterator<#COO, lvls = 0>
+  return
+}
diff --git a/mlir/test/Dialect/SparseTensor/roundtrip.mlir b/mlir/test/Dialect/SparseTensor/roundtrip.mlir
index 12f69c1d37b9..d34071279e51 100644
--- a/mlir/test/Dialect/SparseTensor/roundtrip.mlir
+++ b/mlir/test/Dialect/SparseTensor/roundtrip.mlir
@@ -738,3 +738,28 @@ func.func @sparse_has_runtime() -> i1 {
   %has_runtime = sparse_tensor.has_runtime_library
   return %has_runtime : i1
 }
+
+// -----
+
+#COO = #sparse_tensor.encoding<{
+  map = (i, j) -> (
+    i : compressed(nonunique),
+    j : singleton(soa)
+  )
+}>
+
+// CHECK-LABEL:   func.func @sparse_extract_iter_space(
+// CHECK-SAME:      %[[VAL_0:.*]]: tensor<4x8xf32, #sparse{{[0-9]*}}>,
+// CHECK-SAME:      %[[VAL_1:.*]]: !sparse_tensor.iterator<#sparse{{[0-9]*}}, lvls = 0>)
+// CHECK:           %[[VAL_2:.*]] = sparse_tensor.extract_iteration_space %[[VAL_0]] lvls = 0
+// CHECK:           %[[VAL_3:.*]] = sparse_tensor.extract_iteration_space %[[VAL_0]] at %[[VAL_1]] lvls = 1
+// CHECK:           return %[[VAL_2]], %[[VAL_3]] : !sparse_tensor.iter_space<#sparse{{[0-9]*}}, lvls = 0>, !sparse_tensor.iter_space<#sparse{{[0-9]*}}, lvls = 1>
+// CHECK:         }
+func.func @sparse_extract_iter_space(%sp : tensor<4x8xf32, #COO>, %it1 : !sparse_tensor.iterator<#COO, lvls = 0>)
+  -> (!sparse_tensor.iter_space<#COO, lvls = 0>, !sparse_tensor.iter_space<#COO, lvls = 1>) {
+  // Extracting the iteration space for the first level needs no parent iterator.
+  %l1 = sparse_tensor.extract_iteration_space %sp lvls = 0 : tensor<4x8xf32, #COO>
+  // Extracting the iteration space for the second level needs a parent iterator.
+  %l2 = sparse_tensor.extract_iteration_space %sp at %it1 lvls = 1 : tensor<4x8xf32, #COO>, !sparse_tensor.iterator<#COO, lvls = 0>
+  return %l1, %l2 : !sparse_tensor.iter_space<#COO, lvls = 0>, !sparse_tensor.iter_space<#COO, lvls = 1>
+}
-- 
GitLab


From edb711d2f318b17489692b5f85028fab7ed85b83 Mon Sep 17 00:00:00 2001
From: Noah Goldstein <goldstein.w.n@gmail.com>
Date: Tue, 16 Apr 2024 13:35:23 -0500
Subject: [PATCH 450/464] [InstCombine] Update `vector_reduce_and` tests to
 actually use `llvm.vector.reduce.and`; NFC

---
 llvm/test/Transforms/InstCombine/known-bits.ll | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/llvm/test/Transforms/InstCombine/known-bits.ll b/llvm/test/Transforms/InstCombine/known-bits.ll
index 85a21332b078..8b4249b2c25a 100644
--- a/llvm/test/Transforms/InstCombine/known-bits.ll
+++ b/llvm/test/Transforms/InstCombine/known-bits.ll
@@ -1223,7 +1223,7 @@ define i8 @known_reduce_and(<2 x i8> %xx) {
 ; CHECK-NEXT:    ret i8 1
 ;
   %x = or <2 x i8> %xx, <i8 5, i8 3>
-  %v = call i8 @llvm.vector.reduce.or(<2 x i8> %x)
+  %v = call i8 @llvm.vector.reduce.and(<2 x i8> %x)
   %r = and i8 %v, 1
   ret i8 %r
 }
@@ -1231,12 +1231,12 @@ define i8 @known_reduce_and(<2 x i8> %xx) {
 define i8 @known_reduce_and_fail(<2 x i8> %xx) {
 ; CHECK-LABEL: @known_reduce_and_fail(
 ; CHECK-NEXT:    [[X:%.*]] = or <2 x i8> [[XX:%.*]], <i8 5, i8 3>
-; CHECK-NEXT:    [[V:%.*]] = call i8 @llvm.vector.reduce.or.v2i8(<2 x i8> [[X]])
+; CHECK-NEXT:    [[V:%.*]] = call i8 @llvm.vector.reduce.and.v2i8(<2 x i8> [[X]])
 ; CHECK-NEXT:    [[R:%.*]] = and i8 [[V]], 2
 ; CHECK-NEXT:    ret i8 [[R]]
 ;
   %x = or <2 x i8> %xx, <i8 5, i8 3>
-  %v = call i8 @llvm.vector.reduce.or(<2 x i8> %x)
+  %v = call i8 @llvm.vector.reduce.and(<2 x i8> %x)
   %r = and i8 %v, 2
   ret i8 %r
 }
-- 
GitLab


From 5c6af605b307213453a9a043532b9293db21b5c6 Mon Sep 17 00:00:00 2001
From: mahtohappy <Happy.Kumar@Windriver.com>
Date: Wed, 17 Apr 2024 00:12:14 +0530
Subject: [PATCH 451/464] [Clang][Sema] placement new initializes typedef array
 with correct size (#88902)

Build Failure Fix
Fixes build failures due to #83124
---
 .../{instantiate-new-placement-size.cpp => PR41441.cpp}       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
 rename clang/test/SemaCXX/{instantiate-new-placement-size.cpp => PR41441.cpp} (75%)

diff --git a/clang/test/SemaCXX/instantiate-new-placement-size.cpp b/clang/test/SemaCXX/PR41441.cpp
similarity index 75%
rename from clang/test/SemaCXX/instantiate-new-placement-size.cpp
rename to clang/test/SemaCXX/PR41441.cpp
index 7a29d3dee849..0b012b33fce3 100644
--- a/clang/test/SemaCXX/instantiate-new-placement-size.cpp
+++ b/clang/test/SemaCXX/PR41441.cpp
@@ -1,5 +1,5 @@
-// RUN: %clang -S -fno-discard-value-names -emit-llvm -o - %s | FileCheck %s
-// Issue no: 41441
+// RUN: %clang --target=x86_64-pc-linux -S -fno-discard-value-names -emit-llvm -o - %s | FileCheck %s
+
 #include <new>
 
 // CHECK: call void @llvm.memset.p0.i64(ptr align 1 %x, i8 0, i64 8, i1 false)
-- 
GitLab


From b01879ec1ffbd249f9bf3c4f32308443be6ac36b Mon Sep 17 00:00:00 2001
From: Chao Chen <116223022+chencha3@users.noreply.github.com>
Date: Tue, 16 Apr 2024 13:44:14 -0500
Subject: [PATCH 452/464] [MLIR][XeGPU] Add XeGPU scattered ops (#86594)

- Extended TensorDescAttr with scattered attribute
- Add scattered ops: CreateDescOp, PrefetchOp, LoadGatherOp,
StoreScatterOp, UpdateOffsetOp
- Add a block op: UpdateNdOffsetOp

---------

Co-authored-by: Mehdi Amini <joker.eph@gmail.com>
Co-authored-by: Adam Siemieniuk <adam.siemieniuk@intel.com>
---
 mlir/include/mlir/Dialect/XeGPU/IR/XeGPU.h    |   1 +
 .../mlir/Dialect/XeGPU/IR/XeGPUAttrs.td       |  42 +-
 .../include/mlir/Dialect/XeGPU/IR/XeGPUOps.td | 464 +++++++++++++++---
 .../mlir/Dialect/XeGPU/IR/XeGPUTypes.td       |  41 +-
 mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp    |  21 +
 mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp        | 254 +++++++++-
 mlir/test/Dialect/XeGPU/XeGPUOps.mlir         |  62 +++
 mlir/test/Dialect/XeGPU/invalid.mlir          | 159 ++++++
 8 files changed, 937 insertions(+), 107 deletions(-)
 create mode 100644 mlir/test/Dialect/XeGPU/invalid.mlir

diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPU.h b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPU.h
index 87aabdc015fe..eca9255ff397 100644
--- a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPU.h
+++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPU.h
@@ -12,6 +12,7 @@
 #include "mlir/Bytecode/BytecodeOpInterface.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/Dialect.h"
+#include "mlir/IR/TypeUtilities.h"
 #include "mlir/Interfaces/ShapedOpInterfaces.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "mlir/Interfaces/ViewLikeInterface.h"
diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td
index cd38549f1ccf..6579d07ec262 100644
--- a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td
+++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUAttrs.td
@@ -19,17 +19,36 @@ class XeGPUAttr<string name, string attrMnemonic, list<Trait> traits = [],
 }
 
 def XeGPU_TensorDescAttr: XeGPUAttr<"TensorDesc", "tdesc_attr"> {
+  let summary = [{a composite attribute for `TensorDescType`}];
+  let description = [{`TensorDescAttr` (or `tdesc_attr`) is a composite
+    attribute defined for `TensorDescType` for describing following
+    properties of a `TensorDesc`.
+    1. `memory_scope`: It describes where the data block described by the
+        TensorDesc is located, `Global` device memory or `Shared` local memory.
+        It is default to `Global`.
+    2. `array_length`: It describes how many horizontally consecutive blocks
+        will be loaded by a hardware load instruction. If the TensorDesc shape
+        is 8x16, with array_length = 2. The loaded block shape will be acctually
+        8x32. Its default value is 1.
+    3. `boundary_check`: It is used to indicates the hardware whether to do
+        out-of-boundary check. The default value is true.
+    4. `scattered`: It is used to differenciate TensorDescs created from
+       `create_nd_tdesc` vs from `create_tdesc`.
+  }];
+
   let parameters = (ins
     OptionalParameter<"MemoryScopeAttr">: $memory_scope,
     OptionalParameter<"IntegerAttr", "1">: $array_length,
-    OptionalParameter<"BoolAttr", "true">: $boundary_check
+    OptionalParameter<"BoolAttr", "true">: $boundary_check,
+    OptionalParameter<"BoolAttr", "false">: $scattered
   );
 
   let builders = [
     AttrBuilder<(ins
       CArg<"xegpu::MemoryScope", "xegpu::MemoryScope::Global">:$memory_scope,
       CArg<"int", "1">:$array_length,
-      CArg<"bool", "true">: $boundary_check
+      CArg<"bool", "true">: $boundary_check,
+      CArg<"bool", "false">: $scattered
     )>
   ];
 
@@ -41,15 +60,17 @@ def XeGPU_TensorDescAttr: XeGPUAttr<"TensorDesc", "tdesc_attr"> {
 //===----------------------------------------------------------------------===//
 def XeGPU_MemoryScopeGlobal: I32EnumAttrCase<"Global", 0, "global">;
 def XeGPU_MemoryScopeShared: I32EnumAttrCase<"SLM", 1, "slm">;
-def XeGPU_MemoryScope: I32EnumAttr<"MemoryScope", 
-      "The address space of the memory the tensor descritor is created for", 
+def XeGPU_MemoryScope: I32EnumAttr<"MemoryScope",
+      "The address space of the memory the tensor descritor is created for",
       [XeGPU_MemoryScopeGlobal, XeGPU_MemoryScopeShared]> {
   let genSpecializedAttr = 0;
   let cppNamespace = "::mlir::xegpu";
 }
 
-def XeGPU_MemoryScopeAttr: 
+def XeGPU_MemoryScopeAttr:
   EnumAttr<XeGPU_Dialect, XeGPU_MemoryScope, "memory_scope"> {
+    let summary = [{Describe the location of data described by a `TensorDesc`:
+                 Global device memory (`Global`) or Shared local memory (`SLM`).}];
     let assemblyFormat = "$value";
 }
 
@@ -63,19 +84,18 @@ def XeGPU_CachePolicyInvalid:       I32EnumAttrCase<"READ_INVALIDATE", 3, "read_
 def XeGPU_CachePolicyWriteBack:     I32EnumAttrCase<"WRITE_BACK", 4, "write_back">;            // valid for write only
 def XeGPU_CachePolicyWriteThrough:  I32EnumAttrCase<"WRITE_THROUGH", 5, "write_through">;      // valid for write only
 
-def XeGPU_CachePolicyEnums : I32EnumAttr<"CachePolicy", "Cache policy", 
-  [XeGPU_CachePolicyCached, XeGPU_CachePolicyUncached, 
+def XeGPU_CachePolicyEnums : I32EnumAttr<"CachePolicy", "Cache policy",
+  [XeGPU_CachePolicyCached, XeGPU_CachePolicyUncached,
    XeGPU_CachePolicyStreaming, XeGPU_CachePolicyInvalid,
    XeGPU_CachePolicyWriteBack, XeGPU_CachePolicyWriteThrough]> {
   let genSpecializedAttr = 0;
   let cppNamespace = "::mlir::xegpu";
 }
 
-def XeGPU_CacheHintAttr 
+def XeGPU_CacheHintAttr
   : EnumAttr<XeGPU_Dialect, XeGPU_CachePolicyEnums, "cache_hint"> {
+    let summary = [{Describe the cache settings for prefetch/load/store operators}];
     let assemblyFormat = "`<` $value `>`";
 }
 
-
-
-#endif // MLIR_DIALECT_XEGPU_IR_XEGPUATTRS_TD
+#endif // MLIR_DIALECT_XEGPU_IR_XEGPUATTRS_TD
\ No newline at end of file
diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td
index b8ebd1a40c60..c6f7f83441b9 100644
--- a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td
+++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUOps.td
@@ -47,36 +47,35 @@ class XeGPU_Op<string mnemonic, list<Trait> traits = []>:
 }
 
 
-def XeGPU_CreateNdDescOp: XeGPU_Op<"create_nd_tdesc", [Pure, ViewLikeOpInterface, 
+def XeGPU_CreateNdDescOp: XeGPU_Op<"create_nd_tdesc", [Pure, ViewLikeOpInterface,
                         AttrSizedOperandSegments, OffsetSizeAndStrideOpInterface]> {
 
   let summary = "Create nd-tensor descriptor operation";
   let description = [{
     The "create_nd_tdesc" operation creates a TensorDescType which represents
     a sub-view of a 2D memory region (It can be extended to support n-D memory
-    region if needed in future). Elements in the subview continuous in each 
-    dimention. It encodes the following important information for supporting 
+    region if needed in future). Elements in the subview continuous in each
+    dimension. It encodes the following important information for supporting
     Intel hardware features:
 
-    * source: an object representing (starting address/pointer of) a 2D memory region. 
+    * source: an object representing (starting address/pointer of) a 2D memory region.
         It can be either a 2D memref object, or simply a pointer represented by uint64_t type.
-        for the later case, the shape and layout information of the 2D memory region should 
-        be explicitly passed via `dynamic_shape` and `dynamic_strides` parameters.
-    * offsets: two index values represents offsets from the "source" at the each dimension 
+        for the later case, the shape and layout information of the 2D memory region should
+        be explicitly passed via `shape` and `strides` parameters.
+    * offsets: two index values represents offsets from the "source" at the each dimension
         at which the subview of the target memory will be created. It is encoded via two
-        variables, including "dynamic_offsets" and "static_offsets", such that it can
-        accept various forms, such as, operands (e.g., [%c0, %c]) and attributes (e.g., [2, 4])).
-    * shape: the shape information of the memory region pointed by the "source".  It is 
-        typically encoded via the MemRefType of the source, e.g., memref<4096x4096xf16>. 
-        But if "source" is simply a pointer represented as uint64_t type, or a memref 
-        type without shape information e.g., memref<?x?xf16>, the shape information has 
-        to be explicitly passed via the "dynamic_shape" argument. Currently "dynamic_shape" 
-        only accepts operands(e.g., [%c4096, %c4096]), not attributes(e.g., [4096, 4096]).
-    * strides: the strides of the memory region pointed by the "source". Similar to shape, 
-        it is typically encoded via the MemRefType of the source too. But if "source" is 
-        simply a pointer represented as uint64_t type, or a memref type without shape 
-        information e.g., memref<?x?xf16>, the strides information has to be explicitly 
-        passed via the "dynamic_strides" argument. And it currently only accepts operands two.
+        variables, including "offsets" and "const_offsets", such that it can
+        accept various forms, such as, operands (e.g., [%c0, %c]) and attributes (e.g., [2, 4]).
+    * shape: the shape information of the memory region pointed by the "source".  It is
+        typically encoded via the MemRefType of the source, e.g., memref<4096x4096xf16>.
+        But if "source" is simply a pointer represented as uint64_t type, or a memref
+        type without shape information e.g., memref<?x?xf16>, the shape information has
+        to be explicitly passed via the "shape" and "const_shape" arguments.
+    * strides: the strides of the memory region pointed by the "source". Similar to shape,
+        it is typically encoded via the MemRefType of the source too. But if "source" is
+        simply a pointer represented as uint64_t type, or a memref type without shape
+        information e.g., memref<?x?xf16>, the strides information has to be explicitly
+        passed via the "strides" and "const_strides" argument.
 
     Example 1 (suppose the tensor shape inferred by the compiler is 8x16):
     %0 = memref.alloc() : memref<1024x1024xf32>
@@ -97,10 +96,10 @@ def XeGPU_CreateNdDescOp: XeGPU_Op<"create_nd_tdesc", [Pure, ViewLikeOpInterface
     %1 = xegpu.create_nd_tdesc %0[%c0, %c0], [%h, %w], [%w, %c1]: ui64 -> TensorDesc<8x16xf32>
   }];
 
-  let arguments = (ins 
-    XeGPU_BaseAddrType: $source, 
-    Variadic<Index>: $offsets, 
-    Variadic<Index>: $shape, 
+  let arguments = (ins
+    XeGPU_BaseAddrType: $source,
+    Variadic<Index>: $offsets,
+    Variadic<Index>: $shape,
     Variadic<Index>: $strides,
     DenseI64ArrayAttr: $const_offsets,
     OptionalAttr<DenseI64ArrayAttr>: $const_shape,
@@ -119,12 +118,12 @@ def XeGPU_CreateNdDescOp: XeGPU_Op<"create_nd_tdesc", [Pure, ViewLikeOpInterface
   let hasVerifier = 1;
 
   let builders = [
-    OpBuilder<(ins "Type": $tdesc, "TypedValue<MemRefType>": $source, 
+    OpBuilder<(ins "Type": $tdesc, "TypedValue<MemRefType>": $source,
                    "llvm::ArrayRef<OpFoldResult>": $offsets)>,
 
-    OpBuilder<(ins "Type": $tdesc, "TypedValue<IntegerType> ": $source, 
+    OpBuilder<(ins "Type": $tdesc, "TypedValue<IntegerType> ": $source,
                    "llvm::ArrayRef<OpFoldResult>": $offsets,
-                   "llvm::ArrayRef<OpFoldResult>": $shape, 
+                   "llvm::ArrayRef<OpFoldResult>": $shape,
                    "llvm::ArrayRef<OpFoldResult>": $strides)>
   ];
 
@@ -159,41 +158,41 @@ def XeGPU_CreateNdDescOp: XeGPU_Op<"create_nd_tdesc", [Pure, ViewLikeOpInterface
     }
 
     /// wrapper for matching with OffsetSizeAndStrideOpInterface
-    /// If source is IntegerType or `const_shape` is filled, 
+    /// If source is IntegerType or `const_shape` is filled,
     /// it will return `const_shape`, such that mixes of `shape`
-    /// and `const_shape` will be used to represent the shape of 
+    /// and `const_shape` will be used to represent the shape of
     /// source operand. They overide static shape from source memref type.
     ArrayRef<int64_t> getStaticSizes() {
       auto attr = getConstShapeAttr();
       if (getSourceType().isa<IntegerType>() || attr)
         return attr;
-      
+
       auto memrefType = getSourceType().dyn_cast<MemRefType>();
       assert(memrefType && "Incorrect use of getStaticSizes");
       return memrefType.getShape();
     }
 
     /// wrapper for matching with OffsetSizeAndStrideOpInterface
-    /// If source is IntegerType or `const_strides` is filled, it 
+    /// If source is IntegerType or `const_strides` is filled, it
     /// will return `const_strides`, such that mixes of `strides`
-    /// and `const_strides` will be used to represent the strides of 
+    /// and `const_strides` will be used to represent the strides of
     /// source operand. They overide static strides from source memref type.
     ArrayRef<int64_t> getStaticStrides() {
       auto attr = getConstStridesAttr();
       if (getSourceType().isa<IntegerType>() || attr)
         return attr;
-      
+
       auto memrefType = getSourceType().dyn_cast<MemRefType>();
       assert(memrefType && "Incorrect use of getStaticStrides");
       auto [strides, offset] = getStridesAndOffset(memrefType);
-      // reuse the storage of ConstStridesAttr since strides from 
+      // reuse the storage of ConstStridesAttr since strides from
       // memref is not persistant
       setConstStrides(strides);
       attr = getConstStridesAttr();
       return attr;
     }
 
-    /// Return the expected rank of each of the`static_offsets`, 
+    /// Return the expected rank of each of the`static_offsets`,
     /// `static_shape` and `static_strides` attributes.
     std::array<unsigned, 3> getArrayAttrMaxRanks() {
       unsigned rank;
@@ -204,8 +203,8 @@ def XeGPU_CreateNdDescOp: XeGPU_Op<"create_nd_tdesc", [Pure, ViewLikeOpInterface
       }
       return {rank, rank, rank};
     }
-    
-    /// Return the number of leading operands before the `offsets`, 
+
+    /// Return the number of leading operands before the `offsets`,
     /// `shape` and `strides` operands.
     static unsigned getOffsetSizeAndStrideStartOperandIndex() { return 1; }
 
@@ -214,15 +213,15 @@ def XeGPU_CreateNdDescOp: XeGPU_Op<"create_nd_tdesc", [Pure, ViewLikeOpInterface
 }
 
 def XeGPU_PrefetchNdOp : XeGPU_Op<"prefetch_nd", []> {
-  let summary = "prefetches a nD block to cache";
+  let summary = "prefetches a n-D block to cache";
   let description = [{
-    It issues an instruction to prefetch the data from memory to each 
-    level of the cache based on their cache policy.
+    It issues an instruction to prefetch a block of data from continuous
+    memory regions to each level of the cache based on their cache policy.
 
     Example:
     ```
-      xegpu.prefetch_nd %tdesc {l1_hint = #xegpu.cache_hint<cached>, 
-                                l2_hint = #xegpu.cache_hint<cached>, 
+      xegpu.prefetch_nd %tdesc {l1_hint = #xegpu.cache_hint<cached>,
+                                l2_hint = #xegpu.cache_hint<cached>,
                                 l3_hint = #xegpu.cache_hint<cached>}
         : !xegpu.tensor_desc<8x16xf16>
     ```
@@ -233,34 +232,41 @@ def XeGPU_PrefetchNdOp : XeGPU_Op<"prefetch_nd", []> {
                        OptionalAttr<XeGPU_CacheHintAttr>: $l1_hint,
                        OptionalAttr<XeGPU_CacheHintAttr>: $l2_hint,
                        OptionalAttr<XeGPU_CacheHintAttr>: $l3_hint);
-                       
-  let extraClassDeclaration = extraBaseClassDeclaration;
+
+  let extraClassDeclaration = extraBaseClassDeclaration # [{
+    xegpu::TensorDescType getTensorDescType() {
+      return getTensorDesc().getType();
+    }
+  }];
 
   let assemblyFormat = "$TensorDesc prop-dict attr-dict `:` qualified(type($TensorDesc))";
+
+  let hasVerifier = 1;
 }
 
 
-def XeGPU_LoadNdOp : XeGPU_Op<"load_nd"> {
-  let summary = "loads a n-D block from memory (represented by TensorDesc)" 
+def XeGPU_LoadNdOp : XeGPU_Op<"load_nd", [AllElementTypesMatch<["value", "TensorDesc"]>,
+                                         AllElementCountsMatch<["value", "TensorDesc"]>]> {
+  let summary = "loads a n-D block from memory (represented by TensorDesc)"
                 "to registers (represented by vector)";
   let description = [{
-    LoadNdOp essentially mimics the hardware block read instruction to read 
-    a block of data from memory to register. It takes a set of optional cache 
-    hints for each level of cache, L1, L2 and L3. If hardware does not have a 
+    LoadNdOp essentially mimics the hardware block read instruction to read
+    a block of data from memory to register. It takes a set of optional cache
+    hints for each level of cache, L1, L2 and L3. If hardware does not have a
     correspoding cache, Corresponding cache hint attribute will be masked.
-    vnni transform is an hardware feature for Intel GPU, which is used to 
-    do data packing during the load for B operand of matrix operation, if 
-    the bit width of the data type is less then 32 bits, e.g., fp16. And 
+    vnni transform is an hardware feature for Intel GPU, which is used to
+    do data packing during the load for B operand of matrix operation, if
+    the bit width of the data type is less then 32 bits, e.g., fp16. And
     transpose is another Intel hardware feature, which will do transpose
-    operation when loading the data if the bit width of the data type is 
-    fp32 or fp64. It implies that vnni and transpose cannot exit at the 
+    operation when loading the data if the bit width of the data type is
+    fp32 or fp64. It implies that vnni and transpose cannot exit at the
     same time.
 
     Example:
     ```
       xegpu.load_nd %1 {transpose = [1, 0],
-                        l1_hint = #xegpu.cache_hint<cached>, 
-                        l2_hint = #xegpu.cache_hint<uncached>, 
+                        l1_hint = #xegpu.cache_hint<cached>,
+                        l2_hint = #xegpu.cache_hint<uncached>,
                         l3_hint = #xegpu.cache_hint<streaming>}
               : !xegpu.tensor_desc<8x16xf32> -> vector<16x8xf32>
     ```
@@ -291,20 +297,21 @@ def XeGPU_LoadNdOp : XeGPU_Op<"load_nd"> {
   let hasVerifier = 1;
 }
 
-def XeGPU_StoreNdOp : XeGPU_Op<"store_nd", []> {
+def XeGPU_StoreNdOp : XeGPU_Op<"store_nd", [AllShapesMatch<["value", "TensorDesc"]>,
+                                       AllElementTypesMatch<["value", "TensorDesc"]>]> {
   let summary = "stores a n-D block register region back to memory, currently only supports 2D";
 
   let description = [{
     StoreNdOp essentially mimics the hardware block write instruction io
-    write a block of data from register into the memory region as described 
-    by the TensorDesc. It takes a set of optional cache hints for each level 
-    of cache, L1, L2 and L3. If hardware does not have a correspoding cache, 
+    write a block of data from register into the memory region as described
+    by the TensorDesc. It takes a set of optional cache hints for each level
+    of cache, L1, L2 and L3. If hardware does not have a correspoding cache,
     Corresponding cache hint attribute will be masked.
 
     Example:
     ```
       xegpu.store_nd %3, %2 {l1_hint = #xegpu.cache_hint<uncached>,
-                             l2_hint = #xegpu.cache_hint<write_back>, 
+                             l2_hint = #xegpu.cache_hint<write_back>,
                              l3_hint = #xegpu.cache_hint<write_through>}
                              : vector<8x16xf16>, !xegpu.tensor_desc<8x16xf16>
     ```
@@ -318,11 +325,342 @@ def XeGPU_StoreNdOp : XeGPU_Op<"store_nd", []> {
                        OptionalAttr<XeGPU_CacheHintAttr>: $l2_hint,
                        OptionalAttr<XeGPU_CacheHintAttr>: $l3_hint);
 
-  let extraClassDeclaration = extraBaseClassDeclaration;
+  let extraClassDeclaration = extraBaseClassDeclaration # [{
+    VectorType getValueType() {
+      return llvm::dyn_cast<VectorType>(getValue().getType());
+    }
 
-  let assemblyFormat = [{$value `,` $TensorDesc prop-dict attr-dict 
+    xegpu::TensorDescType getTensorDescType() {
+      return getTensorDesc().getType();
+    }
+  }];
+
+  let assemblyFormat = [{$value `,` $TensorDesc prop-dict attr-dict
                         `:` type($value) `,` qualified(type($TensorDesc))}];
   let hasVerifier = 1;
 }
 
+def XeGPU_UpdateNdOffsetOp : XeGPU_Op<"update_nd_offset",
+                [AllTypesMatch<["TensorDesc", "result"]>]> {
+  let summary = "It updates the offsets for the TensorDesc.";
+  let description = [{The op updates the offset of the given TensorDesc.
+    The offsets are relative offset to the current position in the number
+    of elements. It will result in a same type TensorDesc as the input.
+
+  example:
+  ```
+    %2 = xegpu.update_nd_offset %1, [0, 16]: !xegpu.tensor_desc<8x16xf32>
+  ```
+  }];
+
+  let arguments = (ins
+    XeGPU_TensorDesc: $TensorDesc,
+    Variadic<Index>: $offsets,
+    DenseI64ArrayAttr: $const_offsets);
+
+  let results = (outs XeGPU_TensorDesc: $result);
+
+  let extraClassDeclaration = extraBaseClassDeclaration # [{
+    xegpu::TensorDescType getTensorDescType() {
+      return getTensorDesc().getType();
+    }
+
+    SmallVector<OpFoldResult> getMixedOffsets() {
+      Builder b(getContext());
+      return getMixedValues(getConstOffsets(), getOffsets(), b);
+    }
+
+    size_t getNumOffsets() {
+      return getMixedOffsets().size();
+    }
+
+    OpFoldResult getOffset(unsigned idx) {
+      assert(idx < getNumOffsets() && "Invalid out of bound access.");
+      return getMixedOffsets()[idx];
+    }
+  }];
+
+  let assemblyFormat = [{
+    $TensorDesc `,`
+    custom<DynamicIndexList>($offsets, $const_offsets)
+    attr-dict `:` qualified(type($result))
+  }];
+
+  let hasVerifier = 1;
+}
+
+def XeGPU_CreateDescOp: XeGPU_Op<"create_tdesc", [Pure, ViewLikeOpInterface]> {
+  let summary = "create scattered tensor descriptors (TensorDesc).";
+  let description = [{
+    "create_tdesc" is similar to "create_nd_tdesc" in terms that it creates
+    a Tensor Descriptor (TensorDescType) for a memory region. While "create_nd_tdesc"
+    is for creating continuous subviews, "create_tdesc" is for creating non-continuous
+    (scattered) subviews, allowing each work-item in a subgroup specifying their own offset.
+    It accepts the following parameters:
+
+    * source: a 1D memref or pointer (uint64_t) represents the flattened memory object.
+    * offsets: a array containing offsets of each access point. Its size
+      is fixed to the hardware supportted subgroup size, e.g., 16 on PVC,
+      implying each element in the array corresponds to a work-item (SIMT lane)
+      in the subgroup.
+    * chunk_size: [optional attribute] indicates number of continious
+      elements accessed for each offset, default is 1.
+
+    Example 1. It assumes subgroup size is 4, and accesses a[0], a[16], a[32], a[64]
+    ```
+    %a = memref.alloc() : memref<1024xf32>
+    %1 = xegpu.create_tdesc %a[0, 16, 32, 64]: memref<1024xf32> -> TensorDesc<4xf32>
+    ```
+
+    Example 2. It assumes subgroup size is 4, and each workitem access 8 elements.
+               It will access totally 32 data elements: a[0:7], a[16:23], a[32:39], a[64:71]
+    ```
+    %0 = memref.alloc() : memref<1024xf32>
+    %1 = xegpu.create_tdesc %0[0, 16, 32, 64] {chunk_size = 8}: memref<1024xf32> -> TensorDesc<4x8xf32>
+    ```
+
+    Example 3. It is similar to Example 2, but there is some overlaps among workitems.
+               It accesses: a[0:7], a[4:11], a[8:15], a[12:19]
+    ```
+    %0 = memref.alloc() : memref<1024xf32>
+    %1 = xegpu.create_tdesc %0[0, 4, 8, 12] {chunk_size = 8}: memref<1024xf32> -> TensorDesc<4x8xf32>
+    ```
+
+
+
+
+  }];
+
+  let arguments = (ins XeGPU_BaseAddrType: $source,
+                       Variadic<Index>: $offsets,
+                       DenseI64ArrayAttr: $const_offsets,
+                       DefaultValuedAttr<I64Attr, "1">: $chunk_size);
+  let results = (outs XeGPU_TensorDesc:$TensorDesc);
+
+  let builders = [
+    OpBuilder<(ins "xegpu::TensorDescType": $TensorDesc, "Value": $source,
+                   "llvm::ArrayRef<OpFoldResult>": $offsets,
+                   CArg<"uint32_t", "1"> : $chunk_size)>,
+  ];
+
+  let assemblyFormat = [{
+    $source
+    custom<DynamicIndexList>($offsets, $const_offsets)
+    attr-dict `:`  type($source) `->` qualified(type($TensorDesc))
+  }];
+
+  let extraClassDeclaration = extraBaseClassDeclaration # [{
+    xegpu::TensorDescType getTensorDescType() {
+      return getTensorDesc().getType();
+    }
+
+    SmallVector<OpFoldResult> getMixedOffsets() {
+      Builder b(getContext());
+      return getMixedValues(getConstOffsets(), getOffsets(), b);
+    }
+
+    size_t getNumOffsets() {
+      return getMixedOffsets().size();
+    }
+
+    mlir::Value getViewSource() { return getSource(); }
+
+    OpFoldResult getOffset(unsigned idx) {
+      assert(idx < getNumOffsets() && "Invalid out of bound access.");
+      return getMixedOffsets()[idx];
+    }
+  }];
+
+  let hasVerifier = 1;
+}
+
+def XeGPU_PrefetchOp : XeGPU_Op<"prefetch", []> {
+  let summary = "prefetches a set of scattered data points to cache";
+
+  let description = [{
+    It issues instructions to prefetch a set of scattered data points
+    from memory to each level of the cache based on their cache policy.
+    As compared to prefetch_nd, which works on non-scattered TensorDesc,
+    it works on scattered TensorDesc instead.
+
+    Example:
+    ```
+      xegpu.prefetch %tdesc {l1_hint = #xegpu.cache_hint<cached>,
+                             l2_hint = #xegpu.cache_hint<cached>,
+                             l3_hint = #xegpu.cache_hint<cached>}
+        : !xegpu.tensor_desc<16xf16>
+    ```
+
+  }];
+
+  let arguments = (ins XeGPU_TensorDesc: $TensorDesc,
+                       OptionalAttr<XeGPU_CacheHintAttr>: $l1_hint,
+                       OptionalAttr<XeGPU_CacheHintAttr>: $l2_hint,
+                       OptionalAttr<XeGPU_CacheHintAttr>: $l3_hint);
+
+  let extraClassDeclaration = extraBaseClassDeclaration # [{
+    xegpu::TensorDescType getTensorDescType() {
+      return getTensorDesc().getType();
+    }
+  }];
+
+  let assemblyFormat = "$TensorDesc prop-dict attr-dict `:` qualified(type($TensorDesc))";
+
+  let hasVerifier = 1;
+}
+
+def XeGPU_LoadGatherOp : XeGPU_Op<"load", [AllRanksMatch<["value", "TensorDesc"]>,
+                                    AllElementTypesMatch<["value", "TensorDesc"]>,
+                                   AllElementCountsMatch<["value", "TensorDesc"]>]> {
+  let summary = "load a set of scattered data points from memory.";
+
+  let description = [{ It (aka. load) load data per each work-item. The output
+    describes the data being loaded at the subgroup level, so its size is
+    consistent with the number of work-items in a subgroup. When `chunk_size_per_lane`
+    attribute is larger than 1 in TensorDesc, the output vector will be 2D vector,
+    with dim-1 correspoding to the chunk size.
+
+    The mask operand masks out memory access so that it is safe to pass out-of-boundary
+    addresses/offsets as long as they are masked. It applies to slots of SIMD lanes.
+
+  Example:
+  ```
+    %2 = xegpu.load %1, %0 {transpose = [1, 0],
+                            l1_hint = #xegpu.cache_hint<cached>,
+                            l2_hint = #xegpu.cache_hint<uncached>,
+                            l3_hint = #xegpu.cache_hint<uncached>}
+          : !xegpu.tensor_desc<16xf32, #xegpu.tdesc_attr<scattered=true>>, vector<16xi1>
+            -> vector<16xf32>
+  ```
+
+  }];
+
+  let arguments = (ins XeGPU_TensorDesc: $TensorDesc,
+                       XeGPU_MaskType: $mask,
+                       OptionalAttr<DenseI64ArrayAttr>: $transpose,
+                       OptionalAttr<XeGPU_CacheHintAttr>: $l1_hint,
+                       OptionalAttr<XeGPU_CacheHintAttr>: $l2_hint,
+                       OptionalAttr<XeGPU_CacheHintAttr>: $l3_hint);
+  let results = (outs XeGPU_ValueType: $value);
+
+  let extraClassDeclaration = extraBaseClassDeclaration # [{
+    xegpu::TensorDescType getTensorDescType() {
+      return getTensorDesc().getType();
+    }
+
+    mlir::Type getElementType() {
+      auto type = getValue().getType();
+      return getElementTypeOrSelf(type);
+    }
+
+    Type getValueType() {
+      return getValue().getType();
+    }
+
+    Type getMaskType() {
+      return getMask().getType();
+    }
+
+  }];
+
+  let assemblyFormat = [{$TensorDesc `,` $mask prop-dict attr-dict
+      `:` qualified(type($TensorDesc)) `,` type($mask) `->` type($value)}];
+
+  let hasVerifier = 1;
+}
+
+def XeGPU_StoreScatterOp : XeGPU_Op<"store", [AllShapesMatch<["value", "TensorDesc"]>,
+                                        AllElementTypesMatch<["value", "TensorDesc"]>]> {
+  let summary = "store data to scattered memory locations.";
+  let description = [{ It (aka. store) stores data to scattered memory locations.
+  It has similar semantic to `load_gather`.
+
+  Example:
+  ```
+    %3 = xegpu.store %0, %1, %2 {l1_hint = #xegpu.cache_hint<uncached>,
+                                 l2_hint = #xegpu.cache_hint<write_back>,
+                                 l3_hint = #xegpu.cache_hint<write_through>}
+          : vector<16xf32>, !xegpu.tensor_desc<16xf32, #xegpu.tdesc_attr<scattered=true>>, vector<16xi1>
+  ```
+  }];
+
+  let arguments = (ins
+    XeGPU_ValueType: $value,
+    XeGPU_TensorDesc: $TensorDesc,
+    XeGPU_MaskType: $mask,
+    OptionalAttr<XeGPU_CacheHintAttr>: $l1_hint,
+    OptionalAttr<XeGPU_CacheHintAttr>: $l2_hint,
+    OptionalAttr<XeGPU_CacheHintAttr>: $l3_hint);
+
+  let extraClassDeclaration = extraBaseClassDeclaration # [{
+    xegpu::TensorDescType getTensorDescType() {
+      return getTensorDesc().getType();
+    }
+
+    Type getValueType() {
+      return getValue().getType();
+    }
+
+    Type getMaskType() {
+      return getMask().getType();
+    }
+  }];
+
+  let assemblyFormat = [{$value `,` $TensorDesc `,` $mask prop-dict attr-dict
+            `:` type($value) `,` qualified(type($TensorDesc)) `,` type($mask)}];
+
+  let hasVerifier = 1;
+}
+
+def XeGPU_UpdateOffsetOp: XeGPU_Op<"update_offset",
+          [AllTypesMatch<["TensorDesc", "result"]>]> {
+  let summary = "It updates the offsets for the given tensor descriptor";
+
+  let description = [{It behaves similar to `update_nd_offset` in terms that
+    it updates offset of a TensorDesc, and the offsets are relative offset to
+    the current position in the number of elements. However, `update_nd_offset`
+    is to update the start point of a 2D block, so its offset constains two
+    elements representing the shift in each dimension. `update_offset` is to
+    update the offset per work-item, so its offsets contains values representing
+    shifts for each work-item.
+
+    Example:
+    ```
+      %2 = xegpu.update_offset %1, [32, 32, 32, 32]
+            : !xegpu.tensor_desc<4x2xf32, #xegpu.tdesc_attr<scattered = true>>
+    ```
+  }];
+
+  let arguments = (ins XeGPU_TensorDesc: $TensorDesc,
+                       Variadic<Index>: $offsets,
+                       DenseI64ArrayAttr: $const_offsets);
+  let results = (outs XeGPU_TensorDesc: $result);
+
+  let extraClassDeclaration = extraBaseClassDeclaration # [{
+    xegpu::TensorDescType getTensorDescType() {
+      return getTensorDesc().getType();
+    }
+
+    SmallVector<OpFoldResult> getMixedOffsets() {
+      Builder b(getContext());
+      return getMixedValues(getConstOffsets(), getOffsets(), b);
+    }
+
+    size_t getNumOffsets() {
+      return getMixedOffsets().size();
+    }
+
+    OpFoldResult getOffset(unsigned idx) {
+      assert(idx < getNumOffsets() && "Invalid out of bound access.");
+      return getMixedOffsets()[idx];
+    }
+  }];
+
+  let assemblyFormat = [{
+    $TensorDesc `,`
+    custom<DynamicIndexList>($offsets, $const_offsets)
+    attr-dict `:` qualified(type($TensorDesc))
+  }];
+}
+
 #endif // MLIR_DIALECT_XEGPU_IR_XEGPUOPS_TD
diff --git a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTypes.td b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTypes.td
index 19ac1693712d..4cd4e5411653 100644
--- a/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTypes.td
+++ b/mlir/include/mlir/Dialect/XeGPU/IR/XeGPUTypes.td
@@ -34,10 +34,10 @@ def XeGPU_TensorDesc: XeGPUTypeDef<"TensorDesc", "tensor_desc",
         [ShapedTypeInterface], "::mlir::TensorType"> {
   let summary = "TensorDesc describing regions of interested data.";
   let description = [{
-    TensorDesc is a type designed to describe regions of the interested data as well as some 
-    features that are unique to Intel hardware. Different with the builtin tensor type in MLIR, 
-    it essentially only contains the meta data, and doesn't hold the data by itself. It is designed 
-    to mainly support 2D block load/store and DPAS (matrix multiplication instruction) on Intel GPU. 
+    TensorDesc is a type designed to describe regions of the interested data as well as some
+    features that are unique to Intel hardware. Different with the builtin tensor type in MLIR,
+    it essentially only contains the meta data, and doesn't hold the data by itself. It is designed
+    to mainly support 2D block load/store and DPAS (matrix multiplication instruction) on Intel GPU.
     It encodes the following information:
 
     * shape:  the sizes/shape of the intereted data block, e.g., 8x16 means 8 rows
@@ -46,15 +46,15 @@ def XeGPU_TensorDesc: XeGPUTypeDef<"TensorDesc", "tensor_desc",
               is set or not.
     * element_type: the data type of the data element, e.g., f16, f32.
 
-    Similar to the builtin tensor, it also provides an optinal attribute to encoding 
+    Similar to the builtin tensor, it also provides an optinal attribute to encoding
     the following information via the TensorDescAttr object:
-    * memory_scope (xegpu::MemoryScope): [optional] where the data is located, 
+    * memory_scope (xegpu::MemoryScope): [optional] where the data is located,
                 global memory or shared memory. It is default to Global.
     * array_length (int): [optional] The number of contiguous blocks with size as `shape`,
                that will be loaded by block load at a time. It is default to 1.
-    * boundary_check (bool): [optional] indicates whether the operation detects the boundary 
+    * boundary_check (bool): [optional] indicates whether the operation detects the boundary
                 and pads with zero for out-of-boundary access. It is default to do boundary check.
-    
+
 
     Syntax:
 
@@ -63,7 +63,7 @@ def XeGPU_TensorDesc: XeGPUTypeDef<"TensorDesc", "tensor_desc",
     element-type ::= float-type | integer-type | index-type
     dim-list := (static-dim-list `x`)?
     static-dim-list ::= decimal-literal `x` decimal-literal
-    attr-list = (, memory_scope = value)? (, arr_len = value)? (, boundary_check = value)?
+    attr-list = (, memory_scope = value)? (, arr_len = value)? (, boundary_check = value)? (, scattered = value)?
     ```
 
     Examples:
@@ -84,6 +84,17 @@ def XeGPU_TensorDesc: XeGPUTypeDef<"TensorDesc", "tensor_desc",
                         "mlir::Type": $elementType,
                         OptionalParameter<"mlir::Attribute">: $encoding);
 
+  let builders = [
+    TypeBuilderWithInferredContext<(ins
+      "llvm::ArrayRef<int64_t>": $shape,
+      "mlir::Type": $elementType,
+      CArg<"bool", "false">: $scattered,
+      CArg<"int", "1">: $array_length,
+      CArg<"xegpu::MemoryScope", "xegpu::MemoryScope::Global">:$memory_scope,
+      CArg<"bool", "true">: $boundary_check
+    )>
+  ];
+
   let extraClassDeclaration = [{
     using TensorType::clone;
     using mlir::ShapedType::Trait<TensorDescType>::getElementTypeBitWidth;
@@ -116,7 +127,7 @@ def XeGPU_TensorDesc: XeGPUTypeDef<"TensorDesc", "tensor_desc",
       if (attr && attr.getArrayLength())
         return attr.getArrayLength().getInt();
       // return default value
-      return 1; 
+      return 1;
     }
 
     bool getBoundaryCheck() {
@@ -126,10 +137,18 @@ def XeGPU_TensorDesc: XeGPUTypeDef<"TensorDesc", "tensor_desc",
       // return default value
       return true;
     }
+
+    bool getScattered() {
+      auto attr = getEncodingAsTensorDescAttr();
+      if (attr && attr.getScattered())
+        return attr.getScattered().getValue();
+      // return default value
+      return false;
+    }
   }];
 
   let hasCustomAssemblyFormat = true;
-  
+
 }
 
 #endif // MLIR_DIALECT_XEGPU_IR_XEGPUTYPES_TD
diff --git a/mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp b/mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp
index 0b3f4b9c9dbe..24719fe748fe 100644
--- a/mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp
+++ b/mlir/lib/Dialect/XeGPU/IR/XeGPUDialect.cpp
@@ -32,6 +32,17 @@ void XeGPUDialect::initialize() {
 //===----------------------------------------------------------------------===//
 // XeGPU_TensorDescAttr
 //===----------------------------------------------------------------------===//
+TensorDescAttr TensorDescAttr::get(mlir::MLIRContext *context,
+                                   xegpu::MemoryScope memory_scope,
+                                   int array_length, bool boundary_check,
+                                   bool scattered) {
+  auto scopeAttr = MemoryScopeAttr::get(context, memory_scope);
+  auto lengthAttr =
+      IntegerAttr::get(IntegerType::get(context, 64), array_length);
+  auto boundaryAttr = BoolAttr::get(context, boundary_check);
+  auto scatteredAttr = BoolAttr::get(context, scattered);
+  return Base::get(context, scopeAttr, lengthAttr, boundaryAttr, scatteredAttr);
+}
 
 //===----------------------------------------------------------------------===//
 // XeGPU_TensorDescType
@@ -96,6 +107,16 @@ void TensorDescType::print(::mlir::AsmPrinter &printer) const {
   printer << ">";
 }
 
+TensorDescType TensorDescType::get(llvm::ArrayRef<int64_t> shape,
+                                   mlir::Type elementType, bool scattered,
+                                   int array_length, MemoryScope memory_scope,
+                                   bool boundary_check) {
+  auto context = elementType.getContext();
+  auto attr = TensorDescAttr::get(context, memory_scope, array_length,
+                                  boundary_check, scattered);
+  return Base::get(context, shape, elementType, attr);
+}
+
 } // namespace xegpu
 } // namespace mlir
 
diff --git a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
index 02106f221f32..621986c54d49 100644
--- a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
+++ b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
@@ -9,6 +9,9 @@
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
 #include "mlir/Dialect/XeGPU/IR/XeGPU.h"
 #include "mlir/IR/Builders.h"
+#include "mlir/IR/TypeUtilities.h"
+
+#include "llvm/Support/Debug.h"
 
 #define DEBUG_TYPE "xegpu"
 
@@ -16,8 +19,8 @@ namespace mlir {
 namespace xegpu {
 
 static void transpose(llvm::ArrayRef<int64_t> trans,
-                      std::vector<int64_t> &shape) {
-  std::vector<int64_t> old = shape;
+                      SmallVector<int64_t> &shape) {
+  SmallVector<int64_t> old = shape;
   for (size_t i = 0; i < trans.size(); i++)
     shape[i] = old[trans[i]];
 }
@@ -38,6 +41,38 @@ static std::string makeString(T array, bool breakline = false) {
   return buf;
 }
 
+static SmallVector<int64_t> getShapeOf(Type type) {
+  SmallVector<int64_t> shape;
+  if (auto ty = llvm::dyn_cast<ShapedType>(type))
+    shape = SmallVector<int64_t>(ty.getShape());
+  else
+    shape.push_back(1);
+  return shape;
+}
+
+static int64_t getRankOf(Value val) {
+  auto type = val.getType();
+  if (auto ty = llvm::dyn_cast<ShapedType>(type))
+    return ty.getRank();
+  return 0;
+};
+
+static bool isReadHintOrNone(const CachePolicyAttr &attr) {
+  if (!attr)
+    return true;
+  auto kind = attr.getValue();
+  return kind == CachePolicy::CACHED || kind == CachePolicy::UNCACHED ||
+         kind == CachePolicy::STREAMING || kind == CachePolicy::READ_INVALIDATE;
+}
+
+static bool isWriteHintOrNone(const CachePolicyAttr &attr) {
+  if (!attr)
+    return true;
+  auto kind = attr.getValue();
+  return kind == CachePolicy::CACHED || kind == CachePolicy::UNCACHED ||
+         kind == CachePolicy::WRITE_BACK || kind == CachePolicy::WRITE_THROUGH;
+}
+
 //===----------------------------------------------------------------------===//
 // XeGPU_CreateNdDescOp
 //===----------------------------------------------------------------------===//
@@ -114,6 +149,29 @@ LogicalResult CreateNdDescOp::verify() {
     return emitOpError("TensorDesc should have the same element "
                        "type with the source if it is a memref.\n");
 
+  if (getType().getScattered())
+    return emitOpError("Expects a non-scattered TensorDesc.\n");
+
+  return success();
+}
+
+//===----------------------------------------------------------------------===//
+// XeGPU_PrefetchNdOp
+//===----------------------------------------------------------------------===//
+LogicalResult PrefetchNdOp::verify() {
+  auto tdescTy = getTensorDescType();
+  if (tdescTy.getScattered())
+    return emitOpError("Expects a non-scattered TensorDesc.\n");
+
+  if (!isReadHintOrNone(getL1HintAttr()))
+    return emitOpError("invlid l1_hint: ") << getL1HintAttr();
+
+  if (!isReadHintOrNone(getL2HintAttr()))
+    return emitOpError("invlid l2_hint: ") << getL2HintAttr();
+
+  if (!isReadHintOrNone(getL3HintAttr()))
+    return emitOpError("invlid l3_hint: ") << getL3HintAttr();
+
   return success();
 }
 
@@ -125,22 +183,26 @@ LogicalResult LoadNdOp::verify() {
   auto valueTy = getType();
 
   if (tdescTy.getRank() != 2)
-    return emitOpError(
-        "The TensorDesc for LoadNdOp should be a 2D TensorDesc.");
+    return emitOpError("Expecting a 2D TensorDesc.\n");
+
+  if (tdescTy.getScattered())
+    return emitOpError("Expects a non-scattered TensorDesc.\n");
 
   if (!valueTy)
     return emitOpError("Invalid result, it should be a VectorType.\n");
 
-  auto tdescElemTy = tdescTy.getElementType();
-  auto valueElemTy = valueTy.getElementType();
+  if (!isReadHintOrNone(getL1HintAttr()))
+    return emitOpError("invlid l1_hint: ") << getL1HintAttr();
 
-  if (tdescElemTy != valueElemTy)
-    return emitOpError(
-        "Value should have the same element type as TensorDesc.");
+  if (!isReadHintOrNone(getL2HintAttr()))
+    return emitOpError("invlid l2_hint: ") << getL2HintAttr();
+
+  if (!isReadHintOrNone(getL3HintAttr()))
+    return emitOpError("invlid l3_hint: ") << getL3HintAttr();
 
   auto array_len = tdescTy.getArrayLength();
-  auto tdescShape = tdescTy.getShape().vec();
-  auto valueShape = valueTy.getShape().vec();
+  auto tdescShape = getShapeOf(tdescTy);
+  auto valueShape = getShapeOf(valueTy);
 
   if (getTranspose()) {
     auto trans = getTranspose().value();
@@ -174,26 +236,174 @@ LogicalResult LoadNdOp::verify() {
 // XeGPU_StoreNdOp
 //===----------------------------------------------------------------------===//
 LogicalResult StoreNdOp::verify() {
-  auto dstTy = getTensorDesc().getType();               // Tile
-  auto valTy = getValue().getType().cast<VectorType>(); // Vector
+  auto dstTy = getTensorDescType(); // Tile
+  auto valTy = getValueType();      // Vector
 
   if (dstTy.getRank() != 2)
-    return emitOpError("Expecting a 2D TensorDesc shape.\n");
+    return emitOpError("Expecting a 2D TensorDesc.\n");
+
+  if (dstTy.getScattered())
+    return emitOpError("Expects a non-scattered TensorDesc.\n");
 
   if (!valTy)
     return emitOpError("Exepcting a VectorType result.\n");
 
-  auto dstElemTy = dstTy.getElementType();
-  auto valElemTy = valTy.getElementType();
+  if (!isWriteHintOrNone(getL1HintAttr()))
+    return emitOpError("invlid l1_hint: ") << getL1HintAttr();
+
+  if (!isWriteHintOrNone(getL2HintAttr()))
+    return emitOpError("invlid l2_hint: ") << getL2HintAttr();
+
+  if (!isWriteHintOrNone(getL3HintAttr()))
+    return emitOpError("invlid l3_hint: ") << getL3HintAttr();
+
+  return success();
+}
 
-  if (dstElemTy != valElemTy) {
-    return emitOpError() << "The element type of the value should "
-                            "match the elementtype of the TensorDesc.\n";
+//===----------------------------------------------------------------------===//
+// XeGPU_UpdateNDOffsetOp
+//===----------------------------------------------------------------------===//
+LogicalResult UpdateNdOffsetOp::verify() {
+  auto ty = getTensorDescType();
+  if (ty.getScattered())
+    return emitOpError("Expects a non-scattered TensorDesc.\n");
+
+  // number of offsets specified must match the rank of the tensor descriptor
+  if (ty.getRank() != (int64_t)getNumOffsets()) {
+    return emitOpError("Invalid number of offsets.");
   }
+  return success();
+}
+
+//===----------------------------------------------------------------------===//
+// XeGPU_CreateDescOp
+//===----------------------------------------------------------------------===//
+void CreateDescOp::build(OpBuilder &builder, OperationState &state,
+                         TensorDescType TensorDesc, Value source,
+                         llvm::ArrayRef<OpFoldResult> offsets,
+                         uint32_t chunk_size) {
+  llvm::SmallVector<int64_t> staticOffsets;
+  llvm::SmallVector<Value> dynamicOffsets;
+  dispatchIndexOpFoldResults(offsets, dynamicOffsets, staticOffsets);
+  build(builder, state, TensorDesc, source, dynamicOffsets, staticOffsets,
+        chunk_size);
+}
+
+LogicalResult CreateDescOp::verify() {
+  auto tdescTy = getTensorDescType();
+  auto chunkSize = getChunkSize();
+
+  if (getRankOf(getSource()) > 1)
+    return emitOpError(
+        "Expecting the source is a 1D memref or pointer (uint64_t).");
+
+  if (!tdescTy.getScattered())
+    return emitOpError("Expects a scattered TensorDesc.\n");
+
+  SmallVector<int64_t> shape({(int64_t)getNumOffsets()});
+  if (chunkSize != 1)
+    shape.push_back(chunkSize);
+
+  auto tdescShape = getShapeOf(tdescTy);
+  if (shape != tdescShape)
+    return emitOpError("Incorrect TensorDesc shape. ")
+           << "Expected is " << makeString(shape) << "\n";
+
+  return success();
+}
+
+//===----------------------------------------------------------------------===//
+// XeGPU_PrefetchOp
+//===----------------------------------------------------------------------===//
+LogicalResult PrefetchOp::verify() {
+  auto tdescTy = getTensorDescType();
+  if (!tdescTy.getScattered())
+    return emitOpError("Expects a scattered TensorDesc.\n");
+
+  if (!isReadHintOrNone(getL1HintAttr()))
+    return emitOpError("invlid l1_hint: ") << getL1HintAttr();
+
+  if (!isReadHintOrNone(getL2HintAttr()))
+    return emitOpError("invlid l2_hint: ") << getL2HintAttr();
+
+  if (!isReadHintOrNone(getL3HintAttr()))
+    return emitOpError("invlid l3_hint: ") << getL3HintAttr();
+
+  return success();
+}
+
+//===----------------------------------------------------------------------===//
+// XeGPU_LoadGatherOp
+//===----------------------------------------------------------------------===//
+LogicalResult LoadGatherOp::verify() {
+  auto tdescTy = getTensorDescType();
+  auto maskTy = getMaskType();
+  auto valueTy = getValueType();
+
+  if (!tdescTy.getScattered())
+    return emitOpError("Expects a scattered TensorDesc.\n");
+
+  if (!isReadHintOrNone(getL1HintAttr()))
+    return emitOpError("invlid l1_hint: ") << getL1HintAttr();
+
+  if (!isReadHintOrNone(getL2HintAttr()))
+    return emitOpError("invlid l2_hint: ") << getL2HintAttr();
+
+  if (!isReadHintOrNone(getL3HintAttr()))
+    return emitOpError("invlid l3_hint: ") << getL3HintAttr();
+
+  auto tdescElemTy = tdescTy.getElementType();
+  auto valueElemTy = getElementType();
+  if (tdescElemTy != valueElemTy)
+    return emitOpError(
+        "Value should have the same element type as TensorDesc.");
+
+  auto maskShape = getShapeOf(maskTy);
+  auto valueShape = getShapeOf(valueTy);
+  auto tdescShape = getShapeOf(tdescTy);
+
+  if (tdescShape[0] != maskShape[0])
+    return emitOpError("dim-0 of the Mask and TensorDesc should be the same.");
+
+  if (getTransposeAttr()) {
+    auto trans = getTranspose().value();
+    if (tdescShape.size() < trans.size())
+      emitWarning("Invalid transpose attr. It is ignored.");
+    else
+      transpose(trans, tdescShape);
+  }
+
+  if (valueShape != tdescShape)
+    return emitOpError("Unexpected result shape")
+           << "(Expected shape: " << makeString(tdescShape)
+           << ", Given shape: " << makeString(valueShape) << ").\n";
+
+  return success();
+}
+
+//===----------------------------------------------------------------------===//
+// XeGPU_StoreScatterOp
+//===----------------------------------------------------------------------===//
+LogicalResult StoreScatterOp::verify() {
+  auto tdescTy = getTensorDescType();
+  if (!tdescTy.getScattered())
+    return emitOpError("Expects a scattered TensorDesc.\n");
+
+  if (!isWriteHintOrNone(getL1HintAttr()))
+    return emitOpError("invlid l1_hint: ") << getL1HintAttr();
+
+  if (!isWriteHintOrNone(getL2HintAttr()))
+    return emitOpError("invlid l2_hint: ") << getL2HintAttr();
+
+  if (!isWriteHintOrNone(getL3HintAttr()))
+    return emitOpError("invlid l3_hint: ") << getL3HintAttr();
+
+  auto maskTy = getMaskType();
+  auto maskShape = getShapeOf(maskTy);
+  auto tdescShape = getShapeOf(tdescTy);
+  if (tdescShape[0] != maskShape[0])
+    return emitOpError("dim-0 of the Mask and TensorDesc should be the same.");
 
-  if (dstTy.getShape() != valTy.getShape())
-    return emitOpError()
-           << "The result shape should match the TensorDesc shape.\n";
   return success();
 }
 
diff --git a/mlir/test/Dialect/XeGPU/XeGPUOps.mlir b/mlir/test/Dialect/XeGPU/XeGPUOps.mlir
index 039346adbb85..f0945c79a94a 100644
--- a/mlir/test/Dialect/XeGPU/XeGPUOps.mlir
+++ b/mlir/test/Dialect/XeGPU/XeGPUOps.mlir
@@ -59,4 +59,66 @@ gpu.func @test_store_nd_vc(%dst: memref<24x32xf16>) {
   gpu.return
 }
 
+// CHECK: gpu.func @test_create_update_nd_tdesc_vc(%[[arg0:.*]]: memref<24x32xf32>) {
+gpu.func @test_create_update_nd_tdesc_vc(%src: memref<24x32xf32>) {
+  // CHECK: %[[REG:.*]] = xegpu.create_nd_tdesc %arg0[0, 0] : memref<24x32xf32> -> !xegpu.tensor_desc<8x16xf32>
+  %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf32> -> !xegpu.tensor_desc<8x16xf32>
+  // CHECK: %[[R1:.*]] = xegpu.update_nd_offset %[[REG]], [0, 16] : !xegpu.tensor_desc<8x16xf32>
+  %2 = xegpu.update_nd_offset %1, [0, 16]: !xegpu.tensor_desc<8x16xf32>
+  gpu.return
+}
+
+// CHECK: gpu.func @test_create_tdesc_vc(%[[arg0:.*]]: ui64) {
+gpu.func @test_create_tdesc_vc(%src: ui64) {
+  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %arg0 [0, 8, 16, 24] {chunk_size = 2 : i64} : ui64 -> !xegpu.tensor_desc<4x2xf32, #xegpu.tdesc_attr<scattered = true>>
+  %1 = xegpu.create_tdesc %src[0, 8, 16, 24] {chunk_size = 2} : ui64  -> !xegpu.tensor_desc<4x2xf32, #xegpu.tdesc_attr<scattered = true>>
+  gpu.return
+}
+
+// CHECK: gpu.func @test_prefetch_vc(%[[arg0:.*]]: ui64) {
+gpu.func @test_prefetch_vc(%src: ui64) {
+  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %arg0 [0, 8, 16, 24] {chunk_size = 2 : i64} : ui64 -> !xegpu.tensor_desc<4x2xf32, #xegpu.tdesc_attr<scattered = true>>
+  %1 = xegpu.create_tdesc %src[0, 8, 16, 24] {chunk_size = 2} : ui64  -> !xegpu.tensor_desc<4x2xf32, #xegpu.tdesc_attr<scattered = true>>
+  // CHECK: xegpu.prefetch %[[R0]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<4x2xf32, #xegpu.tdesc_attr<scattered = true>>
+  xegpu.prefetch %1 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}>: !xegpu.tensor_desc<4x2xf32, #xegpu.tdesc_attr<scattered = true>> 
+  gpu.return
+}
+
+// CHECK: gpu.func @test_load_gather_vc(%[[arg0:.*]]: ui64) {
+gpu.func @test_load_gather_vc(%src: ui64) {
+  //CHECK: %[[cst:.*]] = arith.constant dense<true> : vector<4xi1>
+  %0 = arith.constant dense<1>: vector<4xi1>
+  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %arg0 [0, 8, 16, 24] {chunk_size = 2 : i64} : ui64 -> !xegpu.tensor_desc<4x2xf32, #xegpu.tdesc_attr<scattered = true>>
+  %1 = xegpu.create_tdesc %src[0, 8, 16, 24] {chunk_size = 2} : ui64  -> !xegpu.tensor_desc<4x2xf32, #xegpu.tdesc_attr<scattered = true>>
+  //CHECK: %[[R1:.*]] = xegpu.load %[[R0]], %[[cst]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}>
+  //CHECK-SAME: !xegpu.tensor_desc<4x2xf32, #xegpu.tdesc_attr<scattered = true>>, vector<4xi1> -> vector<4x2xf32>
+  %2 = xegpu.load %1, %0 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}>
+        : !xegpu.tensor_desc<4x2xf32, #xegpu.tdesc_attr<scattered = true>>, vector<4xi1> -> vector<4x2xf32>
+  gpu.return
+}
+
+// CHECK: gpu.func @test_store_scatter_vc(%[[arg0:.*]]: ui64) {
+gpu.func @test_store_scatter_vc(%src: ui64) {
+  //CHECK: %[[c0:.*]] = arith.constant dense<true> : vector<4xi1>
+  %0 = arith.constant dense<1>: vector<4xi1>
+  //CHECK: %[[c1:.*]] = arith.constant dense<2.900000e+00> : vector<4x2xf32>
+  %1 = arith.constant dense<2.9>: vector<4x2xf32>
+  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %arg0 [0, 8, 16, 24] {chunk_size = 2 : i64} : ui64 -> !xegpu.tensor_desc<4x2xf32, #xegpu.tdesc_attr<scattered = true>>
+  %2 = xegpu.create_tdesc %src[0, 8, 16, 24] {chunk_size = 2} : ui64  -> !xegpu.tensor_desc<4x2xf32, #xegpu.tdesc_attr<scattered = true>>
+  //CHECK: xegpu.store %[[c1]], %[[R0]], %[[c0]] <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>}>
+  //CHECK-SAME: vector<4x2xf32>, !xegpu.tensor_desc<4x2xf32, #xegpu.tdesc_attr<scattered = true>>, vector<4xi1>
+  xegpu.store %1, %2, %0 <{l1_hint = #xegpu.cache_hint<write_back>, l2_hint = #xegpu.cache_hint<uncached>}>
+        : vector<4x2xf32>, !xegpu.tensor_desc<4x2xf32, #xegpu.tdesc_attr<scattered = true>>, vector<4xi1>
+  gpu.return
+}
+
+// CHECK: gpu.func @test_create_update_tdesc_vc(%[[arg0:.*]]: ui64) {
+gpu.func @test_create_update_tdesc_vc(%src: ui64) {
+  //CHECK: %[[R0:.*]] = xegpu.create_tdesc %arg0 [0, 8, 16, 24] {chunk_size = 2 : i64} : ui64 -> !xegpu.tensor_desc<4x2xf32, #xegpu.tdesc_attr<scattered = true>>
+  %1 = xegpu.create_tdesc %src[0, 8, 16, 24] {chunk_size = 2} : ui64  -> !xegpu.tensor_desc<4x2xf32, #xegpu.tdesc_attr<scattered = true>>
+  //CHECK: %[[R1:.*]] = xegpu.update_offset %[[R0]], [32, 32, 32, 32] : !xegpu.tensor_desc<4x2xf32, #xegpu.tdesc_attr<scattered = true>>
+  %2 = xegpu.update_offset %1, [32, 32, 32, 32] : !xegpu.tensor_desc<4x2xf32, #xegpu.tdesc_attr<scattered = true>>
+  gpu.return
+}
+
 }
\ No newline at end of file
diff --git a/mlir/test/Dialect/XeGPU/invalid.mlir b/mlir/test/Dialect/XeGPU/invalid.mlir
new file mode 100644
index 000000000000..5e29361ec690
--- /dev/null
+++ b/mlir/test/Dialect/XeGPU/invalid.mlir
@@ -0,0 +1,159 @@
+// RUN: mlir-opt %s -split-input-file -verify-diagnostics
+
+// -----
+func.func @test_create_nd_tdesc_vc_1(%src: memref<24xf32>) {
+  // expected-error@+1 {{Expecting the rank of shape, strides, offsets, source memref type (if source is a memref) and TensorDesc should match with each other. They currenlty are 2D.}}
+  %1 = xegpu.create_nd_tdesc %src[0] : memref<24xf32> -> !xegpu.tensor_desc<8x16xf32>
+  return
+}
+
+// -----
+
+func.func @test_create_nd_tdesc_vc_2(%src: memref<24x32xf32>) {
+  // expected-error@+1 {{TensorDesc should have the same element type with the source if it is a memref}}
+  %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf32> -> !xegpu.tensor_desc<8x16xf16>
+  return
+}
+
+// -----
+func.func @test_prefetch_nd_vc_1(%src: memref<24x32xf16>) {
+  %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf16> -> !xegpu.tensor_desc<8x16xf16>
+  // expected-error@+1 {{invlid l1_hint: #xegpu.cache_hint<write_back>}}
+  xegpu.prefetch_nd %1 <{l1_hint = #xegpu.cache_hint<write_back>}>: !xegpu.tensor_desc<8x16xf16>
+  return
+}
+
+// -----
+func.func @test_prefetch_nd_vc_2(%src: memref<24xf16>) {
+  %1 = xegpu.create_tdesc %src[0, 1, 2, 3, 4, 5, 6, 7]
+        : memref<24xf16> -> !xegpu.tensor_desc<8xf16, #xegpu.tdesc_attr<scattered=true>>
+  // expected-error@+1 {{Expects a non-scattered TensorDesc}}
+  xegpu.prefetch_nd %1 <{l1_hint = #xegpu.cache_hint<cached>}>
+        : !xegpu.tensor_desc<8xf16, #xegpu.tdesc_attr<scattered=true>>
+  return
+}
+
+// -----
+func.func @test_load_nd_vc_1(%src: memref<8x16xf16>) {
+  %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<8x16xf16> -> !xegpu.tensor_desc<8x16xf16>
+  // expected-error@+1 {{invlid l1_hint: #xegpu.cache_hint<write_back>}}
+  %2 = xegpu.load_nd %1 <{l1_hint = #xegpu.cache_hint<write_back>}>
+      : !xegpu.tensor_desc<8x16xf16> -> vector<4x16x2xf16>
+  return
+}
+
+// -----
+func.func @test_load_nd_vc_2(%src: memref<16xf16>) {
+  %1 = xegpu.create_tdesc %src[0, 2, 4, 6, 8, 10, 12, 14] {chunk_size = 2}
+        : memref<16xf16> -> !xegpu.tensor_desc<8x2xf16, #xegpu.tdesc_attr<scattered=true>>
+  // expected-error@+1 {{Expects a non-scattered TensorDesc.}}
+  %2 = xegpu.load_nd %1 <{l1_hint = #xegpu.cache_hint<cached>}>
+      : !xegpu.tensor_desc<8x2xf16, #xegpu.tdesc_attr<scattered=true>> -> vector<8x2xf16>
+  return
+}
+
+// -----
+func.func @test_store_nd_vc_1(%dst: memref<24x32xf16>) {
+  %1 = arith.constant dense<1.0>: vector<24x32xf16>
+  %2 = xegpu.create_nd_tdesc %dst[0, 0] : memref<24x32xf16> -> !xegpu.tensor_desc<24x32xf16>
+  // expected-error@+1 {{invlid l1_hint: #xegpu.cache_hint<streaming>}}
+  xegpu.store_nd %1, %2 <{l1_hint = #xegpu.cache_hint<streaming>}>: vector<24x32xf16>, !xegpu.tensor_desc<24x32xf16>
+  return
+}
+
+// -----
+func.func @test_store_nd_vc_2(%dst: memref<16xf16>) {
+  %1 = arith.constant dense<1.0>: vector<8x2xf16>
+  %2 = xegpu.create_tdesc %dst[0, 2, 4, 6, 8, 10, 12, 14] {chunk_size = 2}
+        : memref<16xf16> -> !xegpu.tensor_desc<8x2xf16, #xegpu.tdesc_attr<scattered=true>>
+  // expected-error@+1 {{Expects a non-scattered TensorDesc}}
+  xegpu.store_nd %1, %2 <{l1_hint = #xegpu.cache_hint<streaming>}>
+        : vector<8x2xf16>, !xegpu.tensor_desc<8x2xf16, #xegpu.tdesc_attr<scattered=true>>
+  return
+}
+
+// -----
+func.func @test_update_nd_offset_1(%dst: memref<16xf16>) {
+  %1 = xegpu.create_tdesc %dst[0, 2, 4, 6, 8, 10, 12, 14] {chunk_size = 2}
+        : memref<16xf16> -> !xegpu.tensor_desc<8x2xf16, #xegpu.tdesc_attr<scattered=true>>
+  // expected-error@+1 {{Expects a non-scattered TensorDesc}}
+  xegpu.update_nd_offset %1, [0, 2] : !xegpu.tensor_desc<8x2xf16, #xegpu.tdesc_attr<scattered=true>>
+  return
+}
+
+// -----
+func.func @test_create_tdesc_vc_1(%src: ui64) {
+  // expected-error@+1 {{Expects a scattered TensorDesc}}
+  %1 = xegpu.create_tdesc %src[0, 2, 4, 6, 8, 10, 12, 14] {chunk_size = 2}
+        : ui64 -> !xegpu.tensor_desc<8x2xf16>
+  return
+}
+
+// -----
+func.func @test_create_tdesc_vc_2(%src: ui64) {
+  // expected-error@+1 {{Incorrect TensorDesc shape}}
+  %1 = xegpu.create_tdesc %src[0, 2, 4, 6, 8, 10, 12, 14] {chunk_size = 2}
+        : ui64 -> !xegpu.tensor_desc<8x4xf16, #xegpu.tdesc_attr<scattered = true>>
+  return
+}
+
+// -----
+func.func @test_prefetch_vc_1(%src: memref<24x32xf16>) {
+  %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf16> -> !xegpu.tensor_desc<24x32xf16>
+  // expected-error@+1 {{Expects a scattered TensorDesc}}
+  xegpu.prefetch %1 <{l1_hint = #xegpu.cache_hint<write_back>}>: !xegpu.tensor_desc<24x32xf16>
+  return
+}
+
+// -----
+func.func @test_prefetch_vc_2(%src: ui64) {
+  %1 = xegpu.create_tdesc %src[0, 8, 16, 24] {chunk_size = 2} : ui64  -> !xegpu.tensor_desc<4x2xf32, #xegpu.tdesc_attr<scattered = true>>
+  // expected-error@+1 {{invlid l1_hint: #xegpu.cache_hint<write_back>}}
+  xegpu.prefetch %1 <{l1_hint = #xegpu.cache_hint<write_back>}>: !xegpu.tensor_desc<4x2xf32, #xegpu.tdesc_attr<scattered = true>>
+  return
+}
+
+// -----
+func.func @test_load_gather_vc_1(%src: memref<24x32xf16>) {
+  %0 = arith.constant dense<1>: vector<4xi1>
+  %1 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf16> -> !xegpu.tensor_desc<4x2xf16>
+  // expected-error@+1 {{Expects a scattered TensorDesc}}
+  %2 = xegpu.load %1, %0 <{l1_hint = #xegpu.cache_hint<cached>}>
+      : !xegpu.tensor_desc<4x2xf16>, vector<4xi1> -> vector<4x2xf16>
+  return
+}
+
+// -----
+func.func @test_load_gather_vc_2(%src: ui64) {
+  %0 = arith.constant dense<1>: vector<4xi1>
+  %1 = xegpu.create_tdesc %src[0, 8, 16, 24] {chunk_size = 2} : ui64
+        -> !xegpu.tensor_desc<4x2xf32, #xegpu.tdesc_attr<scattered = true>>
+  // expected-error@+1 {{invlid l1_hint: #xegpu.cache_hint<write_back>}}
+  %2 = xegpu.load %1, %0 <{l1_hint = #xegpu.cache_hint<write_back>}>
+        : !xegpu.tensor_desc<4x2xf32, #xegpu.tdesc_attr<scattered = true>>, vector<4xi1>
+          -> vector<4x2xf32>
+  return
+}
+
+// -----
+func.func @test_store_scatter_vc_1(%src: memref<24x32xf32>) {
+  %0 = arith.constant dense<1>: vector<4xi1>
+  %1 = arith.constant dense<2.9>: vector<4x2xf32>
+  %2 = xegpu.create_nd_tdesc %src[0, 0] : memref<24x32xf32> -> !xegpu.tensor_desc<4x2xf32>
+  // expected-error@+1 {{Expects a scattered TensorDesc}}
+  xegpu.store %1, %2, %0 <{l1_hint = #xegpu.cache_hint<cached>}>
+        : vector<4x2xf32>, !xegpu.tensor_desc<4x2xf32>, vector<4xi1>
+  return
+}
+
+// -----
+func.func @test_store_scatter_vc_2(%src: ui64) {
+  %0 = arith.constant dense<1>: vector<4xi1>
+  %1 = arith.constant dense<2.9>: vector<4x2xf32>
+  %2 = xegpu.create_tdesc %src[0, 8, 16, 24] {chunk_size = 2}
+          : ui64 -> !xegpu.tensor_desc<4x2xf32, #xegpu.tdesc_attr<scattered = true>>
+  // expected-error@+1 {{invlid l1_hint: #xegpu.cache_hint<streaming>}}
+  xegpu.store %1, %2, %0 <{l1_hint = #xegpu.cache_hint<streaming>}> : vector<4x2xf32>,
+          !xegpu.tensor_desc<4x2xf32, #xegpu.tdesc_attr<scattered = true>>, vector<4xi1>
+  return
+}
\ No newline at end of file
-- 
GitLab


From 9ec8c961664de3b3fcc1cbd5238e40ec8c9bdddb Mon Sep 17 00:00:00 2001
From: Samira Bazuzi <bazuzi@google.com>
Date: Tue, 16 Apr 2024 14:46:05 -0400
Subject: [PATCH 453/464] [clang][dataflow] Expose getReferencedDecls and
 relocate free functions. (#88754)

Moves free functions from DataflowEnvironment.h/cc and
DataflowAnalysisContext.h/cc to RecordOps and a new ASTOps and exposes
them as needed for current use and to expose getReferencedDecls for
out-of-tree use.

Minimal change in functionality, only to modify the return type of
getReferenceDecls to return the collected decls instead of using output
params.

Tested with `ninja check-clang-tooling`.
---
 clang/docs/tools/clang-formatted-files.txt    |   2 +
 .../clang/Analysis/FlowSensitive/ASTOps.h     |  98 +++++++
 .../FlowSensitive/DataflowAnalysisContext.h   |  28 +-
 .../FlowSensitive/DataflowEnvironment.h       |  36 ---
 clang/lib/Analysis/FlowSensitive/ASTOps.cpp   | 249 ++++++++++++++++++
 .../lib/Analysis/FlowSensitive/CMakeLists.txt |   1 +
 .../FlowSensitive/DataflowAnalysisContext.cpp |  53 +---
 .../FlowSensitive/DataflowEnvironment.cpp     | 177 +------------
 clang/lib/Analysis/FlowSensitive/Transfer.cpp |   2 +
 9 files changed, 359 insertions(+), 287 deletions(-)
 create mode 100644 clang/include/clang/Analysis/FlowSensitive/ASTOps.h
 create mode 100644 clang/lib/Analysis/FlowSensitive/ASTOps.cpp

diff --git a/clang/docs/tools/clang-formatted-files.txt b/clang/docs/tools/clang-formatted-files.txt
index 3089438c23d9..2252d0ccde96 100644
--- a/clang/docs/tools/clang-formatted-files.txt
+++ b/clang/docs/tools/clang-formatted-files.txt
@@ -123,6 +123,7 @@ clang/include/clang/Analysis/Analyses/CalledOnceCheck.h
 clang/include/clang/Analysis/Analyses/CFGReachabilityAnalysis.h
 clang/include/clang/Analysis/Analyses/ExprMutationAnalyzer.h
 clang/include/clang/Analysis/FlowSensitive/AdornedCFG.h
+clang/include/clang/Analysis/FlowSensitive/ASTOps.h
 clang/include/clang/Analysis/FlowSensitive/DataflowAnalysis.h
 clang/include/clang/Analysis/FlowSensitive/DataflowAnalysisContext.h
 clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h
@@ -307,6 +308,7 @@ clang/lib/Analysis/CalledOnceCheck.cpp
 clang/lib/Analysis/CloneDetection.cpp
 clang/lib/Analysis/CodeInjector.cpp
 clang/lib/Analysis/FlowSensitive/AdornedCFG.cpp
+clang/lib/Analysis/FlowSensitive/ASTOps.cpp
 clang/lib/Analysis/FlowSensitive/DataflowAnalysisContext.cpp
 clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
 clang/lib/Analysis/FlowSensitive/DebugSupport.cpp
diff --git a/clang/include/clang/Analysis/FlowSensitive/ASTOps.h b/clang/include/clang/Analysis/FlowSensitive/ASTOps.h
new file mode 100644
index 000000000000..27ad32c1694f
--- /dev/null
+++ b/clang/include/clang/Analysis/FlowSensitive/ASTOps.h
@@ -0,0 +1,98 @@
+//===-- ASTOps.h -------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//  Operations on AST nodes that are used in flow-sensitive analysis.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_ANALYSIS_FLOWSENSITIVE_ASTOPS_H
+#define LLVM_CLANG_ANALYSIS_FLOWSENSITIVE_ASTOPS_H
+
+#include "clang/AST/Decl.h"
+#include "clang/AST/Expr.h"
+#include "clang/AST/Type.h"
+#include "clang/Analysis/FlowSensitive/StorageLocation.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/SetVector.h"
+
+namespace clang {
+namespace dataflow {
+
+/// Skip past nodes that the CFG does not emit. These nodes are invisible to
+/// flow-sensitive analysis, and should be ignored as they will effectively not
+/// exist.
+///
+///   * `ParenExpr` - The CFG takes the operator precedence into account, but
+///   otherwise omits the node afterwards.
+///
+///   * `ExprWithCleanups` - The CFG will generate the appropriate calls to
+///   destructors and then omit the node.
+///
+const Expr &ignoreCFGOmittedNodes(const Expr &E);
+const Stmt &ignoreCFGOmittedNodes(const Stmt &S);
+
+/// A set of `FieldDecl *`. Use `SmallSetVector` to guarantee deterministic
+/// iteration order.
+using FieldSet = llvm::SmallSetVector<const FieldDecl *, 4>;
+
+/// Returns the set of all fields in the type.
+FieldSet getObjectFields(QualType Type);
+
+/// Returns whether `Fields` and `FieldLocs` contain the same fields.
+bool containsSameFields(const FieldSet &Fields,
+                        const RecordStorageLocation::FieldToLoc &FieldLocs);
+
+/// Helper class for initialization of a record with an `InitListExpr`.
+/// `InitListExpr::inits()` contains the initializers for both the base classes
+/// and the fields of the record; this helper class separates these out into two
+/// different lists. In addition, it deals with special cases associated with
+/// unions.
+class RecordInitListHelper {
+public:
+  // `InitList` must have record type.
+  RecordInitListHelper(const InitListExpr *InitList);
+
+  // Base classes with their associated initializer expressions.
+  ArrayRef<std::pair<const CXXBaseSpecifier *, Expr *>> base_inits() const {
+    return BaseInits;
+  }
+
+  // Fields with their associated initializer expressions.
+  ArrayRef<std::pair<const FieldDecl *, Expr *>> field_inits() const {
+    return FieldInits;
+  }
+
+private:
+  SmallVector<std::pair<const CXXBaseSpecifier *, Expr *>> BaseInits;
+  SmallVector<std::pair<const FieldDecl *, Expr *>> FieldInits;
+
+  // We potentially synthesize an `ImplicitValueInitExpr` for unions. It's a
+  // member variable because we store a pointer to it in `FieldInits`.
+  std::optional<ImplicitValueInitExpr> ImplicitValueInitForUnion;
+};
+
+/// A collection of several types of declarations, all referenced from the same
+/// function.
+struct ReferencedDecls {
+  /// Non-static member variables.
+  FieldSet Fields;
+  /// All variables with static storage duration, notably including static
+  /// member variables and static variables declared within a function.
+  llvm::DenseSet<const VarDecl *> Globals;
+  /// Free functions and member functions which are referenced (but not
+  /// necessarily called).
+  llvm::DenseSet<const FunctionDecl *> Functions;
+};
+
+/// Returns declarations that are declared in or referenced from `FD`.
+ReferencedDecls getReferencedDecls(const FunctionDecl &FD);
+
+} // namespace dataflow
+} // namespace clang
+
+#endif // LLVM_CLANG_ANALYSIS_FLOWSENSITIVE_ASTOPS_H
diff --git a/clang/include/clang/Analysis/FlowSensitive/DataflowAnalysisContext.h b/clang/include/clang/Analysis/FlowSensitive/DataflowAnalysisContext.h
index 909a91059438..aa2c366cb164 100644
--- a/clang/include/clang/Analysis/FlowSensitive/DataflowAnalysisContext.h
+++ b/clang/include/clang/Analysis/FlowSensitive/DataflowAnalysisContext.h
@@ -18,6 +18,7 @@
 #include "clang/AST/Decl.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/TypeOrdering.h"
+#include "clang/Analysis/FlowSensitive/ASTOps.h"
 #include "clang/Analysis/FlowSensitive/AdornedCFG.h"
 #include "clang/Analysis/FlowSensitive/Arena.h"
 #include "clang/Analysis/FlowSensitive/Solver.h"
@@ -30,38 +31,11 @@
 #include <cassert>
 #include <memory>
 #include <optional>
-#include <type_traits>
-#include <utility>
-#include <vector>
 
 namespace clang {
 namespace dataflow {
 class Logger;
 
-/// Skip past nodes that the CFG does not emit. These nodes are invisible to
-/// flow-sensitive analysis, and should be ignored as they will effectively not
-/// exist.
-///
-///   * `ParenExpr` - The CFG takes the operator precedence into account, but
-///   otherwise omits the node afterwards.
-///
-///   * `ExprWithCleanups` - The CFG will generate the appropriate calls to
-///   destructors and then omit the node.
-///
-const Expr &ignoreCFGOmittedNodes(const Expr &E);
-const Stmt &ignoreCFGOmittedNodes(const Stmt &S);
-
-/// A set of `FieldDecl *`. Use `SmallSetVector` to guarantee deterministic
-/// iteration order.
-using FieldSet = llvm::SmallSetVector<const FieldDecl *, 4>;
-
-/// Returns the set of all fields in the type.
-FieldSet getObjectFields(QualType Type);
-
-/// Returns whether `Fields` and `FieldLocs` contain the same fields.
-bool containsSameFields(const FieldSet &Fields,
-                        const RecordStorageLocation::FieldToLoc &FieldLocs);
-
 struct ContextSensitiveOptions {
   /// The maximum depth to analyze. A value of zero is equivalent to disabling
   /// context-sensitive analysis entirely.
diff --git a/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h b/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h
index 706664d7db1c..4277792219c0 100644
--- a/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h
+++ b/clang/include/clang/Analysis/FlowSensitive/DataflowEnvironment.h
@@ -775,42 +775,6 @@ RecordStorageLocation *getImplicitObjectLocation(const CXXMemberCallExpr &MCE,
 RecordStorageLocation *getBaseObjectLocation(const MemberExpr &ME,
                                              const Environment &Env);
 
-/// Returns the fields of a `RecordDecl` that are initialized by an
-/// `InitListExpr`, in the order in which they appear in
-/// `InitListExpr::inits()`.
-/// `Init->getType()` must be a record type.
-std::vector<const FieldDecl *>
-getFieldsForInitListExpr(const InitListExpr *InitList);
-
-/// Helper class for initialization of a record with an `InitListExpr`.
-/// `InitListExpr::inits()` contains the initializers for both the base classes
-/// and the fields of the record; this helper class separates these out into two
-/// different lists. In addition, it deals with special cases associated with
-/// unions.
-class RecordInitListHelper {
-public:
-  // `InitList` must have record type.
-  RecordInitListHelper(const InitListExpr *InitList);
-
-  // Base classes with their associated initializer expressions.
-  ArrayRef<std::pair<const CXXBaseSpecifier *, Expr *>> base_inits() const {
-    return BaseInits;
-  }
-
-  // Fields with their associated initializer expressions.
-  ArrayRef<std::pair<const FieldDecl *, Expr *>> field_inits() const {
-    return FieldInits;
-  }
-
-private:
-  SmallVector<std::pair<const CXXBaseSpecifier *, Expr *>> BaseInits;
-  SmallVector<std::pair<const FieldDecl *, Expr *>> FieldInits;
-
-  // We potentially synthesize an `ImplicitValueInitExpr` for unions. It's a
-  // member variable because we store a pointer to it in `FieldInits`.
-  std::optional<ImplicitValueInitExpr> ImplicitValueInitForUnion;
-};
-
 /// Associates a new `RecordValue` with `Loc` and returns the new value.
 RecordValue &refreshRecordValue(RecordStorageLocation &Loc, Environment &Env);
 
diff --git a/clang/lib/Analysis/FlowSensitive/ASTOps.cpp b/clang/lib/Analysis/FlowSensitive/ASTOps.cpp
new file mode 100644
index 000000000000..75188aef4d1a
--- /dev/null
+++ b/clang/lib/Analysis/FlowSensitive/ASTOps.cpp
@@ -0,0 +1,249 @@
+//===-- ASTOps.cc -------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//  Operations on AST nodes that are used in flow-sensitive analysis.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Analysis/FlowSensitive/ASTOps.h"
+#include "clang/AST/ComputeDependence.h"
+#include "clang/AST/Decl.h"
+#include "clang/AST/DeclBase.h"
+#include "clang/AST/DeclCXX.h"
+#include "clang/AST/Expr.h"
+#include "clang/AST/ExprCXX.h"
+#include "clang/AST/Stmt.h"
+#include "clang/AST/Type.h"
+#include "clang/Analysis/FlowSensitive/StorageLocation.h"
+#include "clang/Basic/LLVM.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/STLExtras.h"
+#include <cassert>
+#include <iterator>
+#include <vector>
+
+#define DEBUG_TYPE "dataflow"
+
+namespace clang::dataflow {
+
+const Expr &ignoreCFGOmittedNodes(const Expr &E) {
+  const Expr *Current = &E;
+  if (auto *EWC = dyn_cast<ExprWithCleanups>(Current)) {
+    Current = EWC->getSubExpr();
+    assert(Current != nullptr);
+  }
+  Current = Current->IgnoreParens();
+  assert(Current != nullptr);
+  return *Current;
+}
+
+const Stmt &ignoreCFGOmittedNodes(const Stmt &S) {
+  if (auto *E = dyn_cast<Expr>(&S))
+    return ignoreCFGOmittedNodes(*E);
+  return S;
+}
+
+// FIXME: Does not precisely handle non-virtual diamond inheritance. A single
+// field decl will be modeled for all instances of the inherited field.
+static void getFieldsFromClassHierarchy(QualType Type, FieldSet &Fields) {
+  if (Type->isIncompleteType() || Type->isDependentType() ||
+      !Type->isRecordType())
+    return;
+
+  for (const FieldDecl *Field : Type->getAsRecordDecl()->fields())
+    Fields.insert(Field);
+  if (auto *CXXRecord = Type->getAsCXXRecordDecl())
+    for (const CXXBaseSpecifier &Base : CXXRecord->bases())
+      getFieldsFromClassHierarchy(Base.getType(), Fields);
+}
+
+/// Gets the set of all fields in the type.
+FieldSet getObjectFields(QualType Type) {
+  FieldSet Fields;
+  getFieldsFromClassHierarchy(Type, Fields);
+  return Fields;
+}
+
+bool containsSameFields(const FieldSet &Fields,
+                        const RecordStorageLocation::FieldToLoc &FieldLocs) {
+  if (Fields.size() != FieldLocs.size())
+    return false;
+  for ([[maybe_unused]] auto [Field, Loc] : FieldLocs)
+    if (!Fields.contains(cast_or_null<FieldDecl>(Field)))
+      return false;
+  return true;
+}
+
+/// Returns the fields of a `RecordDecl` that are initialized by an
+/// `InitListExpr`, in the order in which they appear in
+/// `InitListExpr::inits()`.
+/// `Init->getType()` must be a record type.
+static std::vector<const FieldDecl *>
+getFieldsForInitListExpr(const InitListExpr *InitList) {
+  const RecordDecl *RD = InitList->getType()->getAsRecordDecl();
+  assert(RD != nullptr);
+
+  std::vector<const FieldDecl *> Fields;
+
+  if (InitList->getType()->isUnionType()) {
+    Fields.push_back(InitList->getInitializedFieldInUnion());
+    return Fields;
+  }
+
+  // Unnamed bitfields are only used for padding and do not appear in
+  // `InitListExpr`'s inits. However, those fields do appear in `RecordDecl`'s
+  // field list, and we thus need to remove them before mapping inits to
+  // fields to avoid mapping inits to the wrongs fields.
+  llvm::copy_if(
+      RD->fields(), std::back_inserter(Fields),
+      [](const FieldDecl *Field) { return !Field->isUnnamedBitfield(); });
+  return Fields;
+}
+
+RecordInitListHelper::RecordInitListHelper(const InitListExpr *InitList) {
+  auto *RD = InitList->getType()->getAsCXXRecordDecl();
+  assert(RD != nullptr);
+
+  std::vector<const FieldDecl *> Fields = getFieldsForInitListExpr(InitList);
+  ArrayRef<Expr *> Inits = InitList->inits();
+
+  // Unions initialized with an empty initializer list need special treatment.
+  // For structs/classes initialized with an empty initializer list, Clang
+  // puts `ImplicitValueInitExpr`s in `InitListExpr::inits()`, but for unions,
+  // it doesn't do this -- so we create an `ImplicitValueInitExpr` ourselves.
+  SmallVector<Expr *> InitsForUnion;
+  if (InitList->getType()->isUnionType() && Inits.empty()) {
+    assert(Fields.size() == 1);
+    ImplicitValueInitForUnion.emplace(Fields.front()->getType());
+    InitsForUnion.push_back(&*ImplicitValueInitForUnion);
+    Inits = InitsForUnion;
+  }
+
+  size_t InitIdx = 0;
+
+  assert(Fields.size() + RD->getNumBases() == Inits.size());
+  for (const CXXBaseSpecifier &Base : RD->bases()) {
+    assert(InitIdx < Inits.size());
+    Expr *Init = Inits[InitIdx++];
+    BaseInits.emplace_back(&Base, Init);
+  }
+
+  assert(Fields.size() == Inits.size() - InitIdx);
+  for (const FieldDecl *Field : Fields) {
+    assert(InitIdx < Inits.size());
+    Expr *Init = Inits[InitIdx++];
+    FieldInits.emplace_back(Field, Init);
+  }
+}
+
+static void insertIfGlobal(const Decl &D,
+                           llvm::DenseSet<const VarDecl *> &Globals) {
+  if (auto *V = dyn_cast<VarDecl>(&D))
+    if (V->hasGlobalStorage())
+      Globals.insert(V);
+}
+
+static void insertIfFunction(const Decl &D,
+                             llvm::DenseSet<const FunctionDecl *> &Funcs) {
+  if (auto *FD = dyn_cast<FunctionDecl>(&D))
+    Funcs.insert(FD);
+}
+
+static MemberExpr *getMemberForAccessor(const CXXMemberCallExpr &C) {
+  // Use getCalleeDecl instead of getMethodDecl in order to handle
+  // pointer-to-member calls.
+  const auto *MethodDecl = dyn_cast_or_null<CXXMethodDecl>(C.getCalleeDecl());
+  if (!MethodDecl)
+    return nullptr;
+  auto *Body = dyn_cast_or_null<CompoundStmt>(MethodDecl->getBody());
+  if (!Body || Body->size() != 1)
+    return nullptr;
+  if (auto *RS = dyn_cast<ReturnStmt>(*Body->body_begin()))
+    if (auto *Return = RS->getRetValue())
+      return dyn_cast<MemberExpr>(Return->IgnoreParenImpCasts());
+  return nullptr;
+}
+
+static void getReferencedDecls(const Decl &D, ReferencedDecls &Referenced) {
+  insertIfGlobal(D, Referenced.Globals);
+  insertIfFunction(D, Referenced.Functions);
+  if (const auto *Decomp = dyn_cast<DecompositionDecl>(&D))
+    for (const auto *B : Decomp->bindings())
+      if (auto *ME = dyn_cast_or_null<MemberExpr>(B->getBinding()))
+        // FIXME: should we be using `E->getFoundDecl()`?
+        if (const auto *FD = dyn_cast<FieldDecl>(ME->getMemberDecl()))
+          Referenced.Fields.insert(FD);
+}
+
+/// Traverses `S` and inserts into `Referenced` any declarations that are
+/// declared in or referenced from sub-statements.
+static void getReferencedDecls(const Stmt &S, ReferencedDecls &Referenced) {
+  for (auto *Child : S.children())
+    if (Child != nullptr)
+      getReferencedDecls(*Child, Referenced);
+  if (const auto *DefaultArg = dyn_cast<CXXDefaultArgExpr>(&S))
+    getReferencedDecls(*DefaultArg->getExpr(), Referenced);
+  if (const auto *DefaultInit = dyn_cast<CXXDefaultInitExpr>(&S))
+    getReferencedDecls(*DefaultInit->getExpr(), Referenced);
+
+  if (auto *DS = dyn_cast<DeclStmt>(&S)) {
+    if (DS->isSingleDecl())
+      getReferencedDecls(*DS->getSingleDecl(), Referenced);
+    else
+      for (auto *D : DS->getDeclGroup())
+        getReferencedDecls(*D, Referenced);
+  } else if (auto *E = dyn_cast<DeclRefExpr>(&S)) {
+    insertIfGlobal(*E->getDecl(), Referenced.Globals);
+    insertIfFunction(*E->getDecl(), Referenced.Functions);
+  } else if (const auto *C = dyn_cast<CXXMemberCallExpr>(&S)) {
+    // If this is a method that returns a member variable but does nothing else,
+    // model the field of the return value.
+    if (MemberExpr *E = getMemberForAccessor(*C))
+      if (const auto *FD = dyn_cast<FieldDecl>(E->getMemberDecl()))
+        Referenced.Fields.insert(FD);
+  } else if (auto *E = dyn_cast<MemberExpr>(&S)) {
+    // FIXME: should we be using `E->getFoundDecl()`?
+    const ValueDecl *VD = E->getMemberDecl();
+    insertIfGlobal(*VD, Referenced.Globals);
+    insertIfFunction(*VD, Referenced.Functions);
+    if (const auto *FD = dyn_cast<FieldDecl>(VD))
+      Referenced.Fields.insert(FD);
+  } else if (auto *InitList = dyn_cast<InitListExpr>(&S)) {
+    if (InitList->getType()->isRecordType())
+      for (const auto *FD : getFieldsForInitListExpr(InitList))
+        Referenced.Fields.insert(FD);
+  }
+}
+
+ReferencedDecls getReferencedDecls(const FunctionDecl &FD) {
+  ReferencedDecls Result;
+  // Look for global variable and field references in the
+  // constructor-initializers.
+  if (const auto *CtorDecl = dyn_cast<CXXConstructorDecl>(&FD)) {
+    for (const auto *Init : CtorDecl->inits()) {
+      if (Init->isMemberInitializer()) {
+        Result.Fields.insert(Init->getMember());
+      } else if (Init->isIndirectMemberInitializer()) {
+        for (const auto *I : Init->getIndirectMember()->chain())
+          Result.Fields.insert(cast<FieldDecl>(I));
+      }
+      const Expr *E = Init->getInit();
+      assert(E != nullptr);
+      getReferencedDecls(*E, Result);
+    }
+    // Add all fields mentioned in default member initializers.
+    for (const FieldDecl *F : CtorDecl->getParent()->fields())
+      if (const auto *I = F->getInClassInitializer())
+        getReferencedDecls(*I, Result);
+  }
+  getReferencedDecls(*FD.getBody(), Result);
+
+  return Result;
+}
+
+} // namespace clang::dataflow
diff --git a/clang/lib/Analysis/FlowSensitive/CMakeLists.txt b/clang/lib/Analysis/FlowSensitive/CMakeLists.txt
index a3b5d9adc24b..6631fe27f3d9 100644
--- a/clang/lib/Analysis/FlowSensitive/CMakeLists.txt
+++ b/clang/lib/Analysis/FlowSensitive/CMakeLists.txt
@@ -1,6 +1,7 @@
 add_clang_library(clangAnalysisFlowSensitive
   AdornedCFG.cpp
   Arena.cpp
+  ASTOps.cpp
   DataflowAnalysisContext.cpp
   DataflowEnvironment.cpp
   Formula.cpp
diff --git a/clang/lib/Analysis/FlowSensitive/DataflowAnalysisContext.cpp b/clang/lib/Analysis/FlowSensitive/DataflowAnalysisContext.cpp
index d520539dd253..e94fd39c45dc 100644
--- a/clang/lib/Analysis/FlowSensitive/DataflowAnalysisContext.cpp
+++ b/clang/lib/Analysis/FlowSensitive/DataflowAnalysisContext.cpp
@@ -14,6 +14,7 @@
 
 #include "clang/Analysis/FlowSensitive/DataflowAnalysisContext.h"
 #include "clang/AST/ExprCXX.h"
+#include "clang/Analysis/FlowSensitive/ASTOps.h"
 #include "clang/Analysis/FlowSensitive/DebugSupport.h"
 #include "clang/Analysis/FlowSensitive/Formula.h"
 #include "clang/Analysis/FlowSensitive/Logger.h"
@@ -359,55 +360,3 @@ DataflowAnalysisContext::~DataflowAnalysisContext() = default;
 
 } // namespace dataflow
 } // namespace clang
-
-using namespace clang;
-
-const Expr &clang::dataflow::ignoreCFGOmittedNodes(const Expr &E) {
-  const Expr *Current = &E;
-  if (auto *EWC = dyn_cast<ExprWithCleanups>(Current)) {
-    Current = EWC->getSubExpr();
-    assert(Current != nullptr);
-  }
-  Current = Current->IgnoreParens();
-  assert(Current != nullptr);
-  return *Current;
-}
-
-const Stmt &clang::dataflow::ignoreCFGOmittedNodes(const Stmt &S) {
-  if (auto *E = dyn_cast<Expr>(&S))
-    return ignoreCFGOmittedNodes(*E);
-  return S;
-}
-
-// FIXME: Does not precisely handle non-virtual diamond inheritance. A single
-// field decl will be modeled for all instances of the inherited field.
-static void getFieldsFromClassHierarchy(QualType Type,
-                                        clang::dataflow::FieldSet &Fields) {
-  if (Type->isIncompleteType() || Type->isDependentType() ||
-      !Type->isRecordType())
-    return;
-
-  for (const FieldDecl *Field : Type->getAsRecordDecl()->fields())
-    Fields.insert(Field);
-  if (auto *CXXRecord = Type->getAsCXXRecordDecl())
-    for (const CXXBaseSpecifier &Base : CXXRecord->bases())
-      getFieldsFromClassHierarchy(Base.getType(), Fields);
-}
-
-/// Gets the set of all fields in the type.
-clang::dataflow::FieldSet clang::dataflow::getObjectFields(QualType Type) {
-  FieldSet Fields;
-  getFieldsFromClassHierarchy(Type, Fields);
-  return Fields;
-}
-
-bool clang::dataflow::containsSameFields(
-    const clang::dataflow::FieldSet &Fields,
-    const clang::dataflow::RecordStorageLocation::FieldToLoc &FieldLocs) {
-  if (Fields.size() != FieldLocs.size())
-    return false;
-  for ([[maybe_unused]] auto [Field, Loc] : FieldLocs)
-    if (!Fields.contains(cast_or_null<FieldDecl>(Field)))
-      return false;
-  return true;
-}
diff --git a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
index ee2581143e11..3bf3807268be 100644
--- a/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
+++ b/clang/lib/Analysis/FlowSensitive/DataflowEnvironment.cpp
@@ -17,6 +17,7 @@
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/AST/Type.h"
+#include "clang/Analysis/FlowSensitive/ASTOps.h"
 #include "clang/Analysis/FlowSensitive/DataflowLattice.h"
 #include "clang/Analysis/FlowSensitive/Value.h"
 #include "llvm/ADT/DenseMap.h"
@@ -304,93 +305,6 @@ widenKeyToValueMap(const llvm::MapVector<Key, Value *> &CurMap,
   return WidenedMap;
 }
 
-/// Initializes a global storage value.
-static void insertIfGlobal(const Decl &D,
-                           llvm::DenseSet<const VarDecl *> &Vars) {
-  if (auto *V = dyn_cast<VarDecl>(&D))
-    if (V->hasGlobalStorage())
-      Vars.insert(V);
-}
-
-static void insertIfFunction(const Decl &D,
-                             llvm::DenseSet<const FunctionDecl *> &Funcs) {
-  if (auto *FD = dyn_cast<FunctionDecl>(&D))
-    Funcs.insert(FD);
-}
-
-static MemberExpr *getMemberForAccessor(const CXXMemberCallExpr &C) {
-  // Use getCalleeDecl instead of getMethodDecl in order to handle
-  // pointer-to-member calls.
-  const auto *MethodDecl = dyn_cast_or_null<CXXMethodDecl>(C.getCalleeDecl());
-  if (!MethodDecl)
-    return nullptr;
-  auto *Body = dyn_cast_or_null<CompoundStmt>(MethodDecl->getBody());
-  if (!Body || Body->size() != 1)
-    return nullptr;
-  if (auto *RS = dyn_cast<ReturnStmt>(*Body->body_begin()))
-    if (auto *Return = RS->getRetValue())
-      return dyn_cast<MemberExpr>(Return->IgnoreParenImpCasts());
-  return nullptr;
-}
-
-static void
-getFieldsGlobalsAndFuncs(const Decl &D, FieldSet &Fields,
-                         llvm::DenseSet<const VarDecl *> &Vars,
-                         llvm::DenseSet<const FunctionDecl *> &Funcs) {
-  insertIfGlobal(D, Vars);
-  insertIfFunction(D, Funcs);
-  if (const auto *Decomp = dyn_cast<DecompositionDecl>(&D))
-    for (const auto *B : Decomp->bindings())
-      if (auto *ME = dyn_cast_or_null<MemberExpr>(B->getBinding()))
-        // FIXME: should we be using `E->getFoundDecl()`?
-        if (const auto *FD = dyn_cast<FieldDecl>(ME->getMemberDecl()))
-          Fields.insert(FD);
-}
-
-/// Traverses `S` and inserts into `Fields`, `Vars` and `Funcs` any fields,
-/// global variables and functions that are declared in or referenced from
-/// sub-statements.
-static void
-getFieldsGlobalsAndFuncs(const Stmt &S, FieldSet &Fields,
-                         llvm::DenseSet<const VarDecl *> &Vars,
-                         llvm::DenseSet<const FunctionDecl *> &Funcs) {
-  for (auto *Child : S.children())
-    if (Child != nullptr)
-      getFieldsGlobalsAndFuncs(*Child, Fields, Vars, Funcs);
-  if (const auto *DefaultArg = dyn_cast<CXXDefaultArgExpr>(&S))
-    getFieldsGlobalsAndFuncs(*DefaultArg->getExpr(), Fields, Vars, Funcs);
-  if (const auto *DefaultInit = dyn_cast<CXXDefaultInitExpr>(&S))
-    getFieldsGlobalsAndFuncs(*DefaultInit->getExpr(), Fields, Vars, Funcs);
-
-  if (auto *DS = dyn_cast<DeclStmt>(&S)) {
-    if (DS->isSingleDecl())
-      getFieldsGlobalsAndFuncs(*DS->getSingleDecl(), Fields, Vars, Funcs);
-    else
-      for (auto *D : DS->getDeclGroup())
-        getFieldsGlobalsAndFuncs(*D, Fields, Vars, Funcs);
-  } else if (auto *E = dyn_cast<DeclRefExpr>(&S)) {
-    insertIfGlobal(*E->getDecl(), Vars);
-    insertIfFunction(*E->getDecl(), Funcs);
-  } else if (const auto *C = dyn_cast<CXXMemberCallExpr>(&S)) {
-    // If this is a method that returns a member variable but does nothing else,
-    // model the field of the return value.
-    if (MemberExpr *E = getMemberForAccessor(*C))
-      if (const auto *FD = dyn_cast<FieldDecl>(E->getMemberDecl()))
-        Fields.insert(FD);
-  } else if (auto *E = dyn_cast<MemberExpr>(&S)) {
-    // FIXME: should we be using `E->getFoundDecl()`?
-    const ValueDecl *VD = E->getMemberDecl();
-    insertIfGlobal(*VD, Vars);
-    insertIfFunction(*VD, Funcs);
-    if (const auto *FD = dyn_cast<FieldDecl>(VD))
-      Fields.insert(FD);
-  } else if (auto *InitList = dyn_cast<InitListExpr>(&S)) {
-    if (InitList->getType()->isRecordType())
-      for (const auto *FD : getFieldsForInitListExpr(InitList))
-        Fields.insert(FD);
-  }
-}
-
 namespace {
 
 // Visitor that builds a map from record prvalues to result objects.
@@ -653,36 +567,13 @@ void Environment::initialize() {
 void Environment::initFieldsGlobalsAndFuncs(const FunctionDecl *FuncDecl) {
   assert(FuncDecl->doesThisDeclarationHaveABody());
 
-  FieldSet Fields;
-  llvm::DenseSet<const VarDecl *> Vars;
-  llvm::DenseSet<const FunctionDecl *> Funcs;
-
-  // Look for global variable and field references in the
-  // constructor-initializers.
-  if (const auto *CtorDecl = dyn_cast<CXXConstructorDecl>(FuncDecl)) {
-    for (const auto *Init : CtorDecl->inits()) {
-      if (Init->isMemberInitializer()) {
-        Fields.insert(Init->getMember());
-      } else if (Init->isIndirectMemberInitializer()) {
-        for (const auto *I : Init->getIndirectMember()->chain())
-          Fields.insert(cast<FieldDecl>(I));
-      }
-      const Expr *E = Init->getInit();
-      assert(E != nullptr);
-      getFieldsGlobalsAndFuncs(*E, Fields, Vars, Funcs);
-    }
-    // Add all fields mentioned in default member initializers.
-    for (const FieldDecl *F : CtorDecl->getParent()->fields())
-      if (const auto *I = F->getInClassInitializer())
-          getFieldsGlobalsAndFuncs(*I, Fields, Vars, Funcs);
-  }
-  getFieldsGlobalsAndFuncs(*FuncDecl->getBody(), Fields, Vars, Funcs);
+  ReferencedDecls Referenced = getReferencedDecls(*FuncDecl);
 
   // These have to be added before the lines that follow to ensure that
   // `create*` work correctly for structs.
-  DACtx->addModeledFields(Fields);
+  DACtx->addModeledFields(Referenced.Fields);
 
-  for (const VarDecl *D : Vars) {
+  for (const VarDecl *D : Referenced.Globals) {
     if (getStorageLocation(*D) != nullptr)
       continue;
 
@@ -694,7 +585,7 @@ void Environment::initFieldsGlobalsAndFuncs(const FunctionDecl *FuncDecl) {
     setStorageLocation(*D, createObject(*D, nullptr));
   }
 
-  for (const FunctionDecl *FD : Funcs) {
+  for (const FunctionDecl *FD : Referenced.Functions) {
     if (getStorageLocation(*FD) != nullptr)
       continue;
     auto &Loc = createStorageLocation(*FD);
@@ -1354,64 +1245,6 @@ RecordStorageLocation *getBaseObjectLocation(const MemberExpr &ME,
   return Env.get<RecordStorageLocation>(*Base);
 }
 
-std::vector<const FieldDecl *>
-getFieldsForInitListExpr(const InitListExpr *InitList) {
-  const RecordDecl *RD = InitList->getType()->getAsRecordDecl();
-  assert(RD != nullptr);
-
-  std::vector<const FieldDecl *> Fields;
-
-  if (InitList->getType()->isUnionType()) {
-    Fields.push_back(InitList->getInitializedFieldInUnion());
-    return Fields;
-  }
-
-  // Unnamed bitfields are only used for padding and do not appear in
-  // `InitListExpr`'s inits. However, those fields do appear in `RecordDecl`'s
-  // field list, and we thus need to remove them before mapping inits to
-  // fields to avoid mapping inits to the wrongs fields.
-  llvm::copy_if(
-      RD->fields(), std::back_inserter(Fields),
-      [](const FieldDecl *Field) { return !Field->isUnnamedBitfield(); });
-  return Fields;
-}
-
-RecordInitListHelper::RecordInitListHelper(const InitListExpr *InitList) {
-  auto *RD = InitList->getType()->getAsCXXRecordDecl();
-  assert(RD != nullptr);
-
-  std::vector<const FieldDecl *> Fields = getFieldsForInitListExpr(InitList);
-  ArrayRef<Expr *> Inits = InitList->inits();
-
-  // Unions initialized with an empty initializer list need special treatment.
-  // For structs/classes initialized with an empty initializer list, Clang
-  // puts `ImplicitValueInitExpr`s in `InitListExpr::inits()`, but for unions,
-  // it doesn't do this -- so we create an `ImplicitValueInitExpr` ourselves.
-  SmallVector<Expr *> InitsForUnion;
-  if (InitList->getType()->isUnionType() && Inits.empty()) {
-    assert(Fields.size() == 1);
-    ImplicitValueInitForUnion.emplace(Fields.front()->getType());
-    InitsForUnion.push_back(&*ImplicitValueInitForUnion);
-    Inits = InitsForUnion;
-  }
-
-  size_t InitIdx = 0;
-
-  assert(Fields.size() + RD->getNumBases() == Inits.size());
-  for (const CXXBaseSpecifier &Base : RD->bases()) {
-    assert(InitIdx < Inits.size());
-    Expr *Init = Inits[InitIdx++];
-    BaseInits.emplace_back(&Base, Init);
-  }
-
-  assert(Fields.size() == Inits.size() - InitIdx);
-  for (const FieldDecl *Field : Fields) {
-    assert(InitIdx < Inits.size());
-    Expr *Init = Inits[InitIdx++];
-    FieldInits.emplace_back(Field, Init);
-  }
-}
-
 RecordValue &refreshRecordValue(RecordStorageLocation &Loc, Environment &Env) {
   auto &NewVal = Env.create<RecordValue>(Loc);
   Env.setValue(Loc, NewVal);
diff --git a/clang/lib/Analysis/FlowSensitive/Transfer.cpp b/clang/lib/Analysis/FlowSensitive/Transfer.cpp
index 88a9c0eccbeb..1e034771014e 100644
--- a/clang/lib/Analysis/FlowSensitive/Transfer.cpp
+++ b/clang/lib/Analysis/FlowSensitive/Transfer.cpp
@@ -20,7 +20,9 @@
 #include "clang/AST/OperationKinds.h"
 #include "clang/AST/Stmt.h"
 #include "clang/AST/StmtVisitor.h"
+#include "clang/Analysis/FlowSensitive/ASTOps.h"
 #include "clang/Analysis/FlowSensitive/AdornedCFG.h"
+#include "clang/Analysis/FlowSensitive/DataflowAnalysisContext.h"
 #include "clang/Analysis/FlowSensitive/DataflowEnvironment.h"
 #include "clang/Analysis/FlowSensitive/NoopAnalysis.h"
 #include "clang/Analysis/FlowSensitive/RecordOps.h"
-- 
GitLab


From bbd64c4ddf08be468ab4eb4c161e28bdab6808bb Mon Sep 17 00:00:00 2001
From: Philip Reames <preames@rivosinc.com>
Date: Tue, 16 Apr 2024 11:40:23 -0700
Subject: [PATCH 454/464] [RISCV] Add coverage for strength reduction of mul as
 2^N - 2^M

---
 llvm/test/CodeGen/RISCV/mul.ll | 196 ++++++++++++++++++++++++++++++++-
 1 file changed, 192 insertions(+), 4 deletions(-)

diff --git a/llvm/test/CodeGen/RISCV/mul.ll b/llvm/test/CodeGen/RISCV/mul.ll
index af341dbaadea..364e8c7b38da 100644
--- a/llvm/test/CodeGen/RISCV/mul.ll
+++ b/llvm/test/CodeGen/RISCV/mul.ll
@@ -465,6 +465,192 @@ define i32 @mulhu_constant(i32 %a) nounwind {
   ret i32 %4
 }
 
+define i32 @muli32_p14(i32 %a) nounwind {
+; RV32I-LABEL: muli32_p14:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    li a1, 14
+; RV32I-NEXT:    tail __mulsi3
+;
+; RV32IM-LABEL: muli32_p14:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    li a1, 14
+; RV32IM-NEXT:    mul a0, a0, a1
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: muli32_p14:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    li a1, 14
+; RV64I-NEXT:    call __muldi3
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: muli32_p14:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    li a1, 14
+; RV64IM-NEXT:    mulw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, 14
+  ret i32 %1
+}
+
+define i32 @muli32_p28(i32 %a) nounwind {
+; RV32I-LABEL: muli32_p28:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    li a1, 28
+; RV32I-NEXT:    tail __mulsi3
+;
+; RV32IM-LABEL: muli32_p28:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    li a1, 28
+; RV32IM-NEXT:    mul a0, a0, a1
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: muli32_p28:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    li a1, 28
+; RV64I-NEXT:    call __muldi3
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: muli32_p28:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    li a1, 28
+; RV64IM-NEXT:    mulw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, 28
+  ret i32 %1
+}
+
+define i32 @muli32_p30(i32 %a) nounwind {
+; RV32I-LABEL: muli32_p30:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    li a1, 30
+; RV32I-NEXT:    tail __mulsi3
+;
+; RV32IM-LABEL: muli32_p30:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    li a1, 30
+; RV32IM-NEXT:    mul a0, a0, a1
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: muli32_p30:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    li a1, 30
+; RV64I-NEXT:    call __muldi3
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: muli32_p30:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    li a1, 30
+; RV64IM-NEXT:    mulw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, 30
+  ret i32 %1
+}
+
+define i32 @muli32_p56(i32 %a) nounwind {
+; RV32I-LABEL: muli32_p56:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    li a1, 56
+; RV32I-NEXT:    tail __mulsi3
+;
+; RV32IM-LABEL: muli32_p56:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    li a1, 56
+; RV32IM-NEXT:    mul a0, a0, a1
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: muli32_p56:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    li a1, 56
+; RV64I-NEXT:    call __muldi3
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: muli32_p56:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    li a1, 56
+; RV64IM-NEXT:    mulw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, 56
+  ret i32 %1
+}
+
+define i32 @muli32_p60(i32 %a) nounwind {
+; RV32I-LABEL: muli32_p60:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    li a1, 60
+; RV32I-NEXT:    tail __mulsi3
+;
+; RV32IM-LABEL: muli32_p60:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    li a1, 60
+; RV32IM-NEXT:    mul a0, a0, a1
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: muli32_p60:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    li a1, 60
+; RV64I-NEXT:    call __muldi3
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: muli32_p60:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    li a1, 60
+; RV64IM-NEXT:    mulw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, 60
+  ret i32 %1
+}
+
+define i32 @muli32_p62(i32 %a) nounwind {
+; RV32I-LABEL: muli32_p62:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    li a1, 62
+; RV32I-NEXT:    tail __mulsi3
+;
+; RV32IM-LABEL: muli32_p62:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    li a1, 62
+; RV32IM-NEXT:    mul a0, a0, a1
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: muli32_p62:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
+; RV64I-NEXT:    li a1, 62
+; RV64I-NEXT:    call __muldi3
+; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: muli32_p62:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    li a1, 62
+; RV64IM-NEXT:    mulw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, 62
+  ret i32 %1
+}
+
 define i32 @muli32_p65(i32 %a) nounwind {
 ; RV32I-LABEL: muli32_p65:
 ; RV32I:       # %bb.0:
@@ -600,6 +786,8 @@ define i64 @muli64_p63(i64 %a) nounwind {
   ret i64 %1
 }
 
+
+
 define i32 @muli32_m63(i32 %a) nounwind {
 ; RV32I-LABEL: muli32_m63:
 ; RV32I:       # %bb.0:
@@ -1145,10 +1333,10 @@ define i128 @muli128_m3840(i128 %a) nounwind {
 ; RV32I-NEXT:    sltu a7, a6, a4
 ; RV32I-NEXT:    sub t0, t1, t0
 ; RV32I-NEXT:    mv t1, a7
-; RV32I-NEXT:    beq a5, a3, .LBB30_2
+; RV32I-NEXT:    beq a5, a3, .LBB36_2
 ; RV32I-NEXT:  # %bb.1:
 ; RV32I-NEXT:    sltu t1, a5, a3
-; RV32I-NEXT:  .LBB30_2:
+; RV32I-NEXT:  .LBB36_2:
 ; RV32I-NEXT:    sub a2, a2, a1
 ; RV32I-NEXT:    sltu a1, a2, t1
 ; RV32I-NEXT:    sub a1, t0, a1
@@ -1261,10 +1449,10 @@ define i128 @muli128_m63(i128 %a) nounwind {
 ; RV32I-NEXT:    slli t0, a1, 6
 ; RV32I-NEXT:    or a7, t0, a7
 ; RV32I-NEXT:    mv t0, a5
-; RV32I-NEXT:    beq a1, a7, .LBB31_2
+; RV32I-NEXT:    beq a1, a7, .LBB37_2
 ; RV32I-NEXT:  # %bb.1:
 ; RV32I-NEXT:    sltu t0, a1, a7
-; RV32I-NEXT:  .LBB31_2:
+; RV32I-NEXT:  .LBB37_2:
 ; RV32I-NEXT:    srli t1, a1, 26
 ; RV32I-NEXT:    slli t2, a6, 6
 ; RV32I-NEXT:    or t1, t2, t1
-- 
GitLab


From 8885813ebb0a61014d99ac776b8118d935848cc9 Mon Sep 17 00:00:00 2001
From: Mark de Wever <koraq@xs4all.nl>
Date: Tue, 16 Apr 2024 20:45:12 +0200
Subject: [PATCH 455/464] [libc++][chrono] Disables a test.

This tests seems problematic on different platforms. There is still a
test that ensures coverage, but in an automatic fashion. This test needs
to be investigated.
---
 .../time.zone.members/get_info.sys_time.pass.cpp             | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/get_info.sys_time.pass.cpp b/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/get_info.sys_time.pass.cpp
index a751a2fb6347..d27cf0bd8906 100644
--- a/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/get_info.sys_time.pass.cpp
+++ b/libcxx/test/std/time/time.zone/time.zone.timezone/time.zone.members/get_info.sys_time.pass.cpp
@@ -6,7 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++03, c++11, c++14, c++17
+// TODO TZDB review the test based on review comments in
+// https://github.com/llvm/llvm-project/pull/85619
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20, c++23, c++26
 // UNSUPPORTED: no-filesystem, no-localization, no-tzdb
 
 // XFAIL: libcpp-has-no-incomplete-tzdb
-- 
GitLab


From 0665669876cd7f51f7572cff3bb97485d78f5de5 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Tue, 16 Apr 2024 11:49:25 -0700
Subject: [PATCH 456/464] [Sema] Mark alias/ifunc targets used and consider
 mangled names

https://reviews.llvm.org/D54188 marked "alias" targets as used in C to
fix -Wunused false positives. This patch extends the approach to handle
mangled names to support global scope names in C++ and the
`overloadable` attribute in C.

(Note: we should skip `UsingShadowDecl`, which would trigger an
assertion failure in `ItaniumMangleContextImpl::mangleCXXName`.
See regression test added by commit 1c2afbae9af22b58190c10e3517242d01d89d612.)

In addition, we mark ifunc targets as used to fix #63957 (temporarily
used by xz; ifunc was removed by
https://github.com/tukaani-project/xz/commit/689ae2427342a2ea1206eb5ca08301baf410e7e0)

While our approach has false negatives for namespace scope names, the
majority of alias/ifunc C++ uses (global scope with no overloads) are
handled.

Note: The following function with internal linkage but C language
linkage type is mangled in Clang but not in GCC. This inconsistency
makes alias/ifunc difficult to use in C++ with portability (#88593).
```
extern "C" {
static void f0() {}
// GCC: void g0() __attribute__((alias("_ZL2f0v")));
// Clang: void g0() __attribute__((alias("f0")));
}
```

Pull Request: https://github.com/llvm/llvm-project/pull/87130
---
 clang/lib/Sema/CMakeLists.txt                 |  1 +
 clang/lib/Sema/SemaDeclAttr.cpp               | 46 ++++++++++++++-----
 clang/test/AST/ast-dump-attr-json.cpp         |  1 +
 clang/test/Sema/alias-unused-win.cpp          |  2 +-
 clang/test/Sema/alias-unused.cpp              | 16 ++++---
 .../llvm-project-overlay/clang/BUILD.bazel    |  1 +
 6 files changed, 48 insertions(+), 19 deletions(-)

diff --git a/clang/lib/Sema/CMakeLists.txt b/clang/lib/Sema/CMakeLists.txt
index ab3b813a9ccd..a96439df6642 100644
--- a/clang/lib/Sema/CMakeLists.txt
+++ b/clang/lib/Sema/CMakeLists.txt
@@ -1,5 +1,6 @@
 set(LLVM_LINK_COMPONENTS
   Core
+  Demangle
   FrontendHLSL
   FrontendOpenMP
   MC
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index b7b1fbc625a1..c3bf18a3f79e 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -45,6 +45,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/STLForwardCompat.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/Demangle/Demangle.h"
 #include "llvm/IR/Assumptions.h"
 #include "llvm/MC/MCSectionMachO.h"
 #include "llvm/Support/Error.h"
@@ -1983,6 +1984,38 @@ static void handleWeakRefAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   D->addAttr(::new (S.Context) WeakRefAttr(S.Context, AL));
 }
 
+// Mark alias/ifunc target as used. Due to name mangling, we look up the
+// demangled name ignoring parameters (not supported by microsoftDemangle
+// https://github.com/llvm/llvm-project/issues/88825). This should handle the
+// majority of use cases while leaving namespace scope names unmarked.
+static void markUsedForAliasOrIfunc(Sema &S, Decl *D, const ParsedAttr &AL,
+                                    StringRef Str) {
+  std::unique_ptr<char, llvm::FreeDeleter> Demangled;
+  if (S.getASTContext().getCXXABIKind() != TargetCXXABI::Microsoft)
+    Demangled.reset(llvm::itaniumDemangle(Str, /*ParseParams=*/false));
+  std::unique_ptr<MangleContext> MC(S.Context.createMangleContext());
+  SmallString<256> Name;
+
+  const DeclarationNameInfo Target(
+      &S.Context.Idents.get(Demangled ? Demangled.get() : Str), AL.getLoc());
+  LookupResult LR(S, Target, Sema::LookupOrdinaryName);
+  if (S.LookupName(LR, S.TUScope)) {
+    for (NamedDecl *ND : LR) {
+      if (!isa<FunctionDecl>(ND) && !isa<VarDecl>(ND))
+        continue;
+      if (MC->shouldMangleDeclName(ND)) {
+        llvm::raw_svector_ostream Out(Name);
+        Name.clear();
+        MC->mangleName(GlobalDecl(ND), Out);
+      } else {
+        Name = ND->getIdentifier()->getName();
+      }
+      if (Name == Str)
+        ND->markUsed(S.Context);
+    }
+  }
+}
+
 static void handleIFuncAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   StringRef Str;
   if (!S.checkStringLiteralArgumentAttr(AL, 0, Str))
@@ -1995,6 +2028,7 @@ static void handleIFuncAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
     return;
   }
 
+  markUsedForAliasOrIfunc(S, D, AL, Str);
   D->addAttr(::new (S.Context) IFuncAttr(S.Context, AL, Str));
 }
 
@@ -2029,17 +2063,7 @@ static void handleAliasAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
     }
   }
 
-  // Mark target used to prevent unneeded-internal-declaration warnings.
-  if (!S.LangOpts.CPlusPlus) {
-    // FIXME: demangle Str for C++, as the attribute refers to the mangled
-    // linkage name, not the pre-mangled identifier.
-    const DeclarationNameInfo target(&S.Context.Idents.get(Str), AL.getLoc());
-    LookupResult LR(S, target, Sema::LookupOrdinaryName);
-    if (S.LookupQualifiedName(LR, S.getCurLexicalContext()))
-      for (NamedDecl *ND : LR)
-        ND->markUsed(S.Context);
-  }
-
+  markUsedForAliasOrIfunc(S, D, AL, Str);
   D->addAttr(::new (S.Context) AliasAttr(S.Context, AL, Str));
 }
 
diff --git a/clang/test/AST/ast-dump-attr-json.cpp b/clang/test/AST/ast-dump-attr-json.cpp
index 051c2956abfd..883e584bfedf 100644
--- a/clang/test/AST/ast-dump-attr-json.cpp
+++ b/clang/test/AST/ast-dump-attr-json.cpp
@@ -46,6 +46,7 @@ __thread __attribute__ ((tls_model ("local-exec"))) int tls_model_var;
 // CHECK-NEXT:    "tokLen": 11
 // CHECK-NEXT:   }
 // CHECK-NEXT:  },
+// CHECK-NEXT:  "isUsed": true,
 // CHECK-NEXT:  "name": "global_decl",
 // CHECK-NEXT:  "mangledName": "global_decl",
 // CHECK-NEXT:  "type": {
diff --git a/clang/test/Sema/alias-unused-win.cpp b/clang/test/Sema/alias-unused-win.cpp
index 47c96d411751..97d57a3bbd1e 100644
--- a/clang/test/Sema/alias-unused-win.cpp
+++ b/clang/test/Sema/alias-unused-win.cpp
@@ -7,7 +7,7 @@ extern "C" {
 static int f(void) { return 42; } // cxx-warning{{unused function 'f'}}
 int g(void) __attribute__((alias("f")));
 
-static int foo [] = { 42, 0xDEAD }; // cxx-warning{{variable 'foo' is not needed and will not be emitted}}
+static int foo [] = { 42, 0xDEAD };
 extern typeof(foo) bar __attribute__((unused, alias("foo")));
 
 static int __attribute__((overloadable)) f0(int x) { return x; } // expected-warning{{unused function 'f0'}}
diff --git a/clang/test/Sema/alias-unused.cpp b/clang/test/Sema/alias-unused.cpp
index dc8e46f072d7..c0b541c880e5 100644
--- a/clang/test/Sema/alias-unused.cpp
+++ b/clang/test/Sema/alias-unused.cpp
@@ -14,24 +14,26 @@ extern typeof(foo) bar __attribute__((unused, alias("foo")));
 /// We report a warning in C++ mode because the internal linkage `resolver` gets
 /// mangled as it does not have a language linkage. GCC does not mangle
 /// `resolver` or report a warning.
-static int (*resolver(void))(void) { return f; } // expected-warning{{unused function 'resolver'}}
+static int (*resolver(void))(void) { return f; } // cxx-warning{{unused function 'resolver'}}
 int ifunc(void) __attribute__((ifunc("resolver")));
 
-static int __attribute__((overloadable)) f0(int x) { return x; } // expected-warning{{unused function 'f0'}}
+static int __attribute__((overloadable)) f0(int x) { return x; }
 static float __attribute__((overloadable)) f0(float x) { return x; } // expected-warning{{unused function 'f0'}}
 int g0(void) __attribute__((alias("_ZL2f0i")));
 
 #ifdef __cplusplus
-static int f1() { return 42; } // expected-warning{{unused function 'f1'}}
+static int f1() { return 42; }
 int g1(void) __attribute__((alias("_ZL2f1v")));
 }
 
-static int f2(int) { return 42; } // expected-warning{{unused function 'f2'}}
-static int f2() { return 42; } // expected-warning{{unused function 'f2'}}
+/// We demangle alias/ifunc target and mark all found functions as used.
+
+static int f2(int) { return 42; } // cxx-warning{{unused function 'f2'}}
+static int f2() { return 42; }
 int g2() __attribute__((alias("_ZL2f2v")));
 
-static int (*resolver1())() { return f; } // expected-warning{{unused function 'resolver1'}}
-static int (*resolver1(int))() { return f; } // expected-warning{{unused function 'resolver1'}}
+static int (*resolver1())() { return f; } // cxx-warning{{unused function 'resolver1'}}
+static int (*resolver1(int))() { return f; }
 int ifunc1() __attribute__((ifunc("_ZL9resolver1i")));
 
 /// TODO: We should report "unused function" for f3(int).
diff --git a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
index c2f77e3abca0..725ac6bb3812 100644
--- a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
@@ -1136,6 +1136,7 @@ cc_library(
         "//llvm:AllTargetsAsmParsers",
         "//llvm:AllTargetsCodeGens",
         "//llvm:Core",
+        "//llvm:Demangle",
         "//llvm:FrontendHLSL",
         "//llvm:FrontendOpenMP",
         "//llvm:MC",
-- 
GitLab


From 5422eb0b841521908c2fc60bd9c7fdc11ded12a1 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Tue, 16 Apr 2024 11:50:49 -0700
Subject: [PATCH 457/464] [memprof] Add another constructor to MemProfReader
 (#88952)

This patch enables users of MemProfReader to directly supply mappings
from CallStackId to actual call stacks.

Once the users of the current constructor without CSIdMap switch to
the new constructor, we'll have fewer users of:

- IndexedAllocationInfo::CallStack
- IndexedMemProfRecord::CallSites

bringing us one step closer to the removal of these fields in favor
of:

- IndexedAllocationInfo::CSId
- IndexedMemProfRecord::CallSiteIds
---
 llvm/include/llvm/ProfileData/MemProfReader.h |  9 ++++
 llvm/unittests/ProfileData/MemProfTest.cpp    | 41 +++++++++++++++++++
 2 files changed, 50 insertions(+)

diff --git a/llvm/include/llvm/ProfileData/MemProfReader.h b/llvm/include/llvm/ProfileData/MemProfReader.h
index 1f84fefad03e..7fa8af184dc9 100644
--- a/llvm/include/llvm/ProfileData/MemProfReader.h
+++ b/llvm/include/llvm/ProfileData/MemProfReader.h
@@ -98,6 +98,15 @@ public:
       llvm::DenseMap<FrameId, Frame> FrameIdMap,
       llvm::MapVector<GlobalValue::GUID, IndexedMemProfRecord> ProfData);
 
+  // Initialize the MemProfReader with the frame mappings, call stack mappings,
+  // and profile contents.
+  MemProfReader(
+      llvm::DenseMap<FrameId, Frame> FrameIdMap,
+      llvm::DenseMap<CallStackId, llvm::SmallVector<FrameId>> CSIdMap,
+      llvm::MapVector<GlobalValue::GUID, IndexedMemProfRecord> ProfData)
+      : IdToFrame(std::move(FrameIdMap)), CSIdToCallStack(std::move(CSIdMap)),
+        FunctionProfileData(std::move(ProfData)) {}
+
 protected:
   // A helper method to extract the frame from the IdToFrame map.
   const Frame &idToFrame(const FrameId Id) const {
diff --git a/llvm/unittests/ProfileData/MemProfTest.cpp b/llvm/unittests/ProfileData/MemProfTest.cpp
index ab9227e9df88..f596919ed039 100644
--- a/llvm/unittests/ProfileData/MemProfTest.cpp
+++ b/llvm/unittests/ProfileData/MemProfTest.cpp
@@ -436,6 +436,47 @@ TEST(MemProf, BaseMemProfReader) {
               FrameContains("bar", 10U, 2U, false));
 }
 
+TEST(MemProf, BaseMemProfReaderWithCSIdMap) {
+  llvm::DenseMap<FrameId, Frame> FrameIdMap;
+  Frame F1(/*Hash=*/IndexedMemProfRecord::getGUID("foo"), /*LineOffset=*/20,
+           /*Column=*/5, /*IsInlineFrame=*/true);
+  Frame F2(/*Hash=*/IndexedMemProfRecord::getGUID("bar"), /*LineOffset=*/10,
+           /*Column=*/2, /*IsInlineFrame=*/false);
+  FrameIdMap.insert({F1.hash(), F1});
+  FrameIdMap.insert({F2.hash(), F2});
+
+  llvm::DenseMap<CallStackId, llvm::SmallVector<FrameId>> CSIdMap;
+  llvm::SmallVector<FrameId> CallStack = {F1.hash(), F2.hash()};
+  CallStackId CSId = llvm::memprof::hashCallStack(CallStack);
+  CSIdMap.insert({CSId, CallStack});
+
+  llvm::MapVector<llvm::GlobalValue::GUID, IndexedMemProfRecord> ProfData;
+  IndexedMemProfRecord FakeRecord;
+  MemInfoBlock Block;
+  Block.AllocCount = 1U, Block.TotalAccessDensity = 4,
+  Block.TotalLifetime = 200001;
+  FakeRecord.AllocSites.emplace_back(
+      /*CS=*/llvm::SmallVector<FrameId>(),
+      /*CSId=*/llvm::memprof::hashCallStack(CallStack),
+      /*MB=*/Block);
+  ProfData.insert({F1.hash(), FakeRecord});
+
+  MemProfReader Reader(FrameIdMap, CSIdMap, ProfData);
+
+  llvm::SmallVector<MemProfRecord, 1> Records;
+  for (const auto &KeyRecordPair : Reader) {
+    Records.push_back(KeyRecordPair.second);
+  }
+
+  ASSERT_THAT(Records, SizeIs(1));
+  ASSERT_THAT(Records[0].AllocSites, SizeIs(1));
+  ASSERT_THAT(Records[0].AllocSites[0].CallStack, SizeIs(2));
+  EXPECT_THAT(Records[0].AllocSites[0].CallStack[0],
+              FrameContains("foo", 20U, 5U, true));
+  EXPECT_THAT(Records[0].AllocSites[0].CallStack[1],
+              FrameContains("bar", 10U, 2U, false));
+}
+
 TEST(MemProf, IndexedMemProfRecordToMemProfRecord) {
   // Verify that MemProfRecord can be constructed from IndexedMemProfRecord with
   // CallStackIds only.
-- 
GitLab


From c7657cf7d1ee57f9cb9133164536591a1842b43c Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Tue, 16 Apr 2024 14:54:06 -0400
Subject: [PATCH 458/464] [SLP]Keep externally used GEPs as GEPs, if possible
 instead of extractelement.

If the vectorized GEP instruction can be still kept as a scalar GEP,
better to keep it as scalar instead of extractelement. In many cases it
is more profitable.

Metric: size..text

Program                                                                          size..text
                                                                                 results     results0    diff
                        test-suite :: SingleSource/Benchmarks/Misc/oourafft.test    18911.00    19695.00  4.1%
                   test-suite :: SingleSource/Benchmarks/Misc-C++-EH/spirit.test    59987.00    60707.00  1.2%
       test-suite :: External/SPEC/CFP2017speed/638.imagick_s/638.imagick_s.test  1392209.00  1392753.00  0.0%
        test-suite :: External/SPEC/CFP2017rate/538.imagick_r/538.imagick_r.test  1392209.00  1392753.00  0.0%
           test-suite :: External/SPEC/CINT2006/400.perlbench/400.perlbench.test  1087996.00  1088236.00  0.0%
                         test-suite :: MultiSource/Benchmarks/Bullet/bullet.test   309310.00   309342.00  0.0%
             test-suite :: External/SPEC/CINT2017rate/525.x264_r/525.x264_r.test   664661.00   664693.00  0.0%
            test-suite :: External/SPEC/CINT2017speed/625.x264_s/625.x264_s.test   664661.00   664693.00  0.0%
        test-suite :: External/SPEC/CFP2017rate/526.blender_r/526.blender_r.test 12354636.00 12354908.00  0.0%
                  test-suite :: External/SPEC/CFP2006/453.povray/453.povray.test  1152748.00  1152716.00 -0.0%
                       test-suite :: MultiSource/Applications/oggenc/oggenc.test   191787.00   191771.00 -0.0%
                     test-suite :: SingleSource/UnitTests/matrix-types-spec.test   480796.00   480476.00 -0.1%

Misc/oourafft - Extra code gets vectorized
Misc-C++-EH/spirit - same
CFP2017speed/638.imagick_s
CFP2017rate/538.imagick_r - same, extra code gets vectorized
CINT2006/400.perlbench - some extra 4 x ptr stores vectorized
Bullet/bullet - extra 4 x ptr store vectorized
CINT2017rate/525.x264_r
CINT2017speed/625.x264_s - same
CFP2017rate/526.blender_r - extra 8 x float stores (several), some extra
4 x ptr stores
CFP2006/453.povray - 2 x double loads/stores replaced by 4 x double
loads/stores
Applications/oggenc - extra code is vectorized
UnitTests/matrix-types-spec - extra code gets vectorized

Reviewers: RKSimon

Reviewed By: RKSimon

Pull Request: https://github.com/llvm/llvm-project/pull/88877
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    | 50 ++++++++++++++++++-
 .../SLPVectorizer/X86/extract_in_tree_user.ll |  4 +-
 .../SLPVectorizer/X86/geps-non-pow-2.ll       | 17 ++++---
 .../SLPVectorizer/X86/opaque-ptr.ll           | 19 +++----
 .../X86/reorder-reused-masked-gather2.ll      |  2 +-
 .../SLPVectorizer/X86/stacksave-dependence.ll |  4 +-
 6 files changed, 71 insertions(+), 25 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 0cd3ca32933c..7694627c3b04 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1134,6 +1134,7 @@ public:
     MustGather.clear();
     EntryToLastInstruction.clear();
     ExternalUses.clear();
+    ExternalUsesAsGEPs.clear();
     for (auto &Iter : BlocksSchedules) {
       BlockScheduling *BS = Iter.second.get();
       BS->clear();
@@ -3154,6 +3155,10 @@ private:
   /// after vectorization.
   UserList ExternalUses;
 
+  /// A list of GEPs which can be reaplced by scalar GEPs instead of
+  /// extractelement instructions.
+  SmallPtrSet<Value *, 4> ExternalUsesAsGEPs;
+
   /// Values used only by @llvm.assume calls.
   SmallPtrSet<const Value *, 32> EphValues;
 
@@ -5541,6 +5546,7 @@ void BoUpSLP::buildExternalUses(
                           << FoundLane << " from " << *Scalar << ".\n");
         ScalarToExtUses.try_emplace(Scalar, ExternalUses.size());
         ExternalUses.emplace_back(Scalar, nullptr, FoundLane);
+        continue;
       }
       for (User *U : Scalar->users()) {
         LLVM_DEBUG(dbgs() << "SLP: Checking user:" << *U << ".\n");
@@ -9925,6 +9931,7 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
   SmallVector<APInt> DemandedElts;
   SmallDenseSet<Value *, 4> UsedInserts;
   DenseSet<std::pair<const TreeEntry *, Type *>> VectorCasts;
+  std::optional<DenseMap<Value *, unsigned>> ValueToExtUses;
   for (ExternalUser &EU : ExternalUses) {
     // We only add extract cost once for the same scalar.
     if (!isa_and_nonnull<InsertElementInst>(EU.User) &&
@@ -10033,12 +10040,40 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
         }
       }
     }
+    // Leave the GEPs as is, they are free in most cases and better to keep them
+    // as GEPs.
+    TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
+    if (auto *GEP = dyn_cast<GetElementPtrInst>(EU.Scalar)) {
+      if (!ValueToExtUses) {
+        ValueToExtUses.emplace();
+        for_each(enumerate(ExternalUses), [&](const auto &P) {
+          ValueToExtUses->try_emplace(P.value().Scalar, P.index());
+        });
+      }
+      // Can use original GEP, if no operands vectorized or they are marked as
+      // externally used already.
+      bool CanBeUsedAsGEP = all_of(GEP->operands(), [&](Value *V) {
+        if (!getTreeEntry(V))
+          return true;
+        auto It = ValueToExtUses->find(V);
+        if (It != ValueToExtUses->end()) {
+          // Replace all uses to avoid compiler crash.
+          ExternalUses[It->second].User = nullptr;
+          return true;
+        }
+        return false;
+      });
+      if (CanBeUsedAsGEP) {
+        ExtractCost += TTI->getInstructionCost(GEP, CostKind);
+        ExternalUsesAsGEPs.insert(EU.Scalar);
+        continue;
+      }
+    }
 
     // If we plan to rewrite the tree in a smaller type, we will need to sign
     // extend the extracted value back to the original type. Here, we account
     // for the extract and the added cost of the sign extend if needed.
     auto *VecTy = FixedVectorType::get(EU.Scalar->getType(), BundleWidth);
-    TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
     auto It = MinBWs.find(getTreeEntry(EU.Scalar));
     if (It != MinBWs.end()) {
       auto *MinTy = IntegerType::get(F->getContext(), It->second.first);
@@ -13161,6 +13196,8 @@ Value *BoUpSLP::vectorizeTree(
       if (Scalar->getType() != Vec->getType()) {
         Value *Ex = nullptr;
         Value *ExV = nullptr;
+        auto *GEP = dyn_cast<GetElementPtrInst>(Scalar);
+        bool ReplaceGEP = GEP && ExternalUsesAsGEPs.contains(GEP);
         auto It = ScalarToEEs.find(Scalar);
         if (It != ScalarToEEs.end()) {
           // No need to emit many extracts, just move the only one in the
@@ -13186,6 +13223,15 @@ Value *BoUpSLP::vectorizeTree(
             if (const TreeEntry *ETE = getTreeEntry(V))
               V = ETE->VectorizedValue;
             Ex = Builder.CreateExtractElement(V, ES->getIndexOperand());
+          } else if (ReplaceGEP) {
+            // Leave the GEPs as is, they are free in most cases and better to
+            // keep them as GEPs.
+            auto *CloneGEP = GEP->clone();
+            CloneGEP->insertBefore(*Builder.GetInsertBlock(),
+                                   Builder.GetInsertPoint());
+            if (GEP->hasName())
+              CloneGEP->takeName(GEP);
+            Ex = CloneGEP;
           } else {
             Ex = Builder.CreateExtractElement(Vec, Lane);
           }
@@ -13224,6 +13270,8 @@ Value *BoUpSLP::vectorizeTree(
       assert((ExternallyUsedValues.count(Scalar) ||
               any_of(Scalar->users(),
                      [&](llvm::User *U) {
+                       if (ExternalUsesAsGEPs.contains(U))
+                         return true;
                        TreeEntry *UseEntry = getTreeEntry(U);
                        return UseEntry &&
                               (UseEntry->State == TreeEntry::Vectorize ||
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extract_in_tree_user.ll b/llvm/test/Transforms/SLPVectorizer/X86/extract_in_tree_user.ll
index 096f57d100a5..c600d75ed1e8 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/extract_in_tree_user.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/extract_in_tree_user.ll
@@ -13,7 +13,7 @@ define i32 @fn1() {
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x ptr> [[TMP1]], <2 x ptr> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i64, <2 x ptr> [[TMP2]], <2 x i64> <i64 11, i64 56>
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x ptr> [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP0]], i64 11
 ; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint <2 x ptr> [[TMP3]] to <2 x i64>
 ; CHECK-NEXT:    store <2 x i64> [[TMP5]], ptr [[TMP4]], align 8
 ; CHECK-NEXT:    ret i32 undef
@@ -92,7 +92,7 @@ define void @externally_used_ptrs() {
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP0]], i32 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x ptr> [[TMP1]], <2 x ptr> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i64, <2 x ptr> [[TMP2]], <2 x i64> <i64 56, i64 11>
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x ptr> [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[TMP0]], i64 11
 ; CHECK-NEXT:    [[TMP5:%.*]] = ptrtoint <2 x ptr> [[TMP3]] to <2 x i64>
 ; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x i64>, ptr [[TMP4]], align 8
 ; CHECK-NEXT:    [[TMP7:%.*]] = add <2 x i64> [[TMP5]], [[TMP6]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/geps-non-pow-2.ll b/llvm/test/Transforms/SLPVectorizer/X86/geps-non-pow-2.ll
index aa6797435830..e459cd8c6955 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/geps-non-pow-2.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/geps-non-pow-2.ll
@@ -13,25 +13,26 @@ define dso_local i32 @g() local_unnamed_addr {
 ; CHECK:       while.body:
 ; CHECK-NEXT:    [[C_022:%.*]] = phi ptr [ [[C_022_BE:%.*]], [[WHILE_BODY_BACKEDGE:%.*]] ], [ undef, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[TMP1:%.*]] = phi <2 x ptr> [ [[TMP14:%.*]], [[WHILE_BODY_BACKEDGE]] ], [ undef, [[ENTRY]] ]
-; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, ptr [[C_022]], i64 1
-; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[C_022]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x ptr> [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[TMP9]] to i64
 ; CHECK-NEXT:    [[TMP3:%.*]] = trunc i64 [[TMP2]] to i32
+; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds i32, ptr [[C_022]], i64 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i32, <2 x ptr> [[TMP1]], <2 x i64> <i64 1, i64 1>
 ; CHECK-NEXT:    switch i32 [[TMP3]], label [[WHILE_BODY_BACKEDGE]] [
-; CHECK-NEXT:    i32 2, label [[SW_BB:%.*]]
-; CHECK-NEXT:    i32 4, label [[SW_BB6:%.*]]
+; CHECK-NEXT:      i32 2, label [[SW_BB:%.*]]
+; CHECK-NEXT:      i32 4, label [[SW_BB6:%.*]]
 ; CHECK-NEXT:    ]
 ; CHECK:       sw.bb:
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x ptr> [[TMP4]], i32 0
 ; CHECK-NEXT:    [[TMP6:%.*]] = ptrtoint ptr [[TMP5]] to i64
 ; CHECK-NEXT:    [[TMP7:%.*]] = trunc i64 [[TMP6]] to i32
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i32, <2 x ptr> [[TMP1]], <2 x i64> <i64 2, i64 2>
-; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x ptr> [[TMP4]], i32 1
-; CHECK-NEXT:    store i32 [[TMP7]], ptr [[TMP9]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds i32, ptr [[C_022]], i64 2
+; CHECK-NEXT:    store i32 [[TMP7]], ptr [[INCDEC_PTR1]], align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr i32, <2 x ptr> [[TMP1]], <2 x i64> <i64 2, i64 2>
 ; CHECK-NEXT:    br label [[WHILE_BODY_BACKEDGE]]
 ; CHECK:       sw.bb6:
 ; CHECK-NEXT:    [[INCDEC_PTR8:%.*]] = getelementptr inbounds i32, ptr [[C_022]], i64 2
+; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, ptr [[TMP9]], i64 1
 ; CHECK-NEXT:    [[TMP10:%.*]] = ptrtoint ptr [[INCDEC_PTR]] to i64
 ; CHECK-NEXT:    [[TMP11:%.*]] = trunc i64 [[TMP10]] to i32
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr i32, <2 x ptr> [[TMP1]], <2 x i64> <i64 2, i64 2>
@@ -39,7 +40,7 @@ define dso_local i32 @g() local_unnamed_addr {
 ; CHECK-NEXT:    store i32 [[TMP11]], ptr [[TMP13]], align 4
 ; CHECK-NEXT:    br label [[WHILE_BODY_BACKEDGE]]
 ; CHECK:       while.body.backedge:
-; CHECK-NEXT:    [[C_022_BE]] = phi ptr [ [[INCDEC_PTR]], [[WHILE_BODY]] ], [ [[INCDEC_PTR8]], [[SW_BB6]] ], [ [[INCDEC_PTR5]], [[SW_BB]] ]
+; CHECK-NEXT:    [[C_022_BE]] = phi ptr [ [[INCDEC_PTR1]], [[WHILE_BODY]] ], [ [[INCDEC_PTR8]], [[SW_BB6]] ], [ [[INCDEC_PTR5]], [[SW_BB]] ]
 ; CHECK-NEXT:    [[TMP14]] = phi <2 x ptr> [ [[TMP4]], [[WHILE_BODY]] ], [ [[TMP12]], [[SW_BB6]] ], [ [[TMP8]], [[SW_BB]] ]
 ; CHECK-NEXT:    br label [[WHILE_BODY]]
 ; CHECK:       while.end:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/opaque-ptr.ll b/llvm/test/Transforms/SLPVectorizer/X86/opaque-ptr.ll
index 3801fa5c787b..c40be9690cce 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/opaque-ptr.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/opaque-ptr.ll
@@ -52,17 +52,14 @@ define void @test(ptr %r, ptr %p, ptr %q) #0 {
 
 define void @test2(ptr %a, ptr %b) {
 ; CHECK-LABEL: @test2(
-; CHECK-NEXT:    [[A1:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 1
-; CHECK-NEXT:    [[A2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 2
-; CHECK-NEXT:    [[I1:%.*]] = ptrtoint ptr [[A1]] to i64
-; CHECK-NEXT:    [[B3:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 3
-; CHECK-NEXT:    [[I2:%.*]] = ptrtoint ptr [[B3]] to i64
-; CHECK-NEXT:    [[V1:%.*]] = load i64, ptr [[A1]], align 8
-; CHECK-NEXT:    [[V2:%.*]] = load i64, ptr [[A2]], align 8
-; CHECK-NEXT:    [[ADD1:%.*]] = add i64 [[I1]], [[V1]]
-; CHECK-NEXT:    [[ADD2:%.*]] = add i64 [[I2]], [[V2]]
-; CHECK-NEXT:    store i64 [[ADD1]], ptr [[A1]], align 8
-; CHECK-NEXT:    store i64 [[ADD2]], ptr [[A2]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x ptr> poison, ptr [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x ptr> [[TMP1]], ptr [[B:%.*]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i64, <2 x ptr> [[TMP2]], <2 x i64> <i64 1, i64 3>
+; CHECK-NEXT:    [[A1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 1
+; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint <2 x ptr> [[TMP3]] to <2 x i64>
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i64>, ptr [[A1]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = add <2 x i64> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    store <2 x i64> [[TMP6]], ptr [[A1]], align 8
 ; CHECK-NEXT:    ret void
 ;
   %a1 = getelementptr inbounds i64, ptr %a, i64 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reorder-reused-masked-gather2.ll b/llvm/test/Transforms/SLPVectorizer/X86/reorder-reused-masked-gather2.ll
index ddc2a1b81904..30f328293cda 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reorder-reused-masked-gather2.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reorder-reused-masked-gather2.ll
@@ -9,7 +9,7 @@ define void @"foo"(ptr addrspace(1) %0, ptr addrspace(1) %1) #0 {
 ; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x ptr addrspace(1)> poison, ptr addrspace(1) [[TMP0:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x ptr addrspace(1)> [[TMP3]], <4 x ptr addrspace(1)> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i8, <4 x ptr addrspace(1)> [[TMP4]], <4 x i64> <i64 8, i64 12, i64 28, i64 24>
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x ptr addrspace(1)> [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[TMP0]], i64 8
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[TMP1:%.*]], i64 8
 ; CHECK-NEXT:    [[TMP8:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p1(<4 x ptr addrspace(1)> [[TMP5]], i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x float> poison)
 ; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <4 x float> [[TMP8]], <4 x float> poison, <8 x i32> <i32 0, i32 3, i32 0, i32 3, i32 2, i32 1, i32 2, i32 1>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/stacksave-dependence.ll b/llvm/test/Transforms/SLPVectorizer/X86/stacksave-dependence.ll
index 0125e5fab089..e93c5244dfbe 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/stacksave-dependence.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/stacksave-dependence.ll
@@ -35,7 +35,7 @@ define void @allocas(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x ptr> poison, ptr [[V1]], i32 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x ptr> [[TMP1]], ptr [[V2]], i32 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i8, <2 x ptr> [[TMP2]], <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x ptr> [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[V1]], i32 1
 ; CHECK-NEXT:    store ptr [[TMP4]], ptr [[A:%.*]], align 8
 ; CHECK-NEXT:    store <2 x ptr> [[TMP3]], ptr [[B:%.*]], align 8
 ; CHECK-NEXT:    ret void
@@ -127,7 +127,7 @@ define void @stacksave2(ptr %a, ptr %b, ptr %c) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x ptr> poison, ptr [[V1]], i32 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x ptr> [[TMP1]], ptr [[V2]], i32 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i8, <2 x ptr> [[TMP2]], <2 x i32> <i32 1, i32 1>
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x ptr> [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[V1]], i32 1
 ; CHECK-NEXT:    store ptr [[TMP4]], ptr [[A:%.*]], align 8
 ; CHECK-NEXT:    call void @use(ptr inalloca(i8) [[V2]]) #[[ATTR5:[0-9]+]]
 ; CHECK-NEXT:    call void @llvm.stackrestore.p0(ptr [[STACK]])
-- 
GitLab


From 7d4e8c1f3bbfe976f4871c9cf953f76d771b0eda Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev@outlook.com>
Date: Tue, 16 Apr 2024 14:55:41 -0400
Subject: [PATCH 459/464] [SLP]Attempt to vectorize long stores, if short one
 failed.

We can try to vectorize long store sequences, if short ones were
unsuccessful because of the non-profitable vectorization. It should not
increase compile time significantly (stores are sorted already,
complexity is n x log n), but vectorize extra code.

Metric: size..text

Program                                                                         size..text
                                                                                results     results0    diff
         test-suite :: External/SPEC/CINT2006/400.perlbench/400.perlbench.test  1088012.00  1088236.00  0.0%
                  test-suite :: SingleSource/UnitTests/matrix-types-spec.test   480396.00   480476.00  0.0%
          test-suite :: External/SPEC/CINT2017rate/525.x264_r/525.x264_r.test   664613.00   664661.00  0.0%
         test-suite :: External/SPEC/CINT2017speed/625.x264_s/625.x264_s.test   664613.00   664661.00  0.0%
        test-suite :: External/SPEC/CFP2017rate/510.parest_r/510.parest_r.test  2041105.00  2040961.00 -0.0%
                 test-suite :: MultiSource/Applications/JM/lencod/lencod.test   836563.00   836387.00 -0.0%
                 test-suite :: MultiSource/Benchmarks/7zip/7zip-benchmark.test  1035100.00  1032140.00 -0.3%

In all benchmarks extra code gets vectorized

Reviewers: RKSimon

Reviewed By: RKSimon

Pull Request: https://github.com/llvm/llvm-project/pull/88563
---
 .../Transforms/Vectorize/SLPVectorizer.cpp    | 81 ++++++++++++-------
 .../Transforms/SLPVectorizer/X86/pr46983.ll   | 46 +++--------
 2 files changed, 62 insertions(+), 65 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 7694627c3b04..8ae38550d309 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -15237,39 +15237,60 @@ bool SLPVectorizerPass::vectorizeStores(ArrayRef<StoreInst *> Stores,
         Size *= 2;
       });
       unsigned StartIdx = 0;
-      for (unsigned Size : CandidateVFs) {
-        for (unsigned Cnt = StartIdx, E = Operands.size(); Cnt + Size <= E;) {
-          ArrayRef<Value *> Slice = ArrayRef(Operands).slice(Cnt, Size);
-          assert(
-              all_of(
-                  Slice,
-                  [&](Value *V) {
-                    return cast<StoreInst>(V)->getValueOperand()->getType() ==
-                           cast<StoreInst>(Slice.front())
-                               ->getValueOperand()
-                               ->getType();
-                  }) &&
-              "Expected all operands of same type.");
-          if (!VectorizedStores.count(Slice.front()) &&
-              !VectorizedStores.count(Slice.back()) &&
-              TriedSequences.insert(std::make_pair(Slice.front(), Slice.back()))
-                  .second &&
-              vectorizeStoreChain(Slice, R, Cnt, MinVF)) {
-            // Mark the vectorized stores so that we don't vectorize them again.
-            VectorizedStores.insert(Slice.begin(), Slice.end());
-            Changed = true;
-            // If we vectorized initial block, no need to try to vectorize it
-            // again.
-            if (Cnt == StartIdx)
-              StartIdx += Size;
-            Cnt += Size;
-            continue;
+      unsigned Repeat = 0;
+      constexpr unsigned MaxAttempts = 2;
+      while (true) {
+        ++Repeat;
+        for (unsigned Size : CandidateVFs) {
+          for (unsigned Cnt = StartIdx, E = Operands.size(); Cnt + Size <= E;) {
+            ArrayRef<Value *> Slice = ArrayRef(Operands).slice(Cnt, Size);
+            assert(
+                all_of(
+                    Slice,
+                    [&](Value *V) {
+                      return cast<StoreInst>(V)->getValueOperand()->getType() ==
+                             cast<StoreInst>(Slice.front())
+                                 ->getValueOperand()
+                                 ->getType();
+                    }) &&
+                "Expected all operands of same type.");
+            if (!VectorizedStores.count(Slice.front()) &&
+                !VectorizedStores.count(Slice.back()) &&
+                TriedSequences
+                    .insert(std::make_pair(Slice.front(), Slice.back()))
+                    .second &&
+                vectorizeStoreChain(Slice, R, Cnt, MinVF)) {
+              // Mark the vectorized stores so that we don't vectorize them
+              // again.
+              VectorizedStores.insert(Slice.begin(), Slice.end());
+              Changed = true;
+              // If we vectorized initial block, no need to try to vectorize
+              // it again.
+              if (Cnt == StartIdx)
+                StartIdx += Size;
+              Cnt += Size;
+              continue;
+            }
+            ++Cnt;
+          }
+          // Check if the whole array was vectorized already - exit.
+          if (StartIdx >= Operands.size()) {
+            Repeat = MaxAttempts;
+            break;
           }
-          ++Cnt;
         }
-        // Check if the whole array was vectorized already - exit.
-        if (StartIdx >= Operands.size())
+        // Check if tried all attempts or no need for the last attempts at all.
+        if (Repeat >= MaxAttempts)
           break;
+        const unsigned MaxTotalNum = bit_floor(Operands.size() - StartIdx);
+        if (MaxVF >= MaxTotalNum)
+          break;
+        // Last attempt to vectorize max number of elements, if all previous
+        // attempts were unsuccessful because of the cost issues.
+        CandidateVFs.clear();
+        for (unsigned Size = MaxTotalNum; Size > MaxVF; Size /= 2) {
+          CandidateVFs.push_back(Size);
+        }
       }
     }
   };
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll
index 75505f632a43..3deab0975ce7 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll
@@ -100,41 +100,17 @@ define void @store_i8(ptr nocapture %0, i32 %1, i32 %2) {
 define void @store_i64(ptr nocapture %0, i32 %1, i32 %2) {
 ; SSE-LABEL: @store_i64(
 ; SSE-NEXT:    [[TMP4:%.*]] = zext i32 [[TMP1:%.*]] to i64
-; SSE-NEXT:    [[TMP5:%.*]] = load i64, ptr [[TMP0:%.*]], align 8, !tbaa [[TBAA5:![0-9]+]]
-; SSE-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], [[TMP4]]
-; SSE-NEXT:    [[TMP7:%.*]] = lshr i64 [[TMP6]], 15
-; SSE-NEXT:    [[TMP8:%.*]] = trunc i64 [[TMP7]] to i32
-; SSE-NEXT:    [[TMP9:%.*]] = icmp ult i32 [[TMP8]], 255
-; SSE-NEXT:    [[TMP10:%.*]] = and i64 [[TMP7]], 4294967295
-; SSE-NEXT:    [[TMP11:%.*]] = select i1 [[TMP9]], i64 [[TMP10]], i64 255
-; SSE-NEXT:    store i64 [[TMP11]], ptr [[TMP0]], align 8, !tbaa [[TBAA5]]
-; SSE-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 8
-; SSE-NEXT:    [[TMP13:%.*]] = load i64, ptr [[TMP12]], align 8, !tbaa [[TBAA5]]
-; SSE-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP13]], [[TMP4]]
-; SSE-NEXT:    [[TMP15:%.*]] = lshr i64 [[TMP14]], 15
-; SSE-NEXT:    [[TMP16:%.*]] = trunc i64 [[TMP15]] to i32
-; SSE-NEXT:    [[TMP17:%.*]] = icmp ult i32 [[TMP16]], 255
-; SSE-NEXT:    [[TMP18:%.*]] = and i64 [[TMP15]], 4294967295
-; SSE-NEXT:    [[TMP19:%.*]] = select i1 [[TMP17]], i64 [[TMP18]], i64 255
-; SSE-NEXT:    store i64 [[TMP19]], ptr [[TMP12]], align 8, !tbaa [[TBAA5]]
-; SSE-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 16
-; SSE-NEXT:    [[TMP21:%.*]] = load i64, ptr [[TMP20]], align 8, !tbaa [[TBAA5]]
-; SSE-NEXT:    [[TMP22:%.*]] = mul i64 [[TMP21]], [[TMP4]]
-; SSE-NEXT:    [[TMP23:%.*]] = lshr i64 [[TMP22]], 15
-; SSE-NEXT:    [[TMP24:%.*]] = trunc i64 [[TMP23]] to i32
-; SSE-NEXT:    [[TMP25:%.*]] = icmp ult i32 [[TMP24]], 255
-; SSE-NEXT:    [[TMP26:%.*]] = and i64 [[TMP23]], 4294967295
-; SSE-NEXT:    [[TMP27:%.*]] = select i1 [[TMP25]], i64 [[TMP26]], i64 255
-; SSE-NEXT:    store i64 [[TMP27]], ptr [[TMP20]], align 8, !tbaa [[TBAA5]]
-; SSE-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i64 24
-; SSE-NEXT:    [[TMP29:%.*]] = load i64, ptr [[TMP28]], align 8, !tbaa [[TBAA5]]
-; SSE-NEXT:    [[TMP30:%.*]] = mul i64 [[TMP29]], [[TMP4]]
-; SSE-NEXT:    [[TMP31:%.*]] = lshr i64 [[TMP30]], 15
-; SSE-NEXT:    [[TMP32:%.*]] = trunc i64 [[TMP31]] to i32
-; SSE-NEXT:    [[TMP33:%.*]] = icmp ult i32 [[TMP32]], 255
-; SSE-NEXT:    [[TMP34:%.*]] = and i64 [[TMP31]], 4294967295
-; SSE-NEXT:    [[TMP35:%.*]] = select i1 [[TMP33]], i64 [[TMP34]], i64 255
-; SSE-NEXT:    store i64 [[TMP35]], ptr [[TMP28]], align 8, !tbaa [[TBAA5]]
+; SSE-NEXT:    [[TMP5:%.*]] = load <4 x i64>, ptr [[TMP0:%.*]], align 8, !tbaa [[TBAA5:![0-9]+]]
+; SSE-NEXT:    [[TMP6:%.*]] = insertelement <4 x i64> poison, i64 [[TMP4]], i64 0
+; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP6]], <4 x i64> poison, <4 x i32> zeroinitializer
+; SSE-NEXT:    [[TMP8:%.*]] = mul <4 x i64> [[TMP5]], [[TMP7]]
+; SSE-NEXT:    [[TMP9:%.*]] = lshr <4 x i64> [[TMP8]], <i64 15, i64 15, i64 15, i64 15>
+; SSE-NEXT:    [[TMP10:%.*]] = trunc <4 x i64> [[TMP9]] to <4 x i32>
+; SSE-NEXT:    [[TMP11:%.*]] = icmp ult <4 x i32> [[TMP10]], <i32 255, i32 255, i32 255, i32 255>
+; SSE-NEXT:    [[TMP12:%.*]] = trunc <4 x i64> [[TMP9]] to <4 x i32>
+; SSE-NEXT:    [[TMP13:%.*]] = select <4 x i1> [[TMP11]], <4 x i32> [[TMP12]], <4 x i32> <i32 255, i32 255, i32 255, i32 255>
+; SSE-NEXT:    [[TMP14:%.*]] = zext <4 x i32> [[TMP13]] to <4 x i64>
+; SSE-NEXT:    store <4 x i64> [[TMP14]], ptr [[TMP0]], align 8, !tbaa [[TBAA5]]
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @store_i64(
-- 
GitLab


From 3eff86f82cb59d7dfc88e0cc3d8df8282f24f028 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Tue, 16 Apr 2024 18:59:12 +0000
Subject: [PATCH 460/464] [gn build] Port 9ec8c961664d

---
 .../utils/gn/secondary/clang/lib/Analysis/FlowSensitive/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/clang/lib/Analysis/FlowSensitive/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Analysis/FlowSensitive/BUILD.gn
index 04f20211b3c7..22433459a787 100644
--- a/llvm/utils/gn/secondary/clang/lib/Analysis/FlowSensitive/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Analysis/FlowSensitive/BUILD.gn
@@ -23,6 +23,7 @@ static_library("FlowSensitive") {
     target_gen_dir,
   ]
   sources = [
+    "ASTOps.cpp",
     "AdornedCFG.cpp",
     "Arena.cpp",
     "DataflowAnalysisContext.cpp",
-- 
GitLab


From b0ddbfb77d15e00e08fc36f6ccd8a4fecde465d1 Mon Sep 17 00:00:00 2001
From: Justin Bogner <mail@justinbogner.com>
Date: Tue, 16 Apr 2024 12:09:32 -0700
Subject: [PATCH 461/464] [clang][SPIR-V] Set AS for the SPIR-V logical triple
 (#88939)

This was missed in #88455, causing most of the .hlsl to SPIR-V tests to
fail (such as clang\test\Driver\hlsl-lang-targets-spirv.hlsl)
---
 clang/lib/Basic/Targets/SPIR.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/lib/Basic/Targets/SPIR.h b/clang/lib/Basic/Targets/SPIR.h
index 9a4a8b501460..44265445ff00 100644
--- a/clang/lib/Basic/Targets/SPIR.h
+++ b/clang/lib/Basic/Targets/SPIR.h
@@ -315,7 +315,7 @@ public:
     // SPIR-V IDs are represented with a single 32-bit word.
     SizeType = TargetInfo::UnsignedInt;
     resetDataLayout("e-i64:64-v16:16-v24:32-v32:32-v48:64-"
-                    "v96:128-v192:256-v256:256-v512:512-v1024:1024");
+                    "v96:128-v192:256-v256:256-v512:512-v1024:1024-G1");
   }
 
   void getTargetDefines(const LangOptions &Opts,
-- 
GitLab


From c9731a3dccd381849bfede5e09290c0574efa248 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Tue, 16 Apr 2024 12:10:05 -0700
Subject: [PATCH 462/464] [mlir] Fix a warning about an extraneous semicolon

This patch fixes:

  mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp:58:2: error: extra ';'
  outside of a function is incompatible with C++98
  [-Werror,-Wc++98-compat-extra-semi]
---
 mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
index 621986c54d49..530c50ef74f7 100644
--- a/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
+++ b/mlir/lib/Dialect/XeGPU/IR/XeGPUOps.cpp
@@ -55,7 +55,7 @@ static int64_t getRankOf(Value val) {
   if (auto ty = llvm::dyn_cast<ShapedType>(type))
     return ty.getRank();
   return 0;
-};
+}
 
 static bool isReadHintOrNone(const CachePolicyAttr &attr) {
   if (!attr)
-- 
GitLab


From 0a789ea8a829da345e46d8224d73b2ddaba6969f Mon Sep 17 00:00:00 2001
From: erichkeane <ekeane@nvidia.com>
Date: Tue, 16 Apr 2024 12:12:25 -0700
Subject: [PATCH 463/464] Fix test from #83124 and #88902

This just replaces an '#include<new>' with a declaration of array
placement new.
---
 clang/test/SemaCXX/PR41441.cpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/clang/test/SemaCXX/PR41441.cpp b/clang/test/SemaCXX/PR41441.cpp
index 0b012b33fce3..d0f2917e52f2 100644
--- a/clang/test/SemaCXX/PR41441.cpp
+++ b/clang/test/SemaCXX/PR41441.cpp
@@ -1,6 +1,9 @@
 // RUN: %clang --target=x86_64-pc-linux -S -fno-discard-value-names -emit-llvm -o - %s | FileCheck %s
 
-#include <new>
+namespace std {
+  using size_t = decltype(sizeof(int));
+};
+void* operator new[](std::size_t, void*) noexcept;
 
 // CHECK: call void @llvm.memset.p0.i64(ptr align 1 %x, i8 0, i64 8, i1 false)
 // CHECK: call void @llvm.memset.p0.i64(ptr align 16 %x, i8 0, i64 32, i1 false)
-- 
GitLab


From 9a0a28f8384b2cb534953df33bf124f01f0e0d0e Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Tue, 16 Apr 2024 14:19:12 -0500
Subject: [PATCH 464/464] [Libomptarget] Rework Record & Replay to be a plugin
 member (#88928)

Summary:
Previously, the R&R support was global state initialized by a global
constructor. This is bad because it prevents us from adequately
constraining the lifetime of the library. Additionally, we want to
minimize the amount of global state floating around.

This patch moves the R&R support into a plugin member like everything
else. This means there will be multiple copies of the R&R implementation
floating around, but this was already the case given the fact that we
currently handle everything with dynamic libraries.
---
 .../common/include/PluginInterface.h          | 11 ++++++
 .../common/src/PluginInterface.cpp            | 39 ++++++++++++-------
 2 files changed, 35 insertions(+), 15 deletions(-)

diff --git a/openmp/libomptarget/plugins-nextgen/common/include/PluginInterface.h b/openmp/libomptarget/plugins-nextgen/common/include/PluginInterface.h
index 79e8464bfda5..7f05464f36c1 100644
--- a/openmp/libomptarget/plugins-nextgen/common/include/PluginInterface.h
+++ b/openmp/libomptarget/plugins-nextgen/common/include/PluginInterface.h
@@ -45,6 +45,8 @@
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TargetParser/Triple.h"
 
+struct RecordReplayTy;
+
 namespace llvm {
 namespace omp {
 namespace target {
@@ -1031,6 +1033,12 @@ struct GenericPluginTy {
     return *RPCServer;
   }
 
+  /// Get a reference to the R&R interface for this plugin.
+  RecordReplayTy &getRecordAndReplay() const {
+    assert(RecordReplay && "R&R not initialized");
+    return *RecordReplay;
+  }
+
   /// Get the OpenMP requires flags set for this plugin.
   int64_t getRequiresFlags() const { return RequiresFlags; }
 
@@ -1220,6 +1228,9 @@ private:
 
   /// The interface between the plugin and the GPU for host services.
   RPCServerTy *RPCServer;
+
+  /// The interface into the record-and-replay functionality.
+  RecordReplayTy *RecordReplay;
 };
 
 namespace Plugin {
diff --git a/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp b/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp
index b5f3c45c835f..6df9798f12e3 100644
--- a/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp
@@ -362,8 +362,6 @@ public:
   }
 };
 
-static RecordReplayTy RecordReplay;
-
 // Extract the mapping of host function pointers to device function pointers
 // from the entry table. Functions marked as 'indirect' in OpenMP will have
 // offloading entries generated for them which map the host's function pointer
@@ -473,7 +471,8 @@ GenericKernelTy::getKernelLaunchEnvironment(
   // Ctor/Dtor have no arguments, replaying uses the original kernel launch
   // environment. Older versions of the compiler do not generate a kernel
   // launch environment.
-  if (isCtorOrDtor() || RecordReplay.isReplaying() ||
+  if (isCtorOrDtor() ||
+      GenericDevice.Plugin.getRecordAndReplay().isReplaying() ||
       Version < OMP_KERNEL_ARG_MIN_VERSION_WITH_DYN_PTR)
     return nullptr;
 
@@ -562,6 +561,7 @@ Error GenericKernelTy::launch(GenericDeviceTy &GenericDevice, void **ArgPtrs,
 
   // Record the kernel description after we modified the argument count and num
   // blocks/threads.
+  RecordReplayTy &RecordReplay = GenericDevice.Plugin.getRecordAndReplay();
   if (RecordReplay.isRecording()) {
     RecordReplay.saveImage(getName(), getImage());
     RecordReplay.saveKernelInput(getName(), getImage());
@@ -839,9 +839,6 @@ Error GenericDeviceTy::deinit(GenericPluginTy &Plugin) {
     delete MemoryManager;
   MemoryManager = nullptr;
 
-  if (RecordReplay.isRecordingOrReplaying())
-    RecordReplay.deinit();
-
   if (RPCServer)
     if (auto Err = RPCServer->deinitDevice(*this))
       return Err;
@@ -858,6 +855,7 @@ Error GenericDeviceTy::deinit(GenericPluginTy &Plugin) {
 
   return deinitImpl();
 }
+
 Expected<DeviceImageTy *>
 GenericDeviceTy::loadBinary(GenericPluginTy &Plugin,
                             const __tgt_device_image *InputTgtImage) {
@@ -892,7 +890,8 @@ GenericDeviceTy::loadBinary(GenericPluginTy &Plugin,
     return std::move(Err);
 
   // Setup the global device memory pool if needed.
-  if (!RecordReplay.isReplaying() && shouldSetupDeviceMemoryPool()) {
+  if (!Plugin.getRecordAndReplay().isReplaying() &&
+      shouldSetupDeviceMemoryPool()) {
     uint64_t HeapSize;
     auto SizeOrErr = getDeviceHeapSize(HeapSize);
     if (SizeOrErr) {
@@ -1307,8 +1306,8 @@ Expected<void *> GenericDeviceTy::dataAlloc(int64_t Size, void *HostPtr,
                                             TargetAllocTy Kind) {
   void *Alloc = nullptr;
 
-  if (RecordReplay.isRecordingOrReplaying())
-    return RecordReplay.alloc(Size);
+  if (Plugin.getRecordAndReplay().isRecordingOrReplaying())
+    return Plugin.getRecordAndReplay().alloc(Size);
 
   switch (Kind) {
   case TARGET_ALLOC_DEFAULT:
@@ -1344,7 +1343,7 @@ Expected<void *> GenericDeviceTy::dataAlloc(int64_t Size, void *HostPtr,
 
 Error GenericDeviceTy::dataDelete(void *TgtPtr, TargetAllocTy Kind) {
   // Free is a noop when recording or replaying.
-  if (RecordReplay.isRecordingOrReplaying())
+  if (Plugin.getRecordAndReplay().isRecordingOrReplaying())
     return Plugin::success();
 
   int Res;
@@ -1396,6 +1395,7 @@ Error GenericDeviceTy::launchKernel(void *EntryPtr, void **ArgPtrs,
                                     ptrdiff_t *ArgOffsets,
                                     KernelArgsTy &KernelArgs,
                                     __tgt_async_info *AsyncInfo) {
+  RecordReplayTy &RecordReplay = Plugin.getRecordAndReplay();
   AsyncInfoWrapperTy AsyncInfoWrapper(
       *this, RecordReplay.isRecordingOrReplaying() ? nullptr : AsyncInfo);
 
@@ -1495,6 +1495,9 @@ Error GenericPluginTy::init() {
   RPCServer = new RPCServerTy(*this);
   assert(RPCServer && "Invalid RPC server");
 
+  RecordReplay = new RecordReplayTy();
+  assert(RecordReplay && "Invalid Record and Replay handler");
+
   return Plugin::success();
 }
 
@@ -1508,6 +1511,9 @@ Error GenericPluginTy::deinit() {
     assert(!Devices[DeviceId] && "Device was not deinitialized");
   }
 
+  if (RecordReplay && RecordReplay->isRecordingOrReplaying())
+    RecordReplay->deinit();
+
   // There is no global handler if no device is available.
   if (GlobalHandler)
     delete GlobalHandler;
@@ -1515,6 +1521,9 @@ Error GenericPluginTy::deinit() {
   if (RPCServer)
     delete RPCServer;
 
+  if (RecordReplay)
+    delete RecordReplay;
+
   // Perform last deinitializations on the plugin.
   return deinitImpl();
 }
@@ -1630,12 +1639,12 @@ int32_t GenericPluginTy::initialize_record_replay(int32_t DeviceId,
       isRecord ? RecordReplayTy::RRStatusTy::RRRecording
                : RecordReplayTy::RRStatusTy::RRReplaying;
 
-  if (auto Err = RecordReplay.init(&Device, MemorySize, VAddr, Status,
-                                   SaveOutput, ReqPtrArgOffset)) {
+  if (auto Err = RecordReplay->init(&Device, MemorySize, VAddr, Status,
+                                    SaveOutput, ReqPtrArgOffset)) {
     REPORT("WARNING RR did not intialize RR-properly with %lu bytes"
            "(Error: %s)\n",
            MemorySize, toString(std::move(Err)).data());
-    RecordReplay.setStatus(RecordReplayTy::RRStatusTy::RRDeactivated);
+    RecordReplay->setStatus(RecordReplayTy::RRStatusTy::RRDeactivated);
 
     if (!isRecord) {
       return OFFLOAD_FAIL;
@@ -1984,8 +1993,8 @@ int32_t GenericPluginTy::get_global(__tgt_device_binary Binary, uint64_t Size,
   assert(DevicePtr && "Invalid device global's address");
 
   // Save the loaded globals if we are recording.
-  if (RecordReplay.isRecording())
-    RecordReplay.addEntry(Name, Size, *DevicePtr);
+  if (getRecordAndReplay().isRecording())
+    getRecordAndReplay().addEntry(Name, Size, *DevicePtr);
 
   return OFFLOAD_SUCCESS;
 }
-- 
GitLab